Skip to main content

nautilus_live/node/
mod.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Live trading node built on a single-threaded tokio event loop.
17//!
18//! The node owns system lifecycle and the event loop. Its reconciliation module schedules checks,
19//! manages report futures and deadlines, and dispatches results. The execution manager owns
20//! reconciliation state, discrepancy decisions, and individual reconciliation operations.
21//!
22//! `LiveNode::run()` drives the system through a `tokio::select!` loop that
23//! multiplexes data events, execution events, trading commands, timers, and
24//! periodic maintenance tasks (reconciliation, purge, prune, audit).
25//!
26//! # Threading model
27//!
28//! The core types (`ExecutionManager`, `ExecutionEngine`, `Cache`) use
29//! `Rc<RefCell<..>>` and are `!Send`. All access happens on the same thread.
30//! Pending report futures can retain client borrows while other select branches run.
31//! The client facade defers instrument updates until those borrows are released;
32//! completion and cancellation paths flush the deferred updates. Single-threaded
33//! execution does not by itself prevent conflicting borrows or reentrant callbacks.
34//!
35//! # Startup sequencing
36//!
37//! Startup connects clients in two phases so that instruments are in the
38//! cache before execution clients read them:
39//!
40//! 1. Connect data clients (instruments arrive as buffered `DataEvent`s).
41//! 2. Flush all pending data events and commands into the cache via
42//!    `flush_pending_data`, which loops `try_recv` on the channel receivers
43//!    until no items remain.
44//! 3. Connect execution clients (`load_instruments_from_cache` now finds
45//!    populated instruments).
46//! 4. Drain remaining events, then run reconciliation.
47//!
48//! Both `run()` (integrated event loop) and `start()` (manual lifecycle)
49//! follow this sequence.
50//! Startup aborts if a data or execution client returns a connection error.
51//!
52//! # Reconciliation
53//!
54//! Continuous inflight, open-order, and position checks run on independent intervals. The
55//! shared maintenance timer in the select loop dispatches reconciliation at
56//! the minimum enabled interval. Each dispatch the handler checks which
57//! sub-checks are due based on elapsed nanoseconds and schedules their work.
58//! Continuous checks do not await venue HTTP in the select loop: open-order
59//! and position checks poll bulk venue report futures from the loop.
60//!
61//! # Maintenance dispatcher
62//!
63//! Six periodic tasks share a single coarse `maintenance_timer`:
64//!
65//! - reconciliation (inflight, open, position sub-checks)
66//! - purge closed orders
67//! - purge closed positions
68//! - purge account events
69//! - own-books audit
70//! - recent-fills cache prune
71//!
72//! The runner wakes one timer per loop iteration regardless of how many
73//! maintenance tasks are configured. Each task tracks its own
74//! `next_fire: Instant` and the dispatcher fires the bodies whose deadline
75//! has passed, rescheduling `next = now + interval` (equivalent to
76//! `MissedTickBehavior::Delay`). Disabled tasks anchor on a far-future
77//! `next` that never trips.
78//!
79//! The 100ms timer cadence is the effective floor for any maintenance
80//! interval. Configured intervals below 100ms (the config types allow
81//! `inflight_check_interval_ms` and `own_books_audit_interval_secs` smaller)
82//! become eligible on the next maintenance tick. Event processing and runtime
83//! scheduling can delay dispatch further; the timer does not guarantee a maximum delay.
84
85use std::{
86    any::Any,
87    cell::RefCell,
88    collections::VecDeque,
89    fmt::Debug,
90    rc::{Rc, Weak},
91    time::Duration,
92};
93
94use anyhow::Context;
95use nautilus_common::{
96    actor::{self, Actor, DataActor, DataActorNative},
97    cache::database::{CacheDatabaseAdapter, CacheDatabaseFactory},
98    clients::ExecutionClient,
99    component::Component,
100    enums::{Environment, LogColor},
101    live::{dispatch::DispatchMessage, dst},
102    log_info,
103    messages::{
104        DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent,
105        data::DataCommand,
106        execution::TradingCommand,
107        system::{QueueStateChanged, ReconnectSocket, SocketStateChange, SocketStateChanged},
108    },
109    msgbus::{self, BusMessage, MessagingSwitchboard},
110    runner::{SystemChannel, TimeEventMessage, TradingCommandMessage},
111};
112use nautilus_core::{
113    UUID4,
114    datetime::{mins_to_secs, secs_to_nanos_unchecked},
115};
116use nautilus_execution::client::ExecutionClientAdapter;
117#[cfg(test)]
118use nautilus_model::reports::OrderStatusReport;
119use nautilus_model::{
120    events::OrderEventAny,
121    identifiers::{ClientOrderId, InstrumentId, StrategyId, TraderId},
122    orders::Order,
123};
124use nautilus_network::mode::ReconnectRequestOutcome;
125#[cfg(feature = "python")]
126use nautilus_system::trader::Trader;
127use nautilus_system::{config::NautilusKernelConfig, kernel::NautilusKernel};
128use nautilus_trading::{
129    ExecutionAlgorithm, ExecutionAlgorithmNative,
130    strategy::{Strategy, StrategyNative},
131};
132use tabled::{builder::Builder, settings::Style};
133
134use crate::{
135    dispatch::drain_callbacks,
136    execution::{
137        client::LiveExecutionClient,
138        manager::{ExecutionManager, ExecutionManagerConfig, TargetedOrderQuery},
139    },
140    runner::{AsyncRunner, AsyncRunnerChannels, PendingRunnerEvent},
141    socket::{SocketReconnectLookup, SocketReconnectRegistry},
142};
143
144pub mod builder;
145pub mod config;
146
147mod metrics;
148mod queue;
149mod reconciliation;
150mod state;
151
152#[cfg(feature = "plugin")]
153pub mod plugin;
154
155use builder::ExternalMessageBusIngress;
156pub use builder::LiveNodeBuilder;
157use config::{LiveNodeConfig, PluginConfig, validate_live_environment};
158pub use metrics::{RunnerChannelMetricsSnapshot, RunnerMetricsDelta, RunnerMetricsSnapshot};
159use metrics::{RunnerChannelQueueDepths, RunnerMetrics};
160use queue::{QueueMonitor, QueueStateTransition};
161use reconciliation::{
162    OpenOrderReportCollection, OpenOrderReportTask, PositionFillReportResult, PositionReportTask,
163    PositionReportTaskResult, ReconciliationCheckIntervals, ReconciliationCheckState,
164    ReportTaskOutcome, TargetedOrderReportTask,
165};
166use state::{EngineConnectionStatus, RunningTransition};
167pub use state::{LiveNodeHandle, NodeRunMode, NodeState};
168
169/// Dispatches the run loop performs before yielding to the executor.
170///
171/// A saturated channel keeps every select branch ready, so the loop would otherwise never return
172/// `Pending`. Under a host event loop that starves the adapter I/O tasks feeding those channels,
173/// which shows up as lapsed heartbeats and reconnects rather than as backpressure.
174const DISPATCHES_PER_YIELD: usize = 64;
175
176thread_local! {
177    static NODE_THREAD_OWNER: RefCell<Weak<()>> = const { RefCell::new(Weak::new()) };
178}
179
180type StreamProcessorCallback = dyn Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static;
181
182struct StreamProcessor(Box<StreamProcessorCallback>);
183
184impl Debug for StreamProcessor {
185    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
186        f.debug_struct(stringify!(StreamProcessor)).finish()
187    }
188}
189
190/// High-level abstraction for a live Nautilus system node.
191///
192/// Provides a simplified interface for running live systems
193/// with automatic client management and lifecycle handling.
194///
195/// Only one live node may exist on a thread at a time. Drop the node before building another,
196/// including after disposal, because retained nodes can still access thread-local messaging.
197/// Concurrent nodes in one process remain unsupported, even on separate threads.
198#[derive(Debug)]
199pub struct LiveNode {
200    kernel: NautilusKernel,
201    runner: Option<AsyncRunner>,
202    config: LiveNodeConfig,
203    handle: LiveNodeHandle,
204    exec_manager: ExecutionManager,
205    exec_clients: Vec<LiveExecutionClient>,
206    socket_registry: SocketReconnectRegistry,
207    cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
208    external_msgbus: Option<ExternalMessageBusIngress>,
209    stream_processors: Vec<StreamProcessor>,
210    shutdown_deadline: Option<dst::time::Instant>,
211    #[cfg(feature = "plugin")]
212    plugins: plugin::NodePlugins,
213    _thread_owner: Rc<()>,
214}
215
216impl LiveNode {
217    /// Creates a new `LiveNode` from builder components.
218    ///
219    /// This is an internal constructor used by `LiveNodeBuilder`.
220    #[must_use]
221    #[allow(
222        clippy::too_many_arguments,
223        reason = "builder components have distinct lifecycle roles"
224    )]
225    pub(crate) fn new_from_builder(
226        kernel: NautilusKernel,
227        runner: AsyncRunner,
228        config: LiveNodeConfig,
229        handle: LiveNodeHandle,
230        mut exec_manager: ExecutionManager,
231        exec_clients: Vec<LiveExecutionClient>,
232        socket_registry: SocketReconnectRegistry,
233        cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
234        external_msgbus: Option<ExternalMessageBusIngress>,
235        thread_owner: Rc<()>,
236    ) -> Self {
237        for client in &exec_clients {
238            exec_manager.register_submission_retention(client);
239        }
240
241        Self {
242            kernel,
243            runner: Some(runner),
244            config,
245            handle,
246            exec_manager,
247            exec_clients,
248            socket_registry,
249            cache_database_factory,
250            external_msgbus,
251            stream_processors: Vec::new(),
252            shutdown_deadline: None,
253            #[cfg(feature = "plugin")]
254            plugins: plugin::NodePlugins,
255            _thread_owner: thread_owner,
256        }
257    }
258
259    /// Creates a new [`LiveNodeBuilder`] for fluent configuration.
260    ///
261    /// # Errors
262    ///
263    /// Returns an error if the environment is invalid for live trading.
264    pub fn builder(
265        trader_id: TraderId,
266        environment: Environment,
267    ) -> anyhow::Result<LiveNodeBuilder> {
268        LiveNodeBuilder::new(trader_id, environment)
269    }
270
271    /// Creates a new [`LiveNode`] directly from a kernel name and optional configuration.
272    ///
273    /// This is a convenience method for creating a live node with a pre-configured
274    /// kernel configuration, bypassing the builder pattern. If no config is provided,
275    /// a default configuration will be used.
276    ///
277    /// # Errors
278    ///
279    /// Returns an error if kernel construction fails or another live node exists or is being
280    /// built on this thread.
281    pub fn build(name: String, config: Option<LiveNodeConfig>) -> anyhow::Result<Self> {
282        let thread_owner = Self::acquire_thread()?;
283        let config = config.unwrap_or_default();
284        validate_live_environment(config.environment())?;
285
286        config.validate_runtime_support()?;
287
288        if config.event_store.is_some() {
289            anyhow::bail!(
290                "LiveNodeConfig.event_store is set but LiveNode::build cannot install a factory; \
291                 use LiveNodeBuilder::with_event_store(...) instead"
292            );
293        }
294
295        let handle = LiveNodeHandle::new();
296        let runner = AsyncRunner::new();
297        runner.bind_senders_for_node(handle.clone());
298
299        let kernel = NautilusKernel::new(name, config.clone())?;
300        #[cfg(feature = "python")]
301        if let Some(controller) = config.controller.as_ref() {
302            Trader::add_controller_from_importable_config(&kernel.trader, controller)?;
303        }
304
305        #[cfg(not(feature = "python"))]
306        if let Some(controller) = config.controller.as_ref() {
307            anyhow::bail!(
308                "LiveNodeConfig.controller for importable controller '{}' requires the python feature",
309                controller.controller_path
310            );
311        }
312
313        let exec_manager_config =
314            ExecutionManagerConfig::from(&config.exec_engine).with_trader_id(config.trader_id);
315
316        let exec_manager = ExecutionManager::new(
317            kernel.clock.clone(),
318            kernel.cache.clone(),
319            exec_manager_config,
320        )?;
321
322        let node = Self {
323            kernel,
324            runner: Some(runner),
325            config,
326            handle,
327            exec_manager,
328            exec_clients: Vec::new(),
329            socket_registry: SocketReconnectRegistry::default(),
330            cache_database_factory: None,
331            external_msgbus: None,
332            stream_processors: Vec::new(),
333            shutdown_deadline: None,
334            #[cfg(feature = "plugin")]
335            plugins: plugin::NodePlugins,
336            _thread_owner: thread_owner,
337        };
338
339        node.load_configured_plugins()?;
340
341        log::info!("LiveNode built successfully with kernel config");
342
343        Ok(node)
344    }
345
346    fn acquire_thread() -> anyhow::Result<Rc<()>> {
347        NODE_THREAD_OWNER.with(|slot| {
348            let mut owner = slot.borrow_mut();
349            anyhow::ensure!(
350                owner.upgrade().is_none(),
351                "A LiveNode already exists or is being built on this thread; drop it before building another"
352            );
353            let token = Rc::new(());
354            *owner = Rc::downgrade(&token);
355            Ok(token)
356        })
357    }
358
359    /// Loads and registers plug-ins declared on the node config.
360    ///
361    /// # Errors
362    ///
363    /// Returns an error if plug-ins are configured without host-side support.
364    pub(crate) fn load_configured_plugins(&self) -> anyhow::Result<()> {
365        if self.config.plugins.is_empty() {
366            return Ok(());
367        }
368
369        anyhow::bail!(
370            "LiveNodeConfig.plugins requires host-side plug-in support; nautilus-plugin is the guest SDK only"
371        )
372    }
373
374    /// Loads and registers one plug-in instance.
375    ///
376    /// # Errors
377    ///
378    /// Returns an error if called. Dynamic plug-in hosting lives in the host-side integration.
379    #[expect(
380        clippy::needless_pass_by_value,
381        reason = "signature mirrors the host-enabled API"
382    )]
383    pub fn add_plugin(&mut self, config: PluginConfig) -> anyhow::Result<()> {
384        #[cfg(feature = "plugin")]
385        config.validate_runtime_support(self.config.plugins.len())?;
386        #[cfg(not(feature = "plugin"))]
387        let _ = config;
388
389        anyhow::bail!(
390            "LiveNode::add_plugin requires host-side plug-in support; nautilus-plugin is the guest SDK only"
391        )
392    }
393
394    /// Returns a thread-safe handle to control this node.
395    #[must_use]
396    pub fn handle(&self) -> LiveNodeHandle {
397        self.handle.clone()
398    }
399
400    /// Adds a callback for supported typed external messages.
401    ///
402    /// While the node's run loop services external ingress, processors run in registration order
403    /// before normal inbound streaming filters for supported typed JSON or MessagePack payloads.
404    /// Unsupported encodings do not reach processors. Rust callbacks receive the decoded
405    /// concrete value as [`Any`], allowing them to downcast to the concrete payload type.
406    ///
407    /// Python callbacks receive the serialized payload shape with an added `payload_type` field.
408    /// Native data replies use the shared response representation keyed by response kind.
409    /// Callback exceptions are logged, stop remaining processors, and skip internal republishing.
410    /// The Python wrapper returns an error if the node is no longer available for mutation.
411    ///
412    /// External egress is suppressed while processors run, so synchronous publications remain
413    /// local. During data-client connection, streaming-registered `DataResponse` messages with a
414    /// pending correlation handler bypass processors and can overtake messages buffered until
415    /// normal dispatch starts.
416    pub fn add_stream_processor<F>(&mut self, callback: F)
417    where
418        F: Fn(&dyn Any) + 'static,
419    {
420        self.add_stream_processor_with_mapping(move |message, _| {
421            callback(message);
422            Ok(())
423        });
424    }
425
426    pub(crate) fn add_stream_processor_with_mapping<F>(&mut self, callback: F)
427    where
428        F: Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static,
429    {
430        self.stream_processors
431            .push(StreamProcessor(Box::new(callback)));
432    }
433
434    /// Starts the live node without entering a select loop.
435    ///
436    /// Connects clients, runs reconciliation, and starts the trader, but does
437    /// not consume the runner or drive channel receivers, so channel traffic arriving after
438    /// startup is never serviced. This is a building block for tests and embedding, not a
439    /// lifecycle: use [`run`](Self::run) or [`run_with_mode`](Self::run_with_mode) to run a node.
440    ///
441    /// # Errors
442    ///
443    /// Returns an error if startup fails.
444    pub async fn start(&mut self) -> anyhow::Result<()> {
445        if self.state().is_running() {
446            anyhow::bail!("Already running");
447        }
448
449        if self.external_msgbus.is_some() {
450            log::warn!(
451                "External message bus ingress is configured but LiveNode::start() does not service it; use LiveNode::run()"
452            );
453        }
454
455        self.prepare_cache().await?;
456
457        if let Some(runner) = self.runner.as_ref() {
458            runner.bind_senders_for_node(self.handle.clone());
459        }
460
461        self.handle.set_starting();
462
463        self.kernel.reset_shutdown_flag();
464        self.kernel.start_async().await;
465
466        if self.kernel.is_event_store_replay() {
467            log::info!(
468                "Event-store replay loaded; skipping live client connection and reconciliation",
469            );
470
471            if !self.finish_startup_replay().await? {
472                return Ok(());
473            }
474
475            return Ok(());
476        }
477
478        if self.kernel.is_event_store_replay_configured() {
479            self.abort_startup("Event-store replay did not start")
480                .await?;
481            return Ok(());
482        }
483
484        let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
485
486        // Connect data clients first and flush instrument events into cache
487        if let Err(e) = self.connect_data_phase(connection_deadline).await {
488            return self
489                .abort_startup_with_error("Data client connection failed", e)
490                .await;
491        }
492
493        let (startup_system_events, startup_system_commands) =
494            if let Some(runner) = self.runner.as_mut() {
495                runner.flush_pending_data();
496                (
497                    runner.drain_pending_system_events(),
498                    runner.drain_pending_system_commands(),
499                )
500            } else {
501                (Vec::new(), Vec::new())
502            };
503
504        if let Err(e) = self.connect_exec_clients(connection_deadline).await {
505            return self
506                .abort_startup_with_error("Execution client connection failed", e)
507                .await;
508        }
509
510        if let Some(reason) = self.startup_abort_reason() {
511            self.abort_startup(reason).await?;
512            return Ok(());
513        }
514
515        match self.await_engines_connected(connection_deadline).await {
516            EngineConnectionStatus::Connected => {}
517            EngineConnectionStatus::TimedOut => {
518                return self
519                    .abort_startup_with_error(
520                        "Engine readiness timed out",
521                        anyhow::anyhow!("readiness timeout while waiting for engine connections"),
522                    )
523                    .await;
524            }
525            EngineConnectionStatus::StopRequested => {
526                self.abort_startup("Stop signal received during startup")
527                    .await?;
528                return Ok(());
529            }
530            EngineConnectionStatus::ShutdownRequested => {
531                self.abort_startup("Shutdown signal received during startup")
532                    .await?;
533                return Ok(());
534            }
535        }
536
537        if let Err(e) = self.perform_startup_reconciliation().await {
538            if let Err(finalize_err) = self.abort_startup("Startup reconciliation failed").await {
539                anyhow::bail!(
540                    "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
541                );
542            }
543
544            return Err(e);
545        }
546
547        if let Some(reason) = self.startup_abort_reason() {
548            self.abort_startup(reason).await?;
549            return Ok(());
550        }
551
552        if let Err(e) = self.kernel.start_trader() {
553            return self.abort_after_trader_start_failure(e).await;
554        }
555
556        #[cfg(feature = "plugin")]
557        if let Err(e) = self.plugins.start_controllers() {
558            return self.abort_after_trader_start_failure(e).await;
559        }
560
561        self.process_system_events(startup_system_events);
562        self.process_system_commands(startup_system_commands);
563
564        if !self.finish_startup_trader(None).await? {
565            return Ok(());
566        }
567
568        Ok(())
569    }
570
571    /// Stop the live node.
572    ///
573    /// This method stops the trader, waits for the configured grace period to allow
574    /// residual events to be processed, then finalizes the shutdown sequence.
575    ///
576    /// # Errors
577    ///
578    /// Returns an error if shutdown fails or retained submissions remain unresolved at the end
579    /// of `delay_post_stop`. Teardown still completes; subsequent events cannot clear this result.
580    pub async fn stop(&mut self) -> anyhow::Result<()> {
581        if !self.state().is_running() {
582            anyhow::bail!("Not running");
583        }
584
585        self.handle.set_shutting_down();
586
587        #[cfg(feature = "plugin")]
588        let controller_stop_result = self.plugins.stop_controllers();
589        #[cfg(not(feature = "plugin"))]
590        let controller_stop_result: anyhow::Result<()> = Ok(());
591
592        self.kernel.stop_trader();
593        let delay = self.kernel.delay_post_stop();
594        log::info!("Awaiting residual events ({delay:?})...");
595
596        let residual_events = self.process_runner_for(delay).await;
597        if residual_events > 0 {
598            log::debug!("Processed {residual_events} residual events during shutdown");
599        }
600
601        let stop_result = self.finalize_stop().await;
602        self.drain_runner_pending();
603
604        match (controller_stop_result, stop_result) {
605            (Ok(()), Ok(())) => Ok(()),
606            (Err(controller_err), Ok(())) => Err(controller_err),
607            (Ok(()), Err(stop_err)) => Err(stop_err),
608            (Err(controller_err), Err(stop_err)) => {
609                log::error!("Error stopping plug-in controllers: {controller_err}");
610                Err(stop_err)
611            }
612        }
613    }
614
615    /// Disposes the live node kernel and releases resources.
616    ///
617    /// Discards any retained runner messages and attempts callback cleanup. Logs latched callback
618    /// failures and cleanup rejection; externally retained work can prevent clearing.
619    pub fn dispose(&mut self) {
620        self.close_external_ingress();
621        self.handle.set_stopped();
622        self.kernel.dispose();
623        drop(self.runner.take());
624
625        if let Some(e) = actor::callback_failure() {
626            log::error!("Callback dispatch failed before disposal cleanup: {e}");
627        }
628
629        if let Err(e) = actor::clear_callbacks() {
630            log::error!("Failed to clear callback dispatch during disposal: {e}");
631        }
632    }
633
634    async fn process_runner_for(&mut self, duration: Duration) -> usize {
635        let Some(mut runner) = self.runner.take() else {
636            dst::time::sleep(duration).await;
637            return 0;
638        };
639
640        runner.bind_senders_for_node(self.handle.clone());
641        let deadline = dst::time::Instant::now() + duration;
642        let mut processed = 0;
643
644        loop {
645            tokio::select! {
646                biased;
647
648                () = dst::time::sleep_until(deadline) => break,
649                event = runner.recv() => {
650                    let Some(event) = event else {
651                        dst::time::sleep_until(deadline).await;
652                        break;
653                    };
654
655                    self.process_runner_event(event);
656                    processed += 1;
657                }
658            }
659        }
660
661        self.runner = Some(runner);
662        processed
663    }
664
665    fn drain_runner_pending(&mut self) -> usize {
666        let Some(mut runner) = self.runner.take() else {
667            return 0;
668        };
669
670        let processed = runner.poll_pending(|event| self.process_runner_event(event));
671        self.runner = Some(runner);
672        processed
673    }
674
675    fn process_runner_event(&mut self, event: PendingRunnerEvent) {
676        match event {
677            PendingRunnerEvent::TimeEvent(message) => {
678                let _ = AsyncRunner::handle_time_event(message);
679            }
680            PendingRunnerEvent::SystemEvent(event) => {
681                event.dispatch(|event| self.process_system_event(event));
682            }
683            PendingRunnerEvent::SystemCommand(command) => {
684                command.dispatch(|command| self.process_system_command(command));
685            }
686            PendingRunnerEvent::ExecEvent(event) => {
687                event.dispatch(|event| self.process_exec_event(event));
688            }
689            PendingRunnerEvent::ExecCommand(command) => self.process_exec_command(command),
690            PendingRunnerEvent::DataEvent(event) => AsyncRunner::dispatch_data_event(event),
691            PendingRunnerEvent::DataCommand(command) => AsyncRunner::handle_data_command(command),
692        }
693    }
694
695    fn process_system_events(&self, events: Vec<DispatchMessage<SystemEvent>>) {
696        for event in events {
697            event.dispatch(|event| self.process_system_event(event));
698        }
699    }
700
701    fn process_system_commands(&self, commands: Vec<DispatchMessage<SystemCommand>>) {
702        for command in commands {
703            command.dispatch(|command| self.process_system_command(command));
704        }
705    }
706
707    fn process_system_command(&self, command: SystemCommand) {
708        match command {
709            SystemCommand::ReconnectSocket(command) => {
710                self.process_socket_reconnect(command);
711            }
712        }
713    }
714
715    fn process_socket_reconnect(&self, command: ReconnectSocket) {
716        let outcome = if command.trader_id == self.config.trader_id {
717            Self::request_socket_reconnect(
718                self.socket_registry
719                    .get(command.client_id, command.endpoint),
720            )
721        } else {
722            SocketReconnectDispatchOutcome::InvalidTrader
723        };
724
725        if outcome == SocketReconnectDispatchOutcome::Accepted {
726            log::info!(
727                "Requested socket reconnect for client {} endpoint {}",
728                command.client_id,
729                command.endpoint
730            );
731        } else {
732            log::warn!(
733                "Rejected socket reconnect request for client {} endpoint {}: {outcome:?}",
734                command.client_id,
735                command.endpoint
736            );
737        }
738    }
739
740    fn request_socket_reconnect(lookup: SocketReconnectLookup) -> SocketReconnectDispatchOutcome {
741        match lookup {
742            SocketReconnectLookup::Handle(handle) => match handle.request_reconnect() {
743                ReconnectRequestOutcome::Accepted => SocketReconnectDispatchOutcome::Accepted,
744                ReconnectRequestOutcome::AlreadyReconnecting => {
745                    SocketReconnectDispatchOutcome::AlreadyReconnecting
746                }
747                ReconnectRequestOutcome::Disconnected => {
748                    SocketReconnectDispatchOutcome::Disconnected
749                }
750                ReconnectRequestOutcome::Closed => SocketReconnectDispatchOutcome::Closed,
751                ReconnectRequestOutcome::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
752            },
753            SocketReconnectLookup::ClientNotFound => SocketReconnectDispatchOutcome::UnknownClient,
754            SocketReconnectLookup::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
755            SocketReconnectLookup::EndpointNotFound => {
756                SocketReconnectDispatchOutcome::UnknownEndpoint
757            }
758            SocketReconnectLookup::AmbiguousEndpoint => {
759                SocketReconnectDispatchOutcome::AmbiguousEndpoint
760            }
761        }
762    }
763
764    fn process_system_event(&self, event: SystemEvent) {
765        match event {
766            SystemEvent::SocketState(change) => self.publish_socket_state_change(change),
767        }
768    }
769
770    fn publish_socket_state_change(&self, change: SocketStateChange) {
771        let timestamp = self.kernel.generate_timestamp_ns();
772
773        let event = SocketStateChanged::new(
774            self.config.trader_id,
775            change.client_id,
776            change.venue,
777            change.endpoint,
778            change.state,
779            UUID4::new(),
780            timestamp,
781            timestamp,
782        );
783
784        msgbus::publish_any(
785            MessagingSwitchboard::socket_state_changed_topic(
786                event.client_id,
787                event.endpoint.as_str(),
788            ),
789            event.as_any(),
790        );
791    }
792
793    /// Awaits engine clients to connect with timeout.
794    ///
795    /// Returns the final connection wait status.
796    async fn await_engines_connected(
797        &self,
798        deadline: dst::time::Instant,
799    ) -> EngineConnectionStatus {
800        log::info!(
801            "Awaiting engine connections ({:?} timeout)...",
802            self.config.timeout_connection
803        );
804
805        let interval = Duration::from_millis(100);
806
807        loop {
808            if self.handle.should_stop() {
809                log::warn!("Stop signal received, aborting connection wait");
810                return EngineConnectionStatus::StopRequested;
811            }
812
813            if self.kernel.is_shutdown_requested() {
814                log::warn!("Shutdown signal received, aborting connection wait");
815                return EngineConnectionStatus::ShutdownRequested;
816            }
817
818            if self.kernel.check_engines_connected() {
819                log::info!("All engine clients connected");
820                return EngineConnectionStatus::Connected;
821            }
822
823            let now = dst::time::Instant::now();
824            if now >= deadline {
825                break;
826            }
827
828            dst::time::sleep(interval.min(deadline - now)).await;
829        }
830
831        self.log_connection_status();
832        EngineConnectionStatus::TimedOut
833    }
834
835    /// Awaits engine clients to disconnect with timeout.
836    ///
837    /// Returns an error with client status on timeout.
838    async fn await_engines_disconnected(&self, deadline: dst::time::Instant) -> anyhow::Result<()> {
839        log::info!(
840            "Awaiting engine disconnections ({:?} timeout)...",
841            self.config.timeout_disconnection
842        );
843
844        let timeout = self.config.timeout_disconnection;
845        let interval = Duration::from_millis(100);
846
847        loop {
848            if self.kernel.check_engines_disconnected() {
849                log::info!("All engine clients disconnected");
850                return Ok(());
851            }
852
853            let now = dst::time::Instant::now();
854            if now >= deadline {
855                break;
856            }
857
858            dst::time::sleep(interval.min(deadline - now)).await;
859        }
860
861        log::error!(
862            "Timed out ({:?}) waiting for engines to disconnect\n\
863             DataEngine.check_disconnected() == {}\n\
864             ExecEngine.check_disconnected() == {}",
865            timeout,
866            self.kernel.data_engine().check_disconnected(),
867            self.kernel.exec_engine().borrow().check_disconnected(),
868        );
869        anyhow::bail!("disconnect readiness timeout while waiting for engine disconnections")
870    }
871
872    fn log_connection_status(&self) {
873        let data_status = self.kernel.data_client_connection_status();
874        let exec_status = self.kernel.exec_client_connection_status();
875
876        let mut rows: Vec<ClientStatus> = Vec::new();
877
878        for (client_id, connected) in data_status {
879            rows.push(ClientStatus {
880                client: client_id.to_string(),
881                client_type: "Data",
882                connected,
883            });
884        }
885
886        for (client_id, connected) in exec_status {
887            rows.push(ClientStatus {
888                client: client_id.to_string(),
889                client_type: "Execution",
890                connected,
891            });
892        }
893
894        let table = render_client_statuses(rows);
895
896        log::warn!(
897            "Timed out ({:?}) waiting for engines to connect\n\n{table}\n\n\
898             DataEngine.check_connected() == {}\n\
899             ExecEngine.check_connected() == {}",
900            self.config.timeout_connection,
901            self.kernel.data_engine().check_connected(),
902            self.kernel.exec_engine().borrow().check_connected(),
903        );
904    }
905
906    /// Performs startup reconciliation to align internal state with venue state.
907    ///
908    /// This method queries each execution client for mass status (orders, fills, positions)
909    /// and reconciles any discrepancies with the local cache state.
910    ///
911    /// # Errors
912    ///
913    /// Returns an error if reconciliation fails or times out.
914    async fn perform_startup_reconciliation(&mut self) -> anyhow::Result<()> {
915        if !self.config.exec_engine.reconciliation {
916            log::info!("Startup reconciliation disabled");
917            self.kernel
918                .portfolio
919                .borrow_mut()
920                .initialize_wallet_orders()?;
921            return Ok(());
922        }
923
924        log_info!(
925            "Starting execution state reconciliation...",
926            color = LogColor::Blue
927        );
928
929        let lookback_mins = self
930            .config
931            .exec_engine
932            .reconciliation_lookback_mins
933            .map(u64::from);
934
935        let timeout = self.config.timeout_reconciliation;
936        let start = dst::time::Instant::now();
937        let client_ids = self.kernel.exec_engine.borrow().client_ids();
938
939        for client_id in client_ids {
940            let elapsed = start.elapsed();
941            if elapsed >= timeout {
942                anyhow::bail!("Startup reconciliation timeout reached");
943            }
944
945            let remaining = timeout
946                .checked_sub(elapsed)
947                .expect("elapsed checked against reconciliation timeout");
948
949            log_info!(
950                "Requesting mass status from {}...",
951                client_id,
952                color = LogColor::Blue
953            );
954
955            let client = self
956                .exec_clients
957                .iter()
958                .find(|client| client.client_id() == client_id)
959                .ok_or_else(|| anyhow::anyhow!("Execution client {client_id} has no live handle"))?
960                .clone();
961            let client = Box::new(client);
962            let client = ExecutionClientAdapter::new(client);
963
964            let mass_status_query = client.generate_mass_status(lookback_mins);
965
966            let mass_status_result = dst::time::timeout(remaining, mass_status_query)
967            .await
968            .map_err(|_| {
969                anyhow::anyhow!(
970                    "Startup reconciliation timeout reached while requesting mass status from {client_id}"
971                )
972            })?;
973
974            match mass_status_result {
975                Ok(Some(mass_status)) => {
976                    log_info!(
977                        "Reconciling ExecutionMassStatus for {}",
978                        client_id,
979                        color = LogColor::Blue
980                    );
981
982                    let result = self
983                        .exec_manager
984                        .reconcile_execution_mass_status(&mass_status, &self.kernel.exec_engine);
985
986                    anyhow::ensure!(
987                        self.kernel
988                            .exec_engine
989                            .borrow()
990                            .get_client(&client_id)
991                            .is_some(),
992                        "Execution client {client_id} disappeared during startup reconciliation",
993                    );
994
995                    anyhow::ensure!(
996                        result.unresolved_positions.is_empty(),
997                        "Unresolved positions during startup reconciliation for {client_id}: {}",
998                        result.unresolved_positions.join("; "),
999                    );
1000
1001                    if result.events.is_empty() {
1002                        log_info!(
1003                            "Reconciliation for {} succeeded",
1004                            client_id,
1005                            color = LogColor::Blue
1006                        );
1007                    } else {
1008                        log::info!(
1009                            color = LogColor::Blue as u8;
1010                            "Reconciliation for {} processed {} events",
1011                            client_id,
1012                            result.events.len()
1013                        );
1014                    }
1015
1016                    // Register external orders with execution clients for tracking
1017                    if !result.external_orders.is_empty() {
1018                        let exec_engine = self.kernel.exec_engine.borrow();
1019
1020                        let source_client = exec_engine.get_client(&client_id).ok_or_else(|| {
1021                            anyhow::anyhow!(
1022                                "Execution client {client_id} disappeared during startup reconciliation"
1023                            )
1024                        })?;
1025
1026                        for external in result.external_orders {
1027                            source_client.register_external_order(
1028                                external.client_order_id,
1029                                external.venue_order_id,
1030                                external.instrument_id,
1031                                external.strategy_id,
1032                                external.ts_init,
1033                            );
1034                        }
1035                    }
1036                }
1037                Ok(None) => {
1038                    log::warn!(
1039                        "No mass status available from {client_id} \
1040                         (likely adapter error when generating reports)"
1041                    );
1042                }
1043                Err(e) => {
1044                    return Err(e).context(format!("Failed to get mass status from {client_id}"));
1045                }
1046            }
1047        }
1048
1049        self.kernel.portfolio.borrow_mut().initialize_orders();
1050        self.kernel.portfolio.borrow_mut().initialize_positions();
1051        self.kernel
1052            .portfolio
1053            .borrow_mut()
1054            .initialize_wallet_orders()?;
1055
1056        let elapsed_secs = start.elapsed().as_secs_f64();
1057        log_info!(
1058            "Startup reconciliation completed in {:.2}s",
1059            elapsed_secs,
1060            color = LogColor::Blue
1061        );
1062
1063        Ok(())
1064    }
1065
1066    /// Run the live node with automatic shutdown handling.
1067    ///
1068    /// This method starts the node, runs indefinitely, and handles graceful shutdown
1069    /// on interrupt signals.
1070    ///
1071    /// # Thread Safety
1072    ///
1073    /// The event loop runs directly on the current thread (not spawned) because the
1074    /// msgbus uses thread-local storage. Endpoints registered by the kernel are only
1075    /// accessible from the same thread.
1076    ///
1077    /// # Shutdown Sequence
1078    ///
1079    /// 1. Signal received (SIGINT, SIGTERM, or handle stop).
1080    /// 2. Trader components stopped (triggers order cancellations, etc.).
1081    /// 3. Event loop continues processing residual events for the configured grace period.
1082    /// 4. Kernel finalized, clients disconnected, remaining events drained.
1083    ///
1084    /// # Errors
1085    ///
1086    /// Returns an error if the node fails to start, encounters a runtime error, or shuts down
1087    /// with retained submissions unresolved at the end of `delay_post_stop`.
1088    pub async fn run(&mut self) -> anyhow::Result<()> {
1089        self.run_with_mode(NodeRunMode::Owned).await
1090    }
1091
1092    /// Run the live node under the given mode.
1093    ///
1094    /// [`NodeRunMode::Hosted`] leaves signal handling to the host application. Every other
1095    /// responsibility, including maintenance, reconciliation, external ingress, and the shutdown
1096    /// sequence, is identical across modes so that hosted and owned nodes cannot diverge.
1097    ///
1098    /// # Errors
1099    ///
1100    /// Returns an error if the node fails to start, encounters a runtime error, or shuts down
1101    /// with retained submissions unresolved at the end of `delay_post_stop`.
1102    pub async fn run_with_mode(&mut self, mode: NodeRunMode) -> anyhow::Result<()> {
1103        if self.state().is_running() {
1104            anyhow::bail!("Already running");
1105        }
1106
1107        if self.runner.is_none() {
1108            anyhow::bail!("Runner already consumed - run() called twice");
1109        }
1110
1111        self.prepare_cache().await?;
1112
1113        let Some(runner) = self.runner.take() else {
1114            anyhow::bail!("Runner already consumed - run() called twice");
1115        };
1116
1117        runner.bind_senders_for_node(self.handle.clone());
1118
1119        let AsyncRunnerChannels {
1120            mut time_evt_rx,
1121            mut system_evt_rx,
1122            mut system_cmd_rx,
1123            mut exec_evt_rx,
1124            mut exec_cmd_rx,
1125            mut data_evt_rx,
1126            mut data_cmd_rx,
1127        } = runner.take_channels();
1128
1129        log::info!("Event loop starting");
1130
1131        self.handle.set_starting();
1132        self.kernel.reset_shutdown_flag();
1133        self.kernel.start_async().await;
1134
1135        if self.kernel.is_event_store_replay() {
1136            log::info!(
1137                "Event-store replay loaded; skipping live client connection and reconciliation",
1138            );
1139
1140            if !self.finish_startup_replay().await? {
1141                return Ok(());
1142            }
1143
1144            return Ok(());
1145        }
1146
1147        if self.kernel.is_event_store_replay_configured() {
1148            self.abort_startup("Event-store replay did not start")
1149                .await?;
1150            return Ok(());
1151        }
1152
1153        let mut external_msgbus_rx = match self.take_external_ingress_receiver() {
1154            Ok(rx) => rx,
1155            Err(e) => {
1156                let result = self
1157                    .abort_startup("External message bus ingress failed to start")
1158                    .await;
1159                Self::drain_channels(
1160                    &mut time_evt_rx,
1161                    &mut system_evt_rx,
1162                    &mut system_cmd_rx,
1163                    &mut exec_evt_rx,
1164                    &mut exec_cmd_rx,
1165                    &mut data_evt_rx,
1166                    &mut data_cmd_rx,
1167                );
1168                log::info!("Event loop stopped");
1169
1170                if let Err(finalize_err) = result {
1171                    anyhow::bail!(
1172                        "failed to start external message bus ingress: {e}; failed to finalize startup abort: {finalize_err}"
1173                    );
1174                }
1175
1176                return Err(e);
1177            }
1178        };
1179
1180        let stop_handle = self.handle.clone();
1181        let mut pending = PendingEvents::default();
1182        let mut startup_system_events = Vec::new();
1183        let mut startup_system_commands = Vec::new();
1184        let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
1185
1186        // Startup phase 1: Connect data clients and drain instrument events into cache.
1187        // This ensures the cache is populated before execution clients connect.
1188        let mut pending_external = VecDeque::new();
1189        let data_connect_result = drive_with_event_buffering(
1190            drive_with_external_ingress(
1191                self.connect_data_phase(connection_deadline),
1192                &mut external_msgbus_rx,
1193                &mut pending_external,
1194            ),
1195            &mut pending,
1196            &mut time_evt_rx,
1197            &mut system_evt_rx,
1198            &mut system_cmd_rx,
1199            &mut exec_evt_rx,
1200            &mut exec_cmd_rx,
1201            &mut data_evt_rx,
1202            &mut data_cmd_rx,
1203        )
1204        .await;
1205
1206        self.finish_external_ingress_connection(external_msgbus_rx.as_ref());
1207
1208        if let Err(e) = data_connect_result {
1209            flush_all_pending(
1210                &mut pending,
1211                &mut time_evt_rx,
1212                &mut system_evt_rx,
1213                &mut system_cmd_rx,
1214                &mut exec_evt_rx,
1215                &mut exec_cmd_rx,
1216                &mut data_evt_rx,
1217                &mut data_cmd_rx,
1218            );
1219            let result = self
1220                .abort_startup_with_error("Data client connection failed", e)
1221                .await;
1222            Self::drain_channels(
1223                &mut time_evt_rx,
1224                &mut system_evt_rx,
1225                &mut system_cmd_rx,
1226                &mut exec_evt_rx,
1227                &mut exec_cmd_rx,
1228                &mut data_evt_rx,
1229                &mut data_cmd_rx,
1230            );
1231            log::info!("Event loop stopped");
1232            return result;
1233        }
1234
1235        // Flush any data events still queued in the channel receivers that the
1236        // select loop did not capture before the connect future resolved, then
1237        // drain everything into cache.
1238        flush_pending_data(&mut pending, &mut data_evt_rx, &mut data_cmd_rx);
1239        startup_system_events.extend(pending.take_system_events());
1240        startup_system_commands.extend(pending.take_system_commands());
1241        debug_assert!(
1242            pending.data_evts.is_empty() && pending.data_cmds.is_empty(),
1243            "data must be drained into cache before exec clients connect",
1244        );
1245
1246        // Startup phase 2: Connect execution clients (instruments now in cache)
1247        let engine_connection_result = drive_with_event_buffering(
1248            self.connect_exec_phase(connection_deadline),
1249            &mut pending,
1250            &mut time_evt_rx,
1251            &mut system_evt_rx,
1252            &mut system_cmd_rx,
1253            &mut exec_evt_rx,
1254            &mut exec_cmd_rx,
1255            &mut data_evt_rx,
1256            &mut data_cmd_rx,
1257        )
1258        .await;
1259
1260        // Flush channel receivers and drain all remaining pending events
1261        flush_all_pending(
1262            &mut pending,
1263            &mut time_evt_rx,
1264            &mut system_evt_rx,
1265            &mut system_cmd_rx,
1266            &mut exec_evt_rx,
1267            &mut exec_cmd_rx,
1268            &mut data_evt_rx,
1269            &mut data_cmd_rx,
1270        );
1271        startup_system_events.extend(pending.take_system_events());
1272        startup_system_commands.extend(pending.take_system_commands());
1273        debug_assert!(
1274            pending.is_empty(),
1275            "all startup events must be processed before reconciliation",
1276        );
1277
1278        let engine_connection_status = match engine_connection_result {
1279            Ok(status) => status,
1280            Err(e) => {
1281                let result = self
1282                    .abort_startup_with_error("Execution client connection failed", e)
1283                    .await;
1284                Self::drain_channels(
1285                    &mut time_evt_rx,
1286                    &mut system_evt_rx,
1287                    &mut system_cmd_rx,
1288                    &mut exec_evt_rx,
1289                    &mut exec_cmd_rx,
1290                    &mut data_evt_rx,
1291                    &mut data_cmd_rx,
1292                );
1293                log::info!("Event loop stopped");
1294                return result;
1295            }
1296        };
1297
1298        if engine_connection_status == EngineConnectionStatus::TimedOut {
1299            let result = self
1300                .abort_startup_with_error(
1301                    "Engine readiness timed out",
1302                    anyhow::anyhow!("readiness timeout while waiting for engine connections"),
1303                )
1304                .await;
1305            Self::drain_channels(
1306                &mut time_evt_rx,
1307                &mut system_evt_rx,
1308                &mut system_cmd_rx,
1309                &mut exec_evt_rx,
1310                &mut exec_cmd_rx,
1311                &mut data_evt_rx,
1312                &mut data_cmd_rx,
1313            );
1314            log::info!("Event loop stopped");
1315            return result;
1316        }
1317
1318        if let Some(reason) = engine_connection_status
1319            .abort_reason()
1320            .or_else(|| self.startup_abort_reason())
1321        {
1322            self.abort_startup(reason).await?;
1323            Self::drain_channels(
1324                &mut time_evt_rx,
1325                &mut system_evt_rx,
1326                &mut system_cmd_rx,
1327                &mut exec_evt_rx,
1328                &mut exec_cmd_rx,
1329                &mut data_evt_rx,
1330                &mut data_cmd_rx,
1331            );
1332            log::info!("Event loop stopped");
1333            return Ok(());
1334        }
1335
1336        debug_assert_eq!(engine_connection_status, EngineConnectionStatus::Connected);
1337
1338        // Run reconciliation now that instruments are in cache and start trader
1339        if let Err(e) = self.perform_startup_reconciliation().await {
1340            let result = self.abort_startup("Startup reconciliation failed").await;
1341            Self::drain_channels(
1342                &mut time_evt_rx,
1343                &mut system_evt_rx,
1344                &mut system_cmd_rx,
1345                &mut exec_evt_rx,
1346                &mut exec_cmd_rx,
1347                &mut data_evt_rx,
1348                &mut data_cmd_rx,
1349            );
1350            log::info!("Event loop stopped");
1351
1352            if let Err(finalize_err) = result {
1353                anyhow::bail!(
1354                    "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
1355                );
1356            }
1357
1358            return Err(e);
1359        }
1360
1361        if let Some(reason) = self.startup_abort_reason() {
1362            let result = self.abort_startup(reason).await;
1363            Self::drain_channels(
1364                &mut time_evt_rx,
1365                &mut system_evt_rx,
1366                &mut system_cmd_rx,
1367                &mut exec_evt_rx,
1368                &mut exec_cmd_rx,
1369                &mut data_evt_rx,
1370                &mut data_cmd_rx,
1371            );
1372            log::info!("Event loop stopped");
1373            return result;
1374        }
1375
1376        if let Err(e) = self.kernel.start_trader() {
1377            let result = self.abort_after_trader_start_failure(e).await;
1378            Self::drain_channels(
1379                &mut time_evt_rx,
1380                &mut system_evt_rx,
1381                &mut system_cmd_rx,
1382                &mut exec_evt_rx,
1383                &mut exec_cmd_rx,
1384                &mut data_evt_rx,
1385                &mut data_cmd_rx,
1386            );
1387            log::info!("Event loop stopped");
1388            return result;
1389        }
1390
1391        #[cfg(feature = "plugin")]
1392        if let Err(e) = self.plugins.start_controllers() {
1393            let result = self.abort_after_trader_start_failure(e).await;
1394            Self::drain_channels(
1395                &mut time_evt_rx,
1396                &mut system_evt_rx,
1397                &mut system_cmd_rx,
1398                &mut exec_evt_rx,
1399                &mut exec_cmd_rx,
1400                &mut data_evt_rx,
1401                &mut data_cmd_rx,
1402            );
1403            log::info!("Event loop stopped");
1404            return result;
1405        }
1406
1407        self.process_system_events(startup_system_events);
1408        self.process_system_commands(startup_system_commands);
1409
1410        let finish_result = {
1411            let mut receivers = RunnerReceivers {
1412                time_evt: &mut time_evt_rx,
1413                system_evt: &mut system_evt_rx,
1414                system_cmd: &mut system_cmd_rx,
1415                data_evt: &mut data_evt_rx,
1416                data_cmd: &mut data_cmd_rx,
1417                exec_evt: &mut exec_evt_rx,
1418                exec_cmd: &mut exec_cmd_rx,
1419            };
1420
1421            self.finish_startup_trader(Some(&mut receivers)).await
1422        };
1423
1424        match finish_result {
1425            Ok(true) => {}
1426            result => {
1427                log::info!("Event loop stopped");
1428                return result.map(|_| ());
1429            }
1430        }
1431
1432        let exec_config = &self.config.exec_engine;
1433        let inflight_interval =
1434            Duration::from_millis(u64::from(exec_config.inflight_check_interval_ms));
1435
1436        let open_interval = exec_config
1437            .open_check_interval_secs
1438            .filter(|&s| s > 0.0)
1439            .map_or(Duration::ZERO, |secs| {
1440                Duration::from_nanos(secs_to_nanos_unchecked(secs))
1441            });
1442
1443        let position_interval = exec_config
1444            .position_check_interval_secs
1445            .filter(|&s| s > 0.0)
1446            .map_or(Duration::ZERO, |secs| {
1447                Duration::from_nanos(secs_to_nanos_unchecked(secs))
1448            });
1449
1450        let has_clients = !self
1451            .kernel
1452            .exec_engine
1453            .borrow()
1454            .get_all_clients()
1455            .is_empty();
1456        let recon_enabled = has_clients
1457            && (!inflight_interval.is_zero()
1458                || !open_interval.is_zero()
1459                || !position_interval.is_zero());
1460
1461        let recon_min_interval = if recon_enabled {
1462            let mut intervals = Vec::new();
1463
1464            if !inflight_interval.is_zero() {
1465                intervals.push(inflight_interval);
1466            }
1467
1468            if !open_interval.is_zero() {
1469                intervals.push(open_interval);
1470            }
1471
1472            if !position_interval.is_zero() {
1473                intervals.push(position_interval);
1474            }
1475
1476            intervals
1477                .into_iter()
1478                .min()
1479                .unwrap_or(Duration::from_secs(1))
1480        } else {
1481            Duration::from_secs(1) // Unused, timer won't fire
1482        };
1483
1484        // `reconciliation_startup_delay_secs` is a post-reconciliation grace period:
1485        // startup reconciliation has already completed above, and this delay offsets
1486        // the first periodic tick to let the system stabilize before continuous checks
1487        // begin.
1488        let startup_delay = if self.config.exec_engine.reconciliation {
1489            Duration::from_secs_f64(exec_config.reconciliation_startup_delay_secs)
1490        } else {
1491            Duration::ZERO
1492        };
1493
1494        let recon_start = dst::time::Instant::now() + startup_delay;
1495
1496        let mut last_inflight_check = dst::time::Instant::now();
1497        let mut last_open_check = last_inflight_check;
1498        let mut last_position_check = last_inflight_check;
1499
1500        // Per-task `(interval, next_fire)` schedules dispatched by the
1501        // shared `maintenance_timer` below. See module docs for rationale.
1502        let far_future = Duration::from_hours(24 * 365 * 100);
1503
1504        let make_schedule = |opt_dur: Option<Duration>| -> (Duration, dst::time::Instant) {
1505            let dur = opt_dur.unwrap_or(far_future);
1506            (dur, recon_start + dur)
1507        };
1508
1509        let (recon_interval, mut recon_next) = make_schedule(if recon_enabled {
1510            Some(recon_min_interval)
1511        } else {
1512            None
1513        });
1514
1515        let (purge_orders_interval, mut purge_orders_next) = make_schedule(
1516            exec_config
1517                .purge_closed_orders_interval_mins
1518                .filter(|&m| m > 0)
1519                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1520        );
1521
1522        let (purge_positions_interval, mut purge_positions_next) = make_schedule(
1523            exec_config
1524                .purge_closed_positions_interval_mins
1525                .filter(|&m| m > 0)
1526                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1527        );
1528
1529        let (purge_account_interval, mut purge_account_next) = make_schedule(
1530            exec_config
1531                .purge_account_events_interval_mins
1532                .filter(|&m| m > 0)
1533                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1534        );
1535
1536        let (own_books_interval, mut own_books_next) = make_schedule(
1537            exec_config
1538                .own_books_audit_interval_secs
1539                .filter(|&s| s > 0.0)
1540                .map(Duration::from_secs_f64),
1541        );
1542
1543        let (prune_fills_interval, mut prune_fills_next) =
1544            make_schedule(Some(Duration::from_mins(1)));
1545
1546        let mut maintenance_timer = dst::time::interval(Duration::from_millis(100));
1547        maintenance_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1548
1549        // Stop-check timer is not subject to the reconciliation startup delay,
1550        // so shutdown signals remain responsive from the moment the node reaches
1551        // `Running`. Set `MissedTickBehavior::Skip` so backlog ticks do not fire
1552        // a burst after the select arm was suspended by other branches.
1553        let mut stop_check_timer = dst::time::interval(Duration::from_millis(100));
1554        stop_check_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1555
1556        // Running phase: runs until shutdown deadline expires
1557        let mut residual_events = 0usize;
1558        let mut open_order_report_task: Option<OpenOrderReportTask> = None;
1559        let mut targeted_order_report_task: Option<TargetedOrderReportTask> = None;
1560        let mut position_report_task: Option<PositionReportTask> = None;
1561
1562        // A hosted node never installs signal handlers, so these futures stay pending and their
1563        // listeners are never registered. Both arms resolve to the same type as the real listeners.
1564        let owns_signals = mode.owns_signals();
1565
1566        let ctrl_c = async move {
1567            if owns_signals {
1568                dst::signal::ctrl_c().await
1569            } else {
1570                std::future::pending::<std::io::Result<()>>().await
1571            }
1572        };
1573
1574        let terminate = async move {
1575            if owns_signals {
1576                dst::signal::terminate().await
1577            } else {
1578                std::future::pending::<std::io::Result<()>>().await
1579            }
1580        };
1581
1582        tokio::pin!(ctrl_c);
1583        tokio::pin!(terminate);
1584
1585        let metrics = self.handle.metrics.clone();
1586        let metrics_start = dst::time::Instant::now();
1587        metrics.reset();
1588
1589        let mut queue_monitor = self
1590            .config
1591            .queue_monitor
1592            .as_ref()
1593            .map(|config| QueueMonitor::new(config, metrics.snapshot()));
1594        let mut dispatches_since_yield = 0usize;
1595        let mut skip_time = false;
1596        let mut skip_data = false;
1597
1598        let dispatch_result = loop {
1599            let callbacks_pending = match drain_callbacks().await {
1600                Ok(pending) => pending,
1601                Err(e) => {
1602                    if self.state() == NodeState::Running {
1603                        self.initiate_shutdown();
1604                    }
1605
1606                    log::warn!(
1607                        "Skipping residual events and final buffered dispatch after callback failure"
1608                    );
1609
1610                    break Err(e);
1611                }
1612            };
1613
1614            let shutdown_deadline = self.shutdown_deadline;
1615            let is_shutting_down = self.state() == NodeState::ShuttingDown;
1616            let is_running = self.state() == NodeState::Running;
1617            let commands_empty = data_cmd_rx.is_empty();
1618            let poll_time = !skip_time || (data_evt_rx.is_empty() && commands_empty);
1619            let poll_data = !skip_data || commands_empty;
1620
1621            tokio::select! {
1622                biased;
1623
1624                // Signal branches first so they are always checked
1625                result = &mut ctrl_c, if is_running => {
1626                    match result {
1627                        Ok(()) => log::info!("Received SIGINT, shutting down"),
1628                        Err(e) => log::error!("Failed to listen for SIGINT: {e}"),
1629                    }
1630                    self.initiate_shutdown();
1631                }
1632                result = &mut terminate, if is_running => {
1633                    match result {
1634                        Ok(()) => log::info!("Received SIGTERM, shutting down"),
1635                        Err(e) => log::error!("Failed to listen for SIGTERM: {e}"),
1636                    }
1637                    self.initiate_shutdown();
1638                }
1639                _ = stop_check_timer.tick(), if is_running => {
1640                    if stop_handle.should_stop() {
1641                        log::info!("Received stop signal from handle");
1642                        self.initiate_shutdown();
1643                    } else if self.kernel.is_shutdown_requested() {
1644                        log::info!("Received ShutdownSystem command, shutting down");
1645                        self.initiate_shutdown();
1646                    }
1647                }
1648                () = async {
1649                    match shutdown_deadline {
1650                        Some(deadline) => dst::time::sleep_until(deadline).await,
1651                        None => std::future::pending::<()>().await,
1652                    }
1653                }, if self.state() == NodeState::ShuttingDown => {
1654                    break Ok(());
1655                }
1656                () = std::future::ready(()), if callbacks_pending => {},
1657                result = async {
1658                    match open_order_report_task.as_mut() {
1659                        Some(task) => task.future.as_mut().await,
1660                        None => std::future::pending::<ReportTaskOutcome<OpenOrderReportCollection>>().await,
1661                    }
1662                }, if open_order_report_task.is_some() => {
1663                    let maintenance_start = dst::time::Instant::now();
1664
1665                    drop(open_order_report_task.take());
1666
1667                    match result {
1668                        ReportTaskOutcome::Completed(result) => {
1669                            let result = result.finish();
1670                            self.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
1671                            let client_refs = self
1672                                .exec_clients
1673                                .iter()
1674                                .map(|client| client as &dyn ExecutionClient)
1675                                .collect::<Vec<_>>();
1676                            let reconciliation = self.exec_manager.reconcile_open_order_reports(
1677                                &result.check,
1678                                result.reports,
1679                                &result.queried_clients,
1680                                &result.failed_clients,
1681                                &client_refs,
1682                            );
1683                            self.process_reconciliation_events(&reconciliation.events);
1684                            if !reconciliation.targeted_queries.is_empty() {
1685                                if is_shutting_down {
1686                                    let planned_client_order_ids = reconciliation
1687                                        .targeted_queries
1688                                        .iter()
1689                                        .map(TargetedOrderQuery::client_order_id)
1690                                        .collect::<Vec<_>>();
1691                                    self.cleanup_cancelled_report_tasks(
1692                                        &planned_client_order_ids,
1693                                    );
1694                                } else {
1695                                    targeted_order_report_task = Some(
1696                                        self.start_targeted_order_report_check(
1697                                            reconciliation.targeted_queries,
1698                                        ),
1699                                    );
1700                                }
1701                            }
1702                        }
1703                        ReportTaskOutcome::TimedOut => {
1704                            self.cleanup_cancelled_report_tasks(&[]);
1705                            log::warn!(
1706                                "Open-order report collection expired after {:?}",
1707                                self.config.timeout_reconciliation,
1708                            );
1709                        }
1710                    }
1711                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1712                }
1713                result = async {
1714                    match targeted_order_report_task.as_mut() {
1715                        Some(task) => task.future.as_mut().await,
1716                        None => std::future::pending::<ReportTaskOutcome<()>>().await,
1717                    }
1718                }, if targeted_order_report_task.is_some() => {
1719                    let maintenance_start = dst::time::Instant::now();
1720
1721                    if let Some(task) = targeted_order_report_task.take() {
1722                        self.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
1723                        self.complete_targeted_order_report_check(task, &result);
1724                    }
1725                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1726                }
1727                result = async {
1728                    match position_report_task.as_mut() {
1729                        Some(task) => task.future.as_mut().await,
1730                        None => std::future::pending::<ReportTaskOutcome<PositionReportTaskResult>>().await,
1731                    }
1732                }, if position_report_task.is_some() => {
1733                    let maintenance_start = dst::time::Instant::now();
1734
1735                    drop(position_report_task.take());
1736
1737                    match result {
1738                        ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) => {
1739                            if is_shutting_down {
1740                                self.cleanup_cancelled_report_tasks(&[]);
1741                            } else {
1742                                let result = result.finish();
1743                                position_report_task = self.handle_position_report_result(result);
1744                            }
1745                        }
1746                        ReportTaskOutcome::Completed(PositionReportTaskResult::Fills { position_result, collection }) => {
1747                            let reports = collection.finish();
1748                            self.handle_position_fill_report_result(PositionFillReportResult {
1749                                position_result,
1750                                reports,
1751                            });
1752                        }
1753                        ReportTaskOutcome::TimedOut => {
1754                            self.cleanup_cancelled_report_tasks(&[]);
1755                            log::warn!(
1756                                "Position report collection expired after {:?}",
1757                                self.config.timeout_reconciliation,
1758                            );
1759                        }
1760                    }
1761                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1762                }
1763
1764                // Maintenance dispatcher (before event processing to avoid
1765                // starvation). See module docs for design rationale.
1766                _ = maintenance_timer.tick(), if is_running => {
1767                    let maintenance_start = dst::time::Instant::now();
1768                    metrics.publish_queue_depths(
1769                        RunnerChannelQueueDepths::from_receivers(
1770                            &time_evt_rx,
1771                            &exec_evt_rx,
1772                            &exec_cmd_rx,
1773                            &data_evt_rx,
1774                            &data_cmd_rx,
1775                        ),
1776                        metrics_start.elapsed(),
1777                    );
1778
1779                    if let Some(queue_monitor) = queue_monitor.as_mut() {
1780                        let transitions = queue_monitor.evaluate(metrics.snapshot());
1781                        self.publish_queue_state_transitions(&transitions);
1782                    }
1783
1784                    let mut now = dst::time::Instant::now();
1785
1786                    if recon_enabled && now >= recon_next {
1787                        let recon_intervals = ReconciliationCheckIntervals {
1788                            inflight: inflight_interval,
1789                            open: open_interval,
1790                            position: position_interval,
1791                        };
1792                        let mut recon_state = ReconciliationCheckState {
1793                            last_inflight_check: &mut last_inflight_check,
1794                            last_open_check: &mut last_open_check,
1795                            last_position_check: &mut last_position_check,
1796                            open_order_report_task: &mut open_order_report_task,
1797                            targeted_order_report_task: &mut targeted_order_report_task,
1798                            position_report_task: &mut position_report_task,
1799                        };
1800
1801                        self.run_reconciliation_checks(
1802                            now,
1803                            recon_intervals,
1804                            &mut recon_state,
1805                        );
1806
1807                        now = dst::time::Instant::now();
1808                        recon_next = now + recon_interval;
1809                    }
1810
1811                    if now >= purge_orders_next {
1812                        self.exec_manager.purge_closed_orders();
1813                        purge_orders_next = now + purge_orders_interval;
1814                    }
1815
1816                    if now >= purge_positions_next {
1817                        self.exec_manager.purge_closed_positions();
1818                        purge_positions_next = now + purge_positions_interval;
1819                    }
1820
1821                    if now >= purge_account_next {
1822                        self.exec_manager.purge_account_events();
1823                        purge_account_next = now + purge_account_interval;
1824                    }
1825
1826                    if now >= own_books_next {
1827                        self.kernel.cache().borrow_mut().audit_own_order_books();
1828                        own_books_next = now + own_books_interval;
1829                    }
1830
1831                    if now >= prune_fills_next {
1832                        self.exec_manager.prune_recent_fills_cache(60.0);
1833                        self.exec_manager.prune_processed_fills();
1834                        self.exec_manager.prune_order_local_activity();
1835                        prune_fills_next = now + prune_fills_interval;
1836                    }
1837
1838                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1839                }
1840
1841                // Event processing branches. Exec commands and events are
1842                // ordered ahead of data events so a strategy action (cancel,
1843                // submit, etc.) is not delayed behind a market data backlog
1844                // when the biased select polls receivers each iteration.
1845                Some(handler) = time_evt_rx.recv(), if poll_time => {
1846                    skip_time = true;
1847                    let dispatch_start = dst::time::Instant::now();
1848                    let dispatched = AsyncRunner::handle_time_event(handler);
1849
1850                    if dispatched && is_shutting_down {
1851                        log::debug!("Residual time event");
1852                        residual_events += 1;
1853                    }
1854
1855                    if dispatched {
1856                        record_runner_dispatch(
1857                            &metrics,
1858                            SystemChannel::TimeEvents,
1859                            dispatch_start,
1860                            metrics_start,
1861                        );
1862                    }
1863                }
1864                Some(event) = system_evt_rx.recv() => {
1865                    skip_time = false;
1866
1867                    if is_shutting_down {
1868                        log::debug!("Residual system event: {event}");
1869                        residual_events += 1;
1870                    }
1871                    event.dispatch(|event| self.process_system_event(event));
1872                }
1873                Some(command) = system_cmd_rx.recv() => {
1874                    skip_time = false;
1875
1876                    if is_shutting_down {
1877                        log::debug!("Residual system command: {command}");
1878                        residual_events += 1;
1879                    }
1880                    command.dispatch(|command| self.process_system_command(command));
1881                }
1882                Some(evt) = exec_evt_rx.recv() => {
1883                    skip_time = false;
1884                    let dispatch_start = dst::time::Instant::now();
1885
1886                    if is_shutting_down {
1887                        log::debug!("Residual exec event: {evt}");
1888                        residual_events += 1;
1889                    }
1890
1891                    evt.dispatch(|evt| self.process_exec_event(evt));
1892                    record_runner_dispatch(
1893                        &metrics,
1894                        SystemChannel::ExecEvents,
1895                        dispatch_start,
1896                        metrics_start,
1897                    );
1898                }
1899                Some(cmd) = exec_cmd_rx.recv() => {
1900                    skip_time = false;
1901                    let dispatch_start = dst::time::Instant::now();
1902
1903                    if is_shutting_down {
1904                        log::debug!("Residual exec command: {cmd}");
1905                        residual_events += 1;
1906                    }
1907
1908                    self.process_exec_command(cmd);
1909                    record_runner_dispatch(
1910                        &metrics,
1911                        SystemChannel::ExecCommands,
1912                        dispatch_start,
1913                        metrics_start,
1914                    );
1915                }
1916                message = recv_external_msgbus_message(&mut external_msgbus_rx, &mut pending_external) => {
1917                    skip_time = false;
1918                    let external_msgbus_start = dst::time::Instant::now();
1919
1920                    match message {
1921                        Some(message) => {
1922                            if is_shutting_down {
1923                                log::debug!("Residual external message bus message: {message}");
1924                                residual_events += 1;
1925                            }
1926                            self.process_external_msgbus_message(&message);
1927                        }
1928                        None => {
1929                            log::info!("External message bus ingress closed");
1930                            external_msgbus_rx = None;
1931                            self.close_external_ingress();
1932                        }
1933                    }
1934
1935                    record_runner_external_msgbus(
1936                        &metrics,
1937                        external_msgbus_start,
1938                        metrics_start,
1939                    );
1940                }
1941                Some(evt) = data_evt_rx.recv(), if poll_data => {
1942                    skip_time = false;
1943                    skip_data = true;
1944                    let dispatch_start = dst::time::Instant::now();
1945
1946                    if is_shutting_down {
1947                        log::debug!("Residual data event: {evt:?}");
1948                        residual_events += 1;
1949                    }
1950                    AsyncRunner::dispatch_data_event(evt);
1951                    record_runner_dispatch(
1952                        &metrics,
1953                        SystemChannel::DataEvents,
1954                        dispatch_start,
1955                        metrics_start,
1956                    );
1957                }
1958                Some(cmd) = data_cmd_rx.recv() => {
1959                    skip_time = false;
1960                    skip_data = false;
1961                    let dispatch_start = dst::time::Instant::now();
1962
1963                    if is_shutting_down {
1964                        log::debug!("Residual data command: {cmd:?}");
1965                        residual_events += 1;
1966                    }
1967                    AsyncRunner::handle_data_command(cmd);
1968                    record_runner_dispatch(
1969                        &metrics,
1970                        SystemChannel::DataCommands,
1971                        dispatch_start,
1972                        metrics_start,
1973                    );
1974                }
1975            }
1976
1977            dispatches_since_yield += 1;
1978            if dispatches_since_yield >= DISPATCHES_PER_YIELD {
1979                dispatches_since_yield = 0;
1980                tokio::task::yield_now().await;
1981            }
1982        };
1983
1984        if residual_events > 0 {
1985            log::debug!("Processed {residual_events} residual events during shutdown");
1986        }
1987
1988        self.cancel_report_tasks(
1989            &mut open_order_report_task,
1990            &mut targeted_order_report_task,
1991            &mut position_report_task,
1992            dispatch_result.is_ok(),
1993        );
1994
1995        drop(external_msgbus_rx.take());
1996        let _ = self.kernel.cache().borrow().check_residuals();
1997
1998        let stop_result = self.finalize_stop().await;
1999
2000        if let Err(e) = dispatch_result {
2001            if let Err(stop_err) = stop_result {
2002                log::error!("Failed to finalize node after callback failure: {stop_err}");
2003                // Python converts the outer display message, so preserve both diagnostics there.
2004                let diagnostic = format!("{e}; {stop_err}");
2005                return Err(anyhow::Error::new(e).context(diagnostic));
2006            }
2007
2008            return Err(e.into());
2009        }
2010
2011        // Handle events that arrived during finalize_stop
2012        Self::drain_channels(
2013            &mut time_evt_rx,
2014            &mut system_evt_rx,
2015            &mut system_cmd_rx,
2016            &mut exec_evt_rx,
2017            &mut exec_cmd_rx,
2018            &mut data_evt_rx,
2019            &mut data_cmd_rx,
2020        );
2021
2022        log::info!("Event loop stopped");
2023
2024        stop_result
2025    }
2026
2027    fn publish_queue_state_transitions(&self, transitions: &[QueueStateTransition]) {
2028        for transition in transitions {
2029            let topic = MessagingSwitchboard::queue_state_changed_topic(transition.channel);
2030            let timestamp = self.kernel.generate_timestamp_ns();
2031
2032            let event = QueueStateChanged::new(
2033                self.config.trader_id,
2034                transition.channel,
2035                transition.condition,
2036                transition.state,
2037                transition.queue_depth,
2038                transition.mean_dispatch_ns,
2039                UUID4::new(),
2040                timestamp,
2041                timestamp,
2042            );
2043
2044            msgbus::publish_any(topic, event.as_any());
2045        }
2046    }
2047
2048    #[expect(
2049        clippy::await_holding_refcell_ref,
2050        reason = "cache loading is serialized before the single-threaded live node starts"
2051    )]
2052    async fn prepare_cache(&mut self) -> anyhow::Result<()> {
2053        self.install_cache_database().await?;
2054
2055        let cache = self.kernel.cache();
2056        if !cache.borrow().has_backing() {
2057            return Ok(());
2058        }
2059
2060        if self
2061            .config
2062            .cache
2063            .as_ref()
2064            .is_some_and(|config| config.flush_on_start)
2065        {
2066            cache
2067                .borrow_mut()
2068                .flush_db()
2069                .context("Failed to flush persistent cache")?;
2070            return Ok(());
2071        }
2072
2073        if self.config.exec_engine.load_cache {
2074            self.kernel
2075                .exec_engine()
2076                .borrow_mut()
2077                .load_cache()
2078                .await
2079                .context("Failed to load persistent cache")?;
2080        }
2081
2082        Ok(())
2083    }
2084
2085    /// Returns whether a cache database backing is configured but not yet installed.
2086    #[must_use]
2087    pub const fn has_pending_cache_database(&self) -> bool {
2088        self.cache_database_factory.is_some()
2089    }
2090
2091    /// Constructs the configured cache database backing and installs it on the kernel cache.
2092    ///
2093    /// Construction is deferred to startup so the adapter is built inside the async runtime rather
2094    /// than by blocking the synchronous builder, and so the connection opens only when the node runs.
2095    async fn install_cache_database(&mut self) -> anyhow::Result<()> {
2096        let Some(factory) = self.cache_database_factory.as_ref() else {
2097            return Ok(());
2098        };
2099
2100        let config = self.config.cache.clone().unwrap_or_default();
2101
2102        // Cleared only after a successful install, so a failed startup can be retried rather than
2103        // silently starting without the backing the caller asked for.
2104        let database = factory
2105            .create(self.config.trader_id, self.kernel.instance_id, config)
2106            .await
2107            .context("failed to create cache database backing")?;
2108        self.kernel.cache().borrow_mut().set_database(database);
2109        self.cache_database_factory = None;
2110
2111        Ok(())
2112    }
2113
2114    fn take_external_ingress_receiver(
2115        &mut self,
2116    ) -> anyhow::Result<Option<tokio::sync::mpsc::Receiver<BusMessage>>> {
2117        let Some(external_ingress) = self.external_msgbus.as_mut() else {
2118            return Ok(None);
2119        };
2120
2121        let receiver = external_ingress.take_receiver()?;
2122        log::info!("External message bus ingress started");
2123        Ok(Some(receiver))
2124    }
2125
2126    fn republish_external_msgbus_message(message: &BusMessage) {
2127        if let Err(e) = msgbus::republish_external_message(message) {
2128            log::error!(
2129                "Failed to republish external message bus topic '{}': {e:#}",
2130                message.topic
2131            );
2132        }
2133    }
2134
2135    fn process_external_msgbus_message(&self, message: &BusMessage) {
2136        if self.stream_processors.is_empty() {
2137            Self::republish_external_msgbus_message(message);
2138            return;
2139        }
2140
2141        let mut process = |value: &dyn Any, mapping: &serde_json::Value| {
2142            for processor in &self.stream_processors {
2143                (processor.0)(value, mapping)?;
2144            }
2145
2146            Ok(())
2147        };
2148
2149        if let Err(e) = msgbus::process_external_typed_message(message, &mut process) {
2150            log::error!(
2151                "Failed to process external message bus topic '{}': {e:#}",
2152                message.topic
2153            );
2154        }
2155    }
2156
2157    fn finish_external_ingress_connection(
2158        &mut self,
2159        receiver: Option<&tokio::sync::mpsc::Receiver<BusMessage>>,
2160    ) {
2161        if receiver.is_none() {
2162            self.close_external_ingress();
2163        }
2164    }
2165
2166    fn close_external_ingress(&mut self) {
2167        if let Some(external_ingress) = self.external_msgbus.as_mut()
2168            && !external_ingress.is_closed()
2169        {
2170            external_ingress.close();
2171        }
2172    }
2173
2174    fn drain_exec_events_before_reconciliation(
2175        &mut self,
2176        exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2177    ) {
2178        while let Ok(event) = exec_evt_rx.try_recv() {
2179            event.dispatch(|event| self.process_exec_event(event));
2180        }
2181    }
2182
2183    fn process_reconciliation_events(&mut self, events: &[OrderEventAny]) {
2184        if events.is_empty() {
2185            self.publish_submission_recovery_exhaustions();
2186            return;
2187        }
2188
2189        log::info!(
2190            "Processing {} reconciliation event{}",
2191            events.len(),
2192            if events.len() == 1 { "" } else { "s" }
2193        );
2194
2195        for event in events {
2196            self.exec_manager
2197                .record_local_activity(event.client_order_id());
2198            if let OrderEventAny::Filled(fill) = event {
2199                self.exec_manager
2200                    .record_position_activity(fill.instrument_id, fill.account_id);
2201            }
2202
2203            self.kernel.exec_engine.borrow_mut().process(event);
2204            self.exec_manager
2205                .confirm_submission_outcome(&event.client_order_id());
2206            if let OrderEventAny::Filled(fill) = event {
2207                self.exec_manager.commit_recent_fill_if_applied(fill);
2208            }
2209        }
2210
2211        self.publish_submission_recovery_exhaustions();
2212    }
2213
2214    fn publish_submission_recovery_exhaustions(&mut self) {
2215        for diagnostic in self.exec_manager.take_submission_recovery_exhaustions() {
2216            log::warn!(
2217                "Submission recovery exhausted for {}: {:?} after {} checks; submission remains unresolved pending venue evidence",
2218                diagnostic.client_order_id,
2219                diagnostic.source,
2220                diagnostic.retry_count,
2221            );
2222            msgbus::publish_any(
2223                MessagingSwitchboard::submission_recovery_exhausted_topic(),
2224                &diagnostic,
2225            );
2226        }
2227    }
2228
2229    fn process_exec_event(&mut self, event: ExecutionEvent) {
2230        let clear_closed_orders = matches!(
2231            &event,
2232            ExecutionEvent::Order(_) | ExecutionEvent::OrderCanceledBatch(_)
2233        );
2234        let closed_report_id = match &event {
2235            ExecutionEvent::Report(
2236                ExecutionReport::Order(report) | ExecutionReport::OrderWithFills(report, _),
2237            ) if report.order_status.is_closed() => report.client_order_id,
2238            _ => None,
2239        };
2240        let Some(order_ids) = self.observe_exec_event_before_dispatch(&event) else {
2241            return;
2242        };
2243
2244        self.dispatch_exec_event_and_commit_fill(event);
2245
2246        for client_order_id in &order_ids {
2247            self.exec_manager
2248                .confirm_submission_outcome(client_order_id);
2249            let is_closed = self
2250                .kernel
2251                .cache()
2252                .borrow()
2253                .order(client_order_id)
2254                .is_some_and(|order| order.is_closed());
2255
2256            if is_closed && (clear_closed_orders || closed_report_id == Some(*client_order_id)) {
2257                self.exec_manager
2258                    .clear_recon_tracking(client_order_id, true);
2259            }
2260        }
2261    }
2262
2263    fn process_exec_command(&mut self, message: DispatchMessage<TradingCommandMessage>) {
2264        message.dispatch_trading(|message| {
2265            if message.endpoint() == MessagingSwitchboard::exec_engine_execute() {
2266                self.observe_exec_command_before_dispatch(message.command());
2267            }
2268        });
2269    }
2270
2271    /// Dispatches a normal-ingress execution event, then commits a direct
2272    /// `OrderFilled` to the recent-fills dedup cache only once it is present on
2273    /// its canonical order.
2274    ///
2275    /// The fill candidate is captured from `&evt` BEFORE the value is moved into
2276    /// [`AsyncRunner::handle_exec_event`]; the gated commit runs AFTER dispatch,
2277    /// so a fill the execution engine rejects (unknown order, invalid
2278    /// transition) is never marked and its later `Fill` report stays eligible.
2279    fn dispatch_exec_event_and_commit_fill(&mut self, evt: ExecutionEvent) {
2280        let recent_fill_candidate = match &evt {
2281            ExecutionEvent::Order(OrderEventAny::Filled(fill)) => Some(fill.clone()),
2282            _ => None,
2283        };
2284
2285        AsyncRunner::handle_exec_event(evt);
2286
2287        if let Some(fill) = &recent_fill_candidate {
2288            self.exec_manager.commit_recent_fill_if_applied(fill);
2289        }
2290    }
2291
2292    async fn connect_data_phase(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2293        // A zero remaining budget still admits an immediately-ready connect (an
2294        // empty/ready client set completes on the first poll); a pending connect
2295        // fails closed on that same poll. This keeps a zero `timeout_connection`
2296        // - a supported "do not wait, but allow ready work" configuration -
2297        // working, while still bounding a hung connect.
2298        let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2299        dst::time::timeout(remaining, self.kernel.connect_data_clients())
2300            .await
2301            .map_err(|_| anyhow::anyhow!("data-connect timeout"))?
2302    }
2303
2304    async fn connect_exec_clients(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2305        let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2306        dst::time::timeout(remaining, self.kernel.connect_exec_clients())
2307            .await
2308            .map_err(|_| anyhow::anyhow!("exec-connect timeout"))?
2309    }
2310
2311    /// Connects execution clients and checks all engines are connected.
2312    ///
2313    /// Returns the final connection wait status.
2314    /// Must be called after data clients are connected and instrument events drained.
2315    async fn connect_exec_phase(
2316        &mut self,
2317        deadline: dst::time::Instant,
2318    ) -> anyhow::Result<EngineConnectionStatus> {
2319        self.connect_exec_clients(deadline).await?;
2320        Ok(self.await_engines_connected(deadline).await)
2321    }
2322
2323    fn startup_abort_reason(&self) -> Option<&'static str> {
2324        if self.handle.should_stop() {
2325            Some("Stop signal received during startup")
2326        } else if self.kernel.is_shutdown_requested() {
2327            Some("Shutdown signal received during startup")
2328        } else {
2329            None
2330        }
2331    }
2332
2333    async fn finish_startup_replay(&mut self) -> anyhow::Result<bool> {
2334        match self.handle.try_set_running() {
2335            RunningTransition::Entered => Ok(true),
2336            RunningTransition::StopRequested => {
2337                self.abort_startup("Stop signal received during startup")
2338                    .await?;
2339                Ok(false)
2340            }
2341            RunningTransition::Invalid(control) => {
2342                self.abort_startup_with_error(
2343                    "Invalid lifecycle state during startup",
2344                    anyhow::anyhow!(
2345                        "Invalid LiveNode control state {control:#04x} while entering Running"
2346                    ),
2347                )
2348                .await?;
2349                Ok(false)
2350            }
2351        }
2352    }
2353
2354    async fn finish_startup_trader(
2355        &mut self,
2356        receivers: Option<&mut RunnerReceivers<'_>>,
2357    ) -> anyhow::Result<bool> {
2358        match self.handle.try_set_running() {
2359            RunningTransition::Entered => Ok(true),
2360            RunningTransition::StopRequested => {
2361                self.abort_started_trader("Stop signal received during startup", receivers)
2362                    .await?;
2363                Ok(false)
2364            }
2365            RunningTransition::Invalid(control) => {
2366                let state_err = anyhow::anyhow!(
2367                    "Invalid LiveNode control state {control:#04x} while entering Running"
2368                );
2369
2370                match self
2371                    .abort_started_trader("Invalid lifecycle state during startup", receivers)
2372                    .await
2373                {
2374                    Ok(()) => Err(state_err),
2375                    Err(finalize_err) => {
2376                        anyhow::bail!(
2377                            "{state_err}; failed to finalize startup abort: {finalize_err}"
2378                        )
2379                    }
2380                }
2381            }
2382        }
2383    }
2384
2385    async fn abort_startup(&mut self, reason: &str) -> anyhow::Result<()> {
2386        log::info!("{reason}, aborting startup");
2387        self.handle.set_shutting_down();
2388        self.finalize_stop().await
2389    }
2390
2391    async fn abort_startup_with_error(
2392        &mut self,
2393        reason: &str,
2394        startup_err: anyhow::Error,
2395    ) -> anyhow::Result<()> {
2396        match self.abort_startup(reason).await {
2397            Ok(()) => Err(startup_err),
2398            Err(finalize_err) => {
2399                anyhow::bail!("{startup_err}; failed to finalize startup abort: {finalize_err}")
2400            }
2401        }
2402    }
2403
2404    async fn abort_started_trader(
2405        &mut self,
2406        reason: &str,
2407        mut receivers: Option<&mut RunnerReceivers<'_>>,
2408    ) -> anyhow::Result<()> {
2409        log::info!("{reason}, aborting startup");
2410        self.handle.set_shutting_down();
2411
2412        #[cfg(feature = "plugin")]
2413        let controller_stop_result = self.plugins.stop_controllers();
2414        #[cfg(not(feature = "plugin"))]
2415        let controller_stop_result: anyhow::Result<()> = Ok(());
2416
2417        let trader_stop_result = self.kernel.stop_trader_after_start_failure();
2418        let delay = self.kernel.delay_post_stop();
2419        log::info!("Awaiting residual events ({delay:?})...");
2420
2421        let residual_events = match receivers.as_mut() {
2422            Some(receivers) => self.process_receivers_for(delay, receivers).await,
2423            None => self.process_runner_for(delay).await,
2424        };
2425
2426        if residual_events > 0 {
2427            log::debug!("Processed {residual_events} residual events during shutdown");
2428        }
2429
2430        let finalize_result = self.finalize_stop().await;
2431
2432        if let Some(receivers) = receivers {
2433            Self::drain_channels(
2434                receivers.time_evt,
2435                receivers.system_evt,
2436                receivers.system_cmd,
2437                receivers.exec_evt,
2438                receivers.exec_cmd,
2439                receivers.data_evt,
2440                receivers.data_cmd,
2441            );
2442        } else {
2443            self.drain_runner_pending();
2444        }
2445
2446        let mut errors = Vec::new();
2447
2448        if let Err(e) = controller_stop_result {
2449            errors.push(format!("Failed to stop plug-in controllers: {e}"));
2450        }
2451
2452        if let Err(e) = trader_stop_result {
2453            errors.push(format!("Failed to stop trader: {e}"));
2454        }
2455
2456        if let Err(e) = finalize_result {
2457            errors.push(format!("Failed to finalize startup abort: {e}"));
2458        }
2459
2460        if errors.is_empty() {
2461            Ok(())
2462        } else {
2463            anyhow::bail!("{}", errors.join("; "))
2464        }
2465    }
2466
2467    async fn process_receivers_for(
2468        &mut self,
2469        duration: Duration,
2470        receivers: &mut RunnerReceivers<'_>,
2471    ) -> usize {
2472        let deadline = dst::time::Instant::now() + duration;
2473        let mut processed = 0;
2474
2475        loop {
2476            tokio::select! {
2477                biased;
2478
2479                () = dst::time::sleep_until(deadline) => break,
2480                Some(message) = receivers.time_evt.recv() => {
2481                    let _ = AsyncRunner::handle_time_event(message);
2482                    processed += 1;
2483                }
2484                Some(event) = receivers.system_evt.recv() => {
2485                    event.dispatch(|event| self.process_system_event(event));
2486                    processed += 1;
2487                }
2488                Some(command) = receivers.system_cmd.recv() => {
2489                    command.dispatch(|command| self.process_system_command(command));
2490                    processed += 1;
2491                }
2492                Some(event) = receivers.exec_evt.recv() => {
2493                    event.dispatch(|event| self.process_exec_event(event));
2494                    processed += 1;
2495                }
2496                Some(command) = receivers.exec_cmd.recv() => {
2497                    self.process_exec_command(command);
2498                    processed += 1;
2499                }
2500                Some(event) = receivers.data_evt.recv() => {
2501                    AsyncRunner::dispatch_data_event(event);
2502                    processed += 1;
2503                }
2504                Some(command) = receivers.data_cmd.recv() => {
2505                    AsyncRunner::handle_data_command(command);
2506                    processed += 1;
2507                }
2508            }
2509        }
2510
2511        processed
2512    }
2513
2514    async fn abort_after_trader_start_failure(
2515        &mut self,
2516        start_err: anyhow::Error,
2517    ) -> anyhow::Result<()> {
2518        log::info!("Trader startup failed, aborting startup");
2519        self.handle.set_shutting_down();
2520        let stop_result = self.kernel.stop_trader_after_start_failure();
2521        let finalize_result = self.finalize_stop().await;
2522
2523        match (stop_result, finalize_result) {
2524            (Ok(()), Ok(())) => Err(start_err),
2525            (Err(stop_err), Ok(())) => anyhow::bail!(
2526                "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2527                 {stop_err}"
2528            ),
2529            (Ok(()), Err(finalize_err)) => anyhow::bail!(
2530                "Failed during trader startup: {start_err}; failed to finalize startup abort: \
2531                 {finalize_err}"
2532            ),
2533            (Err(stop_err), Err(finalize_err)) => anyhow::bail!(
2534                "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2535                 {stop_err}; failed to finalize startup abort: {finalize_err}"
2536            ),
2537        }
2538    }
2539
2540    fn initiate_shutdown(&mut self) {
2541        #[cfg(feature = "plugin")]
2542        if let Err(e) = self.plugins.stop_controllers() {
2543            log::error!("Error stopping plug-in controllers: {e}");
2544        }
2545
2546        self.kernel.stop_trader();
2547        let delay = self.kernel.delay_post_stop();
2548        log::info!("Awaiting residual events ({delay:?})...");
2549
2550        self.shutdown_deadline = Some(dst::time::Instant::now() + delay);
2551        self.handle.set_shutting_down();
2552    }
2553
2554    async fn finalize_stop(&mut self) -> anyhow::Result<()> {
2555        // Capture recovery at the grace-period boundary, before disconnection can queue more
2556        // evidence. A later drain must not turn incomplete recovery into a successful shutdown.
2557        let unresolved_submissions = self.exec_manager.unresolved_submission_ids();
2558        self.close_external_ingress();
2559
2560        let timeout = self.config.timeout_disconnection;
2561        let deadline = dst::time::Instant::now() + timeout;
2562
2563        let mut errors = Vec::new();
2564        let (report_result, disconnect_result) = futures_util::join!(
2565            dst::time::timeout(
2566                timeout,
2567                Self::finish_report_tasks(&self.exec_clients, &mut errors)
2568            ),
2569            dst::time::timeout(timeout, self.kernel.disconnect_clients()),
2570        );
2571
2572        if report_result.is_err() {
2573            errors.push("report collection shutdown timeout".to_string());
2574        }
2575
2576        let disconnect_result = match disconnect_result {
2577            Ok(result) => result,
2578            Err(_) => Err(anyhow::anyhow!(
2579                "disconnect timeout while disconnecting clients"
2580            )),
2581        };
2582
2583        if let Err(ref e) = disconnect_result {
2584            log::error!("Error disconnecting clients: {e}");
2585        }
2586
2587        let readiness_result = self.await_engines_disconnected(deadline).await;
2588        let kernel_result = self.kernel.finalize_stop().await;
2589
2590        self.handle.set_stopped();
2591
2592        if !unresolved_submissions.is_empty() {
2593            errors.push(format!(
2594                "Submission recovery incomplete at shutdown: unresolved client order IDs {unresolved_submissions:?}"
2595            ));
2596        }
2597
2598        if let Err(e) = disconnect_result {
2599            errors.push(e.to_string());
2600        }
2601
2602        if let Err(e) = readiness_result {
2603            errors.push(format!("failed while awaiting engine disconnection: {e}"));
2604        }
2605
2606        if let Err(e) = kernel_result {
2607            errors.push(format!("failed while finalizing kernel shutdown: {e}"));
2608        }
2609
2610        if errors.is_empty() {
2611            Ok(())
2612        } else {
2613            anyhow::bail!("{}", errors.join("; "))
2614        }
2615    }
2616
2617    fn drain_channels(
2618        time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
2619        system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
2620        system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
2621        exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2622        exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<
2623            DispatchMessage<TradingCommandMessage>,
2624        >,
2625        data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
2626        data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
2627    ) {
2628        let pending = time_evt_rx.len()
2629            + system_evt_rx.len()
2630            + system_cmd_rx.len()
2631            + exec_evt_rx.len()
2632            + exec_cmd_rx.len()
2633            + data_evt_rx.len()
2634            + data_cmd_rx.len();
2635        log::debug!("Draining shutdown queues ({pending} events initially pending)");
2636
2637        let mut drained = 0;
2638        let log_progress = |processed: usize| {
2639            if processed.is_multiple_of(10_000) {
2640                log::debug!("Processed {processed} events during shutdown drain");
2641            }
2642        };
2643
2644        while let Ok(handler) = time_evt_rx.try_recv() {
2645            let _ = AsyncRunner::handle_time_event(handler);
2646            drained += 1;
2647            log_progress(drained);
2648        }
2649
2650        while system_evt_rx.try_recv().is_ok() {
2651            drained += 1;
2652            log_progress(drained);
2653        }
2654
2655        while system_cmd_rx.try_recv().is_ok() {
2656            drained += 1;
2657            log_progress(drained);
2658        }
2659
2660        while let Ok(evt) = data_evt_rx.try_recv() {
2661            AsyncRunner::dispatch_data_event(evt);
2662            drained += 1;
2663            log_progress(drained);
2664        }
2665
2666        while let Ok(cmd) = data_cmd_rx.try_recv() {
2667            AsyncRunner::handle_data_command(cmd);
2668            drained += 1;
2669            log_progress(drained);
2670        }
2671
2672        while let Ok(evt) = exec_evt_rx.try_recv() {
2673            AsyncRunner::dispatch_exec_event(evt);
2674            drained += 1;
2675            log_progress(drained);
2676        }
2677
2678        while let Ok(cmd) = exec_cmd_rx.try_recv() {
2679            AsyncRunner::handle_trading_command(cmd);
2680            drained += 1;
2681            log_progress(drained);
2682        }
2683
2684        log::debug!("Drained {drained} remaining events during shutdown");
2685    }
2686
2687    fn observe_exec_event_before_dispatch(
2688        &mut self,
2689        evt: &ExecutionEvent,
2690    ) -> Option<Vec<ClientOrderId>> {
2691        let mut order_ids = Vec::new();
2692
2693        match evt {
2694            ExecutionEvent::Order(order_evt) => {
2695                self.exec_manager.observe_order_event(order_evt);
2696                order_ids.push(order_evt.client_order_id());
2697            }
2698            ExecutionEvent::OrderSubmittedBatch(batch) => {
2699                for submitted in &batch.events {
2700                    self.exec_manager
2701                        .record_local_activity(submitted.client_order_id);
2702                }
2703            }
2704            ExecutionEvent::OrderAcceptedBatch(batch) => {
2705                for accepted in &batch.events {
2706                    self.exec_manager
2707                        .observe_order_event(&OrderEventAny::Accepted(*accepted));
2708                    order_ids.push(accepted.client_order_id);
2709                }
2710            }
2711            ExecutionEvent::OrderCanceledBatch(batch) => {
2712                for canceled in &batch.events {
2713                    self.exec_manager
2714                        .observe_order_event(&OrderEventAny::Canceled(*canceled));
2715                    order_ids.push(canceled.client_order_id);
2716                }
2717            }
2718            ExecutionEvent::Report(report) => {
2719                if let ExecutionReport::Fill(fill_report) = report
2720                    && self.exec_manager.is_fill_recently_processed(
2721                        fill_report.account_id,
2722                        fill_report.instrument_id,
2723                        fill_report.trade_id,
2724                    )
2725                {
2726                    log::debug!(
2727                        "Skipping recently processed fill report: {}",
2728                        fill_report.trade_id,
2729                    );
2730                    return None;
2731                }
2732
2733                self.exec_manager.observe_execution_report(report);
2734
2735                if let Some(client_order_id) = self.order_report_client_order_id(report) {
2736                    order_ids.push(client_order_id);
2737                }
2738            }
2739            ExecutionEvent::Account(_) | ExecutionEvent::Data(_) => {}
2740        }
2741
2742        Some(order_ids)
2743    }
2744
2745    fn order_report_client_order_id(&self, report: &ExecutionReport) -> Option<ClientOrderId> {
2746        match report {
2747            ExecutionReport::Order(order_report)
2748            | ExecutionReport::OrderWithFills(order_report, _) => {
2749                order_report.client_order_id.or_else(|| {
2750                    self.kernel
2751                        .cache
2752                        .borrow()
2753                        .client_order_id(&order_report.venue_order_id)
2754                        .copied()
2755                })
2756            }
2757            ExecutionReport::Fill(fill_report) => fill_report.client_order_id.or_else(|| {
2758                self.kernel
2759                    .cache
2760                    .borrow()
2761                    .client_order_id(&fill_report.venue_order_id)
2762                    .copied()
2763            }),
2764            _ => None,
2765        }
2766    }
2767
2768    fn observe_exec_command_before_dispatch(&mut self, cmd: &TradingCommand) {
2769        if matches!(
2770            cmd,
2771            TradingCommand::ModifyOrder(_)
2772                | TradingCommand::ModifyOrders(_)
2773                | TradingCommand::CancelOrder(_)
2774                | TradingCommand::CancelOrders(_)
2775        ) {
2776            let engine = self.kernel.exec_engine.borrow();
2777            let no_route = engine.find_client_for_command(cmd).is_none()
2778                && cmd.client_id().is_none_or(|client_id| {
2779                    engine
2780                        .config()
2781                        .external_clients
2782                        .as_ref()
2783                        .is_none_or(|clients| !clients.contains(&client_id))
2784                });
2785
2786            if no_route {
2787                return;
2788            }
2789        }
2790
2791        let submission_client_id = if matches!(
2792            cmd,
2793            TradingCommand::SubmitOrder(_) | TradingCommand::SubmitOrderList(_)
2794        ) {
2795            self.kernel
2796                .exec_engine
2797                .borrow()
2798                .find_client_for_command(cmd)
2799                .map(|client| client.client_id())
2800        } else {
2801            None
2802        };
2803
2804        match cmd {
2805            TradingCommand::SubmitOrder(submit) => {
2806                self.exec_manager.register_submission(
2807                    &submit.order_init,
2808                    submission_client_id.or(submit.client_id),
2809                );
2810            }
2811            TradingCommand::SubmitOrderList(submit) => {
2812                for order_init in &submit.order_inits {
2813                    self.exec_manager
2814                        .register_submission(order_init, submission_client_id.or(submit.client_id));
2815                }
2816            }
2817            TradingCommand::ModifyOrder(modify) => {
2818                self.exec_manager
2819                    .register_inflight(modify.client_order_id, modify.command_id);
2820            }
2821            TradingCommand::ModifyOrders(modify) => {
2822                for child in &modify.modifies {
2823                    self.exec_manager
2824                        .register_inflight(child.client_order_id, child.command_id);
2825                }
2826            }
2827            TradingCommand::CancelOrder(cancel) => {
2828                self.exec_manager
2829                    .register_inflight(cancel.client_order_id, cancel.command_id);
2830            }
2831            TradingCommand::CancelOrders(cancel) => {
2832                for child in &cancel.cancels {
2833                    self.exec_manager
2834                        .register_inflight(child.client_order_id, child.command_id);
2835                }
2836            }
2837            _ => {}
2838        }
2839    }
2840
2841    /// Gets the node's environment.
2842    #[must_use]
2843    pub fn environment(&self) -> Environment {
2844        self.kernel.environment()
2845    }
2846
2847    /// Gets a reference to the underlying kernel.
2848    #[must_use]
2849    pub const fn kernel(&self) -> &NautilusKernel {
2850        &self.kernel
2851    }
2852
2853    /// Gets an exclusive reference to the underlying kernel.
2854    #[must_use]
2855    pub const fn kernel_mut(&mut self) -> &mut NautilusKernel {
2856        &mut self.kernel
2857    }
2858
2859    /// Gets the node's trader ID.
2860    #[must_use]
2861    pub fn trader_id(&self) -> TraderId {
2862        self.kernel.trader_id()
2863    }
2864
2865    /// Gets the node's instance ID.
2866    #[must_use]
2867    pub const fn instance_id(&self) -> UUID4 {
2868        self.kernel.instance_id()
2869    }
2870
2871    /// Returns the current node state.
2872    #[must_use]
2873    pub fn state(&self) -> NodeState {
2874        self.handle.state()
2875    }
2876
2877    /// Checks if the live node is currently running.
2878    #[must_use]
2879    pub fn is_running(&self) -> bool {
2880        self.state().is_running()
2881    }
2882
2883    /// Sets the cache database adapter for persistence.
2884    ///
2885    /// This allows setting a database adapter (e.g., PostgreSQL, Redis) after the node
2886    /// is built but before it starts running. The database adapter is used to persist
2887    /// cache data for recovery and state management.
2888    ///
2889    /// # Errors
2890    ///
2891    /// Returns an error if the node is already running.
2892    pub fn set_cache_database(
2893        &mut self,
2894        database: Box<dyn CacheDatabaseAdapter>,
2895    ) -> anyhow::Result<()> {
2896        if self.state() != NodeState::Idle {
2897            anyhow::bail!(
2898                "Cannot set cache database while node is running, set it before running the node"
2899            );
2900        }
2901
2902        self.kernel.cache().borrow_mut().set_database(database);
2903        Ok(())
2904    }
2905
2906    /// Returns the execution manager.
2907    #[must_use]
2908    pub fn exec_manager(&self) -> &ExecutionManager {
2909        &self.exec_manager
2910    }
2911
2912    /// Returns a mutable reference to the execution manager.
2913    #[must_use]
2914    pub fn exec_manager_mut(&mut self) -> &mut ExecutionManager {
2915        &mut self.exec_manager
2916    }
2917
2918    /// Adds an actor to the trader.
2919    ///
2920    /// This method provides a high-level interface for adding actors to the underlying
2921    /// trader without requiring direct access to the kernel. Actors should be added
2922    /// after the node is built but before starting the node.
2923    ///
2924    /// # Errors
2925    ///
2926    /// Returns an error if:
2927    /// - The trader is not in a valid state for adding components.
2928    /// - An actor with the same ID is already registered.
2929    /// - The node is currently running.
2930    pub fn add_actor<T>(&mut self, actor: T) -> anyhow::Result<()>
2931    where
2932        T: DataActor + DataActorNative + Component + Actor + 'static,
2933    {
2934        if self.state() != NodeState::Idle {
2935            anyhow::bail!(
2936                "Cannot add actor while node is running, add actors before running the node"
2937            );
2938        }
2939
2940        self.kernel.trader.borrow_mut().add_actor(actor)
2941    }
2942
2943    /// Adds an actor to the live node using a factory function.
2944    ///
2945    /// The factory function is called at registration time to create the actor,
2946    /// avoiding cloning issues with non-cloneable actor types.
2947    ///
2948    /// # Errors
2949    ///
2950    /// Returns an error if:
2951    /// - The node is currently running.
2952    /// - The factory function fails to create the actor.
2953    /// - The underlying trader registration fails.
2954    pub fn add_actor_from_factory<F, T>(&mut self, factory: F) -> anyhow::Result<()>
2955    where
2956        F: FnOnce() -> anyhow::Result<T>,
2957        T: DataActor + DataActorNative + Component + Actor + 'static,
2958    {
2959        if self.state() != NodeState::Idle {
2960            anyhow::bail!(
2961                "Cannot add actor while node is running, add actors before running the node"
2962            );
2963        }
2964
2965        self.kernel
2966            .trader
2967            .borrow_mut()
2968            .add_actor_from_factory(factory)
2969    }
2970
2971    /// Adds a strategy to the trader.
2972    ///
2973    /// Strategies are registered in both the component registry (for lifecycle management)
2974    /// and the actor registry (for data callbacks via msgbus).
2975    ///
2976    /// # Errors
2977    ///
2978    /// Returns an error if:
2979    /// - The node is currently running.
2980    /// - A strategy with the same ID is already registered.
2981    /// - The configured external order instrument IDs repeat an instrument or the cache already
2982    ///   contains a requested claim.
2983    /// - The strategy configures one or more external order instrument IDs and the cache is already
2984    ///   borrowed.
2985    /// - The strategy configures an OMS type override and the execution engine is already borrowed.
2986    pub fn add_strategy<T>(&mut self, mut strategy: T) -> anyhow::Result<()>
2987    where
2988        T: Strategy + StrategyNative + DataActorNative + Component + Debug + 'static,
2989    {
2990        if self.state() != NodeState::Idle {
2991            anyhow::bail!(
2992                "Cannot add strategy while node is running, add strategies before running the node"
2993            );
2994        }
2995
2996        // Capture strategy-owned values before adding the strategy, which moves it
2997        let strategy_id = self
2998            .kernel
2999            .trader
3000            .borrow()
3001            .prepare_strategy_for_registration(&mut strategy)?;
3002        let oms_type = StrategyNative::strategy_core(&strategy).config.oms_type;
3003        let instrument_ids = strategy.external_order_instrument_ids().unwrap_or_default();
3004
3005        if !instrument_ids.is_empty() {
3006            self.register_external_order_claims(strategy_id, &instrument_ids)?;
3007        }
3008
3009        let mut exec_engine = match oms_type
3010            .map(|_| {
3011                self.kernel
3012                    .exec_engine
3013                    .try_borrow_mut()
3014                    .map_err(|e| anyhow::anyhow!("Cannot register OMS type: {e}"))
3015            })
3016            .transpose()
3017        {
3018            Ok(exec_engine) => exec_engine,
3019            Err(e) => {
3020                if !instrument_ids.is_empty()
3021                    && let Err(rollback_error) =
3022                        self.rollback_external_order_claims(strategy_id, &instrument_ids)
3023                {
3024                    anyhow::bail!(
3025                        "{e}; failed to roll back external order claims for {strategy_id}: {rollback_error}"
3026                    );
3027                }
3028
3029                return Err(e);
3030            }
3031        };
3032
3033        if let Err(add_error) = self.kernel.trader.borrow_mut().add_strategy(strategy) {
3034            drop(exec_engine);
3035
3036            if !instrument_ids.is_empty()
3037                && let Err(rollback_error) =
3038                    self.rollback_external_order_claims(strategy_id, &instrument_ids)
3039            {
3040                anyhow::bail!(
3041                    "Failed to add strategy {strategy_id}: {add_error}; failed to roll back external order claims: {rollback_error}"
3042                );
3043            }
3044
3045            return Err(add_error);
3046        }
3047
3048        if let Some(exec_engine) = &mut exec_engine
3049            && let Some(oms_type) = oms_type
3050        {
3051            exec_engine.register_oms_type(strategy_id, oms_type);
3052        }
3053
3054        Ok(())
3055    }
3056
3057    /// Registers external order claims in the shared cache.
3058    ///
3059    /// It can be called while the node is idle, after manual [`start`](Self::start) returns, or
3060    /// after the node stops. A running strategy can update its own claims through
3061    /// `Strategy::set_external_order_instrument_ids`.
3062    ///
3063    /// # Errors
3064    ///
3065    /// Returns an error if the cache is already borrowed, the request repeats an instrument, or any
3066    /// requested instrument already has a claim. The cache remains unchanged.
3067    pub fn register_external_order_claims(
3068        &self,
3069        strategy_id: StrategyId,
3070        instrument_ids: &[InstrumentId],
3071    ) -> anyhow::Result<()> {
3072        self.kernel
3073            .cache
3074            .try_borrow_mut()
3075            .map_err(|e| anyhow::anyhow!("Cannot register external order claims: {e}"))?
3076            .register_external_order_claims(strategy_id, instrument_ids)?;
3077
3078        if !instrument_ids.is_empty() {
3079            log::info!("Registered external order claims for {strategy_id}: {instrument_ids:?}");
3080        }
3081
3082        Ok(())
3083    }
3084
3085    /// Deregisters all external order claims owned by `strategy_id` from the shared cache.
3086    ///
3087    /// The operation is synchronous and can be called while the node is idle, after manual
3088    /// [`start`](Self::start) returns, or after the node stops. It cannot be called while
3089    /// [`run`](Self::run) or [`run_with_mode`](Self::run_with_mode) owns the node.
3090    ///
3091    /// # Errors
3092    ///
3093    /// Returns an error if the cache is already borrowed.
3094    pub fn deregister_external_order_claims(&self, strategy_id: StrategyId) -> anyhow::Result<()> {
3095        self.kernel
3096            .cache
3097            .try_borrow_mut()
3098            .map_err(|e| anyhow::anyhow!("Cannot deregister external order claims: {e}"))?
3099            .set_external_order_claims(strategy_id, &[])?;
3100
3101        Ok(())
3102    }
3103
3104    pub(crate) fn rollback_external_order_claims(
3105        &self,
3106        strategy_id: StrategyId,
3107        instrument_ids: &[InstrumentId],
3108    ) -> anyhow::Result<()> {
3109        let mut cache = self
3110            .kernel
3111            .cache
3112            .try_borrow_mut()
3113            .map_err(|e| anyhow::anyhow!("Cannot roll back external order claims: {e}"))?;
3114        let retained: Vec<_> = cache
3115            .external_order_claim_instrument_ids(Some(strategy_id))
3116            .into_iter()
3117            .filter(|instrument_id| !instrument_ids.contains(instrument_id))
3118            .collect();
3119        cache.set_external_order_claims(strategy_id, &retained)
3120    }
3121
3122    /// Adds an execution algorithm to the trader.
3123    ///
3124    /// Execution algorithms are registered in both the component registry (for lifecycle
3125    /// management) and the actor registry (for data callbacks via msgbus).
3126    ///
3127    /// # Errors
3128    ///
3129    /// Returns an error if:
3130    /// - The node is currently running.
3131    /// - An execution algorithm with the same ID is already registered.
3132    pub fn add_exec_algorithm<T>(&mut self, exec_algorithm: T) -> anyhow::Result<()>
3133    where
3134        T: ExecutionAlgorithm + ExecutionAlgorithmNative + Component + Debug + 'static,
3135    {
3136        if self.state() != NodeState::Idle {
3137            anyhow::bail!(
3138                "Cannot add exec algorithm while node is running, add exec algorithms before running the node"
3139            );
3140        }
3141
3142        self.kernel
3143            .trader
3144            .borrow_mut()
3145            .add_exec_algorithm(exec_algorithm)
3146    }
3147
3148    // Runs reconciliation sub-checks, each gated by its own interval.
3149    // Continuous checks only schedule venue work; they do not await venue I/O
3150    // in the event loop.
3151}
3152
3153#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3154enum SocketReconnectDispatchOutcome {
3155    Accepted,
3156    AlreadyReconnecting,
3157    Disconnected,
3158    Closed,
3159    Unsupported,
3160    InvalidTrader,
3161    UnknownClient,
3162    UnknownEndpoint,
3163    AmbiguousEndpoint,
3164}
3165
3166fn record_runner_dispatch(
3167    metrics: &RunnerMetrics,
3168    channel: SystemChannel,
3169    dispatch_start: dst::time::Instant,
3170    metrics_start: dst::time::Instant,
3171) {
3172    let dispatch_end = dst::time::Instant::now();
3173    metrics.record_dispatch(
3174        channel,
3175        dispatch_end.duration_since(dispatch_start),
3176        dispatch_end.duration_since(metrics_start),
3177    );
3178}
3179
3180fn record_runner_maintenance(
3181    metrics: &RunnerMetrics,
3182    work_start: dst::time::Instant,
3183    metrics_start: dst::time::Instant,
3184) {
3185    let work_end = dst::time::Instant::now();
3186    metrics.record_maintenance(
3187        work_end.duration_since(work_start),
3188        work_end.duration_since(metrics_start),
3189    );
3190}
3191
3192fn record_runner_external_msgbus(
3193    metrics: &RunnerMetrics,
3194    work_start: dst::time::Instant,
3195    metrics_start: dst::time::Instant,
3196) {
3197    let work_end = dst::time::Instant::now();
3198    metrics.record_external_msgbus(
3199        work_end.duration_since(work_start),
3200        work_end.duration_since(metrics_start),
3201    );
3202}
3203
3204async fn drive_with_external_ingress<F, T>(
3205    future: F,
3206    receiver: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3207    pending: &mut VecDeque<BusMessage>,
3208) -> T
3209where
3210    F: std::future::Future<Output = T>,
3211{
3212    tokio::pin!(future);
3213    let capacity = receiver
3214        .as_ref()
3215        .map_or(0, tokio::sync::mpsc::Receiver::max_capacity);
3216
3217    loop {
3218        tokio::select! {
3219            biased;
3220
3221            result = &mut future => return result,
3222            message = async { match receiver.as_mut() { Some(receiver) => receiver.recv().await, None => std::future::pending().await } }, if pending.len() < capacity => {
3223                let Some(message) = message else {
3224                    log::info!("External message bus ingress closed");
3225                    *receiver = None;
3226                    continue;
3227                };
3228
3229                match msgbus::republish_pending_external_response(&message) {
3230                    Ok(true) => {},
3231                    Ok(false) | Err(_) => pending.push_back(message),
3232                }
3233            }
3234        }
3235    }
3236}
3237
3238async fn recv_external_msgbus_message(
3239    rx: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3240    pending: &mut VecDeque<BusMessage>,
3241) -> Option<BusMessage> {
3242    if let Some(message) = pending.pop_front() {
3243        return Some(message);
3244    }
3245
3246    match rx {
3247        Some(rx) => rx.recv().await,
3248        None => std::future::pending::<Option<BusMessage>>().await,
3249    }
3250}
3251
3252struct RunnerReceivers<'a> {
3253    time_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3254    system_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3255    system_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3256    exec_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3257    exec_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3258    data_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3259    data_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3260}
3261
3262/// Flushes data events and commands from both `pending` and the channel receivers
3263/// into the cache, looping until no progress is made.
3264///
3265/// This closes the gap where `drive_with_event_buffering` exits as soon as its
3266/// driven future resolves (biased select), leaving items in the channel receivers
3267/// that were not captured into `pending`.
3268fn flush_pending_data(
3269    pending: &mut PendingEvents,
3270    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3271    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3272) {
3273    loop {
3274        let mut progressed = pending.drain_data();
3275
3276        while let Ok(evt) = data_evt_rx.try_recv() {
3277            AsyncRunner::dispatch_data_event(evt);
3278            progressed = true;
3279        }
3280
3281        while let Ok(cmd) = data_cmd_rx.try_recv() {
3282            AsyncRunner::handle_data_command(cmd);
3283            progressed = true;
3284        }
3285
3286        if !progressed {
3287            break;
3288        }
3289    }
3290}
3291
3292/// Flushes all channel receivers into `pending`, then drains everything.
3293///
3294/// Unlike [`flush_pending_data`] this is a single pass, not a drain-until-quiet
3295/// loop. Sufficient for phase 2 where the goal is to capture items the biased
3296/// select did not poll before the connect future resolved.
3297#[expect(
3298    clippy::too_many_arguments,
3299    reason = "all runner receivers are drained together"
3300)]
3301fn flush_all_pending(
3302    pending: &mut PendingEvents,
3303    time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3304    system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3305    system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3306    exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3307    exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3308    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3309    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3310) {
3311    // Flush channel receivers into pending
3312    while let Ok(handler) = time_evt_rx.try_recv() {
3313        let _ = AsyncRunner::handle_time_event(handler);
3314    }
3315
3316    while let Ok(event) = system_evt_rx.try_recv() {
3317        pending.system_events.push(event);
3318    }
3319
3320    while let Ok(command) = system_cmd_rx.try_recv() {
3321        pending.system_commands.push(command);
3322    }
3323
3324    while let Ok(evt) = data_evt_rx.try_recv() {
3325        pending.data_evts.push(evt);
3326    }
3327
3328    while let Ok(cmd) = data_cmd_rx.try_recv() {
3329        pending.data_cmds.push(cmd);
3330    }
3331
3332    while let Ok(evt) = exec_evt_rx.try_recv() {
3333        pending.push_exec_event(evt);
3334    }
3335
3336    while let Ok(cmd) = exec_cmd_rx.try_recv() {
3337        pending.exec_cmds.push(cmd);
3338    }
3339
3340    pending.drain();
3341}
3342
3343/// Drives a future to completion while buffering channel events.
3344///
3345/// Time events are handled immediately. Account events are forwarded directly.
3346/// All other events are buffered in `pending` for later processing.
3347#[expect(
3348    clippy::too_many_arguments,
3349    reason = "startup buffering owns one future plus the pending state and all runner receivers"
3350)]
3351async fn drive_with_event_buffering<F: std::future::Future>(
3352    future: F,
3353    pending: &mut PendingEvents,
3354    time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3355    system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3356    system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3357    exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3358    exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3359    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3360    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3361) -> F::Output {
3362    tokio::pin!(future);
3363
3364    loop {
3365        tokio::select! {
3366            biased;
3367
3368            result = &mut future => {
3369                break result;
3370            }
3371            Some(handler) = time_evt_rx.recv() => {
3372                let _ = AsyncRunner::handle_time_event(handler);
3373            }
3374            Some(event) = system_evt_rx.recv() => {
3375                pending.system_events.push(event);
3376            }
3377            Some(command) = system_cmd_rx.recv() => {
3378                pending.system_commands.push(command);
3379            }
3380            Some(evt) = exec_evt_rx.recv() => {
3381                pending.push_exec_event(evt);
3382            }
3383            Some(cmd) = exec_cmd_rx.recv() => {
3384                pending.exec_cmds.push(cmd);
3385            }
3386            Some(evt) = data_evt_rx.recv() => {
3387                pending.data_evts.push(evt);
3388            }
3389            Some(cmd) = data_cmd_rx.recv() => {
3390                pending.data_cmds.push(cmd);
3391            }
3392        }
3393    }
3394}
3395
3396#[derive(Default)]
3397struct PendingEvents {
3398    system_events: Vec<DispatchMessage<SystemEvent>>,
3399    system_commands: Vec<DispatchMessage<SystemCommand>>,
3400    data_evts: Vec<DispatchMessage<DataEvent>>,
3401    data_cmds: Vec<DispatchMessage<DataCommand>>,
3402    exec_reports: Vec<DispatchMessage<ExecutionReport>>,
3403    order_evts: Vec<DispatchMessage<OrderEventAny>>,
3404    exec_cmds: Vec<DispatchMessage<TradingCommandMessage>>,
3405}
3406
3407impl PendingEvents {
3408    fn is_empty(&self) -> bool {
3409        self.system_events.is_empty()
3410            && self.system_commands.is_empty()
3411            && self.data_evts.is_empty()
3412            && self.data_cmds.is_empty()
3413            && self.exec_reports.is_empty()
3414            && self.order_evts.is_empty()
3415            && self.exec_cmds.is_empty()
3416    }
3417
3418    /// Drains only data events and commands into the cache.
3419    ///
3420    /// Returns `true` if any events or commands were drained.
3421    fn drain_data(&mut self) -> bool {
3422        let total = self.data_evts.len() + self.data_cmds.len();
3423
3424        if total > 0 {
3425            log::debug!(
3426                "Draining {total} data events/commands into cache \
3427                 (data_evts={}, data_cmds={})",
3428                self.data_evts.len(),
3429                self.data_cmds.len(),
3430            );
3431        }
3432
3433        for evt in self.data_evts.drain(..) {
3434            AsyncRunner::dispatch_data_event(evt);
3435        }
3436
3437        for cmd in self.data_cmds.drain(..) {
3438            AsyncRunner::handle_data_command(cmd);
3439        }
3440
3441        total > 0
3442    }
3443
3444    /// Drains all remaining pending events.
3445    fn drain(&mut self) {
3446        let total = self.data_evts.len()
3447            + self.data_cmds.len()
3448            + self.exec_reports.len()
3449            + self.order_evts.len()
3450            + self.exec_cmds.len();
3451
3452        if total > 0 {
3453            log::debug!(
3454                "Processing {total} events/commands queued during startup \
3455                 (data_evts={}, data_cmds={}, exec_reports={}, order_evts={}, exec_cmds={})",
3456                self.data_evts.len(),
3457                self.data_cmds.len(),
3458                self.exec_reports.len(),
3459                self.order_evts.len(),
3460                self.exec_cmds.len()
3461            );
3462        }
3463
3464        for evt in self.data_evts.drain(..) {
3465            AsyncRunner::dispatch_data_event(evt);
3466        }
3467
3468        for cmd in self.data_cmds.drain(..) {
3469            AsyncRunner::handle_data_command(cmd);
3470        }
3471
3472        for report in self.exec_reports.drain(..) {
3473            report
3474                .dispatch(|report| AsyncRunner::handle_exec_event(ExecutionEvent::Report(report)));
3475        }
3476
3477        for evt in self.order_evts.drain(..) {
3478            evt.dispatch(|evt| AsyncRunner::handle_exec_event(ExecutionEvent::Order(evt)));
3479        }
3480
3481        for cmd in self.exec_cmds.drain(..) {
3482            AsyncRunner::handle_trading_command(cmd);
3483        }
3484    }
3485
3486    fn push_exec_event(&mut self, event: DispatchMessage<ExecutionEvent>) {
3487        let rooted = event.is_rooted();
3488        event.dispatch(|event| {
3489            let owner = std::thread::current().id();
3490
3491            let order = |event| {
3492                if rooted {
3493                    DispatchMessage::new(event, owner)
3494                } else {
3495                    DispatchMessage::from(event)
3496                }
3497            };
3498
3499            // Account events are safe to process immediately. Reports and orders need
3500            // ExecEngine access, which may conflict with the driven startup future's borrow.
3501            match event {
3502                ExecutionEvent::Account(_) | ExecutionEvent::Data(_) => {
3503                    AsyncRunner::handle_exec_event(event);
3504                }
3505                ExecutionEvent::Report(report) => self.exec_reports.push(if rooted {
3506                    DispatchMessage::new(report, owner)
3507                } else {
3508                    report.into()
3509                }),
3510                ExecutionEvent::Order(event) => self.order_evts.push(order(event)),
3511                ExecutionEvent::OrderSubmittedBatch(batch) => {
3512                    for event in batch {
3513                        self.order_evts.push(order(OrderEventAny::Submitted(event)));
3514                    }
3515                }
3516                ExecutionEvent::OrderAcceptedBatch(batch) => {
3517                    for event in batch {
3518                        self.order_evts.push(order(OrderEventAny::Accepted(event)));
3519                    }
3520                }
3521                ExecutionEvent::OrderCanceledBatch(batch) => {
3522                    for event in batch {
3523                        self.order_evts.push(order(OrderEventAny::Canceled(event)));
3524                    }
3525                }
3526            }
3527        });
3528    }
3529
3530    fn take_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
3531        std::mem::take(&mut self.system_events)
3532    }
3533
3534    fn take_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
3535        std::mem::take(&mut self.system_commands)
3536    }
3537}
3538
3539struct ClientStatus {
3540    client: String,
3541    client_type: &'static str,
3542    connected: bool,
3543}
3544
3545fn render_client_statuses(rows: Vec<ClientStatus>) -> String {
3546    let mut builder = Builder::with_capacity(rows.len() + 1, 3);
3547    builder.push_record(["Client", "Type", "Connected"]);
3548
3549    for row in rows {
3550        builder.push_record([
3551            row.client,
3552            row.client_type.to_string(),
3553            row.connected.to_string(),
3554        ]);
3555    }
3556
3557    builder.build().with(Style::rounded()).to_string()
3558}
3559
3560#[cfg(test)]
3561mod tests {
3562    mod report_acceptance;
3563    mod shutdown;
3564
3565    use std::{
3566        cell::{Cell, RefCell},
3567        fmt::Debug,
3568        rc::Rc,
3569        sync::{
3570            Arc,
3571            atomic::{AtomicBool, Ordering},
3572        },
3573    };
3574
3575    use bytes::Bytes;
3576    use indexmap::{IndexMap, IndexSet};
3577    use log::{Level, LevelFilter, Log, Metadata, Record};
3578    #[cfg(feature = "python")]
3579    use nautilus_common::runner::{
3580        SyncDataCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
3581    };
3582    use nautilus_common::{
3583        actor::{
3584            self, CallbackDispatchError, DataActor, DataActorCore, data_actor::DataActorConfig,
3585        },
3586        cache::Cache,
3587        clock::{Clock, VirtualClock},
3588        enums::SerializationEncoding,
3589        live::{
3590            runner::{get_data_event_sender, get_exec_event_sender, get_system_event_sender},
3591            sender::DispatchSender,
3592        },
3593        logging::{logger::LoggerConfig, logging_sync_to_disk, writer::FileWriterConfig},
3594        messages::{
3595            data::{SubscribeCommand, SubscribeQuotes},
3596            execution::{
3597                GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
3598                GeneratePositionStatusReports, QueryAccount, SubmitOrder, SubmitOrderList,
3599                TradingCommand,
3600            },
3601            system::{
3602                QueueCondition, QueueState, ReconnectSocket, SocketState, SocketStateChanged,
3603            },
3604        },
3605        msgbus::{
3606            self, BusMessage, BusPayloadType, MessageBusBacking, MessageBusBackingFactory,
3607            MessageBusConfig, MessageBusExternalEgress, MessageBusExternalIngress,
3608            MessagingSwitchboard, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
3609        },
3610        nautilus_actor,
3611        runner::{SyncTradingCommandSender, TradingCommandSender},
3612        testing::wait_until_async,
3613        timer::{TimeEvent, TimeEventCallback},
3614    };
3615    use nautilus_core::{Params, UUID4, UnixNanos};
3616    use nautilus_execution::{
3617        engine::{ExecutionEngine, SnapshotAnchorer, stubs::StubExecutionClient},
3618        reconciliation::{RECONCILIATION_ORDER_TAG, create_inferred_fill_for_qty},
3619    };
3620    use nautilus_model::{
3621        accounts::{AccountAny, MarginAccount},
3622        data::QuoteTick,
3623        enums::{
3624            AccountType, LiquiditySide, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
3625            TimeInForce,
3626        },
3627        events::{
3628            AccountState, OrderAccepted, OrderAcceptedBatch, OrderFilled, OrderUpdated,
3629            order::spec::{
3630                OrderAcceptedSpec, OrderPendingCancelSpec, OrderPendingUpdateSpec, OrderUpdatedSpec,
3631            },
3632        },
3633        identifiers::{
3634            AccountId, ActorId, ClientId, InstrumentId, OrderListId, PositionId, StrategyId,
3635            TradeId, TraderId, Venue, VenueOrderId,
3636        },
3637        instruments::{
3638            Instrument, InstrumentAny,
3639            stubs::{crypto_perpetual_ethusdt, currency_pair_btcusdt},
3640        },
3641        orders::{
3642            Order, OrderAny, OrderList, OrderTestBuilder,
3643            stubs::{OrderFilledTestBuilder, TestOrderEventStubs},
3644        },
3645        position::PositionReplayEvent,
3646        reports::{FillReport, PositionStatusReport},
3647        types::{AccountBalance, Currency, MarginBalance, Money, Price, Quantity},
3648    };
3649    use nautilus_system::{KernelEventStore, RegisteredComponents, event_store::EventStoreConfig};
3650    use nautilus_testkit::{
3651        cache::TestCacheDatabaseControl,
3652        components::{StateActor, StateStrategy},
3653    };
3654    use nautilus_trading::{
3655        nautilus_strategy,
3656        strategy::{config::StrategyConfig, core::StrategyCore},
3657    };
3658    use parking_lot::Mutex;
3659    use rstest::*;
3660    use rust_decimal::Decimal;
3661    use rust_decimal_macros::dec;
3662    use ustr::Ustr;
3663
3664    use super::{
3665        reconciliation::{
3666            PositionFillReportResult, PositionReportResult, reconciliation_check_due,
3667            request_position_fill_reports,
3668        },
3669        *,
3670    };
3671    use crate::{
3672        execution::{
3673            client::LiveExecutionClient,
3674            manager::{PositionFillReportQuery, ReportClientCoverage, SourcedOrderStatusReport},
3675            submission::{
3676                SubmissionRecoveryExhausted, SubmissionRecoveryPolicy, SubmissionRecoverySource,
3677            },
3678        },
3679        socket::SocketControl,
3680    };
3681
3682    struct ExternalIngressLogCapture {
3683        messages: Mutex<Vec<String>>,
3684    }
3685
3686    static EXTERNAL_INGRESS_LOG_CAPTURE: ExternalIngressLogCapture = ExternalIngressLogCapture {
3687        messages: Mutex::new(Vec::new()),
3688    };
3689
3690    #[derive(Debug)]
3691    enum FillReportClientOutcome {
3692        Reports(Vec<FillReport>),
3693        Failure,
3694    }
3695
3696    #[derive(Debug)]
3697    struct FillReportClient {
3698        client_id: ClientId,
3699        account_id: AccountId,
3700        venue: Venue,
3701        outcome: FillReportClientOutcome,
3702        commands: Rc<RefCell<Vec<GenerateFillReports>>>,
3703    }
3704
3705    #[async_trait::async_trait(?Send)]
3706    impl ExecutionClient for FillReportClient {
3707        fn is_connected(&self) -> bool {
3708            true
3709        }
3710
3711        fn client_id(&self) -> ClientId {
3712            self.client_id
3713        }
3714
3715        fn account_id(&self) -> AccountId {
3716            self.account_id
3717        }
3718
3719        fn venue(&self) -> Venue {
3720            self.venue
3721        }
3722
3723        fn oms_type(&self) -> OmsType {
3724            OmsType::Netting
3725        }
3726
3727        fn get_account(&self) -> Option<AccountAny> {
3728            None
3729        }
3730
3731        fn generate_account_state(
3732            &self,
3733            _balances: Vec<AccountBalance>,
3734            _margins: Vec<MarginBalance>,
3735            _reported: bool,
3736            _ts_event: UnixNanos,
3737            _info: Option<Params>,
3738        ) -> anyhow::Result<()> {
3739            Ok(())
3740        }
3741
3742        fn start(&mut self) -> anyhow::Result<()> {
3743            Ok(())
3744        }
3745
3746        fn stop(&mut self) -> anyhow::Result<()> {
3747            Ok(())
3748        }
3749
3750        async fn generate_order_status_reports(
3751            &self,
3752            _cmd: &GenerateOrderStatusReports,
3753        ) -> anyhow::Result<Vec<OrderStatusReport>> {
3754            match self.outcome {
3755                FillReportClientOutcome::Failure => anyhow::bail!("order reports unavailable"),
3756                FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3757            }
3758        }
3759
3760        async fn generate_position_status_reports(
3761            &self,
3762            _cmd: &GeneratePositionStatusReports,
3763        ) -> anyhow::Result<Vec<PositionStatusReport>> {
3764            match self.outcome {
3765                FillReportClientOutcome::Failure => anyhow::bail!("position reports unavailable"),
3766                FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3767            }
3768        }
3769
3770        async fn generate_fill_reports(
3771            &self,
3772            cmd: GenerateFillReports,
3773        ) -> anyhow::Result<Vec<FillReport>> {
3774            self.commands.borrow_mut().push(cmd);
3775
3776            match &self.outcome {
3777                FillReportClientOutcome::Reports(reports) => Ok(reports.clone()),
3778                FillReportClientOutcome::Failure => anyhow::bail!("fill reports unavailable"),
3779            }
3780        }
3781    }
3782
3783    #[derive(Debug)]
3784    struct OrderReportClient {
3785        client_id: ClientId,
3786        report: OrderStatusReport,
3787        targeted: bool,
3788        queries: Rc<RefCell<Vec<&'static str>>>,
3789    }
3790
3791    #[async_trait::async_trait(?Send)]
3792    impl ExecutionClient for OrderReportClient {
3793        fn is_connected(&self) -> bool {
3794            true
3795        }
3796
3797        fn client_id(&self) -> ClientId {
3798            self.client_id
3799        }
3800
3801        fn account_id(&self) -> AccountId {
3802            self.report.account_id
3803        }
3804
3805        fn venue(&self) -> Venue {
3806            self.report.instrument_id.venue
3807        }
3808
3809        fn oms_type(&self) -> OmsType {
3810            OmsType::Netting
3811        }
3812
3813        fn get_account(&self) -> Option<AccountAny> {
3814            None
3815        }
3816
3817        fn generate_account_state(
3818            &self,
3819            _balances: Vec<AccountBalance>,
3820            _margins: Vec<MarginBalance>,
3821            _reported: bool,
3822            _ts_event: UnixNanos,
3823            _info: Option<Params>,
3824        ) -> anyhow::Result<()> {
3825            Ok(())
3826        }
3827
3828        fn start(&mut self) -> anyhow::Result<()> {
3829            Ok(())
3830        }
3831
3832        fn stop(&mut self) -> anyhow::Result<()> {
3833            Ok(())
3834        }
3835
3836        async fn generate_order_status_reports(
3837            &self,
3838            _command: &GenerateOrderStatusReports,
3839        ) -> anyhow::Result<Vec<OrderStatusReport>> {
3840            self.queries.borrow_mut().push("bulk");
3841            Ok(if self.targeted {
3842                Vec::new()
3843            } else {
3844                vec![self.report.clone()]
3845            })
3846        }
3847
3848        async fn generate_order_status_report(
3849            &self,
3850            _command: &GenerateOrderStatusReport,
3851        ) -> anyhow::Result<Option<OrderStatusReport>> {
3852            self.queries.borrow_mut().push("targeted");
3853            Ok(Some(self.report.clone()))
3854        }
3855    }
3856
3857    #[derive(Debug)]
3858    struct StartupSocketActor {
3859        core: DataActorCore,
3860        received: Rc<RefCell<Vec<SocketStateChanged>>>,
3861    }
3862
3863    impl StartupSocketActor {
3864        fn new(received: Rc<RefCell<Vec<SocketStateChanged>>>) -> Self {
3865            Self {
3866                core: DataActorCore::new(DataActorConfig {
3867                    actor_id: Some(ActorId::from("SOCKET-STARTUP-ACTOR")),
3868                    ..Default::default()
3869                }),
3870                received,
3871            }
3872        }
3873    }
3874
3875    impl DataActor for StartupSocketActor {
3876        fn on_start(&mut self) -> anyhow::Result<()> {
3877            self.subscribe_socket_state(None, None, None);
3878            Ok(())
3879        }
3880
3881        fn on_socket_state(&mut self, event: &SocketStateChanged) -> anyhow::Result<()> {
3882            self.received.borrow_mut().push(event.clone());
3883            Ok(())
3884        }
3885    }
3886
3887    nautilus_actor!(StartupSocketActor);
3888
3889    impl Log for ExternalIngressLogCapture {
3890        fn enabled(&self, metadata: &Metadata<'_>) -> bool {
3891            metadata.level() == Level::Error && metadata.target() == "nautilus_live::node"
3892        }
3893
3894        fn log(&self, record: &Record<'_>) {
3895            if self.enabled(record.metadata()) {
3896                self.messages.lock().push(record.args().to_string());
3897            }
3898        }
3899
3900        fn flush(&self) {}
3901    }
3902
3903    #[derive(Debug)]
3904    struct FailingTimerActor {
3905        core: DataActorCore,
3906        received: Rc<RefCell<Vec<u64>>>,
3907    }
3908
3909    nautilus_actor!(FailingTimerActor);
3910
3911    impl DataActor for FailingTimerActor {
3912        fn on_start(&mut self) -> anyhow::Result<()> {
3913            for timestamp in [17, 23] {
3914                let received = self.received.clone();
3915
3916                let callback = TimeEventCallback::RustLocal(Rc::new(move |_| {
3917                    received.borrow_mut().push(timestamp);
3918
3919                    if timestamp == 17 {
3920                        crate::dispatch::tests::latch_callback_failure();
3921                    }
3922                }));
3923
3924                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
3925                    TimeEvent::new(
3926                        "callback-failure".into(),
3927                        UUID4::new(),
3928                        timestamp.into(),
3929                        timestamp.into(),
3930                    ),
3931                    callback,
3932                ));
3933            }
3934
3935            Ok(())
3936        }
3937    }
3938
3939    #[rstest]
3940    #[tokio::test]
3941    async fn test_callback_failure_stops_later_live_events(
3942        #[values(false, true)] incomplete_submission: bool,
3943    ) {
3944        actor::clear_callbacks().unwrap();
3945
3946        let config = LiveNodeConfig {
3947            exec_engine: crate::config::LiveExecutionEngineConfig {
3948                reconciliation: false,
3949                submission_recovery_policy: SubmissionRecoveryPolicy::RetainUnresolved,
3950                ..Default::default()
3951            },
3952            timeout_connection: Duration::ZERO,
3953            timeout_reconciliation: Duration::ZERO,
3954            timeout_portfolio: Duration::ZERO,
3955            timeout_disconnection: Duration::ZERO,
3956            delay_post_stop: Duration::ZERO,
3957            timeout_shutdown: Duration::ZERO,
3958            ..Default::default()
3959        };
3960
3961        let mut node = LiveNode::build("CallbackFailureNode".to_string(), Some(config)).unwrap();
3962
3963        if incomplete_submission {
3964            let order = OrderTestBuilder::new(OrderType::Market)
3965                .client_order_id(ClientOrderId::from("O-CALLBACK-UNRESOLVED"))
3966                .instrument_id(InstrumentId::from("EUR/USD.SIM"))
3967                .quantity(Quantity::from("1"))
3968                .build();
3969            node.exec_manager
3970                .register_submission(order.init_event(), None);
3971        }
3972        let received = Rc::new(RefCell::new(Vec::new()));
3973        node.add_actor(FailingTimerActor {
3974            core: DataActorCore::new(DataActorConfig {
3975                actor_id: Some(ActorId::from("CALLBACK-FAILURE")),
3976                ..Default::default()
3977            }),
3978            received: received.clone(),
3979        })
3980        .unwrap();
3981
3982        let result = node.run_with_mode(NodeRunMode::Hosted).await;
3983
3984        let error = result.unwrap_err();
3985        assert!(error.to_string().contains("Callback delivery unwound"));
3986        if incomplete_submission {
3987            assert!(error.to_string().contains("O-CALLBACK-UNRESOLVED"));
3988        }
3989        let state = node.state();
3990        let trader_stopped = node.kernel.trader.borrow().is_stopped();
3991        let failure = actor::callback_failure();
3992        node.dispose();
3993
3994        assert_eq!(
3995            error.downcast_ref::<CallbackDispatchError>(),
3996            Some(&CallbackDispatchError::DeliveryUnwound)
3997        );
3998        assert_eq!(*received.borrow(), [17]);
3999        assert_eq!(state, NodeState::Stopped);
4000        assert!(trader_stopped);
4001        assert_eq!(failure, Some(CallbackDispatchError::DeliveryUnwound));
4002        assert_eq!(actor::callback_failure(), None);
4003        assert_eq!(actor::clear_callbacks(), Ok(()));
4004    }
4005
4006    #[rstest]
4007    fn test_render_client_statuses() {
4008        let rows = vec![
4009            ClientStatus {
4010                client: "BINANCE".to_string(),
4011                client_type: "Data",
4012                connected: true,
4013            },
4014            ClientStatus {
4015                client: "SIM".to_string(),
4016                client_type: "Execution",
4017                connected: false,
4018            },
4019        ];
4020
4021        let output = render_client_statuses(rows);
4022        let expected = "╭─────────┬───────────┬───────────╮\n\
4023│ Client  │ Type      │ Connected │\n\
4024├─────────┼───────────┼───────────┤\n\
4025│ BINANCE │ Data      │ true      │\n\
4026│ SIM     │ Execution │ false     │\n\
4027╰─────────┴───────────┴───────────╯";
4028
4029        assert_eq!(output, expected);
4030    }
4031
4032    #[rstest]
4033    fn test_republish_external_msgbus_message_logs_topic_and_error_chain() {
4034        log::set_logger(&EXTERNAL_INGRESS_LOG_CAPTURE).expect("test logger already installed");
4035        log::set_max_level(LevelFilter::Error);
4036        EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock().clear();
4037        let message = BusMessage::with_str_topic(
4038            "data.quotes.AUDUSD.SIM*",
4039            BusPayloadType::Custom(Ustr::from("UnregisteredCustomData")),
4040            Bytes::new(),
4041            SerializationEncoding::Json,
4042        );
4043
4044        LiveNode::republish_external_msgbus_message(&message);
4045
4046        assert_eq!(
4047            *EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock(),
4048            vec![
4049                "Failed to republish external message bus topic 'data.quotes.AUDUSD.SIM*': invalid \
4050                 external message topic: Topic `value` contained invalid characters, was \
4051                 data.quotes.AUDUSD.SIM*"
4052                    .to_string()
4053            ],
4054        );
4055    }
4056
4057    #[rstest]
4058    #[tokio::test]
4059    async fn test_external_startup_prioritizes_completed_connection() {
4060        let message = BusMessage::with_str_topic(
4061            "data.trades.FIRST",
4062            BusPayloadType::TradeTick,
4063            bytes::Bytes::new(),
4064            SerializationEncoding::Json,
4065        );
4066        let (sender, receiver) = tokio::sync::mpsc::channel(1);
4067        sender.send(message.clone()).await.unwrap();
4068        let mut receiver = Some(receiver);
4069        let mut pending = VecDeque::new();
4070
4071        let result =
4072            drive_with_external_ingress(std::future::ready(17), &mut receiver, &mut pending).await;
4073        let queued = receiver.as_mut().unwrap().try_recv().unwrap();
4074
4075        assert_eq!(result, 17);
4076        assert!(pending.is_empty());
4077        assert_eq!(
4078            serde_json::to_value(queued).unwrap(),
4079            serde_json::to_value(message).unwrap()
4080        );
4081    }
4082
4083    #[rstest]
4084    #[tokio::test]
4085    async fn test_external_startup_dispatches_correlated_response_before_buffered_traffic() {
4086        use bytes::Bytes;
4087        use nautilus_common::messages::data::{
4088            CustomDataResponse, DataResponse, DataResponseEnvelope,
4089        };
4090        use nautilus_model::data::DataType;
4091
4092        let (egress, publications, _) = CapturingExternalEgress::new();
4093        let bus = Rc::new(RefCell::new(msgbus::MessageBus::default()));
4094        msgbus::set_message_bus(bus.clone());
4095        bus.borrow_mut()
4096            .set_external_egress_config(
4097                Box::new(egress),
4098                &MessageBusConfig {
4099                    encoding: SerializationEncoding::MsgPack,
4100                    external_streams: Some(vec!["external.test".into()]),
4101                    ..Default::default()
4102                },
4103            )
4104            .unwrap();
4105
4106        bus.borrow_mut()
4107            .add_streaming_type(BusPayloadType::DataResponse);
4108        let id = UUID4::new();
4109        let response = DataResponse::Data(CustomDataResponse::new(
4110            id,
4111            "FEED".into(),
4112            None,
4113            DataType::new("Startup", None, None),
4114            Bytes::from_static(b"connected"),
4115            Some(101.into()),
4116            Some(103.into()),
4117            107.into(),
4118            None,
4119        ));
4120        msgbus::publish_any("responses.data.FEED".into(), &response);
4121        let message = BusMessage::with_str_topic(
4122            "responses.data.FEED",
4123            BusPayloadType::DataResponse,
4124            publications.borrow()[0].payload.clone(),
4125            SerializationEncoding::MsgPack,
4126        );
4127        let (done, connected) = tokio::sync::oneshot::channel();
4128        let done = Rc::new(RefCell::new(Some(done)));
4129        msgbus::register_response_handler(
4130            &id,
4131            TypedHandler::from(move |envelope: &DataResponseEnvelope| {
4132                let Some(DataResponse::Data(reply)) = envelope.response() else {
4133                    panic!("expected custom reply");
4134                };
4135
4136                done.borrow_mut()
4137                    .take()
4138                    .unwrap()
4139                    .send(reply.clone())
4140                    .unwrap();
4141            }),
4142        );
4143
4144        msgbus::subscribe_any(
4145            message.topic.as_str().into(),
4146            ShareableMessageHandler::from_typed::<DataResponse, _>(|reply| {
4147                msgbus::send_response(reply.correlation_id(), reply);
4148            }),
4149            None,
4150        );
4151
4152        let unrelated = BusMessage::with_str_topic(
4153            "data.trades.FIRST",
4154            BusPayloadType::TradeTick,
4155            Bytes::new(),
4156            SerializationEncoding::Json,
4157        );
4158        let (sender, receiver) = tokio::sync::mpsc::channel(2);
4159        sender.send(unrelated.clone()).await.unwrap();
4160        sender.send(message).await.unwrap();
4161        let mut receiver = Some(receiver);
4162        let mut pending = VecDeque::new();
4163        let actual = tokio::time::timeout(
4164            Duration::from_secs(1),
4165            drive_with_external_ingress(connected, &mut receiver, &mut pending),
4166        )
4167        .await
4168        .unwrap()
4169        .unwrap();
4170
4171        assert_eq!(actual.correlation_id, id);
4172        assert_eq!(actual.client_id, ClientId::from("FEED"));
4173        assert_eq!(actual.venue, None);
4174        assert_eq!(actual.data_type, DataType::new("Startup", None, None));
4175        assert_eq!(
4176            actual.data.downcast_ref::<Bytes>(),
4177            Some(&Bytes::from_static(b"connected"))
4178        );
4179        assert_eq!(actual.start, Some(101.into()));
4180        assert_eq!(actual.end, Some(103.into()));
4181        assert_eq!(actual.ts_init, UnixNanos::from(107));
4182        assert_eq!(actual.params, None);
4183        assert_eq!(publications.borrow().len(), 1);
4184        assert_eq!(pending.len(), 1);
4185        assert_eq!(
4186            serde_json::to_value(pending.pop_front().unwrap()).unwrap(),
4187            serde_json::to_value(unrelated).unwrap()
4188        );
4189        assert!(receiver.as_ref().unwrap().is_empty());
4190    }
4191
4192    #[rstest]
4193    #[tokio::test]
4194    async fn test_external_startup_defers_unrelated_messages_with_bounded_fifo() {
4195        let (sender, receiver) = tokio::sync::mpsc::channel(2);
4196        let mut receiver = Some(receiver);
4197
4198        let message = |name: &str| {
4199            BusMessage::with_str_topic(
4200                name,
4201                BusPayloadType::TradeTick,
4202                bytes::Bytes::new(),
4203                SerializationEncoding::Json,
4204            )
4205        };
4206
4207        let first = message("data.trades.FIRST");
4208        let second = message("data.trades.SECOND");
4209        let third = message("data.trades.THIRD");
4210        sender.send(first.clone()).await.unwrap();
4211        sender.send(second.clone()).await.unwrap();
4212        let (done, connected) = tokio::sync::oneshot::channel();
4213        let mut pending = VecDeque::new();
4214        let mut connection = Box::pin(drive_with_external_ingress(
4215            connected,
4216            &mut receiver,
4217            &mut pending,
4218        ));
4219        assert!(futures_util::poll!(&mut connection).is_pending());
4220        sender.send(third.clone()).await.unwrap();
4221        assert!(futures_util::poll!(&mut connection).is_pending());
4222        done.send(()).unwrap();
4223        connection.await.unwrap();
4224
4225        assert_eq!(pending.len(), 2);
4226        assert_eq!(receiver.as_ref().unwrap().len(), 1);
4227
4228        for expected in [first, second, third] {
4229            let received = recv_external_msgbus_message(&mut receiver, &mut pending)
4230                .await
4231                .unwrap();
4232            assert_eq!(
4233                serde_json::to_value(received).unwrap(),
4234                serde_json::to_value(expected).unwrap()
4235            );
4236        }
4237    }
4238
4239    #[rstest]
4240    #[tokio::test]
4241    async fn test_external_startup_retains_receiver_closure_for_lifecycle_cleanup() {
4242        let (sender, receiver) = tokio::sync::mpsc::channel(2);
4243        drop(sender);
4244        let mut receiver = Some(receiver);
4245        let mut pending = VecDeque::new();
4246        let (done, connected) = tokio::sync::oneshot::channel();
4247        let mut connection = Box::pin(drive_with_external_ingress(
4248            connected,
4249            &mut receiver,
4250            &mut pending,
4251        ));
4252        assert!(futures_util::poll!(&mut connection).is_pending());
4253        done.send(()).unwrap();
4254        connection.await.unwrap();
4255
4256        assert!(receiver.is_none());
4257        assert!(pending.is_empty());
4258    }
4259
4260    #[rstest]
4261    #[case(None, 2)]
4262    #[case(Some(SystemChannel::DataEvents), 1)]
4263    #[case(Some(SystemChannel::ExecCommands), 1)]
4264    fn test_publish_queue_state_transitions_reaches_typed_subscriber(
4265        #[case] channel: Option<SystemChannel>,
4266        #[case] expected_count: usize,
4267    ) {
4268        let config = LiveNodeConfig {
4269            trader_id: TraderId::from("QUEUE-001"),
4270            exec_engine: crate::config::LiveExecutionEngineConfig {
4271                reconciliation: false,
4272                ..Default::default()
4273            },
4274            ..Default::default()
4275        };
4276
4277        let node = LiveNode::build("QueuePublicationNode".to_string(), Some(config)).unwrap();
4278        let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4279
4280        let handler = ShareableMessageHandler::from_typed({
4281            let received = received.clone();
4282            move |event: &QueueStateChanged| received.borrow_mut().push(event.clone())
4283        });
4284
4285        msgbus::subscribe_any(
4286            MessagingSwitchboard::queue_state_changed_pattern(channel),
4287            handler,
4288            None,
4289        );
4290
4291        let transitions = [
4292            QueueStateTransition {
4293                channel: SystemChannel::DataEvents,
4294                condition: QueueCondition::Backlogged,
4295                state: QueueState::Triggered,
4296                queue_depth: 17,
4297                mean_dispatch_ns: 23,
4298            },
4299            QueueStateTransition {
4300                channel: SystemChannel::ExecCommands,
4301                condition: QueueCondition::Slow,
4302                state: QueueState::Triggered,
4303                queue_depth: 17,
4304                mean_dispatch_ns: 23,
4305            },
4306        ];
4307
4308        node.publish_queue_state_transitions(&transitions);
4309
4310        let events = received.borrow();
4311        assert_eq!(events.len(), expected_count);
4312
4313        let expected = transitions
4314            .into_iter()
4315            .filter(|transition| channel.is_none_or(|channel| channel == transition.channel));
4316
4317        for (event, transition) in events.iter().zip(expected) {
4318            assert_eq!(event.trader_id, TraderId::from("QUEUE-001"));
4319            assert_eq!(event.channel, transition.channel);
4320            assert_eq!(event.condition, transition.condition);
4321            assert_eq!(event.state, transition.state);
4322            assert_eq!(event.queue_depth, transition.queue_depth);
4323            assert_eq!(event.mean_dispatch_ns, transition.mean_dispatch_ns);
4324            assert_ne!(event.event_id, UUID4::default());
4325            assert_ne!(event.ts_event, UnixNanos::default());
4326            assert_eq!(event.ts_init, event.ts_event);
4327        }
4328
4329        if channel.is_none() {
4330            assert_ne!(events[0].event_id, events[1].event_id);
4331        }
4332
4333        drop(events);
4334        msgbus::get_message_bus().borrow_mut().dispose();
4335    }
4336
4337    #[rstest]
4338    #[case(None, None)]
4339    #[case(Some(ClientId::from("BINANCE")), None)]
4340    #[case(None, Some("binance-futures-market-streams"))]
4341    #[case(
4342        Some(ClientId::from("BINANCE")),
4343        Some("binance-futures-market-streams")
4344    )]
4345    fn test_process_socket_state_change_reaches_typed_subscriber(
4346        #[case] client_id: Option<ClientId>,
4347        #[case] endpoint: Option<&str>,
4348    ) {
4349        let config = LiveNodeConfig {
4350            trader_id: TraderId::from("SOCKET-001"),
4351            exec_engine: crate::config::LiveExecutionEngineConfig {
4352                reconciliation: false,
4353                ..Default::default()
4354            },
4355            ..Default::default()
4356        };
4357
4358        let node = LiveNode::build("SocketPublicationNode".to_string(), Some(config)).unwrap();
4359        let received = Rc::new(RefCell::new(Vec::<SocketStateChanged>::new()));
4360
4361        let handler = ShareableMessageHandler::from_typed({
4362            let received = received.clone();
4363            move |event: &SocketStateChanged| received.borrow_mut().push(event.clone())
4364        });
4365
4366        msgbus::subscribe_any(
4367            MessagingSwitchboard::socket_state_changed_pattern(client_id, endpoint),
4368            handler,
4369            None,
4370        );
4371
4372        let change = SocketStateChange::new(
4373            ClientId::from("BINANCE"),
4374            Some(Venue::from("BINANCE")),
4375            Ustr::from("binance-futures-market-streams"),
4376            SocketState::Disconnected,
4377        );
4378
4379        node.process_system_event(SystemEvent::SocketState(change));
4380
4381        let events = received.borrow();
4382        assert_eq!(events.len(), 1);
4383        assert_eq!(events[0].trader_id, TraderId::from("SOCKET-001"));
4384        assert_eq!(events[0].client_id, ClientId::from("BINANCE"));
4385        assert_eq!(events[0].venue, Some(Venue::from("BINANCE")));
4386        assert_eq!(
4387            events[0].endpoint,
4388            Ustr::from("binance-futures-market-streams")
4389        );
4390        assert_eq!(events[0].state, SocketState::Disconnected);
4391        assert_ne!(events[0].event_id, UUID4::default());
4392        assert_ne!(events[0].ts_event, UnixNanos::default());
4393        assert_eq!(events[0].ts_init, events[0].ts_event);
4394        drop(events);
4395        msgbus::get_message_bus().borrow_mut().dispose();
4396    }
4397
4398    #[rstest]
4399    #[case::accepted(
4400        ReconnectRequestOutcome::Accepted,
4401        SocketReconnectDispatchOutcome::Accepted
4402    )]
4403    #[case::already_reconnecting(
4404        ReconnectRequestOutcome::AlreadyReconnecting,
4405        SocketReconnectDispatchOutcome::AlreadyReconnecting
4406    )]
4407    #[case::disconnected(
4408        ReconnectRequestOutcome::Disconnected,
4409        SocketReconnectDispatchOutcome::Disconnected
4410    )]
4411    #[case::closed(
4412        ReconnectRequestOutcome::Closed,
4413        SocketReconnectDispatchOutcome::Closed
4414    )]
4415    #[case::unsupported(
4416        ReconnectRequestOutcome::Unsupported,
4417        SocketReconnectDispatchOutcome::Unsupported
4418    )]
4419    fn test_request_socket_reconnect_maps_transport_outcome(
4420        #[case] transport: ReconnectRequestOutcome,
4421        #[case] expected: SocketReconnectDispatchOutcome,
4422    ) {
4423        let registry = SocketReconnectRegistry::default();
4424        let client_id = ClientId::from("TEST");
4425        let endpoint = Ustr::from("test-streams");
4426        let control = SocketControl::with_registry(client_id, None, endpoint, &registry);
4427        let _sink = control.sink();
4428        control.register(move || transport);
4429
4430        let outcome = LiveNode::request_socket_reconnect(registry.get(client_id, endpoint));
4431
4432        assert_eq!(outcome, expected);
4433    }
4434
4435    #[rstest]
4436    #[case::client_not_found(
4437        SocketReconnectLookup::ClientNotFound,
4438        SocketReconnectDispatchOutcome::UnknownClient
4439    )]
4440    #[case::unsupported(
4441        SocketReconnectLookup::Unsupported,
4442        SocketReconnectDispatchOutcome::Unsupported
4443    )]
4444    #[case::endpoint_not_found(
4445        SocketReconnectLookup::EndpointNotFound,
4446        SocketReconnectDispatchOutcome::UnknownEndpoint
4447    )]
4448    #[case::ambiguous(
4449        SocketReconnectLookup::AmbiguousEndpoint,
4450        SocketReconnectDispatchOutcome::AmbiguousEndpoint
4451    )]
4452    fn test_request_socket_reconnect_maps_lookup_failure(
4453        #[case] lookup: SocketReconnectLookup,
4454        #[case] expected: SocketReconnectDispatchOutcome,
4455    ) {
4456        assert_eq!(LiveNode::request_socket_reconnect(lookup), expected);
4457    }
4458
4459    #[rstest]
4460    fn test_system_dispatch_restores_context(#[values(false, true)] startup: bool) {
4461        let trader_id = TraderId::from("SOCKET-001");
4462
4463        let config = LiveNodeConfig {
4464            trader_id,
4465            ..Default::default()
4466        };
4467
4468        let mut node = LiveNode::build("SystemDispatchNode".to_string(), Some(config)).unwrap();
4469        let client_id = ClientId::from("TEST");
4470        let endpoint = Ustr::from("test-streams");
4471        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
4472        let sender = DispatchSender::new(tx);
4473        let events = sender.clone();
4474        msgbus::subscribe_any(
4475            MessagingSwitchboard::socket_state_changed_pattern(
4476                Some(client_id),
4477                Some(endpoint.as_str()),
4478            ),
4479            ShareableMessageHandler::from_typed(move |_: &SocketStateChanged| {
4480                events.send(17u32).unwrap();
4481            }),
4482            None,
4483        );
4484
4485        let control =
4486            SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4487        let _sink = control.sink();
4488        control.register(move || {
4489            sender.send(23u32).unwrap();
4490            ReconnectRequestOutcome::Accepted
4491        });
4492
4493        let event = SystemEvent::SocketState(SocketStateChange::new(
4494            client_id,
4495            None,
4496            endpoint,
4497            SocketState::Connected,
4498        ));
4499        let command = SystemCommand::ReconnectSocket(ReconnectSocket::new(
4500            trader_id,
4501            client_id,
4502            endpoint,
4503            31.into(),
4504        ));
4505
4506        if startup {
4507            let mut pending = PendingEvents::default();
4508            pending.system_events.push(event.into());
4509            pending.system_commands.push(command.into());
4510            node.process_system_events(pending.take_system_events());
4511            node.process_system_commands(pending.take_system_commands());
4512            assert!(pending.is_empty());
4513        } else {
4514            node.process_runner_event(PendingRunnerEvent::SystemEvent(event.into()));
4515            node.process_runner_event(PendingRunnerEvent::SystemCommand(command.into()));
4516        }
4517
4518        let event_child = rx.try_recv().unwrap();
4519        let command_child = rx.try_recv().unwrap();
4520        assert!(event_child.is_rooted());
4521        assert!(command_child.is_rooted());
4522        assert_eq!(event_child.dispatch(|value| value), 17);
4523        assert_eq!(command_child.dispatch(|value| value), 23);
4524        assert!(rx.is_empty());
4525        msgbus::get_message_bus().borrow_mut().dispose();
4526    }
4527
4528    #[rstest]
4529    fn test_process_socket_reconnect_routes_only_matching_trader() {
4530        let trader_id = TraderId::from("SOCKET-001");
4531
4532        let config = LiveNodeConfig {
4533            trader_id,
4534            exec_engine: crate::config::LiveExecutionEngineConfig {
4535                reconciliation: false,
4536                ..Default::default()
4537            },
4538            ..Default::default()
4539        };
4540
4541        let node = LiveNode::build("SocketReconnectNode".to_string(), Some(config)).unwrap();
4542        let client_id = ClientId::from("TEST");
4543        let endpoint = Ustr::from("test-streams");
4544        let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
4545        let request_count = Arc::clone(&requests);
4546        let control =
4547            SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4548        let _sink = control.sink();
4549        control.register(move || {
4550            request_count.fetch_add(1, Ordering::SeqCst);
4551            ReconnectRequestOutcome::Accepted
4552        });
4553
4554        node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4555            TraderId::from("OTHER-001"),
4556            client_id,
4557            endpoint,
4558            UnixNanos::default(),
4559        )));
4560        assert_eq!(requests.load(Ordering::SeqCst), 0);
4561
4562        node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4563            trader_id,
4564            client_id,
4565            endpoint,
4566            UnixNanos::default(),
4567        )));
4568        assert_eq!(requests.load(Ordering::SeqCst), 1);
4569    }
4570
4571    #[rstest]
4572    #[tokio::test]
4573    async fn test_start_publishes_socket_change_after_actor_subscribes() {
4574        let config = LiveNodeConfig {
4575            trader_id: TraderId::from("SOCKET-STARTUP-001"),
4576            exec_engine: crate::config::LiveExecutionEngineConfig {
4577                reconciliation: false,
4578                ..Default::default()
4579            },
4580            timeout_connection: Duration::ZERO,
4581            timeout_reconciliation: Duration::ZERO,
4582            timeout_portfolio: Duration::ZERO,
4583            timeout_disconnection: Duration::ZERO,
4584            delay_post_stop: Duration::ZERO,
4585            timeout_shutdown: Duration::ZERO,
4586            ..Default::default()
4587        };
4588
4589        let mut node = LiveNode::build("SocketStartupNode".to_string(), Some(config)).unwrap();
4590        let received = Rc::new(RefCell::new(Vec::new()));
4591        node.add_actor(StartupSocketActor::new(Rc::clone(&received)))
4592            .unwrap();
4593        node.runner.as_ref().unwrap().bind_senders();
4594
4595        let change = SocketStateChange::new(
4596            ClientId::from("BINANCE"),
4597            Some(Venue::from("BINANCE")),
4598            Ustr::from("binance-futures-market-streams"),
4599            SocketState::Connected,
4600        );
4601        get_system_event_sender()
4602            .send(SystemEvent::SocketState(change))
4603            .unwrap();
4604
4605        node.start().await.unwrap();
4606
4607        {
4608            let events = received.borrow();
4609            assert_eq!(events.len(), 1);
4610            assert_eq!(events[0].trader_id, TraderId::from("SOCKET-STARTUP-001"));
4611            assert_eq!(events[0].client_id, change.client_id);
4612            assert_eq!(events[0].venue, change.venue);
4613            assert_eq!(events[0].endpoint, change.endpoint);
4614            assert_eq!(events[0].state, change.state);
4615            assert_ne!(events[0].event_id, UUID4::default());
4616            assert_ne!(events[0].ts_event, UnixNanos::default());
4617            assert_eq!(events[0].ts_init, events[0].ts_event);
4618        }
4619
4620        node.stop().await.unwrap();
4621        node.dispose();
4622    }
4623
4624    #[rstest]
4625    #[tokio::test(flavor = "current_thread")]
4626    async fn test_run_publishes_queue_state_after_dispatch_sample() {
4627        let config = LiveNodeConfig {
4628            trader_id: TraderId::from("QUEUE-RUN-001"),
4629            queue_monitor: Some(crate::config::QueueMonitorConfig {
4630                queue_depth_trigger: usize::MAX,
4631                queue_depth_clear: 0,
4632                mean_dispatch_ns_trigger: 1,
4633                mean_dispatch_ns_clear: 0,
4634            }),
4635            exec_engine: crate::config::LiveExecutionEngineConfig {
4636                reconciliation: false,
4637                ..Default::default()
4638            },
4639            delay_post_stop: Duration::ZERO,
4640            ..Default::default()
4641        };
4642
4643        let mut node = LiveNode::build("QueueMonitorRunNode".to_string(), Some(config)).unwrap();
4644        let handle = node.handle();
4645        let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4646
4647        let handler = ShareableMessageHandler::from_typed({
4648            let received = received.clone();
4649            let stop_handle = handle.clone();
4650
4651            move |event: &QueueStateChanged| {
4652                received.borrow_mut().push(event.clone());
4653                stop_handle.stop();
4654            }
4655        });
4656
4657        msgbus::subscribe_any(
4658            MessagingSwitchboard::queue_state_changed_pattern(None),
4659            handler,
4660            None,
4661        );
4662        let drive_handle = handle.clone();
4663
4664        let result = tokio::time::timeout(Duration::from_secs(5), async {
4665            let run = node.run();
4666            tokio::pin!(run);
4667
4668            let drive = async move {
4669                wait_until_async(
4670                    || async { drive_handle.is_running() },
4671                    Duration::from_secs(2),
4672                )
4673                .await;
4674                get_data_event_sender().send(stub_data_event()).unwrap();
4675            };
4676
4677            let (run_result, ()) = tokio::join!(run, drive);
4678            run_result
4679        })
4680        .await;
4681
4682        assert!(
4683            result.is_ok(),
4684            "queue state event should arrive before timeout"
4685        );
4686        assert!(result.unwrap().is_ok(), "run() should succeed");
4687        let events = received.borrow();
4688        assert_eq!(events.len(), 1);
4689        assert_eq!(events[0].trader_id, TraderId::from("QUEUE-RUN-001"));
4690        assert_eq!(events[0].channel, SystemChannel::DataEvents);
4691        assert_eq!(events[0].condition, QueueCondition::Slow);
4692        assert_eq!(events[0].state, QueueState::Triggered);
4693        assert_eq!(events[0].queue_depth, 0);
4694        assert!(events[0].mean_dispatch_ns > 0);
4695        assert_ne!(events[0].event_id, UUID4::default());
4696        assert_ne!(events[0].ts_event, UnixNanos::default());
4697        assert_eq!(events[0].ts_init, events[0].ts_event);
4698        drop(events);
4699        msgbus::get_message_bus().borrow_mut().dispose();
4700    }
4701
4702    #[rstest]
4703    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4704    async fn test_run_dispatch_progress_under_load(
4705        #[values(false, true)] execution: bool,
4706        #[values(false, true)] slow: bool,
4707    ) {
4708        let config = LiveNodeConfig {
4709            delay_post_stop: Duration::ZERO,
4710            ..Default::default()
4711        };
4712        let mut node = LiveNode::build("FairDispatchNode".to_string(), Some(config)).unwrap();
4713        let handle = node.handle();
4714        let stop = handle.clone();
4715        let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
4716            if slow {
4717                // Make housekeeping ready after each timer callback
4718                std::thread::sleep(Duration::from_millis(110));
4719            }
4720
4721            if event.ts_init == UnixNanos::from(8) {
4722                // Check progress before shutdown can drain the queues
4723                let snapshot = stop.metrics_snapshot();
4724                assert_eq!(snapshot.time_events.dispatched, 7);
4725
4726                if execution {
4727                    assert!((1..16).contains(&snapshot.exec_events.dispatched));
4728                    assert_eq!(snapshot.data_events.dispatched, 0);
4729                } else {
4730                    assert!(snapshot.data_events.dispatched > 0);
4731                    assert_eq!(snapshot.data_commands.dispatched, 1);
4732                }
4733                stop.stop();
4734            }
4735        }));
4736
4737        let drive = async {
4738            wait_until_async(|| async { handle.is_running() }, Duration::from_secs(2)).await;
4739
4740            for ts in (1..=16_u64).map(UnixNanos::from) {
4741                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
4742                    TimeEvent::new("fairness".into(), UUID4::new(), ts, ts),
4743                    callback.clone(),
4744                ));
4745                get_data_event_sender().send(stub_data_event()).unwrap();
4746
4747                if execution {
4748                    get_exec_event_sender().send(stub_exec_event()).unwrap();
4749                }
4750            }
4751            nautilus_common::runner::get_data_cmd_sender().execute(stub_data_command());
4752        };
4753        tokio::time::timeout(Duration::from_secs(5), async {
4754            let (result, ()) = tokio::join!(node.run(), drive);
4755            result.unwrap();
4756        })
4757        .await
4758        .unwrap();
4759        assert!(handle.should_stop());
4760    }
4761
4762    #[rstest]
4763    fn test_observe_exec_event_before_dispatch_skips_recent_fill_report() {
4764        let config = LiveNodeConfig {
4765            exec_engine: crate::config::LiveExecutionEngineConfig {
4766                reconciliation: false,
4767                ..Default::default()
4768            },
4769            ..Default::default()
4770        };
4771
4772        let mut node = LiveNode::build("FillSkipNode".to_string(), Some(config)).unwrap();
4773        let event = stub_exec_event();
4774        let account_id = AccountId::from("TEST-001");
4775        let instrument_id = InstrumentId::from("TEST.VENUE");
4776        let trade_id = TradeId::from("T-001");
4777
4778        let close_ids = node.observe_exec_event_before_dispatch(&event);
4779        assert_eq!(close_ids, Some(Vec::new()));
4780        assert!(
4781            !node
4782                .exec_manager
4783                .is_fill_recently_processed(account_id, instrument_id, trade_id)
4784        );
4785
4786        node.exec_manager
4787            .mark_fill_processed(account_id, instrument_id, trade_id);
4788
4789        let close_ids = node.observe_exec_event_before_dispatch(&event);
4790        assert_eq!(close_ids, None);
4791    }
4792
4793    #[rstest]
4794    #[case(false, false, OrderStatus::Canceled, 1)]
4795    #[case(false, true, OrderStatus::Accepted, 0)]
4796    #[case(true, false, OrderStatus::Canceled, 1)]
4797    #[case(true, true, OrderStatus::Accepted, 0)]
4798    fn test_process_exec_event_clears_terminal_activity_only_after_cached_order_closes(
4799        #[case] with_fills: bool,
4800        #[case] superseded: bool,
4801        #[case] expected_status: OrderStatus,
4802        #[case] expected_query_count: usize,
4803    ) {
4804        let config = LiveNodeConfig {
4805            exec_engine: crate::config::LiveExecutionEngineConfig {
4806                reconciliation: true,
4807                open_check_threshold_ms: 5_000,
4808                single_order_query_delay_ms: 0,
4809                ..Default::default()
4810            },
4811            ..Default::default()
4812        };
4813
4814        let mut node = LiveNode::build("TerminalReportNode".to_string(), Some(config)).unwrap();
4815        let client_order_id = ClientOrderId::from("O-TERMINAL-REPORT");
4816        let old_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-OLD");
4817        let new_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-NEW");
4818        let account_id = AccountId::from("TEST-001");
4819        let client_id = ClientId::from("TEST");
4820        let instrument = crypto_perpetual_ethusdt();
4821        let instrument_id = instrument.id();
4822        let account = AccountAny::Margin(MarginAccount::new(
4823            AccountState::new(
4824                account_id,
4825                AccountType::Margin,
4826                vec![AccountBalance::new(
4827                    Money::from("1000000 USDT"),
4828                    Money::from("0 USDT"),
4829                    Money::from("1000000 USDT"),
4830                )],
4831                Vec::new(),
4832                true,
4833                UUID4::new(),
4834                UnixNanos::default(),
4835                UnixNanos::default(),
4836                Some(Currency::USDT()),
4837            ),
4838            true,
4839        ));
4840        node.kernel.cache.borrow_mut().add_account(account).unwrap();
4841        node.kernel
4842            .cache
4843            .borrow_mut()
4844            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
4845            .unwrap();
4846        insert_accepted_limit_order_in_node(
4847            &node,
4848            account_id,
4849            client_id,
4850            instrument_id,
4851            client_order_id,
4852            old_venue_order_id,
4853        );
4854
4855        if superseded {
4856            let order = node
4857                .kernel
4858                .cache
4859                .borrow()
4860                .order_owned(&client_order_id)
4861                .unwrap();
4862            let pending_update = OrderPendingUpdateSpec::builder()
4863                .trader_id(order.trader_id())
4864                .strategy_id(order.strategy_id())
4865                .instrument_id(order.instrument_id())
4866                .client_order_id(client_order_id)
4867                .account_id(account_id)
4868                .venue_order_id(old_venue_order_id)
4869                .build();
4870            node.kernel
4871                .cache
4872                .borrow_mut()
4873                .update_order(&OrderEventAny::PendingUpdate(pending_update))
4874                .unwrap();
4875            let order = node
4876                .kernel
4877                .cache
4878                .borrow()
4879                .order_owned(&client_order_id)
4880                .unwrap();
4881            let updated = OrderUpdatedSpec::builder()
4882                .trader_id(order.trader_id())
4883                .strategy_id(order.strategy_id())
4884                .instrument_id(order.instrument_id())
4885                .client_order_id(client_order_id)
4886                .quantity(order.quantity())
4887                .venue_order_id(new_venue_order_id)
4888                .account_id(account_id)
4889                .build();
4890            node.kernel
4891                .cache
4892                .borrow_mut()
4893                .update_order(&OrderEventAny::Updated(updated))
4894                .unwrap();
4895        }
4896
4897        let report = OrderStatusReport::new(
4898            account_id,
4899            instrument_id,
4900            Some(client_order_id),
4901            old_venue_order_id,
4902            OrderSide::Buy.into(),
4903            OrderType::Limit,
4904            TimeInForce::Gtc,
4905            OrderStatus::Canceled,
4906            Quantity::from("10.0"),
4907            Quantity::from("0.0"),
4908            UnixNanos::from(1_000),
4909            UnixNanos::from(2_000),
4910            UnixNanos::from(3_000),
4911            None,
4912        );
4913
4914        let report = if with_fills {
4915            ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
4916        } else {
4917            ExecutionReport::Order(Box::new(report))
4918        };
4919
4920        let event = ExecutionEvent::Report(report);
4921
4922        node.process_exec_event(event);
4923
4924        let order = node
4925            .kernel
4926            .cache
4927            .borrow()
4928            .order_owned(&client_order_id)
4929            .unwrap();
4930
4931        let expected_venue_order_id = if superseded {
4932            new_venue_order_id
4933        } else {
4934            old_venue_order_id
4935        };
4936
4937        assert_eq!(order.status(), expected_status);
4938        assert_eq!(order.venue_order_id(), Some(expected_venue_order_id));
4939
4940        if !superseded {
4941            let replacement = OrderTestBuilder::new(OrderType::Limit)
4942                .client_order_id(client_order_id)
4943                .instrument_id(instrument_id)
4944                .quantity(Quantity::from("20.0"))
4945                .price(Price::from("200.0"))
4946                .build();
4947            let submitted = TestOrderEventStubs::submitted(&replacement, account_id);
4948            node.kernel
4949                .cache
4950                .borrow_mut()
4951                .add_order(replacement, None, Some(client_id), true)
4952                .unwrap();
4953            let replacement = node
4954                .kernel
4955                .cache
4956                .borrow_mut()
4957                .update_order(&submitted)
4958                .unwrap();
4959            let accepted =
4960                TestOrderEventStubs::accepted(&replacement, account_id, old_venue_order_id);
4961            node.kernel
4962                .cache
4963                .borrow_mut()
4964                .update_order(&accepted)
4965                .unwrap();
4966        }
4967
4968        assert_eq!(
4969            node.exec_manager.check_open_order_queries().len(),
4970            expected_query_count,
4971        );
4972    }
4973
4974    #[rstest]
4975    fn test_rejected_direct_fill_stays_eligible_for_later_report() {
4976        let (mut node, mut fill_event, _) = recent_fill_test_fixture("RejectedDirectFillNode");
4977
4978        let OrderEventAny::Filled(fill) = &mut fill_event else {
4979            unreachable!();
4980        };
4981
4982        fill.client_order_id = ClientOrderId::from("O-UNKNOWN");
4983        fill.venue_order_id = VenueOrderId::from("V-UNKNOWN");
4984        let fill = fill.clone();
4985        let report_event = fill_report_event(&fill);
4986        let event = ExecutionEvent::Order(OrderEventAny::Filled(fill.clone()));
4987
4988        assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4989        let marked_before_dispatch = is_recent_fill(&node, &fill);
4990
4991        node.dispatch_exec_event_and_commit_fill(event);
4992
4993        let marked_after_dispatch = is_recent_fill(&node, &fill);
4994        let later_report_is_eligible = node
4995            .observe_exec_event_before_dispatch(&report_event)
4996            .is_some();
4997        assert_eq!(
4998            (
4999                marked_before_dispatch,
5000                marked_after_dispatch,
5001                later_report_is_eligible,
5002            ),
5003            (false, false, true),
5004        );
5005    }
5006
5007    #[rstest]
5008    fn test_applied_direct_fill_commits_and_skips_later_report() {
5009        let (mut node, fill_event, _) = recent_fill_test_fixture("AppliedDirectFillNode");
5010
5011        let OrderEventAny::Filled(fill) = &fill_event else {
5012            unreachable!();
5013        };
5014
5015        let fill = fill.clone();
5016        let report_event = fill_report_event(&fill);
5017        let event = ExecutionEvent::Order(fill_event);
5018
5019        assert!(node.observe_exec_event_before_dispatch(&event).is_some());
5020        assert!(!is_recent_fill(&node, &fill));
5021
5022        node.dispatch_exec_event_and_commit_fill(event);
5023
5024        assert!(is_recent_fill(&node, &fill));
5025        assert_eq!(node.observe_exec_event_before_dispatch(&report_event), None);
5026    }
5027
5028    #[rstest]
5029    fn test_canonical_duplicate_fill_counts_as_applied() {
5030        let (mut node, fill_event, _) = recent_fill_test_fixture("DuplicateDirectFillNode");
5031
5032        let OrderEventAny::Filled(fill) = &fill_event else {
5033            unreachable!();
5034        };
5035
5036        let mut fill = fill.clone();
5037        node.kernel
5038            .cache
5039            .borrow_mut()
5040            .update_order(&fill_event)
5041            .unwrap();
5042        fill.client_order_id = ClientOrderId::from("O-DUPLICATE-UNKNOWN");
5043
5044        node.exec_manager.commit_recent_fill_if_applied(&fill);
5045
5046        assert!(is_recent_fill(&node, &fill));
5047    }
5048
5049    #[rstest]
5050    #[case(false)]
5051    #[case(true)]
5052    fn test_continuous_reconciliation_commits_only_applied_fill(#[case] applied: bool) {
5053        let (mut node, mut fill_event, _) = recent_fill_test_fixture(if applied {
5054            "AppliedContinuousFillNode"
5055        } else {
5056            "RejectedContinuousFillNode"
5057        });
5058
5059        if !applied {
5060            let OrderEventAny::Filled(fill) = &mut fill_event else {
5061                unreachable!();
5062            };
5063
5064            fill.client_order_id = ClientOrderId::from("O-CONTINUOUS-UNKNOWN");
5065            fill.venue_order_id = VenueOrderId::from("V-CONTINUOUS-UNKNOWN");
5066        }
5067
5068        let OrderEventAny::Filled(fill) = &fill_event else {
5069            unreachable!();
5070        };
5071
5072        let fill = fill.clone();
5073
5074        node.process_reconciliation_events(&[fill_event]);
5075
5076        assert_eq!(is_recent_fill(&node, &fill), applied);
5077    }
5078
5079    #[rstest]
5080    fn test_recent_fill_commit_requires_account_and_instrument_match() {
5081        let (mut node, fill_event, _) = recent_fill_test_fixture("MismatchedDirectFillNode");
5082        node.kernel
5083            .cache
5084            .borrow_mut()
5085            .update_order(&fill_event)
5086            .unwrap();
5087
5088        let OrderEventAny::Filled(fill) = fill_event else {
5089            unreachable!();
5090        };
5091
5092        let mut account_mismatch = fill.clone();
5093        account_mismatch.account_id = AccountId::from("OTHER-001");
5094        let mut instrument_mismatch = fill;
5095        instrument_mismatch.instrument_id = InstrumentId::from("OTHER.VENUE");
5096
5097        node.exec_manager
5098            .commit_recent_fill_if_applied(&account_mismatch);
5099        node.exec_manager
5100            .commit_recent_fill_if_applied(&instrument_mismatch);
5101
5102        assert!(!is_recent_fill(&node, &account_mismatch));
5103        assert!(!is_recent_fill(&node, &instrument_mismatch));
5104    }
5105
5106    #[rstest]
5107    fn test_applied_inferred_fill_remains_recently_processed() {
5108        let (mut node, _, instrument) = recent_fill_test_fixture("InferredFillNode");
5109        let client_order_id = ClientOrderId::from("O-RECENT-FILL");
5110        let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
5111        let account_id = AccountId::from("TEST-001");
5112        let order = node
5113            .kernel
5114            .cache
5115            .borrow()
5116            .order_owned(&client_order_id)
5117            .unwrap();
5118        let report = OrderStatusReport::new(
5119            account_id,
5120            instrument.id(),
5121            Some(client_order_id),
5122            venue_order_id,
5123            OrderSide::Buy.into(),
5124            OrderType::Limit,
5125            TimeInForce::Gtc,
5126            OrderStatus::PartiallyFilled,
5127            Quantity::from("10.0"),
5128            Quantity::from("1.0"),
5129            UnixNanos::from(1_000),
5130            UnixNanos::from(1_000),
5131            UnixNanos::from(1_000),
5132            None,
5133        )
5134        .with_avg_px(dec!(100.0));
5135        let inferred = create_inferred_fill_for_qty(
5136            &order,
5137            &report,
5138            &account_id,
5139            &instrument,
5140            Quantity::from("1.0"),
5141            UnixNanos::from(1_000),
5142            None,
5143        )
5144        .unwrap();
5145
5146        let OrderEventAny::Filled(fill) = &inferred else {
5147            unreachable!();
5148        };
5149
5150        let fill = fill.clone();
5151
5152        node.process_reconciliation_events(&[inferred]);
5153
5154        assert!(fill.reconciliation);
5155        assert!(is_recent_fill(&node, &fill));
5156    }
5157
5158    #[rstest]
5159    fn test_drain_exec_events_before_reconciliation_applies_real_fill_first() {
5160        let (mut node, fill_event, instrument) = recent_fill_test_fixture("DeferredFillNode");
5161
5162        let OrderEventAny::Filled(fill) = fill_event else {
5163            unreachable!();
5164        };
5165
5166        let account_id = fill.account_id;
5167        let venue_order_id = fill.venue_order_id;
5168        let client_id = ClientId::from("TEST-EVIDENCE");
5169        node.kernel
5170            .exec_engine
5171            .borrow_mut()
5172            .register_client(Box::new(StubExecutionClient::new(
5173                client_id,
5174                account_id,
5175                instrument.id().venue,
5176                OmsType::Netting,
5177                None,
5178            )))
5179            .unwrap();
5180
5181        let report = FillReport::new(
5182            account_id,
5183            instrument.id(),
5184            venue_order_id,
5185            fill.trade_id,
5186            fill.order_side,
5187            fill.last_qty,
5188            fill.last_px,
5189            fill.commission.unwrap(),
5190            fill.liquidity_side,
5191            Some(fill.client_order_id),
5192            None,
5193            fill.ts_event,
5194            fill.ts_init,
5195            None,
5196        );
5197        let (exec_evt_tx, mut exec_evt_rx) =
5198            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
5199        exec_evt_tx
5200            .send(ExecutionEvent::Report(ExecutionReport::Fill(Box::new(report))).into())
5201            .unwrap();
5202
5203        node.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
5204
5205        assert!(exec_evt_rx.try_recv().is_err());
5206        let order = node
5207            .kernel
5208            .cache
5209            .borrow()
5210            .order_owned(&fill.client_order_id)
5211            .unwrap();
5212        assert_eq!(order.filled_qty(), fill.last_qty);
5213        assert_eq!(order.trade_ids(), &[&fill.trade_id]);
5214
5215        let report = OrderStatusReport::new(
5216            account_id,
5217            instrument.id(),
5218            Some(fill.client_order_id),
5219            venue_order_id,
5220            OrderSide::Buy.into(),
5221            OrderType::Limit,
5222            TimeInForce::Gtc,
5223            OrderStatus::PartiallyFilled,
5224            Quantity::from("10.0"),
5225            fill.last_qty,
5226            UnixNanos::from(1_000),
5227            UnixNanos::from(1_000),
5228            UnixNanos::from(1_000),
5229            None,
5230        )
5231        .with_avg_px(dec!(100.0));
5232        let engine = node.kernel.exec_engine.borrow();
5233        let clients = engine.get_all_clients();
5234        let check = node
5235            .exec_manager
5236            .prepare_open_order_report_check(UUID4::new(), &clients);
5237        let reconciliation = node.exec_manager.reconcile_open_order_reports(
5238            &check,
5239            vec![SourcedOrderStatusReport { client_id, report }],
5240            &IndexSet::new(),
5241            &IndexSet::new(),
5242            &clients,
5243        );
5244        drop(engine);
5245        node.process_reconciliation_events(&reconciliation.events);
5246
5247        let order = node
5248            .kernel
5249            .cache
5250            .borrow()
5251            .order_owned(&fill.client_order_id)
5252            .unwrap();
5253        assert_eq!(order.filled_qty(), fill.last_qty);
5254        assert_eq!(order.trade_ids(), &[&fill.trade_id]);
5255    }
5256
5257    #[rstest]
5258    #[tokio::test]
5259    async fn test_request_position_fill_reports_keeps_failed_query_unsuccessful() {
5260        let client_id = ClientId::from("POSITION-FILLS");
5261        let account_id = AccountId::from("POSITION-FILLS-001");
5262        let instrument_id = crypto_perpetual_ethusdt().id();
5263        let commands = Rc::new(RefCell::new(Vec::new()));
5264
5265        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5266            client_id,
5267            account_id,
5268            venue: instrument_id.venue,
5269            outcome: FillReportClientOutcome::Failure,
5270            commands: commands.clone(),
5271        }));
5272
5273        let command = GenerateFillReports::new(
5274            UUID4::new(),
5275            UnixNanos::from(2_000),
5276            Some(instrument_id),
5277            None,
5278            Some(UnixNanos::from(1_000)),
5279            Some(UnixNanos::from(2_000)),
5280            None,
5281            None,
5282        );
5283
5284        let result = request_position_fill_reports(
5285            vec![client],
5286            vec![PositionFillReportQuery {
5287                key: (instrument_id, account_id),
5288                client_id,
5289                command: command.clone(),
5290            }],
5291        )
5292        .await
5293        .finish();
5294
5295        assert_eq!(*commands.borrow(), vec![command]);
5296        assert!(result.is_empty());
5297    }
5298
5299    #[rstest]
5300    #[tokio::test]
5301    async fn test_request_position_fill_reports_accepts_scoped_response() {
5302        let client_id = ClientId::from("POSITION-FILLS");
5303        let account_id = AccountId::from("POSITION-FILLS-001");
5304        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5305        let instrument_id = instrument.id();
5306
5307        let report_b = FillReport::new(
5308            account_id,
5309            instrument_id,
5310            VenueOrderId::from("V-POSITION-FILLS"),
5311            TradeId::from("T-POSITION-FILLS-B"),
5312            OrderSide::Buy,
5313            Quantity::from("1.0"),
5314            Price::from("100.0"),
5315            Money::zero(instrument.quote_currency()),
5316            LiquiditySide::Taker,
5317            Some(ClientOrderId::from("O-POSITION-FILLS")),
5318            None,
5319            UnixNanos::from(1_500),
5320            UnixNanos::from(2_000),
5321            None,
5322        );
5323        let mut report_a = report_b.clone();
5324        report_a.trade_id = TradeId::from("T-POSITION-FILLS-A");
5325        let commands = Rc::new(RefCell::new(Vec::new()));
5326
5327        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5328            client_id,
5329            account_id,
5330            venue: instrument_id.venue,
5331            outcome: FillReportClientOutcome::Reports(vec![report_b.clone(), report_a.clone()]),
5332            commands,
5333        }));
5334
5335        let command = GenerateFillReports::new(
5336            UUID4::new(),
5337            UnixNanos::from(2_000),
5338            Some(instrument_id),
5339            None,
5340            Some(UnixNanos::from(1_000)),
5341            Some(UnixNanos::from(2_000)),
5342            None,
5343            None,
5344        );
5345        let key = (instrument_id, account_id);
5346
5347        let result = request_position_fill_reports(
5348            vec![client],
5349            vec![PositionFillReportQuery {
5350                key,
5351                client_id,
5352                command,
5353            }],
5354        )
5355        .await
5356        .finish();
5357
5358        assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5359        assert_eq!(result, IndexMap::from([(key, vec![report_a, report_b])]));
5360    }
5361
5362    #[rstest]
5363    #[case("OTHER-001", "ETHUSDT-PERP.BINANCE", 1_500, "1.0")]
5364    #[case("POSITION-FILLS-001", "BTCUSDT-PERP.BINANCE", 1_500, "1.0")]
5365    #[case("POSITION-FILLS-001", "ETHUSDT-PERP.BINANCE", 1_500, "0.0")]
5366    #[tokio::test]
5367    async fn test_request_position_fill_reports_rejects_out_of_scope_response(
5368        #[case] report_account: &str,
5369        #[case] report_instrument: &str,
5370        #[case] ts_event: u64,
5371        #[case] quantity: &str,
5372    ) {
5373        let client_id = ClientId::from("POSITION-FILLS");
5374        let account_id = AccountId::from("POSITION-FILLS-001");
5375        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5376        let instrument_id = instrument.id();
5377
5378        let report = FillReport::new(
5379            AccountId::from(report_account),
5380            InstrumentId::from(report_instrument),
5381            VenueOrderId::from("V-POSITION-FILLS"),
5382            TradeId::from("T-POSITION-FILLS"),
5383            OrderSide::Buy,
5384            Quantity::from(quantity),
5385            Price::from("100.0"),
5386            Money::zero(instrument.quote_currency()),
5387            LiquiditySide::Taker,
5388            Some(ClientOrderId::from("O-POSITION-FILLS")),
5389            None,
5390            UnixNanos::from(ts_event),
5391            UnixNanos::from(2_000),
5392            None,
5393        );
5394
5395        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5396            client_id,
5397            account_id,
5398            venue: instrument_id.venue,
5399            outcome: FillReportClientOutcome::Reports(vec![report]),
5400            commands: Rc::new(RefCell::new(Vec::new())),
5401        }));
5402
5403        let command = GenerateFillReports::new(
5404            UUID4::new(),
5405            UnixNanos::from(2_000),
5406            Some(instrument_id),
5407            None,
5408            Some(UnixNanos::from(1_000)),
5409            Some(UnixNanos::from(2_000)),
5410            None,
5411            None,
5412        );
5413        let key = (instrument_id, account_id);
5414
5415        let result = request_position_fill_reports(
5416            vec![client],
5417            vec![PositionFillReportQuery {
5418                key,
5419                client_id,
5420                command,
5421            }],
5422        )
5423        .await
5424        .finish();
5425
5426        assert!(result.is_empty());
5427    }
5428
5429    #[rstest]
5430    #[case(999)]
5431    #[case(2_001)]
5432    #[tokio::test]
5433    async fn test_request_position_fill_reports_filters_time_window_superset(
5434        #[case] ts_event: u64,
5435    ) {
5436        let client_id = ClientId::from("POSITION-FILLS");
5437        let account_id = AccountId::from("POSITION-FILLS-001");
5438        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5439        let instrument_id = instrument.id();
5440
5441        let report = FillReport::new(
5442            account_id,
5443            instrument_id,
5444            VenueOrderId::from("V-POSITION-FILLS"),
5445            TradeId::from("T-POSITION-FILLS"),
5446            OrderSide::Buy,
5447            Quantity::from("1.0"),
5448            Price::from("100.0"),
5449            Money::zero(instrument.quote_currency()),
5450            LiquiditySide::Taker,
5451            Some(ClientOrderId::from("O-POSITION-FILLS")),
5452            None,
5453            UnixNanos::from(ts_event),
5454            UnixNanos::from(2_000),
5455            None,
5456        );
5457
5458        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5459            client_id,
5460            account_id,
5461            venue: instrument_id.venue,
5462            outcome: FillReportClientOutcome::Reports(vec![report]),
5463            commands: Rc::new(RefCell::new(Vec::new())),
5464        }));
5465
5466        let command = GenerateFillReports::new(
5467            UUID4::new(),
5468            UnixNanos::from(2_000),
5469            Some(instrument_id),
5470            None,
5471            Some(UnixNanos::from(1_000)),
5472            Some(UnixNanos::from(2_000)),
5473            None,
5474            None,
5475        );
5476        let key = (instrument_id, account_id);
5477
5478        let result = request_position_fill_reports(
5479            vec![client],
5480            vec![PositionFillReportQuery {
5481                key,
5482                client_id,
5483                command,
5484            }],
5485        )
5486        .await
5487        .finish();
5488
5489        assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5490        assert_eq!(result, IndexMap::from([(key, Vec::new())]));
5491    }
5492
5493    #[rstest]
5494    #[case::query_error(false)]
5495    #[case::missing_client(true)]
5496    #[tokio::test]
5497    async fn test_position_fill_collection_discards_partial_failure(
5498        #[case] missing_client: bool,
5499        #[values(false, true)] failure_first: bool,
5500        position_fill_query: (FillReport, PositionFillReportQuery),
5501    ) {
5502        let (report, query) = position_fill_query;
5503        let failed_client_id = ClientId::from("FAILED-FILLS");
5504        let healthy_key = (report.instrument_id, AccountId::from("HEALTHY-001"));
5505        let mut healthy_report = report.clone();
5506        healthy_report.account_id = healthy_key.1;
5507        let healthy_client_id = ClientId::from("HEALTHY-FILLS");
5508        let empty_key = (report.instrument_id, AccountId::from("EMPTY-001"));
5509        let empty_client_id = ClientId::from("EMPTY-FILLS");
5510        let mut clients = vec![
5511            LiveExecutionClient::new(Box::new(FillReportClient {
5512                client_id: query.client_id,
5513                account_id: query.key.1,
5514                venue: query.key.0.venue,
5515                outcome: FillReportClientOutcome::Reports(vec![report]),
5516                commands: Rc::new(RefCell::new(Vec::new())),
5517            })),
5518            LiveExecutionClient::new(Box::new(FillReportClient {
5519                client_id: healthy_client_id,
5520                account_id: healthy_key.1,
5521                venue: healthy_key.0.venue,
5522                outcome: FillReportClientOutcome::Reports(vec![healthy_report.clone()]),
5523                commands: Rc::new(RefCell::new(Vec::new())),
5524            })),
5525            LiveExecutionClient::new(Box::new(FillReportClient {
5526                client_id: empty_client_id,
5527                account_id: empty_key.1,
5528                venue: empty_key.0.venue,
5529                outcome: FillReportClientOutcome::Reports(Vec::new()),
5530                commands: Rc::new(RefCell::new(Vec::new())),
5531            })),
5532        ];
5533
5534        if !missing_client {
5535            clients.push(LiveExecutionClient::new(Box::new(FillReportClient {
5536                client_id: failed_client_id,
5537                account_id: query.key.1,
5538                venue: query.key.0.venue,
5539                outcome: FillReportClientOutcome::Failure,
5540                commands: Rc::new(RefCell::new(Vec::new())),
5541            })));
5542        }
5543
5544        let failed_query = PositionFillReportQuery {
5545            client_id: failed_client_id,
5546            key: query.key,
5547            command: query.command.clone(),
5548        };
5549
5550        let healthy_query = PositionFillReportQuery {
5551            key: healthy_key,
5552            client_id: healthy_client_id,
5553            command: query.command.clone(),
5554        };
5555
5556        let empty_query = PositionFillReportQuery {
5557            key: empty_key,
5558            client_id: empty_client_id,
5559            command: query.command.clone(),
5560        };
5561
5562        let mut queries = vec![query, failed_query];
5563
5564        if failure_first {
5565            queries.reverse();
5566        }
5567
5568        queries.insert(0, empty_query);
5569        queries.push(healthy_query);
5570
5571        let result = request_position_fill_reports(clients, queries)
5572            .await
5573            .finish();
5574
5575        assert_eq!(
5576            result.keys().copied().collect::<Vec<_>>(),
5577            vec![empty_key, healthy_key]
5578        );
5579        assert_eq!(
5580            result,
5581            IndexMap::from([(empty_key, Vec::new()), (healthy_key, vec![healthy_report])])
5582        );
5583    }
5584
5585    #[rstest]
5586    #[case::same_economics("metadata", true)]
5587    #[case::venue_order("venue_order", false)]
5588    #[case::side("side", false)]
5589    #[case::quantity("quantity", false)]
5590    #[case::price("price", false)]
5591    #[case::commission("commission", false)]
5592    #[case::liquidity("liquidity", false)]
5593    #[case::average("average", false)]
5594    #[case::event_time("event_time", false)]
5595    #[case::client_order("client_order", false)]
5596    #[case::position("position", false)]
5597    #[tokio::test]
5598    async fn test_position_fill_collection_validates_duplicate_economics(
5599        #[case] changed: &str,
5600        #[case] accepted: bool,
5601        position_fill_query: (FillReport, PositionFillReportQuery),
5602    ) {
5603        let (report, query) = position_fill_query;
5604        let mut duplicate = report.clone();
5605
5606        match changed {
5607            "metadata" => {
5608                duplicate.report_id = UUID4::new();
5609                duplicate.ts_init = UnixNanos::from(3_000);
5610            }
5611            "venue_order" => duplicate.venue_order_id = VenueOrderId::from("OTHER-ORDER"),
5612            "side" => duplicate.order_side = OrderSide::Sell,
5613            "quantity" => duplicate.last_qty = Quantity::from("2.0"),
5614            "price" => duplicate.last_px = Price::from("101.0"),
5615            "commission" => duplicate.commission = Money::from("0.25 USDT"),
5616            "liquidity" => duplicate.liquidity_side = LiquiditySide::Maker,
5617            "average" => duplicate.avg_px = Some(dec!(100.5)),
5618            "event_time" => duplicate.ts_event = UnixNanos::from(1_501),
5619            "client_order" => duplicate.client_order_id = None,
5620            "position" => duplicate.venue_position_id = Some(PositionId::from("OTHER-POSITION")),
5621            _ => unreachable!(),
5622        }
5623
5624        let key = query.key;
5625
5626        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5627            client_id: query.client_id,
5628            account_id: key.1,
5629            venue: key.0.venue,
5630            outcome: FillReportClientOutcome::Reports(vec![report.clone(), duplicate]),
5631            commands: Rc::new(RefCell::new(Vec::new())),
5632        }));
5633
5634        let result = request_position_fill_reports(vec![client], vec![query])
5635            .await
5636            .finish();
5637
5638        if accepted {
5639            assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5640            assert_eq!(result, IndexMap::from([(key, vec![report])]));
5641        } else {
5642            assert!(result.is_empty());
5643        }
5644    }
5645
5646    #[fixture]
5647    fn position_fill_query() -> (FillReport, PositionFillReportQuery) {
5648        let account_id = AccountId::from("POSITION-FILLS-001");
5649        let instrument_id = crypto_perpetual_ethusdt().id();
5650
5651        let report = FillReport::new(
5652            account_id,
5653            instrument_id,
5654            VenueOrderId::from("V-POSITION-FILLS"),
5655            TradeId::from("T-POSITION-FILLS"),
5656            OrderSide::Buy,
5657            Quantity::from("1.0"),
5658            Price::from("100.0"),
5659            Money::from("0.10 USDT"),
5660            LiquiditySide::Taker,
5661            Some(ClientOrderId::from("O-POSITION-FILLS")),
5662            None,
5663            UnixNanos::from(1_500),
5664            UnixNanos::from(2_000),
5665            None,
5666        );
5667
5668        let query = PositionFillReportQuery {
5669            key: (instrument_id, account_id),
5670            client_id: ClientId::from("POSITION-FILLS"),
5671            command: GenerateFillReports::new(
5672                UUID4::new(),
5673                UnixNanos::from(2_000),
5674                Some(instrument_id),
5675                None,
5676                Some(UnixNanos::from(1_000)),
5677                Some(UnixNanos::from(2_000)),
5678                None,
5679                None,
5680            ),
5681        };
5682
5683        (report, query)
5684    }
5685
5686    #[rstest]
5687    #[case::failed_query(false)]
5688    #[case::local_activity(true)]
5689    fn test_position_fallback_preserves_deferred_venue_only_retries(#[case] local_activity: bool) {
5690        let (mut node, report, _) =
5691            position_fill_test_fixture("PositionRetryRetentionNode", Quantity::from("1.0"));
5692        let active_key = (report.instrument_id, report.account_id);
5693        let deferred_key = (report.instrument_id, AccountId::from("SECOND-001"));
5694        let deferred_reports = vec![
5695            PositionStatusReport::new(
5696                deferred_key.1,
5697                deferred_key.0,
5698                PositionSide::Long,
5699                Quantity::from("2.0"),
5700                UnixNanos::from(1_000),
5701                UnixNanos::from(1_000),
5702                None,
5703                None,
5704                Some(dec!(100.0)),
5705            ),
5706            PositionStatusReport::new(
5707                deferred_key.1,
5708                deferred_key.0,
5709                PositionSide::Short,
5710                Quantity::from("1.0"),
5711                UnixNanos::from(1_000),
5712                UnixNanos::from(1_000),
5713                None,
5714                None,
5715                Some(dec!(100.0)),
5716            ),
5717        ];
5718        let mut check = node
5719            .exec_manager
5720            .prepare_position_report_check(UUID4::new(), &[]);
5721        check.client_coverage.clear();
5722        let events = node.exec_manager.reconcile_position_reports(
5723            &check,
5724            deferred_reports.clone(),
5725            &IndexSet::new(),
5726            &IndexSet::new(),
5727        );
5728        assert!(events.is_empty());
5729        assert_eq!(
5730            node.exec_manager.position_recon_retry_count(&deferred_key),
5731            1
5732        );
5733
5734        let mut position_result = position_report_result(&node, report);
5735        position_result.reports.extend(deferred_reports);
5736        position_result.check.client_coverage.insert(
5737            deferred_key,
5738            ReportClientCoverage::Resolved(IndexSet::from([ClientId::from("SECOND")])),
5739        );
5740        position_result
5741            .check
5742            .activity_revisions
5743            .insert(deferred_key, 0);
5744        position_result
5745            .queried_clients
5746            .insert(ClientId::from("SECOND"));
5747        let mut reports = IndexMap::from([(active_key, Vec::new())]);
5748        if local_activity {
5749            reports.insert(deferred_key, Vec::new());
5750            node.exec_manager
5751                .record_position_activity(deferred_key.0, deferred_key.1);
5752        }
5753
5754        node.handle_position_fill_report_result(PositionFillReportResult {
5755            position_result,
5756            reports,
5757        });
5758
5759        assert_eq!(
5760            node.exec_manager.position_recon_retry_count(&deferred_key),
5761            1
5762        );
5763        {
5764            let cache = node.kernel.cache.borrow();
5765            let positions =
5766                cache.positions_open(None, Some(&active_key.0), None, Some(&active_key.1), None);
5767            assert_eq!(
5768                positions
5769                    .iter()
5770                    .map(|position| position.quantity)
5771                    .collect::<Vec<_>>(),
5772                vec![Quantity::from("1.0"), Quantity::from("1.0")],
5773            );
5774            assert_eq!(
5775                cache
5776                    .positions_open(
5777                        None,
5778                        Some(&deferred_key.0),
5779                        None,
5780                        Some(&deferred_key.1),
5781                        None,
5782                    )
5783                    .len(),
5784                0
5785            );
5786        }
5787
5788        let mut fresh_check = node
5789            .exec_manager
5790            .prepare_position_report_check(UUID4::new(), &[]);
5791        node.exec_manager.plan_position_fill_reports(
5792            &mut fresh_check,
5793            &[],
5794            &IndexSet::new(),
5795            &IndexSet::new(),
5796            &[],
5797        );
5798
5799        assert_eq!(
5800            node.exec_manager.position_recon_retry_count(&deferred_key),
5801            0
5802        );
5803    }
5804
5805    #[rstest]
5806    fn test_position_fill_report_result_applies_authoritative_fill_without_synthetic_order() {
5807        let (mut node, venue_report, fill_report) =
5808            position_fill_test_fixture("AuthoritativePositionFillNode", Quantity::from("1.0"));
5809        let key = (venue_report.instrument_id, venue_report.account_id);
5810        let revision = node.exec_manager.position_activity_revision(&key);
5811        let position_result = position_report_result(&node, venue_report);
5812
5813        node.handle_position_fill_report_result(PositionFillReportResult {
5814            position_result,
5815            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5816        });
5817
5818        let cache = node.kernel.cache.borrow();
5819        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5820        assert_eq!(
5821            positions
5822                .iter()
5823                .map(|position| position.quantity)
5824                .sum::<Quantity>(),
5825            Quantity::from("2.0")
5826        );
5827        assert_eq!(
5828            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5829            1
5830        );
5831        drop(positions);
5832        drop(cache);
5833        assert!(
5834            node.exec_manager
5835                .position_contains_fill_report(&fill_report)
5836        );
5837        assert_eq!(
5838            node.exec_manager.position_activity_revision(&key),
5839            revision + 1
5840        );
5841    }
5842
5843    #[rstest]
5844    fn test_position_fill_prefix_rechecks_history_after_dispatch_callback() {
5845        let (mut node, venue_report, mut first) =
5846            position_fill_test_fixture("PositionPrefixCallbackNode", Quantity::from("0.5"));
5847        first.commission = Money::from("0.13 USDT");
5848        let key = (first.instrument_id, first.account_id);
5849        let client_order_id = first.client_order_id.unwrap();
5850        let position_result = position_report_result(&node, venue_report.clone());
5851        node.handle_position_fill_report_result(PositionFillReportResult {
5852            position_result,
5853            reports: IndexMap::from([(key, vec![first.clone()])]),
5854        });
5855
5856        let mut second = first.clone();
5857        second.trade_id = TradeId::from("T-PREFIX-SECOND");
5858        second.last_qty = Quantity::from("0.75");
5859        second.commission = Money::from("0.29 USDT");
5860        let mut third = first.clone();
5861        third.trade_id = TradeId::from("T-PREFIX-THIRD");
5862        third.last_qty = Quantity::from("0.25");
5863        third.commission = Money::from("0.07 USDT");
5864        let revision = node.exec_manager.position_activity_revision(&key);
5865        let received = Rc::new(RefCell::new(Vec::new()));
5866        let cache = node.kernel.cache.clone();
5867        let callback_received = received.clone();
5868        let second_trade_id = second.trade_id;
5869
5870        let handler = TypedHandler::from(move |event: &OrderEventAny| {
5871            let OrderEventAny::Filled(fill) = event else {
5872                return;
5873            };
5874
5875            callback_received.borrow_mut().push(fill.trade_id);
5876            if fill.trade_id == second_trade_id {
5877                let mut cache = cache.borrow_mut();
5878                let position_id = *cache.position_id(&fill.client_order_id).unwrap();
5879                let mut position = cache.position_mut(&position_id).unwrap();
5880                position.replay_events.retain(|event| !matches!(event, PositionReplayEvent::Filled(fill) if fill.trade_id == second_trade_id));
5881            }
5882        });
5883
5884        let strategy_id = node
5885            .kernel
5886            .cache
5887            .borrow()
5888            .order(&client_order_id)
5889            .unwrap()
5890            .strategy_id();
5891        let topic = nautilus_common::msgbus::switchboard::get_event_order_topic(strategy_id);
5892        msgbus::subscribe_order_events(topic.into(), handler.clone(), None);
5893        let position_result = position_report_result(&node, venue_report);
5894        node.handle_position_fill_report_result(PositionFillReportResult {
5895            position_result,
5896            reports: IndexMap::from([(
5897                key,
5898                vec![first.clone(), first.clone(), second.clone(), third.clone()],
5899            )]),
5900        });
5901
5902        msgbus::unsubscribe_order_events(topic.into(), &handler);
5903        let cache = node.kernel.cache.borrow();
5904        let order = cache.order(&client_order_id).unwrap();
5905        let position_id = cache.position_id(&client_order_id).unwrap();
5906        let position = cache.position_ref(position_id).unwrap();
5907
5908        assert_eq!(*received.borrow(), vec![second.trade_id]);
5909        assert_eq!(order.filled_qty(), Quantity::from("2.25"));
5910        assert_eq!(order.events().len(), 6);
5911        assert_eq!(
5912            order.trade_ids(),
5913            vec![
5914                &TradeId::from("T-POSITION-INITIAL"),
5915                &first.trade_id,
5916                &second.trade_id
5917            ]
5918        );
5919        assert_eq!(
5920            order.commissions().values().copied().collect::<Vec<_>>(),
5921            vec![Money::from("0.42 USDT")]
5922        );
5923        assert_eq!(position.quantity, Quantity::from("2.25"));
5924        assert_eq!(position.commissions(), vec![Money::from("0.42 USDT")]);
5925        assert_eq!(position.replay_events.len(), 2);
5926        assert_eq!(
5927            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5928            1
5929        );
5930        assert_eq!(
5931            node.exec_manager.position_activity_revision(&key),
5932            revision + 1
5933        );
5934        assert!(node.exec_manager.position_contains_fill_report(&first));
5935        assert!(!node.exec_manager.position_contains_fill_report(&second));
5936        assert!(!node.exec_manager.position_contains_fill_report(&third));
5937        assert!(
5938            !node
5939                .exec_manager
5940                .is_fill_recently_processed(key.1, key.0, second.trade_id)
5941        );
5942        assert!(
5943            !node
5944                .exec_manager
5945                .is_fill_recently_processed(key.1, key.0, third.trade_id)
5946        );
5947    }
5948
5949    #[rstest]
5950    fn test_position_fill_report_match_ignores_adapter_timestamp_source() {
5951        let (mut node, venue_report, fill_report) =
5952            position_fill_test_fixture("PositionFillTimestampNode", Quantity::from("1.0"));
5953        let key = (venue_report.instrument_id, venue_report.account_id);
5954        let position_result = position_report_result(&node, venue_report);
5955
5956        node.handle_position_fill_report_result(PositionFillReportResult {
5957            position_result,
5958            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5959        });
5960
5961        let mut rest_report = fill_report;
5962        rest_report.ts_event = UnixNanos::from(2_000);
5963
5964        assert!(
5965            node.exec_manager
5966                .position_contains_fill_report(&rest_report)
5967        );
5968    }
5969
5970    #[rstest]
5971    fn test_position_fill_report_result_falls_back_when_order_contains_inferred_fill() {
5972        let (mut node, venue_report, fill_report) =
5973            position_fill_test_fixture("InferredPositionFillNode", Quantity::from("1.0"));
5974        let key = (venue_report.instrument_id, venue_report.account_id);
5975        let client_order_id = fill_report.client_order_id.unwrap();
5976        apply_inferred_position_fill(&mut node, &fill_report);
5977
5978        let venue_report = PositionStatusReport::new(
5979            key.1,
5980            key.0,
5981            PositionSide::Long,
5982            Quantity::from("3.0"),
5983            venue_report.ts_last,
5984            venue_report.ts_init,
5985            None,
5986            None,
5987            venue_report.avg_px_open,
5988        );
5989        let mut later_fill_report = fill_report.clone();
5990        later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
5991        later_fill_report.ts_event = UnixNanos::from(1_001);
5992        let position_result = position_report_result(&node, venue_report);
5993
5994        node.handle_position_fill_report_result(PositionFillReportResult {
5995            position_result,
5996            reports: IndexMap::from([(key, vec![fill_report.clone(), later_fill_report.clone()])]),
5997        });
5998
5999        let cache = node.kernel.cache.borrow();
6000        let order = cache.order(&client_order_id).unwrap();
6001        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6002        assert_eq!(order.filled_qty(), Quantity::from("2.0"));
6003        assert!(!order.trade_ids().contains(&&fill_report.trade_id));
6004        assert!(!order.trade_ids().contains(&&later_fill_report.trade_id));
6005        assert_eq!(
6006            positions
6007                .iter()
6008                .map(|position| position.quantity)
6009                .sum::<Quantity>(),
6010            Quantity::from("3.0")
6011        );
6012        assert_eq!(
6013            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6014            2
6015        );
6016    }
6017
6018    #[rstest]
6019    fn test_position_fill_report_result_falls_back_when_reconciled_position_includes_fill() {
6020        let (mut node, _, _) =
6021            position_fill_test_fixture("ReconciledPositionFillNode", Quantity::from("1.0"));
6022        let account_id = AccountId::from("TEST-001");
6023        let instrument = InstrumentAny::CurrencyPair(currency_pair_btcusdt());
6024        let key = (instrument.id(), account_id);
6025        let commission = Money::zero(instrument.quote_currency());
6026        node.kernel
6027            .cache
6028            .borrow_mut()
6029            .add_instrument(instrument.clone())
6030            .unwrap();
6031
6032        let order = OrderTestBuilder::new(OrderType::Market)
6033            .strategy_id(StrategyId::external())
6034            .instrument_id(instrument.id())
6035            .client_order_id(ClientOrderId::from("O-RECONCILED"))
6036            .quantity(Quantity::from("1.000000"))
6037            .tags(vec![Ustr::from(RECONCILIATION_ORDER_TAG)])
6038            .build();
6039        let submitted = TestOrderEventStubs::submitted(&order, account_id);
6040        let accepted =
6041            TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-RECONCILED"));
6042        node.kernel
6043            .cache
6044            .borrow_mut()
6045            .add_order(
6046                order.clone(),
6047                None,
6048                Some(ClientId::from("POSITION-FILLS")),
6049                false,
6050            )
6051            .unwrap();
6052        node.process_reconciliation_events(&[submitted, accepted]);
6053        let order = node
6054            .kernel
6055            .cache
6056            .borrow()
6057            .order_owned(&order.client_order_id())
6058            .unwrap();
6059        let opening_fill = TestOrderEventStubs::filled(
6060            &order,
6061            &instrument,
6062            Some(TradeId::from("T-RECONCILED")),
6063            None,
6064            Some(Price::from("100.00")),
6065            Some(Quantity::from("1.000000")),
6066            Some(LiquiditySide::Taker),
6067            Some(commission),
6068            Some(UnixNanos::from(2_000)),
6069            Some(account_id),
6070        );
6071        node.process_reconciliation_events(&[opening_fill]);
6072
6073        // A real fill between the position report query and reconciliation stays off the position
6074        let held_fill_report = FillReport::new(
6075            account_id,
6076            instrument.id(),
6077            VenueOrderId::from("V-HELD"),
6078            TradeId::from("T-HELD"),
6079            OrderSide::Buy,
6080            Quantity::from("1.000000"),
6081            Price::from("100.00"),
6082            commission,
6083            LiquiditySide::Taker,
6084            None,
6085            None,
6086            UnixNanos::from(1_000),
6087            UnixNanos::from(1_000),
6088            None,
6089        );
6090        node.kernel
6091            .exec_engine
6092            .borrow_mut()
6093            .reconcile_fill_report(&held_fill_report);
6094
6095        let venue_report = PositionStatusReport::new(
6096            account_id,
6097            instrument.id(),
6098            PositionSide::Long,
6099            Quantity::from("2.000000"),
6100            UnixNanos::from(3_000),
6101            UnixNanos::from(3_000),
6102            None,
6103            None,
6104            Some(dec!(100.00)),
6105        );
6106        let position_result = position_report_result(&node, venue_report);
6107
6108        node.handle_position_fill_report_result(PositionFillReportResult {
6109            position_result,
6110            reports: IndexMap::from([(key, vec![held_fill_report.clone()])]),
6111        });
6112
6113        let cache = node.kernel.cache.borrow();
6114        let held_order = cache.order(&ClientOrderId::from("V-HELD")).unwrap();
6115        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6116        assert_eq!(held_order.filled_qty(), Quantity::from("1.000000"));
6117        assert_eq!(
6118            positions
6119                .iter()
6120                .map(|position| position.quantity)
6121                .sum::<Quantity>(),
6122            Quantity::from("2.000000")
6123        );
6124        assert!(
6125            positions
6126                .iter()
6127                .all(|position| !position.trade_ids.contains(&held_fill_report.trade_id))
6128        );
6129    }
6130
6131    #[rstest]
6132    fn test_position_fill_report_validates_hedge_identity_before_inferred_fallback() {
6133        let (mut node, _, mut fill_report) =
6134            position_fill_test_fixture("InferredHedgeIdentityNode", Quantity::from("1.0"));
6135        apply_inferred_position_fill(&mut node, &fill_report);
6136        let conflicting_position_id = PositionId::from("P-POSITION-CONFLICT");
6137        fill_report.venue_position_id = Some(conflicting_position_id);
6138
6139        let error = node
6140            .exec_manager
6141            .prepare_position_fill_report(&mut fill_report, &[])
6142            .unwrap_err();
6143
6144        assert!(
6145            error.to_string().contains(&format!(
6146                "position ID {conflicting_position_id} conflicts with cached order position"
6147            )),
6148            "{error:#}"
6149        );
6150    }
6151
6152    #[rstest]
6153    #[case::not_applied_exactly("100.0", "O-POSITION-FILLS")]
6154    #[case::preparation_error("1.0", "O-POSITION-CONFLICT")]
6155    fn test_position_fill_report_result_falls_back_after_unapplied_fill_expires(
6156        #[case] authoritative_qty: &str,
6157        #[case] client_order_id: &str,
6158    ) {
6159        let (mut node, venue_report, mut fill_report) = position_fill_test_fixture(
6160            "UnappliedPositionFillNode",
6161            Quantity::from(authoritative_qty),
6162        );
6163        fill_report.client_order_id = Some(ClientOrderId::from(client_order_id));
6164        let reports = [fill_report.clone()];
6165
6166        let quantities = (0..2)
6167            .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6168            .collect::<Vec<_>>();
6169
6170        assert_eq!(
6171            quantities,
6172            vec![Quantity::from("1.0"), Quantity::from("2.0")]
6173        );
6174        assert!(
6175            !node
6176                .exec_manager
6177                .position_contains_fill_report(&fill_report)
6178        );
6179    }
6180
6181    #[rstest]
6182    fn test_position_fill_report_result_expires_one_unapplied_fill_per_check() {
6183        let (mut node, venue_report, fill_report) =
6184            position_fill_test_fixture("UnappliedPositionFillsNode", Quantity::from("100.0"));
6185        let mut later_fill_report = fill_report.clone();
6186        later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
6187        later_fill_report.ts_event = UnixNanos::from(1_001);
6188        let reports = [fill_report.clone(), later_fill_report.clone()];
6189
6190        let quantities = (0..3)
6191            .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6192            .collect::<Vec<_>>();
6193
6194        assert_eq!(
6195            quantities,
6196            vec![
6197                Quantity::from("1.0"),
6198                Quantity::from("1.0"),
6199                Quantity::from("2.0"),
6200            ]
6201        );
6202        assert!(
6203            !node
6204                .exec_manager
6205                .position_contains_fill_report(&fill_report)
6206        );
6207        assert!(
6208            !node
6209                .exec_manager
6210                .position_contains_fill_report(&later_fill_report)
6211        );
6212    }
6213
6214    #[rstest]
6215    #[case::within_grace(60_000, true)]
6216    #[case::after_grace(0, false)]
6217    fn test_position_fill_report_result_retries_refused_fill_only_within_grace(
6218        #[case] position_check_threshold_ms: u32,
6219        #[case] expected_applied: bool,
6220    ) {
6221        let (mut node, venue_report, fill_report) = position_fill_test_fixture_with_threshold(
6222            "TransientPositionFillNode",
6223            Quantity::from("1.0"),
6224            position_check_threshold_ms,
6225        );
6226        let mut conflicting_report = fill_report.clone();
6227        conflicting_report.client_order_id = Some(ClientOrderId::from("O-POSITION-CONFLICT"));
6228
6229        let quantities = [conflicting_report, fill_report.clone()]
6230            .into_iter()
6231            .map(|report| position_quantity_after_check(&mut node, &venue_report, &[report]))
6232            .collect::<Vec<_>>();
6233
6234        assert_eq!(
6235            quantities,
6236            vec![Quantity::from("1.0"), Quantity::from("2.0")]
6237        );
6238        assert_eq!(
6239            node.exec_manager
6240                .position_contains_fill_report(&fill_report),
6241            expected_applied
6242        );
6243    }
6244
6245    #[rstest]
6246    fn test_position_fill_report_result_synthesizes_only_residual_after_fresh_report() {
6247        let (mut node, venue_report, fill_report) =
6248            position_fill_test_fixture("ResidualPositionFillNode", Quantity::from("0.5"));
6249        let key = (venue_report.instrument_id, venue_report.account_id);
6250        let first_result = position_report_result(&node, venue_report.clone());
6251
6252        node.handle_position_fill_report_result(PositionFillReportResult {
6253            position_result: first_result,
6254            reports: IndexMap::from([(key, vec![fill_report])]),
6255        });
6256
6257        let fresh_result = position_report_result(&node, venue_report);
6258        node.handle_position_fill_report_result(PositionFillReportResult {
6259            position_result: fresh_result,
6260            reports: IndexMap::from([(key, Vec::new())]),
6261        });
6262
6263        let cache = node.kernel.cache.borrow();
6264        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6265        assert_eq!(
6266            positions
6267                .iter()
6268                .map(|position| position.quantity)
6269                .sum::<Quantity>(),
6270            Quantity::from("2.0")
6271        );
6272        assert_eq!(
6273            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6274            2
6275        );
6276    }
6277
6278    #[rstest]
6279    #[case::below_limit(63)]
6280    #[case::at_limit(64)]
6281    #[case::above_limit(65)]
6282    fn test_position_fill_dispatch_limit_resumes_without_duplicate_economics(#[case] count: usize) {
6283        let (mut node, venue_report, fill_report) =
6284            position_fill_test_fixture("PositionFillLimitNode", Quantity::from("0.1"));
6285        let expected_quantity = dec!(1) + Decimal::new(i64::try_from(count).unwrap(), 1);
6286
6287        let venue_report = PositionStatusReport::new(
6288            venue_report.account_id,
6289            venue_report.instrument_id,
6290            PositionSide::Long,
6291            Quantity::from_decimal_dp(expected_quantity, 1).unwrap(),
6292            venue_report.ts_last,
6293            venue_report.ts_init,
6294            None,
6295            None,
6296            venue_report.avg_px_open,
6297        );
6298        let key = (venue_report.instrument_id, venue_report.account_id);
6299        let client_order_id = fill_report.client_order_id.unwrap();
6300
6301        let reports = (0..count)
6302            .map(|index| {
6303                let mut report = fill_report.clone();
6304                report.trade_id = TradeId::from(format!("T-LIMIT-{index:03}"));
6305                report.ts_event = UnixNanos::from(1_000 + index as u64);
6306                report
6307            })
6308            .collect::<Vec<_>>();
6309
6310        let position_result = position_report_result(&node, venue_report.clone());
6311
6312        node.handle_position_fill_report_result(PositionFillReportResult {
6313            position_result,
6314            reports: IndexMap::from([(key, reports.clone())]),
6315        });
6316
6317        {
6318            let cache = node.kernel.cache.borrow();
6319            let order = cache.order(&client_order_id).unwrap();
6320            let applied = count.min(64);
6321            assert_eq!(
6322                order.filled_qty().as_decimal(),
6323                dec!(1) + Decimal::new(i64::try_from(applied).unwrap(), 1)
6324            );
6325            assert_eq!(order.trade_ids().len(), applied + 1);
6326
6327            for (index, report) in reports.iter().enumerate() {
6328                assert_eq!(
6329                    order.trade_ids().contains(&&report.trade_id),
6330                    index < applied
6331                );
6332            }
6333
6334            assert_eq!(
6335                cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6336                1
6337            );
6338        }
6339
6340        // A fresh cycle replays the full response; only the deferred fill may change exposure
6341        let position_result = position_report_result(&node, venue_report);
6342        node.handle_position_fill_report_result(PositionFillReportResult {
6343            position_result,
6344            reports: IndexMap::from([(key, reports)]),
6345        });
6346
6347        let cache = node.kernel.cache.borrow();
6348        let order = cache.order(&client_order_id).unwrap();
6349        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6350        assert_eq!(order.filled_qty().as_decimal(), expected_quantity);
6351        assert_eq!(order.trade_ids().len(), count + 1);
6352        assert_eq!(positions.len(), 1);
6353        assert_eq!(positions[0].quantity.as_decimal(), expected_quantity);
6354        assert_eq!(positions[0].side, PositionSide::Long);
6355        assert_eq!(
6356            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6357            1
6358        );
6359    }
6360
6361    #[rstest]
6362    #[case::client_order(0)]
6363    #[case::venue_order(1)]
6364    #[case::side(2)]
6365    #[case::position(3)]
6366    fn test_position_fill_conflict_blocks_remaining_fills_and_fallback(#[case] conflict: u8) {
6367        let (mut node, venue_report, valid_report) =
6368            position_fill_test_fixture("ConflictingPositionFillNode", Quantity::from("1.0"));
6369        let key = (venue_report.instrument_id, venue_report.account_id);
6370        let client_order_id = valid_report.client_order_id.unwrap();
6371        let revision = node.exec_manager.position_activity_revision(&key);
6372        let position_result = position_report_result(&node, venue_report);
6373        let mut conflicting_report = valid_report.clone();
6374        conflicting_report.trade_id = TradeId::from("T-CONFLICTING-FILL");
6375
6376        match conflict {
6377            0 => conflicting_report.client_order_id = Some(ClientOrderId::from("O-CONFLICT")),
6378            1 => conflicting_report.venue_order_id = VenueOrderId::from("V-CONFLICT"),
6379            2 => conflicting_report.order_side = OrderSide::Sell,
6380            3 => conflicting_report.venue_position_id = Some(PositionId::from("P-CONFLICT")),
6381            _ => unreachable!(),
6382        }
6383
6384        node.handle_position_fill_report_result(PositionFillReportResult {
6385            position_result,
6386            reports: IndexMap::from([(key, vec![conflicting_report, valid_report])]),
6387        });
6388
6389        let cache = node.kernel.cache.borrow();
6390        let order = cache.order(&client_order_id).unwrap();
6391        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6392        assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6393        assert_eq!(
6394            order.trade_ids(),
6395            vec![&TradeId::from("T-POSITION-INITIAL")]
6396        );
6397        assert_eq!(positions.len(), 1);
6398        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6399        assert_eq!(
6400            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6401            1
6402        );
6403        assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
6404    }
6405
6406    #[rstest]
6407    fn test_position_fill_report_failure_does_not_trigger_synthetic_fallback() {
6408        let (mut node, venue_report, _) =
6409            position_fill_test_fixture("FailedPositionFillNode", Quantity::from("1.0"));
6410        let key = (venue_report.instrument_id, venue_report.account_id);
6411        let position_result = position_report_result(&node, venue_report);
6412
6413        node.handle_position_fill_report_result(PositionFillReportResult {
6414            position_result,
6415            reports: IndexMap::new(),
6416        });
6417
6418        let cache = node.kernel.cache.borrow();
6419        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6420        assert_eq!(positions.len(), 1);
6421        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6422        assert_eq!(
6423            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6424            1
6425        );
6426    }
6427
6428    #[rstest]
6429    fn test_position_fill_report_result_falls_back_for_unattributable_hedge_fill() {
6430        let (mut node, mut venue_report, mut fill_report) =
6431            position_fill_test_fixture("UnattributedHedgeFillNode", Quantity::from("1.0"));
6432        node.kernel
6433            .exec_engine
6434            .borrow_mut()
6435            .register_oms_type(StrategyId::from("EXTERNAL"), OmsType::Hedging);
6436        let key = (venue_report.instrument_id, venue_report.account_id);
6437
6438        let position_id = {
6439            let cache = node.kernel.cache.borrow();
6440            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6441            assert_eq!(positions.len(), 1);
6442            positions[0].id
6443        };
6444
6445        venue_report.venue_position_id = Some(position_id);
6446        fill_report.client_order_id = None;
6447        fill_report.venue_order_id = VenueOrderId::from("V-POSITION-EXTERNAL");
6448        let position_result = position_report_result(&node, venue_report);
6449
6450        node.handle_position_fill_report_result(PositionFillReportResult {
6451            position_result,
6452            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6453        });
6454
6455        let cache = node.kernel.cache.borrow();
6456        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6457        assert_eq!(positions.len(), 1);
6458        assert_eq!(positions[0].id, position_id);
6459        assert_eq!(positions[0].quantity, Quantity::from("2.0"));
6460        assert_eq!(
6461            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6462            2
6463        );
6464        drop(positions);
6465        drop(cache);
6466        assert!(
6467            !node
6468                .exec_manager
6469                .position_contains_fill_report(&fill_report)
6470        );
6471    }
6472
6473    #[rstest]
6474    fn test_bundled_fill_report_invalidates_prepared_position_reconciliation() {
6475        let (mut node, venue_report, fill_report) =
6476            position_fill_test_fixture("BundledPositionActivityNode", Quantity::from("1.0"));
6477        let key = (venue_report.instrument_id, venue_report.account_id);
6478        let client_order_id = fill_report.client_order_id.unwrap();
6479        let position_result = position_report_result(&node, venue_report);
6480        let revision = node.exec_manager.position_activity_revision(&key);
6481
6482        let order_report = OrderStatusReport::new(
6483            key.1,
6484            key.0,
6485            Some(client_order_id),
6486            fill_report.venue_order_id,
6487            OrderSide::Buy.into(),
6488            OrderType::Limit,
6489            TimeInForce::Gtc,
6490            OrderStatus::PartiallyFilled,
6491            Quantity::from("10.0"),
6492            Quantity::from("2.0"),
6493            UnixNanos::from(1_000),
6494            UnixNanos::from(1_000),
6495            UnixNanos::from(2_000),
6496            None,
6497        );
6498        let event = ExecutionEvent::Report(ExecutionReport::OrderWithFills(
6499            Box::new(order_report),
6500            vec![fill_report.clone()],
6501        ));
6502
6503        assert_eq!(
6504            node.observe_exec_event_before_dispatch(&event),
6505            Some(vec![client_order_id])
6506        );
6507
6508        assert_eq!(
6509            node.exec_manager.position_activity_revision(&key),
6510            revision + 1
6511        );
6512        assert!(
6513            !node
6514                .exec_manager
6515                .position_report_check_is_current(&position_result.check, &key)
6516        );
6517
6518        node.handle_position_fill_report_result(PositionFillReportResult {
6519            position_result,
6520            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6521        });
6522
6523        let cache = node.kernel.cache.borrow();
6524        let order = cache.order(&client_order_id).unwrap();
6525        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6526        assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6527        assert!(!order.trade_ids().contains(&&fill_report.trade_id));
6528        assert_eq!(positions.len(), 1);
6529        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6530        assert_eq!(positions[0].side, PositionSide::Long);
6531        assert_eq!(
6532            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6533            1
6534        );
6535    }
6536
6537    #[rstest]
6538    fn test_position_fill_report_result_defers_after_local_position_activity() {
6539        let (mut node, venue_report, fill_report) =
6540            position_fill_test_fixture("StalePositionFillNode", Quantity::from("1.0"));
6541        let key = (venue_report.instrument_id, venue_report.account_id);
6542        let position_result = position_report_result(&node, venue_report);
6543        node.exec_manager.record_position_activity(key.0, key.1);
6544
6545        node.handle_position_fill_report_result(PositionFillReportResult {
6546            position_result,
6547            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6548        });
6549
6550        let cache = node.kernel.cache.borrow();
6551        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6552        assert_eq!(positions.len(), 1);
6553        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6554        assert_eq!(
6555            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6556            1
6557        );
6558        drop(positions);
6559        drop(cache);
6560        assert!(
6561            !node
6562                .exec_manager
6563                .position_contains_fill_report(&fill_report)
6564        );
6565    }
6566
6567    #[rstest]
6568    fn test_observe_exec_event_before_dispatch_accepted_batch_stamps_local_activity() {
6569        let config = LiveNodeConfig {
6570            exec_engine: crate::config::LiveExecutionEngineConfig {
6571                reconciliation: true,
6572                open_check_threshold_ms: 5_000,
6573                single_order_query_delay_ms: 0,
6574                ..Default::default()
6575            },
6576            ..Default::default()
6577        };
6578
6579        let mut node = LiveNode::build("AcceptedBatchNode".to_string(), Some(config)).unwrap();
6580        let account_id = AccountId::from("TEST-ACCEPTED-BATCH-001");
6581        let client_id = ClientId::from("TEST-ACCEPTED-BATCH");
6582        let instrument = crypto_perpetual_ethusdt();
6583        let instrument_id = instrument.id();
6584        let client_order_id = ClientOrderId::from("O-ACCEPTED-BATCH");
6585        let venue_order_id = VenueOrderId::from("V-ACCEPTED-BATCH");
6586
6587        node.kernel
6588            .cache
6589            .borrow_mut()
6590            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6591            .unwrap();
6592        insert_accepted_limit_order_in_node(
6593            &node,
6594            account_id,
6595            client_id,
6596            instrument_id,
6597            client_order_id,
6598            venue_order_id,
6599        );
6600
6601        assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
6602
6603        let accepted = OrderAcceptedSpec::builder()
6604            .instrument_id(instrument_id)
6605            .client_order_id(client_order_id)
6606            .venue_order_id(venue_order_id)
6607            .account_id(account_id)
6608            .build();
6609        let event = ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![accepted]));
6610
6611        let close_ids = node.observe_exec_event_before_dispatch(&event);
6612
6613        assert_eq!(close_ids, Some(vec![client_order_id]));
6614        assert!(node.exec_manager.check_open_order_queries().is_empty());
6615    }
6616
6617    #[rstest]
6618    #[cfg_attr(
6619        not(all(feature = "simulation", madsim)),
6620        tokio::test(start_paused = true)
6621    )]
6622    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6623    async fn test_batch_cancel_command_registers_each_child_for_inflight_timeout() {
6624        use nautilus_common::messages::execution::{BatchCancelOrders, CancelOrder};
6625        use nautilus_model::{
6626            events::order::spec::OrderPendingCancelSpec, identifiers::ClientOrderId,
6627        };
6628
6629        let config = LiveNodeConfig {
6630            exec_engine: crate::config::LiveExecutionEngineConfig {
6631                reconciliation: true,
6632                inflight_check_threshold_ms: 100,
6633                inflight_check_retries: 1,
6634                ..Default::default()
6635            },
6636            ..Default::default()
6637        };
6638
6639        let mut node = LiveNode::build("BatchCancelNode".to_string(), Some(config)).unwrap();
6640        let trader_id = TraderId::from("TESTER-001");
6641        let strategy_id = StrategyId::from("S-BATCH-CANCEL");
6642        let account_id = AccountId::from("TEST-001");
6643        let instrument = crypto_perpetual_ethusdt();
6644        let instrument_id = instrument.id();
6645        node.kernel
6646            .exec_engine
6647            .borrow_mut()
6648            .register_client(Box::new(StubExecutionClient::new(
6649                ClientId::from("STUB"),
6650                account_id,
6651                instrument_id.venue,
6652                OmsType::Netting,
6653                None,
6654            )))
6655            .unwrap();
6656        node.kernel
6657            .exec_engine
6658            .borrow_mut()
6659            .set_default_client(ClientId::from("STUB"))
6660            .unwrap();
6661        let child_ids = [
6662            ClientOrderId::from("O-BATCH-CANCEL-1"),
6663            ClientOrderId::from("O-BATCH-CANCEL-2"),
6664        ];
6665        node.kernel
6666            .cache
6667            .borrow_mut()
6668            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6669            .unwrap();
6670
6671        for client_order_id in child_ids {
6672            let order = OrderTestBuilder::new(OrderType::Limit)
6673                .trader_id(trader_id)
6674                .strategy_id(strategy_id)
6675                .client_order_id(client_order_id)
6676                .instrument_id(instrument_id)
6677                .quantity(Quantity::from("10.0"))
6678                .price(Price::from("100.0"))
6679                .build();
6680            let venue_order_id = VenueOrderId::from(format!("V-{client_order_id}").as_str());
6681            // Model the production batch-cancel path: each child is accepted, then
6682            // moved to PendingCancel, so an inflight timeout must emit a Canceled
6683            // event (not a Submitted-order rejection).
6684            let submitted = TestOrderEventStubs::submitted(&order, account_id);
6685            let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
6686            let pending_cancel = OrderEventAny::PendingCancel(
6687                OrderPendingCancelSpec::builder()
6688                    .trader_id(trader_id)
6689                    .strategy_id(strategy_id)
6690                    .instrument_id(instrument_id)
6691                    .client_order_id(client_order_id)
6692                    .account_id(account_id)
6693                    .venue_order_id(venue_order_id)
6694                    .build(),
6695            );
6696            let mut cache = node.kernel.cache.borrow_mut();
6697            cache.add_order(order, None, None, false).unwrap();
6698            cache.update_order(&submitted).unwrap();
6699            cache.update_order(&accepted).unwrap();
6700            cache.update_order(&pending_cancel).unwrap();
6701        }
6702
6703        let cancels = child_ids
6704            .into_iter()
6705            .map(|client_order_id| {
6706                CancelOrder::new(
6707                    trader_id,
6708                    None,
6709                    strategy_id,
6710                    instrument_id,
6711                    client_order_id,
6712                    None,
6713                    UUID4::new(),
6714                    UnixNanos::default(),
6715                    None,
6716                    None,
6717                )
6718            })
6719            .collect();
6720
6721        let command = TradingCommand::CancelOrders(BatchCancelOrders::new(
6722            trader_id,
6723            None,
6724            strategy_id,
6725            instrument_id,
6726            cancels,
6727            UUID4::new(),
6728            UnixNanos::default(),
6729            None,
6730            None,
6731        ));
6732
6733        node.observe_exec_command_before_dispatch(&command);
6734        advance_clock(Duration::from_millis(101)).await;
6735        let result = node.exec_manager.check_inflight_orders();
6736        let timed_out_ids = result
6737            .events
6738            .iter()
6739            .map(OrderEventAny::client_order_id)
6740            .collect::<IndexSet<_>>();
6741
6742        assert_eq!(timed_out_ids, IndexSet::from(child_ids));
6743        assert_eq!(result.events.len(), child_ids.len());
6744        assert!(
6745            result
6746                .events
6747                .iter()
6748                .all(|event| matches!(event, OrderEventAny::Canceled(_))),
6749            "batch-cancel children must time out as Canceled events",
6750        );
6751    }
6752
6753    #[rstest]
6754    #[cfg_attr(
6755        not(all(feature = "simulation", madsim)),
6756        tokio::test(start_paused = true)
6757    )]
6758    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6759    async fn test_risk_bound_command_does_not_register_inflight() {
6760        let config = LiveNodeConfig {
6761            exec_engine: crate::config::LiveExecutionEngineConfig {
6762                reconciliation: true,
6763                inflight_check_threshold_ms: 100,
6764                inflight_check_retries: 2,
6765                ..Default::default()
6766            },
6767            ..Default::default()
6768        };
6769
6770        let mut node = LiveNode::build("RiskBoundNode".to_string(), Some(config)).unwrap();
6771        msgbus::register_trading_command_endpoint(
6772            MessagingSwitchboard::risk_engine_execute(),
6773            TypedIntoHandler::from(|_: TradingCommand| {}),
6774        );
6775        let instrument = crypto_perpetual_ethusdt();
6776        let instrument_id = instrument.id();
6777        let order = OrderTestBuilder::new(OrderType::Limit)
6778            .trader_id(node.trader_id())
6779            .strategy_id(StrategyId::from("S-RISK-DENIED"))
6780            .instrument_id(instrument_id)
6781            .side(OrderSide::Buy)
6782            .quantity(Quantity::from("1.000"))
6783            .price(Price::from("100.00"))
6784            .build();
6785        let client_order_id = order.client_order_id();
6786
6787        {
6788            let mut cache = node.kernel.cache.borrow_mut();
6789            cache
6790                .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6791                .unwrap();
6792            cache.add_order(order.clone(), None, None, false).unwrap();
6793        }
6794
6795        let submit_order = SubmitOrder::new(
6796            order.trader_id(),
6797            None,
6798            order.strategy_id(),
6799            instrument_id,
6800            client_order_id,
6801            order.init_event().clone(),
6802            None,
6803            None,
6804            None,
6805            UUID4::new(),
6806            UnixNanos::default(),
6807            None,
6808        );
6809        node.process_exec_command(
6810            TradingCommandMessage::new(
6811                MessagingSwitchboard::risk_engine_execute(),
6812                TradingCommand::SubmitOrder(submit_order),
6813            )
6814            .into(),
6815        );
6816
6817        advance_clock(Duration::from_millis(101)).await;
6818        let result = node.exec_manager.check_inflight_orders();
6819        let status = node
6820            .kernel
6821            .cache
6822            .borrow()
6823            .order(&client_order_id)
6824            .unwrap()
6825            .status();
6826
6827        assert_eq!(status, OrderStatus::Initialized);
6828        assert_eq!(
6829            node.exec_manager.recon_check_retry_count(&client_order_id),
6830            0
6831        );
6832        assert!(result.events.is_empty());
6833        assert!(result.queries.is_empty());
6834    }
6835
6836    #[rstest]
6837    #[cfg_attr(
6838        not(all(feature = "simulation", madsim)),
6839        tokio::test(start_paused = true)
6840    )]
6841    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6842    async fn test_external_cancel_command_still_registers_inflight() {
6843        use nautilus_common::messages::execution::CancelOrder;
6844        let external = ClientId::from("EXTERNAL");
6845
6846        let config = LiveNodeConfig {
6847            exec_engine: crate::config::LiveExecutionEngineConfig {
6848                reconciliation: true,
6849                inflight_check_threshold_ms: 100,
6850                inflight_check_retries: 2,
6851                external_clients: Some(vec![external]),
6852                ..Default::default()
6853            },
6854            ..Default::default()
6855        };
6856
6857        let mut node = LiveNode::build("ExternalCancelNode".to_string(), Some(config)).unwrap();
6858        let instrument = crypto_perpetual_ethusdt();
6859        let mut order = OrderTestBuilder::new(OrderType::Limit)
6860            .trader_id(node.trader_id())
6861            .strategy_id(StrategyId::from("S-EXTERNAL"))
6862            .instrument_id(instrument.id())
6863            .quantity(Quantity::from("9.003"))
6864            .price(Price::from("113.47"))
6865            .build();
6866        let venue = VenueOrderId::from("V-EXTERNAL");
6867        order
6868            .apply(TestOrderEventStubs::accepted(
6869                &order,
6870                AccountId::from("SIM-001"),
6871                venue,
6872            ))
6873            .unwrap();
6874        let id = order.client_order_id();
6875        let command = TradingCommand::CancelOrder(CancelOrder::new(
6876            order.trader_id(),
6877            Some(external),
6878            order.strategy_id(),
6879            order.instrument_id(),
6880            id,
6881            Some(venue),
6882            UUID4::new(),
6883            17.into(),
6884            None,
6885            None,
6886        ));
6887        {
6888            let mut cache = node.kernel.cache.borrow_mut();
6889            cache.add_instrument(instrument.into()).unwrap();
6890            cache
6891                .add_order(order.clone(), None, Some(external), false)
6892                .unwrap();
6893        }
6894
6895        node.process_exec_command(
6896            TradingCommandMessage::new(MessagingSwitchboard::exec_engine_execute(), command).into(),
6897        );
6898        advance_clock(Duration::from_millis(101)).await;
6899        let result = node.exec_manager.check_inflight_orders();
6900
6901        let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
6902            panic!("expected external query");
6903        };
6904
6905        assert_eq!(
6906            (query.client_order_id, query.venue_order_id, query.client_id),
6907            (id, Some(venue), Some(external))
6908        );
6909        assert!(result.events.is_empty());
6910        assert_eq!(
6911            serde_json::to_value(node.kernel.cache.borrow().order(&id).unwrap().clone()).unwrap(),
6912            serde_json::to_value(&order).unwrap()
6913        );
6914    }
6915
6916    #[rstest]
6917    #[case::modify(false, false)]
6918    #[case::batch_modify(false, true)]
6919    #[case::cancel(true, false)]
6920    #[case::batch_cancel(true, true)]
6921    #[cfg_attr(
6922        not(all(feature = "simulation", madsim)),
6923        tokio::test(start_paused = true)
6924    )]
6925    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6926    async fn test_native_command_denial_does_not_leave_inflight_tracking(
6927        #[case] cancel: bool,
6928        #[case] batch: bool,
6929        #[values(false, true)] old_pending: bool,
6930    ) {
6931        use nautilus_common::messages::execution::{
6932            BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
6933        };
6934        use nautilus_model::events::order::spec::{OrderModifyDeniedSpec, OrderPendingUpdateSpec};
6935
6936        let config = LiveNodeConfig {
6937            exec_engine: crate::config::LiveExecutionEngineConfig {
6938                reconciliation: true,
6939                inflight_check_threshold_ms: 100,
6940                inflight_check_retries: 2,
6941                ..Default::default()
6942            },
6943            ..Default::default()
6944        };
6945
6946        let mut node = LiveNode::build("LocalDenialNode".to_string(), Some(config)).unwrap();
6947        let instrument = crypto_perpetual_ethusdt();
6948        let mut order = OrderTestBuilder::new(OrderType::Limit)
6949            .trader_id(node.trader_id())
6950            .strategy_id(StrategyId::from("S-LOCAL-DENIED"))
6951            .instrument_id(instrument.id())
6952            .side(OrderSide::Buy)
6953            .quantity(Quantity::from("9.003"))
6954            .price(Price::from("113.47"))
6955            .build();
6956        let account_id = AccountId::from("SIM-001");
6957        let venue_id = VenueOrderId::from("V-LOCAL-DENIED");
6958        order
6959            .apply(TestOrderEventStubs::accepted(&order, account_id, venue_id))
6960            .unwrap();
6961        let id = order.client_order_id();
6962        let old_cause = UUID4::new();
6963
6964        if old_pending {
6965            let mut marker = OrderPendingUpdateSpec::builder()
6966                .trader_id(order.trader_id())
6967                .strategy_id(order.strategy_id())
6968                .instrument_id(order.instrument_id())
6969                .client_order_id(id)
6970                .venue_order_id(venue_id)
6971                .account_id(account_id)
6972                .build();
6973            marker.causation_id = Some(old_cause);
6974            order.apply(OrderEventAny::PendingUpdate(marker)).unwrap();
6975            node.exec_manager.register_inflight(id, old_cause);
6976        }
6977
6978        let expected_status = order.status();
6979        let events_before = order.event_count();
6980        {
6981            let mut cache = node.kernel.cache.borrow_mut();
6982            cache.add_instrument(instrument.into()).unwrap();
6983            cache.add_order(order.clone(), None, None, false).unwrap();
6984        }
6985
6986        let cause = UUID4::new();
6987
6988        let command = if cancel {
6989            let command = CancelOrder::new(
6990                order.trader_id(),
6991                None,
6992                order.strategy_id(),
6993                order.instrument_id(),
6994                id,
6995                Some(venue_id),
6996                cause,
6997                17.into(),
6998                None,
6999                None,
7000            );
7001
7002            if batch {
7003                TradingCommand::CancelOrders(BatchCancelOrders::new(
7004                    order.trader_id(),
7005                    None,
7006                    order.strategy_id(),
7007                    order.instrument_id(),
7008                    vec![command],
7009                    UUID4::new(),
7010                    19.into(),
7011                    None,
7012                    None,
7013                ))
7014            } else {
7015                TradingCommand::CancelOrder(command)
7016            }
7017        } else {
7018            let command = ModifyOrder::new(
7019                order.trader_id(),
7020                None,
7021                order.strategy_id(),
7022                order.instrument_id(),
7023                id,
7024                Some(venue_id),
7025                Some(Quantity::from("8.001")),
7026                None,
7027                None,
7028                cause,
7029                17.into(),
7030                None,
7031                None,
7032            );
7033
7034            if batch {
7035                TradingCommand::ModifyOrders(BatchModifyOrders::new(
7036                    order.trader_id(),
7037                    None,
7038                    order.strategy_id(),
7039                    order.instrument_id(),
7040                    vec![command],
7041                    UUID4::new(),
7042                    19.into(),
7043                    None,
7044                    None,
7045                ))
7046            } else {
7047                TradingCommand::ModifyOrder(command)
7048            }
7049        };
7050
7051        node.process_exec_command(
7052            TradingCommandMessage::new(MessagingSwitchboard::exec_engine_execute(), command).into(),
7053        );
7054        {
7055            let cache = node.kernel.cache.borrow();
7056            let cached = cache.order(&id).unwrap();
7057            match cached.last_event() {
7058                OrderEventAny::ModifyDenied(event) if !cancel => {
7059                    assert_eq!(event.causation_id, Some(cause));
7060                }
7061                OrderEventAny::CancelDenied(event) if cancel => {
7062                    assert_eq!(event.causation_id, Some(cause));
7063                }
7064                other => panic!("expected native command denial, was {other:?}"),
7065            }
7066
7067            assert_eq!(cached.status(), expected_status);
7068            assert_eq!(cached.quantity(), order.quantity());
7069            assert_eq!(cached.price(), order.price());
7070            assert_eq!(cached.filled_qty(), order.filled_qty());
7071            assert_eq!(cached.event_count(), events_before + 1);
7072        }
7073
7074        advance_clock(Duration::from_millis(101)).await;
7075        let result = node.exec_manager.check_inflight_orders();
7076        assert_eq!(result.queries.len(), usize::from(old_pending));
7077        assert!(result.events.is_empty());
7078
7079        if old_pending {
7080            let refusal = OrderModifyDeniedSpec::builder()
7081                .trader_id(order.trader_id())
7082                .strategy_id(order.strategy_id())
7083                .instrument_id(order.instrument_id())
7084                .client_order_id(id)
7085                .venue_order_id(venue_id)
7086                .account_id(account_id)
7087                .causation_id(old_cause)
7088                .build();
7089            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::ModifyDenied(refusal)));
7090        }
7091
7092        assert_eq!(node.exec_manager.recon_check_retry_count(&id), 0);
7093    }
7094
7095    #[rstest]
7096    #[cfg_attr(
7097        not(all(feature = "simulation", madsim)),
7098        tokio::test(start_paused = true)
7099    )]
7100    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7101    async fn test_risk_approved_command_registers_inflight() {
7102        let config = LiveNodeConfig {
7103            risk_engine: crate::config::LiveRiskEngineConfig {
7104                bypass: true,
7105                ..Default::default()
7106            },
7107            exec_engine: crate::config::LiveExecutionEngineConfig {
7108                reconciliation: true,
7109                inflight_check_threshold_ms: 100,
7110                inflight_check_retries: 2,
7111                ..Default::default()
7112            },
7113            ..Default::default()
7114        };
7115
7116        let mut node = LiveNode::build("RiskApprovedNode".to_string(), Some(config)).unwrap();
7117        msgbus::register_trading_command_endpoint(
7118            MessagingSwitchboard::exec_engine_execute(),
7119            TypedIntoHandler::from(|_: TradingCommand| {}),
7120        );
7121        let instrument = crypto_perpetual_ethusdt();
7122        let instrument_id = instrument.id();
7123        let order = OrderTestBuilder::new(OrderType::Limit)
7124            .trader_id(node.trader_id())
7125            .strategy_id(StrategyId::from("S-RISK-APPROVED"))
7126            .instrument_id(instrument_id)
7127            .side(OrderSide::Buy)
7128            .quantity(Quantity::from("1.000"))
7129            .price(Price::from("100.00"))
7130            .build();
7131        let client_order_id = order.client_order_id();
7132
7133        {
7134            let mut cache = node.kernel.cache.borrow_mut();
7135            cache
7136                .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7137                .unwrap();
7138            cache.add_order(order.clone(), None, None, false).unwrap();
7139        }
7140
7141        let submit_order = SubmitOrder::new(
7142            order.trader_id(),
7143            None,
7144            order.strategy_id(),
7145            instrument_id,
7146            client_order_id,
7147            order.init_event().clone(),
7148            None,
7149            None,
7150            None,
7151            UUID4::new(),
7152            UnixNanos::default(),
7153            None,
7154        );
7155        node.process_exec_command(
7156            TradingCommandMessage::new(
7157                MessagingSwitchboard::risk_engine_execute(),
7158                TradingCommand::SubmitOrder(submit_order),
7159            )
7160            .into(),
7161        );
7162
7163        advance_clock(Duration::from_millis(101)).await;
7164        let result = node.exec_manager.check_inflight_orders();
7165
7166        let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
7167            panic!("expected one query order command");
7168        };
7169
7170        assert_eq!(query.client_order_id, client_order_id);
7171        assert_eq!(
7172            node.exec_manager.recon_check_retry_count(&client_order_id),
7173            1
7174        );
7175        assert!(result.events.is_empty());
7176    }
7177
7178    #[rstest]
7179    fn test_live_node_builder_clock_factory_drives_kernel_clock() {
7180        let calls = Rc::new(Cell::new(0usize));
7181        let calls_in_factory = calls.clone();
7182        let sentinel = UnixNanos::from(123_456_789_u64);
7183
7184        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7185            .unwrap()
7186            .with_reconciliation(false)
7187            .with_clock_factory(move || {
7188                calls_in_factory.set(calls_in_factory.get() + 1);
7189                let mut clock = VirtualClock::new();
7190                clock.advance_time(sentinel, true);
7191                Rc::new(RefCell::new(clock)) as Rc<RefCell<dyn Clock>>
7192            })
7193            .build()
7194            .unwrap();
7195
7196        assert_eq!(node.kernel().clock().borrow().timestamp_ns(), sentinel);
7197        assert_eq!(calls.get(), 1);
7198    }
7199
7200    #[derive(Debug)]
7201    struct ReplayKernelEventStore {
7202        fail_restore: bool,
7203    }
7204
7205    impl KernelEventStore for ReplayKernelEventStore {
7206        fn restore_parent_cache(
7207            &mut self,
7208            _instance_id: UUID4,
7209            _cache: &mut Cache,
7210        ) -> anyhow::Result<()> {
7211            if self.fail_restore {
7212                anyhow::bail!("replay restore failed");
7213            }
7214
7215            Ok(())
7216        }
7217
7218        fn open(
7219            &mut self,
7220            _instance_id: UUID4,
7221            _components: &RegisteredComponents,
7222            _environment: Environment,
7223        ) -> anyhow::Result<()> {
7224            Ok(())
7225        }
7226
7227        fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
7228            None
7229        }
7230
7231        fn seal(&mut self, _ts_init: UnixNanos) {}
7232
7233        fn run_id(&self) -> Option<&str> {
7234            Some("replay-child")
7235        }
7236
7237        fn parent_run_id(&self) -> Option<&str> {
7238            Some("seed-run")
7239        }
7240
7241        fn is_event_store_replay_configured(&self) -> bool {
7242            true
7243        }
7244
7245        fn is_halted(&self) -> bool {
7246            false
7247        }
7248    }
7249
7250    #[derive(Debug)]
7251    struct TestStrategy {
7252        core: StrategyCore,
7253    }
7254
7255    impl TestStrategy {
7256        fn new(config: StrategyConfig) -> Self {
7257            Self {
7258                core: StrategyCore::new(config),
7259            }
7260        }
7261    }
7262
7263    impl DataActor for TestStrategy {}
7264
7265    nautilus_strategy!(TestStrategy, {
7266        fn external_order_instrument_ids(&self) -> Option<Vec<InstrumentId>> {
7267            self.core.config.external_order_instrument_ids.clone()
7268        }
7269    });
7270
7271    fn live_node_with_replay_store(fail_restore: bool) -> LiveNode {
7272        // load_state must be true: the kernel rejects event-store replay otherwise,
7273        // and LiveNodeConfig defaults it to false.
7274        let builder = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
7275            .unwrap()
7276            .with_exec_engine_config(crate::config::LiveExecutionEngineConfig {
7277                reconciliation: false,
7278                ..Default::default()
7279            })
7280            .with_load_state(true)
7281            .with_name("TestKernel")
7282            .with_event_store(move |_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
7283                Ok(Box::new(ReplayKernelEventStore { fail_restore }) as Box<dyn KernelEventStore>)
7284            });
7285
7286        builder.build().unwrap()
7287    }
7288
7289    #[rstest]
7290    fn test_add_strategy_registers_external_order_claims_with_manager_and_engine() {
7291        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7292            .unwrap()
7293            .with_reconciliation(false)
7294            .with_delay_post_stop_secs(0)
7295            .with_timeout_connection(1)
7296            .build()
7297            .unwrap();
7298        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7299        let strategy_id = StrategyId::from("CLAIMS-001");
7300
7301        node.add_strategy(TestStrategy::new(StrategyConfig {
7302            strategy_id: Some(strategy_id),
7303            external_order_instrument_ids: Some(vec![instrument_id]),
7304            ..Default::default()
7305        }))
7306        .unwrap();
7307
7308        assert_eq!(
7309            node.exec_manager.get_external_order_claim(&instrument_id),
7310            Some(strategy_id)
7311        );
7312
7313        {
7314            let exec_engine = node.kernel().exec_engine.borrow();
7315            assert_eq!(
7316                exec_engine.get_external_order_claim(&instrument_id),
7317                Some(strategy_id)
7318            );
7319        }
7320    }
7321
7322    #[rstest]
7323    fn test_register_external_order_claims_after_build_is_visible_to_manager_and_engine() {
7324        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7325            .unwrap()
7326            .with_reconciliation(false)
7327            .build()
7328            .unwrap();
7329        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7330        let strategy_id = StrategyId::from("CLAIMS-001");
7331
7332        node.register_external_order_claims(strategy_id, &[instrument_id])
7333            .unwrap();
7334
7335        assert_eq!(
7336            node.exec_manager.get_external_order_claim(&instrument_id),
7337            Some(strategy_id)
7338        );
7339        assert_eq!(
7340            node.kernel
7341                .exec_engine
7342                .borrow()
7343                .get_external_order_claim(&instrument_id),
7344            Some(strategy_id)
7345        );
7346    }
7347
7348    #[rstest]
7349    #[tokio::test]
7350    async fn test_register_external_order_claims_while_running_is_visible_to_manager_and_engine() {
7351        let mut node = live_node_with_replay_store(false);
7352        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7353        let strategy_id = StrategyId::from("CLAIMS-001");
7354
7355        node.start().await.unwrap();
7356        assert_eq!(node.state(), NodeState::Running);
7357
7358        node.register_external_order_claims(strategy_id, &[instrument_id])
7359            .unwrap();
7360
7361        assert_eq!(
7362            node.exec_manager.get_external_order_claim(&instrument_id),
7363            Some(strategy_id)
7364        );
7365        assert_eq!(
7366            node.kernel
7367                .exec_engine
7368                .borrow()
7369                .get_external_order_claim(&instrument_id),
7370            Some(strategy_id)
7371        );
7372    }
7373
7374    #[rstest]
7375    fn test_register_external_order_claims_conflicting_batch_leaves_new_claims_absent() {
7376        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7377            .unwrap()
7378            .with_reconciliation(false)
7379            .build()
7380            .unwrap();
7381        let existing_instrument = InstrumentId::from("AUDUSD.SIM");
7382        let new_instruments = [
7383            InstrumentId::from("EURUSD.SIM"),
7384            InstrumentId::from("GBPUSD.SIM"),
7385        ];
7386        let existing_strategy_id = StrategyId::from("CLAIMS-001");
7387        let new_strategy_id = StrategyId::from("CLAIMS-002");
7388        node.register_external_order_claims(existing_strategy_id, &[existing_instrument])
7389            .unwrap();
7390
7391        let result = node.register_external_order_claims(
7392            new_strategy_id,
7393            &[new_instruments[0], existing_instrument, new_instruments[1]],
7394        );
7395
7396        assert!(result.is_err());
7397        assert_eq!(
7398            node.exec_manager
7399                .get_external_order_claim(&existing_instrument),
7400            Some(existing_strategy_id)
7401        );
7402
7403        for instrument_id in new_instruments {
7404            assert_eq!(
7405                node.exec_manager.get_external_order_claim(&instrument_id),
7406                None
7407            );
7408            assert_eq!(
7409                node.kernel
7410                    .exec_engine
7411                    .borrow()
7412                    .get_external_order_claim(&instrument_id),
7413                None
7414            );
7415        }
7416    }
7417
7418    #[rstest]
7419    fn test_deregister_external_order_claims_allows_successor_to_claim() {
7420        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7421            .unwrap()
7422            .with_reconciliation(false)
7423            .build()
7424            .unwrap();
7425        let instruments = [
7426            InstrumentId::from("AUDUSD.SIM"),
7427            InstrumentId::from("EURUSD.SIM"),
7428        ];
7429        let first_strategy_id = StrategyId::from("CLAIMS-001");
7430        let successor_strategy_id = StrategyId::from("CLAIMS-002");
7431        node.register_external_order_claims(first_strategy_id, &instruments)
7432            .unwrap();
7433
7434        node.deregister_external_order_claims(first_strategy_id)
7435            .unwrap();
7436        node.register_external_order_claims(successor_strategy_id, &instruments)
7437            .unwrap();
7438
7439        for instrument_id in instruments {
7440            assert_eq!(
7441                node.exec_manager.get_external_order_claim(&instrument_id),
7442                Some(successor_strategy_id)
7443            );
7444            assert_eq!(
7445                node.kernel
7446                    .exec_engine
7447                    .borrow()
7448                    .get_external_order_claim(&instrument_id),
7449                Some(successor_strategy_id)
7450            );
7451        }
7452    }
7453
7454    #[rstest]
7455    fn test_deregister_external_order_claims_without_claims_is_idempotent() {
7456        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7457            .unwrap()
7458            .with_reconciliation(false)
7459            .build()
7460            .unwrap();
7461        let strategy_id = StrategyId::from("CLAIMS-001");
7462
7463        node.deregister_external_order_claims(strategy_id).unwrap();
7464        node.deregister_external_order_claims(strategy_id).unwrap();
7465    }
7466
7467    #[rstest]
7468    fn test_add_strategy_rejects_duplicate_external_order_claim_without_overwriting() {
7469        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7470            .unwrap()
7471            .with_reconciliation(false)
7472            .with_delay_post_stop_secs(0)
7473            .with_timeout_connection(1)
7474            .build()
7475            .unwrap();
7476        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7477        let strategy_id = StrategyId::from("CLAIMS-001");
7478        let duplicate_strategy_id = StrategyId::from("OTHER-002");
7479
7480        node.add_strategy(TestStrategy::new(StrategyConfig {
7481            strategy_id: Some(strategy_id),
7482            external_order_instrument_ids: Some(vec![instrument_id]),
7483            ..Default::default()
7484        }))
7485        .unwrap();
7486
7487        let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7488            strategy_id: Some(duplicate_strategy_id),
7489            external_order_instrument_ids: Some(vec![instrument_id]),
7490            ..Default::default()
7491        }));
7492
7493        assert!(result.is_err());
7494        assert!(
7495            result
7496                .unwrap_err()
7497                .to_string()
7498                .contains("already exists for CLAIMS-001")
7499        );
7500        assert_eq!(
7501            node.exec_manager.get_external_order_claim(&instrument_id),
7502            Some(strategy_id)
7503        );
7504
7505        {
7506            let exec_engine = node.kernel().exec_engine.borrow();
7507            assert_eq!(
7508                exec_engine.get_external_order_claim(&instrument_id),
7509                Some(strategy_id)
7510            );
7511        }
7512    }
7513
7514    #[rstest]
7515    fn test_add_strategy_rejects_repeated_external_order_claim_without_registering() {
7516        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7517            .unwrap()
7518            .with_reconciliation(false)
7519            .with_delay_post_stop_secs(0)
7520            .with_timeout_connection(1)
7521            .build()
7522            .unwrap();
7523        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7524        let strategy_id = StrategyId::from("CLAIMS-001");
7525
7526        let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7527            strategy_id: Some(strategy_id),
7528            external_order_instrument_ids: Some(vec![instrument_id, instrument_id]),
7529            ..Default::default()
7530        }));
7531
7532        assert!(result.is_err());
7533        assert!(
7534            result
7535                .unwrap_err()
7536                .to_string()
7537                .contains("appears more than once for CLAIMS-001")
7538        );
7539        assert_eq!(
7540            node.exec_manager.get_external_order_claim(&instrument_id),
7541            None
7542        );
7543
7544        {
7545            let exec_engine = node.kernel().exec_engine.borrow();
7546            assert_eq!(exec_engine.get_external_order_claim(&instrument_id), None);
7547        }
7548    }
7549
7550    #[rstest]
7551    fn test_add_strategy_failure_restores_external_order_claims() {
7552        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7553            .unwrap()
7554            .with_reconciliation(false)
7555            .with_delay_post_stop_secs(0)
7556            .with_timeout_connection(1)
7557            .build()
7558            .unwrap();
7559        let existing_instrument_id = InstrumentId::from("AUDUSD.SIM");
7560        let configured_instrument_id = InstrumentId::from("EURUSD.SIM");
7561        let strategy_id = StrategyId::from("CLAIMS-001");
7562        node.register_external_order_claims(strategy_id, &[existing_instrument_id])
7563            .unwrap();
7564
7565        let mut strategy = TestStrategy::new(StrategyConfig {
7566            strategy_id: Some(strategy_id),
7567            external_order_instrument_ids: Some(vec![configured_instrument_id]),
7568            ..Default::default()
7569        });
7570
7571        strategy
7572            .core
7573            .register(
7574                node.trader_id(),
7575                node.kernel.clock(),
7576                node.kernel.cache.clone(),
7577                node.kernel.portfolio.clone(),
7578            )
7579            .unwrap();
7580
7581        let result = node.add_strategy(strategy);
7582
7583        assert!(result.is_err());
7584        assert!(
7585            result
7586                .unwrap_err()
7587                .to_string()
7588                .contains("already registered with trader")
7589        );
7590        assert_eq!(
7591            node.exec_manager
7592                .get_external_order_claim(&existing_instrument_id),
7593            Some(strategy_id)
7594        );
7595        assert_eq!(
7596            node.kernel
7597                .exec_engine
7598                .borrow()
7599                .get_external_order_claim(&existing_instrument_id),
7600            Some(strategy_id)
7601        );
7602        assert_eq!(
7603            node.exec_manager
7604                .get_external_order_claim(&configured_instrument_id),
7605            None
7606        );
7607        assert_eq!(
7608            node.kernel
7609                .exec_engine
7610                .borrow()
7611                .get_external_order_claim(&configured_instrument_id),
7612            None
7613        );
7614    }
7615
7616    #[rstest]
7617    fn test_add_strategy_without_claims_or_oms_type_does_not_require_engine_borrow() {
7618        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7619            .unwrap()
7620            .with_reconciliation(false)
7621            .build()
7622            .unwrap();
7623        let exec_engine = node.kernel.exec_engine.clone();
7624        let _engine_borrow = exec_engine.borrow_mut();
7625
7626        node.add_strategy(TestStrategy::new(StrategyConfig {
7627            strategy_id: Some(StrategyId::from("NOCLAIMS-001")),
7628            ..Default::default()
7629        }))
7630        .unwrap();
7631    }
7632
7633    #[rstest]
7634    fn test_add_strategy_registers_configured_hedging_oms_type() {
7635        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7636            .unwrap()
7637            .with_reconciliation(false)
7638            .with_delay_post_stop_secs(0)
7639            .with_timeout_connection(1)
7640            .build()
7641            .unwrap();
7642        let strategy_id = StrategyId::from("FUNDING_ARBITRAGE-001");
7643
7644        node.add_strategy(TestStrategy::new(StrategyConfig {
7645            strategy_id: Some(strategy_id),
7646            oms_type: Some(OmsType::Hedging),
7647            ..Default::default()
7648        }))
7649        .unwrap();
7650
7651        let instrument = crypto_perpetual_ethusdt();
7652        let instrument_id = instrument.id();
7653        let client_id = ClientId::from("STUB");
7654
7655        node.kernel
7656            .cache
7657            .borrow_mut()
7658            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7659            .unwrap();
7660        node.kernel
7661            .exec_engine
7662            .borrow_mut()
7663            .register_client(Box::new(StubExecutionClient::new(
7664                client_id,
7665                AccountId::from("TEST-ACCOUNT"),
7666                instrument_id.venue,
7667                OmsType::Netting,
7668                None,
7669            )))
7670            .unwrap();
7671
7672        let order = OrderTestBuilder::new(OrderType::Market)
7673            .trader_id(node.trader_id())
7674            .strategy_id(strategy_id)
7675            .instrument_id(instrument_id)
7676            .quantity(Quantity::from("1.000"))
7677            .build();
7678        let position_id = PositionId::new("CUSTOM-POSITION-001");
7679
7680        node.kernel
7681            .cache
7682            .borrow_mut()
7683            .add_order(order.clone(), Some(position_id), Some(client_id), true)
7684            .unwrap();
7685
7686        let submit_order = SubmitOrder::new(
7687            order.trader_id(),
7688            Some(client_id),
7689            strategy_id,
7690            instrument_id,
7691            order.client_order_id(),
7692            order.init_event().clone(),
7693            order.exec_algorithm_id(),
7694            Some(position_id),
7695            None,
7696            UUID4::new(),
7697            UnixNanos::default(),
7698            None,
7699        );
7700
7701        node.kernel
7702            .exec_engine
7703            .borrow()
7704            .execute(TradingCommand::SubmitOrder(submit_order));
7705
7706        let exec_engine = node.kernel.exec_engine.borrow();
7707        let cache = exec_engine.cache().borrow();
7708        let cached_order = cache
7709            .order(&order.client_order_id())
7710            .expect("Order should be cached");
7711
7712        assert_eq!(cached_order.status(), OrderStatus::Initialized);
7713    }
7714
7715    #[cfg(all(feature = "simulation", madsim))]
7716    async fn advance_clock(d: Duration) {
7717        madsim::time::advance(d);
7718        madsim::task::yield_now().await;
7719    }
7720
7721    #[cfg(not(all(feature = "simulation", madsim)))]
7722    async fn advance_clock(d: Duration) {
7723        tokio::time::advance(d).await;
7724    }
7725
7726    #[rstest]
7727    #[case::single_default(SubmissionRecoveryPolicy::ResolveLocally, false, None, false)]
7728    #[case::list_default(SubmissionRecoveryPolicy::ResolveLocally, true, None, false)]
7729    #[case::single_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, false, None, false)]
7730    #[case::list_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, true, None, false)]
7731    #[case::unapplied_cancel_default(
7732        SubmissionRecoveryPolicy::ResolveLocally,
7733        false,
7734        Some(OrderStatus::PendingCancel),
7735        false
7736    )]
7737    #[case::unapplied_modify_default(
7738        SubmissionRecoveryPolicy::ResolveLocally,
7739        false,
7740        Some(OrderStatus::PendingUpdate),
7741        false
7742    )]
7743    #[case::unapplied_cancel_diagnostics(
7744        SubmissionRecoveryPolicy::RetainUnresolved,
7745        false,
7746        Some(OrderStatus::PendingCancel),
7747        false
7748    )]
7749    #[case::unapplied_modify_diagnostics(
7750        SubmissionRecoveryPolicy::RetainUnresolved,
7751        false,
7752        Some(OrderStatus::PendingUpdate),
7753        false
7754    )]
7755    #[case::first_fill_cancel_default(
7756        SubmissionRecoveryPolicy::ResolveLocally,
7757        false,
7758        Some(OrderStatus::PendingCancel),
7759        true
7760    )]
7761    #[case::first_fill_modify_default(
7762        SubmissionRecoveryPolicy::ResolveLocally,
7763        false,
7764        Some(OrderStatus::PendingUpdate),
7765        true
7766    )]
7767    #[case::first_fill_cancel_diagnostics(
7768        SubmissionRecoveryPolicy::RetainUnresolved,
7769        false,
7770        Some(OrderStatus::PendingCancel),
7771        true
7772    )]
7773    #[case::first_fill_modify_diagnostics(
7774        SubmissionRecoveryPolicy::RetainUnresolved,
7775        false,
7776        Some(OrderStatus::PendingUpdate),
7777        true
7778    )]
7779    #[cfg_attr(
7780        not(all(feature = "simulation", madsim)),
7781        tokio::test(start_paused = true)
7782    )]
7783    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7784    async fn test_submission_registry_publishes_after_native_resolution(
7785        #[case] policy: SubmissionRecoveryPolicy,
7786        #[case] order_list: bool,
7787        #[case] pending_status: Option<OrderStatus>,
7788        #[case] first_fill: bool,
7789    ) {
7790        let config = LiveNodeConfig {
7791            exec_engine: crate::config::LiveExecutionEngineConfig {
7792                inflight_check_threshold_ms: 100,
7793                inflight_check_retries: 2,
7794                submission_recovery_policy: policy,
7795                ..Default::default()
7796            },
7797            ..Default::default()
7798        };
7799        let mut node = LiveNode::build("SubmissionRegistryNode".to_string(), Some(config)).unwrap();
7800        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7801        let client_id = ClientId::from("TEST-SUBMISSION");
7802        let account_id = AccountId::from("TEST-001");
7803        let strategy_id = StrategyId::from("SUBMISSION-001");
7804        let account = AccountAny::Margin(MarginAccount::new(
7805            AccountState::new(
7806                account_id,
7807                AccountType::Margin,
7808                vec![AccountBalance::new(
7809                    Money::from("1000000 USDT"),
7810                    Money::from("0 USDT"),
7811                    Money::from("1000000 USDT"),
7812                )],
7813                Vec::new(),
7814                true,
7815                UUID4::new(),
7816                UnixNanos::default(),
7817                UnixNanos::default(),
7818                Some(Currency::USDT()),
7819            ),
7820            true,
7821        ));
7822        node.kernel.cache.borrow_mut().add_account(account).unwrap();
7823        node.kernel
7824            .cache
7825            .borrow_mut()
7826            .add_instrument(instrument.clone())
7827            .unwrap();
7828        let orders: Vec<_> = (0..if order_list { 2 } else { 1 })
7829            .map(|index| {
7830                OrderTestBuilder::new(OrderType::Limit)
7831                    .trader_id(node.trader_id())
7832                    .strategy_id(strategy_id)
7833                    .client_order_id(ClientOrderId::from(
7834                        format!("O-SUBMISSION-{index}").as_str(),
7835                    ))
7836                    .instrument_id(instrument.id())
7837                    .quantity(Quantity::from("1.000"))
7838                    .price(Price::from("100.00"))
7839                    .build()
7840            })
7841            .collect();
7842        let command = if order_list {
7843            TradingCommand::SubmitOrderList(SubmitOrderList::new(
7844                node.trader_id(),
7845                Some(client_id),
7846                strategy_id,
7847                OrderList::new(
7848                    OrderListId::from("OL-SUBMISSION"),
7849                    instrument.id(),
7850                    strategy_id,
7851                    orders.iter().map(Order::client_order_id).collect(),
7852                    UnixNanos::default(),
7853                ),
7854                orders
7855                    .iter()
7856                    .map(|order| order.init_event().clone())
7857                    .collect(),
7858                None,
7859                None,
7860                None,
7861                UUID4::new(),
7862                UnixNanos::default(),
7863                None,
7864            ))
7865        } else {
7866            TradingCommand::SubmitOrder(SubmitOrder::from_order(
7867                &orders[0],
7868                node.trader_id(),
7869                Some(client_id),
7870                None,
7871                UUID4::new(),
7872                UnixNanos::default(),
7873            ))
7874        };
7875        node.observe_exec_command_before_dispatch(&command);
7876
7877        for order in &orders {
7878            let submitted = TestOrderEventStubs::submitted(order, account_id);
7879            node.kernel
7880                .cache
7881                .borrow_mut()
7882                .add_order(order.clone(), None, Some(client_id), false)
7883                .unwrap();
7884            node.process_exec_event(ExecutionEvent::Order(submitted));
7885        }
7886
7887        let expected_status = match pending_status {
7888            Some(status) if !first_fill => status,
7889            Some(_) => OrderStatus::Canceled,
7890            None if policy == SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Submitted,
7891            None => OrderStatus::Rejected,
7892        };
7893        let diagnostics = Rc::new(RefCell::new(Vec::new()));
7894        let handler = ShareableMessageHandler::from_typed({
7895            let diagnostics = diagnostics.clone();
7896            let cache = node.kernel.cache.clone();
7897            move |diagnostic: &SubmissionRecoveryExhausted| {
7898                let cache = cache.borrow_mut();
7899                let order = cache.order(&diagnostic.client_order_id).unwrap();
7900                assert_eq!(order.status(), expected_status);
7901                assert!(diagnostic.ts_event >= order.ts_last());
7902                diagnostics.borrow_mut().push(diagnostic.clone());
7903            }
7904        });
7905        let topic = MessagingSwitchboard::submission_recovery_exhausted_topic();
7906        msgbus::subscribe_any(topic.into(), handler.clone(), None);
7907        let queries = Rc::new(RefCell::new(Vec::new()));
7908        msgbus::register_trading_command_endpoint(
7909            MessagingSwitchboard::exec_engine_execute(),
7910            TypedIntoHandler::from({
7911                let queries = queries.clone();
7912                move |command: TradingCommand| queries.borrow_mut().push(command)
7913            }),
7914        );
7915
7916        let mut last_inflight_check = dst::time::Instant::now();
7917        let mut last_open_check = last_inflight_check;
7918        let mut last_position_check = last_inflight_check;
7919        let mut open_order_report_task = None;
7920        let mut targeted_order_report_task = None;
7921        let mut position_report_task = None;
7922
7923        for check in 1..=3 {
7924            advance_clock(Duration::from_millis(101)).await;
7925            node.run_reconciliation_checks(
7926                dst::time::Instant::now(),
7927                ReconciliationCheckIntervals {
7928                    inflight: Duration::from_millis(100),
7929                    open: Duration::ZERO,
7930                    position: Duration::ZERO,
7931                },
7932                &mut ReconciliationCheckState {
7933                    last_inflight_check: &mut last_inflight_check,
7934                    last_open_check: &mut last_open_check,
7935                    last_position_check: &mut last_position_check,
7936                    open_order_report_task: &mut open_order_report_task,
7937                    targeted_order_report_task: &mut targeted_order_report_task,
7938                    position_report_task: &mut position_report_task,
7939                },
7940            );
7941
7942            if check == 1 {
7943                assert!(diagnostics.borrow().is_empty());
7944
7945                if let Some(pending_status) = pending_status {
7946                    let order = &orders[0];
7947                    let pending = if pending_status == OrderStatus::PendingCancel {
7948                        OrderEventAny::PendingCancel(
7949                            OrderPendingCancelSpec::builder()
7950                                .trader_id(node.trader_id())
7951                                .strategy_id(strategy_id)
7952                                .instrument_id(instrument.id())
7953                                .client_order_id(order.client_order_id())
7954                                .account_id(account_id)
7955                                .build(),
7956                        )
7957                    } else {
7958                        OrderEventAny::PendingUpdate(
7959                            OrderPendingUpdateSpec::builder()
7960                                .trader_id(node.trader_id())
7961                                .strategy_id(strategy_id)
7962                                .instrument_id(instrument.id())
7963                                .client_order_id(order.client_order_id())
7964                                .account_id(account_id)
7965                                .build(),
7966                        )
7967                    };
7968                    node.process_exec_event(ExecutionEvent::Order(pending));
7969                    node.exec_manager
7970                        .register_inflight(order.client_order_id(), UUID4::new());
7971
7972                    if first_fill {
7973                        let cached = node
7974                            .kernel
7975                            .cache
7976                            .borrow()
7977                            .order(&order.client_order_id())
7978                            .unwrap()
7979                            .clone();
7980                        let OrderEventAny::Filled(mut fill) =
7981                            OrderFilledTestBuilder::new(&cached, &instrument)
7982                                .last_qty(Quantity::from("0.400"))
7983                                .last_px(Price::from("100.00"))
7984                                .commission(Money::from("0 USDT"))
7985                                .without_position_id()
7986                                .build()
7987                        else {
7988                            unreachable!()
7989                        };
7990                        fill.venue_order_id = VenueOrderId::from("V-SUBMISSION");
7991                        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
7992                    } else {
7993                        let report = OrderStatusReport::new(
7994                            account_id,
7995                            instrument.id(),
7996                            Some(order.client_order_id()),
7997                            VenueOrderId::from("V-SUBMISSION"),
7998                            Some(order.order_side()),
7999                            order.order_type(),
8000                            order.time_in_force(),
8001                            OrderStatus::Submitted,
8002                            order.quantity(),
8003                            Quantity::zero(3),
8004                            UnixNanos::default(),
8005                            UnixNanos::default(),
8006                            UnixNanos::default(),
8007                            None,
8008                        )
8009                        .with_price(Price::from("100.00"));
8010                        node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(
8011                            Box::new(report),
8012                        )));
8013                    }
8014                    let cache = node.kernel.cache.borrow();
8015                    let cached = cache.order(&order.client_order_id()).unwrap();
8016                    assert_eq!(cached.status(), pending_status);
8017                    assert_eq!(
8018                        cached.filled_qty(),
8019                        if first_fill {
8020                            Quantity::from("0.400")
8021                        } else {
8022                            Quantity::zero(3)
8023                        }
8024                    );
8025                    assert!(
8026                        !cached
8027                            .events()
8028                            .iter()
8029                            .any(|event| matches!(event, OrderEventAny::Accepted(_)))
8030                    );
8031                }
8032            }
8033        }
8034
8035        assert_eq!(
8036            queries.borrow().len(),
8037            orders.len()
8038                + usize::from(first_fill && policy == SubmissionRecoveryPolicy::ResolveLocally)
8039        );
8040
8041        for (command, order) in queries.borrow().iter().zip(orders.iter().cycle()) {
8042            assert!(matches!(command, TradingCommand::QueryOrder(query)
8043                if query.client_order_id == order.client_order_id()
8044                && query.client_id == Some(client_id)));
8045            assert_eq!(
8046                node.kernel
8047                    .cache
8048                    .borrow()
8049                    .order(&order.client_order_id())
8050                    .unwrap()
8051                    .status(),
8052                if first_fill && policy == SubmissionRecoveryPolicy::RetainUnresolved {
8053                    pending_status.unwrap()
8054                } else {
8055                    expected_status
8056                }
8057            );
8058        }
8059        let diagnostics = diagnostics.borrow();
8060        assert_eq!(
8061            diagnostics.len(),
8062            if policy == SubmissionRecoveryPolicy::RetainUnresolved && pending_status.is_none() {
8063                orders.len()
8064            } else {
8065                0
8066            }
8067        );
8068
8069        for (diagnostic, order) in diagnostics.iter().zip(&orders) {
8070            assert_eq!(diagnostic.trader_id, node.trader_id());
8071            assert_eq!(diagnostic.client_id, Some(client_id));
8072            assert_eq!(diagnostic.strategy_id, strategy_id);
8073            assert_eq!(diagnostic.instrument_id, instrument.id());
8074            assert_eq!(diagnostic.client_order_id, order.client_order_id());
8075            assert_eq!(diagnostic.source, SubmissionRecoverySource::Inflight);
8076            assert_eq!(diagnostic.retry_count, 2);
8077        }
8078        msgbus::unsubscribe_any(topic.into(), &handler);
8079        ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
8080    }
8081
8082    #[derive(Debug)]
8083    struct SubmissionCallbackStrategy {
8084        core: StrategyCore,
8085        cancel: bool,
8086        batch: bool,
8087        callback: &'static str,
8088        client_id: ClientId,
8089    }
8090
8091    impl DataActor for SubmissionCallbackStrategy {}
8092
8093    impl SubmissionCallbackStrategy {
8094        fn queue_command(&mut self, client_order_id: ClientOrderId) {
8095            match (self.cancel, self.batch) {
8096                (true, true) => {
8097                    self.cancel_orders(vec![client_order_id], Some(self.client_id), None)
8098                        .unwrap();
8099                }
8100                (true, false) => {
8101                    self.cancel_order(client_order_id, Some(self.client_id), None)
8102                        .unwrap();
8103                }
8104                (false, true) => {
8105                    self.modify_orders(
8106                        vec![(client_order_id, None, Some(Price::from("101.00")), None)],
8107                        Some(self.client_id),
8108                        None,
8109                    )
8110                    .unwrap();
8111                }
8112                (false, false) => self
8113                    .modify_order(
8114                        client_order_id,
8115                        None,
8116                        Some(Price::from("101.00")),
8117                        None,
8118                        Some(self.client_id),
8119                        None,
8120                    )
8121                    .unwrap(),
8122            }
8123        }
8124    }
8125
8126    nautilus_strategy!(SubmissionCallbackStrategy, {
8127        fn on_order_accepted(&mut self, event: OrderAccepted) {
8128            if self.callback == "accepted" {
8129                self.queue_command(event.client_order_id);
8130            }
8131        }
8132
8133        fn on_order_updated(&mut self, event: OrderUpdated) {
8134            if self.callback == "updated" {
8135                self.queue_command(event.client_order_id);
8136            }
8137        }
8138    });
8139
8140    #[rstest]
8141    #[case::native(false, "accepted")]
8142    #[case::report(true, "accepted")]
8143    #[case::updated(false, "updated")]
8144    #[cfg_attr(
8145        not(all(feature = "simulation", madsim)),
8146        tokio::test(start_paused = true)
8147    )]
8148    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8149    async fn test_submission_registry_ack_callback_starts_command_budget_at_dispatch(
8150        #[values(
8151            SubmissionRecoveryPolicy::ResolveLocally,
8152            SubmissionRecoveryPolicy::RetainUnresolved
8153        )]
8154        policy: SubmissionRecoveryPolicy,
8155        #[values(false, true)] cancel: bool,
8156        #[values(false, true)] batch: bool,
8157        #[case] report: bool,
8158        #[case] callback: &'static str,
8159    ) {
8160        let config = LiveNodeConfig {
8161            exec_engine: crate::config::LiveExecutionEngineConfig {
8162                inflight_check_threshold_ms: 100,
8163                inflight_check_retries: 2,
8164                submission_recovery_policy: policy,
8165                ..Default::default()
8166            },
8167            ..Default::default()
8168        };
8169        let mut node = LiveNode::build("SubmissionCallbackNode".to_string(), Some(config)).unwrap();
8170        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8171        let account_id = AccountId::from("TEST-001");
8172        let strategy_id = StrategyId::from("CALLBACK-001");
8173        let client_id = ClientId::from("TEST-CALLBACK");
8174        let account = AccountAny::Margin(MarginAccount::new(
8175            AccountState::new(
8176                account_id,
8177                AccountType::Margin,
8178                vec![AccountBalance::new(
8179                    Money::from("1000000 USDT"),
8180                    Money::from("0 USDT"),
8181                    Money::from("1000000 USDT"),
8182                )],
8183                Vec::new(),
8184                true,
8185                UUID4::new(),
8186                UnixNanos::default(),
8187                UnixNanos::default(),
8188                Some(Currency::USDT()),
8189            ),
8190            true,
8191        ));
8192        node.kernel.cache.borrow_mut().add_account(account).unwrap();
8193        node.kernel
8194            .cache
8195            .borrow_mut()
8196            .add_instrument(instrument.clone())
8197            .unwrap();
8198        node.kernel
8199            .exec_engine
8200            .borrow_mut()
8201            .register_client(Box::new(StubExecutionClient::new(
8202                client_id,
8203                account_id,
8204                instrument.id().venue,
8205                OmsType::Netting,
8206                None,
8207            )))
8208            .unwrap();
8209        node.add_strategy(SubmissionCallbackStrategy {
8210            core: StrategyCore::new(StrategyConfig {
8211                strategy_id: Some(strategy_id),
8212                ..Default::default()
8213            }),
8214            cancel,
8215            batch,
8216            callback,
8217            client_id,
8218        })
8219        .unwrap();
8220        node.kernel.trader.borrow_mut().initialize().unwrap();
8221        node.kernel.start_trader().unwrap();
8222        let runner = node.runner.take().unwrap();
8223        runner.bind_senders_for_node(node.handle.clone());
8224        let mut channels = runner.take_channels();
8225        let order = OrderTestBuilder::new(OrderType::Limit)
8226            .trader_id(node.trader_id())
8227            .strategy_id(strategy_id)
8228            .client_order_id(ClientOrderId::from("O-CALLBACK"))
8229            .instrument_id(instrument.id())
8230            .quantity(Quantity::from("1.000"))
8231            .price(Price::from("100.00"))
8232            .build();
8233        let client_order_id = order.client_order_id();
8234        let submit = TradingCommand::SubmitOrder(SubmitOrder::from_order(
8235            &order,
8236            node.trader_id(),
8237            Some(client_id),
8238            None,
8239            UUID4::new(),
8240            UnixNanos::default(),
8241        ));
8242        node.observe_exec_command_before_dispatch(&submit);
8243        node.kernel
8244            .cache
8245            .borrow_mut()
8246            .add_order(order.clone(), None, Some(client_id), false)
8247            .unwrap();
8248        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
8249            &order, account_id,
8250        )));
8251        advance_clock(Duration::from_millis(101)).await;
8252        let first = node.exec_manager.check_inflight_orders();
8253        assert_eq!(first.queries.len(), 1);
8254        assert!(first.events.is_empty());
8255
8256        advance_clock(Duration::from_millis(101)).await;
8257        let accepted =
8258            TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-CALLBACK"));
8259
8260        if callback == "updated" {
8261            let mut update = OrderUpdatedSpec::builder()
8262                .trader_id(node.trader_id())
8263                .strategy_id(strategy_id)
8264                .instrument_id(instrument.id())
8265                .client_order_id(client_order_id)
8266                .account_id(account_id)
8267                .quantity(order.quantity())
8268                .price(order.price().unwrap())
8269                .build();
8270            update.venue_order_id = Some(VenueOrderId::from("V-CALLBACK"));
8271            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
8272        } else if report {
8273            let accepted = OrderStatusReport::new(
8274                account_id,
8275                instrument.id(),
8276                Some(client_order_id),
8277                VenueOrderId::from("V-CALLBACK"),
8278                Some(order.order_side()),
8279                order.order_type(),
8280                order.time_in_force(),
8281                OrderStatus::Accepted,
8282                order.quantity(),
8283                Quantity::zero(3),
8284                UnixNanos::default(),
8285                UnixNanos::default(),
8286                UnixNanos::default(),
8287                None,
8288            )
8289            .with_price(Price::from("100.00"));
8290            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
8291                accepted,
8292            ))));
8293        } else {
8294            node.process_exec_event(ExecutionEvent::Order(accepted));
8295        }
8296
8297        let pending_status = if cancel {
8298            OrderStatus::PendingCancel
8299        } else {
8300            OrderStatus::PendingUpdate
8301        };
8302        assert_eq!(channels.exec_cmd_rx.len(), 1);
8303        assert_eq!(
8304            node.kernel
8305                .cache
8306                .borrow()
8307                .order(&client_order_id)
8308                .unwrap()
8309                .status(),
8310            pending_status
8311        );
8312
8313        // Maintenance may win the runner selection before the queued command dispatches
8314        for _ in 0..3 {
8315            advance_clock(Duration::from_millis(101)).await;
8316            let maintenance = node.exec_manager.check_inflight_orders();
8317            assert!(maintenance.events.is_empty());
8318            assert!(maintenance.queries.is_empty());
8319            assert!(
8320                node.exec_manager
8321                    .take_submission_recovery_exhaustions()
8322                    .is_empty()
8323            );
8324        }
8325
8326        let command = channels.exec_cmd_rx.try_recv().unwrap();
8327        node.process_exec_command(command);
8328        assert!(channels.exec_cmd_rx.is_empty());
8329        let immediate = node.exec_manager.check_inflight_orders();
8330        assert!(immediate.events.is_empty());
8331        assert!(immediate.queries.is_empty());
8332        advance_clock(Duration::from_millis(101)).await;
8333        let first_command_check = node.exec_manager.check_inflight_orders();
8334        assert!(first_command_check.events.is_empty());
8335        assert_eq!(first_command_check.queries.len(), 1);
8336        advance_clock(Duration::from_millis(101)).await;
8337        let exhausted = node.exec_manager.check_inflight_orders();
8338        assert!(exhausted.queries.is_empty());
8339        assert!(
8340            matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
8341            if event.client_order_id == client_order_id)
8342        );
8343        assert!(
8344            node.exec_manager
8345                .take_submission_recovery_exhaustions()
8346                .is_empty()
8347        );
8348        node.process_reconciliation_events(&exhausted.events);
8349        assert_eq!(
8350            node.kernel
8351                .cache
8352                .borrow()
8353                .order(&client_order_id)
8354                .unwrap()
8355                .status(),
8356            OrderStatus::Canceled
8357        );
8358    }
8359
8360    #[cfg_attr(
8361        not(all(feature = "simulation", madsim)),
8362        tokio::test(start_paused = true)
8363    )]
8364    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8365    async fn test_reconciliation_check_due_uses_monotonic_elapsed_time() {
8366        let last = dst::time::Instant::now();
8367        let interval = Duration::from_millis(100);
8368
8369        assert!(!reconciliation_check_due(last, last, Duration::ZERO));
8370        assert!(!reconciliation_check_due(last, last, interval));
8371
8372        advance_clock(Duration::from_millis(99)).await;
8373        let before_interval = dst::time::Instant::now();
8374        assert!(!reconciliation_check_due(before_interval, last, interval));
8375
8376        advance_clock(Duration::from_millis(1)).await;
8377        let at_interval = dst::time::Instant::now();
8378        assert!(reconciliation_check_due(at_interval, last, interval));
8379
8380        assert!(!reconciliation_check_due(last, at_interval, interval));
8381    }
8382
8383    #[cfg_attr(
8384        not(all(feature = "simulation", madsim)),
8385        tokio::test(start_paused = true)
8386    )]
8387    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8388    async fn test_run_reconciliation_checks_does_not_publish_open_order_queries() {
8389        let config = LiveNodeConfig {
8390            exec_engine: crate::config::LiveExecutionEngineConfig {
8391                reconciliation: true,
8392                open_check_interval_secs: Some(1.0),
8393                position_check_interval_secs: Some(1.0),
8394                max_single_order_queries_per_cycle: 5,
8395                ..Default::default()
8396            },
8397            ..Default::default()
8398        };
8399
8400        let mut node =
8401            LiveNode::build("ReconciliationFallbackNode".to_string(), Some(config)).unwrap();
8402        let client_id = ClientId::from("TEST-QUERY");
8403        let account_id = AccountId::from("TEST-QUERY-001");
8404
8405        let trading_commands = Rc::new(RefCell::new(Vec::new()));
8406        msgbus::register_trading_command_endpoint(
8407            MessagingSwitchboard::exec_engine_execute(),
8408            TypedIntoHandler::from({
8409                let trading_commands = trading_commands.clone();
8410                move |command: TradingCommand| {
8411                    trading_commands.borrow_mut().push(command);
8412                }
8413            }),
8414        );
8415
8416        let venue_order_id = VenueOrderId::from("V-NODE-QUERY-001");
8417        let instrument = crypto_perpetual_ethusdt();
8418        let instrument_id = instrument.id();
8419        let client_order_id = ClientOrderId::from("O-NODE-QUERY-001");
8420
8421        node.kernel
8422            .cache
8423            .borrow_mut()
8424            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
8425            .unwrap();
8426        insert_accepted_limit_order_in_node(
8427            &node,
8428            account_id,
8429            client_id,
8430            instrument_id,
8431            client_order_id,
8432            venue_order_id,
8433        );
8434
8435        let last = dst::time::Instant::now();
8436        advance_clock(Duration::from_nanos(1)).await;
8437        let now = dst::time::Instant::now();
8438        let mut last_inflight_check = last;
8439        let mut last_open_check = last;
8440        let mut last_position_check = last;
8441        let mut open_order_report_task = None;
8442        let mut targeted_order_report_task = None;
8443        let mut position_report_task = None;
8444
8445        node.run_reconciliation_checks(
8446            now,
8447            ReconciliationCheckIntervals {
8448                inflight: Duration::ZERO,
8449                open: Duration::from_nanos(1),
8450                position: Duration::ZERO,
8451            },
8452            &mut ReconciliationCheckState {
8453                last_inflight_check: &mut last_inflight_check,
8454                last_open_check: &mut last_open_check,
8455                last_position_check: &mut last_position_check,
8456                open_order_report_task: &mut open_order_report_task,
8457                targeted_order_report_task: &mut targeted_order_report_task,
8458                position_report_task: &mut position_report_task,
8459            },
8460        );
8461
8462        let commands = trading_commands.borrow();
8463
8464        assert!(commands.is_empty());
8465        assert!(open_order_report_task.is_none());
8466        assert!(targeted_order_report_task.is_none());
8467        assert!(position_report_task.is_none());
8468
8469        ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
8470    }
8471
8472    #[rstest]
8473    #[tokio::test]
8474    async fn test_recurring_report_failure_preserves_client_coverage(
8475        #[values(false, true)] positions: bool,
8476        #[values(false, true)] failure_first: bool,
8477    ) {
8478        let mut node = LiveNode::build("ReportFailureNode".to_string(), None).unwrap();
8479        let failed_id = ClientId::from("FAILED");
8480        let healthy_id = ClientId::from("HEALTHY");
8481
8482        for (client_id, outcome) in [
8483            (failed_id, FillReportClientOutcome::Failure),
8484            (healthy_id, FillReportClientOutcome::Reports(Vec::new())),
8485        ] {
8486            node.exec_clients
8487                .push(LiveExecutionClient::new(Box::new(FillReportClient {
8488                    client_id,
8489                    account_id: AccountId::from("TEST-001"),
8490                    venue: Venue::from("BINANCE"),
8491                    outcome,
8492                    commands: Rc::new(RefCell::new(Vec::new())),
8493                })));
8494        }
8495
8496        if !failure_first {
8497            node.exec_clients.reverse();
8498        }
8499
8500        let last = dst::time::Instant::now();
8501        let now = last + Duration::from_secs(1);
8502        let mut last_inflight = last;
8503        let mut last_open = last;
8504        let mut last_position = last;
8505        let mut open_task = None;
8506        let mut targeted_task = None;
8507        let mut position_task = None;
8508
8509        node.run_reconciliation_checks(
8510            now,
8511            ReconciliationCheckIntervals {
8512                inflight: Duration::ZERO,
8513                open: if positions {
8514                    Duration::ZERO
8515                } else {
8516                    Duration::from_secs(1)
8517                },
8518                position: if positions {
8519                    Duration::from_secs(1)
8520                } else {
8521                    Duration::ZERO
8522                },
8523            },
8524            &mut ReconciliationCheckState {
8525                last_inflight_check: &mut last_inflight,
8526                last_open_check: &mut last_open,
8527                last_position_check: &mut last_position,
8528                open_order_report_task: &mut open_task,
8529                targeted_order_report_task: &mut targeted_task,
8530                position_report_task: &mut position_task,
8531            },
8532        );
8533
8534        let (queried_clients, failed_clients) = if positions {
8535            let ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) =
8536                position_task.unwrap().future.await
8537            else {
8538                panic!("position collection should complete despite one client failure");
8539            };
8540
8541            let result = result.finish();
8542            assert!(result.reports.is_empty());
8543            assert!(open_task.is_none());
8544            (result.queried_clients, result.failed_clients)
8545        } else {
8546            let ReportTaskOutcome::Completed(result) = open_task.unwrap().future.await else {
8547                panic!("order collection should complete despite one client failure");
8548            };
8549
8550            let result = result.finish();
8551
8552            assert!(result.reports.is_empty());
8553            assert!(position_task.is_none());
8554            (result.queried_clients, result.failed_clients)
8555        };
8556
8557        assert_eq!(queried_clients, IndexSet::from([failed_id, healthy_id]));
8558        assert_eq!(failed_clients, IndexSet::from([failed_id]));
8559        assert!(targeted_task.is_none());
8560    }
8561
8562    #[rstest]
8563    #[case::inflight(true, false)]
8564    #[case::open(false, false)]
8565    #[case::positions(false, true)]
8566    #[tokio::test]
8567    async fn test_reconciliation_shutdown_preserves_pending_checks(
8568        #[case] inflight: bool,
8569        #[case] positions: bool,
8570    ) {
8571        let mut node = LiveNode::build("ShutdownChecksNode".to_string(), None).unwrap();
8572        node.handle.set_shutting_down();
8573        let last = dst::time::Instant::now();
8574        let now = last + Duration::from_secs(1);
8575        let mut last_inflight = last;
8576        let mut last_open = last;
8577        let mut last_position = last;
8578        let mut open_task = None;
8579        let mut targeted_task = None;
8580        let mut position_task = None;
8581
8582        node.run_reconciliation_checks(
8583            now,
8584            ReconciliationCheckIntervals {
8585                inflight: if inflight {
8586                    Duration::from_secs(1)
8587                } else {
8588                    Duration::ZERO
8589                },
8590                open: if !inflight && !positions {
8591                    Duration::from_secs(1)
8592                } else {
8593                    Duration::ZERO
8594                },
8595                position: if positions {
8596                    Duration::from_secs(1)
8597                } else {
8598                    Duration::ZERO
8599                },
8600            },
8601            &mut ReconciliationCheckState {
8602                last_inflight_check: &mut last_inflight,
8603                last_open_check: &mut last_open,
8604                last_position_check: &mut last_position,
8605                open_order_report_task: &mut open_task,
8606                targeted_order_report_task: &mut targeted_task,
8607                position_report_task: &mut position_task,
8608            },
8609        );
8610
8611        assert_eq!(
8612            (last_inflight, last_open, last_position),
8613            (last, last, last)
8614        );
8615        assert!(open_task.is_none());
8616        assert!(targeted_task.is_none());
8617        assert!(position_task.is_none());
8618    }
8619
8620    fn insert_accepted_limit_order_in_node(
8621        node: &LiveNode,
8622        account_id: AccountId,
8623        client_id: ClientId,
8624        instrument_id: InstrumentId,
8625        client_order_id: ClientOrderId,
8626        venue_order_id: VenueOrderId,
8627    ) {
8628        let order = OrderTestBuilder::new(OrderType::Limit)
8629            .client_order_id(client_order_id)
8630            .instrument_id(instrument_id)
8631            .quantity(Quantity::from("10.0"))
8632            .price(Price::from("100.0"))
8633            .build();
8634        let submitted = TestOrderEventStubs::submitted(&order, account_id);
8635        node.kernel
8636            .cache
8637            .borrow_mut()
8638            .add_order(order, None, Some(client_id), false)
8639            .unwrap();
8640        let order = node
8641            .kernel
8642            .cache
8643            .borrow_mut()
8644            .update_order(&submitted)
8645            .unwrap();
8646        let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
8647        node.kernel
8648            .cache
8649            .borrow_mut()
8650            .update_order(&accepted)
8651            .unwrap();
8652    }
8653
8654    fn recent_fill_test_fixture(name: &str) -> (LiveNode, OrderEventAny, InstrumentAny) {
8655        let config = LiveNodeConfig {
8656            exec_engine: crate::config::LiveExecutionEngineConfig {
8657                reconciliation: true,
8658                ..Default::default()
8659            },
8660            ..Default::default()
8661        };
8662
8663        let node = LiveNode::build(name.to_string(), Some(config)).unwrap();
8664        let account_id = AccountId::from("TEST-001");
8665        let client_id = ClientId::from("TEST-RECENT-FILL");
8666        let client_order_id = ClientOrderId::from("O-RECENT-FILL");
8667        let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
8668        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8669        node.kernel
8670            .cache
8671            .borrow_mut()
8672            .add_instrument(instrument.clone())
8673            .unwrap();
8674        insert_accepted_limit_order_in_node(
8675            &node,
8676            account_id,
8677            client_id,
8678            instrument.id(),
8679            client_order_id,
8680            venue_order_id,
8681        );
8682        let order = node
8683            .kernel
8684            .cache
8685            .borrow()
8686            .order_owned(&client_order_id)
8687            .unwrap();
8688        let fill = TestOrderEventStubs::filled(
8689            &order,
8690            &instrument,
8691            Some(TradeId::from("T-RECENT-FILL")),
8692            None,
8693            Some(Price::from("100.0")),
8694            Some(Quantity::from("1.0")),
8695            Some(LiquiditySide::Taker),
8696            None,
8697            None,
8698            Some(account_id),
8699        );
8700
8701        (node, fill, instrument)
8702    }
8703
8704    fn apply_inferred_position_fill(node: &mut LiveNode, fill_report: &FillReport) {
8705        let client_order_id = fill_report.client_order_id.unwrap();
8706        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8707        let order = node
8708            .kernel
8709            .cache
8710            .borrow()
8711            .order_owned(&client_order_id)
8712            .unwrap();
8713        let order_report = OrderStatusReport::new(
8714            fill_report.account_id,
8715            fill_report.instrument_id,
8716            Some(client_order_id),
8717            fill_report.venue_order_id,
8718            OrderSide::Buy.into(),
8719            OrderType::Limit,
8720            TimeInForce::Gtc,
8721            OrderStatus::PartiallyFilled,
8722            Quantity::from("10.0"),
8723            Quantity::from("2.0"),
8724            UnixNanos::from(1_000),
8725            UnixNanos::from(1_000),
8726            UnixNanos::from(1_000),
8727            None,
8728        )
8729        .with_avg_px(dec!(100.0));
8730        let inferred = create_inferred_fill_for_qty(
8731            &order,
8732            &order_report,
8733            &fill_report.account_id,
8734            &instrument,
8735            Quantity::from("1.0"),
8736            UnixNanos::from(1_000),
8737            None,
8738        )
8739        .unwrap();
8740
8741        node.process_reconciliation_events(&[inferred]);
8742    }
8743
8744    #[rstest]
8745    #[case::rejected(false, "10.0", "10.0", "1.0")]
8746    #[case::dust(true, "9.004", "10.004", "10.004")]
8747    #[case::price_improvement(true, "12.5", "13.5", "13.5")]
8748    fn test_position_check_overfill_raises_before_fill_when_client_allows(
8749        #[case] allow: bool,
8750        #[case] fill_qty: &str,
8751        #[case] expected_qty: &str,
8752        #[case] expected_filled: &str,
8753    ) {
8754        let (mut node, venue_report, mut fill_report) =
8755            position_fill_test_fixture("PositionOverfillRaise", Quantity::from(fill_qty));
8756        fill_report.commission = Money::from("1.25 USDT");
8757        let key = (fill_report.instrument_id, fill_report.account_id);
8758
8759        let client = StubExecutionClient::new(
8760            ClientId::from("POSITION-FILLS"),
8761            AccountId::from("TEST-001"),
8762            fill_report.instrument_id.venue,
8763            OmsType::Netting,
8764            None,
8765        );
8766
8767        let client = if allow {
8768            client.with_allows_reconciliation_overfill()
8769        } else {
8770            client
8771        };
8772
8773        node.kernel
8774            .exec_engine
8775            .borrow_mut()
8776            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8777            .unwrap();
8778
8779        let position_result = position_report_result(&node, venue_report);
8780        node.handle_position_fill_report_result(PositionFillReportResult {
8781            position_result,
8782            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
8783        });
8784
8785        let order = node
8786            .kernel
8787            .cache
8788            .borrow()
8789            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8790            .unwrap();
8791        assert_eq!(order.quantity(), Quantity::from(expected_qty));
8792        assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8793        let applied = order
8794            .trade_ids()
8795            .iter()
8796            .any(|trade_id| trade_id.as_str() == "T-POSITION-AUTHORITATIVE");
8797        assert_eq!(applied, allow);
8798        let revision = node.exec_manager.position_activity_revision(&key);
8799
8800        if !allow {
8801            return;
8802        }
8803
8804        assert_eq!(revision, 2);
8805
8806        {
8807            let cache = node.kernel.cache.borrow();
8808            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8809            assert_eq!(positions.len(), 1);
8810            assert_eq!(positions[0].quantity, Quantity::from(expected_qty));
8811            assert_eq!(positions[0].commissions(), vec![Money::from("1.25 USDT")]);
8812            assert_eq!(positions[0].trade_ids.len(), 2);
8813        }
8814
8815        let replay_report = position_report_result(
8816            &node,
8817            PositionStatusReport::new(
8818                key.1,
8819                key.0,
8820                PositionSide::Long,
8821                Quantity::from(expected_qty),
8822                UnixNanos::from(2_000),
8823                UnixNanos::from(2_000),
8824                None,
8825                None,
8826                Some(dec!(100.0)),
8827            ),
8828        );
8829        node.handle_position_fill_report_result(PositionFillReportResult {
8830            position_result: replay_report,
8831            reports: IndexMap::from([(key, vec![fill_report])]),
8832        });
8833
8834        let order = node
8835            .kernel
8836            .cache
8837            .borrow()
8838            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8839            .unwrap();
8840        assert_eq!(order.quantity(), Quantity::from(expected_qty));
8841        assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8842        assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
8843    }
8844
8845    #[rstest]
8846    fn test_position_check_applies_fitting_fill_then_overfill_in_one_cycle() {
8847        let (mut node, venue_report, mut overfill) =
8848            position_fill_test_fixture("PositionSplitOverfill", Quantity::from("2.0"));
8849        overfill.commission = Money::from("0.20 USDT");
8850        let key = (overfill.instrument_id, overfill.account_id);
8851        let client = StubExecutionClient::new(
8852            ClientId::from("POSITION-FILLS"),
8853            AccountId::from("TEST-001"),
8854            overfill.instrument_id.venue,
8855            OmsType::Netting,
8856            None,
8857        )
8858        .with_allows_reconciliation_overfill();
8859        node.kernel
8860            .exec_engine
8861            .borrow_mut()
8862            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8863            .unwrap();
8864
8865        let mut fitting = overfill.clone();
8866        fitting.trade_id = TradeId::from("T-POSITION-FITTING");
8867        fitting.last_qty = Quantity::from("8.0");
8868        fitting.commission = Money::from("0.80 USDT");
8869        let position_result = position_report_result(&node, venue_report);
8870        node.handle_position_fill_report_result(PositionFillReportResult {
8871            position_result,
8872            reports: IndexMap::from([(key, vec![fitting, overfill])]),
8873        });
8874
8875        let order = node
8876            .kernel
8877            .cache
8878            .borrow()
8879            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8880            .unwrap();
8881        assert_eq!(order.quantity(), Quantity::from("11.0"));
8882        assert_eq!(order.filled_qty(), Quantity::from("11.0"));
8883        assert_eq!(order.trade_ids().len(), 3);
8884        assert_eq!(node.exec_manager.position_activity_revision(&key), 3);
8885        {
8886            let cache = node.kernel.cache.borrow();
8887            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8888            assert_eq!(positions.len(), 1);
8889            assert_eq!(positions[0].quantity, Quantity::from("11.0"));
8890            assert_eq!(positions[0].commissions(), vec![Money::from("1.00 USDT")]);
8891        }
8892    }
8893
8894    #[rstest]
8895    fn test_position_check_second_overfill_does_not_raise_filled_order() {
8896        let (mut node, venue_report, mut first) =
8897            position_fill_test_fixture("PositionSecondOverfill", Quantity::from("12.0"));
8898        first.commission = Money::from("1.20 USDT");
8899        let key = (first.instrument_id, first.account_id);
8900        let client = StubExecutionClient::new(
8901            ClientId::from("POSITION-FILLS"),
8902            AccountId::from("TEST-001"),
8903            first.instrument_id.venue,
8904            OmsType::Netting,
8905            None,
8906        )
8907        .with_allows_reconciliation_overfill();
8908        node.kernel
8909            .exec_engine
8910            .borrow_mut()
8911            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8912            .unwrap();
8913
8914        let mut second = first.clone();
8915        second.trade_id = TradeId::from("T-POSITION-LATER");
8916        second.last_qty = Quantity::from("1.0");
8917        second.commission = Money::from("0.10 USDT");
8918        second.ts_event = UnixNanos::from(1_001);
8919        let position_result = position_report_result(&node, venue_report);
8920        node.handle_position_fill_report_result(PositionFillReportResult {
8921            position_result,
8922            reports: IndexMap::from([(key, vec![first, second])]),
8923        });
8924
8925        let order = node
8926            .kernel
8927            .cache
8928            .borrow()
8929            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8930            .unwrap();
8931        assert_eq!(order.quantity(), Quantity::from("13.0"));
8932        assert_eq!(order.filled_qty(), Quantity::from("13.0"));
8933        assert_eq!(order.status(), OrderStatus::Filled);
8934        assert_eq!(order.trade_ids().len(), 2);
8935        assert!(
8936            order
8937                .trade_ids()
8938                .iter()
8939                .all(|trade_id| trade_id.as_str() != "T-POSITION-LATER")
8940        );
8941        {
8942            let cache = node.kernel.cache.borrow();
8943            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8944            assert_eq!(positions.len(), 1);
8945            assert_eq!(positions[0].quantity, Quantity::from("13.0"));
8946            assert_eq!(positions[0].commissions(), vec![Money::from("1.20 USDT")]);
8947        }
8948    }
8949
8950    #[rstest]
8951    fn test_allow_overfills_applies_fill_without_raising_quantity() {
8952        let config = LiveNodeConfig {
8953            exec_engine: crate::config::LiveExecutionEngineConfig {
8954                reconciliation: true,
8955                allow_overfills: true,
8956                ..Default::default()
8957            },
8958            ..Default::default()
8959        };
8960
8961        let node = LiveNode::build("AllowOverfillNoRaise".to_string(), Some(config)).unwrap();
8962        let account_id = AccountId::from("TEST-001");
8963        let client_id = ClientId::from("POSITION-FILLS");
8964        let client_order_id = ClientOrderId::from("O-ALLOW");
8965        let venue_order_id = VenueOrderId::from("V-ALLOW");
8966        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8967        node.kernel
8968            .cache
8969            .borrow_mut()
8970            .add_instrument(instrument.clone())
8971            .unwrap();
8972        insert_accepted_limit_order_in_node(
8973            &node,
8974            account_id,
8975            client_id,
8976            instrument.id(),
8977            client_order_id,
8978            venue_order_id,
8979        );
8980        let client = StubExecutionClient::new(
8981            client_id,
8982            account_id,
8983            instrument.id().venue,
8984            OmsType::Netting,
8985            None,
8986        )
8987        .with_allows_reconciliation_overfill();
8988        node.kernel
8989            .exec_engine
8990            .borrow_mut()
8991            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8992            .unwrap();
8993
8994        let fill = FillReport::new(
8995            account_id,
8996            instrument.id(),
8997            venue_order_id,
8998            TradeId::from("T-ALLOW"),
8999            OrderSide::Buy,
9000            Quantity::from("12.0"),
9001            Price::from("100.0"),
9002            Money::from("1.00 USDT"),
9003            LiquiditySide::Taker,
9004            Some(client_order_id),
9005            None,
9006            UnixNanos::from(1_000),
9007            UnixNanos::from(1_000),
9008            None,
9009        );
9010        node.kernel
9011            .exec_engine
9012            .borrow_mut()
9013            .reconcile_fill_report(&fill);
9014
9015        let order = node
9016            .kernel
9017            .cache
9018            .borrow()
9019            .order_owned(&client_order_id)
9020            .unwrap();
9021        assert_eq!(order.quantity(), Quantity::from("10.0"));
9022        assert_eq!(order.filled_qty(), Quantity::from("12.0"));
9023        assert_eq!(order.trade_ids().len(), 1);
9024    }
9025
9026    fn position_fill_test_fixture(
9027        name: &str,
9028        authoritative_qty: Quantity,
9029    ) -> (LiveNode, PositionStatusReport, FillReport) {
9030        position_fill_test_fixture_with_threshold(name, authoritative_qty, 0)
9031    }
9032
9033    fn position_fill_test_fixture_with_threshold(
9034        name: &str,
9035        authoritative_qty: Quantity,
9036        position_check_threshold_ms: u32,
9037    ) -> (LiveNode, PositionStatusReport, FillReport) {
9038        let config = LiveNodeConfig {
9039            exec_engine: crate::config::LiveExecutionEngineConfig {
9040                reconciliation: true,
9041                position_check_threshold_ms,
9042                ..Default::default()
9043            },
9044            ..Default::default()
9045        };
9046
9047        let mut node = LiveNode::build(name.to_string(), Some(config)).unwrap();
9048        let account_id = AccountId::from("TEST-001");
9049        let client_id = ClientId::from("POSITION-FILLS");
9050        let client_order_id = ClientOrderId::from("O-POSITION-FILLS");
9051        let venue_order_id = VenueOrderId::from("V-POSITION-FILLS");
9052        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
9053        let account = AccountAny::Margin(MarginAccount::new(
9054            AccountState::new(
9055                account_id,
9056                AccountType::Margin,
9057                vec![AccountBalance::new(
9058                    Money::from("1000000 USDT"),
9059                    Money::from("0 USDT"),
9060                    Money::from("1000000 USDT"),
9061                )],
9062                Vec::new(),
9063                true,
9064                UUID4::new(),
9065                UnixNanos::default(),
9066                UnixNanos::default(),
9067                Some(Currency::USDT()),
9068            ),
9069            true,
9070        ));
9071        node.kernel.cache.borrow_mut().add_account(account).unwrap();
9072        node.kernel
9073            .cache
9074            .borrow_mut()
9075            .add_instrument(instrument.clone())
9076            .unwrap();
9077        insert_accepted_limit_order_in_node(
9078            &node,
9079            account_id,
9080            client_id,
9081            instrument.id(),
9082            client_order_id,
9083            venue_order_id,
9084        );
9085        let order = node
9086            .kernel
9087            .cache
9088            .borrow()
9089            .order_owned(&client_order_id)
9090            .unwrap();
9091        let mut initial_fill = TestOrderEventStubs::filled(
9092            &order,
9093            &instrument,
9094            Some(TradeId::from("T-POSITION-INITIAL")),
9095            None,
9096            Some(Price::from("100.0")),
9097            Some(Quantity::from("1.0")),
9098            Some(LiquiditySide::Taker),
9099            None,
9100            None,
9101            Some(account_id),
9102        );
9103
9104        let OrderEventAny::Filled(fill) = &mut initial_fill else {
9105            unreachable!();
9106        };
9107
9108        fill.commission = Some(Money::zero(instrument.quote_currency()));
9109        node.process_reconciliation_events(&[initial_fill]);
9110
9111        let ts_event = UnixNanos::from(1_000);
9112
9113        let fill_report = FillReport::new(
9114            account_id,
9115            instrument.id(),
9116            venue_order_id,
9117            TradeId::from("T-POSITION-AUTHORITATIVE"),
9118            OrderSide::Buy,
9119            authoritative_qty,
9120            Price::from("100.0"),
9121            Money::zero(instrument.quote_currency()),
9122            LiquiditySide::Taker,
9123            Some(client_order_id),
9124            None,
9125            ts_event,
9126            ts_event,
9127            None,
9128        );
9129
9130        let venue_report = PositionStatusReport::new(
9131            account_id,
9132            instrument.id(),
9133            PositionSide::Long,
9134            Quantity::from("2.0"),
9135            ts_event,
9136            ts_event,
9137            None,
9138            None,
9139            Some(dec!(100.0)),
9140        );
9141
9142        (node, venue_report, fill_report)
9143    }
9144
9145    fn position_report_result(
9146        node: &LiveNode,
9147        report: PositionStatusReport,
9148    ) -> PositionReportResult {
9149        let client_id = ClientId::from("POSITION-FILLS");
9150        let key = (report.instrument_id, report.account_id);
9151        let mut check = node
9152            .exec_manager
9153            .prepare_position_report_check(UUID4::new(), &[]);
9154        check.client_coverage.insert(
9155            key,
9156            ReportClientCoverage::Resolved(IndexSet::from([client_id])),
9157        );
9158
9159        PositionReportResult {
9160            check,
9161            reports: vec![report],
9162            queried_clients: IndexSet::from([client_id]),
9163            failed_clients: IndexSet::new(),
9164        }
9165    }
9166
9167    fn position_quantity_after_check(
9168        node: &mut LiveNode,
9169        venue_report: &PositionStatusReport,
9170        reports: &[FillReport],
9171    ) -> Quantity {
9172        let key = (venue_report.instrument_id, venue_report.account_id);
9173        let position_result = position_report_result(node, venue_report.clone());
9174        node.handle_position_fill_report_result(PositionFillReportResult {
9175            position_result,
9176            reports: IndexMap::from([(key, reports.to_vec())]),
9177        });
9178
9179        node.kernel
9180            .cache
9181            .borrow()
9182            .positions_open(None, Some(&key.0), None, Some(&key.1), None)
9183            .iter()
9184            .map(|position| position.quantity)
9185            .sum()
9186    }
9187
9188    fn fill_report_event(fill: &OrderFilled) -> ExecutionEvent {
9189        ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
9190            fill.account_id,
9191            fill.instrument_id,
9192            fill.venue_order_id,
9193            fill.trade_id,
9194            fill.order_side,
9195            fill.last_qty,
9196            fill.last_px,
9197            fill.commission
9198                .unwrap_or_else(|| Money::zero(fill.currency)),
9199            fill.liquidity_side,
9200            Some(fill.client_order_id),
9201            fill.position_id,
9202            fill.ts_event,
9203            fill.ts_init,
9204            None,
9205        ))))
9206    }
9207
9208    fn is_recent_fill(node: &LiveNode, fill: &OrderFilled) -> bool {
9209        node.exec_manager.is_fill_recently_processed(
9210            fill.account_id,
9211            fill.instrument_id,
9212            fill.trade_id,
9213        )
9214    }
9215
9216    #[rstest]
9217    #[case(0, NodeState::Idle)]
9218    #[case(1, NodeState::Starting)]
9219    #[case(2, NodeState::Running)]
9220    #[case(3, NodeState::ShuttingDown)]
9221    #[case(4, NodeState::Stopped)]
9222    fn test_node_state_from_u8_valid(#[case] value: u8, #[case] expected: NodeState) {
9223        assert_eq!(NodeState::from_u8(value), expected);
9224    }
9225
9226    #[rstest]
9227    #[case(5)]
9228    #[case(255)]
9229    #[should_panic(expected = "Invalid NodeState value")]
9230    fn test_node_state_from_u8_invalid_panics(#[case] value: u8) {
9231        let _ = NodeState::from_u8(value);
9232    }
9233
9234    #[rstest]
9235    fn test_node_state_roundtrip() {
9236        for state in [
9237            NodeState::Idle,
9238            NodeState::Starting,
9239            NodeState::Running,
9240            NodeState::ShuttingDown,
9241            NodeState::Stopped,
9242        ] {
9243            assert_eq!(NodeState::from_u8(state.as_u8()), state);
9244        }
9245    }
9246
9247    #[rstest]
9248    fn test_node_state_is_running_only_for_running() {
9249        assert!(!NodeState::Idle.is_running());
9250        assert!(!NodeState::Starting.is_running());
9251        assert!(NodeState::Running.is_running());
9252        assert!(!NodeState::ShuttingDown.is_running());
9253        assert!(!NodeState::Stopped.is_running());
9254    }
9255
9256    #[rstest]
9257    #[tokio::test]
9258    async fn test_await_engines_connected_returns_stop_requested() {
9259        let node = LiveNode::build("TestNode".to_string(), None).unwrap();
9260        let handle = node.handle();
9261
9262        handle.stop();
9263
9264        let deadline = dst::time::Instant::now() + Duration::from_secs(1);
9265        let status = node.await_engines_connected(deadline).await;
9266
9267        assert_eq!(status, EngineConnectionStatus::StopRequested);
9268        assert!(handle.should_stop());
9269    }
9270
9271    #[rstest]
9272    #[tokio::test]
9273    async fn test_await_engines_connected_returns_shutdown_requested() {
9274        let node = LiveNode::build("TestNode".to_string(), None).unwrap();
9275
9276        node.kernel().shutdown_flag().set(true);
9277
9278        let deadline = dst::time::Instant::now() + Duration::from_secs(1);
9279        let status = node.await_engines_connected(deadline).await;
9280
9281        assert_eq!(status, EngineConnectionStatus::ShutdownRequested);
9282    }
9283
9284    #[rstest]
9285    #[tokio::test]
9286    async fn test_start_stop_request_aborts_startup_without_running() {
9287        let config = LiveNodeConfig {
9288            exec_engine: crate::config::LiveExecutionEngineConfig {
9289                reconciliation: false,
9290                ..Default::default()
9291            },
9292            timeout_disconnection: Duration::from_millis(50),
9293            ..Default::default()
9294        };
9295
9296        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9297        let handle = node.handle();
9298
9299        handle.stop();
9300        node.start().await.unwrap();
9301
9302        assert_eq!(handle.state(), NodeState::Stopped);
9303        assert!(handle.should_stop());
9304        assert!(!handle.is_running());
9305    }
9306
9307    #[rstest]
9308    #[tokio::test(start_paused = true)]
9309    async fn test_stop_processes_residual_exec_event_during_grace_period() {
9310        let config = LiveNodeConfig {
9311            exec_engine: crate::config::LiveExecutionEngineConfig {
9312                reconciliation: false,
9313                ..Default::default()
9314            },
9315            timeout_connection: Duration::ZERO,
9316            timeout_reconciliation: Duration::ZERO,
9317            timeout_portfolio: Duration::ZERO,
9318            timeout_disconnection: Duration::ZERO,
9319            delay_post_stop: Duration::from_millis(20),
9320            timeout_shutdown: Duration::ZERO,
9321            ..Default::default()
9322        };
9323
9324        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9325        let order = OrderTestBuilder::new(OrderType::Market)
9326            .instrument_id(InstrumentId::from("EUR/USD.SIM"))
9327            .quantity(Quantity::from("2"))
9328            .build();
9329        let client_order_id = order.client_order_id();
9330        let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-STOP-001"));
9331
9332        node.kernel
9333            .cache()
9334            .borrow_mut()
9335            .add_order(order, None, None, false)
9336            .unwrap();
9337
9338        node.start().await.unwrap();
9339        let exec_event_sender = get_exec_event_sender();
9340
9341        let send_residual = tokio::spawn(async move {
9342            tokio::time::sleep(Duration::from_millis(1)).await;
9343            exec_event_sender
9344                .send(ExecutionEvent::Order(submitted))
9345                .unwrap();
9346        });
9347
9348        node.stop().await.unwrap();
9349        send_residual.await.unwrap();
9350
9351        assert_eq!(
9352            node.kernel
9353                .cache()
9354                .borrow()
9355                .order(&client_order_id)
9356                .unwrap()
9357                .status(),
9358            OrderStatus::Submitted
9359        );
9360
9361        node.dispose();
9362    }
9363
9364    #[rstest]
9365    #[tokio::test]
9366    async fn test_live_state_persistence_loads_before_start_and_saves_after_stop() {
9367        let actor_id = ActorId::from("LIVE-STATE-ACTOR");
9368        let strategy_id = StrategyId::from("LIVE-STATE-STRATEGY-001");
9369        let actor_load = IndexMap::from([("actor-load".to_string(), b"actor-loaded".to_vec())]);
9370        let strategy_load =
9371            IndexMap::from([("strategy-load".to_string(), b"strategy-loaded".to_vec())]);
9372        let actor_save = IndexMap::from([("actor-save".to_string(), b"actor-saved".to_vec())]);
9373        let strategy_save =
9374            IndexMap::from([("strategy-save".to_string(), b"strategy-saved".to_vec())]);
9375        let (database, control) = TestCacheDatabaseControl::create();
9376        control.set_actor_state(actor_id, &actor_load);
9377        control.set_strategy_state(strategy_id, &strategy_load);
9378
9379        let config = LiveNodeConfig {
9380            load_state: true,
9381            save_state: true,
9382            exec_engine: crate::config::LiveExecutionEngineConfig {
9383                reconciliation: false,
9384                ..Default::default()
9385            },
9386            timeout_connection: Duration::ZERO,
9387            timeout_reconciliation: Duration::ZERO,
9388            timeout_portfolio: Duration::ZERO,
9389            timeout_disconnection: Duration::ZERO,
9390            delay_post_stop: Duration::ZERO,
9391            timeout_shutdown: Duration::ZERO,
9392            ..Default::default()
9393        };
9394
9395        let mut node = LiveNode::build("StatePersistenceNode".to_string(), Some(config)).unwrap();
9396        node.set_cache_database(Box::new(database)).unwrap();
9397        node.add_actor(StateActor::new(
9398            actor_id,
9399            control.clone(),
9400            actor_save.clone(),
9401        ))
9402        .unwrap();
9403        node.add_strategy(StateStrategy::new(
9404            strategy_id,
9405            control.clone(),
9406            strategy_save.clone(),
9407        ))
9408        .unwrap();
9409
9410        node.start().await.unwrap();
9411        node.stop().await.unwrap();
9412        node.dispose();
9413
9414        assert_eq!(
9415            control.events(),
9416            vec![
9417                "actor.load:LIVE-STATE-ACTOR",
9418                "actor.on_load",
9419                "strategy.load:LIVE-STATE-STRATEGY-001",
9420                "strategy.on_load",
9421                "actor.on_start",
9422                "strategy.on_start",
9423                "actor.on_stop",
9424                "strategy.on_stop",
9425                "actor.on_save",
9426                "actor.update:LIVE-STATE-ACTOR",
9427                "strategy.on_save",
9428                "strategy.update:LIVE-STATE-STRATEGY-001",
9429                "database.close",
9430            ]
9431        );
9432        assert_eq!(control.actor_state(&actor_id), Some(actor_save));
9433        assert_eq!(control.strategy_state(&strategy_id), Some(strategy_save));
9434        assert_eq!(node.state(), NodeState::Stopped);
9435    }
9436
9437    #[rstest]
9438    #[tokio::test]
9439    async fn test_live_state_persistence_reports_callback_errors_after_shutdown() {
9440        let actor_id = ActorId::from("LIVE-FAIL-SAVE-ACTOR");
9441        let strategy_id = StrategyId::from("LIVE-FAIL-SAVE-STRATEGY-001");
9442        let (database, control) = TestCacheDatabaseControl::create();
9443
9444        let config = LiveNodeConfig {
9445            save_state: true,
9446            exec_engine: crate::config::LiveExecutionEngineConfig {
9447                reconciliation: false,
9448                ..Default::default()
9449            },
9450            timeout_connection: Duration::ZERO,
9451            timeout_reconciliation: Duration::ZERO,
9452            timeout_portfolio: Duration::ZERO,
9453            timeout_disconnection: Duration::ZERO,
9454            delay_post_stop: Duration::ZERO,
9455            timeout_shutdown: Duration::ZERO,
9456            ..Default::default()
9457        };
9458
9459        let mut node =
9460            LiveNode::build("StatePersistenceErrorNode".to_string(), Some(config)).unwrap();
9461        node.set_cache_database(Box::new(database)).unwrap();
9462        node.add_actor(
9463            StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_save(),
9464        )
9465        .unwrap();
9466        node.add_strategy(
9467            StateStrategy::new(strategy_id, control.clone(), IndexMap::new()).with_fail_save(),
9468        )
9469        .unwrap();
9470
9471        node.start().await.unwrap();
9472        let error = node.stop().await.unwrap_err();
9473        node.dispose();
9474
9475        assert_eq!(
9476            error.to_string(),
9477            "failed while finalizing kernel shutdown: Failed to save component state: actor \
9478             LIVE-FAIL-SAVE-ACTOR callback: test actor on_save failure; strategy \
9479             LIVE-FAIL-SAVE-STRATEGY-001 callback: test strategy on_save failure"
9480        );
9481        assert_eq!(
9482            control.events(),
9483            vec![
9484                "actor.on_start",
9485                "strategy.on_start",
9486                "actor.on_stop",
9487                "strategy.on_stop",
9488                "actor.on_save",
9489                "strategy.on_save",
9490                "database.close",
9491            ]
9492        );
9493        assert_eq!(node.state(), NodeState::Stopped);
9494    }
9495
9496    #[rstest]
9497    #[tokio::test]
9498    async fn test_stop_drains_queued_exec_event_after_zero_grace() {
9499        let config = LiveNodeConfig {
9500            exec_engine: crate::config::LiveExecutionEngineConfig {
9501                reconciliation: false,
9502                ..Default::default()
9503            },
9504            timeout_connection: Duration::ZERO,
9505            timeout_reconciliation: Duration::ZERO,
9506            timeout_portfolio: Duration::ZERO,
9507            timeout_disconnection: Duration::ZERO,
9508            delay_post_stop: Duration::ZERO,
9509            timeout_shutdown: Duration::ZERO,
9510            ..Default::default()
9511        };
9512
9513        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9514        let order = OrderTestBuilder::new(OrderType::Market)
9515            .instrument_id(InstrumentId::from("GBP/USD.SIM"))
9516            .quantity(Quantity::from("3"))
9517            .build();
9518        let client_order_id = order.client_order_id();
9519        let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-DRAIN-001"));
9520
9521        node.kernel
9522            .cache()
9523            .borrow_mut()
9524            .add_order(order, None, None, false)
9525            .unwrap();
9526
9527        node.start().await.unwrap();
9528        get_exec_event_sender()
9529            .send(ExecutionEvent::Order(submitted))
9530            .unwrap();
9531
9532        node.stop().await.unwrap();
9533
9534        assert_eq!(
9535            node.kernel
9536                .cache()
9537                .borrow()
9538                .order(&client_order_id)
9539                .unwrap()
9540                .status(),
9541            OrderStatus::Submitted
9542        );
9543
9544        node.dispose();
9545    }
9546
9547    #[rstest]
9548    #[case::manual(false)]
9549    #[case::integrated(true)]
9550    fn test_shutdown_drain_logs_progress(
9551        #[case] integrated: bool,
9552        #[values(0, 9_999, 10_000, 20_001)] pending: usize,
9553    ) {
9554        struct DrainLogCapture(Mutex<Vec<String>>);
9555
9556        impl Log for DrainLogCapture {
9557            fn enabled(&self, metadata: &Metadata<'_>) -> bool {
9558                metadata.level() == Level::Debug
9559            }
9560
9561            fn log(&self, record: &Record<'_>) {
9562                assert_eq!(record.level(), Level::Debug);
9563                self.0.lock().push(record.args().to_string());
9564            }
9565
9566            fn flush(&self) {}
9567        }
9568
9569        static LOGS: DrainLogCapture = DrainLogCapture(Mutex::new(Vec::new()));
9570        log::set_logger(&LOGS).unwrap();
9571        log::set_max_level(LevelFilter::Debug);
9572
9573        let mut runner = AsyncRunner::new();
9574        runner.bind_senders();
9575        let event = TimeEvent::new("drain".into(), UUID4::new(), 0.into(), 0.into());
9576
9577        for _ in 0..pending {
9578            nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9579                event.clone(),
9580                TimeEventCallback::from(|_: TimeEvent| {}),
9581            ));
9582        }
9583
9584        if integrated {
9585            let mut channels = runner.take_channels();
9586            LiveNode::drain_channels(
9587                &mut channels.time_evt_rx,
9588                &mut channels.system_evt_rx,
9589                &mut channels.system_cmd_rx,
9590                &mut channels.exec_evt_rx,
9591                &mut channels.exec_cmd_rx,
9592                &mut channels.data_evt_rx,
9593                &mut channels.data_cmd_rx,
9594            );
9595        } else {
9596            assert_eq!(runner.poll_pending(|_| {}), pending);
9597        }
9598
9599        let mut expected = vec![format!(
9600            "Draining shutdown queues ({pending} events initially pending)"
9601        )];
9602        expected.extend(
9603            (10_000..=pending)
9604                .step_by(10_000)
9605                .map(|processed| format!("Processed {processed} events during shutdown drain")),
9606        );
9607        expected.push(format!(
9608            "Drained {pending} remaining events during shutdown"
9609        ));
9610        assert_eq!(*LOGS.0.lock(), expected);
9611    }
9612
9613    #[rstest]
9614    #[tokio::test]
9615    async fn test_start_event_store_replay_skips_live_connections() {
9616        let mut node = live_node_with_replay_store(false);
9617        let handle = node.handle();
9618
9619        node.start().await.unwrap();
9620
9621        assert_eq!(handle.state(), NodeState::Running);
9622        assert!(handle.is_running());
9623        assert!(node.kernel.is_event_store_replay());
9624        assert!(node.runner.is_some());
9625    }
9626
9627    #[rstest]
9628    #[tokio::test]
9629    async fn test_start_event_store_replay_preserves_stop_request() {
9630        let mut node = live_node_with_replay_store(false);
9631        let handle = node.handle();
9632        handle.stop();
9633
9634        node.start().await.unwrap();
9635
9636        assert_eq!(handle.state(), NodeState::Stopped);
9637        assert!(handle.should_stop());
9638        assert!(node.kernel.is_event_store_replay());
9639        assert!(node.runner.is_some());
9640    }
9641
9642    #[rstest]
9643    #[tokio::test]
9644    async fn test_start_event_store_replay_config_failure_aborts_startup() {
9645        let mut node = live_node_with_replay_store(true);
9646        let handle = node.handle();
9647
9648        node.start().await.unwrap();
9649
9650        assert_eq!(handle.state(), NodeState::Stopped);
9651        assert!(!handle.is_running());
9652        assert!(node.kernel.is_event_store_replay_configured());
9653        assert!(!node.kernel.is_event_store_replay());
9654        assert!(node.runner.is_some());
9655    }
9656
9657    #[rstest]
9658    #[tokio::test]
9659    async fn test_run_event_store_replay_consumes_runner_and_stops_before_connections() {
9660        let mut node = live_node_with_replay_store(false);
9661        let handle = node.handle();
9662
9663        node.run().await.unwrap();
9664
9665        assert_eq!(handle.state(), NodeState::Running);
9666        assert!(handle.is_running());
9667        assert!(node.kernel.is_event_store_replay());
9668        assert!(node.runner.is_none());
9669    }
9670
9671    #[rstest]
9672    #[tokio::test]
9673    async fn test_run_event_store_replay_preserves_stop_request() {
9674        let mut node = live_node_with_replay_store(false);
9675        let handle = node.handle();
9676        handle.stop();
9677
9678        node.run().await.unwrap();
9679
9680        assert_eq!(handle.state(), NodeState::Stopped);
9681        assert!(handle.should_stop());
9682        assert!(node.kernel.is_event_store_replay());
9683        assert!(node.runner.is_none());
9684    }
9685
9686    #[rstest]
9687    #[tokio::test]
9688    async fn test_run_event_store_replay_config_failure_aborts_startup() {
9689        let mut node = live_node_with_replay_store(true);
9690        let handle = node.handle();
9691
9692        node.run().await.unwrap();
9693
9694        assert_eq!(handle.state(), NodeState::Stopped);
9695        assert!(!handle.is_running());
9696        assert!(node.kernel.is_event_store_replay_configured());
9697        assert!(!node.kernel.is_event_store_replay());
9698        assert!(node.runner.is_none());
9699    }
9700
9701    #[rstest]
9702    fn test_build_rejects_event_store_config_without_factory() {
9703        let config = LiveNodeConfig {
9704            event_store: Some(EventStoreConfig::default()),
9705            exec_engine: crate::config::LiveExecutionEngineConfig {
9706                reconciliation: false,
9707                ..Default::default()
9708            },
9709            ..Default::default()
9710        };
9711
9712        let err = LiveNodeBuilder::from_config(config)
9713            .expect("builder")
9714            .build()
9715            .expect_err("should reject event_store config without factory");
9716
9717        assert!(
9718            err.to_string().contains("with_event_store"),
9719            "error message should mention with_event_store, was: {err}"
9720        );
9721    }
9722
9723    #[rstest]
9724    fn test_direct_build_rejects_event_store_config() {
9725        let config = LiveNodeConfig {
9726            event_store: Some(EventStoreConfig::default()),
9727            exec_engine: crate::config::LiveExecutionEngineConfig {
9728                reconciliation: false,
9729                ..Default::default()
9730            },
9731            ..Default::default()
9732        };
9733
9734        let err = LiveNode::build("TestNode".to_string(), Some(config))
9735            .expect_err("LiveNode::build should reject event_store config");
9736
9737        assert!(
9738            err.to_string().contains("with_event_store"),
9739            "error message should mention with_event_store, was: {err}"
9740        );
9741    }
9742
9743    #[rstest]
9744    fn test_dispose_before_start_is_idempotent() {
9745        let mut node = LiveNode::build("TestNode".to_string(), None).unwrap();
9746        node.add_strategy(TestStrategy::new(StrategyConfig {
9747            strategy_id: Some(StrategyId::from("DISPOSAL-001")),
9748            ..Default::default()
9749        }))
9750        .unwrap();
9751
9752        node.dispose();
9753        node.dispose();
9754
9755        assert!(node.kernel.trader().borrow().is_disposed());
9756        assert_eq!(node.kernel.trader().borrow().component_count(), 0);
9757        assert_eq!(node.state(), NodeState::Stopped);
9758    }
9759
9760    #[rstest]
9761    fn test_dispose_releases_retained_callback_roots(
9762        #[values(false, true)] fatal: bool,
9763        #[values(false, true)] external: bool,
9764    ) {
9765        actor::clear_callbacks().unwrap();
9766        let directory =
9767            std::env::temp_dir().join(format!("nautilus-callback-disposal-{}", UUID4::new()));
9768
9769        let config = LiveNodeConfig {
9770            logging: LoggerConfig {
9771                fileout_level: LevelFilter::Info,
9772                file_config: Some(FileWriterConfig {
9773                    directory: Some(directory.to_str().unwrap().to_string()),
9774                    file_name: Some("disposal".to_string()),
9775                    ..Default::default()
9776                }),
9777                ..Default::default()
9778            },
9779            ..Default::default()
9780        };
9781
9782        let mut node = LiveNode::build("CallbackDisposalNode".to_string(), Some(config)).unwrap();
9783        let received = Rc::new(RefCell::new(Vec::new()));
9784        let observed = received.clone();
9785
9786        let retained = DispatchMessage::from(()).dispatch(|()| {
9787            nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9788                TimeEvent::new("disposal".into(), UUID4::new(), 17.into(), 23.into()),
9789                TimeEventCallback::RustLocal(Rc::new(move |_| {
9790                    observed.borrow_mut().push("delivered");
9791                })),
9792            ));
9793
9794            external.then(|| DispatchMessage::new((), std::thread::current().id()))
9795        });
9796
9797        if let Some(retained) = &retained {
9798            assert!(retained.is_rooted());
9799        }
9800
9801        assert_eq!(actor::clear_callbacks(), Err(CallbackDispatchError::Active));
9802
9803        if fatal {
9804            crate::dispatch::tests::latch_callback_failure();
9805        }
9806
9807        node.dispose();
9808
9809        assert!(node.runner.is_none());
9810        assert!(received.borrow().is_empty());
9811        assert_eq!(Rc::strong_count(&received), 1);
9812        assert_eq!(
9813            actor::callback_failure(),
9814            (fatal && external).then_some(CallbackDispatchError::DeliveryUnwound)
9815        );
9816        assert_eq!(
9817            actor::clear_callbacks(),
9818            if external {
9819                Err(CallbackDispatchError::Active)
9820            } else {
9821                Ok(())
9822            }
9823        );
9824
9825        logging_sync_to_disk().unwrap();
9826        let output = std::fs::read_to_string(directory.join("disposal.log")).unwrap();
9827        drop(retained);
9828        node.dispose();
9829
9830        assert_eq!(actor::callback_failure(), None);
9831        assert_eq!(actor::clear_callbacks(), Ok(()));
9832        drop(node);
9833        std::fs::remove_dir_all(directory).unwrap();
9834
9835        let errors: Vec<_> = output
9836            .lines()
9837            .filter(|line| line.contains("[ERROR]"))
9838            .filter_map(|line| {
9839                line.split_once(".nautilus_live::node: ")
9840                    .map(|(_, text)| text)
9841            })
9842            .collect();
9843
9844        let mut expected = Vec::new();
9845
9846        if fatal {
9847            expected.push(
9848                "Callback dispatch failed before disposal cleanup: Callback delivery unwound",
9849            );
9850        }
9851
9852        if external {
9853            expected.push(
9854                "Failed to clear callback dispatch during disposal: Callback work or access is still active",
9855            );
9856        }
9857
9858        assert_eq!(errors, expected);
9859    }
9860
9861    #[tokio::test]
9862    async fn test_dispose_releases_stop_generated_callback_roots() {
9863        actor::clear_callbacks().unwrap();
9864
9865        let config = LiveNodeConfig {
9866            exec_engine: crate::config::LiveExecutionEngineConfig {
9867                reconciliation: false,
9868                ..Default::default()
9869            },
9870            timeout_connection: Duration::ZERO,
9871            timeout_reconciliation: Duration::ZERO,
9872            timeout_portfolio: Duration::ZERO,
9873            timeout_shutdown: Duration::ZERO,
9874            ..Default::default()
9875        };
9876
9877        let mut node =
9878            LiveNode::build("StopCallbackDisposalNode".to_string(), Some(config)).unwrap();
9879        let received = Rc::new(RefCell::new(Vec::new()));
9880        node.add_actor(StopCallbackActor {
9881            core: DataActorCore::new(DataActorConfig {
9882                actor_id: Some(ActorId::from("STOP-CALLBACK")),
9883                ..Default::default()
9884            }),
9885            received: received.clone(),
9886        })
9887        .unwrap();
9888
9889        node.start().await.unwrap();
9890
9891        assert!(node.kernel.trader().borrow().is_running());
9892        assert_eq!(actor::clear_callbacks(), Ok(()));
9893
9894        node.dispose();
9895
9896        assert_eq!(*received.borrow(), ["stop", "queued"]);
9897        assert_eq!(Rc::strong_count(&received), 1);
9898        assert!(node.runner.is_none());
9899        assert!(node.kernel.trader().borrow().is_disposed());
9900        assert_eq!(node.state(), NodeState::Stopped);
9901        assert_eq!(actor::clear_callbacks(), Ok(()));
9902    }
9903
9904    #[derive(Debug)]
9905    struct StopCallbackActor {
9906        core: DataActorCore,
9907        received: Rc<RefCell<Vec<&'static str>>>,
9908    }
9909
9910    nautilus_actor!(StopCallbackActor);
9911
9912    impl DataActor for StopCallbackActor {
9913        fn on_stop(&mut self) -> anyhow::Result<()> {
9914            self.received.borrow_mut().push("stop");
9915            let received = self.received.clone();
9916            DispatchMessage::from(()).dispatch(|()| {
9917                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9918                    TimeEvent::new("stop-disposal".into(), UUID4::new(), 31.into(), 37.into()),
9919                    TimeEventCallback::RustLocal(Rc::new(move |_| {
9920                        received.borrow_mut().push("delivered");
9921                    })),
9922                ));
9923            });
9924
9925            let clear_result = actor::clear_callbacks();
9926            anyhow::ensure!(
9927                clear_result == Err(CallbackDispatchError::Active),
9928                "Expected active callback roots during stop, received {clear_result:?}"
9929            );
9930            self.received.borrow_mut().push("queued");
9931            Ok(())
9932        }
9933    }
9934
9935    #[rstest]
9936    fn test_handle_initial_state() {
9937        let handle = LiveNodeHandle::new();
9938
9939        assert_eq!(handle.state(), NodeState::Idle);
9940        assert!(!handle.should_stop());
9941        assert!(!handle.is_running());
9942    }
9943
9944    #[rstest]
9945    fn test_handle_initial_metrics_snapshot_is_zero() {
9946        let handle = LiveNodeHandle::new();
9947
9948        assert_eq!(handle.metrics_snapshot(), RunnerMetricsSnapshot::default());
9949    }
9950
9951    #[rstest]
9952    fn test_record_runner_dispatch_updates_selected_channel() {
9953        let metrics = RunnerMetrics::default();
9954        let dispatch_start = dst::time::Instant::now();
9955        let metrics_start = dispatch_start
9956            .checked_sub(Duration::from_micros(1))
9957            .expect("test instant should support a one-microsecond lookback");
9958
9959        record_runner_dispatch(
9960            &metrics,
9961            SystemChannel::DataCommands,
9962            dispatch_start,
9963            metrics_start,
9964        );
9965        let snapshot = metrics.snapshot();
9966
9967        assert_eq!(snapshot.time_events.dispatched, 0);
9968        assert_eq!(snapshot.exec_events.dispatched, 0);
9969        assert_eq!(snapshot.exec_commands.dispatched, 0);
9970        assert_eq!(snapshot.data_events.dispatched, 0);
9971        assert_eq!(snapshot.data_commands.dispatched, 1);
9972        assert_eq!(
9973            snapshot.data_commands.last_dispatch_at_ns,
9974            snapshot.elapsed_ns
9975        );
9976        assert_eq!(
9977            snapshot.data_commands.dispatch_busy_ns,
9978            snapshot.dispatch_busy_ns
9979        );
9980        assert!(snapshot.dispatch_busy_ns < snapshot.elapsed_ns);
9981    }
9982
9983    #[rstest]
9984    fn test_handle_stop_sets_flag() {
9985        let handle = LiveNodeHandle::new();
9986
9987        handle.stop();
9988
9989        assert!(handle.should_stop());
9990    }
9991
9992    #[rstest]
9993    fn test_handle_stop_blocks_running_transition() {
9994        let handle = LiveNodeHandle::new();
9995        handle.set_starting();
9996        handle.stop();
9997
9998        let transition = handle.try_set_running();
9999
10000        assert_eq!(transition, RunningTransition::StopRequested);
10001        assert_eq!(handle.state(), NodeState::Starting);
10002        assert!(handle.should_stop());
10003        assert!(!handle.is_running());
10004    }
10005
10006    #[rstest]
10007    fn test_handle_stop_after_running_transition_remains_pending() {
10008        let handle = LiveNodeHandle::new();
10009        handle.set_starting();
10010
10011        let transition = handle.try_set_running();
10012        handle.stop();
10013
10014        assert_eq!(transition, RunningTransition::Entered);
10015        assert_eq!(handle.state(), NodeState::Running);
10016        assert!(handle.should_stop());
10017        assert!(handle.is_running());
10018    }
10019
10020    #[rstest]
10021    fn test_handle_node_state_transitions() {
10022        let handle = LiveNodeHandle::new();
10023        assert_eq!(handle.state(), NodeState::Idle);
10024
10025        handle.set_starting();
10026        assert_eq!(handle.state(), NodeState::Starting);
10027        assert!(!handle.is_running());
10028
10029        assert_eq!(handle.try_set_running(), RunningTransition::Entered);
10030        assert_eq!(handle.state(), NodeState::Running);
10031        assert!(handle.is_running());
10032
10033        handle.set_shutting_down();
10034        assert_eq!(handle.state(), NodeState::ShuttingDown);
10035        assert!(!handle.is_running());
10036
10037        handle.set_stopped();
10038        assert_eq!(handle.state(), NodeState::Stopped);
10039        assert!(!handle.is_running());
10040    }
10041
10042    #[rstest]
10043    fn test_handle_clone_shares_state_bidirectionally() {
10044        let handle1 = LiveNodeHandle::new();
10045        let handle2 = handle1.clone();
10046
10047        handle1.set_starting();
10048        let transition = handle2.try_set_running();
10049        handle1.stop();
10050
10051        assert_eq!(transition, RunningTransition::Entered);
10052        assert_eq!(handle1.state(), NodeState::Running);
10053        assert!(handle2.should_stop());
10054    }
10055
10056    #[rstest]
10057    fn test_handle_stop_flag_survives_non_running_state_changes() {
10058        let handle = LiveNodeHandle::new();
10059
10060        handle.set_starting();
10061        handle.stop();
10062        handle.set_shutting_down();
10063        handle.set_stopped();
10064
10065        assert_eq!(handle.state(), NodeState::Stopped);
10066        assert!(handle.should_stop());
10067    }
10068
10069    #[rstest]
10070    fn test_builder_creation() {
10071        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
10072
10073        assert!(result.is_ok());
10074    }
10075
10076    #[rstest]
10077    fn test_stream_processor_receives_unregistered_typed_payload() {
10078        let mut node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10079            .unwrap()
10080            .build()
10081            .unwrap();
10082        let command = SubscribeCommand::Quotes(SubscribeQuotes::new(
10083            InstrumentId::from("AUD/USD.SIM"),
10084            Some(ClientId::from("EXTERNAL")),
10085            Some(Venue::from("SIM")),
10086            UUID4::from("00000000-0000-4000-8000-000000000001"),
10087            UnixNanos::from(1),
10088            None,
10089            None,
10090        ));
10091        let expected = serde_json::to_value(&command).unwrap();
10092        let received = Rc::new(RefCell::new(Vec::new()));
10093        let steps = Rc::new(RefCell::new(Vec::new()));
10094        let received_processor = received.clone();
10095        let first_steps = steps.clone();
10096        node.add_stream_processor(move |message| {
10097            let command = message
10098                .downcast_ref::<SubscribeCommand>()
10099                .expect("processor must receive the decoded concrete command");
10100            received_processor
10101                .borrow_mut()
10102                .push(serde_json::to_value(command).unwrap());
10103            first_steps.borrow_mut().push(1);
10104        });
10105
10106        let second_steps = steps.clone();
10107        node.add_stream_processor(move |_| second_steps.borrow_mut().push(2));
10108        let republished = Rc::new(RefCell::new(Vec::new()));
10109        let republished_handler = republished.clone();
10110        let subscriber_steps = steps.clone();
10111        msgbus::subscribe_any(
10112            "external.test".into(),
10113            ShareableMessageHandler::from_typed(move |command: &SubscribeCommand| {
10114                republished_handler
10115                    .borrow_mut()
10116                    .push(serde_json::to_value(command).unwrap());
10117                subscriber_steps.borrow_mut().push(3);
10118            }),
10119            None,
10120        );
10121
10122        let message = BusMessage::with_str_topic(
10123            "external.test",
10124            BusPayloadType::SubscribeCommand,
10125            Bytes::from(serde_json::to_vec(&command).unwrap()),
10126            SerializationEncoding::Json,
10127        );
10128
10129        assert!(
10130            !msgbus::get_message_bus()
10131                .borrow()
10132                .is_streaming_type(BusPayloadType::SubscribeCommand)
10133        );
10134        node.process_external_msgbus_message(&message);
10135
10136        assert_eq!(*received.borrow(), vec![expected.clone()]);
10137        assert_eq!(*steps.borrow(), vec![1, 2]);
10138        assert!(republished.borrow().is_empty());
10139
10140        received.borrow_mut().clear();
10141        steps.borrow_mut().clear();
10142        msgbus::get_message_bus()
10143            .borrow_mut()
10144            .add_streaming_type(BusPayloadType::SubscribeCommand);
10145        node.process_external_msgbus_message(&message);
10146
10147        assert_eq!(*received.borrow(), vec![expected.clone()]);
10148        assert_eq!(*republished.borrow(), vec![expected]);
10149        assert_eq!(*steps.borrow(), vec![1, 2, 3]);
10150        msgbus::get_message_bus().borrow_mut().dispose();
10151    }
10152
10153    #[rstest]
10154    fn test_builder_rejects_backtest() {
10155        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Backtest);
10156
10157        assert!(result.is_err());
10158        assert!(result.unwrap_err().to_string().contains("Backtest"));
10159    }
10160
10161    #[rstest]
10162    fn test_builder_accepts_live_environment() {
10163        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live);
10164
10165        assert!(result.is_ok());
10166    }
10167
10168    #[rstest]
10169    fn test_builder_accepts_sandbox_environment() {
10170        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
10171
10172        assert!(result.is_ok());
10173    }
10174
10175    #[rstest]
10176    fn test_builder_fluent_api_chaining() {
10177        let builder = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live)
10178            .unwrap()
10179            .with_name("TestNode")
10180            .with_instance_id(UUID4::new())
10181            .with_load_state(false)
10182            .with_save_state(true)
10183            .with_timeout_connection(30)
10184            .with_timeout_reconciliation(60)
10185            .with_reconciliation(true)
10186            .with_reconciliation_lookback_mins(120)
10187            .with_timeout_portfolio(10)
10188            .with_timeout_disconnection_secs(5)
10189            .with_delay_post_stop_secs(3)
10190            .with_delay_shutdown_secs(10);
10191
10192        assert_eq!(builder.name(), "TestNode");
10193    }
10194
10195    #[rstest]
10196    fn test_builder_with_external_msgbus_egress_uses_configured_encoding() {
10197        let (external_egress, publications, closed) = CapturingExternalEgress::new();
10198
10199        let msgbus_config = MessageBusConfig {
10200            encoding: SerializationEncoding::Json,
10201            ..Default::default()
10202        };
10203
10204        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10205            .unwrap()
10206            .with_msgbus_config(msgbus_config)
10207            .with_external_msgbus_egress(Box::new(external_egress))
10208            .build()
10209            .expect("node builds with external message bus egress");
10210        let quote = QuoteTick::default();
10211
10212        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
10213
10214        let publications = publications.borrow();
10215        assert_eq!(publications.len(), 1);
10216        assert_eq!(publications[0].topic, "data.quotes.TEST");
10217        assert_eq!(
10218            serde_json::from_slice::<QuoteTick>(&publications[0].payload)
10219                .expect("JSON payload must decode as QuoteTick"),
10220            quote
10221        );
10222        drop(publications);
10223
10224        msgbus::get_message_bus().borrow_mut().dispose();
10225        assert!(closed.get());
10226        drop(node);
10227    }
10228
10229    #[rstest]
10230    #[tokio::test(flavor = "current_thread")]
10231    async fn test_builder_with_external_msgbus_factory_installs_egress_and_ingress() {
10232        let quote = QuoteTick::default();
10233        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10234        let publications = Arc::new(Mutex::new(Vec::new()));
10235        let closed = Arc::new(AtomicBool::new(false));
10236        let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), Some(rx));
10237
10238        let msgbus_config = MessageBusConfig {
10239            external_streams: Some(vec!["stream".to_string()]),
10240            ..Default::default()
10241        };
10242
10243        let config = LiveNodeConfig {
10244            environment: Environment::Sandbox,
10245            msgbus: Some(msgbus_config),
10246            exec_engine: crate::config::LiveExecutionEngineConfig {
10247                reconciliation: false,
10248                ..Default::default()
10249            },
10250            delay_post_stop: Duration::ZERO,
10251            timeout_connection: Duration::from_millis(500),
10252            timeout_disconnection: Duration::from_millis(500),
10253            ..Default::default()
10254        };
10255
10256        let mut node = LiveNodeBuilder::from_config(config)
10257            .unwrap()
10258            .with_external_msgbus_factory(Box::new(factory))
10259            .build()
10260            .expect("node builds with external message bus factory");
10261
10262        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
10263        {
10264            let publications = publications.lock();
10265            assert_eq!(publications.len(), 1);
10266            assert_eq!(publications[0].topic, "data.quotes.TEST");
10267            assert_eq!(
10268                serde_json::from_slice::<QuoteTick>(&publications[0].payload)
10269                    .expect("JSON payload must decode as QuoteTick"),
10270                quote
10271            );
10272        }
10273
10274        let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
10275        let handle = node.handle();
10276
10277        // Stopping from `drive` rather than here, so the node cannot finish `run` before the
10278        // republished quote is observed.
10279        let handler = TypedHandler::from({
10280            let received = received.clone();
10281            move |quote: &QuoteTick| {
10282                received.borrow_mut().push(*quote);
10283            }
10284        });
10285
10286        msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
10287        msgbus::get_message_bus()
10288            .borrow_mut()
10289            .add_streaming_type(BusPayloadType::QuoteTick);
10290
10291        let payload =
10292            Bytes::from(serde_json::to_vec(&quote).expect("QuoteTick should serialize as JSON"));
10293        let message = BusMessage::with_str_topic(
10294            "data.quotes.TEST",
10295            BusPayloadType::QuoteTick,
10296            payload,
10297            SerializationEncoding::Json,
10298        );
10299
10300        tokio::time::timeout(Duration::from_secs(30), async {
10301            let run = node.run();
10302            tokio::pin!(run);
10303
10304            let drive = async {
10305                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10306
10307                tx.send(message)
10308                    .await
10309                    .expect("external ingress receiver should be open");
10310
10311                wait_until_async(
10312                    || async { received.borrow().len() == 1 },
10313                    Duration::from_secs(10),
10314                )
10315                .await;
10316                assert_eq!(*received.borrow(), vec![quote]);
10317                handle.stop();
10318            };
10319
10320            tokio::select! {
10321                biased;
10322
10323                () = drive => {}
10324                result = &mut run => {
10325                    panic!("node stopped before factory ingress was republished: {result:?}");
10326                }
10327            }
10328
10329            run.await.expect("node should stop cleanly");
10330        })
10331        .await
10332        .expect("live node should republish factory ingress and stop before timeout");
10333
10334        assert_eq!(handle.state(), NodeState::Stopped);
10335        assert!(closed.load(Ordering::Relaxed));
10336        msgbus::get_message_bus().borrow_mut().dispose();
10337    }
10338
10339    #[rstest]
10340    #[tokio::test(flavor = "current_thread")]
10341    async fn test_builder_with_external_msgbus_factory_without_streams_runs_without_ingress() {
10342        let quote = QuoteTick::default();
10343        let publications = Arc::new(Mutex::new(Vec::new()));
10344        let closed = Arc::new(AtomicBool::new(false));
10345        let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), None);
10346
10347        let config = LiveNodeConfig {
10348            environment: Environment::Sandbox,
10349            msgbus: Some(MessageBusConfig::default()),
10350            exec_engine: crate::config::LiveExecutionEngineConfig {
10351                reconciliation: false,
10352                ..Default::default()
10353            },
10354            delay_post_stop: Duration::ZERO,
10355            timeout_connection: Duration::from_millis(500),
10356            timeout_disconnection: Duration::from_millis(500),
10357            ..Default::default()
10358        };
10359
10360        let mut node = LiveNodeBuilder::from_config(config)
10361            .unwrap()
10362            .with_external_msgbus_factory(Box::new(factory))
10363            .build()
10364            .expect("node builds with egress-only message bus factory");
10365        let handle = node.handle();
10366
10367        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
10368        {
10369            let publications = publications.lock();
10370            assert_eq!(publications.len(), 1);
10371            assert_eq!(publications[0].topic, "data.quotes.TEST");
10372        }
10373
10374        tokio::time::timeout(Duration::from_secs(30), async {
10375            let run = node.run();
10376            tokio::pin!(run);
10377
10378            let drive = async {
10379                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10380                handle.stop();
10381            };
10382
10383            tokio::select! {
10384                biased;
10385
10386                () = drive => {}
10387                result = &mut run => {
10388                    panic!("node stopped before egress-only factory run was observed: {result:?}");
10389                }
10390            }
10391
10392            run.await.expect("node should stop cleanly");
10393        })
10394        .await
10395        .expect("live node should run without external ingress before timeout");
10396
10397        assert_eq!(handle.state(), NodeState::Stopped);
10398        msgbus::get_message_bus().borrow_mut().dispose();
10399        assert!(closed.load(Ordering::Relaxed));
10400    }
10401
10402    #[rstest]
10403    fn test_builder_with_external_msgbus_factory_rejects_injected_surfaces() {
10404        let (external_egress, _publications, _closed) = CapturingExternalEgress::new();
10405
10406        let egress_factory = CapturingBackingFactory::new(
10407            Arc::new(Mutex::new(Vec::new())),
10408            Arc::new(AtomicBool::new(false)),
10409            None,
10410        );
10411        let egress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10412            .unwrap()
10413            .with_external_msgbus_factory(Box::new(egress_factory))
10414            .with_external_msgbus_egress(Box::new(external_egress))
10415            .build()
10416            .expect_err("builder should reject factory plus injected egress");
10417
10418        assert!(
10419            egress_error
10420                .to_string()
10421                .contains("cannot be combined with injected egress or ingress")
10422        );
10423
10424        let (_tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10425
10426        let ingress_factory = CapturingBackingFactory::new(
10427            Arc::new(Mutex::new(Vec::new())),
10428            Arc::new(AtomicBool::new(false)),
10429            None,
10430        );
10431        let ingress = CapturingExternalIngress::new(rx, Rc::new(Cell::new(false)));
10432        let ingress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10433            .unwrap()
10434            .with_external_msgbus_factory(Box::new(ingress_factory))
10435            .with_external_ingress(Box::new(ingress))
10436            .build()
10437            .expect_err("builder should reject factory plus injected ingress");
10438
10439        assert!(
10440            ingress_error
10441                .to_string()
10442                .contains("cannot be combined with injected egress or ingress")
10443        );
10444    }
10445
10446    #[rstest]
10447    #[tokio::test(flavor = "current_thread")]
10448    async fn test_run_republishes_external_ingress_on_local_msgbus() {
10449        let quote = QuoteTick::default();
10450        let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
10451        let payload =
10452            Bytes::from(serde_json::to_vec(&quote).expect("QuoteTick should serialize as JSON"));
10453        let message = BusMessage::with_str_topic(
10454            "data.quotes.TEST",
10455            BusPayloadType::QuoteTick,
10456            payload,
10457            SerializationEncoding::Json,
10458        );
10459        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10460        let closed = Rc::new(Cell::new(false));
10461        let ingress = CapturingExternalIngress::new(rx, closed.clone());
10462
10463        let config = LiveNodeConfig {
10464            environment: Environment::Sandbox,
10465            exec_engine: crate::config::LiveExecutionEngineConfig {
10466                reconciliation: false,
10467                ..Default::default()
10468            },
10469            delay_post_stop: Duration::ZERO,
10470            timeout_connection: Duration::from_millis(500),
10471            timeout_disconnection: Duration::from_millis(500),
10472            ..Default::default()
10473        };
10474
10475        let mut node = LiveNodeBuilder::from_config(config)
10476            .unwrap()
10477            .with_external_ingress(Box::new(ingress))
10478            .build()
10479            .expect("node builds with external message bus ingress");
10480        let handle = node.handle();
10481
10482        let handler = TypedHandler::from({
10483            let received = received.clone();
10484            move |quote: &QuoteTick| {
10485                received.borrow_mut().push(*quote);
10486            }
10487        });
10488
10489        msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
10490        msgbus::get_message_bus()
10491            .borrow_mut()
10492            .add_streaming_type(BusPayloadType::QuoteTick);
10493
10494        tokio::time::timeout(Duration::from_secs(30), async {
10495            let run = node.run();
10496            tokio::pin!(run);
10497
10498            let drive = async {
10499                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10500
10501                tx.send(message)
10502                    .await
10503                    .expect("external ingress receiver should be open");
10504
10505                wait_until_async(
10506                    || async { received.borrow().len() == 1 },
10507                    Duration::from_secs(10),
10508                )
10509                .await;
10510                assert_eq!(*received.borrow(), vec![quote]);
10511                handle.stop();
10512            };
10513
10514            tokio::select! {
10515                biased;
10516
10517                () = drive => {}
10518                result = &mut run => {
10519                    panic!("node stopped before external message was republished: {result:?}");
10520                }
10521            }
10522
10523            run.await.expect("node should stop cleanly");
10524        })
10525        .await
10526        .expect("live node should republish ingress and stop before timeout");
10527
10528        assert_eq!(handle.state(), NodeState::Stopped);
10529        assert!(closed.get());
10530        msgbus::get_message_bus().borrow_mut().dispose();
10531    }
10532
10533    #[rstest]
10534    #[tokio::test(flavor = "current_thread")]
10535    async fn test_run_closes_external_ingress_when_receiver_closes() {
10536        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10537        let closed = Rc::new(Cell::new(false));
10538        let ingress = CapturingExternalIngress::new(rx, closed.clone());
10539
10540        let config = LiveNodeConfig {
10541            environment: Environment::Sandbox,
10542            exec_engine: crate::config::LiveExecutionEngineConfig {
10543                reconciliation: false,
10544                ..Default::default()
10545            },
10546            delay_post_stop: Duration::ZERO,
10547            timeout_connection: Duration::from_millis(500),
10548            timeout_disconnection: Duration::from_millis(500),
10549            ..Default::default()
10550        };
10551
10552        let mut node = LiveNodeBuilder::from_config(config)
10553            .unwrap()
10554            .with_external_ingress(Box::new(ingress))
10555            .build()
10556            .expect("node builds with external message bus ingress");
10557        let handle = node.handle();
10558
10559        tokio::time::timeout(Duration::from_secs(30), async {
10560            let run = node.run();
10561            tokio::pin!(run);
10562
10563            let drive = async {
10564                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10565
10566                drop(tx);
10567
10568                wait_until_async(|| async { closed.get() }, Duration::from_secs(10)).await;
10569                assert!(
10570                    handle.is_running(),
10571                    "node should keep running after ingress closes"
10572                );
10573                handle.stop();
10574            };
10575
10576            tokio::select! {
10577                biased;
10578
10579                () = drive => {}
10580                result = &mut run => {
10581                    panic!("node stopped before ingress close was observed: {result:?}");
10582                }
10583            }
10584
10585            run.await.expect("node should stop cleanly");
10586        })
10587        .await
10588        .expect("live node should close ingress and stop before timeout");
10589
10590        assert_eq!(handle.state(), NodeState::Stopped);
10591    }
10592
10593    #[rstest]
10594    #[tokio::test(flavor = "current_thread")]
10595    async fn test_run_aborts_startup_when_external_ingress_receiver_unavailable() {
10596        let closed = Rc::new(Cell::new(false));
10597        let ingress = FailingExternalIngress::new(closed.clone());
10598
10599        let config = LiveNodeConfig {
10600            environment: Environment::Sandbox,
10601            exec_engine: crate::config::LiveExecutionEngineConfig {
10602                reconciliation: false,
10603                ..Default::default()
10604            },
10605            delay_post_stop: Duration::ZERO,
10606            timeout_connection: Duration::from_millis(500),
10607            timeout_disconnection: Duration::from_millis(500),
10608            ..Default::default()
10609        };
10610
10611        let mut node = LiveNodeBuilder::from_config(config)
10612            .unwrap()
10613            .with_external_ingress(Box::new(ingress))
10614            .build()
10615            .expect("node builds with external message bus ingress");
10616        let handle = node.handle();
10617
10618        let err = node.run().await.expect_err("run should fail");
10619
10620        assert!(
10621            err.to_string()
10622                .contains("external ingress receiver unavailable")
10623        );
10624        assert_eq!(handle.state(), NodeState::Stopped);
10625        assert!(closed.get());
10626    }
10627
10628    #[rstest]
10629    fn test_node_build_rejects_existing_node_preserving_account_delivery(
10630        #[values(false, true)] first_builder: bool,
10631        #[values(false, true)] second_builder: bool,
10632    ) {
10633        let config = LiveNodeConfig {
10634            trader_id: TraderId::from("PROBE-001"),
10635            ..Default::default()
10636        };
10637
10638        let mut node = if first_builder {
10639            LiveNodeBuilder::from_config(config)
10640                .unwrap()
10641                .build()
10642                .unwrap()
10643        } else {
10644            LiveNode::build("Original".to_string(), Some(config)).unwrap()
10645        };
10646
10647        let bus = msgbus::get_message_bus();
10648
10649        let ExecutionEvent::Account(before) = stub_account_event() else {
10650            unreachable!()
10651        };
10652
10653        get_exec_event_sender()
10654            .send(ExecutionEvent::Account(before.clone()))
10655            .unwrap();
10656        assert_eq!(node.drain_runner_pending(), 1);
10657        assert_eq!(
10658            node.kernel
10659                .cache
10660                .borrow()
10661                .account(&before.account_id)
10662                .unwrap()
10663                .last_event(),
10664            Some(before)
10665        );
10666
10667        let result = if second_builder {
10668            LiveNodeBuilder::from_config(LiveNodeConfig::default())
10669                .unwrap()
10670                .build()
10671        } else {
10672            LiveNode::build("Extra".to_string(), None)
10673        };
10674
10675        assert_eq!(
10676            result.unwrap_err().to_string(),
10677            "A LiveNode already exists or is being built on this thread; drop it before building another"
10678        );
10679
10680        let ExecutionEvent::Account(after) = stub_account_event() else {
10681            unreachable!()
10682        };
10683
10684        get_exec_event_sender()
10685            .send(ExecutionEvent::Account(after.clone()))
10686            .unwrap();
10687        assert_eq!(node.drain_runner_pending(), 1);
10688        assert_eq!(node.trader_id(), TraderId::from("PROBE-001"));
10689        assert!(Rc::ptr_eq(&bus, &msgbus::get_message_bus()));
10690        assert_eq!(
10691            node.kernel
10692                .cache
10693                .borrow()
10694                .account(&after.account_id)
10695                .unwrap()
10696                .last_event(),
10697            Some(after)
10698        );
10699    }
10700
10701    #[rstest]
10702    fn test_node_build_rejects_reentry_and_releases_thread_after_factory_failure() {
10703        let result = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
10704            .unwrap()
10705            .with_event_store(|_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
10706                let error = LiveNode::build("Reentrant".to_string(), None).unwrap_err();
10707                assert!(
10708                    error
10709                        .to_string()
10710                        .contains("already exists or is being built")
10711                );
10712                anyhow::bail!("Event store construction failed")
10713            })
10714            .build();
10715
10716        assert_eq!(
10717            result.unwrap_err().to_string(),
10718            "Event store construction failed"
10719        );
10720
10721        let node = LiveNode::build("Retry".to_string(), None).unwrap();
10722        assert_eq!(node.state(), NodeState::Idle);
10723    }
10724
10725    #[rstest]
10726    fn test_node_build_allows_separate_threads() {
10727        let node = LiveNode::build("First".to_string(), None).unwrap();
10728
10729        let other_state = std::thread::spawn(|| {
10730            let other = LiveNode::build("OtherThread".to_string(), None).unwrap();
10731            other.state()
10732        })
10733        .join()
10734        .unwrap();
10735
10736        assert_eq!(node.state(), NodeState::Idle);
10737        assert_eq!(other_state, NodeState::Idle);
10738    }
10739
10740    #[rstest]
10741    fn test_node_build_releases_thread_after_drop() {
10742        let mut node = LiveNode::build("First".to_string(), None).unwrap();
10743        node.dispose();
10744        assert!(LiveNode::build("WhileRetained".to_string(), None).is_err());
10745        drop(node);
10746
10747        let replacement = LiveNode::build("Replacement".to_string(), None).unwrap();
10748        assert_eq!(replacement.state(), NodeState::Idle);
10749    }
10750
10751    #[rstest]
10752    fn test_node_build_releases_thread_after_failure(#[values(false, true)] builder: bool) {
10753        let config = LiveNodeConfig {
10754            event_store: Some(EventStoreConfig::default()),
10755            ..Default::default()
10756        };
10757
10758        let result = if builder {
10759            LiveNodeBuilder::from_config(config).unwrap().build()
10760        } else {
10761            LiveNode::build("Failure".to_string(), Some(config))
10762        };
10763
10764        assert!(result.unwrap_err().to_string().contains("factory"));
10765
10766        let node = LiveNode::build("Retry".to_string(), None).unwrap();
10767        assert_eq!(node.state(), NodeState::Idle);
10768    }
10769
10770    #[cfg(feature = "python")]
10771    #[rstest]
10772    fn test_node_build_and_initial_state() {
10773        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10774            .unwrap()
10775            .with_name("TestNode")
10776            .build()
10777            .unwrap();
10778
10779        assert_eq!(node.state(), NodeState::Idle);
10780        assert!(!node.is_running());
10781        assert_eq!(node.environment(), Environment::Sandbox);
10782        assert_eq!(node.trader_id(), TraderId::from("TRADER-001"));
10783    }
10784
10785    #[cfg(feature = "python")]
10786    #[rstest]
10787    fn test_node_build_replaces_stale_runner_senders() {
10788        replace_data_cmd_sender(Arc::new(SyncDataCommandSender));
10789        replace_exec_cmd_sender(Arc::new(SyncTradingCommandSender));
10790
10791        let first = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10792            .unwrap()
10793            .with_name("FirstNode")
10794            .build()
10795            .unwrap();
10796
10797        assert_eq!(first.state(), NodeState::Idle);
10798        drop(first);
10799
10800        let second = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10801            .unwrap()
10802            .with_name("SecondNode")
10803            .build()
10804            .unwrap();
10805
10806        assert_eq!(second.state(), NodeState::Idle);
10807        assert!(!second.is_running());
10808    }
10809
10810    #[cfg(feature = "python")]
10811    #[rstest]
10812    fn test_node_handle_reflects_node_state() {
10813        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10814            .unwrap()
10815            .with_name("TestNode")
10816            .build()
10817            .unwrap();
10818
10819        let handle = node.handle();
10820
10821        assert_eq!(handle.state(), NodeState::Idle);
10822        assert!(!handle.is_running());
10823    }
10824
10825    #[rstest]
10826    fn test_pending_drain_data_returns_false_when_empty() {
10827        let mut pending = PendingEvents::default();
10828
10829        assert!(!pending.drain_data());
10830    }
10831
10832    #[rstest]
10833    fn test_pending_drain_data_returns_true_when_non_empty() {
10834        use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10835
10836        let mut pending = PendingEvents::default();
10837        pending.data_evts.push(
10838            DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10839                .into(),
10840        );
10841
10842        assert!(pending.drain_data());
10843        assert!(pending.data_evts.is_empty());
10844    }
10845
10846    fn stub_data_event() -> DataEvent {
10847        use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10848
10849        DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10850    }
10851
10852    fn stub_data_command() -> DataCommand {
10853        use nautilus_common::messages::data::{SubscribeCommand, subscribe::SubscribeInstruments};
10854        use nautilus_core::{UUID4, UnixNanos};
10855        use nautilus_model::identifiers::Venue;
10856
10857        DataCommand::Subscribe(SubscribeCommand::Instruments(SubscribeInstruments::new(
10858            None,
10859            Venue::from("TEST"),
10860            UUID4::new(),
10861            UnixNanos::default(),
10862            None,
10863            None,
10864        )))
10865    }
10866
10867    fn stub_system_command() -> SystemCommand {
10868        SystemCommand::ReconnectSocket(ReconnectSocket::new(
10869            TraderId::from("TRADER-001"),
10870            ClientId::from("POLYMARKET"),
10871            Ustr::from("polymarket-market-streams"),
10872            UnixNanos::default(),
10873        ))
10874    }
10875
10876    #[rstest]
10877    fn test_flush_pending_data_drains_events_and_commands() {
10878        let (evt_tx, mut evt_rx) =
10879            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10880        let (cmd_tx, mut cmd_rx) =
10881            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10882
10883        let mut pending = PendingEvents::default();
10884
10885        // Pre-load pending (items captured by the select loop)
10886        pending.data_evts.push((stub_data_event()).into());
10887        pending.data_cmds.push(stub_data_command().into());
10888
10889        // Pre-load channels (items missed by the select loop)
10890        evt_tx.send((stub_data_event()).into()).unwrap();
10891        cmd_tx.send(stub_data_command().into()).unwrap();
10892
10893        flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10894
10895        assert!(pending.data_evts.is_empty());
10896        assert!(pending.data_cmds.is_empty());
10897        assert!(evt_rx.try_recv().is_err());
10898        assert!(cmd_rx.try_recv().is_err());
10899    }
10900
10901    #[rstest]
10902    fn test_flush_pending_data_drains_mixed_sources() {
10903        let (evt_tx, mut evt_rx) =
10904            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10905        let (cmd_tx, mut cmd_rx) =
10906            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10907
10908        let mut pending = PendingEvents::default();
10909
10910        // First pass: pending has an event, channel has a command
10911        pending.data_evts.push((stub_data_event()).into());
10912        cmd_tx.send(stub_data_command().into()).unwrap();
10913
10914        // Second pass: channel has items that simulate arrival during first drain
10915        evt_tx.send((stub_data_event()).into()).unwrap();
10916        evt_tx.send((stub_data_event()).into()).unwrap();
10917        cmd_tx.send(stub_data_command().into()).unwrap();
10918
10919        flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10920
10921        assert!(pending.data_evts.is_empty());
10922        assert!(pending.data_cmds.is_empty());
10923        assert!(evt_rx.try_recv().is_err());
10924        assert!(cmd_rx.try_recv().is_err());
10925    }
10926
10927    #[rstest]
10928    fn test_pending_system_events_stay_separate_from_data() {
10929        let mut pending = PendingEvents::default();
10930
10931        let change = SocketStateChange::new(
10932            ClientId::from("BINANCE"),
10933            Some(Venue::from("BINANCE")),
10934            ustr::Ustr::from("binance-futures-market-streams"),
10935            SocketState::Connected,
10936        );
10937
10938        pending
10939            .system_events
10940            .push(SystemEvent::SocketState(change).into());
10941        let system_events = pending
10942            .take_system_events()
10943            .into_iter()
10944            .map(|message| message.dispatch(|value| value))
10945            .collect::<Vec<_>>();
10946
10947        assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10948        assert!(pending.is_empty());
10949    }
10950
10951    #[rstest]
10952    fn test_pending_system_commands_stay_separate_from_data() {
10953        let mut pending = PendingEvents::default();
10954        let command = stub_system_command();
10955
10956        pending.system_commands.push(command.into());
10957        let system_commands = pending
10958            .take_system_commands()
10959            .into_iter()
10960            .map(|message| message.dispatch(|value| value))
10961            .collect::<Vec<_>>();
10962
10963        assert_eq!(system_commands, vec![command]);
10964        assert!(pending.is_empty());
10965    }
10966
10967    fn stub_time_event_handler() -> TimeEventMessage {
10968        use std::rc::Rc;
10969
10970        use nautilus_common::{
10971            runner::TimeEventMessage,
10972            timer::{TimeEvent, TimeEventCallback},
10973        };
10974        use nautilus_core::{UUID4, UnixNanos};
10975        use ustr::Ustr;
10976
10977        TimeEventMessage::new(
10978            TimeEvent::new(
10979                Ustr::from("test-timer"),
10980                UUID4::new(),
10981                UnixNanos::default(),
10982                UnixNanos::default(),
10983            ),
10984            TimeEventCallback::RustLocal(Rc::new(|_| {})),
10985        )
10986    }
10987
10988    fn stub_trading_command_message() -> TradingCommandMessage {
10989        use nautilus_common::messages::execution::query::QueryAccount;
10990        use nautilus_core::{UUID4, UnixNanos};
10991        use nautilus_model::identifiers::AccountId;
10992
10993        TradingCommandMessage::new(
10994            MessagingSwitchboard::exec_engine_execute(),
10995            TradingCommand::QueryAccount(QueryAccount::new(
10996                TraderId::from("TESTER-001"),
10997                None,
10998                AccountId::from("TEST-001"),
10999                UUID4::new(),
11000                UnixNanos::default(),
11001                None,
11002                None, // correlation_id
11003            )),
11004        )
11005    }
11006
11007    fn stub_exec_event() -> ExecutionEvent {
11008        use nautilus_model::{
11009            enums::{LiquiditySide, OrderSide},
11010            identifiers::{AccountId, InstrumentId, TradeId, VenueOrderId},
11011            reports::FillReport,
11012            types::{Money, Price, Quantity},
11013        };
11014
11015        ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
11016            AccountId::from("TEST-001"),
11017            InstrumentId::from("TEST.VENUE"),
11018            VenueOrderId::from("V-001"),
11019            TradeId::from("T-001"),
11020            OrderSide::Buy,
11021            Quantity::from("1.0"),
11022            Price::from("100.0"),
11023            Money::from("0.01 USD"),
11024            LiquiditySide::Maker,
11025            None,
11026            None,
11027            nautilus_core::UnixNanos::default(),
11028            nautilus_core::UnixNanos::default(),
11029            None,
11030        ))))
11031    }
11032
11033    #[rstest]
11034    fn test_flush_all_pending_drains_buffered_channels() {
11035        let (time_tx, mut time_rx) =
11036            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11037        let (system_evt_tx, mut system_evt_rx) =
11038            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11039        let (system_cmd_tx, mut system_cmd_rx) =
11040            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11041        let (data_evt_tx, mut data_evt_rx) =
11042            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11043        let (data_cmd_tx, mut data_cmd_rx) =
11044            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11045        let (exec_evt_tx, mut exec_evt_rx) =
11046            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11047        let (exec_cmd_tx, mut exec_cmd_rx) =
11048            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11049
11050        let mut pending = PendingEvents::default();
11051
11052        // Pre-load pending with data items
11053        pending.data_evts.push((stub_data_event()).into());
11054        pending.data_cmds.push(stub_data_command().into());
11055
11056        // Pre-load all channel types
11057        time_tx.send((stub_time_event_handler()).into()).unwrap();
11058
11059        let change = SocketStateChange::new(
11060            ClientId::from("BINANCE"),
11061            Some(Venue::from("BINANCE")),
11062            Ustr::from("binance-futures-market-streams"),
11063            SocketState::Connected,
11064        );
11065        system_evt_tx
11066            .send((SystemEvent::SocketState(change)).into())
11067            .unwrap();
11068        system_cmd_tx.send((stub_system_command()).into()).unwrap();
11069        data_evt_tx.send((stub_data_event()).into()).unwrap();
11070        data_cmd_tx.send(stub_data_command().into()).unwrap();
11071        exec_evt_tx.send((stub_exec_event()).into()).unwrap();
11072        exec_cmd_tx
11073            .send(stub_trading_command_message().into())
11074            .unwrap();
11075
11076        flush_all_pending(
11077            &mut pending,
11078            &mut time_rx,
11079            &mut system_evt_rx,
11080            &mut system_cmd_rx,
11081            &mut exec_evt_rx,
11082            &mut exec_cmd_rx,
11083            &mut data_evt_rx,
11084            &mut data_cmd_rx,
11085        );
11086
11087        let system_events = pending
11088            .take_system_events()
11089            .into_iter()
11090            .map(|message| message.dispatch(|value| value))
11091            .collect::<Vec<_>>();
11092        let system_commands = pending
11093            .take_system_commands()
11094            .into_iter()
11095            .map(|message| message.dispatch(|value| value))
11096            .collect::<Vec<_>>();
11097        assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
11098        assert_eq!(system_commands, vec![stub_system_command()]);
11099        assert!(pending.data_evts.is_empty());
11100        assert!(pending.data_cmds.is_empty());
11101        assert!(pending.exec_reports.is_empty());
11102        assert!(pending.exec_cmds.is_empty());
11103        assert!(pending.order_evts.is_empty());
11104        assert!(time_rx.try_recv().is_err());
11105        assert!(system_evt_rx.try_recv().is_err());
11106        assert!(system_cmd_rx.try_recv().is_err());
11107        assert!(data_evt_rx.try_recv().is_err());
11108        assert!(data_cmd_rx.try_recv().is_err());
11109        assert!(exec_evt_rx.try_recv().is_err());
11110        assert!(exec_cmd_rx.try_recv().is_err());
11111    }
11112
11113    fn stub_order_event() -> ExecutionEvent {
11114        use nautilus_model::events::order::spec::OrderSubmittedSpec;
11115
11116        ExecutionEvent::Order(OrderEventAny::Submitted(
11117            OrderSubmittedSpec::builder().build(),
11118        ))
11119    }
11120
11121    fn stub_account_event() -> ExecutionEvent {
11122        use nautilus_core::{UUID4, UnixNanos};
11123        use nautilus_model::{
11124            enums::AccountType, events::account::state::AccountState, identifiers::AccountId,
11125        };
11126
11127        ExecutionEvent::Account(AccountState::new(
11128            AccountId::from("TEST-001"),
11129            AccountType::Cash,
11130            vec![],
11131            vec![],
11132            true,
11133            UUID4::new(),
11134            UnixNanos::default(),
11135            UnixNanos::default(),
11136            None,
11137        ))
11138    }
11139
11140    #[rstest]
11141    fn test_flush_all_pending_routes_order_event_to_order_evts() {
11142        let (_time_tx, mut time_rx) =
11143            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11144        let (_system_evt_tx, mut system_evt_rx) =
11145            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11146        let (_system_cmd_tx, mut system_cmd_rx) =
11147            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11148        let (_data_evt_tx, mut data_evt_rx) =
11149            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11150        let (_data_cmd_tx, mut data_cmd_rx) =
11151            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11152        let (exec_evt_tx, mut exec_evt_rx) =
11153            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11154        let (_exec_cmd_tx, mut exec_cmd_rx) =
11155            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11156
11157        let mut pending = PendingEvents::default();
11158
11159        exec_evt_tx.send((stub_order_event()).into()).unwrap();
11160        exec_evt_tx.send((stub_exec_event()).into()).unwrap();
11161
11162        flush_all_pending(
11163            &mut pending,
11164            &mut time_rx,
11165            &mut system_evt_rx,
11166            &mut system_cmd_rx,
11167            &mut exec_evt_rx,
11168            &mut exec_cmd_rx,
11169            &mut data_evt_rx,
11170            &mut data_cmd_rx,
11171        );
11172
11173        // Both order and report events are drained by pending.drain()
11174        assert!(pending.order_evts.is_empty());
11175        assert!(pending.exec_reports.is_empty());
11176        assert!(exec_evt_rx.try_recv().is_err());
11177    }
11178
11179    #[rstest]
11180    fn test_flush_all_pending_routes_account_event_immediately() {
11181        let (_time_tx, mut time_rx) =
11182            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11183        let (_system_evt_tx, mut system_evt_rx) =
11184            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11185        let (_system_cmd_tx, mut system_cmd_rx) =
11186            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11187        let (_data_evt_tx, mut data_evt_rx) =
11188            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11189        let (_data_cmd_tx, mut data_cmd_rx) =
11190            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11191        let (exec_evt_tx, mut exec_evt_rx) =
11192            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11193        let (_exec_cmd_tx, mut exec_cmd_rx) =
11194            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11195
11196        let mut pending = PendingEvents::default();
11197
11198        exec_evt_tx.send((stub_account_event()).into()).unwrap();
11199
11200        flush_all_pending(
11201            &mut pending,
11202            &mut time_rx,
11203            &mut system_evt_rx,
11204            &mut system_cmd_rx,
11205            &mut exec_evt_rx,
11206            &mut exec_cmd_rx,
11207            &mut data_evt_rx,
11208            &mut data_cmd_rx,
11209        );
11210
11211        // Account events are forwarded immediately, never buffered in pending
11212        assert!(pending.exec_reports.is_empty());
11213        assert!(pending.order_evts.is_empty());
11214        assert!(pending.exec_cmds.is_empty());
11215        assert!(exec_evt_rx.try_recv().is_err());
11216    }
11217
11218    #[rstest]
11219    fn test_pending_is_empty_when_default() {
11220        let pending = PendingEvents::default();
11221
11222        assert!(pending.is_empty());
11223    }
11224
11225    #[rstest]
11226    fn test_pending_is_empty_false_with_data_evt() {
11227        let mut pending = PendingEvents::default();
11228        pending.data_evts.push((stub_data_event()).into());
11229
11230        assert!(!pending.is_empty());
11231    }
11232
11233    #[rstest]
11234    fn test_pending_is_empty_false_with_data_cmd() {
11235        let mut pending = PendingEvents::default();
11236        pending.data_cmds.push(stub_data_command().into());
11237
11238        assert!(!pending.is_empty());
11239    }
11240
11241    #[rstest]
11242    fn test_pending_is_empty_false_with_exec_cmd() {
11243        let mut pending = PendingEvents::default();
11244        pending
11245            .exec_cmds
11246            .push(stub_trading_command_message().into());
11247
11248        assert!(!pending.is_empty());
11249    }
11250
11251    #[rstest]
11252    fn test_pending_drain_preserves_trading_command_target() {
11253        std::thread::spawn(|| {
11254            msgbus::get_message_bus().borrow_mut().dispose();
11255            let risk_commands = Rc::new(RefCell::new(Vec::new()));
11256            let exec_commands = Rc::new(RefCell::new(Vec::new()));
11257
11258            let risk_commands_handler = risk_commands.clone();
11259            msgbus::register_trading_command_endpoint(
11260                MessagingSwitchboard::risk_engine_execute(),
11261                TypedIntoHandler::from(move |command: TradingCommand| {
11262                    risk_commands_handler.borrow_mut().push(command);
11263                }),
11264            );
11265
11266            let exec_commands_handler = exec_commands.clone();
11267            msgbus::register_trading_command_endpoint(
11268                MessagingSwitchboard::exec_engine_execute(),
11269                TypedIntoHandler::from(move |command: TradingCommand| {
11270                    exec_commands_handler.borrow_mut().push(command);
11271                }),
11272            );
11273
11274            let mut pending = PendingEvents::default();
11275            pending.exec_cmds.push(
11276                TradingCommandMessage::new(
11277                    MessagingSwitchboard::risk_engine_execute(),
11278                    TradingCommand::QueryAccount(QueryAccount::new(
11279                        TraderId::from("TESTER-001"),
11280                        None,
11281                        AccountId::from("TEST-001"),
11282                        UUID4::new(),
11283                        UnixNanos::default(),
11284                        None,
11285                        None,
11286                    )),
11287                )
11288                .into(),
11289            );
11290
11291            pending.drain();
11292
11293            assert!(pending.is_empty());
11294            assert_eq!(risk_commands.borrow().len(), 1);
11295            assert!(matches!(
11296                &risk_commands.borrow()[0],
11297                TradingCommand::QueryAccount(_)
11298            ));
11299            assert_eq!(exec_commands.borrow().as_slice(), &[]);
11300        })
11301        .join()
11302        .unwrap();
11303    }
11304
11305    #[rstest]
11306    fn test_pending_is_empty_false_with_exec_report() {
11307        let mut pending = PendingEvents::default();
11308
11309        if let ExecutionEvent::Report(report) = stub_exec_event() {
11310            pending.exec_reports.push((report).into());
11311        }
11312
11313        assert!(!pending.is_empty());
11314    }
11315
11316    #[rstest]
11317    fn test_pending_is_empty_false_with_order_evt() {
11318        let mut pending = PendingEvents::default();
11319
11320        if let ExecutionEvent::Order(order_evt) = stub_order_event() {
11321            pending.order_evts.push((order_evt).into());
11322        }
11323
11324        assert!(!pending.is_empty());
11325    }
11326
11327    fn stub_submitted_batch_event() -> ExecutionEvent {
11328        use nautilus_model::{
11329            events::{OrderSubmittedBatch, order::spec::OrderSubmittedSpec},
11330            identifiers::ClientOrderId,
11331        };
11332
11333        let events = vec![
11334            OrderSubmittedSpec::builder()
11335                .client_order_id(ClientOrderId::from("O-001"))
11336                .build(),
11337            OrderSubmittedSpec::builder()
11338                .client_order_id(ClientOrderId::from("O-002"))
11339                .build(),
11340        ];
11341
11342        ExecutionEvent::OrderSubmittedBatch(OrderSubmittedBatch::new(events))
11343    }
11344
11345    fn stub_canceled_batch_event() -> ExecutionEvent {
11346        use nautilus_model::{
11347            events::{OrderCanceledBatch, order::spec::OrderCanceledSpec},
11348            identifiers::ClientOrderId,
11349        };
11350
11351        let events = vec![
11352            OrderCanceledSpec::builder()
11353                .client_order_id(ClientOrderId::from("O-001"))
11354                .build(),
11355            OrderCanceledSpec::builder()
11356                .client_order_id(ClientOrderId::from("O-002"))
11357                .build(),
11358        ];
11359
11360        ExecutionEvent::OrderCanceledBatch(OrderCanceledBatch::new(events))
11361    }
11362
11363    #[rstest]
11364    fn test_flush_all_pending_buffers_submitted_batch_as_individual_events() {
11365        let (_time_tx, mut time_rx) =
11366            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11367        let (_system_evt_tx, mut system_evt_rx) =
11368            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11369        let (_system_cmd_tx, mut system_cmd_rx) =
11370            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11371        let (_data_evt_tx, mut data_evt_rx) =
11372            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11373        let (_data_cmd_tx, mut data_cmd_rx) =
11374            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11375        let (exec_evt_tx, mut exec_evt_rx) =
11376            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11377        let (_exec_cmd_tx, mut exec_cmd_rx) =
11378            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11379
11380        let mut pending = PendingEvents::default();
11381
11382        exec_evt_tx
11383            .send((stub_submitted_batch_event()).into())
11384            .unwrap();
11385
11386        flush_all_pending(
11387            &mut pending,
11388            &mut time_rx,
11389            &mut system_evt_rx,
11390            &mut system_cmd_rx,
11391            &mut exec_evt_rx,
11392            &mut exec_cmd_rx,
11393            &mut data_evt_rx,
11394            &mut data_cmd_rx,
11395        );
11396
11397        // Batch should be unpacked into individual Submitted events then drained
11398        assert!(pending.order_evts.is_empty());
11399        assert!(exec_evt_rx.try_recv().is_err());
11400    }
11401
11402    #[rstest]
11403    fn test_flush_all_pending_buffers_canceled_batch_as_individual_events() {
11404        let (_time_tx, mut time_rx) =
11405            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11406        let (_system_evt_tx, mut system_evt_rx) =
11407            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11408        let (_system_cmd_tx, mut system_cmd_rx) =
11409            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11410        let (_data_evt_tx, mut data_evt_rx) =
11411            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11412        let (_data_cmd_tx, mut data_cmd_rx) =
11413            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11414        let (exec_evt_tx, mut exec_evt_rx) =
11415            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11416        let (_exec_cmd_tx, mut exec_cmd_rx) =
11417            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11418
11419        let mut pending = PendingEvents::default();
11420
11421        exec_evt_tx
11422            .send((stub_canceled_batch_event()).into())
11423            .unwrap();
11424
11425        flush_all_pending(
11426            &mut pending,
11427            &mut time_rx,
11428            &mut system_evt_rx,
11429            &mut system_cmd_rx,
11430            &mut exec_evt_rx,
11431            &mut exec_cmd_rx,
11432            &mut data_evt_rx,
11433            &mut data_cmd_rx,
11434        );
11435
11436        // Batch should be unpacked into individual Canceled events then drained
11437        assert!(pending.order_evts.is_empty());
11438        assert!(exec_evt_rx.try_recv().is_err());
11439    }
11440
11441    #[rstest]
11442    fn test_flush_all_pending_expands_batch_into_order_evts_before_drain() {
11443        use nautilus_model::identifiers::ClientOrderId;
11444
11445        let (exec_evt_tx, mut exec_evt_rx) =
11446            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11447
11448        exec_evt_tx
11449            .send((stub_canceled_batch_event()).into())
11450            .unwrap();
11451
11452        let mut pending = PendingEvents::default();
11453
11454        while let Ok(evt) = exec_evt_rx.try_recv() {
11455            pending.push_exec_event(evt);
11456        }
11457
11458        let events: Vec<_> = pending
11459            .order_evts
11460            .drain(..)
11461            .map(|event| event.dispatch(|event| event))
11462            .collect();
11463        assert_eq!(events.len(), 2);
11464        assert!(
11465            matches!(&events[0], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-001"))
11466        );
11467        assert!(
11468            matches!(&events[1], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-002"))
11469        );
11470    }
11471
11472    #[rstest]
11473    #[cfg_attr(
11474        not(all(feature = "simulation", madsim)),
11475        tokio::test(start_paused = true)
11476    )]
11477    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11478    async fn test_submission_confirmation_prevents_duplicate_tracking(
11479        #[values(false, true)] report_confirmation: bool,
11480        #[values(false, true)] exhausted: bool,
11481    ) {
11482        let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
11483        let client_order_id = order.client_order_id();
11484        let account_id = AccountId::from("TEST-001");
11485        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11486        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11487            OrderPendingUpdateSpec::builder()
11488                .trader_id(order.trader_id())
11489                .strategy_id(order.strategy_id())
11490                .instrument_id(order.instrument_id())
11491                .client_order_id(client_order_id)
11492                .account_id(account_id)
11493                .build(),
11494        )));
11495        advance_clock(Duration::from_millis(101)).await;
11496        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11497
11498        if exhausted {
11499            advance_clock(Duration::from_millis(101)).await;
11500            let result = node.exec_manager.check_inflight_orders();
11501            assert!(result.events.is_empty());
11502            assert!(result.queries.is_empty());
11503            assert_eq!(
11504                node.exec_manager
11505                    .take_submission_recovery_exhaustions()
11506                    .len(),
11507                1
11508            );
11509        }
11510
11511        if report_confirmation {
11512            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11513                OrderStatusReport::new(
11514                    account_id,
11515                    order.instrument_id(),
11516                    Some(client_order_id),
11517                    venue_order_id,
11518                    Some(order.order_side()),
11519                    order.order_type(),
11520                    order.time_in_force(),
11521                    OrderStatus::Submitted,
11522                    order.quantity(),
11523                    Quantity::zero(3),
11524                    UnixNanos::default(),
11525                    UnixNanos::default(),
11526                    UnixNanos::default(),
11527                    None,
11528                )
11529                .with_price(order.price().unwrap()),
11530            ))));
11531            node.exec_manager
11532                .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11533            advance_clock(Duration::from_millis(101)).await;
11534            let result = node.exec_manager.check_inflight_orders();
11535            assert!(result.queries.is_empty());
11536            assert!(result.events.is_empty());
11537        }
11538
11539        let mut update = OrderUpdatedSpec::builder()
11540            .trader_id(order.trader_id())
11541            .strategy_id(order.strategy_id())
11542            .instrument_id(order.instrument_id())
11543            .client_order_id(client_order_id)
11544            .account_id(account_id)
11545            .quantity(order.quantity())
11546            .price(order.price().unwrap())
11547            .build();
11548
11549        if !report_confirmation {
11550            update.venue_order_id = Some(venue_order_id);
11551        }
11552        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11553        assert_eq!(
11554            node.kernel
11555                .cache
11556                .borrow()
11557                .order(&client_order_id)
11558                .unwrap()
11559                .status(),
11560            OrderStatus::Submitted
11561        );
11562
11563        for _ in 0..3 {
11564            node.exec_manager
11565                .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11566            advance_clock(Duration::from_millis(101)).await;
11567            let result = node.exec_manager.check_inflight_orders();
11568            assert!(result.queries.is_empty());
11569            assert!(result.events.is_empty());
11570            assert!(
11571                node.exec_manager
11572                    .take_submission_recovery_exhaustions()
11573                    .is_empty()
11574            );
11575        }
11576    }
11577
11578    #[rstest]
11579    #[cfg_attr(
11580        not(all(feature = "simulation", madsim)),
11581        tokio::test(start_paused = true)
11582    )]
11583    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11584    async fn test_submission_report_confirmation_preserves_later_command_budget(
11585        #[values(false, true)] cancel: bool,
11586        #[values(false, true)] later_fill: bool,
11587        #[values(false, true)] repeated_report: bool,
11588        #[values(
11589            SubmissionRecoveryPolicy::ResolveLocally,
11590            SubmissionRecoveryPolicy::RetainUnresolved
11591        )]
11592        policy: SubmissionRecoveryPolicy,
11593    ) {
11594        let (mut node, order, instrument) = venue_evidence_node(policy);
11595        let client_order_id = order.client_order_id();
11596        let account_id = AccountId::from("TEST-001");
11597        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11598        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11599            OrderPendingUpdateSpec::builder()
11600                .trader_id(order.trader_id())
11601                .strategy_id(order.strategy_id())
11602                .instrument_id(order.instrument_id())
11603                .client_order_id(client_order_id)
11604                .account_id(account_id)
11605                .build(),
11606        )));
11607        node.exec_manager
11608            .register_inflight(client_order_id, UUID4::new());
11609        advance_clock(Duration::from_millis(101)).await;
11610        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11611        let report = ExecutionReport::Order(Box::new(
11612            OrderStatusReport::new(
11613                account_id,
11614                order.instrument_id(),
11615                Some(client_order_id),
11616                venue_order_id,
11617                Some(order.order_side()),
11618                order.order_type(),
11619                order.time_in_force(),
11620                OrderStatus::Submitted,
11621                order.quantity(),
11622                Quantity::zero(3),
11623                UnixNanos::default(),
11624                UnixNanos::default(),
11625                UnixNanos::default(),
11626                None,
11627            )
11628            .with_price(order.price().unwrap()),
11629        ));
11630        node.process_exec_event(ExecutionEvent::Report(report.clone()));
11631        advance_clock(Duration::from_millis(101)).await;
11632        let result = node.exec_manager.check_inflight_orders();
11633        assert!(result.events.is_empty());
11634        assert!(result.queries.is_empty());
11635
11636        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(
11637            OrderUpdatedSpec::builder()
11638                .trader_id(order.trader_id())
11639                .strategy_id(order.strategy_id())
11640                .instrument_id(order.instrument_id())
11641                .client_order_id(client_order_id)
11642                .account_id(account_id)
11643                .quantity(order.quantity())
11644                .price(order.price().unwrap())
11645                .build(),
11646        )));
11647        let pending = if cancel {
11648            OrderEventAny::PendingCancel(
11649                OrderPendingCancelSpec::builder()
11650                    .trader_id(order.trader_id())
11651                    .strategy_id(order.strategy_id())
11652                    .instrument_id(order.instrument_id())
11653                    .client_order_id(client_order_id)
11654                    .account_id(account_id)
11655                    .build(),
11656            )
11657        } else {
11658            OrderEventAny::PendingUpdate(
11659                OrderPendingUpdateSpec::builder()
11660                    .trader_id(order.trader_id())
11661                    .strategy_id(order.strategy_id())
11662                    .instrument_id(order.instrument_id())
11663                    .client_order_id(client_order_id)
11664                    .account_id(account_id)
11665                    .build(),
11666            )
11667        };
11668        node.process_exec_event(ExecutionEvent::Order(pending));
11669        node.exec_manager
11670            .register_inflight(client_order_id, UUID4::new());
11671
11672        if repeated_report {
11673            node.process_exec_event(ExecutionEvent::Report(report));
11674        }
11675        advance_clock(Duration::from_millis(50)).await;
11676
11677        if later_fill {
11678            let cached = node
11679                .kernel
11680                .cache
11681                .borrow()
11682                .order_owned(&client_order_id)
11683                .unwrap();
11684            let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
11685                .last_qty(Quantity::from("0.400"))
11686                .last_px(order.price().unwrap())
11687                .commission(Money::from("0 USDT"))
11688                .without_position_id()
11689                .build()
11690            else {
11691                unreachable!()
11692            };
11693            fill.venue_order_id = venue_order_id;
11694            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
11695        }
11696        let early = node.exec_manager.check_inflight_orders();
11697        assert!(early.events.is_empty());
11698        assert!(early.queries.is_empty());
11699        advance_clock(Duration::from_millis(51)).await;
11700        let first = node.exec_manager.check_inflight_orders();
11701        assert_eq!(first.queries.len(), usize::from(!repeated_report));
11702        assert!(first.events.is_empty());
11703        advance_clock(Duration::from_millis(101)).await;
11704        let second = node.exec_manager.check_inflight_orders();
11705        assert!(second.queries.is_empty());
11706        if repeated_report {
11707            assert!(second.events.is_empty());
11708        } else {
11709            assert!(
11710                matches!(&second.events[..], [OrderEventAny::Canceled(event)] if event.client_order_id == client_order_id)
11711            );
11712        }
11713        assert!(
11714            node.exec_manager
11715                .take_submission_recovery_exhaustions()
11716                .is_empty()
11717        );
11718    }
11719
11720    #[rstest]
11721    #[cfg_attr(
11722        not(all(feature = "simulation", madsim)),
11723        tokio::test(start_paused = true)
11724    )]
11725    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11726    async fn test_confirmed_submission_command_dispatch_starts_own_budget(
11727        #[values(false, true)] report_confirmation: bool,
11728        #[values(false, true)] cancel: bool,
11729        #[values(false, true)] batch: bool,
11730        #[values(false, true)] pending_before_dispatch: bool,
11731        #[values(false, true)] missing_order_check: bool,
11732        #[values(
11733            SubmissionRecoveryPolicy::ResolveLocally,
11734            SubmissionRecoveryPolicy::RetainUnresolved
11735        )]
11736        policy: SubmissionRecoveryPolicy,
11737    ) {
11738        let (mut node, order, instrument) = venue_evidence_node_with_config(
11739            crate::config::LiveExecutionEngineConfig {
11740                reconciliation: true,
11741                inflight_check_threshold_ms: 100,
11742                inflight_check_retries: 2,
11743                open_check_threshold_ms: 5_000,
11744                open_check_open_only: !missing_order_check,
11745                open_check_lookback_mins: None,
11746                open_check_missing_retries: 5,
11747                single_order_query_delay_ms: 0,
11748                submission_recovery_policy: policy,
11749                ..Default::default()
11750            },
11751            OrderType::Limit,
11752            true,
11753        );
11754        let client_order_id = order.client_order_id();
11755        let client_id = ClientId::from("TEST-EVIDENCE");
11756        let account_id = AccountId::from("TEST-001");
11757        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11758        node.kernel
11759            .exec_engine
11760            .borrow_mut()
11761            .register_client(Box::new(StubExecutionClient::new(
11762                client_id,
11763                account_id,
11764                instrument.id().venue,
11765                OmsType::Netting,
11766                None,
11767            )))
11768            .unwrap();
11769
11770        // Consume part of the original submission budget before venue confirmation
11771        advance_clock(Duration::from_millis(101)).await;
11772        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11773
11774        if report_confirmation {
11775            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11776                OrderPendingUpdateSpec::builder()
11777                    .trader_id(order.trader_id())
11778                    .strategy_id(order.strategy_id())
11779                    .instrument_id(order.instrument_id())
11780                    .client_order_id(client_order_id)
11781                    .account_id(account_id)
11782                    .build(),
11783            )));
11784            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11785                OrderStatusReport::new(
11786                    account_id,
11787                    order.instrument_id(),
11788                    Some(client_order_id),
11789                    venue_order_id,
11790                    Some(order.order_side()),
11791                    order.order_type(),
11792                    order.time_in_force(),
11793                    OrderStatus::Submitted,
11794                    order.quantity(),
11795                    Quantity::zero(3),
11796                    UnixNanos::default(),
11797                    UnixNanos::default(),
11798                    UnixNanos::default(),
11799                    None,
11800                )
11801                .with_price(order.price().unwrap()),
11802            ))));
11803        }
11804
11805        let mut update = OrderUpdatedSpec::builder()
11806            .trader_id(order.trader_id())
11807            .strategy_id(order.strategy_id())
11808            .instrument_id(order.instrument_id())
11809            .client_order_id(client_order_id)
11810            .account_id(account_id)
11811            .quantity(order.quantity())
11812            .price(order.price().unwrap())
11813            .build();
11814
11815        if !report_confirmation {
11816            update.venue_order_id = Some(venue_order_id);
11817        }
11818        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11819        assert_eq!(
11820            node.kernel
11821                .cache
11822                .borrow()
11823                .order(&client_order_id)
11824                .unwrap()
11825                .status(),
11826            OrderStatus::Submitted,
11827        );
11828        advance_clock(Duration::from_millis(101)).await;
11829        let confirmed = node.exec_manager.check_inflight_orders();
11830        assert!(confirmed.queries.is_empty());
11831        assert!(confirmed.events.is_empty());
11832
11833        let pending = if cancel {
11834            OrderEventAny::PendingCancel(
11835                OrderPendingCancelSpec::builder()
11836                    .trader_id(order.trader_id())
11837                    .strategy_id(order.strategy_id())
11838                    .instrument_id(order.instrument_id())
11839                    .client_order_id(client_order_id)
11840                    .account_id(account_id)
11841                    .build(),
11842            )
11843        } else {
11844            OrderEventAny::PendingUpdate(
11845                OrderPendingUpdateSpec::builder()
11846                    .trader_id(order.trader_id())
11847                    .strategy_id(order.strategy_id())
11848                    .instrument_id(order.instrument_id())
11849                    .client_order_id(client_order_id)
11850                    .account_id(account_id)
11851                    .build(),
11852            )
11853        };
11854        let command = venue_evidence_command(&order, cancel, batch);
11855
11856        if pending_before_dispatch {
11857            node.process_exec_event(ExecutionEvent::Order(pending.clone()));
11858        }
11859        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11860            MessagingSwitchboard::exec_engine_execute(),
11861            command,
11862        )));
11863        advance_clock(Duration::from_millis(50)).await;
11864
11865        if missing_order_check {
11866            let engine = node.kernel.exec_engine.borrow();
11867            let clients = engine.get_all_clients();
11868            let check = node
11869                .exec_manager
11870                .prepare_open_order_report_check(UUID4::new(), &clients);
11871            assert_eq!(check.filtered_orders.len(), 1);
11872            let missing = node.exec_manager.reconcile_open_order_reports(
11873                &check,
11874                Vec::new(),
11875                &IndexSet::from([client_id]),
11876                &IndexSet::new(),
11877                &clients,
11878            );
11879            assert!(missing.events.is_empty());
11880            assert!(missing.targeted_queries.is_empty());
11881            assert_eq!(
11882                node.exec_manager.recon_check_retry_count(&client_order_id),
11883                1
11884            );
11885        }
11886        let early = node.exec_manager.check_inflight_orders();
11887        assert!(early.queries.is_empty());
11888        assert!(early.events.is_empty());
11889
11890        if !pending_before_dispatch {
11891            // The queued pending event must not be needed to register the command,
11892            // nor shift its timeout start from dispatch to event application.
11893            assert_eq!(
11894                node.kernel
11895                    .cache
11896                    .borrow()
11897                    .order(&client_order_id)
11898                    .unwrap()
11899                    .status(),
11900                OrderStatus::Submitted,
11901            );
11902            node.process_exec_event(ExecutionEvent::Order(pending));
11903        }
11904
11905        if policy == SubmissionRecoveryPolicy::RetainUnresolved {
11906            node.exec_manager
11907                .register_submission(order.init_event(), Some(client_id));
11908        }
11909        advance_clock(Duration::from_millis(51)).await;
11910        let first = node.exec_manager.check_inflight_orders();
11911        assert_eq!(first.queries.len(), 1);
11912        assert!(
11913            matches!(&first.queries[0], TradingCommand::QueryOrder(query)
11914            if query.client_order_id == client_order_id)
11915        );
11916        assert!(first.events.is_empty());
11917        advance_clock(Duration::from_millis(101)).await;
11918        let exhausted = node.exec_manager.check_inflight_orders();
11919        assert!(exhausted.queries.is_empty());
11920        assert!(
11921            matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11922                if event.client_order_id == client_order_id)
11923        );
11924        assert!(
11925            node.exec_manager
11926                .take_submission_recovery_exhaustions()
11927                .is_empty()
11928        );
11929        node.process_reconciliation_events(&exhausted.events);
11930        assert_eq!(
11931            node.kernel
11932                .cache
11933                .borrow()
11934                .order(&client_order_id)
11935                .unwrap()
11936                .status(),
11937            OrderStatus::Canceled,
11938        );
11939    }
11940
11941    #[rstest]
11942    #[cfg_attr(
11943        not(all(feature = "simulation", madsim)),
11944        tokio::test(start_paused = true)
11945    )]
11946    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11947    async fn test_submission_command_report_cleanup_matches_default(
11948        #[values(OrderStatus::Submitted, OrderStatus::PartiallyFilled)] report_status: OrderStatus,
11949        #[values(
11950            SubmissionRecoveryPolicy::ResolveLocally,
11951            SubmissionRecoveryPolicy::RetainUnresolved
11952        )]
11953        policy: SubmissionRecoveryPolicy,
11954    ) {
11955        let (mut node, order, instrument) = venue_evidence_node(policy);
11956        let client_order_id = order.client_order_id();
11957        let client_id = ClientId::from("TEST-EVIDENCE");
11958        let account_id = AccountId::from("TEST-001");
11959        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11960        node.kernel
11961            .exec_engine
11962            .borrow_mut()
11963            .register_client(Box::new(StubExecutionClient::new(
11964                client_id,
11965                account_id,
11966                instrument.id().venue,
11967                OmsType::Netting,
11968                None,
11969            )))
11970            .unwrap();
11971        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
11972            &order,
11973            account_id,
11974            venue_order_id,
11975        )));
11976        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11977            MessagingSwitchboard::exec_engine_execute(),
11978            venue_evidence_command(&order, true, false),
11979        )));
11980        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11981            OrderPendingCancelSpec::builder()
11982                .trader_id(order.trader_id())
11983                .strategy_id(order.strategy_id())
11984                .instrument_id(order.instrument_id())
11985                .client_order_id(client_order_id)
11986                .account_id(account_id)
11987                .venue_order_id(venue_order_id)
11988                .build(),
11989        )));
11990        advance_clock(Duration::from_millis(101)).await;
11991        let first = node.exec_manager.check_inflight_orders();
11992        assert_eq!(first.queries.len(), 1);
11993        assert!(first.events.is_empty());
11994
11995        let filled_qty = if report_status == OrderStatus::PartiallyFilled {
11996            Quantity::from("0.400")
11997        } else {
11998            Quantity::zero(3)
11999        };
12000
12001        let report = OrderStatusReport::new(
12002            account_id,
12003            order.instrument_id(),
12004            Some(client_order_id),
12005            venue_order_id,
12006            Some(order.order_side()),
12007            order.order_type(),
12008            order.time_in_force(),
12009            report_status,
12010            order.quantity(),
12011            filled_qty,
12012            UnixNanos::default(),
12013            UnixNanos::default(),
12014            UnixNanos::default(),
12015            None,
12016        )
12017        .with_price(order.price().unwrap())
12018        .with_avg_px(dec!(100.00));
12019        node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12020            report,
12021        ))));
12022        advance_clock(Duration::from_millis(101)).await;
12023        let exhausted = node.exec_manager.check_inflight_orders();
12024
12025        assert!(exhausted.queries.is_empty());
12026        assert!(exhausted.events.is_empty());
12027        assert!(
12028            node.exec_manager
12029                .take_submission_recovery_exhaustions()
12030                .is_empty()
12031        );
12032        let cache = node.kernel.cache.borrow();
12033        let cached = cache.order_ref(&client_order_id).unwrap();
12034        assert_eq!(cached.status(), OrderStatus::PendingCancel);
12035        assert_eq!(cached.filled_qty(), filled_qty);
12036    }
12037
12038    #[rstest]
12039    #[case::accepted_modify(false, "accepted")]
12040    #[case::accepted_cancel(true, "accepted")]
12041    #[case::accepted_batch_modify(false, "accepted_batch")]
12042    #[case::accepted_batch_cancel(true, "accepted_batch")]
12043    #[case::updated_cancel(true, "updated")]
12044    #[case::triggered_cancel(true, "triggered")]
12045    #[cfg_attr(
12046        not(all(feature = "simulation", madsim)),
12047        tokio::test(start_paused = true)
12048    )]
12049    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12050    async fn test_submission_acknowledgement_retires_overlapping_command_recovery(
12051        #[case] cancel: bool,
12052        #[case] evidence: &str,
12053        #[values(false, true)] batch: bool,
12054        #[values(false, true)] report_first: bool,
12055        #[values(
12056            SubmissionRecoveryPolicy::ResolveLocally,
12057            SubmissionRecoveryPolicy::RetainUnresolved
12058        )]
12059        policy: SubmissionRecoveryPolicy,
12060    ) {
12061        use nautilus_model::events::order::spec::OrderTriggeredSpec;
12062
12063        let triggered = evidence == "triggered";
12064
12065        let order_type = if triggered {
12066            OrderType::StopLimit
12067        } else {
12068            OrderType::Limit
12069        };
12070        let (mut node, order, instrument) = venue_evidence_node_with_order_type(policy, order_type);
12071        let client_order_id = order.client_order_id();
12072        let account_id = AccountId::from("TEST-001");
12073        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12074        node.kernel
12075            .exec_engine
12076            .borrow_mut()
12077            .register_client(Box::new(StubExecutionClient::new(
12078                ClientId::from("TEST-EVIDENCE"),
12079                account_id,
12080                instrument.id().venue,
12081                OmsType::Netting,
12082                None,
12083            )))
12084            .unwrap();
12085        advance_clock(Duration::from_millis(101)).await;
12086        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12087
12088        if triggered || report_first {
12089            // Evidence can overtake a different, queued command intent
12090            let previous_pending = if cancel {
12091                OrderEventAny::PendingUpdate(
12092                    OrderPendingUpdateSpec::builder()
12093                        .trader_id(order.trader_id())
12094                        .strategy_id(order.strategy_id())
12095                        .instrument_id(order.instrument_id())
12096                        .client_order_id(client_order_id)
12097                        .account_id(account_id)
12098                        .build(),
12099                )
12100            } else {
12101                OrderEventAny::PendingCancel(
12102                    OrderPendingCancelSpec::builder()
12103                        .trader_id(order.trader_id())
12104                        .strategy_id(order.strategy_id())
12105                        .instrument_id(order.instrument_id())
12106                        .client_order_id(client_order_id)
12107                        .account_id(account_id)
12108                        .build(),
12109                )
12110            };
12111            node.process_exec_event(ExecutionEvent::Order(previous_pending));
12112        }
12113        advance_clock(Duration::from_millis(10)).await;
12114        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12115            MessagingSwitchboard::exec_engine_execute(),
12116            venue_evidence_command(&order, cancel, batch),
12117        )));
12118
12119        if report_first {
12120            let report = OrderStatusReport::new(
12121                account_id,
12122                instrument.id(),
12123                Some(client_order_id),
12124                venue_order_id,
12125                Some(order.order_side()),
12126                order.order_type(),
12127                order.time_in_force(),
12128                OrderStatus::Submitted,
12129                order.quantity(),
12130                Quantity::zero(3),
12131                UnixNanos::default(),
12132                UnixNanos::default(),
12133                UnixNanos::default(),
12134                None,
12135            )
12136            .with_price(order.price().unwrap());
12137            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12138                report,
12139            ))));
12140            assert!(
12141                node.kernel
12142                    .cache
12143                    .borrow()
12144                    .order(&client_order_id)
12145                    .unwrap()
12146                    .venue_order_id()
12147                    .is_none()
12148            );
12149        }
12150        advance_clock(Duration::from_millis(40)).await;
12151        let acknowledgement = if triggered {
12152            OrderEventAny::Triggered(
12153                OrderTriggeredSpec::builder()
12154                    .trader_id(order.trader_id())
12155                    .strategy_id(order.strategy_id())
12156                    .instrument_id(order.instrument_id())
12157                    .client_order_id(client_order_id)
12158                    .account_id(account_id)
12159                    .venue_order_id(venue_order_id)
12160                    .build(),
12161            )
12162        } else if evidence == "updated" {
12163            OrderEventAny::Updated(
12164                OrderUpdatedSpec::builder()
12165                    .trader_id(order.trader_id())
12166                    .strategy_id(order.strategy_id())
12167                    .instrument_id(order.instrument_id())
12168                    .client_order_id(client_order_id)
12169                    .account_id(account_id)
12170                    .venue_order_id(venue_order_id)
12171                    .quantity(order.quantity())
12172                    .price(order.price().unwrap())
12173                    .build(),
12174            )
12175        } else {
12176            TestOrderEventStubs::accepted(&order, account_id, venue_order_id)
12177        };
12178
12179        if evidence == "accepted_batch" {
12180            let OrderEventAny::Accepted(accepted) = acknowledgement else {
12181                unreachable!()
12182            };
12183            node.process_exec_event(ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(
12184                vec![accepted],
12185            )));
12186        } else {
12187            node.process_exec_event(ExecutionEvent::Order(acknowledgement));
12188        }
12189        assert_eq!(
12190            node.kernel
12191                .cache
12192                .borrow()
12193                .order(&client_order_id)
12194                .unwrap()
12195                .status(),
12196            match evidence {
12197                "triggered" => OrderStatus::Triggered,
12198                "updated" => OrderStatus::Submitted,
12199                _ => OrderStatus::Accepted,
12200            }
12201        );
12202        advance_clock(Duration::from_millis(20)).await;
12203        let pending = if cancel {
12204            OrderEventAny::PendingCancel(
12205                OrderPendingCancelSpec::builder()
12206                    .trader_id(order.trader_id())
12207                    .strategy_id(order.strategy_id())
12208                    .instrument_id(order.instrument_id())
12209                    .client_order_id(client_order_id)
12210                    .account_id(account_id)
12211                    .build(),
12212            )
12213        } else {
12214            OrderEventAny::PendingUpdate(
12215                OrderPendingUpdateSpec::builder()
12216                    .trader_id(order.trader_id())
12217                    .strategy_id(order.strategy_id())
12218                    .instrument_id(order.instrument_id())
12219                    .client_order_id(client_order_id)
12220                    .account_id(account_id)
12221                    .build(),
12222            )
12223        };
12224        node.process_exec_event(ExecutionEvent::Order(pending));
12225        let early = node.exec_manager.check_inflight_orders();
12226        assert!(early.queries.is_empty());
12227        assert!(early.events.is_empty());
12228        advance_clock(Duration::from_millis(41)).await;
12229        let first = node.exec_manager.check_inflight_orders();
12230        // The default does not clear inflight tracking on Triggered events.
12231        let recover_command =
12232            triggered && !report_first && policy == SubmissionRecoveryPolicy::ResolveLocally;
12233        assert_eq!(first.queries.len(), usize::from(recover_command));
12234        assert!(first.events.is_empty());
12235        advance_clock(Duration::from_millis(101)).await;
12236        let exhausted = node.exec_manager.check_inflight_orders();
12237        assert!(exhausted.queries.is_empty());
12238        assert!(
12239            node.exec_manager
12240                .take_submission_recovery_exhaustions()
12241                .is_empty()
12242        );
12243
12244        if recover_command {
12245            assert!(
12246                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12247                if event.client_order_id == client_order_id)
12248            );
12249        } else {
12250            assert!(exhausted.events.is_empty());
12251        }
12252    }
12253
12254    #[rstest]
12255    #[case::submission_only(None, false)]
12256    #[case::modify(Some(false), false)]
12257    #[case::cancel(Some(true), false)]
12258    #[case::batch_modify(Some(false), true)]
12259    #[case::batch_cancel(Some(true), true)]
12260    #[cfg_attr(
12261        not(all(feature = "simulation", madsim)),
12262        tokio::test(start_paused = true)
12263    )]
12264    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12265    async fn test_unconfirmed_submission_bulk_confirmation_before_registration(
12266        #[case] cancel: Option<bool>,
12267        #[case] batch: bool,
12268        #[values(false, true)] missing_order_check: bool,
12269        #[values(
12270            SubmissionRecoveryPolicy::ResolveLocally,
12271            SubmissionRecoveryPolicy::RetainUnresolved
12272        )]
12273        policy: SubmissionRecoveryPolicy,
12274    ) {
12275        let (mut node, order, instrument) = venue_evidence_node_with_config(
12276            crate::config::LiveExecutionEngineConfig {
12277                reconciliation: true,
12278                inflight_check_threshold_ms: 100,
12279                inflight_check_retries: 2,
12280                open_check_threshold_ms: 1,
12281                open_check_open_only: false,
12282                open_check_lookback_mins: None,
12283                open_check_missing_retries: 5,
12284                single_order_query_delay_ms: 0,
12285                submission_recovery_policy: policy,
12286                ..Default::default()
12287            },
12288            OrderType::Limit,
12289            false,
12290        );
12291        let client_order_id = order.client_order_id();
12292        let client_id = ClientId::from("TEST-EVIDENCE");
12293        let account_id = AccountId::from("TEST-001");
12294        node.kernel
12295            .exec_engine
12296            .borrow_mut()
12297            .register_client(Box::new(StubExecutionClient::new(
12298                client_id,
12299                account_id,
12300                instrument.id().venue,
12301                OmsType::Netting,
12302                None,
12303            )))
12304            .unwrap();
12305        let queries = Rc::new(RefCell::new(Vec::new()));
12306        let client = OrderReportClient {
12307            client_id,
12308            report: OrderStatusReport::new(
12309                account_id,
12310                instrument.id(),
12311                Some(client_order_id),
12312                VenueOrderId::from("V-VENUE-EVIDENCE"),
12313                Some(order.order_side()),
12314                order.order_type(),
12315                order.time_in_force(),
12316                OrderStatus::Submitted,
12317                order.quantity(),
12318                Quantity::zero(3),
12319                UnixNanos::default(),
12320                UnixNanos::default(),
12321                UnixNanos::default(),
12322                None,
12323            )
12324            .with_price(order.price().unwrap()),
12325            targeted: false,
12326            queries: queries.clone(),
12327        };
12328        advance_clock(Duration::from_millis(2)).await;
12329        let events = node.exec_manager.check_open_orders(&[&client]).await;
12330        assert!(events.is_empty());
12331        assert_eq!(*queries.borrow(), vec!["bulk"]);
12332        assert_eq!(
12333            node.kernel
12334                .cache
12335                .borrow()
12336                .order(&client_order_id)
12337                .unwrap()
12338                .status(),
12339            OrderStatus::Submitted
12340        );
12341        advance_clock(Duration::from_millis(2)).await;
12342
12343        if missing_order_check {
12344            let engine = node.kernel.exec_engine.borrow();
12345            let clients = engine.get_all_clients();
12346            let check = node
12347                .exec_manager
12348                .prepare_open_order_report_check(UUID4::new(), &clients);
12349            assert_eq!(check.filtered_orders.len(), 1);
12350            let missing = node.exec_manager.reconcile_open_order_reports(
12351                &check,
12352                Vec::new(),
12353                &IndexSet::from([client_id]),
12354                &IndexSet::new(),
12355                &clients,
12356            );
12357            assert!(missing.events.is_empty());
12358            assert!(missing.targeted_queries.is_empty());
12359            assert_eq!(
12360                node.exec_manager.recon_check_retry_count(&client_order_id),
12361                1
12362            );
12363        }
12364        node.exec_manager
12365            .register_submission(order.init_event(), Some(client_id));
12366        advance_clock(Duration::from_millis(101)).await;
12367        let duplicate = node.exec_manager.check_inflight_orders();
12368        assert!(duplicate.events.is_empty());
12369        assert_eq!(
12370            duplicate.queries.len(),
12371            usize::from(policy == SubmissionRecoveryPolicy::ResolveLocally)
12372        );
12373
12374        if let Some(cancel) = cancel {
12375            node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12376                MessagingSwitchboard::exec_engine_execute(),
12377                venue_evidence_command(&order, cancel, batch),
12378            )));
12379            let pending = if cancel {
12380                OrderEventAny::PendingCancel(
12381                    OrderPendingCancelSpec::builder()
12382                        .trader_id(order.trader_id())
12383                        .strategy_id(order.strategy_id())
12384                        .instrument_id(order.instrument_id())
12385                        .client_order_id(client_order_id)
12386                        .account_id(account_id)
12387                        .build(),
12388                )
12389            } else {
12390                OrderEventAny::PendingUpdate(
12391                    OrderPendingUpdateSpec::builder()
12392                        .trader_id(order.trader_id())
12393                        .strategy_id(order.strategy_id())
12394                        .instrument_id(order.instrument_id())
12395                        .client_order_id(client_order_id)
12396                        .account_id(account_id)
12397                        .build(),
12398                )
12399            };
12400            node.process_exec_event(ExecutionEvent::Order(pending));
12401        }
12402        advance_clock(Duration::from_millis(101)).await;
12403        let first = node.exec_manager.check_inflight_orders();
12404        assert_eq!(first.queries.len(), usize::from(cancel.is_some()));
12405
12406        if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
12407            assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
12408                if event.client_order_id == client_order_id));
12409        } else {
12410            assert!(first.events.is_empty());
12411        }
12412        node.process_reconciliation_events(&first.events);
12413        advance_clock(Duration::from_millis(101)).await;
12414        let exhausted = node.exec_manager.check_inflight_orders();
12415        assert!(exhausted.queries.is_empty());
12416
12417        if cancel.is_some() {
12418            assert!(
12419                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12420                if event.client_order_id == client_order_id)
12421            );
12422        } else {
12423            assert!(exhausted.events.is_empty());
12424        }
12425        assert!(
12426            node.exec_manager
12427                .take_submission_recovery_exhaustions()
12428                .is_empty()
12429        );
12430    }
12431
12432    #[rstest]
12433    #[case::submission_only(None, false)]
12434    #[case::modify(Some(false), false)]
12435    #[case::cancel(Some(true), false)]
12436    #[case::batch_modify(Some(false), true)]
12437    #[case::batch_cancel(Some(true), true)]
12438    #[cfg_attr(
12439        not(all(feature = "simulation", madsim)),
12440        tokio::test(start_paused = true)
12441    )]
12442    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12443    async fn test_unconfirmed_submission_query_report_confirms_before_exhaustion(
12444        #[case] cancel: Option<bool>,
12445        #[case] batch: bool,
12446        #[values(false, true)] targeted: bool,
12447        #[values(
12448            SubmissionRecoveryPolicy::ResolveLocally,
12449            SubmissionRecoveryPolicy::RetainUnresolved
12450        )]
12451        policy: SubmissionRecoveryPolicy,
12452    ) {
12453        let (mut node, order, instrument) = venue_evidence_node_with_config(
12454            crate::config::LiveExecutionEngineConfig {
12455                reconciliation: true,
12456                inflight_check_threshold_ms: 100,
12457                inflight_check_retries: 2,
12458                open_check_threshold_ms: 1,
12459                open_check_open_only: false,
12460                open_check_lookback_mins: None,
12461                open_check_missing_retries: 1,
12462                single_order_query_delay_ms: 0,
12463                submission_recovery_policy: policy,
12464                ..Default::default()
12465            },
12466            OrderType::Limit,
12467            true,
12468        );
12469        let client_order_id = order.client_order_id();
12470        let client_id = ClientId::from("TEST-EVIDENCE");
12471        let account_id = AccountId::from("TEST-001");
12472        node.kernel
12473            .exec_engine
12474            .borrow_mut()
12475            .register_client(Box::new(StubExecutionClient::new(
12476                client_id,
12477                account_id,
12478                instrument.id().venue,
12479                OmsType::Netting,
12480                None,
12481            )))
12482            .unwrap();
12483        advance_clock(Duration::from_millis(101)).await;
12484        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12485        advance_clock(Duration::from_millis(10)).await;
12486
12487        if let Some(cancel) = cancel {
12488            node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12489                MessagingSwitchboard::exec_engine_execute(),
12490                venue_evidence_command(&order, cancel, batch),
12491            )));
12492            let pending = if cancel {
12493                OrderEventAny::PendingCancel(
12494                    OrderPendingCancelSpec::builder()
12495                        .trader_id(order.trader_id())
12496                        .strategy_id(order.strategy_id())
12497                        .instrument_id(order.instrument_id())
12498                        .client_order_id(client_order_id)
12499                        .account_id(account_id)
12500                        .build(),
12501                )
12502            } else {
12503                OrderEventAny::PendingUpdate(
12504                    OrderPendingUpdateSpec::builder()
12505                        .trader_id(order.trader_id())
12506                        .strategy_id(order.strategy_id())
12507                        .instrument_id(order.instrument_id())
12508                        .client_order_id(client_order_id)
12509                        .account_id(account_id)
12510                        .build(),
12511                )
12512            };
12513            node.process_exec_event(ExecutionEvent::Order(pending));
12514        }
12515        let queries = Rc::new(RefCell::new(Vec::new()));
12516        let client = OrderReportClient {
12517            client_id,
12518            report: OrderStatusReport::new(
12519                account_id,
12520                order.instrument_id(),
12521                Some(client_order_id),
12522                VenueOrderId::from("V-VENUE-EVIDENCE"),
12523                Some(order.order_side()),
12524                order.order_type(),
12525                order.time_in_force(),
12526                OrderStatus::Submitted,
12527                order.quantity(),
12528                Quantity::zero(3),
12529                UnixNanos::default(),
12530                UnixNanos::default(),
12531                UnixNanos::default(),
12532                None,
12533            )
12534            .with_price(order.price().unwrap()),
12535            targeted,
12536            queries: queries.clone(),
12537        };
12538        advance_clock(Duration::from_millis(40)).await;
12539        let events = node.exec_manager.check_open_orders(&[&client]).await;
12540        assert!(events.is_empty());
12541        assert_eq!(
12542            *queries.borrow(),
12543            if targeted {
12544                vec!["bulk", "targeted"]
12545            } else {
12546                vec!["bulk"]
12547            }
12548        );
12549        node.process_reconciliation_events(&events);
12550        advance_clock(Duration::from_millis(61)).await;
12551        let first = node.exec_manager.check_inflight_orders();
12552        let recover_command =
12553            cancel.is_some() && policy == SubmissionRecoveryPolicy::ResolveLocally;
12554        assert_eq!(first.queries.len(), usize::from(recover_command));
12555
12556        if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
12557            assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
12558                if event.client_order_id == client_order_id));
12559        } else {
12560            assert!(first.events.is_empty());
12561        }
12562        node.process_reconciliation_events(&first.events);
12563        advance_clock(Duration::from_millis(101)).await;
12564        let exhausted = node.exec_manager.check_inflight_orders();
12565        assert!(exhausted.queries.is_empty());
12566
12567        if recover_command {
12568            assert!(
12569                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12570                if event.client_order_id == client_order_id)
12571            );
12572        } else {
12573            assert!(exhausted.events.is_empty());
12574        }
12575        assert!(
12576            node.exec_manager
12577                .take_submission_recovery_exhaustions()
12578                .is_empty()
12579        );
12580    }
12581
12582    #[rstest]
12583    #[case::local_update_before_pending("updated")]
12584    #[case::modify_rejected_after_pending("modify_rejected")]
12585    #[case::cancel_rejected_after_pending("cancel_rejected")]
12586    #[case::modify_rejected_after_cancel_pending("modify_rejected_after_cancel_pending")]
12587    #[cfg_attr(
12588        not(all(feature = "simulation", madsim)),
12589        tokio::test(start_paused = true)
12590    )]
12591    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12592    async fn test_unconfirmed_submission_completed_command_does_not_revive_budget(
12593        #[case] completion: &str,
12594        #[values(
12595            SubmissionRecoveryPolicy::ResolveLocally,
12596            SubmissionRecoveryPolicy::RetainUnresolved
12597        )]
12598        policy: SubmissionRecoveryPolicy,
12599    ) {
12600        use nautilus_model::events::order::spec::{
12601            OrderCancelRejectedSpec, OrderModifyRejectedSpec,
12602        };
12603
12604        let (mut node, order, instrument) = venue_evidence_node(policy);
12605        let client_order_id = order.client_order_id();
12606        let client_id = ClientId::from("TEST-EVIDENCE");
12607        let account_id = AccountId::from("TEST-001");
12608        let cancel = completion == "cancel_rejected";
12609        let later_cancel_pending = completion == "modify_rejected_after_cancel_pending";
12610        node.kernel
12611            .exec_engine
12612            .borrow_mut()
12613            .register_client(Box::new(StubExecutionClient::new(
12614                client_id,
12615                account_id,
12616                instrument.id().venue,
12617                OmsType::Netting,
12618                None,
12619            )))
12620            .unwrap();
12621        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12622            MessagingSwitchboard::exec_engine_execute(),
12623            venue_evidence_command(&order, cancel, false),
12624        )));
12625        advance_clock(Duration::from_millis(10)).await;
12626
12627        if completion != "updated" {
12628            let pending = if cancel {
12629                OrderEventAny::PendingCancel(
12630                    OrderPendingCancelSpec::builder()
12631                        .trader_id(order.trader_id())
12632                        .strategy_id(order.strategy_id())
12633                        .instrument_id(order.instrument_id())
12634                        .client_order_id(client_order_id)
12635                        .account_id(account_id)
12636                        .build(),
12637                )
12638            } else {
12639                OrderEventAny::PendingUpdate(
12640                    OrderPendingUpdateSpec::builder()
12641                        .trader_id(order.trader_id())
12642                        .strategy_id(order.strategy_id())
12643                        .instrument_id(order.instrument_id())
12644                        .client_order_id(client_order_id)
12645                        .account_id(account_id)
12646                        .build(),
12647                )
12648            };
12649            node.process_exec_event(ExecutionEvent::Order(pending));
12650        }
12651
12652        if later_cancel_pending {
12653            // The cancel is pending in native state, but has not dispatched
12654            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12655                OrderPendingCancelSpec::builder()
12656                    .trader_id(order.trader_id())
12657                    .strategy_id(order.strategy_id())
12658                    .instrument_id(order.instrument_id())
12659                    .client_order_id(client_order_id)
12660                    .account_id(account_id)
12661                    .build(),
12662            )));
12663        }
12664        advance_clock(Duration::from_millis(10)).await;
12665        let completed = match completion {
12666            "updated" => OrderEventAny::Updated(
12667                OrderUpdatedSpec::builder()
12668                    .trader_id(order.trader_id())
12669                    .strategy_id(order.strategy_id())
12670                    .instrument_id(order.instrument_id())
12671                    .client_order_id(client_order_id)
12672                    .account_id(account_id)
12673                    .quantity(order.quantity())
12674                    .price(Price::from("101.00"))
12675                    .build(),
12676            ),
12677            "modify_rejected" | "modify_rejected_after_cancel_pending" => {
12678                OrderEventAny::ModifyRejected(
12679                    OrderModifyRejectedSpec::builder()
12680                        .trader_id(order.trader_id())
12681                        .strategy_id(order.strategy_id())
12682                        .instrument_id(order.instrument_id())
12683                        .client_order_id(client_order_id)
12684                        .account_id(account_id)
12685                        .build(),
12686                )
12687            }
12688            "cancel_rejected" => OrderEventAny::CancelRejected(
12689                OrderCancelRejectedSpec::builder()
12690                    .trader_id(order.trader_id())
12691                    .strategy_id(order.strategy_id())
12692                    .instrument_id(order.instrument_id())
12693                    .client_order_id(client_order_id)
12694                    .account_id(account_id)
12695                    .build(),
12696            ),
12697            _ => unreachable!(),
12698        };
12699        node.process_exec_event(ExecutionEvent::Order(completed));
12700        assert_eq!(
12701            node.kernel
12702                .cache
12703                .borrow()
12704                .order(&client_order_id)
12705                .unwrap()
12706                .status(),
12707            if later_cancel_pending {
12708                OrderStatus::PendingCancel
12709            } else {
12710                OrderStatus::Submitted
12711            }
12712        );
12713
12714        // A newer pending event is queued, but that command has not dispatched
12715        advance_clock(Duration::from_millis(10)).await;
12716
12717        if !later_cancel_pending {
12718            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
12719                OrderPendingUpdateSpec::builder()
12720                    .trader_id(order.trader_id())
12721                    .strategy_id(order.strategy_id())
12722                    .instrument_id(order.instrument_id())
12723                    .client_order_id(client_order_id)
12724                    .account_id(account_id)
12725                    .build(),
12726            )));
12727        }
12728        advance_clock(Duration::from_millis(10)).await;
12729        let cached = node
12730            .kernel
12731            .cache
12732            .borrow()
12733            .order_owned(&client_order_id)
12734            .unwrap();
12735        let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
12736            .last_qty(Quantity::from("0.400"))
12737            .last_px(order.price().unwrap())
12738            .commission(Money::from("0 USDT"))
12739            .without_position_id()
12740            .build()
12741        else {
12742            unreachable!()
12743        };
12744        fill.venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12745        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
12746
12747        for _ in 0..3 {
12748            advance_clock(Duration::from_millis(101)).await;
12749            let result = node.exec_manager.check_inflight_orders();
12750            assert!(result.queries.is_empty());
12751            assert!(result.events.is_empty());
12752            assert!(
12753                node.exec_manager
12754                    .take_submission_recovery_exhaustions()
12755                    .is_empty()
12756            );
12757        }
12758        assert_eq!(
12759            node.kernel
12760                .cache
12761                .borrow()
12762                .order(&client_order_id)
12763                .unwrap()
12764                .status(),
12765            if later_cancel_pending {
12766                OrderStatus::PendingCancel
12767            } else {
12768                OrderStatus::PendingUpdate
12769            }
12770        );
12771    }
12772
12773    #[rstest]
12774    #[case::wrong_account("account")]
12775    #[case::wrong_instrument("instrument")]
12776    #[case::conflicting_report_venue("report_venue")]
12777    #[case::unapplied_venue_update("update_venue")]
12778    #[case::local_update("local_update")]
12779    #[cfg_attr(
12780        not(all(feature = "simulation", madsim)),
12781        tokio::test(start_paused = true)
12782    )]
12783    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12784    async fn test_submission_confirmation_requires_matching_venue_evidence(#[case] invalid: &str) {
12785        let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
12786        let client_order_id = order.client_order_id();
12787        let account_id = AccountId::from("TEST-001");
12788        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12789
12790        if invalid.ends_with("venue") {
12791            insert_accepted_limit_order_in_node(
12792                &node,
12793                account_id,
12794                ClientId::from("TEST-EVIDENCE"),
12795                order.instrument_id(),
12796                ClientOrderId::from("O-OTHER"),
12797                venue_order_id,
12798            );
12799        }
12800        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12801            OrderPendingCancelSpec::builder()
12802                .trader_id(order.trader_id())
12803                .strategy_id(order.strategy_id())
12804                .instrument_id(order.instrument_id())
12805                .client_order_id(client_order_id)
12806                .account_id(account_id)
12807                .build(),
12808        )));
12809        advance_clock(Duration::from_millis(101)).await;
12810        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12811
12812        if invalid == "update_venue" || invalid == "local_update" {
12813            let mut updated = OrderUpdatedSpec::builder()
12814                .trader_id(order.trader_id())
12815                .strategy_id(order.strategy_id())
12816                .instrument_id(order.instrument_id())
12817                .client_order_id(client_order_id)
12818                .account_id(account_id)
12819                .quantity(order.quantity())
12820                .price(order.price().unwrap())
12821                .build();
12822
12823            if invalid == "update_venue" {
12824                updated.venue_order_id = Some(venue_order_id);
12825            }
12826            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(updated)));
12827        } else {
12828            let report = OrderStatusReport::new(
12829                if invalid == "account" {
12830                    AccountId::from("OTHER-001")
12831                } else {
12832                    account_id
12833                },
12834                if invalid == "instrument" {
12835                    InstrumentId::from("OTHER.BINANCE")
12836                } else {
12837                    order.instrument_id()
12838                },
12839                Some(client_order_id),
12840                venue_order_id,
12841                Some(order.order_side()),
12842                order.order_type(),
12843                order.time_in_force(),
12844                OrderStatus::Submitted,
12845                order.quantity(),
12846                Quantity::zero(3),
12847                UnixNanos::default(),
12848                UnixNanos::default(),
12849                UnixNanos::default(),
12850                None,
12851            )
12852            .with_price(order.price().unwrap());
12853            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12854                report,
12855            ))));
12856        }
12857        assert!(
12858            node.kernel
12859                .cache
12860                .borrow()
12861                .order(&client_order_id)
12862                .unwrap()
12863                .venue_order_id()
12864                .is_none()
12865        );
12866        advance_clock(Duration::from_millis(101)).await;
12867        let result = node.exec_manager.check_inflight_orders();
12868        assert!(result.events.is_empty());
12869        assert!(result.queries.is_empty());
12870        let cached = node
12871            .kernel
12872            .cache
12873            .borrow()
12874            .order_owned(&client_order_id)
12875            .unwrap();
12876        assert_eq!(cached.status(), OrderStatus::PendingCancel);
12877        let diagnostics = node.exec_manager.take_submission_recovery_exhaustions();
12878        assert_eq!(diagnostics.len(), 1);
12879        assert_eq!(diagnostics[0].client_order_id, client_order_id);
12880    }
12881
12882    fn venue_evidence_command(order: &OrderAny, cancel: bool, batch: bool) -> TradingCommand {
12883        use nautilus_common::messages::execution::{
12884            BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
12885        };
12886
12887        let client_id = ClientId::from("TEST-EVIDENCE");
12888        let client_order_id = order.client_order_id();
12889        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12890
12891        if cancel {
12892            let cancel = CancelOrder::new(
12893                order.trader_id(),
12894                Some(client_id),
12895                order.strategy_id(),
12896                order.instrument_id(),
12897                client_order_id,
12898                Some(venue_order_id),
12899                UUID4::new(),
12900                UnixNanos::default(),
12901                None,
12902                None,
12903            );
12904
12905            if batch {
12906                TradingCommand::CancelOrders(BatchCancelOrders::new(
12907                    order.trader_id(),
12908                    Some(client_id),
12909                    order.strategy_id(),
12910                    order.instrument_id(),
12911                    vec![cancel],
12912                    UUID4::new(),
12913                    UnixNanos::default(),
12914                    None,
12915                    None,
12916                ))
12917            } else {
12918                TradingCommand::CancelOrder(cancel)
12919            }
12920        } else {
12921            let modify = ModifyOrder::new(
12922                order.trader_id(),
12923                Some(client_id),
12924                order.strategy_id(),
12925                order.instrument_id(),
12926                client_order_id,
12927                Some(venue_order_id),
12928                None,
12929                Some(Price::from("101.00")),
12930                None,
12931                UUID4::new(),
12932                UnixNanos::default(),
12933                None,
12934                None,
12935            );
12936
12937            if batch {
12938                TradingCommand::ModifyOrders(BatchModifyOrders::new(
12939                    order.trader_id(),
12940                    Some(client_id),
12941                    order.strategy_id(),
12942                    order.instrument_id(),
12943                    vec![modify],
12944                    UUID4::new(),
12945                    UnixNanos::default(),
12946                    None,
12947                    None,
12948                ))
12949            } else {
12950                TradingCommand::ModifyOrder(modify)
12951            }
12952        }
12953    }
12954
12955    fn venue_evidence_node(
12956        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12957    ) -> (LiveNode, OrderAny, InstrumentAny) {
12958        venue_evidence_node_with_order_type(policy, OrderType::Limit)
12959    }
12960
12961    fn venue_evidence_node_with_order_type(
12962        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12963        order_type: OrderType,
12964    ) -> (LiveNode, OrderAny, InstrumentAny) {
12965        venue_evidence_node_with_config(
12966            crate::config::LiveExecutionEngineConfig {
12967                reconciliation: true,
12968                inflight_check_threshold_ms: 100,
12969                inflight_check_retries: 2,
12970                open_check_threshold_ms: 5_000,
12971                single_order_query_delay_ms: 0,
12972                submission_recovery_policy: policy,
12973                ..Default::default()
12974            },
12975            order_type,
12976            true,
12977        )
12978    }
12979
12980    fn venue_evidence_node_with_config(
12981        exec_engine: crate::config::LiveExecutionEngineConfig,
12982        order_type: OrderType,
12983        register_submission: bool,
12984    ) -> (LiveNode, OrderAny, InstrumentAny) {
12985        let config = LiveNodeConfig {
12986            exec_engine,
12987            ..Default::default()
12988        };
12989        let mut node = LiveNode::build("VenueEvidenceNode".to_string(), Some(config)).unwrap();
12990        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
12991        let account_id = AccountId::from("TEST-001");
12992        let client_id = ClientId::from("TEST-EVIDENCE");
12993        let account = AccountAny::Margin(MarginAccount::new(
12994            AccountState::new(
12995                account_id,
12996                AccountType::Margin,
12997                vec![AccountBalance::new(
12998                    Money::from("1000000 USDT"),
12999                    Money::from("0 USDT"),
13000                    Money::from("1000000 USDT"),
13001                )],
13002                Vec::new(),
13003                true,
13004                UUID4::new(),
13005                UnixNanos::default(),
13006                UnixNanos::default(),
13007                Some(Currency::USDT()),
13008            ),
13009            true,
13010        ));
13011        node.kernel.cache.borrow_mut().add_account(account).unwrap();
13012        node.kernel
13013            .cache
13014            .borrow_mut()
13015            .add_instrument(instrument.clone())
13016            .unwrap();
13017        let mut builder = OrderTestBuilder::new(order_type);
13018        builder
13019            .trader_id(node.trader_id())
13020            .client_order_id(ClientOrderId::from("O-VENUE-EVIDENCE"))
13021            .instrument_id(instrument.id())
13022            .quantity(Quantity::from("1.000"))
13023            .price(Price::from("100.00"));
13024
13025        if order_type == OrderType::StopLimit {
13026            builder.trigger_price(Price::from("110.00"));
13027        }
13028        let order = builder.build();
13029        let command = TradingCommand::SubmitOrder(SubmitOrder::from_order(
13030            &order,
13031            node.trader_id(),
13032            Some(client_id),
13033            None,
13034            UUID4::new(),
13035            UnixNanos::default(),
13036        ));
13037
13038        if register_submission {
13039            node.observe_exec_command_before_dispatch(&command);
13040        }
13041        node.kernel
13042            .cache
13043            .borrow_mut()
13044            .add_order(order.clone(), None, Some(client_id), false)
13045            .unwrap();
13046        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
13047            &order, account_id,
13048        )));
13049        (node, order, instrument)
13050    }
13051
13052    #[rstest]
13053    #[case::partial("0.400", OrderStatus::PartiallyFilled)]
13054    #[case::full("1.000", OrderStatus::Filled)]
13055    #[tokio::test(start_paused = true)]
13056    async fn test_retained_submission_resolves_late_fill(
13057        #[case] quantity: &str,
13058        #[case] expected_status: OrderStatus,
13059    ) {
13060        let (mut node, order, instrument) =
13061            venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
13062        let client_order_id = order.client_order_id();
13063
13064        for _ in 0..2 {
13065            advance_clock(Duration::from_millis(101)).await;
13066            assert!(node.exec_manager.check_inflight_orders().events.is_empty());
13067        }
13068
13069        assert_eq!(
13070            node.exec_manager
13071                .take_submission_recovery_exhaustions()
13072                .len(),
13073            1
13074        );
13075        let fill = TestOrderEventStubs::filled(
13076            &order,
13077            &instrument,
13078            Some(TradeId::from("T-LATE-RETAINED")),
13079            None,
13080            Some(Price::from("100.00")),
13081            Some(Quantity::from(quantity)),
13082            Some(LiquiditySide::Taker),
13083            None,
13084            None,
13085            Some(AccountId::from("TEST-001")),
13086        );
13087        node.process_exec_event(ExecutionEvent::Order(fill));
13088        advance_clock(Duration::from_millis(101)).await;
13089        let result = node.exec_manager.check_inflight_orders();
13090        let cache = node.kernel.cache.borrow();
13091        let cached = cache.order_ref(&client_order_id).unwrap();
13092        assert_eq!(cached.status(), expected_status);
13093        assert_eq!(cached.filled_qty(), Quantity::from(quantity));
13094        assert_eq!(cached.trade_ids(), vec![&TradeId::from("T-LATE-RETAINED")]);
13095        assert!(result.events.is_empty());
13096        assert!(result.queries.is_empty());
13097        assert!(
13098            node.exec_manager
13099                .take_submission_recovery_exhaustions()
13100                .is_empty()
13101        );
13102    }
13103
13104    #[rstest]
13105    #[case::submitted_report("order")]
13106    #[case::submitted_report_with_fills("order_with_fills")]
13107    #[case::venue_update("updated")]
13108    #[cfg_attr(
13109        not(all(feature = "simulation", madsim)),
13110        tokio::test(start_paused = true)
13111    )]
13112    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
13113    async fn test_venue_evidence_stops_submission_timeout(
13114        #[case] evidence: &str,
13115        #[values(
13116            OrderStatus::Submitted,
13117            OrderStatus::PendingCancel,
13118            OrderStatus::PendingUpdate
13119        )]
13120        status: OrderStatus,
13121        #[values(
13122            crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
13123            crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
13124        )]
13125        policy: crate::execution::submission::SubmissionRecoveryPolicy,
13126    ) {
13127        let (mut node, order, _) = venue_evidence_node(policy);
13128        let client_order_id = order.client_order_id();
13129        let account_id = AccountId::from("TEST-001");
13130        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
13131        let pending = match status {
13132            OrderStatus::PendingCancel => Some(OrderEventAny::PendingCancel(
13133                OrderPendingCancelSpec::builder()
13134                    .trader_id(order.trader_id())
13135                    .strategy_id(order.strategy_id())
13136                    .instrument_id(order.instrument_id())
13137                    .client_order_id(client_order_id)
13138                    .account_id(account_id)
13139                    .build(),
13140            )),
13141            OrderStatus::PendingUpdate => Some(OrderEventAny::PendingUpdate(
13142                OrderPendingUpdateSpec::builder()
13143                    .trader_id(order.trader_id())
13144                    .strategy_id(order.strategy_id())
13145                    .instrument_id(order.instrument_id())
13146                    .client_order_id(client_order_id)
13147                    .account_id(account_id)
13148                    .build(),
13149            )),
13150            _ => None,
13151        };
13152
13153        if let Some(pending) = pending {
13154            node.process_exec_event(ExecutionEvent::Order(pending));
13155        }
13156        advance_clock(Duration::from_millis(101)).await;
13157        let first = node.exec_manager.check_inflight_orders();
13158        assert_eq!(first.queries.len(), 1);
13159        assert!(first.events.is_empty());
13160        advance_clock(Duration::from_millis(94)).await;
13161
13162        let event = if evidence == "updated" {
13163            ExecutionEvent::Order(OrderEventAny::Updated(
13164                OrderUpdatedSpec::builder()
13165                    .trader_id(order.trader_id())
13166                    .strategy_id(order.strategy_id())
13167                    .instrument_id(order.instrument_id())
13168                    .client_order_id(client_order_id)
13169                    .account_id(account_id)
13170                    .venue_order_id(venue_order_id)
13171                    .quantity(order.quantity())
13172                    .price(Price::from("101.00"))
13173                    .build(),
13174            ))
13175        } else {
13176            let report = OrderStatusReport::new(
13177                account_id,
13178                order.instrument_id(),
13179                Some(client_order_id),
13180                venue_order_id,
13181                Some(order.order_side()),
13182                order.order_type(),
13183                order.time_in_force(),
13184                OrderStatus::Submitted,
13185                order.quantity(),
13186                Quantity::zero(3),
13187                UnixNanos::default(),
13188                UnixNanos::default(),
13189                UnixNanos::default(),
13190                None,
13191            )
13192            .with_price(Price::from("100.00"));
13193            ExecutionEvent::Report(if evidence == "order" {
13194                ExecutionReport::Order(Box::new(report))
13195            } else {
13196                ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
13197            })
13198        };
13199        node.process_exec_event(event);
13200
13201        let expected_status = if evidence == "updated" && status != OrderStatus::PendingCancel {
13202            OrderStatus::Submitted
13203        } else if status == OrderStatus::Submitted {
13204            OrderStatus::Accepted
13205        } else {
13206            status
13207        };
13208        assert_eq!(
13209            node.kernel
13210                .cache
13211                .borrow()
13212                .order(&client_order_id)
13213                .unwrap()
13214                .status(),
13215            expected_status,
13216        );
13217
13218        for _ in 0..3 {
13219            advance_clock(Duration::from_millis(101)).await;
13220            let result = node.exec_manager.check_inflight_orders();
13221            assert!(result.queries.is_empty());
13222            assert!(result.events.is_empty());
13223        }
13224        assert_eq!(
13225            node.exec_manager.recon_check_retry_count(&client_order_id),
13226            0
13227        );
13228        assert!(node.exec_manager.check_open_order_queries().is_empty());
13229        advance_clock(Duration::from_secs(5)).await;
13230        assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
13231    }
13232
13233    #[rstest]
13234    #[case::fill("fill", false)]
13235    #[case::terminal_order("order", true)]
13236    #[case::terminal_order_with_fills("order_with_fills", true)]
13237    #[case::working_order_for_closed_cache("order", false)]
13238    #[case::working_order_with_fills_for_closed_cache("order_with_fills", false)]
13239    #[cfg_attr(
13240        not(all(feature = "simulation", madsim)),
13241        tokio::test(start_paused = true)
13242    )]
13243    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
13244    async fn test_report_cleanup_preserves_original_settling_predicate(
13245        #[case] report_kind: &str,
13246        #[case] terminal_report: bool,
13247        #[values(false, true)] explicit_client_id: bool,
13248        #[values(
13249            crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
13250            crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
13251        )]
13252        policy: crate::execution::submission::SubmissionRecoveryPolicy,
13253    ) {
13254        let (mut node, order, instrument) = venue_evidence_node(policy);
13255        let client_order_id = order.client_order_id();
13256        let account_id = AccountId::from("TEST-001");
13257        let client_id = ClientId::from("TEST-EVIDENCE");
13258        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
13259        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
13260            &order,
13261            account_id,
13262            venue_order_id,
13263        )));
13264        let accepted = node
13265            .kernel
13266            .cache
13267            .borrow()
13268            .order_owned(&client_order_id)
13269            .unwrap();
13270        let OrderEventAny::Filled(fill) = OrderFilledTestBuilder::new(&accepted, &instrument)
13271            .last_qty(accepted.quantity())
13272            .last_px(Price::from("100.00"))
13273            .commission(Money::from("0 USDT"))
13274            .without_position_id()
13275            .build()
13276        else {
13277            unreachable!()
13278        };
13279        let mut fill_report = FillReport::new(
13280            account_id,
13281            instrument.id(),
13282            venue_order_id,
13283            fill.trade_id,
13284            fill.order_side,
13285            fill.last_qty,
13286            fill.last_px,
13287            fill.commission.unwrap(),
13288            fill.liquidity_side,
13289            Some(client_order_id),
13290            None,
13291            fill.ts_event,
13292            fill.ts_init,
13293            None,
13294        );
13295
13296        if !explicit_client_id {
13297            fill_report.client_order_id = None;
13298        }
13299        let event = if report_kind == "fill" {
13300            ExecutionReport::Fill(Box::new(fill_report))
13301        } else {
13302            if !terminal_report {
13303                node.kernel
13304                    .cache
13305                    .borrow_mut()
13306                    .update_order(&OrderEventAny::Filled(fill))
13307                    .unwrap();
13308            }
13309            let report = OrderStatusReport::new(
13310                account_id,
13311                instrument.id(),
13312                explicit_client_id.then_some(client_order_id),
13313                venue_order_id,
13314                Some(order.order_side()),
13315                order.order_type(),
13316                order.time_in_force(),
13317                if terminal_report {
13318                    OrderStatus::Filled
13319                } else {
13320                    OrderStatus::Accepted
13321                },
13322                order.quantity(),
13323                order.quantity(),
13324                UnixNanos::default(),
13325                UnixNanos::default(),
13326                UnixNanos::default(),
13327                None,
13328            )
13329            .with_price(Price::from("100.00"))
13330            .with_avg_px(dec!(100.00));
13331
13332            if report_kind == "order" {
13333                ExecutionReport::Order(Box::new(report))
13334            } else {
13335                ExecutionReport::OrderWithFills(Box::new(report), vec![fill_report])
13336            }
13337        };
13338        node.process_exec_event(ExecutionEvent::Report(event));
13339        assert_eq!(
13340            node.kernel
13341                .cache
13342                .borrow()
13343                .order(&client_order_id)
13344                .unwrap()
13345                .status(),
13346            OrderStatus::Filled
13347        );
13348
13349        // Replace only cached state, without an observer stamp, to expose the existing grace gate
13350        node.kernel
13351            .cache
13352            .borrow_mut()
13353            .add_order(order.clone(), None, Some(client_id), true)
13354            .unwrap();
13355        node.kernel
13356            .cache
13357            .borrow_mut()
13358            .update_order(&TestOrderEventStubs::submitted(&order, account_id))
13359            .unwrap();
13360        node.kernel
13361            .cache
13362            .borrow_mut()
13363            .update_order(&TestOrderEventStubs::accepted(
13364                &order,
13365                account_id,
13366                venue_order_id,
13367            ))
13368            .unwrap();
13369        let should_clear = terminal_report && explicit_client_id;
13370        assert_eq!(
13371            node.exec_manager.check_open_order_queries().len(),
13372            usize::from(should_clear)
13373        );
13374
13375        if !should_clear {
13376            advance_clock(Duration::from_secs(5)).await;
13377            assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
13378        }
13379    }
13380
13381    #[rstest]
13382    #[case(stub_order_event, 1)]
13383    #[case(stub_submitted_batch_event, 2)]
13384    #[case(stub_accepted_batch_event, 2)]
13385    #[case(stub_canceled_batch_event, 2)]
13386    fn test_pending_events_preserve_roots_when_splitting_batches(
13387        #[values(false, true)] rooted: bool,
13388        #[case] make_event: fn() -> ExecutionEvent,
13389        #[case] orders: usize,
13390    ) {
13391        let runner = AsyncRunner::new();
13392        runner.bind_senders();
13393        let mut channels = runner.take_channels();
13394
13395        let send = move || {
13396            get_data_event_sender().send(stub_data_event()).unwrap();
13397            get_exec_event_sender().send(stub_exec_event()).unwrap();
13398            get_exec_event_sender().send(make_event()).unwrap();
13399        };
13400
13401        if rooted {
13402            msgbus::register_trading_command_endpoint(
13403                MessagingSwitchboard::risk_engine_execute(),
13404                TypedIntoHandler::from(move |_| send()),
13405            );
13406            TradingCommandSender::execute(
13407                &SyncTradingCommandSender,
13408                TradingCommandMessage::new(
13409                    MessagingSwitchboard::risk_engine_execute(),
13410                    TradingCommand::QueryAccount(QueryAccount::new(
13411                        "TRADER-001".into(),
13412                        None,
13413                        "SIM-001".into(),
13414                        UUID4::new(),
13415                        37.into(),
13416                        None,
13417                        None,
13418                    )),
13419                ),
13420            );
13421            nautilus_common::runner::drain_trading_cmd_queue();
13422        } else {
13423            send();
13424        }
13425
13426        let mut pending = PendingEvents::default();
13427        pending
13428            .data_evts
13429            .push(channels.data_evt_rx.try_recv().unwrap());
13430        while let Ok(event) = channels.exec_evt_rx.try_recv() {
13431            pending.push_exec_event(event);
13432        }
13433
13434        assert_eq!(pending.data_evts.len(), 1);
13435        assert_eq!(pending.exec_reports.len(), 1);
13436        assert_eq!(pending.order_evts.len(), orders);
13437        assert_eq!(pending.data_evts[0].is_rooted(), rooted);
13438        assert_eq!(pending.exec_reports[0].is_rooted(), rooted);
13439        assert!(
13440            pending
13441                .order_evts
13442                .iter()
13443                .all(|event| event.is_rooted() == rooted)
13444        );
13445
13446        let results = std::thread::spawn(move || {
13447            let mut results = Vec::new();
13448
13449            for event in pending.data_evts {
13450                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13451                    || event.dispatch(drop),
13452                )));
13453            }
13454
13455            for event in pending.exec_reports {
13456                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13457                    || event.dispatch(drop),
13458                )));
13459            }
13460
13461            for event in pending.order_evts {
13462                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13463                    || event.dispatch(drop),
13464                )));
13465            }
13466
13467            results
13468        })
13469        .join()
13470        .unwrap();
13471
13472        assert_eq!(results.len(), orders + 2);
13473
13474        for result in results {
13475            if rooted {
13476                let error = result.unwrap_err();
13477                assert!(
13478                    error
13479                        .downcast_ref::<String>()
13480                        .unwrap()
13481                        .contains("command context dispatched outside its owner thread")
13482                );
13483            } else {
13484                result.unwrap();
13485            }
13486        }
13487    }
13488
13489    fn stub_accepted_batch_event() -> ExecutionEvent {
13490        ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![
13491            OrderAcceptedSpec::builder()
13492                .client_order_id(ClientOrderId::from("O-017"))
13493                .build(),
13494            OrderAcceptedSpec::builder()
13495                .client_order_id(ClientOrderId::from("O-023"))
13496                .build(),
13497        ]))
13498    }
13499
13500    #[derive(Debug)]
13501    struct CapturedEgressMessage {
13502        topic: String,
13503        payload: Bytes,
13504    }
13505
13506    type CapturedEgressMessages = Rc<RefCell<Vec<CapturedEgressMessage>>>;
13507    type SharedClosed = Rc<Cell<bool>>;
13508
13509    #[derive(Debug)]
13510    struct CapturingExternalIngress {
13511        rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13512        closed: SharedClosed,
13513    }
13514
13515    impl CapturingExternalIngress {
13516        fn new(rx: tokio::sync::mpsc::Receiver<BusMessage>, closed: SharedClosed) -> Self {
13517            Self {
13518                rx: Some(rx),
13519                closed,
13520            }
13521        }
13522    }
13523
13524    impl MessageBusExternalIngress for CapturingExternalIngress {
13525        fn is_closed(&self) -> bool {
13526            self.closed.get()
13527        }
13528
13529        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13530            self.rx
13531                .take()
13532                .ok_or_else(|| anyhow::anyhow!("external ingress receiver already taken"))
13533        }
13534
13535        fn close(&mut self) {
13536            self.closed.set(true);
13537        }
13538    }
13539
13540    #[derive(Debug)]
13541    struct FailingExternalIngress {
13542        closed: SharedClosed,
13543    }
13544
13545    impl FailingExternalIngress {
13546        fn new(closed: SharedClosed) -> Self {
13547            Self { closed }
13548        }
13549    }
13550
13551    impl MessageBusExternalIngress for FailingExternalIngress {
13552        fn is_closed(&self) -> bool {
13553            self.closed.get()
13554        }
13555
13556        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13557            anyhow::bail!("external ingress receiver unavailable")
13558        }
13559
13560        fn close(&mut self) {
13561            self.closed.set(true);
13562        }
13563    }
13564
13565    struct CapturingExternalEgress {
13566        publications: CapturedEgressMessages,
13567        closed: SharedClosed,
13568    }
13569
13570    impl CapturingExternalEgress {
13571        fn new() -> (Self, CapturedEgressMessages, SharedClosed) {
13572            let publications = Rc::new(RefCell::new(Vec::new()));
13573            let closed = Rc::new(Cell::new(false));
13574            (
13575                Self {
13576                    publications: publications.clone(),
13577                    closed: closed.clone(),
13578                },
13579                publications,
13580                closed,
13581            )
13582        }
13583    }
13584
13585    impl MessageBusExternalEgress for CapturingExternalEgress {
13586        fn is_closed(&self) -> bool {
13587            self.closed.get()
13588        }
13589
13590        fn publish(&self, message: BusMessage) {
13591            self.publications.borrow_mut().push(CapturedEgressMessage {
13592                topic: message.topic.to_string(),
13593                payload: message.payload,
13594            });
13595        }
13596
13597        fn close(&mut self) {
13598            self.closed.set(true);
13599        }
13600    }
13601
13602    struct CapturingBackingFactory {
13603        publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13604        closed: Arc<AtomicBool>,
13605        rx: Mutex<Option<tokio::sync::mpsc::Receiver<BusMessage>>>,
13606    }
13607
13608    impl CapturingBackingFactory {
13609        fn new(
13610            publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13611            closed: Arc<AtomicBool>,
13612            rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13613        ) -> Self {
13614            Self {
13615                publications,
13616                closed,
13617                rx: Mutex::new(rx),
13618            }
13619        }
13620    }
13621
13622    impl Debug for CapturingBackingFactory {
13623        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
13624            f.debug_struct(stringify!(CapturingBackingFactory))
13625                .finish_non_exhaustive()
13626        }
13627    }
13628
13629    impl MessageBusBackingFactory for CapturingBackingFactory {
13630        fn create(
13631            &self,
13632            _trader_id: TraderId,
13633            _instance_id: UUID4,
13634            _config: MessageBusConfig,
13635        ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
13636            let rx = self.rx.lock().take();
13637            Ok(Box::new(CapturingBacking {
13638                publications: self.publications.clone(),
13639                closed: self.closed.clone(),
13640                rx,
13641            }))
13642        }
13643    }
13644
13645    struct CapturingBacking {
13646        publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13647        closed: Arc<AtomicBool>,
13648        rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13649    }
13650
13651    impl MessageBusBacking for CapturingBacking {
13652        fn is_closed(&self) -> bool {
13653            self.closed.load(Ordering::Relaxed)
13654        }
13655
13656        fn publish(&self, message: BusMessage) {
13657            self.publications.lock().push(CapturedEgressMessage {
13658                topic: message.topic.to_string(),
13659                payload: message.payload,
13660            });
13661        }
13662
13663        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13664            self.rx
13665                .take()
13666                .ok_or_else(|| anyhow::anyhow!("external ingress receiver unavailable"))
13667        }
13668
13669        fn close(&mut self) {
13670            self.closed.store(true, Ordering::Relaxed);
13671        }
13672    }
13673}