Skip to main content

nautilus_data/engine/
mod.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Provides a high-performance `DataEngine` for all environments.
17//!
18//! The `DataEngine` is the central component of the entire data stack.
19//! The data engines primary responsibility is to orchestrate interactions between
20//! the `DataClient` instances, and the rest of the platform. This includes sending
21//! requests to, and receiving responses from, data endpoints via its registered
22//! data clients.
23//!
24//! The engine employs a simple fan-in fan-out messaging pattern to execute
25//! `DataCommand` type messages, and process `DataResponse` messages or market data
26//! objects.
27//!
28//! Alternative implementations can be written on top of the generic engine - which
29//! just need to override the `execute`, `process`, `send`, and `receive` methods.
30
31pub mod bar;
32pub mod book;
33pub mod config;
34
35mod commands;
36mod dispatch;
37mod futures;
38mod handlers;
39mod option_chain;
40mod registry;
41mod requests;
42mod responses;
43mod spread_quote;
44mod synthetic;
45mod time_range;
46
47#[cfg(feature = "defi")]
48pub mod pool;
49
50#[cfg(feature = "streaming")]
51mod streaming;
52
53use std::{
54    any::{Any, type_name},
55    cell::RefCell,
56    collections::VecDeque,
57    fmt::{Debug, Display},
58    mem,
59    num::NonZeroUsize,
60    rc::Rc,
61    str::FromStr,
62};
63
64use ::futures::future::join_all;
65use ahash::{AHashMap, AHashSet};
66use anyhow::Context;
67pub use bar::BarAggregatorSubscription;
68use bar::{BarAggregationSubscription, BarAggregatorKey, BarAggregatorWarmup, bar_aggregator_key};
69use book::{
70    BookDeltasKey, BookSnapshotInfos, BookSnapshotKey, BookSnapshotSource, BookSnapshotter,
71    BookSubscription, BookSubscriptionOwner, BookUpdater,
72};
73pub(crate) use commands::{DeferredCommand, DeferredCommandQueue};
74use config::DataEngineConfig;
75use dispatch::{log_error_on_cache_insert, process_engine_bar};
76use futures::{ContinuousFutureRoller, ContinuousFutureSubscriptionState, datetime_to_unix_nanos};
77use handlers::{
78    BAR_AGGREGATOR_PRIORITY, BarBarHandler, BarQuoteHandler, BarTradeHandler, SpreadQuoteHandler,
79};
80use indexmap::IndexMap;
81#[cfg(feature = "defi")]
82use nautilus_common::messages::defi::PoolSnapshotResponse;
83use nautilus_common::{
84    cache::Cache,
85    clock::Clock,
86    logging::{RECV, RES},
87    messages::data::{
88        BarsResponse, BookDeltasResponse, BookDepthResponse, CustomDataResponse, DataCommand,
89        DataRequestFailed, DataResponse, DataResponseEnvelope, IdentifierCountError,
90        OptionChainReferencePriceResponse, PARAMS_IS_PARENT, QuoteRouting, RequestBars,
91        RequestCommand, RequestJoin, RequestOptionChainReferencePrice, RequestQuotes,
92        RequestTrades, SubscribeBars, SubscribeBookDeltas, SubscribeBookDepth, SubscribeCommand,
93        SubscribeOptionChain, SubscribeOptionGreeks, SubscribeQuotes, SubscribeTrades,
94        UnsubscribeBars, UnsubscribeBookDeltas, UnsubscribeBookDepth, UnsubscribeBookSnapshots,
95        UnsubscribeCommand, UnsubscribeInstrumentStatus, UnsubscribeOptionChain,
96        UnsubscribeOptionGreeks, UnsubscribeQuotes, UnsubscribeTrades, ensure_routable_bar_types,
97        ensure_routable_instrument_ids, is_parent_subscription, quote_routing,
98    },
99    msgbus::{
100        self, BusPayloadType, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
101        get_message_bus,
102        switchboard::{self, MessagingSwitchboard},
103    },
104    runner::get_data_cmd_sender,
105    timer::{TimeEvent, TimeEventCallback},
106};
107use nautilus_core::{
108    DurationNanos, Params, UUID4, UnixNanos, WeakCell,
109    correctness::{FAILED, check_key_in_map, check_key_not_in_map, check_predicate_true},
110    datetime::NANOSECONDS_IN_DAY,
111};
112#[cfg(feature = "defi")]
113use nautilus_model::defi::DefiData;
114use nautilus_model::{
115    data::{
116        Bar, BarType, CustomData, Data, DataRef, DataType, FundingRateUpdate, HasTsInit,
117        IndexPriceUpdate, InstrumentClose, InstrumentStatus, MarkPriceUpdate, OrderBookDelta,
118        OrderBookDeltas, OrderBookDepth, QuoteTick, TradeTick,
119        option_chain::{OptionGreeks, StrikeRange},
120    },
121    enums::{
122        AggregationSource, BarAggregation, BookType, InstrumentClass, MarketStatusAction,
123        PriceType, RecordFlag,
124    },
125    identifiers::{
126        ClientId, GENERIC_SPREAD_ID_SEPARATOR, InstrumentId, OptionSeriesId, Venue,
127        parse_generic_spread_id_legs,
128    },
129    instruments::{Instrument, InstrumentAny, SyntheticInstrument},
130    orderbook::OrderBook,
131    types::{Price, Quantity},
132};
133use option_chain::{
134    OptionChainBootstrapper, OptionChainGreeksBootstrap, PendingOptionChainRequest,
135};
136use requests::{
137    ContinuousFutureRequest, ContinuousFutureRequestState, ContinuousFutureSegment,
138    ContinuousFutureSource, RequestBarAggregation, continuous_future_request_from_bars,
139    continuous_future_subscription_from_bars, has_continuous_future_params,
140    request_bar_aggregation_from_params, request_params, response_params,
141};
142use responses::{
143    log_if_empty_response, parent_request_window, rebind_response_correlation,
144    rebuild_pipeline_response, same_instrument,
145};
146use serde_json::Value;
147use spread_quote::SpreadQuoteState;
148#[cfg(feature = "streaming")]
149use streaming::CatalogMap;
150use time_range::{
151    TimeRangePipelineState, has_time_range_pipeline_params, is_time_range_pipeline_variant,
152};
153use ustr::Ustr;
154
155#[cfg(feature = "defi")]
156#[allow(unused_imports)] // Brings DeFi impl blocks into scope
157use crate::defi::engine as _;
158#[cfg(feature = "defi")]
159use crate::engine::pool::PoolUpdater;
160use crate::{
161    aggregation::{
162        BarAggregator, RenkoBarAggregator, SpreadQuoteAggregator, TickBarAggregator,
163        TickImbalanceBarAggregator, TickRunsBarAggregator, TimeBarAggregator, ValueBarAggregator,
164        ValueImbalanceBarAggregator, ValueRunsBarAggregator, VolumeBarAggregator,
165        VolumeImbalanceBarAggregator, VolumeRunsBarAggregator,
166    },
167    client::DataClientAdapter,
168    option_chains::OptionChainManager,
169    subscription::{
170        SubscriptionKey, SubscriptionRegistry, SubscriptionRelease, split_subscribe,
171        split_unsubscribe,
172    },
173};
174
175/// Provides a high-performance `DataEngine` for all environments.
176#[derive(Debug)]
177pub struct DataEngine {
178    clock: Rc<RefCell<dyn Clock>>,
179    cache: Rc<RefCell<Cache>>,
180    config: DataEngineConfig,
181    is_stopped: bool,
182    msgbus_priority: u32,
183    subscriptions_external: SubscriptionRegistry<(ClientId, SubscriptionKey), SubscribeCommand>,
184    clients: IndexMap<ClientId, DataClientAdapter>,
185    default_client_id: Option<ClientId>,
186    routing_map: IndexMap<Venue, ClientId>,
187    book_intervals: AHashMap<NonZeroUsize, BookSnapshotInfos>,
188    book_snapshot_counts: IndexMap<BookSnapshotKey, usize>,
189    book_snapshot_sources: AHashMap<InstrumentId, BookSnapshotSource>,
190    book_deltas_counts: IndexMap<BookDeltasKey, usize>,
191    book_depth_counts: IndexMap<BookDeltasKey, usize>,
192    book_updaters: AHashMap<InstrumentId, Rc<BookUpdater>>,
193    book_subscriptions: AHashMap<InstrumentId, BookSubscription>,
194    book_subscription_owners: AHashMap<InstrumentId, Vec<Rc<BookSubscriptionOwner>>>,
195    book_snapshotters: AHashMap<NonZeroUsize, Rc<BookSnapshotter>>,
196    bar_aggregators: IndexMap<BarAggregatorKey, Rc<RefCell<Box<dyn BarAggregator>>>>,
197    bar_aggregator_handlers: AHashMap<BarAggregatorKey, Vec<BarAggregatorSubscription>>,
198    bar_aggregator_warmups: AHashMap<BarAggregatorKey, BarAggregatorWarmup>,
199    bar_aggregators_pending_stop: AHashSet<BarAggregatorKey>,
200    subscriptions_bar_aggregation: AHashMap<BarType, BarAggregationSubscription>,
201    request_bar_aggregations: AHashMap<UUID4, RequestBarAggregation>,
202    inflight_requests: AHashMap<UUID4, RequestCommand>,
203    request_pipeline_parent_request: AHashMap<UUID4, RequestCommand>,
204    request_pipeline_n_components: AHashMap<UUID4, usize>,
205    request_pipeline_parent_request_id: AHashMap<UUID4, (UUID4, usize)>,
206    request_pipeline_responses: AHashMap<UUID4, Vec<Option<Vec<DataResponseEnvelope>>>>,
207    time_range_pipeline_requests: AHashMap<UUID4, TimeRangePipelineState>,
208    time_range_pipeline_parent_request_id: AHashMap<UUID4, UUID4>,
209    parent_join_request_id: AHashMap<UUID4, UUID4>,
210    pending_join_requests: AHashMap<UUID4, RequestJoin>,
211    continuous_future_requests: AHashMap<UUID4, ContinuousFutureRequestState>,
212    continuous_future_parent_request_id: AHashMap<UUID4, UUID4>,
213    continuous_future_subscriptions: AHashMap<BarType, ContinuousFutureSubscriptionState>,
214    continuous_future_roller: Option<Rc<ContinuousFutureRoller>>,
215    spread_quote_states: AHashMap<InstrumentId, SpreadQuoteState>,
216    option_chain_managers: AHashMap<OptionSeriesId, Rc<RefCell<OptionChainManager>>>,
217    option_chain_instrument_index: AHashMap<InstrumentId, OptionSeriesId>,
218    deferred_cmd_queue: DeferredCommandQueue,
219    option_chain_bootstrapper: Option<Rc<OptionChainBootstrapper>>,
220    pending_option_chain_requests: AHashMap<UUID4, PendingOptionChainRequest>,
221    option_chain_greeks_bootstraps: AHashMap<OptionSeriesId, OptionChainGreeksBootstrap>,
222    synthetic_quote_feeds: AHashMap<InstrumentId, Vec<SyntheticInstrument>>,
223    synthetic_trade_feeds: AHashMap<InstrumentId, Vec<SyntheticInstrument>>,
224    subscribed_synthetic_quotes: AHashMap<InstrumentId, usize>,
225    subscribed_synthetic_trades: AHashMap<InstrumentId, usize>,
226    buffered_deltas_map: AHashMap<InstrumentId, OrderBookDeltas>,
227    deltas_frame: Vec<OrderBookDelta>,
228    command_count: u64,
229    data_count: u64,
230    request_count: u64,
231    response_count: u64,
232    #[cfg(feature = "streaming")]
233    catalogs: CatalogMap,
234    #[cfg(feature = "defi")]
235    pub(crate) pool_updaters: AHashMap<InstrumentId, Rc<PoolUpdater>>,
236    #[cfg(feature = "defi")]
237    pub(crate) pool_snapshot_pending: AHashMap<InstrumentId, UUID4>,
238    #[cfg(feature = "defi")]
239    pub(crate) pool_event_buffers: AHashMap<InstrumentId, Vec<DefiData>>,
240}
241
242impl DataEngine {
243    /// Creates a new [`DataEngine`] instance.
244    #[must_use]
245    pub fn new(
246        clock: Rc<RefCell<dyn Clock>>,
247        cache: Rc<RefCell<Cache>>,
248        config: Option<DataEngineConfig>,
249    ) -> Self {
250        let config = config.unwrap_or_default();
251        if let Some(client_ids) = &config.external_clients {
252            let mut cache = cache.borrow_mut();
253            for client_id in client_ids {
254                cache.add_external_data_client(*client_id);
255            }
256        }
257
258        Self {
259            clock,
260            cache,
261            config,
262            is_stopped: false,
263            msgbus_priority: 10, // High-priority for built-in component
264            subscriptions_external: SubscriptionRegistry::default(),
265            clients: IndexMap::new(),
266            default_client_id: None,
267            routing_map: IndexMap::new(),
268            book_intervals: AHashMap::new(),
269            book_snapshot_counts: IndexMap::new(),
270            book_snapshot_sources: AHashMap::new(),
271            book_deltas_counts: IndexMap::new(),
272            book_depth_counts: IndexMap::new(),
273            book_updaters: AHashMap::new(),
274            book_subscriptions: AHashMap::new(),
275            book_subscription_owners: AHashMap::new(),
276            book_snapshotters: AHashMap::new(),
277            bar_aggregators: IndexMap::new(),
278            bar_aggregator_handlers: AHashMap::new(),
279            bar_aggregator_warmups: AHashMap::new(),
280            bar_aggregators_pending_stop: AHashSet::new(),
281            subscriptions_bar_aggregation: AHashMap::new(),
282            request_bar_aggregations: AHashMap::new(),
283            inflight_requests: AHashMap::new(),
284            request_pipeline_parent_request: AHashMap::new(),
285            request_pipeline_n_components: AHashMap::new(),
286            request_pipeline_parent_request_id: AHashMap::new(),
287            request_pipeline_responses: AHashMap::new(),
288            time_range_pipeline_requests: AHashMap::new(),
289            time_range_pipeline_parent_request_id: AHashMap::new(),
290            parent_join_request_id: AHashMap::new(),
291            pending_join_requests: AHashMap::new(),
292            continuous_future_requests: AHashMap::new(),
293            continuous_future_parent_request_id: AHashMap::new(),
294            continuous_future_subscriptions: AHashMap::new(),
295            continuous_future_roller: None,
296            spread_quote_states: AHashMap::new(),
297            option_chain_managers: AHashMap::new(),
298            option_chain_instrument_index: AHashMap::new(),
299            deferred_cmd_queue: Rc::new(RefCell::new(VecDeque::new())),
300            option_chain_bootstrapper: None,
301            pending_option_chain_requests: AHashMap::new(),
302            option_chain_greeks_bootstraps: AHashMap::new(),
303            synthetic_quote_feeds: AHashMap::new(),
304            synthetic_trade_feeds: AHashMap::new(),
305            subscribed_synthetic_quotes: AHashMap::new(),
306            subscribed_synthetic_trades: AHashMap::new(),
307            buffered_deltas_map: AHashMap::new(),
308            deltas_frame: Vec::new(),
309            command_count: 0,
310            data_count: 0,
311            request_count: 0,
312            response_count: 0,
313            #[cfg(feature = "streaming")]
314            catalogs: CatalogMap::new(),
315            #[cfg(feature = "defi")]
316            pool_updaters: AHashMap::new(),
317            #[cfg(feature = "defi")]
318            pool_snapshot_pending: AHashMap::new(),
319            #[cfg(feature = "defi")]
320            pool_event_buffers: AHashMap::new(),
321        }
322    }
323
324    /// Returns a reference to the clock.
325    #[must_use]
326    pub fn clock(&self) -> &Rc<RefCell<dyn Clock>> {
327        &self.clock
328    }
329
330    /// Returns a reference to the cache.
331    #[must_use]
332    pub fn cache(&self) -> &Rc<RefCell<Cache>> {
333        &self.cache
334    }
335
336    /// Returns a reference to the configuration.
337    #[must_use]
338    pub const fn config(&self) -> &DataEngineConfig {
339        &self.config
340    }
341
342    #[cfg(feature = "defi")]
343    #[must_use]
344    pub(crate) const fn msgbus_priority(&self) -> u32 {
345        self.msgbus_priority
346    }
347
348    /// Registers all message bus handlers for the data engine.
349    ///
350    /// Sets external market-data ingress eligibility from the cache's external data-client registrations.
351    pub fn register_msgbus_handlers(engine: &Rc<RefCell<Self>>) {
352        let weak = WeakCell::from(Rc::downgrade(engine));
353        engine.borrow_mut().continuous_future_roller =
354            Some(Rc::new(ContinuousFutureRoller::new(engine)));
355        engine.borrow_mut().option_chain_bootstrapper =
356            Some(Rc::new(OptionChainBootstrapper::new(engine)));
357
358        let weak1 = weak.clone();
359        msgbus::register_data_command_endpoint(
360            MessagingSwitchboard::data_engine_execute(),
361            TypedIntoHandler::from(move |cmd: DataCommand| {
362                if let Some(rc) = weak1.upgrade() {
363                    rc.borrow_mut().execute(cmd);
364                }
365            }),
366        );
367
368        msgbus::register_data_command_endpoint(
369            MessagingSwitchboard::data_engine_queue_execute(),
370            TypedIntoHandler::from(move |cmd: DataCommand| {
371                get_data_cmd_sender().execute(cmd);
372            }),
373        );
374
375        // Register process handler (polymorphic - uses Any)
376        let weak2 = weak.clone();
377        msgbus::register_any(
378            MessagingSwitchboard::data_engine_process(),
379            ShareableMessageHandler::from_any(move |data: &dyn Any| {
380                if let Some(rc) = weak2.upgrade() {
381                    rc.borrow_mut().process(data);
382                }
383            }),
384        );
385
386        // Register process_data handler (typed - takes ownership)
387        let weak3 = weak.clone();
388        msgbus::register_data_endpoint(
389            MessagingSwitchboard::data_engine_process_data(),
390            TypedIntoHandler::from(move |data: Data| {
391                if let Some(rc) = weak3.upgrade() {
392                    rc.borrow_mut().process_data(data);
393                }
394            }),
395        );
396
397        let external_data_ingress = engine.borrow().cache.borrow().has_external_data_clients();
398        get_message_bus()
399            .borrow_mut()
400            .set_external_data_ingress(external_data_ingress);
401
402        // Register process_defi_data handler (typed - takes ownership)
403        #[cfg(feature = "defi")]
404        {
405            let weak4 = weak.clone();
406            msgbus::register_defi_data_endpoint(
407                MessagingSwitchboard::data_engine_process_defi_data(),
408                TypedIntoHandler::from(move |data: DefiData| {
409                    if let Some(rc) = weak4.upgrade() {
410                        rc.borrow_mut().process_defi_data(data);
411                    }
412                }),
413            );
414        }
415
416        let weak5 = weak;
417        msgbus::register_data_response_endpoint(
418            MessagingSwitchboard::data_engine_response(),
419            TypedIntoHandler::from(move |resp: DataResponse| {
420                if let Some(rc) = weak5.upgrade() {
421                    rc.borrow_mut().response(resp);
422                }
423            }),
424        );
425    }
426
427    /// Returns the total count of data commands received by the engine.
428    #[must_use]
429    pub const fn command_count(&self) -> u64 {
430        self.command_count
431    }
432
433    /// Returns the total count of data stream objects received by the engine.
434    #[must_use]
435    pub const fn data_count(&self) -> u64 {
436        self.data_count
437    }
438
439    #[cfg(feature = "defi")]
440    pub(crate) const fn increment_data_count(&mut self) {
441        self.data_count += 1;
442    }
443
444    /// Returns the total count of data requests received by the engine.
445    #[must_use]
446    pub const fn request_count(&self) -> u64 {
447        self.request_count
448    }
449
450    /// Returns the total count of data responses received by the engine.
451    #[must_use]
452    pub const fn response_count(&self) -> u64 {
453        self.response_count
454    }
455
456    /// Returns the number of request pipelines awaiting leg responses.
457    #[must_use]
458    pub fn request_pipeline_count(&self) -> usize {
459        self.request_pipeline_parent_request.len()
460    }
461
462    /// Returns the number of time-range pipelines awaiting child responses.
463    #[must_use]
464    pub fn time_range_pipeline_count(&self) -> usize {
465        self.time_range_pipeline_requests.len()
466    }
467
468    /// Returns the number of `RequestJoin` originals awaiting finalization.
469    #[must_use]
470    pub fn pending_join_request_count(&self) -> usize {
471        self.pending_join_requests.len()
472    }
473
474    /// Starts all registered data clients and re-arms bar aggregator timers.
475    pub fn start(&mut self) {
476        self.is_stopped = false;
477
478        for client in self.get_clients_mut() {
479            client.is_stopped = false;
480            if client.start().is_err() {
481                log::error!("Failed to start data client {}", client.client_id());
482            }
483        }
484
485        for ((_, request_id), aggregator) in &self.bar_aggregators {
486            // Request-scoped or historical aggregators run on private clocks;
487            // re-arming them here would perturb an in-flight request's timer state
488            let is_subscription = request_id.is_none() && !aggregator.borrow().is_historical();
489            if is_subscription && aggregator.borrow().bar_type().spec().is_time_aggregated() {
490                aggregator
491                    .borrow_mut()
492                    .start_timer(Some(Rc::clone(aggregator)));
493            }
494        }
495
496        for state in self.spread_quote_states.values() {
497            state
498                .aggregator
499                .borrow_mut()
500                .start_timer(Some(Rc::clone(&state.aggregator)));
501        }
502    }
503
504    /// Stops all registered data clients and bar aggregator timers.
505    ///
506    /// Blocks new physical subscriptions until [`Self::start`] or [`Self::reset`].
507    /// Data processing, subscription ownership, and unsubscribes remain enabled.
508    /// Blocked subscriptions are not automatically replayed on start or connect.
509    pub fn stop(&mut self) {
510        self.is_stopped = true;
511
512        for client in self.get_clients_mut() {
513            client.is_stopped = true;
514            if client.stop().is_err() {
515                log::error!("Failed to stop data client {}", client.client_id());
516            }
517        }
518
519        for aggregator in self.bar_aggregators.values() {
520            aggregator.borrow_mut().stop();
521        }
522
523        for state in self.spread_quote_states.values() {
524            state.aggregator.borrow_mut().stop_timer();
525        }
526    }
527
528    /// Resets all registered data clients and clears engine state.
529    pub fn reset(&mut self) {
530        self.is_stopped = false;
531
532        for client in self.get_clients_mut() {
533            client.is_stopped = false;
534            match client.reset() {
535                Ok(()) => client.clear_subscription_state(),
536                Err(_) => log::error!("Failed to reset data client {}", client.client_id()),
537            }
538        }
539
540        let keys: Vec<BarAggregatorKey> = self.bar_aggregators.keys().copied().collect();
541        for (bar_type, request_id) in keys {
542            if let Err(e) = self.stop_bar_aggregator(bar_type, request_id) {
543                log::error!("Error stopping bar aggregator during reset for {bar_type}: {e}");
544            }
545        }
546        self.subscriptions_bar_aggregation.clear();
547
548        self.request_bar_aggregations.clear();
549        self.inflight_requests.clear();
550        self.request_pipeline_parent_request.clear();
551        self.request_pipeline_n_components.clear();
552        self.request_pipeline_parent_request_id.clear();
553        self.request_pipeline_responses.clear();
554        self.time_range_pipeline_requests.clear();
555        self.time_range_pipeline_parent_request_id.clear();
556        self.parent_join_request_id.clear();
557        self.pending_join_requests.clear();
558        self.continuous_future_requests.clear();
559        self.continuous_future_parent_request_id.clear();
560
561        for state in self.continuous_future_subscriptions.values_mut() {
562            if let Some(name) = state.timer_name.take() {
563                self.clock.borrow_mut().cancel_timer(&name);
564            }
565        }
566        self.continuous_future_subscriptions.clear();
567
568        let spread_ids: Vec<InstrumentId> = self.spread_quote_states.keys().copied().collect();
569        for spread_id in spread_ids {
570            self.stop_spread_quote_aggregation(spread_id);
571        }
572
573        // Tear down option chain managers to unregister their msgbus handlers
574        let managers: Vec<_> = self.option_chain_managers.drain().collect();
575        for (_, manager) in managers {
576            manager.borrow_mut().teardown(&self.clock);
577        }
578
579        self.option_chain_instrument_index.clear();
580        self.cancel_pending_option_chain_requests(None);
581        self.clear_option_chain_greeks_bootstraps();
582
583        // Unsubscribe BookUpdaters before dropping; otherwise the typed router
584        // keeps dispatching to abandoned updaters. `book_updaters` is keyed by
585        // per-underlying id, so the literal per-underlying topic is the same
586        // string the subscribe path used.
587        let book_updaters: Vec<(InstrumentId, Rc<BookUpdater>)> =
588            self.book_updaters.drain().collect();
589        for (instrument_id, updater) in book_updaters {
590            let deltas_topic = switchboard::get_book_deltas_topic(instrument_id);
591            let depth_topic = switchboard::get_book_depth_topic(instrument_id);
592            let deltas_handler: TypedHandler<OrderBookDeltas> =
593                TypedHandler::new(Rc::clone(&updater));
594            let depth_handler: TypedHandler<OrderBookDepth> = TypedHandler::new(updater);
595            msgbus::unsubscribe_book_deltas(deltas_topic.into(), &deltas_handler);
596            msgbus::unsubscribe_book_depth(depth_topic.into(), &depth_handler);
597        }
598
599        self.book_subscriptions.clear();
600        self.book_subscription_owners.clear();
601
602        self.book_deltas_counts.clear();
603        self.book_depth_counts.clear();
604        self.book_intervals.clear();
605        self.book_snapshot_counts.clear();
606        self.book_snapshot_sources.clear();
607        self.book_snapshotters.clear();
608        self.buffered_deltas_map.clear();
609        self.deltas_frame.clear();
610
611        self.synthetic_quote_feeds.clear();
612        self.synthetic_trade_feeds.clear();
613        self.subscribed_synthetic_quotes.clear();
614        self.subscribed_synthetic_trades.clear();
615        self.subscriptions_external.clear();
616
617        self.deferred_cmd_queue.borrow_mut().clear();
618
619        #[cfg(feature = "defi")]
620        self.clear_pool_updaters();
621
622        self.clock.borrow_mut().cancel_timers();
623
624        self.command_count = 0;
625        self.data_count = 0;
626        self.request_count = 0;
627        self.response_count = 0;
628    }
629
630    /// Disposes the engine, stopping all clients and canceling any timers.
631    pub fn dispose(&mut self) {
632        for client in self.get_clients_mut() {
633            if client.dispose().is_err() {
634                log::error!("Failed to dispose data client {}", client.client_id());
635            }
636        }
637
638        self.clear_option_chain_greeks_bootstraps();
639
640        // Continuous-future source handlers live outside bar_aggregator_handlers,
641        // so release them before dropping the aggregators
642        let mut cf_sources = Vec::new();
643
644        for state in self.continuous_future_subscriptions.values_mut() {
645            if let Some(name) = state.timer_name.take() {
646                self.clock.borrow_mut().cancel_timer(&name);
647            }
648
649            if let Some(subscription) = state.active_source_subscription.take() {
650                cf_sources.push((state.target_bar_type, subscription));
651            }
652        }
653
654        for (target_bar_type, subscription) in cf_sources {
655            self.unsubscribe_continuous_future_source(target_bar_type, subscription);
656        }
657        self.continuous_future_subscriptions.clear();
658
659        // Unsubscribe aggregator msgbus handlers so the typed routers don't keep
660        // entries pointing at dropped aggregators
661        let keys: Vec<BarAggregatorKey> = self.bar_aggregators.keys().copied().collect();
662        for (bar_type, request_id) in keys {
663            if let Err(e) = self.stop_bar_aggregator(bar_type, request_id) {
664                log::error!("Error stopping bar aggregator during dispose for {bar_type}: {e}");
665            }
666        }
667
668        self.subscriptions_external.clear();
669
670        self.clock.borrow_mut().cancel_timers();
671    }
672
673    /// Connects all registered data clients concurrently.
674    ///
675    /// Successful clients remain connected if another client fails.
676    /// Logs identify failed clients without provider error text, which can contain credentials.
677    ///
678    /// # Errors
679    ///
680    /// Returns an error if any client fails to connect, listing failed client IDs without provider
681    /// error text.
682    pub async fn connect(&mut self) -> anyhow::Result<()> {
683        let futures: Vec<_> = self
684            .get_clients_mut()
685            .into_iter()
686            .map(|client| async move {
687                let client_id = client.client_id();
688                (client_id, client.connect().await)
689            })
690            .collect();
691
692        let results = join_all(futures).await;
693        let mut failed_clients = Vec::new();
694
695        for (client_id, result) in results {
696            if result.is_err() {
697                log::error!("Failed to connect data client {client_id}: connection failed");
698                failed_clients.push(client_id.to_string());
699            }
700        }
701
702        if failed_clients.is_empty() {
703            Ok(())
704        } else {
705            anyhow::bail!(
706                "Failed to connect data clients: {}",
707                failed_clients.join(", ")
708            )
709        }
710    }
711
712    /// Disconnects all registered data clients concurrently.
713    ///
714    /// # Errors
715    ///
716    /// Returns an error if any client fails to disconnect.
717    pub async fn disconnect(&mut self) -> anyhow::Result<()> {
718        let futures: Vec<_> = self
719            .get_clients_mut()
720            .into_iter()
721            .map(|client| async move {
722                let client_id = client.client_id();
723                (client_id, client.disconnect().await)
724            })
725            .collect();
726
727        let results = join_all(futures).await;
728
729        // A closed session cannot answer, so discard its pending bootstraps
730        #[cfg(feature = "defi")]
731        self.abandon_pool_snapshots();
732
733        let failed_clients: Vec<_> = results
734            .into_iter()
735            .filter_map(|(client_id, result)| result.err().map(|_| client_id.to_string()))
736            .collect();
737
738        if failed_clients.is_empty() {
739            Ok(())
740        } else {
741            anyhow::bail!(
742                "Failed to disconnect data clients: {}",
743                failed_clients.join(", ")
744            )
745        }
746    }
747
748    /// Returns `true` if all registered data clients are currently connected.
749    #[must_use]
750    pub fn check_connected(&self) -> bool {
751        self.get_clients()
752            .iter()
753            .all(|client| client.is_connected())
754    }
755
756    /// Returns `true` if all registered data clients are currently disconnected.
757    #[must_use]
758    pub fn check_disconnected(&self) -> bool {
759        self.get_clients()
760            .iter()
761            .all(|client| !client.is_connected())
762    }
763
764    /// Returns all custom data types currently subscribed across all clients.
765    #[must_use]
766    pub fn subscribed_custom_data(&self) -> Vec<DataType> {
767        self.collect_subscriptions(|client| &client.subscriptions_custom)
768    }
769
770    /// Returns all instrument IDs currently subscribed across all clients.
771    #[must_use]
772    pub fn subscribed_instruments(&self) -> Vec<InstrumentId> {
773        self.collect_subscriptions(|client| &client.subscriptions_instrument)
774    }
775
776    /// Returns all instrument IDs for which quote subscriptions exist.
777    #[must_use]
778    pub fn subscribed_quotes(&self) -> Vec<InstrumentId> {
779        self.collect_subscriptions(|client| &client.subscriptions_quotes)
780    }
781
782    /// Returns all instrument IDs for which trade subscriptions exist.
783    #[must_use]
784    pub fn subscribed_trades(&self) -> Vec<InstrumentId> {
785        self.collect_subscriptions(|client| &client.subscriptions_trades)
786    }
787
788    /// Returns all bar types currently subscribed across all clients,
789    /// including internally aggregated subscriptions (v1 parity).
790    #[must_use]
791    pub fn subscribed_bars(&self) -> Vec<BarType> {
792        let mut subscribed = self.collect_subscriptions(|client| &client.subscriptions_bars);
793        subscribed.extend(
794            self.bar_aggregators
795                .keys()
796                .filter(|key| key.1.is_none() && !self.bar_aggregators_pending_stop.contains(*key))
797                .map(|(bar_type, _)| *bar_type),
798        );
799        subscribed
800    }
801
802    /// Returns all instrument IDs for which mark price subscriptions exist.
803    #[must_use]
804    pub fn subscribed_mark_prices(&self) -> Vec<InstrumentId> {
805        self.collect_subscriptions(|client| &client.subscriptions_mark_prices)
806    }
807
808    /// Returns all instrument IDs for which index price subscriptions exist.
809    #[must_use]
810    pub fn subscribed_index_prices(&self) -> Vec<InstrumentId> {
811        self.collect_subscriptions(|client| &client.subscriptions_index_prices)
812    }
813
814    /// Returns all instrument IDs for which funding rate subscriptions exist.
815    #[must_use]
816    pub fn subscribed_funding_rates(&self) -> Vec<InstrumentId> {
817        self.collect_subscriptions(|client| &client.subscriptions_funding_rates)
818    }
819
820    /// Returns all instrument IDs for which status subscriptions exist.
821    #[must_use]
822    pub fn subscribed_instrument_status(&self) -> Vec<InstrumentId> {
823        self.collect_subscriptions(|client| &client.subscriptions_instrument_status)
824    }
825
826    /// Returns all instrument IDs for which instrument close subscriptions exist.
827    #[must_use]
828    pub fn subscribed_instrument_close(&self) -> Vec<InstrumentId> {
829        self.collect_subscriptions(|client| &client.subscriptions_instrument_close)
830    }
831
832    /// Executes a `DataCommand` by delegating to subscribe, unsubscribe, or request handlers.
833    ///
834    /// This is the final synchronous dispatch point for data commands. Runtime command producers
835    /// should send to `DataEngine.queue_execute`, which lets the runner sequence command execution
836    /// before this method runs. The engine also calls this method for child commands generated while
837    /// processing a parent command, where immediate in-engine ordering matters.
838    ///
839    /// Errors during execution are logged.
840    pub fn execute(&mut self, cmd: DataCommand) {
841        match &cmd {
842            DataCommand::Subscribe(_) | DataCommand::Unsubscribe(_) => self.command_count += 1,
843            DataCommand::Request(_) => self.request_count += 1,
844            #[cfg(feature = "defi")]
845            DataCommand::DefiRequest(_) => self.request_count += 1,
846            #[cfg(feature = "defi")]
847            DataCommand::DefiSubscribe(_) | DataCommand::DefiUnsubscribe(_) => {
848                self.command_count += 1;
849            }
850            _ => {}
851        }
852
853        let command_context = match &cmd {
854            DataCommand::Subscribe(c) => (
855                "Subscribe",
856                c.client_id().copied(),
857                c.venue().copied(),
858                Some(c.command_id()),
859            ),
860            DataCommand::Unsubscribe(c) => (
861                "Unsubscribe",
862                c.client_id().copied(),
863                c.venue().copied(),
864                Some(c.command_id()),
865            ),
866            DataCommand::Request(c) => (
867                "Request",
868                c.client_id().copied(),
869                c.venue().copied(),
870                Some(*c.request_id()),
871            ),
872            #[cfg(feature = "defi")]
873            DataCommand::DefiRequest(c) => (
874                "DefiRequest",
875                c.client_id().copied(),
876                c.venue().copied(),
877                Some(*c.request_id()),
878            ),
879            #[cfg(feature = "defi")]
880            DataCommand::DefiSubscribe(c) => (
881                "DefiSubscribe",
882                c.client_id().copied(),
883                c.venue().copied(),
884                Some(c.command_id()),
885            ),
886            #[cfg(feature = "defi")]
887            DataCommand::DefiUnsubscribe(c) => (
888                "DefiUnsubscribe",
889                c.client_id().copied(),
890                c.venue().copied(),
891                Some(c.command_id()),
892            ),
893            _ => ("Unknown", None, None, None),
894        };
895
896        let result = match cmd {
897            DataCommand::Subscribe(c) => self.execute_subscribe(c),
898            DataCommand::Unsubscribe(c) => self.execute_unsubscribe(&c),
899            DataCommand::Request(c) => self.execute_request(c),
900            #[cfg(feature = "defi")]
901            DataCommand::DefiRequest(c) => self.execute_defi_request(c),
902            #[cfg(feature = "defi")]
903            DataCommand::DefiSubscribe(c) => self.execute_defi_subscribe(c),
904            #[cfg(feature = "defi")]
905            DataCommand::DefiUnsubscribe(c) => self.execute_defi_unsubscribe(&c),
906            _ => {
907                log::warn!("Unhandled DataCommand variant");
908                Ok(())
909            }
910        };
911
912        if let Err(e) = result {
913            log::error!("Data command failed: {command_context:?}: {e}");
914        }
915    }
916
917    /// Handles a subscribe command, updating internal state and forwarding to the client.
918    ///
919    /// # Errors
920    ///
921    /// Returns an error if the subscription is invalid (e.g., synthetic instrument for book data),
922    /// or if the underlying client operation fails.
923    pub fn execute_subscribe(&mut self, cmd: SubscribeCommand) -> anyhow::Result<()> {
924        cmd.validate_routing(&self.cache.borrow())?;
925        if let Some(client_id) = cmd.client_id().copied()
926            && self.is_external_client(client_id)
927        {
928            return self.subscribe_external(client_id, &cmd);
929        }
930
931        self.retain_intent(&cmd)?;
932
933        let retained = cmd.clone();
934        let (groups, result) = self.prepare_subscribe_groups(cmd);
935        self.dispatch_subscribe_groups(&retained, groups);
936        result
937    }
938
939    fn retain_intent(&mut self, cmd: &SubscribeCommand) -> anyhow::Result<()> {
940        let retain_intent = match cmd {
941            SubscribeCommand::Quotes(command) => {
942                self.quote_routing(&command.instrument_ids, command.params.as_ref())?
943                    == QuoteRouting::Client
944            }
945            SubscribeCommand::Trades(command) => !command.instrument_ids[0].is_synthetic(),
946            _ => false,
947        };
948
949        if let Some(client) = self.get_command_client(cmd.client_id(), cmd.venue()) {
950            if let SubscribeCommand::Bars(command) = cmd {
951                let external: Vec<_> = command
952                    .bar_types
953                    .iter()
954                    .copied()
955                    .filter(BarType::is_externally_aggregated)
956                    .collect();
957
958                if !external.is_empty() {
959                    let intent = SubscribeCommand::Bars(command.with_bar_types(external));
960                    client.retain_intent(&intent);
961                }
962            } else if retain_intent {
963                client.retain_intent(cmd);
964            }
965        }
966
967        Ok(())
968    }
969
970    fn prepare_subscribe_groups(
971        &mut self,
972        cmd: SubscribeCommand,
973    ) -> (Vec<SubscribeCommand>, anyhow::Result<()>) {
974        let members = if matches!(
975            &cmd,
976            SubscribeCommand::Quotes(_) | SubscribeCommand::Trades(_)
977        ) {
978            vec![cmd]
979        } else {
980            split_subscribe(cmd)
981        };
982
983        let mut groups: IndexMap<Option<Value>, Vec<SubscribeCommand>> = IndexMap::new();
984        let mut result = Ok(());
985
986        for member in members {
987            let prepared = match self.prepare_subscribe_member(&member) {
988                Ok(Some(prepared)) => prepared,
989                Ok(None) => continue,
990                Err(e) => {
991                    log::error!(
992                        "Subscribe preparation failed: command={}, client={:?}, instrument_ids={:?}, bar_types={:?}: {e}",
993                        member.command_id(),
994                        member.client_id(),
995                        member.instrument_ids(),
996                        member.bar_types(),
997                    );
998
999                    if result.is_ok() {
1000                        result = Err(e);
1001                    }
1002
1003                    continue;
1004                }
1005            };
1006
1007            // Members share one parent; only catalog-inferred params can differ
1008            let params = prepared.params().map(|params| {
1009                Value::Object(params.iter().map(|(k, v)| (k.clone(), v.clone())).collect())
1010            });
1011
1012            groups.entry(params).or_default().push(prepared);
1013        }
1014
1015        let groups = groups
1016            .into_values()
1017            .map(|mut members| {
1018                if members.len() == 1 {
1019                    return members.pop().expect("nonempty group");
1020                }
1021
1022                let first = &members[0];
1023                if first.instrument_ids().is_some() {
1024                    let ids: Vec<_> = members
1025                        .iter()
1026                        .flat_map(|member| member.instrument_ids().expect("same parent kind"))
1027                        .copied()
1028                        .collect();
1029                    first.with_instrument_ids(ids).expect("instrument command")
1030                } else if first.bar_types().is_some() {
1031                    let types: Vec<_> = members
1032                        .iter()
1033                        .flat_map(|member| member.bar_types().expect("same parent kind"))
1034                        .copied()
1035                        .collect();
1036                    first.with_bar_types(types).expect("bar command")
1037                } else {
1038                    first.clone()
1039                }
1040            })
1041            .collect();
1042
1043        (groups, result)
1044    }
1045
1046    fn dispatch_subscribe_groups(&mut self, cmd: &SubscribeCommand, groups: Vec<SubscribeCommand>) {
1047        for group in groups {
1048            let retained = if let Some(ids) = group.instrument_ids() {
1049                cmd.with_instrument_ids(ids).expect("instrument parent")
1050            } else if let Some(types) = group.bar_types() {
1051                cmd.with_bar_types(types).expect("bar parent")
1052            } else {
1053                cmd.clone()
1054            };
1055
1056            let retain_on_failure = !matches!(
1057                &group,
1058                SubscribeCommand::BookDeltas(_) | SubscribeCommand::BookDepth(_)
1059            );
1060
1061            if let Some(client) = self.get_command_client(group.client_id(), group.venue()) {
1062                client.execute_subscribe_with_retained(group, retained, retain_on_failure);
1063            } else {
1064                log::error!(
1065                    "Cannot handle command: no client found for client_id={:?}, venue={:?}",
1066                    group.client_id(),
1067                    group.venue()
1068                );
1069            }
1070        }
1071    }
1072
1073    fn prepare_subscribe_member(
1074        &mut self,
1075        cmd: &SubscribeCommand,
1076    ) -> anyhow::Result<Option<SubscribeCommand>> {
1077        // Update internal engine state
1078        match cmd {
1079            SubscribeCommand::BookDeltas(book_cmd) => {
1080                if !self.subscribe_book_deltas(book_cmd)? && self.client_subscription_active(cmd) {
1081                    return Ok(None);
1082                }
1083            }
1084            SubscribeCommand::BookDepth(book_cmd) => {
1085                if !self.subscribe_book_depth(book_cmd)? && self.client_subscription_active(cmd) {
1086                    return Ok(None);
1087                }
1088            }
1089            SubscribeCommand::BookSnapshots(cmd) => {
1090                // Handles client forwarding internally (forwards as BookDeltas)
1091                self.subscribe_book_snapshots(cmd)?;
1092                return Ok(None);
1093            }
1094            SubscribeCommand::Bars(cmd) if has_continuous_future_params(cmd.params.as_ref()) => {
1095                self.subscribe_continuous_future_bars(cmd)?;
1096                return Ok(None);
1097            }
1098            SubscribeCommand::Bars(cmd) => {
1099                self.subscribe_bars(cmd)?;
1100                if cmd
1101                    .single_bar_type()
1102                    .expect("member preparation receives one bar type")
1103                    .is_internally_aggregated()
1104                {
1105                    return Ok(None);
1106                }
1107            }
1108            SubscribeCommand::OptionChain(cmd) => {
1109                self.subscribe_option_chain(cmd);
1110                return Ok(None);
1111            }
1112            SubscribeCommand::Quotes(cmd) if self.subscribe_quotes(cmd)? => {
1113                return Ok(None);
1114            }
1115            SubscribeCommand::Trades(cmd) if self.subscribe_trades(cmd) => {
1116                return Ok(None);
1117            }
1118            SubscribeCommand::Instrument(cmd)
1119                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1120            {
1121                anyhow::bail!("Cannot subscribe for synthetic instrument `Instrument` data");
1122            }
1123            SubscribeCommand::InstrumentStatus(cmd)
1124                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1125            {
1126                anyhow::bail!("Cannot subscribe for synthetic instrument `InstrumentStatus` data");
1127            }
1128            SubscribeCommand::InstrumentClose(cmd)
1129                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1130            {
1131                anyhow::bail!("Cannot subscribe for synthetic instrument `InstrumentClose` data");
1132            }
1133            SubscribeCommand::OptionGreeks(cmd)
1134                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1135            {
1136                anyhow::bail!("Cannot subscribe for synthetic instrument `OptionGreeks` data");
1137            }
1138            _ => {} // Do nothing else
1139        }
1140
1141        let cmd = cmd.clone();
1142        #[cfg(feature = "streaming")]
1143        let cmd = self.subscribe_command_with_prefilled_start_ns(cmd)?;
1144
1145        Ok(Some(cmd))
1146    }
1147
1148    fn client_subscription_active(&mut self, cmd: &SubscribeCommand) -> bool {
1149        self.get_command_client(cmd.client_id(), cmd.venue())
1150            .is_some_and(|client| client.has_active_subscription(cmd))
1151    }
1152
1153    /// Handles an unsubscribe command, updating internal state and forwarding to the client.
1154    ///
1155    /// # Errors
1156    ///
1157    /// Returns an error if the underlying client operation fails.
1158    pub fn execute_unsubscribe(&mut self, cmd: &UnsubscribeCommand) -> anyhow::Result<()> {
1159        if let Some(bar_types) = cmd.bar_types() {
1160            ensure_routable_bar_types(bar_types, cmd.client_id().is_some())?;
1161        }
1162
1163        if let Some(ids) = cmd.instrument_ids() {
1164            ensure_routable_instrument_ids(ids, cmd.client_id().is_some())?;
1165        }
1166
1167        if let UnsubscribeCommand::Trades(trade) = cmd
1168            && !trade
1169                .instrument_ids
1170                .iter()
1171                .all(|id| id.is_synthetic() == trade.instrument_ids[0].is_synthetic())
1172        {
1173            anyhow::bail!("trade identifier lists cannot mix client and synthetic routes");
1174        }
1175
1176        if let Some(client_id) = cmd.client_id().copied()
1177            && self.is_external_client(client_id)
1178        {
1179            self.unsubscribe_external(client_id, cmd);
1180            return Ok(());
1181        }
1182
1183        if matches!(
1184            &cmd,
1185            UnsubscribeCommand::Instrument(_)
1186                | UnsubscribeCommand::InstrumentStatus(_)
1187                | UnsubscribeCommand::InstrumentClose(_)
1188                | UnsubscribeCommand::OptionGreeks(_)
1189                | UnsubscribeCommand::BookDeltas(_)
1190                | UnsubscribeCommand::BookDepth(_)
1191                | UnsubscribeCommand::BookSnapshots(_)
1192        ) && cmd
1193            .instrument_ids()
1194            .is_some_and(|ids| ids.len() > 1 && ids.iter().any(InstrumentId::is_synthetic))
1195        {
1196            anyhow::bail!("Cannot unsubscribe synthetic instruments for this data type");
1197        }
1198
1199        let (prepared, result) = self.prepare_unsubscribe_members(cmd);
1200
1201        for command in UnsubscribeCommand::group(prepared) {
1202            if let Some(client) = self.get_command_client(command.client_id(), command.venue()) {
1203                client.execute_unsubscribe(&command);
1204            } else {
1205                log::error!(
1206                    "Cannot handle command: no client found for client_id={:?}, venue={:?}",
1207                    command.client_id(),
1208                    command.venue()
1209                );
1210            }
1211        }
1212
1213        result
1214    }
1215
1216    fn prepare_unsubscribe_members(
1217        &mut self,
1218        cmd: &UnsubscribeCommand,
1219    ) -> (Vec<UnsubscribeCommand>, anyhow::Result<()>) {
1220        let members = if matches!(
1221            cmd,
1222            UnsubscribeCommand::Quotes(_) | UnsubscribeCommand::Trades(_)
1223        ) {
1224            vec![cmd.clone()]
1225        } else {
1226            split_unsubscribe(cmd)
1227        };
1228
1229        let mut prepared = Vec::new();
1230        let mut result = Ok(());
1231
1232        for member in members {
1233            match self.prepare_unsubscribe_member(&member) {
1234                Ok(true) => prepared.push(member),
1235                Ok(false) => {}
1236                Err(e) => {
1237                    log::error!(
1238                        "Unsubscribe preparation failed: command={}, client={:?}, instrument_ids={:?}, bar_types={:?}: {e}",
1239                        member.command_id(),
1240                        member.client_id(),
1241                        member.instrument_ids(),
1242                        member.bar_types(),
1243                    );
1244
1245                    if result.is_ok() {
1246                        result = Err(e);
1247                    }
1248                }
1249            }
1250        }
1251
1252        (prepared, result)
1253    }
1254
1255    fn prepare_unsubscribe_member(&mut self, cmd: &UnsubscribeCommand) -> anyhow::Result<bool> {
1256        if matches!(
1257            cmd,
1258            UnsubscribeCommand::BookDeltas(_)
1259                | UnsubscribeCommand::BookDepth(_)
1260                | UnsubscribeCommand::BookSnapshots(_)
1261        ) && !self.release_book_subscription(cmd)
1262        {
1263            return Ok(false);
1264        }
1265
1266        match &cmd {
1267            UnsubscribeCommand::BookDeltas(cmd) if !self.unsubscribe_book_deltas(cmd) => {
1268                return Ok(false);
1269            }
1270            UnsubscribeCommand::BookDepth(cmd) if !self.unsubscribe_book_depth(cmd) => {
1271                return Ok(false);
1272            }
1273            UnsubscribeCommand::BookSnapshots(cmd) => {
1274                // Handles client forwarding internally (forwards as BookDeltas)
1275                self.unsubscribe_book_snapshots(cmd);
1276                return Ok(false);
1277            }
1278            UnsubscribeCommand::Bars(cmd)
1279                if self.continuous_future_subscriptions.contains_key(
1280                    &cmd.single_bar_type()
1281                        .expect("member preparation receives one bar type")
1282                        .standard(),
1283                ) =>
1284            {
1285                // Don't tear down the chain while other actors remain subscribed
1286                let topic = switchboard::get_bars_topic(
1287                    cmd.single_bar_type()
1288                        .expect("member preparation receives one bar type")
1289                        .standard(),
1290                );
1291
1292                if msgbus::exact_subscriber_count_bars(topic) == 0 {
1293                    self.unsubscribe_continuous_future_bars(cmd);
1294                }
1295
1296                return Ok(false);
1297            }
1298            UnsubscribeCommand::Bars(cmd) => {
1299                self.unsubscribe_bars(cmd);
1300                if cmd
1301                    .single_bar_type()
1302                    .expect("member preparation receives one bar type")
1303                    .is_internally_aggregated()
1304                {
1305                    return Ok(false);
1306                }
1307            }
1308            UnsubscribeCommand::OptionChain(cmd) => {
1309                self.unsubscribe_option_chain(cmd);
1310                return Ok(false);
1311            }
1312            UnsubscribeCommand::Quotes(cmd) if self.unsubscribe_quotes(cmd)? => {
1313                return Ok(false);
1314            }
1315            UnsubscribeCommand::Trades(cmd) if self.unsubscribe_trades(cmd) => {
1316                return Ok(false);
1317            }
1318            UnsubscribeCommand::Instrument(cmd)
1319                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1320            {
1321                anyhow::bail!("Cannot unsubscribe from synthetic instrument `Instrument` data");
1322            }
1323            UnsubscribeCommand::InstrumentStatus(cmd)
1324                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1325            {
1326                anyhow::bail!(
1327                    "Cannot unsubscribe from synthetic instrument `InstrumentStatus` data"
1328                );
1329            }
1330            UnsubscribeCommand::InstrumentClose(cmd)
1331                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1332            {
1333                anyhow::bail!(
1334                    "Cannot unsubscribe from synthetic instrument `InstrumentClose` data"
1335                );
1336            }
1337            UnsubscribeCommand::OptionGreeks(cmd)
1338                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1339            {
1340                anyhow::bail!("Cannot unsubscribe from synthetic instrument `OptionGreeks` data");
1341            }
1342            _ => {}
1343        }
1344
1345        Ok(true)
1346    }
1347
1348    fn subscribe_external(
1349        &mut self,
1350        client_id: ClientId,
1351        cmd: &SubscribeCommand,
1352    ) -> anyhow::Result<()> {
1353        let (dispatch, result) = match cmd {
1354            SubscribeCommand::OptionChain(command) => {
1355                self.retain_external_option_chain(client_id, command, cmd);
1356                (Some(cmd.clone()), Ok(()))
1357            }
1358            _ => self.prepare_external_subscribe(client_id, cmd),
1359        };
1360
1361        let Some(dispatch) = dispatch else {
1362            return result;
1363        };
1364
1365        if self.is_stopped {
1366            log::debug!(
1367                "Skipped external subscribe after stop: client_id={client_id}, command_id={}",
1368                dispatch.command_id()
1369            );
1370            return result;
1371        }
1372
1373        register_external_streaming_type(&dispatch);
1374        publish_external_data_command(client_id, &dispatch);
1375
1376        if self.config.debug {
1377            log::debug!("Skipping subscribe command for external client {client_id}: {cmd:?}");
1378        }
1379
1380        result
1381    }
1382
1383    fn prepare_external_subscribe(
1384        &mut self,
1385        client_id: ClientId,
1386        cmd: &SubscribeCommand,
1387    ) -> (Option<SubscribeCommand>, anyhow::Result<()>) {
1388        let mut result = Ok(());
1389        let mut acquired = false;
1390        let mut fresh_ids = Vec::new();
1391        let mut fresh_bar_types = Vec::new();
1392
1393        for member in split_subscribe(cmd.clone()) {
1394            let key = SubscriptionKey::from_subscribe(&member);
1395            let instrument_id = member.instrument_ids().and_then(|ids| ids.first()).copied();
1396            let bar_type = member.bar_types().and_then(|types| types.first()).copied();
1397
1398            match self.retain_external_subscription(client_id, key.clone(), member) {
1399                Ok(false) => continue,
1400                Ok(true) => {}
1401                Err(e) => {
1402                    log::error!(
1403                        "External subscribe preparation failed: command={}, client={client_id}, subscription={}: {e}",
1404                        cmd.command_id(),
1405                        key.log_context(),
1406                    );
1407
1408                    if result.is_ok() {
1409                        result = Err(e);
1410                    }
1411
1412                    continue;
1413                }
1414            }
1415
1416            acquired = true;
1417            fresh_ids.extend(instrument_id);
1418            fresh_bar_types.extend(bar_type);
1419        }
1420
1421        if !acquired {
1422            return (None, result);
1423        }
1424
1425        let dispatch = if cmd.bar_types().is_some() {
1426            cmd.with_bar_types(fresh_bar_types)
1427                .expect("bar command has types")
1428        } else {
1429            cmd.with_instrument_ids(fresh_ids)
1430                .unwrap_or_else(|| cmd.clone())
1431        };
1432
1433        (Some(dispatch), result)
1434    }
1435
1436    fn retain_external_subscription(
1437        &mut self,
1438        client_id: ClientId,
1439        key: SubscriptionKey,
1440        command: SubscribeCommand,
1441    ) -> anyhow::Result<bool> {
1442        let key = (client_id, key);
1443
1444        if self
1445            .subscriptions_external
1446            .get_mut(&key)
1447            .is_some_and(|active| active.acquisitions.contains(&command.command_id()))
1448        {
1449            return Ok(false);
1450        }
1451
1452        // External owners share the first acquisition's source parameters
1453        let params = self.subscriptions_external.get_mut(&key).map_or_else(
1454            || command.params().cloned(),
1455            |active| active.command.params().cloned(),
1456        );
1457
1458        match &command {
1459            SubscribeCommand::BookDeltas(command) => {
1460                let mut command = command.clone();
1461                command.params = params;
1462                self.subscribe_book_deltas(&command)?;
1463            }
1464            SubscribeCommand::BookDepth(command) => {
1465                let mut command = command.clone();
1466                command.params = params;
1467                self.subscribe_book_depth(&command)?;
1468            }
1469            _ => {}
1470        }
1471
1472        Ok(self
1473            .subscriptions_external
1474            .retain(key, command.command_id(), command))
1475    }
1476
1477    fn unsubscribe_external(&mut self, client_id: ClientId, cmd: &UnsubscribeCommand) {
1478        let mut releases = IndexMap::new();
1479
1480        for member in split_unsubscribe(cmd) {
1481            let key = (client_id, SubscriptionKey::from_unsubscribe(&member));
1482            if matches!(
1483                member,
1484                UnsubscribeCommand::BookDeltas(_) | UnsubscribeCommand::BookDepth(_)
1485            ) && self.subscriptions_external.contains(&key)
1486            {
1487                if !self.release_book_subscription(&member) {
1488                    continue;
1489                }
1490
1491                match &member {
1492                    UnsubscribeCommand::BookDeltas(command) => {
1493                        self.unsubscribe_book_deltas(command);
1494                    }
1495                    UnsubscribeCommand::BookDepth(command) => {
1496                        self.unsubscribe_book_depth(command);
1497                    }
1498                    _ => unreachable!("only delta and depth subscriptions retain external books"),
1499                }
1500            }
1501
1502            let (acquisition_id, command) = match self.subscriptions_external.release(&key) {
1503                SubscriptionRelease::Retained => continue,
1504                SubscriptionRelease::Final(subscribe) => (
1505                    subscribe.command_id(),
1506                    subscribe.into_unsubscribe(
1507                        cmd.command_id(),
1508                        cmd.ts_init(),
1509                        cmd.correlation_id(),
1510                    ),
1511                ),
1512                SubscriptionRelease::Untracked => (cmd.command_id(), member),
1513            };
1514
1515            self.subscriptions_external.remove(&key);
1516            releases
1517                .entry(acquisition_id)
1518                .or_insert_with(Vec::new)
1519                .push(command);
1520        }
1521
1522        for command in releases.into_values().flat_map(UnsubscribeCommand::group) {
1523            publish_external_data_command(client_id, &command);
1524        }
1525
1526        if self.config.debug {
1527            log::debug!("Skipping unsubscribe command for external client {client_id}: {cmd:?}");
1528        }
1529    }
1530
1531    fn subscribe_quotes(&mut self, cmd: &SubscribeQuotes) -> anyhow::Result<bool> {
1532        match self.quote_routing(&cmd.instrument_ids, cmd.params.as_ref())? {
1533            QuoteRouting::Client => Ok(false),
1534            QuoteRouting::Synthetic => {
1535                for id in &cmd.instrument_ids {
1536                    self.subscribe_synthetic_quotes(*id);
1537                }
1538
1539                Ok(true)
1540            }
1541            QuoteRouting::Spread => {
1542                for id in &cmd.instrument_ids {
1543                    let member = cmd.with_instrument_ids(*id);
1544                    self.subscribe_spread_quotes(&member);
1545                }
1546
1547                Ok(true)
1548            }
1549        }
1550    }
1551
1552    fn unsubscribe_quotes(&mut self, cmd: &UnsubscribeQuotes) -> anyhow::Result<bool> {
1553        match self.quote_routing(&cmd.instrument_ids, cmd.params.as_ref())? {
1554            QuoteRouting::Client => Ok(false),
1555            QuoteRouting::Synthetic => {
1556                for id in &cmd.instrument_ids {
1557                    self.unsubscribe_synthetic_quotes(*id);
1558                }
1559
1560                Ok(true)
1561            }
1562            QuoteRouting::Spread => {
1563                for id in &cmd.instrument_ids {
1564                    let member = cmd.with_instrument_ids(*id);
1565                    self.unsubscribe_spread_quotes(&member);
1566                }
1567
1568                Ok(true)
1569            }
1570        }
1571    }
1572
1573    fn subscribe_trades(&mut self, cmd: &SubscribeTrades) -> bool {
1574        if !cmd.instrument_ids[0].is_synthetic() {
1575            return false;
1576        }
1577
1578        for id in &cmd.instrument_ids {
1579            self.subscribe_synthetic_trades(*id);
1580        }
1581
1582        true
1583    }
1584
1585    fn unsubscribe_trades(&mut self, cmd: &UnsubscribeTrades) -> bool {
1586        if !cmd.instrument_ids[0].is_synthetic() {
1587            return false;
1588        }
1589
1590        for id in &cmd.instrument_ids {
1591            self.unsubscribe_synthetic_trades(*id);
1592        }
1593
1594        true
1595    }
1596
1597    fn quote_routing(
1598        &self,
1599        ids: &[InstrumentId],
1600        params: Option<&Params>,
1601    ) -> anyhow::Result<QuoteRouting> {
1602        quote_routing(ids, None, params, &self.cache.borrow())
1603    }
1604
1605    /// Sends a [`RequestCommand`] to a suitable data client implementation.
1606    ///
1607    /// Submission errors also send the requester a terminal failure envelope.
1608    ///
1609    /// # Errors
1610    ///
1611    /// Returns an error if validation fails, no client is found for the given client ID or venue,
1612    /// or the client rejects direct submission.
1613    pub fn execute_request(&mut self, req: RequestCommand) -> anyhow::Result<()> {
1614        let request_id = *req.request_id();
1615        let result = self.execute_request_inner(req);
1616        if let Err(e) = &result {
1617            self.process_response_envelope(DataResponseEnvelope::terminal(
1618                request_id,
1619                Some(e.to_string()),
1620            ));
1621        }
1622
1623        result
1624    }
1625
1626    fn execute_request_inner(&mut self, req: RequestCommand) -> anyhow::Result<()> {
1627        Self::validate_request(&req)?;
1628        #[cfg(feature = "streaming")]
1629        self.validate_catalog_update(&req)?;
1630
1631        // Skip requests for external clients
1632        if let Some(cid) = req.client_id()
1633            && self.is_external_client(*cid)
1634        {
1635            if self.config.debug {
1636                log::debug!("Skipping data request for external client {cid}: {req:?}");
1637            }
1638            return Ok(());
1639        }
1640
1641        if let RequestCommand::Join(join) = req {
1642            return self.handle_request_join(join);
1643        }
1644
1645        if has_continuous_future_params(request_params(&req)) {
1646            return self.execute_continuous_future_request(req);
1647        }
1648
1649        let request_id = *req.request_id();
1650        self.prepare_request_bar_aggregators(&req)?;
1651
1652        if has_time_range_pipeline_params(request_params(&req))
1653            && is_time_range_pipeline_variant(&req)
1654        {
1655            let result = self.execute_time_range_pipeline_request(req);
1656            if result.is_err() {
1657                self.cleanup_request_bar_aggregators(&request_id);
1658            }
1659            return result;
1660        }
1661
1662        #[cfg(feature = "streaming")]
1663        if self.catalogs_registered() && streaming::is_date_range_variant(&req) {
1664            let result = self.dispatch_date_range_request(req);
1665            if result.is_err() {
1666                self.cleanup_request_bar_aggregators(&request_id);
1667            }
1668            return result;
1669        }
1670
1671        let result = self.dispatch_request_to_client(req);
1672
1673        if result.is_err() {
1674            self.cleanup_request_bar_aggregators(&request_id);
1675        }
1676
1677        result.map(|_| ())
1678    }
1679
1680    fn validate_request(req: &RequestCommand) -> anyhow::Result<()> {
1681        let (instrument_ids, requires_single, limit) = match req {
1682            RequestCommand::Instrument(cmd) => (&cmd.instrument_ids, false, None),
1683            RequestCommand::BookSnapshot(cmd) => (&cmd.instrument_ids, true, None),
1684            RequestCommand::BookDeltas(cmd) => (&cmd.instrument_ids, true, None),
1685            RequestCommand::BookDepth(cmd) => (&cmd.instrument_ids, true, None),
1686            RequestCommand::Quotes(cmd) => (&cmd.instrument_ids, false, cmd.limit),
1687            RequestCommand::Trades(cmd) => (&cmd.instrument_ids, false, cmd.limit),
1688            RequestCommand::FundingRates(cmd) => (&cmd.instrument_ids, true, None),
1689            RequestCommand::Bars(cmd) => {
1690                ensure_routable_bar_types(&cmd.bar_types, cmd.client_id.is_some())?;
1691                cmd.single_bar_type()?;
1692                return Ok(());
1693            }
1694            _ => return Ok(()),
1695        };
1696
1697        ensure_routable_instrument_ids(instrument_ids, req.client_id().is_some())?;
1698        anyhow::ensure!(
1699            !requires_single || instrument_ids.len() == 1,
1700            "request requires one instrument ID for this data type, received {}",
1701            instrument_ids.len(),
1702        );
1703        anyhow::ensure!(
1704            instrument_ids.len() == 1 || limit.is_none(),
1705            "A record limit is unsupported for multi-instrument requests",
1706        );
1707        Ok(())
1708    }
1709
1710    pub(super) fn dispatch_request_to_client(
1711        &mut self,
1712        req: RequestCommand,
1713    ) -> anyhow::Result<ClientId> {
1714        self.inflight_requests
1715            .insert(*req.request_id(), req.clone());
1716        let client_id = req.client_id().copied();
1717        let venue = req.venue().copied();
1718        let Some(client) = self.get_client(client_id.as_ref(), venue.as_ref()) else {
1719            anyhow::bail!("Cannot handle request: no client found for {client_id:?} {venue:?}");
1720        };
1721        let resolved_client_id = client.client_id();
1722
1723        #[rustfmt::skip]
1724        let result = match req {
1725            RequestCommand::Data(req) => client.request_data(req),
1726            RequestCommand::Instrument(req) => client.request_instrument(req),
1727            RequestCommand::Instruments(req) => client.request_instruments(req),
1728            RequestCommand::BookSnapshot(req) => client.request_book_snapshot(req),
1729            RequestCommand::BookDeltas(req) => client.request_book_deltas(req),
1730            RequestCommand::BookDepth(req) => client.request_book_depth(req),
1731            RequestCommand::Quotes(req) => client.request_quotes(req),
1732            RequestCommand::Trades(req) => client.request_trades(req),
1733            RequestCommand::FundingRates(req) => client.request_funding_rates(req),
1734            RequestCommand::OptionChainReferencePrice(req) => client.request_option_chain_reference_price(req),
1735            RequestCommand::Bars(req) => client.request_bars(req),
1736            RequestCommand::Join(_) => anyhow::bail!("RequestJoin must be handled by handle_request_join"),
1737        };
1738
1739        result.map_err(|e| {
1740            let context = format!("Data client {resolved_client_id} request failed");
1741
1742            let context = if let Some(rejection) = e.downcast_ref::<IdentifierCountError>() {
1743                format!("{context}: {rejection}")
1744            } else {
1745                context
1746            };
1747
1748            e.context(context)
1749        })?;
1750
1751        Ok(resolved_client_id)
1752    }
1753
1754    /// Processes a dynamically-typed data message.
1755    ///
1756    /// Supports instruments, response envelopes, request failures, funding rates, option greeks,
1757    /// instrument status, and custom data; unrecognized types are logged as errors.
1758    pub fn process(&mut self, data: &dyn Any) {
1759        if let Some(response) = data.downcast_ref::<DataResponseEnvelope>() {
1760            self.response_envelope(response.clone());
1761            return;
1762        }
1763
1764        if let Some(failure) = data.downcast_ref::<DataRequestFailed>() {
1765            self.request_failed(failure.clone());
1766            return;
1767        }
1768
1769        self.data_count += 1;
1770
1771        if let Some(instrument) = data.downcast_ref::<InstrumentAny>() {
1772            self.handle_instrument(instrument);
1773        } else if let Some(funding_rate) = data.downcast_ref::<FundingRateUpdate>() {
1774            self.handle_funding_rate(*funding_rate);
1775        } else if let Some(option_greeks) = data.downcast_ref::<OptionGreeks>() {
1776            self.cache.borrow_mut().add_option_greeks(*option_greeks);
1777            self.feed_option_greeks_to_pre_bootstrap_chain(option_greeks);
1778            let topic = switchboard::get_option_greeks_topic(option_greeks.instrument_id);
1779            msgbus::publish_option_greeks(topic, option_greeks);
1780            self.drain_deferred_commands();
1781        } else if let Some(status) = data.downcast_ref::<InstrumentStatus>() {
1782            self.handle_instrument_status(*status);
1783        } else if let Some(custom) = data.downcast_ref::<CustomData>() {
1784            self.handle_custom_data(custom);
1785        } else {
1786            #[cfg(feature = "defi")]
1787            if let Some(response) = data.downcast_ref::<PoolSnapshotResponse>() {
1788                self.handle_pool_snapshot_response(response);
1789                return;
1790            }
1791
1792            log::error!("Cannot process data {data:?}, type is unrecognized");
1793        }
1794    }
1795
1796    /// Processes a `Data` enum instance, dispatching to live handlers.
1797    #[allow(
1798        clippy::needless_pass_by_value,
1799        reason = "callers hand over ownership; the payload is only moved when a DeFi handler consumes it"
1800    )]
1801    pub fn process_data(&mut self, data: Data) {
1802        #[cfg(feature = "defi")]
1803        let data = match data {
1804            Data::Defi(defi) => {
1805                self.process_defi_data(*defi);
1806                return;
1807            }
1808            data => data,
1809        };
1810
1811        self.process_data_ref(DataRef::from(&data));
1812    }
1813
1814    /// Processes a borrowed `Data` enum view, dispatching to live handlers.
1815    ///
1816    /// DeFi payloads are cloned because the DeFi handler may buffer them while a pool snapshot
1817    /// is pending; no other variant clones its payload.
1818    pub fn process_data_ref(&mut self, data: DataRef<'_>) {
1819        #[cfg(feature = "defi")]
1820        let data = match data {
1821            DataRef::Defi(defi) => {
1822                self.process_defi_data(defi.clone());
1823                return;
1824            }
1825            data => data,
1826        };
1827
1828        self.data_count += 1;
1829
1830        match data {
1831            DataRef::Instrument(instrument) => self.handle_instrument(instrument),
1832            DataRef::BookDelta(delta) => self.handle_delta(*delta),
1833            DataRef::BookDeltas(deltas) => self.handle_deltas(deltas),
1834            DataRef::BookDepth(depth) => self.handle_depth(depth),
1835            DataRef::Quote(quote) => {
1836                self.handle_quote(*quote);
1837                self.drain_deferred_commands();
1838            }
1839            DataRef::Trade(trade) => self.handle_trade(*trade),
1840            DataRef::Bar(bar) => self.handle_bar(*bar),
1841            DataRef::MarkPrice(mark_price) => {
1842                self.handle_mark_price(*mark_price);
1843                self.drain_deferred_commands();
1844            }
1845            DataRef::IndexPrice(index_price) => {
1846                self.handle_index_price(*index_price);
1847                self.drain_deferred_commands();
1848            }
1849            DataRef::FundingRate(funding_rate) => {
1850                self.handle_funding_rate(*funding_rate);
1851                self.drain_deferred_commands();
1852            }
1853            DataRef::OptionGreeks(greeks) => {
1854                self.cache.borrow_mut().add_option_greeks(*greeks);
1855                self.feed_option_greeks_to_pre_bootstrap_chain(greeks);
1856                let topic = switchboard::get_option_greeks_topic(greeks.instrument_id);
1857                msgbus::publish_option_greeks(topic, greeks);
1858                self.drain_deferred_commands();
1859            }
1860            DataRef::InstrumentStatus(status) => {
1861                self.handle_instrument_status(*status);
1862                self.drain_deferred_commands();
1863            }
1864            DataRef::InstrumentClose(close) => self.handle_instrument_close(*close),
1865            DataRef::Custom(custom) => self.handle_custom_data(custom),
1866            #[cfg(feature = "defi")]
1867            DataRef::Defi(_) => unreachable!("handled before market data dispatch"),
1868            #[cfg(not(feature = "defi"))]
1869            #[allow(
1870                unreachable_patterns,
1871                reason = "DeFi variants can exist without this crate's defi feature"
1872            )]
1873            other => log_defi_data_dropped(other),
1874        }
1875    }
1876
1877    /// Processes a `Data` instance through the pipeline bus path.
1878    ///
1879    /// Pipeline mode publishes each item on the `data.pipeline.` topic family and gates cache
1880    /// writes on `disable_historical_cache`. None of the live-only side effects (synthetic
1881    /// republish, option-chain expiry, depth-derived quotes, deferred-command drains) run in this
1882    /// path.
1883    pub fn process_pipeline(&mut self, data: Data) {
1884        #[cfg(feature = "defi")]
1885        let data = match data {
1886            Data::Defi(defi) => {
1887                self.process_defi_data(*defi);
1888                return;
1889            }
1890            data => data,
1891        };
1892
1893        self.data_count += 1;
1894
1895        match data {
1896            Data::Instrument(instrument) => self.handle_instrument(&instrument),
1897            Data::BookDelta(delta) => self.handle_delta_pipeline(delta),
1898            Data::BookDeltas(deltas) => self.handle_deltas_pipeline(&deltas),
1899            Data::BookDepth(depth) => self.handle_depth_pipeline(&depth),
1900            Data::Quote(quote) => self.handle_quote_pipeline(quote),
1901            Data::Trade(trade) => self.handle_trade_pipeline(trade),
1902            Data::Bar(bar) => self.handle_bar_pipeline(bar),
1903            Data::MarkPrice(mark_price) => self.handle_mark_price_pipeline(mark_price),
1904            Data::IndexPrice(index_price) => self.handle_index_price_pipeline(index_price),
1905            Data::FundingRate(funding_rate) => {
1906                self.handle_funding_rate_pipeline(funding_rate);
1907            }
1908            Data::OptionGreeks(greeks) => self.handle_option_greeks_pipeline(greeks),
1909            Data::InstrumentStatus(status) => self.handle_instrument_status_pipeline(status),
1910            Data::InstrumentClose(close) => self.handle_instrument_close_pipeline(close),
1911            Data::Custom(custom) => self.handle_custom_data_pipeline(&custom),
1912            #[cfg(feature = "defi")]
1913            Data::Defi(_) => unreachable!("handled before market data dispatch"),
1914            #[cfg(not(feature = "defi"))]
1915            #[allow(
1916                unreachable_patterns,
1917                reason = "DeFi variants can exist without this crate's defi feature"
1918            )]
1919            other => log_defi_data_dropped(DataRef::from(&other)),
1920        }
1921    }
1922
1923    /// Processes a payload-only reply as a completed request.
1924    ///
1925    /// Payload-only replies do not certify coverage for catalog write-back. Use
1926    /// [`Self::response_envelope`] to report explicit source completion.
1927    pub fn response(&mut self, mut resp: DataResponse) {
1928        resp.trim_to_bounds();
1929        let request = self.inflight_requests.remove(resp.correlation_id());
1930        #[cfg(feature = "streaming")]
1931        if request
1932            .as_ref()
1933            .and_then(request_params)
1934            .is_some_and(|params| params.get_bool(streaming::PARAM_UPDATE_CATALOG) == Some(true))
1935        {
1936            log::warn!(
1937                "Catalog write-back requires an explicit completed response envelope, request={}",
1938                resp.correlation_id()
1939            );
1940        }
1941
1942        #[cfg(not(feature = "streaming"))]
1943        let _ = request;
1944        let envelope = DataResponseEnvelope::new(resp, true, None);
1945        self.process_response_envelope(envelope);
1946    }
1947
1948    /// Completes failed adapter work while preserving any returned rows.
1949    pub fn request_failed(&mut self, failure: DataRequestFailed) {
1950        log::error!(
1951            "Data request {} failed: {}",
1952            failure.request_id,
1953            failure.reason
1954        );
1955
1956        let envelope = if let Some(mut response) = failure.response {
1957            response.trim_to_bounds();
1958            DataResponseEnvelope::new(response, true, None)
1959                .with_correlation_id(failure.request_id)
1960                .with_failure(failure.reason)
1961        } else {
1962            DataResponseEnvelope::terminal(failure.request_id, Some(failure.reason))
1963        };
1964
1965        self.process_response_envelope(envelope);
1966    }
1967
1968    /// Processes an adapter envelope, retaining intermediate request handlers.
1969    pub fn response_envelope(&mut self, mut envelope: DataResponseEnvelope) {
1970        if let Some(response) = envelope.response_mut() {
1971            response.trim_to_bounds();
1972        }
1973
1974        self.process_response_envelope(envelope);
1975    }
1976
1977    fn process_response_envelope(&mut self, mut envelope: DataResponseEnvelope) {
1978        let accepted = match envelope.response() {
1979            Some(DataResponse::Instrument(response)) => self.cache_instrument(&response.data),
1980            Some(DataResponse::Instruments(response)) => {
1981                let mut accepted = true;
1982                for instrument in &response.data {
1983                    accepted &= self.cache_instrument(instrument);
1984                }
1985
1986                accepted
1987            }
1988            _ => true,
1989        };
1990
1991        if !accepted {
1992            envelope = DataResponseEnvelope::terminal(
1993                *envelope.correlation_id(),
1994                Some("Instrument currency integrity check failed".into()),
1995            );
1996        }
1997
1998        let request = self.inflight_requests.remove(envelope.correlation_id());
1999        #[cfg(feature = "streaming")]
2000        let envelope = match (&request, envelope.response()) {
2001            (Some(request), Some(response))
2002                if envelope.completed() && envelope.failure_reason().is_none() =>
2003            {
2004                match self.update_catalog_from_response(response, request) {
2005                    Ok(()) => envelope,
2006                    Err(e) => envelope.with_failure(e.to_string()),
2007                }
2008            }
2009            _ => envelope,
2010        };
2011
2012        #[cfg(not(feature = "streaming"))]
2013        let _ = request;
2014
2015        let Some(envelope) = self.handle_request_pipeline_response(envelope) else {
2016            return;
2017        };
2018
2019        if self
2020            .request_pipeline_parent_request_id
2021            .contains_key(envelope.correlation_id())
2022        {
2023            self.process_response_envelope(envelope);
2024            return;
2025        }
2026
2027        if self.handle_continuous_future_child_response(&envelope)
2028            || self.handle_time_range_pipeline_child_response(&envelope)
2029        {
2030            return;
2031        }
2032
2033        let Some(resp) = envelope.response().cloned() else {
2034            self.finish_empty_response(&envelope);
2035            return;
2036        };
2037
2038        self.route_response_payload(resp, &envelope);
2039    }
2040
2041    fn finish_empty_response(&mut self, envelope: &DataResponseEnvelope) {
2042        if let Some(original_id) = self
2043            .parent_join_request_id
2044            .remove(envelope.correlation_id())
2045        {
2046            if let Some(original) = self.pending_join_requests.remove(&original_id) {
2047                for child_id in original.request_ids {
2048                    msgbus::send_response_envelope(&DataResponseEnvelope::terminal(
2049                        child_id,
2050                        envelope.failure_reason().map(str::to_owned),
2051                    ));
2052                }
2053            }
2054
2055            self.process_response_envelope(DataResponseEnvelope::terminal(
2056                original_id,
2057                envelope.failure_reason().map(str::to_owned),
2058            ));
2059            return;
2060        }
2061
2062        self.cleanup_request_bar_aggregators(envelope.correlation_id());
2063        msgbus::send_response_envelope(envelope);
2064    }
2065
2066    fn route_response_payload(&mut self, resp: DataResponse, envelope: &DataResponseEnvelope) {
2067        let resp = rebind_response_correlation(resp, *envelope.correlation_id());
2068        if log::log_enabled!(log::Level::Debug) {
2069            let correlation_id = resp.correlation_id();
2070            match resp.record_count() {
2071                Some(count) => log::debug!(
2072                    "{RECV}{RES} {} correlation_id={correlation_id} records={count}",
2073                    resp.kind(),
2074                ),
2075                None => log::debug!(
2076                    "{RECV}{RES} {} correlation_id={correlation_id}",
2077                    resp.kind(),
2078                ),
2079            }
2080        }
2081        log::trace!("{RECV}{RES} {resp:?}");
2082
2083        self.response_count += 1;
2084
2085        if self
2086            .parent_join_request_id
2087            .contains_key(resp.correlation_id())
2088        {
2089            self.finalize_request_join(resp, envelope.failure_reason().map(str::to_owned));
2090            return;
2091        }
2092
2093        if !self.handle_response_payload(&resp) {
2094            return;
2095        }
2096
2097        self.process_request_bar_aggregation_response(&resp, envelope.completed());
2098
2099        msgbus::send_response_envelope(envelope);
2100    }
2101
2102    fn handle_response_payload(&mut self, resp: &DataResponse) -> bool {
2103        let correlation_id = *resp.correlation_id();
2104
2105        match resp {
2106            DataResponse::Instrument(_) | DataResponse::Instruments(_) => {}
2107            DataResponse::Quotes(r) => {
2108                if !log_if_empty_response(
2109                    &r.data,
2110                    &format_args!("{:?}", r.instrument_ids),
2111                    &correlation_id,
2112                ) {
2113                    self.handle_quotes(&r.data);
2114                }
2115            }
2116            DataResponse::Trades(r) => {
2117                if !log_if_empty_response(
2118                    &r.data,
2119                    &format_args!("{:?}", r.instrument_ids),
2120                    &correlation_id,
2121                ) {
2122                    self.handle_trades(&r.data);
2123                }
2124            }
2125            DataResponse::FundingRates(r) => {
2126                if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2127                    self.handle_funding_rates(&r.data);
2128                }
2129            }
2130            DataResponse::Bars(r) => {
2131                if !log_if_empty_response(&r.data, &r.bar_type, &correlation_id) {
2132                    self.handle_bars(&r.data);
2133                }
2134            }
2135            DataResponse::Book(r) => self.handle_book_response(&r.data),
2136            DataResponse::BookDeltas(r) => {
2137                if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2138                    self.handle_book_deltas_response(r);
2139                }
2140            }
2141            DataResponse::BookDepth(r) => {
2142                if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2143                    self.handle_book_depth_response(r);
2144                }
2145            }
2146            DataResponse::OptionChainReferencePrice(r) => {
2147                self.process_request_bar_aggregation_response(resp, true);
2148                self.handle_option_chain_reference_price_response(&correlation_id, r);
2149                return false;
2150            }
2151            DataResponse::Data(_) => {}
2152        }
2153
2154        true
2155    }
2156
2157    /// Registers a parent request whose response will be rebuilt from `n_components` leg responses.
2158    pub fn new_request_pipeline(&mut self, parent: RequestCommand, n_components: usize) {
2159        let parent_id = *parent.request_id();
2160        self.request_pipeline_n_components
2161            .insert(parent_id, n_components);
2162        self.request_pipeline_parent_request
2163            .insert(parent_id, parent);
2164        self.request_pipeline_responses
2165            .insert(parent_id, Vec::with_capacity(n_components));
2166    }
2167
2168    /// Registers a leg `request_id` as a child of the pipeline keyed by `parent_id`.
2169    ///
2170    /// # Panics
2171    ///
2172    /// Panics if the parent pipeline is not registered.
2173    pub fn register_request_pipeline_leg(&mut self, leg_id: UUID4, parent_id: UUID4) {
2174        let slots = self
2175            .request_pipeline_responses
2176            .get_mut(&parent_id)
2177            .expect("pipeline registered before its legs");
2178        let index = slots.len();
2179        slots.push(None);
2180        self.request_pipeline_parent_request_id
2181            .insert(leg_id, (parent_id, index));
2182    }
2183
2184    fn handle_request_pipeline_response(
2185        &mut self,
2186        envelope: DataResponseEnvelope,
2187    ) -> Option<DataResponseEnvelope> {
2188        let leg_id = *envelope.correlation_id();
2189
2190        let Some(&(parent_id, index)) = self.request_pipeline_parent_request_id.get(&leg_id) else {
2191            return Some(envelope);
2192        };
2193
2194        let completed = envelope.completed();
2195        let slots = self.request_pipeline_responses.get_mut(&parent_id)?;
2196        slots[index].get_or_insert_with(Vec::new).push(envelope);
2197
2198        if !completed {
2199            return None;
2200        }
2201
2202        self.request_pipeline_parent_request_id.remove(&leg_id);
2203        let expected = *self.request_pipeline_n_components.get(&parent_id)?;
2204
2205        if slots
2206            .iter()
2207            .flatten()
2208            .filter(|slot| slot.last().is_some_and(DataResponseEnvelope::completed))
2209            .count()
2210            < expected
2211        {
2212            return None;
2213        }
2214
2215        let slots = self.request_pipeline_responses.remove(&parent_id)?;
2216        self.request_pipeline_n_components.remove(&parent_id);
2217        let parent = self.request_pipeline_parent_request.remove(&parent_id);
2218        let failures = slots
2219            .iter()
2220            .flatten()
2221            .flatten()
2222            .filter_map(|value| value.failure_reason())
2223            .collect::<Vec<_>>();
2224        let failure_reason = (!failures.is_empty()).then(|| failures.join("; "));
2225        let legs = slots
2226            .into_iter()
2227            .flatten()
2228            .flatten()
2229            .filter_map(|value| value.response().cloned())
2230            .collect::<Vec<_>>();
2231
2232        if legs.is_empty() {
2233            return Some(DataResponseEnvelope::terminal(parent_id, failure_reason));
2234        }
2235
2236        let (parent_start, parent_end) = parent_request_window(parent.as_ref());
2237        if self.emit_partitioned_book_responses(parent_id, parent.as_ref(), &legs) {
2238            return Some(DataResponseEnvelope::terminal(parent_id, failure_reason));
2239        }
2240
2241        let rebuilt = rebuild_pipeline_response(parent_id, parent.as_ref(), legs);
2242
2243        let Some(mut rebuilt) = rebuilt else {
2244            let reason = failure_reason
2245                .unwrap_or_else(|| "Request pipeline could not rebuild its response".to_owned());
2246
2247            return Some(DataResponseEnvelope::terminal(parent_id, Some(reason)));
2248        };
2249
2250        if let DataResponse::BookDeltas(r) = &mut rebuilt {
2251            self.book_deltas_snapshot_replay(r);
2252        }
2253
2254        // Client definitions can have a newer ts_init than the requested history
2255        if !matches!(parent, Some(RequestCommand::Instrument(_)))
2256            && (parent_start.is_some() || parent_end.is_some())
2257        {
2258            rebuilt.trim_to_bounds();
2259        }
2260
2261        let result = DataResponseEnvelope::new(rebuilt, true, None);
2262        Some(if let Some(reason) = failure_reason {
2263            result.with_failure(reason)
2264        } else {
2265            result
2266        })
2267    }
2268
2269    fn emit_partitioned_book_responses(
2270        &mut self,
2271        parent_id: UUID4,
2272        parent: Option<&RequestCommand>,
2273        legs: &[DataResponse],
2274    ) -> bool {
2275        if !matches!(parent, Some(RequestCommand::BookDeltas(_))) {
2276            return false;
2277        }
2278
2279        let mut ids = legs.iter().filter_map(|r| match r {
2280            DataResponse::BookDeltas(r) => Some(r.instrument_id),
2281            _ => None,
2282        });
2283
2284        let Some(first) = ids.next() else {
2285            return false;
2286        };
2287
2288        if !ids.any(|id| !same_instrument(id, first)) {
2289            return false;
2290        }
2291
2292        let (start, end) = parent_request_window(parent);
2293
2294        for response in legs {
2295            let child_id = *response.correlation_id();
2296            let mut response = rebind_response_correlation(response.clone(), parent_id);
2297            if let DataResponse::BookDeltas(r) = &mut response {
2298                r.start = start.or(r.start);
2299                r.end = end.or(r.end);
2300                self.book_deltas_snapshot_replay(r);
2301            }
2302
2303            response.trim_to_bounds();
2304            self.process_response_envelope(DataResponseEnvelope::new(
2305                response,
2306                false,
2307                Some(child_id),
2308            ));
2309        }
2310
2311        true
2312    }
2313
2314    fn handle_request_join(&mut self, req: RequestJoin) -> anyhow::Result<()> {
2315        if has_time_range_pipeline_params(req.params.as_ref()) {
2316            return self.execute_time_range_pipeline_request(RequestCommand::Join(req));
2317        }
2318
2319        let now_ns = self.clock.borrow().timestamp_ns();
2320        let now_dt = now_ns.to_datetime_utc();
2321        let zero = jiff::Timestamp::UNIX_EPOCH;
2322        let start = req.start.unwrap_or(zero).min(now_dt);
2323        let end = req.end.unwrap_or(now_dt).min(now_dt);
2324        let dated = req.with_dates(Some(start), Some(end), now_ns);
2325
2326        let original_id = req.request_id;
2327        let dated_id = dated.request_id;
2328
2329        self.pending_join_requests.insert(original_id, req);
2330        self.parent_join_request_id.insert(dated_id, original_id);
2331
2332        let leg_ids: Vec<UUID4> = dated.request_ids.clone();
2333        self.new_request_pipeline(RequestCommand::Join(dated), leg_ids.len());
2334        for leg_id in leg_ids {
2335            self.register_request_pipeline_leg(leg_id, dated_id);
2336        }
2337
2338        Ok(())
2339    }
2340
2341    fn finalize_request_join(&mut self, resp: DataResponse, failure_reason: Option<String>) {
2342        let dated_id = *resp.correlation_id();
2343        let Some(original_id) = self.parent_join_request_id.remove(&dated_id) else {
2344            log::error!("parent_join_request_id missing for dated correlation {dated_id}");
2345            return;
2346        };
2347
2348        let Some(original) = self.pending_join_requests.remove(&original_id) else {
2349            log::error!("pending_join_requests missing for original {original_id}");
2350            return;
2351        };
2352
2353        for leg_request_id in &original.request_ids {
2354            let terminal = DataResponseEnvelope::terminal(*leg_request_id, failure_reason.clone());
2355            msgbus::send_response_envelope(&terminal);
2356        }
2357
2358        // Route the final join response through the normal response path so
2359        // bounds-trim against the parent window runs and the per-variant
2360        // handlers (cache writes, request bar aggregators) fire. The pipeline
2361        // and join staging maps for this request have already been popped, so
2362        // the recursive call cannot re-enter either gate.
2363        let final_resp = rebind_response_correlation(resp, original_id);
2364        let envelope = DataResponseEnvelope::new(final_resp, true, None);
2365        self.process_response_envelope(if let Some(reason) = failure_reason {
2366            envelope.with_failure(reason)
2367        } else {
2368            envelope
2369        });
2370    }
2371}
2372
2373fn register_external_streaming_type(cmd: &SubscribeCommand) {
2374    if let Some(payload_type) = streaming_payload_type(cmd) {
2375        msgbus::get_message_bus()
2376            .borrow_mut()
2377            .add_streaming_type(payload_type);
2378    }
2379}
2380
2381fn publish_external_data_command<T>(client_id: ClientId, command: &T)
2382where
2383    T: Any,
2384{
2385    let topic = format!("commands.data.{client_id}");
2386    msgbus::publish_any(topic.into(), command);
2387}
2388
2389#[rustfmt::skip]
2390fn streaming_payload_type(cmd: &SubscribeCommand) -> Option<BusPayloadType> {
2391    match cmd {
2392        SubscribeCommand::Data(cmd) => Some(BusPayloadType::Custom(Ustr::from(
2393            cmd.data_type.type_name(),
2394        ))),
2395        SubscribeCommand::Instrument(_) | SubscribeCommand::Instruments(_) => Some(BusPayloadType::Instrument),
2396        SubscribeCommand::BookDeltas(_) | SubscribeCommand::BookSnapshots(_) => Some(BusPayloadType::OrderBookDeltas),
2397        SubscribeCommand::BookDepth(_) => Some(BusPayloadType::OrderBookDepth),
2398        SubscribeCommand::Quotes(_) => Some(BusPayloadType::QuoteTick),
2399        SubscribeCommand::Trades(_) => Some(BusPayloadType::TradeTick),
2400        SubscribeCommand::Bars(_) => Some(BusPayloadType::Bar),
2401        SubscribeCommand::MarkPrices(_) => Some(BusPayloadType::MarkPriceUpdate),
2402        SubscribeCommand::IndexPrices(_) => Some(BusPayloadType::IndexPriceUpdate),
2403        SubscribeCommand::FundingRates(_) => Some(BusPayloadType::FundingRateUpdate),
2404        SubscribeCommand::OptionGreeks(_) => Some(BusPayloadType::OptionGreeks),
2405        SubscribeCommand::InstrumentStatus(_)
2406        | SubscribeCommand::InstrumentClose(_)
2407        | SubscribeCommand::OptionChain(_) => None,
2408    }
2409}
2410
2411#[cfg(not(feature = "defi"))]
2412fn log_defi_data_dropped(data: DataRef<'_>) {
2413    log::error!("Cannot process data {data:?}, nautilus-data built without its `defi` feature");
2414}