Skip to main content

nautilus_data/engine/
mod.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Provides a high-performance `DataEngine` for all environments.
17//!
18//! The `DataEngine` is the central component of the entire data stack.
19//! The data engines primary responsibility is to orchestrate interactions between
20//! the `DataClient` instances, and the rest of the platform. This includes sending
21//! requests to, and receiving responses from, data endpoints via its registered
22//! data clients.
23//!
24//! The engine employs a simple fan-in fan-out messaging pattern to execute
25//! `DataCommand` type messages, and process `DataResponse` messages or market data
26//! objects.
27//!
28//! Alternative implementations can be written on top of the generic engine - which
29//! just need to override the `execute`, `process`, `send`, and `receive` methods.
30
31pub mod bar;
32pub mod book;
33pub mod config;
34
35mod commands;
36mod dispatch;
37mod futures;
38mod handlers;
39mod option_chain;
40mod registry;
41mod requests;
42mod responses;
43mod spread_quote;
44mod synthetic;
45mod time_range;
46
47#[cfg(feature = "defi")]
48pub mod pool;
49
50#[cfg(feature = "streaming")]
51mod streaming;
52
53use std::{
54    any::{Any, type_name},
55    cell::RefCell,
56    collections::VecDeque,
57    fmt::{Debug, Display},
58    mem,
59    num::NonZeroUsize,
60    rc::Rc,
61    str::FromStr,
62};
63
64use ::futures::future::join_all;
65use ahash::{AHashMap, AHashSet};
66use anyhow::Context;
67pub use bar::BarAggregatorSubscription;
68use bar::{BarAggregationSubscription, BarAggregatorKey, BarAggregatorWarmup, bar_aggregator_key};
69use book::{
70    BookDeltasKey, BookSnapshotInfos, BookSnapshotKey, BookSnapshotSource, BookSnapshotter,
71    BookSubscription, BookSubscriptionOwner, BookUpdater,
72};
73pub(crate) use commands::{DeferredCommand, DeferredCommandQueue};
74use config::DataEngineConfig;
75use dispatch::{log_error_on_cache_insert, process_engine_bar};
76use futures::{ContinuousFutureRoller, ContinuousFutureSubscriptionState, datetime_to_unix_nanos};
77use handlers::{
78    BAR_AGGREGATOR_PRIORITY, BarBarHandler, BarQuoteHandler, BarTradeHandler, SpreadQuoteHandler,
79};
80use indexmap::IndexMap;
81#[cfg(feature = "defi")]
82use nautilus_common::messages::defi::PoolSnapshotResponse;
83use nautilus_common::{
84    cache::Cache,
85    clock::Clock,
86    logging::{RECV, RES},
87    messages::data::{
88        BarsResponse, BookDeltasResponse, BookDepthResponse, CustomDataResponse, DataCommand,
89        DataRequestFailed, DataResponse, DataResponseEnvelope, IdentifierCountError,
90        OptionChainReferencePriceResponse, PARAMS_IS_PARENT, QuoteRouting, RequestBars,
91        RequestCommand, RequestJoin, RequestOptionChainReferencePrice, RequestQuotes,
92        RequestTrades, SubscribeBars, SubscribeBookDeltas, SubscribeBookDepth, SubscribeCommand,
93        SubscribeOptionChain, SubscribeOptionGreeks, SubscribeQuotes, SubscribeTrades,
94        UnsubscribeBars, UnsubscribeBookDeltas, UnsubscribeBookDepth, UnsubscribeBookSnapshots,
95        UnsubscribeCommand, UnsubscribeInstrumentStatus, UnsubscribeOptionChain,
96        UnsubscribeOptionGreeks, UnsubscribeQuotes, UnsubscribeTrades, ensure_routable_bar_types,
97        ensure_routable_instrument_ids, is_parent_subscription, quote_routing,
98    },
99    msgbus::{
100        self, BusPayloadType, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
101        get_message_bus,
102        switchboard::{self, MessagingSwitchboard},
103    },
104    runner::get_data_cmd_sender,
105    timer::{TimeEvent, TimeEventCallback},
106};
107use nautilus_core::{
108    DurationNanos, Params, UUID4, UnixNanos, WeakCell,
109    correctness::{FAILED, check_key_in_map, check_key_not_in_map, check_predicate_true},
110    datetime::NANOSECONDS_IN_DAY,
111};
112#[cfg(feature = "defi")]
113use nautilus_model::defi::DefiData;
114use nautilus_model::{
115    data::{
116        Bar, BarType, CustomData, Data, DataRef, DataType, FundingRateUpdate, HasTsInit,
117        IndexPriceUpdate, InstrumentClose, InstrumentStatus, MarkPriceUpdate, OrderBookDelta,
118        OrderBookDeltas, OrderBookDepth, QuoteTick, TradeTick,
119        option_chain::{OptionGreeks, StrikeRange},
120    },
121    enums::{
122        AggregationSource, BarAggregation, BookType, InstrumentClass, MarketStatusAction,
123        PriceType, RecordFlag,
124    },
125    identifiers::{
126        ClientId, GENERIC_SPREAD_ID_SEPARATOR, InstrumentId, OptionSeriesId, Venue,
127        parse_generic_spread_id_legs,
128    },
129    instruments::{Instrument, InstrumentAny, SyntheticInstrument},
130    orderbook::OrderBook,
131    types::{Price, Quantity},
132};
133use option_chain::{
134    OptionChainBootstrapper, OptionChainGreeksBootstrap, PendingOptionChainRequest,
135};
136use requests::{
137    ContinuousFutureRequest, ContinuousFutureRequestState, ContinuousFutureSegment,
138    ContinuousFutureSource, RequestBarAggregation, continuous_future_request_from_bars,
139    continuous_future_subscription_from_bars, has_continuous_future_params,
140    request_bar_aggregation_from_params, request_params, response_params,
141};
142use responses::{
143    log_if_empty_response, parent_request_window, rebind_response_correlation,
144    rebuild_pipeline_response, same_instrument,
145};
146use serde_json::Value;
147use spread_quote::SpreadQuoteState;
148#[cfg(feature = "streaming")]
149use streaming::CatalogMap;
150use time_range::{
151    TimeRangePipelineState, has_time_range_pipeline_params, is_time_range_pipeline_variant,
152};
153use ustr::Ustr;
154
155#[cfg(feature = "defi")]
156#[allow(unused_imports)] // Brings DeFi impl blocks into scope
157use crate::defi::engine as _;
158#[cfg(feature = "defi")]
159use crate::engine::pool::PoolUpdater;
160use crate::{
161    aggregation::{
162        BarAggregator, RenkoBarAggregator, SpreadQuoteAggregator, TickBarAggregator,
163        TickImbalanceBarAggregator, TickRunsBarAggregator, TimeBarAggregator, ValueBarAggregator,
164        ValueImbalanceBarAggregator, ValueRunsBarAggregator, VolumeBarAggregator,
165        VolumeImbalanceBarAggregator, VolumeRunsBarAggregator,
166    },
167    client::DataClientAdapter,
168    option_chains::OptionChainManager,
169    subscription::{
170        SubscriptionKey, SubscriptionRegistry, SubscriptionRelease, split_subscribe,
171        split_unsubscribe,
172    },
173};
174
175/// Provides a high-performance `DataEngine` for all environments.
176#[derive(Debug)]
177pub struct DataEngine {
178    clock: Rc<RefCell<dyn Clock>>,
179    cache: Rc<RefCell<Cache>>,
180    config: DataEngineConfig,
181    is_stopped: bool,
182    msgbus_priority: u32,
183    subscriptions_external: SubscriptionRegistry<(ClientId, SubscriptionKey), SubscribeCommand>,
184    clients: IndexMap<ClientId, DataClientAdapter>,
185    default_client_id: Option<ClientId>,
186    routing_map: IndexMap<Venue, ClientId>,
187    book_intervals: AHashMap<NonZeroUsize, BookSnapshotInfos>,
188    book_snapshot_counts: IndexMap<BookSnapshotKey, usize>,
189    book_snapshot_sources: AHashMap<InstrumentId, BookSnapshotSource>,
190    book_deltas_counts: IndexMap<BookDeltasKey, usize>,
191    book_depth_counts: IndexMap<BookDeltasKey, usize>,
192    book_updaters: AHashMap<InstrumentId, Rc<BookUpdater>>,
193    book_subscriptions: AHashMap<InstrumentId, BookSubscription>,
194    book_subscription_owners: AHashMap<InstrumentId, Vec<Rc<BookSubscriptionOwner>>>,
195    book_snapshotters: AHashMap<NonZeroUsize, Rc<BookSnapshotter>>,
196    bar_aggregators: IndexMap<BarAggregatorKey, Rc<RefCell<Box<dyn BarAggregator>>>>,
197    bar_aggregator_handlers: AHashMap<BarAggregatorKey, Vec<BarAggregatorSubscription>>,
198    bar_aggregator_warmups: AHashMap<BarAggregatorKey, BarAggregatorWarmup>,
199    bar_aggregators_pending_stop: AHashSet<BarAggregatorKey>,
200    subscriptions_bar_aggregation: AHashMap<BarType, BarAggregationSubscription>,
201    request_bar_aggregations: AHashMap<UUID4, RequestBarAggregation>,
202    inflight_requests: AHashMap<UUID4, RequestCommand>,
203    request_pipeline_parent_request: AHashMap<UUID4, RequestCommand>,
204    request_pipeline_n_components: AHashMap<UUID4, usize>,
205    request_pipeline_parent_request_id: AHashMap<UUID4, (UUID4, usize)>,
206    request_pipeline_responses: AHashMap<UUID4, Vec<Option<Vec<DataResponseEnvelope>>>>,
207    time_range_pipeline_requests: AHashMap<UUID4, TimeRangePipelineState>,
208    time_range_pipeline_parent_request_id: AHashMap<UUID4, UUID4>,
209    parent_join_request_id: AHashMap<UUID4, UUID4>,
210    pending_join_requests: AHashMap<UUID4, RequestJoin>,
211    continuous_future_requests: AHashMap<UUID4, ContinuousFutureRequestState>,
212    continuous_future_parent_request_id: AHashMap<UUID4, UUID4>,
213    continuous_future_subscriptions: AHashMap<BarType, ContinuousFutureSubscriptionState>,
214    continuous_future_roller: Option<Rc<ContinuousFutureRoller>>,
215    spread_quote_states: AHashMap<InstrumentId, SpreadQuoteState>,
216    option_chain_managers: AHashMap<OptionSeriesId, Rc<RefCell<OptionChainManager>>>,
217    option_chain_instrument_index: AHashMap<InstrumentId, OptionSeriesId>,
218    deferred_cmd_queue: DeferredCommandQueue,
219    option_chain_bootstrapper: Option<Rc<OptionChainBootstrapper>>,
220    pending_option_chain_requests: AHashMap<UUID4, PendingOptionChainRequest>,
221    option_chain_greeks_bootstraps: AHashMap<OptionSeriesId, OptionChainGreeksBootstrap>,
222    synthetic_quote_feeds: AHashMap<InstrumentId, Vec<SyntheticInstrument>>,
223    synthetic_trade_feeds: AHashMap<InstrumentId, Vec<SyntheticInstrument>>,
224    subscribed_synthetic_quotes: AHashMap<InstrumentId, usize>,
225    subscribed_synthetic_trades: AHashMap<InstrumentId, usize>,
226    buffered_deltas_map: AHashMap<InstrumentId, OrderBookDeltas>,
227    deltas_frame: Vec<OrderBookDelta>,
228    command_count: u64,
229    data_count: u64,
230    request_count: u64,
231    response_count: u64,
232    #[cfg(feature = "streaming")]
233    catalogs: CatalogMap,
234    #[cfg(feature = "defi")]
235    pub(crate) pool_updaters: AHashMap<InstrumentId, Rc<PoolUpdater>>,
236    #[cfg(feature = "defi")]
237    pub(crate) pool_snapshot_pending: AHashMap<InstrumentId, UUID4>,
238    #[cfg(feature = "defi")]
239    pub(crate) pool_event_buffers: AHashMap<InstrumentId, Vec<DefiData>>,
240}
241
242impl DataEngine {
243    /// Creates a new [`DataEngine`] instance.
244    #[must_use]
245    pub fn new(
246        clock: Rc<RefCell<dyn Clock>>,
247        cache: Rc<RefCell<Cache>>,
248        config: Option<DataEngineConfig>,
249    ) -> Self {
250        let config = config.unwrap_or_default();
251        if let Some(client_ids) = &config.external_clients {
252            let mut cache = cache.borrow_mut();
253            for client_id in client_ids {
254                cache.add_external_data_client(*client_id);
255            }
256        }
257
258        Self {
259            clock,
260            cache,
261            config,
262            is_stopped: false,
263            msgbus_priority: 10, // High-priority for built-in component
264            subscriptions_external: SubscriptionRegistry::default(),
265            clients: IndexMap::new(),
266            default_client_id: None,
267            routing_map: IndexMap::new(),
268            book_intervals: AHashMap::new(),
269            book_snapshot_counts: IndexMap::new(),
270            book_snapshot_sources: AHashMap::new(),
271            book_deltas_counts: IndexMap::new(),
272            book_depth_counts: IndexMap::new(),
273            book_updaters: AHashMap::new(),
274            book_subscriptions: AHashMap::new(),
275            book_subscription_owners: AHashMap::new(),
276            book_snapshotters: AHashMap::new(),
277            bar_aggregators: IndexMap::new(),
278            bar_aggregator_handlers: AHashMap::new(),
279            bar_aggregator_warmups: AHashMap::new(),
280            bar_aggregators_pending_stop: AHashSet::new(),
281            subscriptions_bar_aggregation: AHashMap::new(),
282            request_bar_aggregations: AHashMap::new(),
283            inflight_requests: AHashMap::new(),
284            request_pipeline_parent_request: AHashMap::new(),
285            request_pipeline_n_components: AHashMap::new(),
286            request_pipeline_parent_request_id: AHashMap::new(),
287            request_pipeline_responses: AHashMap::new(),
288            time_range_pipeline_requests: AHashMap::new(),
289            time_range_pipeline_parent_request_id: AHashMap::new(),
290            parent_join_request_id: AHashMap::new(),
291            pending_join_requests: AHashMap::new(),
292            continuous_future_requests: AHashMap::new(),
293            continuous_future_parent_request_id: AHashMap::new(),
294            continuous_future_subscriptions: AHashMap::new(),
295            continuous_future_roller: None,
296            spread_quote_states: AHashMap::new(),
297            option_chain_managers: AHashMap::new(),
298            option_chain_instrument_index: AHashMap::new(),
299            deferred_cmd_queue: Rc::new(RefCell::new(VecDeque::new())),
300            option_chain_bootstrapper: None,
301            pending_option_chain_requests: AHashMap::new(),
302            option_chain_greeks_bootstraps: AHashMap::new(),
303            synthetic_quote_feeds: AHashMap::new(),
304            synthetic_trade_feeds: AHashMap::new(),
305            subscribed_synthetic_quotes: AHashMap::new(),
306            subscribed_synthetic_trades: AHashMap::new(),
307            buffered_deltas_map: AHashMap::new(),
308            deltas_frame: Vec::new(),
309            command_count: 0,
310            data_count: 0,
311            request_count: 0,
312            response_count: 0,
313            #[cfg(feature = "streaming")]
314            catalogs: CatalogMap::new(),
315            #[cfg(feature = "defi")]
316            pool_updaters: AHashMap::new(),
317            #[cfg(feature = "defi")]
318            pool_snapshot_pending: AHashMap::new(),
319            #[cfg(feature = "defi")]
320            pool_event_buffers: AHashMap::new(),
321        }
322    }
323
324    /// Returns a reference to the clock.
325    #[must_use]
326    pub fn clock(&self) -> &Rc<RefCell<dyn Clock>> {
327        &self.clock
328    }
329
330    /// Returns a reference to the cache.
331    #[must_use]
332    pub fn cache(&self) -> &Rc<RefCell<Cache>> {
333        &self.cache
334    }
335
336    /// Returns a reference to the configuration.
337    #[must_use]
338    pub const fn config(&self) -> &DataEngineConfig {
339        &self.config
340    }
341
342    #[cfg(feature = "defi")]
343    #[must_use]
344    pub(crate) const fn msgbus_priority(&self) -> u32 {
345        self.msgbus_priority
346    }
347
348    /// Registers all message bus handlers for the data engine.
349    ///
350    /// Sets external market-data ingress eligibility from the cache's external data-client registrations.
351    pub fn register_msgbus_handlers(engine: &Rc<RefCell<Self>>) {
352        let weak = WeakCell::from(Rc::downgrade(engine));
353        engine.borrow_mut().continuous_future_roller =
354            Some(Rc::new(ContinuousFutureRoller::new(engine)));
355        engine.borrow_mut().option_chain_bootstrapper =
356            Some(Rc::new(OptionChainBootstrapper::new(engine)));
357
358        let weak1 = weak.clone();
359        msgbus::register_data_command_endpoint(
360            MessagingSwitchboard::data_engine_execute(),
361            TypedIntoHandler::from(move |cmd: DataCommand| {
362                if let Some(rc) = weak1.upgrade() {
363                    rc.borrow_mut().execute(cmd);
364                }
365            }),
366        );
367
368        msgbus::register_data_command_endpoint(
369            MessagingSwitchboard::data_engine_queue_execute(),
370            TypedIntoHandler::from(move |cmd: DataCommand| {
371                get_data_cmd_sender().execute(cmd);
372            }),
373        );
374
375        // Register process handler (polymorphic - uses Any)
376        let weak2 = weak.clone();
377        msgbus::register_any(
378            MessagingSwitchboard::data_engine_process(),
379            ShareableMessageHandler::from_any(move |data: &dyn Any| {
380                if let Some(rc) = weak2.upgrade() {
381                    rc.borrow_mut().process(data);
382                }
383            }),
384        );
385
386        // Register process_data handler (typed - takes ownership)
387        let weak3 = weak.clone();
388        msgbus::register_data_endpoint(
389            MessagingSwitchboard::data_engine_process_data(),
390            TypedIntoHandler::from(move |data: Data| {
391                if let Some(rc) = weak3.upgrade() {
392                    rc.borrow_mut().process_data(data);
393                }
394            }),
395        );
396
397        let external_data_ingress = engine.borrow().cache.borrow().has_external_data_clients();
398        get_message_bus()
399            .borrow_mut()
400            .set_external_data_ingress(external_data_ingress);
401
402        // Register process_defi_data handler (typed - takes ownership)
403        #[cfg(feature = "defi")]
404        {
405            let weak4 = weak.clone();
406            msgbus::register_defi_data_endpoint(
407                MessagingSwitchboard::data_engine_process_defi_data(),
408                TypedIntoHandler::from(move |data: DefiData| {
409                    if let Some(rc) = weak4.upgrade() {
410                        rc.borrow_mut().process_defi_data(data);
411                    }
412                }),
413            );
414        }
415
416        let weak5 = weak;
417        msgbus::register_data_response_endpoint(
418            MessagingSwitchboard::data_engine_response(),
419            TypedIntoHandler::from(move |resp: DataResponse| {
420                if let Some(rc) = weak5.upgrade() {
421                    rc.borrow_mut().response(resp);
422                }
423            }),
424        );
425    }
426
427    /// Returns the total count of data commands received by the engine.
428    #[must_use]
429    pub const fn command_count(&self) -> u64 {
430        self.command_count
431    }
432
433    /// Returns the total count of data stream objects received by the engine.
434    #[must_use]
435    pub const fn data_count(&self) -> u64 {
436        self.data_count
437    }
438
439    #[cfg(feature = "defi")]
440    pub(crate) const fn increment_data_count(&mut self) {
441        self.data_count += 1;
442    }
443
444    /// Returns the total count of data requests received by the engine.
445    #[must_use]
446    pub const fn request_count(&self) -> u64 {
447        self.request_count
448    }
449
450    /// Returns the total count of data responses received by the engine.
451    #[must_use]
452    pub const fn response_count(&self) -> u64 {
453        self.response_count
454    }
455
456    /// Returns the number of request pipelines awaiting leg responses.
457    #[must_use]
458    pub fn request_pipeline_count(&self) -> usize {
459        self.request_pipeline_parent_request.len()
460    }
461
462    /// Returns the number of time-range pipelines awaiting child responses.
463    #[must_use]
464    pub fn time_range_pipeline_count(&self) -> usize {
465        self.time_range_pipeline_requests.len()
466    }
467
468    /// Returns the number of `RequestJoin` originals awaiting finalization.
469    #[must_use]
470    pub fn pending_join_request_count(&self) -> usize {
471        self.pending_join_requests.len()
472    }
473
474    /// Starts all registered data clients and re-arms bar aggregator timers.
475    pub fn start(&mut self) {
476        self.is_stopped = false;
477
478        for client in self.get_clients_mut() {
479            client.is_stopped = false;
480            if client.start().is_err() {
481                log::error!("Failed to start data client {}", client.client_id());
482            }
483        }
484
485        for ((_, request_id), aggregator) in &self.bar_aggregators {
486            // Request-scoped or historical aggregators run on private clocks;
487            // re-arming them here would perturb an in-flight request's timer state
488            let is_subscription = request_id.is_none() && !aggregator.borrow().is_historical();
489            if is_subscription && aggregator.borrow().bar_type().spec().is_time_aggregated() {
490                aggregator
491                    .borrow_mut()
492                    .start_timer(Some(Rc::clone(aggregator)));
493            }
494        }
495
496        for state in self.spread_quote_states.values() {
497            state
498                .aggregator
499                .borrow_mut()
500                .start_timer(Some(Rc::clone(&state.aggregator)));
501        }
502    }
503
504    /// Stops all registered data clients and bar aggregator timers.
505    ///
506    /// Blocks new physical subscriptions until [`Self::start`] or [`Self::reset`].
507    /// Data processing, subscription ownership, and unsubscribes remain enabled.
508    /// Blocked subscriptions are not automatically replayed on start or connect.
509    pub fn stop(&mut self) {
510        self.is_stopped = true;
511
512        for client in self.get_clients_mut() {
513            client.is_stopped = true;
514            if client.stop().is_err() {
515                log::error!("Failed to stop data client {}", client.client_id());
516            }
517        }
518
519        for aggregator in self.bar_aggregators.values() {
520            aggregator.borrow_mut().stop();
521        }
522
523        for state in self.spread_quote_states.values() {
524            state.aggregator.borrow_mut().stop_timer();
525        }
526    }
527
528    /// Resets all registered data clients and clears engine state.
529    pub fn reset(&mut self) {
530        self.is_stopped = false;
531
532        for client in self.get_clients_mut() {
533            client.is_stopped = false;
534            match client.reset() {
535                Ok(()) => client.clear_subscription_state(),
536                Err(_) => log::error!("Failed to reset data client {}", client.client_id()),
537            }
538        }
539
540        self.clear_bar_aggregators();
541        self.clear_request_state();
542
543        for state in self.continuous_future_subscriptions.values_mut() {
544            if let Some(name) = state.timer_name.take() {
545                self.clock.borrow_mut().cancel_timer(&name);
546            }
547        }
548        self.continuous_future_subscriptions.clear();
549
550        let spread_ids: Vec<InstrumentId> = self.spread_quote_states.keys().copied().collect();
551        for spread_id in spread_ids {
552            self.stop_spread_quote_aggregation(spread_id);
553        }
554
555        self.clear_option_chain_managers();
556
557        self.clear_book_state();
558
559        self.synthetic_quote_feeds.clear();
560        self.synthetic_trade_feeds.clear();
561        self.subscribed_synthetic_quotes.clear();
562        self.subscribed_synthetic_trades.clear();
563        self.subscriptions_external.clear();
564
565        self.deferred_cmd_queue.borrow_mut().clear();
566
567        #[cfg(feature = "defi")]
568        self.clear_pool_updaters();
569
570        self.clock.borrow_mut().cancel_timers();
571
572        self.command_count = 0;
573        self.data_count = 0;
574        self.request_count = 0;
575        self.response_count = 0;
576    }
577
578    fn clear_request_state(&mut self) {
579        self.request_bar_aggregations.clear();
580        self.inflight_requests.clear();
581        self.request_pipeline_parent_request.clear();
582        self.request_pipeline_n_components.clear();
583        self.request_pipeline_parent_request_id.clear();
584        self.request_pipeline_responses.clear();
585        self.time_range_pipeline_requests.clear();
586        self.time_range_pipeline_parent_request_id.clear();
587        self.parent_join_request_id.clear();
588        self.pending_join_requests.clear();
589        self.continuous_future_requests.clear();
590        self.continuous_future_parent_request_id.clear();
591    }
592
593    /// Disposes the engine, stopping all clients and canceling any timers.
594    pub fn dispose(&mut self) {
595        for client in self.get_clients_mut() {
596            if client.dispose().is_err() {
597                log::error!("Failed to dispose data client {}", client.client_id());
598            }
599        }
600
601        self.clear_option_chain_greeks_bootstraps();
602
603        // Continuous-future source handlers live outside bar_aggregator_handlers,
604        // so release them before dropping the aggregators
605        let mut cf_sources = Vec::new();
606
607        for state in self.continuous_future_subscriptions.values_mut() {
608            if let Some(name) = state.timer_name.take() {
609                self.clock.borrow_mut().cancel_timer(&name);
610            }
611
612            if let Some(subscription) = state.active_source_subscription.take() {
613                cf_sources.push((state.target_bar_type, subscription));
614            }
615        }
616
617        for (target_bar_type, subscription) in cf_sources {
618            self.unsubscribe_continuous_future_source(target_bar_type, subscription);
619        }
620        self.continuous_future_subscriptions.clear();
621
622        // Unsubscribe aggregator msgbus handlers so the typed routers don't keep
623        // entries pointing at dropped aggregators
624        let keys: Vec<BarAggregatorKey> = self.bar_aggregators.keys().copied().collect();
625        for (bar_type, request_id) in keys {
626            if let Err(e) = self.stop_bar_aggregator(bar_type, request_id) {
627                log::error!("Error stopping bar aggregator during dispose for {bar_type}: {e}");
628            }
629        }
630
631        self.subscriptions_external.clear();
632
633        self.clock.borrow_mut().cancel_timers();
634    }
635
636    /// Connects all registered data clients concurrently.
637    ///
638    /// Successful clients remain connected if another client fails.
639    /// Logs identify failed clients without provider error text, which can contain credentials.
640    ///
641    /// # Errors
642    ///
643    /// Returns an error if any client fails to connect, listing failed client IDs without provider
644    /// error text.
645    pub async fn connect(&mut self) -> anyhow::Result<()> {
646        let futures: Vec<_> = self
647            .get_clients_mut()
648            .into_iter()
649            .map(|client| async move {
650                let client_id = client.client_id();
651                (client_id, client.connect().await)
652            })
653            .collect();
654
655        let results = join_all(futures).await;
656        let mut failed_clients = Vec::new();
657
658        for (client_id, result) in results {
659            if result.is_err() {
660                log::error!("Failed to connect data client {client_id}: connection failed");
661                failed_clients.push(client_id.to_string());
662            }
663        }
664
665        if failed_clients.is_empty() {
666            Ok(())
667        } else {
668            anyhow::bail!(
669                "Failed to connect data clients: {}",
670                failed_clients.join(", ")
671            )
672        }
673    }
674
675    /// Disconnects all registered data clients concurrently.
676    ///
677    /// # Errors
678    ///
679    /// Returns an error if any client fails to disconnect.
680    pub async fn disconnect(&mut self) -> anyhow::Result<()> {
681        let futures: Vec<_> = self
682            .get_clients_mut()
683            .into_iter()
684            .map(|client| async move {
685                let client_id = client.client_id();
686                (client_id, client.disconnect().await)
687            })
688            .collect();
689
690        let results = join_all(futures).await;
691
692        // A closed session cannot answer, so discard its pending bootstraps
693        #[cfg(feature = "defi")]
694        self.abandon_pool_snapshots();
695
696        let failed_clients: Vec<_> = results
697            .into_iter()
698            .filter_map(|(client_id, result)| result.err().map(|_| client_id.to_string()))
699            .collect();
700
701        if failed_clients.is_empty() {
702            Ok(())
703        } else {
704            anyhow::bail!(
705                "Failed to disconnect data clients: {}",
706                failed_clients.join(", ")
707            )
708        }
709    }
710
711    /// Returns `true` if all registered data clients are currently connected.
712    #[must_use]
713    pub fn check_connected(&self) -> bool {
714        self.get_clients()
715            .iter()
716            .all(|client| client.is_connected())
717    }
718
719    /// Returns `true` if all registered data clients are currently disconnected.
720    #[must_use]
721    pub fn check_disconnected(&self) -> bool {
722        self.get_clients()
723            .iter()
724            .all(|client| !client.is_connected())
725    }
726
727    /// Returns all custom data types currently subscribed across all clients.
728    #[must_use]
729    pub fn subscribed_custom_data(&self) -> Vec<DataType> {
730        self.collect_subscriptions(|client| &client.subscriptions_custom)
731    }
732
733    /// Returns all instrument IDs currently subscribed across all clients.
734    #[must_use]
735    pub fn subscribed_instruments(&self) -> Vec<InstrumentId> {
736        self.collect_subscriptions(|client| &client.subscriptions_instrument)
737    }
738
739    /// Returns all instrument IDs for which quote subscriptions exist.
740    #[must_use]
741    pub fn subscribed_quotes(&self) -> Vec<InstrumentId> {
742        self.collect_subscriptions(|client| &client.subscriptions_quotes)
743    }
744
745    /// Returns all instrument IDs for which trade subscriptions exist.
746    #[must_use]
747    pub fn subscribed_trades(&self) -> Vec<InstrumentId> {
748        self.collect_subscriptions(|client| &client.subscriptions_trades)
749    }
750
751    /// Returns all bar types currently subscribed across all clients,
752    /// including internally aggregated subscriptions (v1 parity).
753    #[must_use]
754    pub fn subscribed_bars(&self) -> Vec<BarType> {
755        let mut subscribed = self.collect_subscriptions(|client| &client.subscriptions_bars);
756        subscribed.extend(
757            self.bar_aggregators
758                .keys()
759                .filter(|key| key.1.is_none() && !self.bar_aggregators_pending_stop.contains(*key))
760                .map(|(bar_type, _)| *bar_type),
761        );
762        subscribed
763    }
764
765    /// Returns all instrument IDs for which mark price subscriptions exist.
766    #[must_use]
767    pub fn subscribed_mark_prices(&self) -> Vec<InstrumentId> {
768        self.collect_subscriptions(|client| &client.subscriptions_mark_prices)
769    }
770
771    /// Returns all instrument IDs for which index price subscriptions exist.
772    #[must_use]
773    pub fn subscribed_index_prices(&self) -> Vec<InstrumentId> {
774        self.collect_subscriptions(|client| &client.subscriptions_index_prices)
775    }
776
777    /// Returns all instrument IDs for which funding rate subscriptions exist.
778    #[must_use]
779    pub fn subscribed_funding_rates(&self) -> Vec<InstrumentId> {
780        self.collect_subscriptions(|client| &client.subscriptions_funding_rates)
781    }
782
783    /// Returns all instrument IDs for which status subscriptions exist.
784    #[must_use]
785    pub fn subscribed_instrument_status(&self) -> Vec<InstrumentId> {
786        self.collect_subscriptions(|client| &client.subscriptions_instrument_status)
787    }
788
789    /// Returns all instrument IDs for which instrument close subscriptions exist.
790    #[must_use]
791    pub fn subscribed_instrument_close(&self) -> Vec<InstrumentId> {
792        self.collect_subscriptions(|client| &client.subscriptions_instrument_close)
793    }
794
795    /// Executes a `DataCommand` by delegating to subscribe, unsubscribe, or request handlers.
796    ///
797    /// This is the final synchronous dispatch point for data commands. Runtime command producers
798    /// should send to `DataEngine.queue_execute`, which lets the runner sequence command execution
799    /// before this method runs. The engine also calls this method for child commands generated while
800    /// processing a parent command, where immediate in-engine ordering matters.
801    ///
802    /// Errors during execution are logged.
803    pub fn execute(&mut self, cmd: DataCommand) {
804        match &cmd {
805            DataCommand::Subscribe(_) | DataCommand::Unsubscribe(_) => self.command_count += 1,
806            DataCommand::Request(_) => self.request_count += 1,
807            #[cfg(feature = "defi")]
808            DataCommand::DefiRequest(_) => self.request_count += 1,
809            #[cfg(feature = "defi")]
810            DataCommand::DefiSubscribe(_) | DataCommand::DefiUnsubscribe(_) => {
811                self.command_count += 1;
812            }
813            _ => {}
814        }
815
816        let command_context = match &cmd {
817            DataCommand::Subscribe(c) => (
818                "Subscribe",
819                c.client_id().copied(),
820                c.venue().copied(),
821                Some(c.command_id()),
822            ),
823            DataCommand::Unsubscribe(c) => (
824                "Unsubscribe",
825                c.client_id().copied(),
826                c.venue().copied(),
827                Some(c.command_id()),
828            ),
829            DataCommand::Request(c) => (
830                "Request",
831                c.client_id().copied(),
832                c.venue().copied(),
833                Some(*c.request_id()),
834            ),
835            #[cfg(feature = "defi")]
836            DataCommand::DefiRequest(c) => (
837                "DefiRequest",
838                c.client_id().copied(),
839                c.venue().copied(),
840                Some(*c.request_id()),
841            ),
842            #[cfg(feature = "defi")]
843            DataCommand::DefiSubscribe(c) => (
844                "DefiSubscribe",
845                c.client_id().copied(),
846                c.venue().copied(),
847                Some(c.command_id()),
848            ),
849            #[cfg(feature = "defi")]
850            DataCommand::DefiUnsubscribe(c) => (
851                "DefiUnsubscribe",
852                c.client_id().copied(),
853                c.venue().copied(),
854                Some(c.command_id()),
855            ),
856            _ => ("Unknown", None, None, None),
857        };
858
859        let result = match cmd {
860            DataCommand::Subscribe(c) => self.execute_subscribe(c),
861            DataCommand::Unsubscribe(c) => self.execute_unsubscribe(&c),
862            DataCommand::Request(c) => self.execute_request(c),
863            #[cfg(feature = "defi")]
864            DataCommand::DefiRequest(c) => self.execute_defi_request(c),
865            #[cfg(feature = "defi")]
866            DataCommand::DefiSubscribe(c) => self.execute_defi_subscribe(c),
867            #[cfg(feature = "defi")]
868            DataCommand::DefiUnsubscribe(c) => self.execute_defi_unsubscribe(&c),
869            _ => {
870                log::warn!("Unhandled DataCommand variant");
871                Ok(())
872            }
873        };
874
875        if let Err(e) = result {
876            log::error!("Data command failed: {command_context:?}: {e}");
877        }
878    }
879
880    /// Handles a subscribe command, updating internal state and forwarding to the client.
881    ///
882    /// # Errors
883    ///
884    /// Returns an error if the subscription is invalid (e.g., synthetic instrument for book data),
885    /// or if the underlying client operation fails.
886    pub fn execute_subscribe(&mut self, cmd: SubscribeCommand) -> anyhow::Result<()> {
887        cmd.validate_routing(&self.cache.borrow())?;
888        if let Some(client_id) = cmd.client_id().copied()
889            && self.is_external_client(client_id)
890        {
891            return self.subscribe_external(client_id, &cmd);
892        }
893
894        self.retain_intent(&cmd)?;
895
896        let retained = cmd.clone();
897        let (groups, result) = self.prepare_subscribe_groups(cmd);
898        self.dispatch_subscribe_groups(&retained, groups);
899        result
900    }
901
902    fn retain_intent(&mut self, cmd: &SubscribeCommand) -> anyhow::Result<()> {
903        let retain_intent = match cmd {
904            SubscribeCommand::Quotes(command) => {
905                self.quote_routing(&command.instrument_ids, command.params.as_ref())?
906                    == QuoteRouting::Client
907            }
908            SubscribeCommand::Trades(command) => !command.instrument_ids[0].is_synthetic(),
909            _ => false,
910        };
911
912        if let Some(client) = self.get_command_client(cmd.client_id(), cmd.venue()) {
913            if let SubscribeCommand::Bars(command) = cmd {
914                let external: Vec<_> = command
915                    .bar_types
916                    .iter()
917                    .copied()
918                    .filter(BarType::is_externally_aggregated)
919                    .collect();
920
921                if !external.is_empty() {
922                    let intent = SubscribeCommand::Bars(command.with_bar_types(external));
923                    client.retain_intent(&intent);
924                }
925            } else if retain_intent {
926                client.retain_intent(cmd);
927            }
928        }
929
930        Ok(())
931    }
932
933    fn prepare_subscribe_groups(
934        &mut self,
935        cmd: SubscribeCommand,
936    ) -> (Vec<SubscribeCommand>, anyhow::Result<()>) {
937        let members = if matches!(
938            &cmd,
939            SubscribeCommand::Quotes(_) | SubscribeCommand::Trades(_)
940        ) {
941            vec![cmd]
942        } else {
943            split_subscribe(cmd)
944        };
945
946        let mut groups: IndexMap<Option<Value>, Vec<SubscribeCommand>> = IndexMap::new();
947        let mut result = Ok(());
948
949        for member in members {
950            let prepared = match self.prepare_subscribe_member(&member) {
951                Ok(Some(prepared)) => prepared,
952                Ok(None) => continue,
953                Err(e) => {
954                    log::error!(
955                        "Subscribe preparation failed: command={}, client={:?}, instrument_ids={:?}, bar_types={:?}: {e}",
956                        member.command_id(),
957                        member.client_id(),
958                        member.instrument_ids(),
959                        member.bar_types(),
960                    );
961
962                    if result.is_ok() {
963                        result = Err(e);
964                    }
965
966                    continue;
967                }
968            };
969
970            // Members share one parent; only catalog-inferred params can differ
971            let params = prepared.params().map(|params| {
972                Value::Object(params.iter().map(|(k, v)| (k.clone(), v.clone())).collect())
973            });
974
975            groups.entry(params).or_default().push(prepared);
976        }
977
978        let groups = groups
979            .into_values()
980            .map(|mut members| {
981                if members.len() == 1 {
982                    return members.pop().expect("nonempty group");
983                }
984
985                let first = &members[0];
986                if first.instrument_ids().is_some() {
987                    let ids: Vec<_> = members
988                        .iter()
989                        .flat_map(|member| member.instrument_ids().expect("same parent kind"))
990                        .copied()
991                        .collect();
992                    first.with_instrument_ids(ids).expect("instrument command")
993                } else if first.bar_types().is_some() {
994                    let types: Vec<_> = members
995                        .iter()
996                        .flat_map(|member| member.bar_types().expect("same parent kind"))
997                        .copied()
998                        .collect();
999                    first.with_bar_types(types).expect("bar command")
1000                } else {
1001                    first.clone()
1002                }
1003            })
1004            .collect();
1005
1006        (groups, result)
1007    }
1008
1009    fn dispatch_subscribe_groups(&mut self, cmd: &SubscribeCommand, groups: Vec<SubscribeCommand>) {
1010        for group in groups {
1011            let retained = if let Some(ids) = group.instrument_ids() {
1012                cmd.with_instrument_ids(ids).expect("instrument parent")
1013            } else if let Some(types) = group.bar_types() {
1014                cmd.with_bar_types(types).expect("bar parent")
1015            } else {
1016                cmd.clone()
1017            };
1018
1019            let retain_on_failure = !matches!(
1020                &group,
1021                SubscribeCommand::BookDeltas(_) | SubscribeCommand::BookDepth(_)
1022            );
1023
1024            if let Some(client) = self.get_command_client(group.client_id(), group.venue()) {
1025                client.execute_subscribe_with_retained(group, retained, retain_on_failure);
1026            } else {
1027                log::error!(
1028                    "Cannot handle command: no client found for client_id={:?}, venue={:?}",
1029                    group.client_id(),
1030                    group.venue()
1031                );
1032            }
1033        }
1034    }
1035
1036    fn prepare_subscribe_member(
1037        &mut self,
1038        cmd: &SubscribeCommand,
1039    ) -> anyhow::Result<Option<SubscribeCommand>> {
1040        // Update internal engine state
1041        match cmd {
1042            SubscribeCommand::BookDeltas(book_cmd) => {
1043                if !self.subscribe_book_deltas(book_cmd)? && self.client_subscription_active(cmd) {
1044                    return Ok(None);
1045                }
1046            }
1047            SubscribeCommand::BookDepth(book_cmd) => {
1048                if !self.subscribe_book_depth(book_cmd)? && self.client_subscription_active(cmd) {
1049                    return Ok(None);
1050                }
1051            }
1052            SubscribeCommand::BookSnapshots(cmd) => {
1053                // Handles client forwarding internally (forwards as BookDeltas)
1054                self.subscribe_book_snapshots(cmd)?;
1055                return Ok(None);
1056            }
1057            SubscribeCommand::Bars(cmd) if has_continuous_future_params(cmd.params.as_ref()) => {
1058                self.subscribe_continuous_future_bars(cmd)?;
1059                return Ok(None);
1060            }
1061            SubscribeCommand::Bars(cmd) => {
1062                self.subscribe_bars(cmd)?;
1063                if cmd
1064                    .single_bar_type()
1065                    .expect("member preparation receives one bar type")
1066                    .is_internally_aggregated()
1067                {
1068                    return Ok(None);
1069                }
1070            }
1071            SubscribeCommand::OptionChain(cmd) => {
1072                self.subscribe_option_chain(cmd);
1073                return Ok(None);
1074            }
1075            SubscribeCommand::Quotes(cmd) if self.subscribe_quotes(cmd)? => {
1076                return Ok(None);
1077            }
1078            SubscribeCommand::Trades(cmd) if self.subscribe_trades(cmd) => {
1079                return Ok(None);
1080            }
1081            SubscribeCommand::Instrument(cmd)
1082                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1083            {
1084                anyhow::bail!("Cannot subscribe for synthetic instrument `Instrument` data");
1085            }
1086            SubscribeCommand::InstrumentStatus(cmd)
1087                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1088            {
1089                anyhow::bail!("Cannot subscribe for synthetic instrument `InstrumentStatus` data");
1090            }
1091            SubscribeCommand::InstrumentClose(cmd)
1092                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1093            {
1094                anyhow::bail!("Cannot subscribe for synthetic instrument `InstrumentClose` data");
1095            }
1096            SubscribeCommand::OptionGreeks(cmd)
1097                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1098            {
1099                anyhow::bail!("Cannot subscribe for synthetic instrument `OptionGreeks` data");
1100            }
1101            _ => {} // Do nothing else
1102        }
1103
1104        let cmd = cmd.clone();
1105        #[cfg(feature = "streaming")]
1106        let cmd = self.subscribe_command_with_prefilled_start_ns(cmd)?;
1107
1108        Ok(Some(cmd))
1109    }
1110
1111    fn client_subscription_active(&mut self, cmd: &SubscribeCommand) -> bool {
1112        self.get_command_client(cmd.client_id(), cmd.venue())
1113            .is_some_and(|client| client.has_active_subscription(cmd))
1114    }
1115
1116    /// Handles an unsubscribe command, updating internal state and forwarding to the client.
1117    ///
1118    /// # Errors
1119    ///
1120    /// Returns an error if the underlying client operation fails.
1121    pub fn execute_unsubscribe(&mut self, cmd: &UnsubscribeCommand) -> anyhow::Result<()> {
1122        if let Some(bar_types) = cmd.bar_types() {
1123            ensure_routable_bar_types(bar_types, cmd.client_id().is_some())?;
1124        }
1125
1126        if let Some(ids) = cmd.instrument_ids() {
1127            ensure_routable_instrument_ids(ids, cmd.client_id().is_some())?;
1128        }
1129
1130        if let UnsubscribeCommand::Trades(trade) = cmd
1131            && !trade
1132                .instrument_ids
1133                .iter()
1134                .all(|id| id.is_synthetic() == trade.instrument_ids[0].is_synthetic())
1135        {
1136            anyhow::bail!("trade identifier lists cannot mix client and synthetic routes");
1137        }
1138
1139        if let Some(client_id) = cmd.client_id().copied()
1140            && self.is_external_client(client_id)
1141        {
1142            self.unsubscribe_external(client_id, cmd);
1143            return Ok(());
1144        }
1145
1146        if matches!(
1147            &cmd,
1148            UnsubscribeCommand::Instrument(_)
1149                | UnsubscribeCommand::InstrumentStatus(_)
1150                | UnsubscribeCommand::InstrumentClose(_)
1151                | UnsubscribeCommand::OptionGreeks(_)
1152                | UnsubscribeCommand::BookDeltas(_)
1153                | UnsubscribeCommand::BookDepth(_)
1154                | UnsubscribeCommand::BookSnapshots(_)
1155        ) && cmd
1156            .instrument_ids()
1157            .is_some_and(|ids| ids.len() > 1 && ids.iter().any(InstrumentId::is_synthetic))
1158        {
1159            anyhow::bail!("Cannot unsubscribe synthetic instruments for this data type");
1160        }
1161
1162        let (prepared, result) = self.prepare_unsubscribe_members(cmd);
1163
1164        for command in UnsubscribeCommand::group(prepared) {
1165            if let Some(client) = self.get_command_client(command.client_id(), command.venue()) {
1166                client.execute_unsubscribe(&command);
1167            } else {
1168                log::error!(
1169                    "Cannot handle command: no client found for client_id={:?}, venue={:?}",
1170                    command.client_id(),
1171                    command.venue()
1172                );
1173            }
1174        }
1175
1176        result
1177    }
1178
1179    fn prepare_unsubscribe_members(
1180        &mut self,
1181        cmd: &UnsubscribeCommand,
1182    ) -> (Vec<UnsubscribeCommand>, anyhow::Result<()>) {
1183        let members = if matches!(
1184            cmd,
1185            UnsubscribeCommand::Quotes(_) | UnsubscribeCommand::Trades(_)
1186        ) {
1187            vec![cmd.clone()]
1188        } else {
1189            split_unsubscribe(cmd)
1190        };
1191
1192        let mut prepared = Vec::new();
1193        let mut result = Ok(());
1194
1195        for member in members {
1196            match self.prepare_unsubscribe_member(&member) {
1197                Ok(true) => prepared.push(member),
1198                Ok(false) => {}
1199                Err(e) => {
1200                    log::error!(
1201                        "Unsubscribe preparation failed: command={}, client={:?}, instrument_ids={:?}, bar_types={:?}: {e}",
1202                        member.command_id(),
1203                        member.client_id(),
1204                        member.instrument_ids(),
1205                        member.bar_types(),
1206                    );
1207
1208                    if result.is_ok() {
1209                        result = Err(e);
1210                    }
1211                }
1212            }
1213        }
1214
1215        (prepared, result)
1216    }
1217
1218    fn prepare_unsubscribe_member(&mut self, cmd: &UnsubscribeCommand) -> anyhow::Result<bool> {
1219        if matches!(
1220            cmd,
1221            UnsubscribeCommand::BookDeltas(_)
1222                | UnsubscribeCommand::BookDepth(_)
1223                | UnsubscribeCommand::BookSnapshots(_)
1224        ) && !self.release_book_subscription(cmd)
1225        {
1226            return Ok(false);
1227        }
1228
1229        match &cmd {
1230            UnsubscribeCommand::BookDeltas(cmd) if !self.unsubscribe_book_deltas(cmd) => {
1231                return Ok(false);
1232            }
1233            UnsubscribeCommand::BookDepth(cmd) if !self.unsubscribe_book_depth(cmd) => {
1234                return Ok(false);
1235            }
1236            UnsubscribeCommand::BookSnapshots(cmd) => {
1237                // Handles client forwarding internally (forwards as BookDeltas)
1238                self.unsubscribe_book_snapshots(cmd);
1239                return Ok(false);
1240            }
1241            UnsubscribeCommand::Bars(cmd)
1242                if self.continuous_future_subscriptions.contains_key(
1243                    &cmd.single_bar_type()
1244                        .expect("member preparation receives one bar type")
1245                        .standard(),
1246                ) =>
1247            {
1248                // Don't tear down the chain while other actors remain subscribed
1249                let topic = switchboard::get_bars_topic(
1250                    cmd.single_bar_type()
1251                        .expect("member preparation receives one bar type")
1252                        .standard(),
1253                );
1254
1255                if msgbus::exact_subscriber_count_bars(topic) == 0 {
1256                    self.unsubscribe_continuous_future_bars(cmd);
1257                }
1258
1259                return Ok(false);
1260            }
1261            UnsubscribeCommand::Bars(cmd) => {
1262                self.unsubscribe_bars(cmd);
1263                if cmd
1264                    .single_bar_type()
1265                    .expect("member preparation receives one bar type")
1266                    .is_internally_aggregated()
1267                {
1268                    return Ok(false);
1269                }
1270            }
1271            UnsubscribeCommand::OptionChain(cmd) => {
1272                self.unsubscribe_option_chain(cmd);
1273                return Ok(false);
1274            }
1275            UnsubscribeCommand::Quotes(cmd) if self.unsubscribe_quotes(cmd)? => {
1276                return Ok(false);
1277            }
1278            UnsubscribeCommand::Trades(cmd) if self.unsubscribe_trades(cmd) => {
1279                return Ok(false);
1280            }
1281            UnsubscribeCommand::Instrument(cmd)
1282                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1283            {
1284                anyhow::bail!("Cannot unsubscribe from synthetic instrument `Instrument` data");
1285            }
1286            UnsubscribeCommand::InstrumentStatus(cmd)
1287                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1288            {
1289                anyhow::bail!(
1290                    "Cannot unsubscribe from synthetic instrument `InstrumentStatus` data"
1291                );
1292            }
1293            UnsubscribeCommand::InstrumentClose(cmd)
1294                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1295            {
1296                anyhow::bail!(
1297                    "Cannot unsubscribe from synthetic instrument `InstrumentClose` data"
1298                );
1299            }
1300            UnsubscribeCommand::OptionGreeks(cmd)
1301                if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1302            {
1303                anyhow::bail!("Cannot unsubscribe from synthetic instrument `OptionGreeks` data");
1304            }
1305            _ => {}
1306        }
1307
1308        Ok(true)
1309    }
1310
1311    fn subscribe_external(
1312        &mut self,
1313        client_id: ClientId,
1314        cmd: &SubscribeCommand,
1315    ) -> anyhow::Result<()> {
1316        let (dispatch, result) = match cmd {
1317            SubscribeCommand::OptionChain(command) => {
1318                self.retain_external_option_chain(client_id, command, cmd);
1319                (Some(cmd.clone()), Ok(()))
1320            }
1321            _ => self.prepare_external_subscribe(client_id, cmd),
1322        };
1323
1324        let Some(dispatch) = dispatch else {
1325            return result;
1326        };
1327
1328        if self.is_stopped {
1329            log::debug!(
1330                "Skipped external subscribe after stop: client_id={client_id}, command_id={}",
1331                dispatch.command_id()
1332            );
1333            return result;
1334        }
1335
1336        register_external_streaming_type(&dispatch);
1337        publish_external_data_command(client_id, &dispatch);
1338
1339        if self.config.debug {
1340            log::debug!("Skipping subscribe command for external client {client_id}: {cmd:?}");
1341        }
1342
1343        result
1344    }
1345
1346    fn prepare_external_subscribe(
1347        &mut self,
1348        client_id: ClientId,
1349        cmd: &SubscribeCommand,
1350    ) -> (Option<SubscribeCommand>, anyhow::Result<()>) {
1351        let mut result = Ok(());
1352        let mut acquired = false;
1353        let mut fresh_ids = Vec::new();
1354        let mut fresh_bar_types = Vec::new();
1355
1356        for member in split_subscribe(cmd.clone()) {
1357            let key = SubscriptionKey::from_subscribe(&member);
1358            let instrument_id = member.instrument_ids().and_then(|ids| ids.first()).copied();
1359            let bar_type = member.bar_types().and_then(|types| types.first()).copied();
1360
1361            match self.retain_external_subscription(client_id, key.clone(), member) {
1362                Ok(false) => continue,
1363                Ok(true) => {}
1364                Err(e) => {
1365                    log::error!(
1366                        "External subscribe preparation failed: command={}, client={client_id}, subscription={}: {e}",
1367                        cmd.command_id(),
1368                        key.log_context(),
1369                    );
1370
1371                    if result.is_ok() {
1372                        result = Err(e);
1373                    }
1374
1375                    continue;
1376                }
1377            }
1378
1379            acquired = true;
1380            fresh_ids.extend(instrument_id);
1381            fresh_bar_types.extend(bar_type);
1382        }
1383
1384        if !acquired {
1385            return (None, result);
1386        }
1387
1388        let dispatch = if cmd.bar_types().is_some() {
1389            cmd.with_bar_types(fresh_bar_types)
1390                .expect("bar command has types")
1391        } else {
1392            cmd.with_instrument_ids(fresh_ids)
1393                .unwrap_or_else(|| cmd.clone())
1394        };
1395
1396        (Some(dispatch), result)
1397    }
1398
1399    fn retain_external_subscription(
1400        &mut self,
1401        client_id: ClientId,
1402        key: SubscriptionKey,
1403        command: SubscribeCommand,
1404    ) -> anyhow::Result<bool> {
1405        let key = (client_id, key);
1406
1407        if self
1408            .subscriptions_external
1409            .get_mut(&key)
1410            .is_some_and(|active| active.acquisitions.contains(&command.command_id()))
1411        {
1412            return Ok(false);
1413        }
1414
1415        // External owners share the first acquisition's source parameters
1416        let params = self.subscriptions_external.get_mut(&key).map_or_else(
1417            || command.params().cloned(),
1418            |active| active.command.params().cloned(),
1419        );
1420
1421        match &command {
1422            SubscribeCommand::BookDeltas(command) => {
1423                let mut command = command.clone();
1424                command.params = params;
1425                self.subscribe_book_deltas(&command)?;
1426            }
1427            SubscribeCommand::BookDepth(command) => {
1428                let mut command = command.clone();
1429                command.params = params;
1430                self.subscribe_book_depth(&command)?;
1431            }
1432            _ => {}
1433        }
1434
1435        Ok(self
1436            .subscriptions_external
1437            .retain(key, command.command_id(), command))
1438    }
1439
1440    fn unsubscribe_external(&mut self, client_id: ClientId, cmd: &UnsubscribeCommand) {
1441        let mut releases = IndexMap::new();
1442
1443        for member in split_unsubscribe(cmd) {
1444            let key = (client_id, SubscriptionKey::from_unsubscribe(&member));
1445            if matches!(
1446                member,
1447                UnsubscribeCommand::BookDeltas(_) | UnsubscribeCommand::BookDepth(_)
1448            ) && self.subscriptions_external.contains(&key)
1449            {
1450                if !self.release_book_subscription(&member) {
1451                    continue;
1452                }
1453
1454                match &member {
1455                    UnsubscribeCommand::BookDeltas(command) => {
1456                        self.unsubscribe_book_deltas(command);
1457                    }
1458                    UnsubscribeCommand::BookDepth(command) => {
1459                        self.unsubscribe_book_depth(command);
1460                    }
1461                    _ => unreachable!("only delta and depth subscriptions retain external books"),
1462                }
1463            }
1464
1465            let (acquisition_id, command) = match self.subscriptions_external.release(&key) {
1466                SubscriptionRelease::Retained => continue,
1467                SubscriptionRelease::Final(subscribe) => (
1468                    subscribe.command_id(),
1469                    subscribe.into_unsubscribe(
1470                        cmd.command_id(),
1471                        cmd.ts_init(),
1472                        cmd.correlation_id(),
1473                    ),
1474                ),
1475                SubscriptionRelease::Untracked => (cmd.command_id(), member),
1476            };
1477
1478            self.subscriptions_external.remove(&key);
1479            releases
1480                .entry(acquisition_id)
1481                .or_insert_with(Vec::new)
1482                .push(command);
1483        }
1484
1485        for command in releases.into_values().flat_map(UnsubscribeCommand::group) {
1486            publish_external_data_command(client_id, &command);
1487        }
1488
1489        if self.config.debug {
1490            log::debug!("Skipping unsubscribe command for external client {client_id}: {cmd:?}");
1491        }
1492    }
1493
1494    fn subscribe_quotes(&mut self, cmd: &SubscribeQuotes) -> anyhow::Result<bool> {
1495        match self.quote_routing(&cmd.instrument_ids, cmd.params.as_ref())? {
1496            QuoteRouting::Client => Ok(false),
1497            QuoteRouting::Synthetic => {
1498                for id in &cmd.instrument_ids {
1499                    self.subscribe_synthetic_quotes(*id);
1500                }
1501
1502                Ok(true)
1503            }
1504            QuoteRouting::Spread => {
1505                for id in &cmd.instrument_ids {
1506                    let member = cmd.with_instrument_ids(*id);
1507                    self.subscribe_spread_quotes(&member);
1508                }
1509
1510                Ok(true)
1511            }
1512        }
1513    }
1514
1515    fn unsubscribe_quotes(&mut self, cmd: &UnsubscribeQuotes) -> anyhow::Result<bool> {
1516        match self.quote_routing(&cmd.instrument_ids, cmd.params.as_ref())? {
1517            QuoteRouting::Client => Ok(false),
1518            QuoteRouting::Synthetic => {
1519                for id in &cmd.instrument_ids {
1520                    self.unsubscribe_synthetic_quotes(*id);
1521                }
1522
1523                Ok(true)
1524            }
1525            QuoteRouting::Spread => {
1526                for id in &cmd.instrument_ids {
1527                    let member = cmd.with_instrument_ids(*id);
1528                    self.unsubscribe_spread_quotes(&member);
1529                }
1530
1531                Ok(true)
1532            }
1533        }
1534    }
1535
1536    fn subscribe_trades(&mut self, cmd: &SubscribeTrades) -> bool {
1537        if !cmd.instrument_ids[0].is_synthetic() {
1538            return false;
1539        }
1540
1541        for id in &cmd.instrument_ids {
1542            self.subscribe_synthetic_trades(*id);
1543        }
1544
1545        true
1546    }
1547
1548    fn unsubscribe_trades(&mut self, cmd: &UnsubscribeTrades) -> bool {
1549        if !cmd.instrument_ids[0].is_synthetic() {
1550            return false;
1551        }
1552
1553        for id in &cmd.instrument_ids {
1554            self.unsubscribe_synthetic_trades(*id);
1555        }
1556
1557        true
1558    }
1559
1560    fn quote_routing(
1561        &self,
1562        ids: &[InstrumentId],
1563        params: Option<&Params>,
1564    ) -> anyhow::Result<QuoteRouting> {
1565        quote_routing(ids, None, params, &self.cache.borrow())
1566    }
1567
1568    /// Sends a [`RequestCommand`] to a suitable data client implementation.
1569    ///
1570    /// Submission errors also send the requester a terminal failure envelope.
1571    ///
1572    /// # Errors
1573    ///
1574    /// Returns an error if validation fails, no client is found for the given client ID or venue,
1575    /// or the client rejects direct submission.
1576    pub fn execute_request(&mut self, req: RequestCommand) -> anyhow::Result<()> {
1577        let request_id = *req.request_id();
1578        let result = self.execute_request_inner(req);
1579        if let Err(e) = &result {
1580            self.process_response_envelope(DataResponseEnvelope::terminal(
1581                request_id,
1582                Some(e.to_string()),
1583            ));
1584        }
1585
1586        result
1587    }
1588
1589    fn execute_request_inner(&mut self, req: RequestCommand) -> anyhow::Result<()> {
1590        Self::validate_request(&req)?;
1591        #[cfg(feature = "streaming")]
1592        self.validate_catalog_update(&req)?;
1593
1594        // Skip requests for external clients
1595        if let Some(cid) = req.client_id()
1596            && self.is_external_client(*cid)
1597        {
1598            if self.config.debug {
1599                log::debug!("Skipping data request for external client {cid}: {req:?}");
1600            }
1601            return Ok(());
1602        }
1603
1604        if let RequestCommand::Join(join) = req {
1605            return self.handle_request_join(join);
1606        }
1607
1608        if has_continuous_future_params(request_params(&req)) {
1609            return self.execute_continuous_future_request(req);
1610        }
1611
1612        let request_id = *req.request_id();
1613        self.prepare_request_bar_aggregators(&req)?;
1614
1615        if has_time_range_pipeline_params(request_params(&req))
1616            && is_time_range_pipeline_variant(&req)
1617        {
1618            let result = self.execute_time_range_pipeline_request(req);
1619            if result.is_err() {
1620                self.cleanup_request_bar_aggregators(&request_id);
1621            }
1622            return result;
1623        }
1624
1625        #[cfg(feature = "streaming")]
1626        if self.catalogs_registered() && streaming::is_date_range_variant(&req) {
1627            let result = self.dispatch_date_range_request(req);
1628            if result.is_err() {
1629                self.cleanup_request_bar_aggregators(&request_id);
1630            }
1631            return result;
1632        }
1633
1634        let result = self.dispatch_request_to_client(req);
1635
1636        if result.is_err() {
1637            self.cleanup_request_bar_aggregators(&request_id);
1638        }
1639
1640        result.map(|_| ())
1641    }
1642
1643    fn validate_request(req: &RequestCommand) -> anyhow::Result<()> {
1644        let (instrument_ids, requires_single, limit) = match req {
1645            RequestCommand::Instrument(cmd) => (&cmd.instrument_ids, false, None),
1646            RequestCommand::BookSnapshot(cmd) => (&cmd.instrument_ids, true, None),
1647            RequestCommand::BookDeltas(cmd) => (&cmd.instrument_ids, true, None),
1648            RequestCommand::BookDepth(cmd) => (&cmd.instrument_ids, true, None),
1649            RequestCommand::Quotes(cmd) => (&cmd.instrument_ids, false, cmd.limit),
1650            RequestCommand::Trades(cmd) => (&cmd.instrument_ids, false, cmd.limit),
1651            RequestCommand::FundingRates(cmd) => (&cmd.instrument_ids, true, None),
1652            RequestCommand::Bars(cmd) => {
1653                ensure_routable_bar_types(&cmd.bar_types, cmd.client_id.is_some())?;
1654                cmd.single_bar_type()?;
1655                return Ok(());
1656            }
1657            _ => return Ok(()),
1658        };
1659
1660        ensure_routable_instrument_ids(instrument_ids, req.client_id().is_some())?;
1661        anyhow::ensure!(
1662            !requires_single || instrument_ids.len() == 1,
1663            "request requires one instrument ID for this data type, received {}",
1664            instrument_ids.len(),
1665        );
1666        anyhow::ensure!(
1667            instrument_ids.len() == 1 || limit.is_none(),
1668            "A record limit is unsupported for multi-instrument requests",
1669        );
1670        Ok(())
1671    }
1672
1673    pub(super) fn dispatch_request_to_client(
1674        &mut self,
1675        req: RequestCommand,
1676    ) -> anyhow::Result<ClientId> {
1677        self.inflight_requests
1678            .insert(*req.request_id(), req.clone());
1679        let client_id = req.client_id().copied();
1680        let venue = req.venue().copied();
1681        let Some(client) = self.get_client(client_id.as_ref(), venue.as_ref()) else {
1682            anyhow::bail!("Cannot handle request: no client found for {client_id:?} {venue:?}");
1683        };
1684        let resolved_client_id = client.client_id();
1685
1686        #[rustfmt::skip]
1687        let result = match req {
1688            RequestCommand::Data(req) => client.request_data(req),
1689            RequestCommand::Instrument(req) => client.request_instrument(req),
1690            RequestCommand::Instruments(req) => client.request_instruments(req),
1691            RequestCommand::BookSnapshot(req) => client.request_book_snapshot(req),
1692            RequestCommand::BookDeltas(req) => client.request_book_deltas(req),
1693            RequestCommand::BookDepth(req) => client.request_book_depth(req),
1694            RequestCommand::Quotes(req) => client.request_quotes(req),
1695            RequestCommand::Trades(req) => client.request_trades(req),
1696            RequestCommand::FundingRates(req) => client.request_funding_rates(req),
1697            RequestCommand::OptionChainReferencePrice(req) => client.request_option_chain_reference_price(req),
1698            RequestCommand::Bars(req) => client.request_bars(req),
1699            RequestCommand::Join(_) => anyhow::bail!("RequestJoin must be handled by handle_request_join"),
1700        };
1701
1702        result.map_err(|e| {
1703            let context = format!("Data client {resolved_client_id} request failed");
1704
1705            let context = if let Some(rejection) = e.downcast_ref::<IdentifierCountError>() {
1706                format!("{context}: {rejection}")
1707            } else {
1708                context
1709            };
1710
1711            e.context(context)
1712        })?;
1713
1714        Ok(resolved_client_id)
1715    }
1716
1717    /// Processes a dynamically-typed data message.
1718    ///
1719    /// Supports instruments, response envelopes, request failures, funding rates, option greeks,
1720    /// instrument status, and custom data; unrecognized types are logged as errors.
1721    pub fn process(&mut self, data: &dyn Any) {
1722        if let Some(response) = data.downcast_ref::<DataResponseEnvelope>() {
1723            self.response_envelope(response.clone());
1724            return;
1725        }
1726
1727        if let Some(failure) = data.downcast_ref::<DataRequestFailed>() {
1728            self.request_failed(failure.clone());
1729            return;
1730        }
1731
1732        self.data_count += 1;
1733
1734        if let Some(instrument) = data.downcast_ref::<InstrumentAny>() {
1735            self.handle_instrument(instrument);
1736        } else if let Some(funding_rate) = data.downcast_ref::<FundingRateUpdate>() {
1737            self.handle_funding_rate(*funding_rate);
1738        } else if let Some(option_greeks) = data.downcast_ref::<OptionGreeks>() {
1739            self.cache.borrow_mut().add_option_greeks(*option_greeks);
1740            self.feed_option_greeks_to_pre_bootstrap_chain(option_greeks);
1741            let topic = switchboard::get_option_greeks_topic(option_greeks.instrument_id);
1742            msgbus::publish_option_greeks(topic, option_greeks);
1743            self.drain_deferred_commands();
1744        } else if let Some(status) = data.downcast_ref::<InstrumentStatus>() {
1745            self.handle_instrument_status(*status);
1746        } else if let Some(custom) = data.downcast_ref::<CustomData>() {
1747            self.handle_custom_data(custom);
1748        } else {
1749            #[cfg(feature = "defi")]
1750            if let Some(response) = data.downcast_ref::<PoolSnapshotResponse>() {
1751                self.handle_pool_snapshot_response(response);
1752                return;
1753            }
1754
1755            log::error!("Cannot process data {data:?}, type is unrecognized");
1756        }
1757    }
1758
1759    /// Processes a `Data` enum instance, dispatching to live handlers.
1760    #[allow(
1761        clippy::needless_pass_by_value,
1762        reason = "callers hand over ownership; the payload is only moved when a DeFi handler consumes it"
1763    )]
1764    pub fn process_data(&mut self, data: Data) {
1765        #[cfg(feature = "defi")]
1766        let data = match data {
1767            Data::Defi(defi) => {
1768                self.process_defi_data(*defi);
1769                return;
1770            }
1771            data => data,
1772        };
1773
1774        self.process_data_ref(DataRef::from(&data));
1775    }
1776
1777    /// Processes a borrowed `Data` enum view, dispatching to live handlers.
1778    ///
1779    /// DeFi payloads are cloned because the DeFi handler may buffer them while a pool snapshot
1780    /// is pending; no other variant clones its payload.
1781    pub fn process_data_ref(&mut self, data: DataRef<'_>) {
1782        #[cfg(feature = "defi")]
1783        let data = match data {
1784            DataRef::Defi(defi) => {
1785                self.process_defi_data(defi.clone());
1786                return;
1787            }
1788            data => data,
1789        };
1790
1791        self.data_count += 1;
1792
1793        match data {
1794            DataRef::Instrument(instrument) => self.handle_instrument(instrument),
1795            DataRef::BookDelta(delta) => self.handle_delta(*delta),
1796            DataRef::BookDeltas(deltas) => self.handle_deltas(deltas),
1797            DataRef::BookDepth(depth) => self.handle_depth(depth),
1798            DataRef::Quote(quote) => {
1799                self.handle_quote(*quote);
1800                self.drain_deferred_commands();
1801            }
1802            DataRef::Trade(trade) => self.handle_trade(*trade),
1803            DataRef::Bar(bar) => self.handle_bar(*bar),
1804            DataRef::MarkPrice(mark_price) => {
1805                self.handle_mark_price(*mark_price);
1806                self.drain_deferred_commands();
1807            }
1808            DataRef::IndexPrice(index_price) => {
1809                self.handle_index_price(*index_price);
1810                self.drain_deferred_commands();
1811            }
1812            DataRef::FundingRate(funding_rate) => {
1813                self.handle_funding_rate(*funding_rate);
1814                self.drain_deferred_commands();
1815            }
1816            DataRef::OptionGreeks(greeks) => {
1817                self.cache.borrow_mut().add_option_greeks(*greeks);
1818                self.feed_option_greeks_to_pre_bootstrap_chain(greeks);
1819                let topic = switchboard::get_option_greeks_topic(greeks.instrument_id);
1820                msgbus::publish_option_greeks(topic, greeks);
1821                self.drain_deferred_commands();
1822            }
1823            DataRef::InstrumentStatus(status) => {
1824                self.handle_instrument_status(*status);
1825                self.drain_deferred_commands();
1826            }
1827            DataRef::InstrumentClose(close) => self.handle_instrument_close(*close),
1828            DataRef::Custom(custom) => self.handle_custom_data(custom),
1829            #[cfg(feature = "defi")]
1830            DataRef::Defi(_) => unreachable!("handled before market data dispatch"),
1831            #[cfg(not(feature = "defi"))]
1832            #[allow(
1833                unreachable_patterns,
1834                reason = "DeFi variants can exist without this crate's defi feature"
1835            )]
1836            other => log_defi_data_dropped(other),
1837        }
1838    }
1839
1840    /// Processes a `Data` instance through the pipeline bus path.
1841    ///
1842    /// Pipeline mode publishes each item on the `data.pipeline.` topic family and gates cache
1843    /// writes on `disable_historical_cache`. None of the live-only side effects (synthetic
1844    /// republish, option-chain expiry, depth-derived quotes, deferred-command drains) run in this
1845    /// path.
1846    pub fn process_pipeline(&mut self, data: Data) {
1847        #[cfg(feature = "defi")]
1848        let data = match data {
1849            Data::Defi(defi) => {
1850                self.process_defi_data(*defi);
1851                return;
1852            }
1853            data => data,
1854        };
1855
1856        self.data_count += 1;
1857
1858        match data {
1859            Data::Instrument(instrument) => self.handle_instrument(&instrument),
1860            Data::BookDelta(delta) => self.handle_delta_pipeline(delta),
1861            Data::BookDeltas(deltas) => self.handle_deltas_pipeline(&deltas),
1862            Data::BookDepth(depth) => self.handle_depth_pipeline(&depth),
1863            Data::Quote(quote) => self.handle_quote_pipeline(quote),
1864            Data::Trade(trade) => self.handle_trade_pipeline(trade),
1865            Data::Bar(bar) => self.handle_bar_pipeline(bar),
1866            Data::MarkPrice(mark_price) => self.handle_mark_price_pipeline(mark_price),
1867            Data::IndexPrice(index_price) => self.handle_index_price_pipeline(index_price),
1868            Data::FundingRate(funding_rate) => {
1869                self.handle_funding_rate_pipeline(funding_rate);
1870            }
1871            Data::OptionGreeks(greeks) => self.handle_option_greeks_pipeline(greeks),
1872            Data::InstrumentStatus(status) => self.handle_instrument_status_pipeline(status),
1873            Data::InstrumentClose(close) => self.handle_instrument_close_pipeline(close),
1874            Data::Custom(custom) => self.handle_custom_data_pipeline(&custom),
1875            #[cfg(feature = "defi")]
1876            Data::Defi(_) => unreachable!("handled before market data dispatch"),
1877            #[cfg(not(feature = "defi"))]
1878            #[allow(
1879                unreachable_patterns,
1880                reason = "DeFi variants can exist without this crate's defi feature"
1881            )]
1882            other => log_defi_data_dropped(DataRef::from(&other)),
1883        }
1884    }
1885
1886    /// Processes a payload-only reply as a completed request.
1887    ///
1888    /// Payload-only replies do not certify coverage for catalog write-back. Use
1889    /// [`Self::response_envelope`] to report explicit source completion.
1890    pub fn response(&mut self, mut resp: DataResponse) {
1891        resp.trim_to_bounds();
1892        let request = self.inflight_requests.remove(resp.correlation_id());
1893        #[cfg(feature = "streaming")]
1894        if request
1895            .as_ref()
1896            .and_then(request_params)
1897            .is_some_and(|params| params.get_bool(streaming::PARAM_UPDATE_CATALOG) == Some(true))
1898        {
1899            log::warn!(
1900                "Catalog write-back requires an explicit completed response envelope, request={}",
1901                resp.correlation_id()
1902            );
1903        }
1904
1905        #[cfg(not(feature = "streaming"))]
1906        let _ = request;
1907        let envelope = DataResponseEnvelope::new(resp, true, None);
1908        self.process_response_envelope(envelope);
1909    }
1910
1911    /// Completes failed adapter work while preserving any returned rows.
1912    pub fn request_failed(&mut self, failure: DataRequestFailed) {
1913        log::error!(
1914            "Data request {} failed: {}",
1915            failure.request_id,
1916            failure.reason
1917        );
1918
1919        let envelope = if let Some(mut response) = failure.response {
1920            response.trim_to_bounds();
1921            DataResponseEnvelope::new(response, true, None)
1922                .with_correlation_id(failure.request_id)
1923                .with_failure(failure.reason)
1924        } else {
1925            DataResponseEnvelope::terminal(failure.request_id, Some(failure.reason))
1926        };
1927
1928        self.process_response_envelope(envelope);
1929    }
1930
1931    /// Processes an adapter envelope, retaining intermediate request handlers.
1932    pub fn response_envelope(&mut self, mut envelope: DataResponseEnvelope) {
1933        if let Some(response) = envelope.response_mut() {
1934            response.trim_to_bounds();
1935        }
1936
1937        self.process_response_envelope(envelope);
1938    }
1939
1940    fn process_response_envelope(&mut self, mut envelope: DataResponseEnvelope) {
1941        let accepted = match envelope.response() {
1942            Some(DataResponse::Instrument(response)) => self.cache_instrument(&response.data),
1943            Some(DataResponse::Instruments(response)) => {
1944                let mut accepted = true;
1945                for instrument in &response.data {
1946                    accepted &= self.cache_instrument(instrument);
1947                }
1948
1949                accepted
1950            }
1951            _ => true,
1952        };
1953
1954        if !accepted {
1955            envelope = DataResponseEnvelope::terminal(
1956                *envelope.correlation_id(),
1957                Some("Instrument currency integrity check failed".into()),
1958            );
1959        }
1960
1961        let request = self.inflight_requests.remove(envelope.correlation_id());
1962        #[cfg(feature = "streaming")]
1963        let envelope = match (&request, envelope.response()) {
1964            (Some(request), Some(response))
1965                if envelope.completed() && envelope.failure_reason().is_none() =>
1966            {
1967                match self.update_catalog_from_response(response, request) {
1968                    Ok(()) => envelope,
1969                    Err(e) => envelope.with_failure(e.to_string()),
1970                }
1971            }
1972            _ => envelope,
1973        };
1974
1975        #[cfg(not(feature = "streaming"))]
1976        let _ = request;
1977
1978        let Some(envelope) = self.handle_request_pipeline_response(envelope) else {
1979            return;
1980        };
1981
1982        if self
1983            .request_pipeline_parent_request_id
1984            .contains_key(envelope.correlation_id())
1985        {
1986            self.process_response_envelope(envelope);
1987            return;
1988        }
1989
1990        if self.handle_continuous_future_child_response(&envelope)
1991            || self.handle_time_range_pipeline_child_response(&envelope)
1992        {
1993            return;
1994        }
1995
1996        let Some(resp) = envelope.response().cloned() else {
1997            self.finish_empty_response(&envelope);
1998            return;
1999        };
2000
2001        self.route_response_payload(resp, &envelope);
2002    }
2003
2004    fn finish_empty_response(&mut self, envelope: &DataResponseEnvelope) {
2005        if let Some(original_id) = self
2006            .parent_join_request_id
2007            .remove(envelope.correlation_id())
2008        {
2009            if let Some(original) = self.pending_join_requests.remove(&original_id) {
2010                for child_id in original.request_ids {
2011                    msgbus::send_response_envelope(&DataResponseEnvelope::terminal(
2012                        child_id,
2013                        envelope.failure_reason().map(str::to_owned),
2014                    ));
2015                }
2016            }
2017
2018            self.process_response_envelope(DataResponseEnvelope::terminal(
2019                original_id,
2020                envelope.failure_reason().map(str::to_owned),
2021            ));
2022            return;
2023        }
2024
2025        self.cleanup_request_bar_aggregators(envelope.correlation_id());
2026        msgbus::send_response_envelope(envelope);
2027    }
2028
2029    fn route_response_payload(&mut self, resp: DataResponse, envelope: &DataResponseEnvelope) {
2030        let resp = rebind_response_correlation(resp, *envelope.correlation_id());
2031        if log::log_enabled!(log::Level::Debug) {
2032            let correlation_id = resp.correlation_id();
2033            match resp.record_count() {
2034                Some(count) => log::debug!(
2035                    "{RECV}{RES} {} correlation_id={correlation_id} records={count}",
2036                    resp.kind(),
2037                ),
2038                None => log::debug!(
2039                    "{RECV}{RES} {} correlation_id={correlation_id}",
2040                    resp.kind(),
2041                ),
2042            }
2043        }
2044        log::trace!("{RECV}{RES} {resp:?}");
2045
2046        self.response_count += 1;
2047
2048        if self
2049            .parent_join_request_id
2050            .contains_key(resp.correlation_id())
2051        {
2052            self.finalize_request_join(resp, envelope.failure_reason().map(str::to_owned));
2053            return;
2054        }
2055
2056        if !self.handle_response_payload(&resp) {
2057            return;
2058        }
2059
2060        self.process_request_bar_aggregation_response(&resp, envelope.completed());
2061
2062        msgbus::send_response_envelope(envelope);
2063    }
2064
2065    fn handle_response_payload(&mut self, resp: &DataResponse) -> bool {
2066        let correlation_id = *resp.correlation_id();
2067
2068        match resp {
2069            DataResponse::Instrument(_) | DataResponse::Instruments(_) => {}
2070            DataResponse::Quotes(r) => {
2071                if !log_if_empty_response(
2072                    &r.data,
2073                    &format_args!("{:?}", r.instrument_ids),
2074                    &correlation_id,
2075                ) {
2076                    self.handle_quotes(&r.data);
2077                }
2078            }
2079            DataResponse::Trades(r) => {
2080                if !log_if_empty_response(
2081                    &r.data,
2082                    &format_args!("{:?}", r.instrument_ids),
2083                    &correlation_id,
2084                ) {
2085                    self.handle_trades(&r.data);
2086                }
2087            }
2088            DataResponse::FundingRates(r) => {
2089                if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2090                    self.handle_funding_rates(&r.data);
2091                }
2092            }
2093            DataResponse::Bars(r) => {
2094                if !log_if_empty_response(&r.data, &r.bar_type, &correlation_id) {
2095                    self.handle_bars(&r.data);
2096                }
2097            }
2098            DataResponse::Book(r) => self.handle_book_response(&r.data),
2099            DataResponse::BookDeltas(r) => {
2100                if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2101                    self.handle_book_deltas_response(r);
2102                }
2103            }
2104            DataResponse::BookDepth(r) => {
2105                if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2106                    self.handle_book_depth_response(r);
2107                }
2108            }
2109            DataResponse::OptionChainReferencePrice(r) => {
2110                self.process_request_bar_aggregation_response(resp, true);
2111                self.handle_option_chain_reference_price_response(&correlation_id, r);
2112                return false;
2113            }
2114            DataResponse::Data(_) => {}
2115        }
2116
2117        true
2118    }
2119
2120    /// Registers a parent request whose response will be rebuilt from `n_components` leg responses.
2121    pub fn new_request_pipeline(&mut self, parent: RequestCommand, n_components: usize) {
2122        let parent_id = *parent.request_id();
2123        self.request_pipeline_n_components
2124            .insert(parent_id, n_components);
2125        self.request_pipeline_parent_request
2126            .insert(parent_id, parent);
2127        self.request_pipeline_responses
2128            .insert(parent_id, Vec::with_capacity(n_components));
2129    }
2130
2131    /// Registers a leg `request_id` as a child of the pipeline keyed by `parent_id`.
2132    ///
2133    /// # Panics
2134    ///
2135    /// Panics if the parent pipeline is not registered.
2136    pub fn register_request_pipeline_leg(&mut self, leg_id: UUID4, parent_id: UUID4) {
2137        let slots = self
2138            .request_pipeline_responses
2139            .get_mut(&parent_id)
2140            .expect("pipeline registered before its legs");
2141        let index = slots.len();
2142        slots.push(None);
2143        self.request_pipeline_parent_request_id
2144            .insert(leg_id, (parent_id, index));
2145    }
2146
2147    fn handle_request_pipeline_response(
2148        &mut self,
2149        envelope: DataResponseEnvelope,
2150    ) -> Option<DataResponseEnvelope> {
2151        let leg_id = *envelope.correlation_id();
2152
2153        let Some(&(parent_id, index)) = self.request_pipeline_parent_request_id.get(&leg_id) else {
2154            return Some(envelope);
2155        };
2156
2157        let completed = envelope.completed();
2158        let slots = self.request_pipeline_responses.get_mut(&parent_id)?;
2159        slots[index].get_or_insert_with(Vec::new).push(envelope);
2160
2161        if !completed {
2162            return None;
2163        }
2164
2165        self.request_pipeline_parent_request_id.remove(&leg_id);
2166        let expected = *self.request_pipeline_n_components.get(&parent_id)?;
2167
2168        if slots
2169            .iter()
2170            .flatten()
2171            .filter(|slot| slot.last().is_some_and(DataResponseEnvelope::completed))
2172            .count()
2173            < expected
2174        {
2175            return None;
2176        }
2177
2178        let slots = self.request_pipeline_responses.remove(&parent_id)?;
2179        self.request_pipeline_n_components.remove(&parent_id);
2180        let parent = self.request_pipeline_parent_request.remove(&parent_id);
2181        let failures = slots
2182            .iter()
2183            .flatten()
2184            .flatten()
2185            .filter_map(|value| value.failure_reason())
2186            .collect::<Vec<_>>();
2187        let failure_reason = (!failures.is_empty()).then(|| failures.join("; "));
2188        let legs = slots
2189            .into_iter()
2190            .flatten()
2191            .flatten()
2192            .filter_map(|value| value.response().cloned())
2193            .collect::<Vec<_>>();
2194
2195        if legs.is_empty() {
2196            return Some(DataResponseEnvelope::terminal(parent_id, failure_reason));
2197        }
2198
2199        let (parent_start, parent_end) = parent_request_window(parent.as_ref());
2200        if self.emit_partitioned_book_responses(parent_id, parent.as_ref(), &legs) {
2201            return Some(DataResponseEnvelope::terminal(parent_id, failure_reason));
2202        }
2203
2204        let rebuilt = rebuild_pipeline_response(parent_id, parent.as_ref(), legs);
2205
2206        let Some(mut rebuilt) = rebuilt else {
2207            let reason = failure_reason
2208                .unwrap_or_else(|| "Request pipeline could not rebuild its response".to_owned());
2209
2210            return Some(DataResponseEnvelope::terminal(parent_id, Some(reason)));
2211        };
2212
2213        if let DataResponse::BookDeltas(r) = &mut rebuilt {
2214            self.book_deltas_snapshot_replay(r);
2215        }
2216
2217        // Client definitions can have a newer ts_init than the requested history
2218        if !matches!(parent, Some(RequestCommand::Instrument(_)))
2219            && (parent_start.is_some() || parent_end.is_some())
2220        {
2221            rebuilt.trim_to_bounds();
2222        }
2223
2224        let result = DataResponseEnvelope::new(rebuilt, true, None);
2225        Some(if let Some(reason) = failure_reason {
2226            result.with_failure(reason)
2227        } else {
2228            result
2229        })
2230    }
2231
2232    fn emit_partitioned_book_responses(
2233        &mut self,
2234        parent_id: UUID4,
2235        parent: Option<&RequestCommand>,
2236        legs: &[DataResponse],
2237    ) -> bool {
2238        if !matches!(parent, Some(RequestCommand::BookDeltas(_))) {
2239            return false;
2240        }
2241
2242        let mut ids = legs.iter().filter_map(|r| match r {
2243            DataResponse::BookDeltas(r) => Some(r.instrument_id),
2244            _ => None,
2245        });
2246
2247        let Some(first) = ids.next() else {
2248            return false;
2249        };
2250
2251        if !ids.any(|id| !same_instrument(id, first)) {
2252            return false;
2253        }
2254
2255        let (start, end) = parent_request_window(parent);
2256
2257        for response in legs {
2258            let child_id = *response.correlation_id();
2259            let mut response = rebind_response_correlation(response.clone(), parent_id);
2260            if let DataResponse::BookDeltas(r) = &mut response {
2261                r.start = start.or(r.start);
2262                r.end = end.or(r.end);
2263                self.book_deltas_snapshot_replay(r);
2264            }
2265
2266            response.trim_to_bounds();
2267            self.process_response_envelope(DataResponseEnvelope::new(
2268                response,
2269                false,
2270                Some(child_id),
2271            ));
2272        }
2273
2274        true
2275    }
2276
2277    fn handle_request_join(&mut self, req: RequestJoin) -> anyhow::Result<()> {
2278        if has_time_range_pipeline_params(req.params.as_ref()) {
2279            return self.execute_time_range_pipeline_request(RequestCommand::Join(req));
2280        }
2281
2282        let now_ns = self.clock.borrow().timestamp_ns();
2283        let now_dt = now_ns.to_datetime_utc();
2284        let zero = jiff::Timestamp::UNIX_EPOCH;
2285        let start = req.start.unwrap_or(zero).min(now_dt);
2286        let end = req.end.unwrap_or(now_dt).min(now_dt);
2287        let dated = req.with_dates(Some(start), Some(end), now_ns);
2288
2289        let original_id = req.request_id;
2290        let dated_id = dated.request_id;
2291
2292        self.pending_join_requests.insert(original_id, req);
2293        self.parent_join_request_id.insert(dated_id, original_id);
2294
2295        let leg_ids: Vec<UUID4> = dated.request_ids.clone();
2296        self.new_request_pipeline(RequestCommand::Join(dated), leg_ids.len());
2297        for leg_id in leg_ids {
2298            self.register_request_pipeline_leg(leg_id, dated_id);
2299        }
2300
2301        Ok(())
2302    }
2303
2304    fn finalize_request_join(&mut self, resp: DataResponse, failure_reason: Option<String>) {
2305        let dated_id = *resp.correlation_id();
2306        let Some(original_id) = self.parent_join_request_id.remove(&dated_id) else {
2307            log::error!("parent_join_request_id missing for dated correlation {dated_id}");
2308            return;
2309        };
2310
2311        let Some(original) = self.pending_join_requests.remove(&original_id) else {
2312            log::error!("pending_join_requests missing for original {original_id}");
2313            return;
2314        };
2315
2316        for leg_request_id in &original.request_ids {
2317            let terminal = DataResponseEnvelope::terminal(*leg_request_id, failure_reason.clone());
2318            msgbus::send_response_envelope(&terminal);
2319        }
2320
2321        // Route the final join response through the normal response path so
2322        // bounds-trim against the parent window runs and the per-variant
2323        // handlers (cache writes, request bar aggregators) fire. The pipeline
2324        // and join staging maps for this request have already been popped, so
2325        // the recursive call cannot re-enter either gate.
2326        let final_resp = rebind_response_correlation(resp, original_id);
2327        let envelope = DataResponseEnvelope::new(final_resp, true, None);
2328        self.process_response_envelope(if let Some(reason) = failure_reason {
2329            envelope.with_failure(reason)
2330        } else {
2331            envelope
2332        });
2333    }
2334}
2335
2336fn register_external_streaming_type(cmd: &SubscribeCommand) {
2337    if let Some(payload_type) = streaming_payload_type(cmd) {
2338        msgbus::get_message_bus()
2339            .borrow_mut()
2340            .add_streaming_type(payload_type);
2341    }
2342}
2343
2344fn publish_external_data_command<T>(client_id: ClientId, command: &T)
2345where
2346    T: Any,
2347{
2348    let topic = format!("commands.data.{client_id}");
2349    msgbus::publish_any(topic.into(), command);
2350}
2351
2352#[rustfmt::skip]
2353fn streaming_payload_type(cmd: &SubscribeCommand) -> Option<BusPayloadType> {
2354    match cmd {
2355        SubscribeCommand::Data(cmd) => Some(BusPayloadType::Custom(Ustr::from(
2356            cmd.data_type.type_name(),
2357        ))),
2358        SubscribeCommand::Instrument(_) | SubscribeCommand::Instruments(_) => Some(BusPayloadType::Instrument),
2359        SubscribeCommand::BookDeltas(_) | SubscribeCommand::BookSnapshots(_) => Some(BusPayloadType::OrderBookDeltas),
2360        SubscribeCommand::BookDepth(_) => Some(BusPayloadType::OrderBookDepth),
2361        SubscribeCommand::Quotes(_) => Some(BusPayloadType::QuoteTick),
2362        SubscribeCommand::Trades(_) => Some(BusPayloadType::TradeTick),
2363        SubscribeCommand::Bars(_) => Some(BusPayloadType::Bar),
2364        SubscribeCommand::MarkPrices(_) => Some(BusPayloadType::MarkPriceUpdate),
2365        SubscribeCommand::IndexPrices(_) => Some(BusPayloadType::IndexPriceUpdate),
2366        SubscribeCommand::FundingRates(_) => Some(BusPayloadType::FundingRateUpdate),
2367        SubscribeCommand::OptionGreeks(_) => Some(BusPayloadType::OptionGreeks),
2368        SubscribeCommand::InstrumentStatus(_)
2369        | SubscribeCommand::InstrumentClose(_)
2370        | SubscribeCommand::OptionChain(_) => None,
2371    }
2372}
2373
2374#[cfg(not(feature = "defi"))]
2375fn log_defi_data_dropped(data: DataRef<'_>) {
2376    log::error!("Cannot process data {data:?}, nautilus-data built without its `defi` feature");
2377}