1pub mod bar;
32pub mod book;
33pub mod config;
34
35mod commands;
36mod dispatch;
37mod futures;
38mod handlers;
39mod option_chain;
40mod registry;
41mod requests;
42mod responses;
43mod spread_quote;
44mod synthetic;
45mod time_range;
46
47#[cfg(feature = "defi")]
48pub mod pool;
49
50#[cfg(feature = "streaming")]
51mod streaming;
52
53use std::{
54 any::{Any, type_name},
55 cell::RefCell,
56 collections::VecDeque,
57 fmt::{Debug, Display},
58 mem,
59 num::NonZeroUsize,
60 rc::Rc,
61 str::FromStr,
62};
63
64use ::futures::future::join_all;
65use ahash::{AHashMap, AHashSet};
66use anyhow::Context;
67pub use bar::BarAggregatorSubscription;
68use bar::{BarAggregationSubscription, BarAggregatorKey, BarAggregatorWarmup, bar_aggregator_key};
69use book::{
70 BookDeltasKey, BookSnapshotInfos, BookSnapshotKey, BookSnapshotSource, BookSnapshotter,
71 BookSubscription, BookSubscriptionOwner, BookUpdater,
72};
73pub(crate) use commands::{DeferredCommand, DeferredCommandQueue};
74use config::DataEngineConfig;
75use dispatch::{log_error_on_cache_insert, process_engine_bar};
76use futures::{ContinuousFutureRoller, ContinuousFutureSubscriptionState, datetime_to_unix_nanos};
77use handlers::{
78 BAR_AGGREGATOR_PRIORITY, BarBarHandler, BarQuoteHandler, BarTradeHandler, SpreadQuoteHandler,
79};
80use indexmap::IndexMap;
81#[cfg(feature = "defi")]
82use nautilus_common::messages::defi::PoolSnapshotResponse;
83use nautilus_common::{
84 cache::Cache,
85 clock::Clock,
86 logging::{RECV, RES},
87 messages::data::{
88 BarsResponse, BookDeltasResponse, BookDepthResponse, CustomDataResponse, DataCommand,
89 DataRequestFailed, DataResponse, DataResponseEnvelope, IdentifierCountError,
90 OptionChainReferencePriceResponse, PARAMS_IS_PARENT, QuoteRouting, RequestBars,
91 RequestCommand, RequestJoin, RequestOptionChainReferencePrice, RequestQuotes,
92 RequestTrades, SubscribeBars, SubscribeBookDeltas, SubscribeBookDepth, SubscribeCommand,
93 SubscribeOptionChain, SubscribeOptionGreeks, SubscribeQuotes, SubscribeTrades,
94 UnsubscribeBars, UnsubscribeBookDeltas, UnsubscribeBookDepth, UnsubscribeBookSnapshots,
95 UnsubscribeCommand, UnsubscribeInstrumentStatus, UnsubscribeOptionChain,
96 UnsubscribeOptionGreeks, UnsubscribeQuotes, UnsubscribeTrades, ensure_routable_bar_types,
97 ensure_routable_instrument_ids, is_parent_subscription, quote_routing,
98 },
99 msgbus::{
100 self, BusPayloadType, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
101 get_message_bus,
102 switchboard::{self, MessagingSwitchboard},
103 },
104 runner::get_data_cmd_sender,
105 timer::{TimeEvent, TimeEventCallback},
106};
107use nautilus_core::{
108 DurationNanos, Params, UUID4, UnixNanos, WeakCell,
109 correctness::{FAILED, check_key_in_map, check_key_not_in_map, check_predicate_true},
110 datetime::NANOSECONDS_IN_DAY,
111};
112#[cfg(feature = "defi")]
113use nautilus_model::defi::DefiData;
114use nautilus_model::{
115 data::{
116 Bar, BarType, CustomData, Data, DataRef, DataType, FundingRateUpdate, HasTsInit,
117 IndexPriceUpdate, InstrumentClose, InstrumentStatus, MarkPriceUpdate, OrderBookDelta,
118 OrderBookDeltas, OrderBookDepth, QuoteTick, TradeTick,
119 option_chain::{OptionGreeks, StrikeRange},
120 },
121 enums::{
122 AggregationSource, BarAggregation, BookType, InstrumentClass, MarketStatusAction,
123 PriceType, RecordFlag,
124 },
125 identifiers::{
126 ClientId, GENERIC_SPREAD_ID_SEPARATOR, InstrumentId, OptionSeriesId, Venue,
127 parse_generic_spread_id_legs,
128 },
129 instruments::{Instrument, InstrumentAny, SyntheticInstrument},
130 orderbook::OrderBook,
131 types::{Price, Quantity},
132};
133use option_chain::{
134 OptionChainBootstrapper, OptionChainGreeksBootstrap, PendingOptionChainRequest,
135};
136use requests::{
137 ContinuousFutureRequest, ContinuousFutureRequestState, ContinuousFutureSegment,
138 ContinuousFutureSource, RequestBarAggregation, continuous_future_request_from_bars,
139 continuous_future_subscription_from_bars, has_continuous_future_params,
140 request_bar_aggregation_from_params, request_params, response_params,
141};
142use responses::{
143 log_if_empty_response, parent_request_window, rebind_response_correlation,
144 rebuild_pipeline_response, same_instrument,
145};
146use serde_json::Value;
147use spread_quote::SpreadQuoteState;
148#[cfg(feature = "streaming")]
149use streaming::CatalogMap;
150use time_range::{
151 TimeRangePipelineState, has_time_range_pipeline_params, is_time_range_pipeline_variant,
152};
153use ustr::Ustr;
154
155#[cfg(feature = "defi")]
156#[allow(unused_imports)] use crate::defi::engine as _;
158#[cfg(feature = "defi")]
159use crate::engine::pool::PoolUpdater;
160use crate::{
161 aggregation::{
162 BarAggregator, RenkoBarAggregator, SpreadQuoteAggregator, TickBarAggregator,
163 TickImbalanceBarAggregator, TickRunsBarAggregator, TimeBarAggregator, ValueBarAggregator,
164 ValueImbalanceBarAggregator, ValueRunsBarAggregator, VolumeBarAggregator,
165 VolumeImbalanceBarAggregator, VolumeRunsBarAggregator,
166 },
167 client::DataClientAdapter,
168 option_chains::OptionChainManager,
169 subscription::{
170 SubscriptionKey, SubscriptionRegistry, SubscriptionRelease, split_subscribe,
171 split_unsubscribe,
172 },
173};
174
175#[derive(Debug)]
177pub struct DataEngine {
178 clock: Rc<RefCell<dyn Clock>>,
179 cache: Rc<RefCell<Cache>>,
180 config: DataEngineConfig,
181 is_stopped: bool,
182 msgbus_priority: u32,
183 subscriptions_external: SubscriptionRegistry<(ClientId, SubscriptionKey), SubscribeCommand>,
184 clients: IndexMap<ClientId, DataClientAdapter>,
185 default_client_id: Option<ClientId>,
186 routing_map: IndexMap<Venue, ClientId>,
187 book_intervals: AHashMap<NonZeroUsize, BookSnapshotInfos>,
188 book_snapshot_counts: IndexMap<BookSnapshotKey, usize>,
189 book_snapshot_sources: AHashMap<InstrumentId, BookSnapshotSource>,
190 book_deltas_counts: IndexMap<BookDeltasKey, usize>,
191 book_depth_counts: IndexMap<BookDeltasKey, usize>,
192 book_updaters: AHashMap<InstrumentId, Rc<BookUpdater>>,
193 book_subscriptions: AHashMap<InstrumentId, BookSubscription>,
194 book_subscription_owners: AHashMap<InstrumentId, Vec<Rc<BookSubscriptionOwner>>>,
195 book_snapshotters: AHashMap<NonZeroUsize, Rc<BookSnapshotter>>,
196 bar_aggregators: IndexMap<BarAggregatorKey, Rc<RefCell<Box<dyn BarAggregator>>>>,
197 bar_aggregator_handlers: AHashMap<BarAggregatorKey, Vec<BarAggregatorSubscription>>,
198 bar_aggregator_warmups: AHashMap<BarAggregatorKey, BarAggregatorWarmup>,
199 bar_aggregators_pending_stop: AHashSet<BarAggregatorKey>,
200 subscriptions_bar_aggregation: AHashMap<BarType, BarAggregationSubscription>,
201 request_bar_aggregations: AHashMap<UUID4, RequestBarAggregation>,
202 inflight_requests: AHashMap<UUID4, RequestCommand>,
203 request_pipeline_parent_request: AHashMap<UUID4, RequestCommand>,
204 request_pipeline_n_components: AHashMap<UUID4, usize>,
205 request_pipeline_parent_request_id: AHashMap<UUID4, (UUID4, usize)>,
206 request_pipeline_responses: AHashMap<UUID4, Vec<Option<Vec<DataResponseEnvelope>>>>,
207 time_range_pipeline_requests: AHashMap<UUID4, TimeRangePipelineState>,
208 time_range_pipeline_parent_request_id: AHashMap<UUID4, UUID4>,
209 parent_join_request_id: AHashMap<UUID4, UUID4>,
210 pending_join_requests: AHashMap<UUID4, RequestJoin>,
211 continuous_future_requests: AHashMap<UUID4, ContinuousFutureRequestState>,
212 continuous_future_parent_request_id: AHashMap<UUID4, UUID4>,
213 continuous_future_subscriptions: AHashMap<BarType, ContinuousFutureSubscriptionState>,
214 continuous_future_roller: Option<Rc<ContinuousFutureRoller>>,
215 spread_quote_states: AHashMap<InstrumentId, SpreadQuoteState>,
216 option_chain_managers: AHashMap<OptionSeriesId, Rc<RefCell<OptionChainManager>>>,
217 option_chain_instrument_index: AHashMap<InstrumentId, OptionSeriesId>,
218 deferred_cmd_queue: DeferredCommandQueue,
219 option_chain_bootstrapper: Option<Rc<OptionChainBootstrapper>>,
220 pending_option_chain_requests: AHashMap<UUID4, PendingOptionChainRequest>,
221 option_chain_greeks_bootstraps: AHashMap<OptionSeriesId, OptionChainGreeksBootstrap>,
222 synthetic_quote_feeds: AHashMap<InstrumentId, Vec<SyntheticInstrument>>,
223 synthetic_trade_feeds: AHashMap<InstrumentId, Vec<SyntheticInstrument>>,
224 subscribed_synthetic_quotes: AHashMap<InstrumentId, usize>,
225 subscribed_synthetic_trades: AHashMap<InstrumentId, usize>,
226 buffered_deltas_map: AHashMap<InstrumentId, OrderBookDeltas>,
227 deltas_frame: Vec<OrderBookDelta>,
228 command_count: u64,
229 data_count: u64,
230 request_count: u64,
231 response_count: u64,
232 #[cfg(feature = "streaming")]
233 catalogs: CatalogMap,
234 #[cfg(feature = "defi")]
235 pub(crate) pool_updaters: AHashMap<InstrumentId, Rc<PoolUpdater>>,
236 #[cfg(feature = "defi")]
237 pub(crate) pool_snapshot_pending: AHashMap<InstrumentId, UUID4>,
238 #[cfg(feature = "defi")]
239 pub(crate) pool_event_buffers: AHashMap<InstrumentId, Vec<DefiData>>,
240}
241
242impl DataEngine {
243 #[must_use]
245 pub fn new(
246 clock: Rc<RefCell<dyn Clock>>,
247 cache: Rc<RefCell<Cache>>,
248 config: Option<DataEngineConfig>,
249 ) -> Self {
250 let config = config.unwrap_or_default();
251 if let Some(client_ids) = &config.external_clients {
252 let mut cache = cache.borrow_mut();
253 for client_id in client_ids {
254 cache.add_external_data_client(*client_id);
255 }
256 }
257
258 Self {
259 clock,
260 cache,
261 config,
262 is_stopped: false,
263 msgbus_priority: 10, subscriptions_external: SubscriptionRegistry::default(),
265 clients: IndexMap::new(),
266 default_client_id: None,
267 routing_map: IndexMap::new(),
268 book_intervals: AHashMap::new(),
269 book_snapshot_counts: IndexMap::new(),
270 book_snapshot_sources: AHashMap::new(),
271 book_deltas_counts: IndexMap::new(),
272 book_depth_counts: IndexMap::new(),
273 book_updaters: AHashMap::new(),
274 book_subscriptions: AHashMap::new(),
275 book_subscription_owners: AHashMap::new(),
276 book_snapshotters: AHashMap::new(),
277 bar_aggregators: IndexMap::new(),
278 bar_aggregator_handlers: AHashMap::new(),
279 bar_aggregator_warmups: AHashMap::new(),
280 bar_aggregators_pending_stop: AHashSet::new(),
281 subscriptions_bar_aggregation: AHashMap::new(),
282 request_bar_aggregations: AHashMap::new(),
283 inflight_requests: AHashMap::new(),
284 request_pipeline_parent_request: AHashMap::new(),
285 request_pipeline_n_components: AHashMap::new(),
286 request_pipeline_parent_request_id: AHashMap::new(),
287 request_pipeline_responses: AHashMap::new(),
288 time_range_pipeline_requests: AHashMap::new(),
289 time_range_pipeline_parent_request_id: AHashMap::new(),
290 parent_join_request_id: AHashMap::new(),
291 pending_join_requests: AHashMap::new(),
292 continuous_future_requests: AHashMap::new(),
293 continuous_future_parent_request_id: AHashMap::new(),
294 continuous_future_subscriptions: AHashMap::new(),
295 continuous_future_roller: None,
296 spread_quote_states: AHashMap::new(),
297 option_chain_managers: AHashMap::new(),
298 option_chain_instrument_index: AHashMap::new(),
299 deferred_cmd_queue: Rc::new(RefCell::new(VecDeque::new())),
300 option_chain_bootstrapper: None,
301 pending_option_chain_requests: AHashMap::new(),
302 option_chain_greeks_bootstraps: AHashMap::new(),
303 synthetic_quote_feeds: AHashMap::new(),
304 synthetic_trade_feeds: AHashMap::new(),
305 subscribed_synthetic_quotes: AHashMap::new(),
306 subscribed_synthetic_trades: AHashMap::new(),
307 buffered_deltas_map: AHashMap::new(),
308 deltas_frame: Vec::new(),
309 command_count: 0,
310 data_count: 0,
311 request_count: 0,
312 response_count: 0,
313 #[cfg(feature = "streaming")]
314 catalogs: CatalogMap::new(),
315 #[cfg(feature = "defi")]
316 pool_updaters: AHashMap::new(),
317 #[cfg(feature = "defi")]
318 pool_snapshot_pending: AHashMap::new(),
319 #[cfg(feature = "defi")]
320 pool_event_buffers: AHashMap::new(),
321 }
322 }
323
324 #[must_use]
326 pub fn clock(&self) -> &Rc<RefCell<dyn Clock>> {
327 &self.clock
328 }
329
330 #[must_use]
332 pub fn cache(&self) -> &Rc<RefCell<Cache>> {
333 &self.cache
334 }
335
336 #[must_use]
338 pub const fn config(&self) -> &DataEngineConfig {
339 &self.config
340 }
341
342 #[cfg(feature = "defi")]
343 #[must_use]
344 pub(crate) const fn msgbus_priority(&self) -> u32 {
345 self.msgbus_priority
346 }
347
348 pub fn register_msgbus_handlers(engine: &Rc<RefCell<Self>>) {
352 let weak = WeakCell::from(Rc::downgrade(engine));
353 engine.borrow_mut().continuous_future_roller =
354 Some(Rc::new(ContinuousFutureRoller::new(engine)));
355 engine.borrow_mut().option_chain_bootstrapper =
356 Some(Rc::new(OptionChainBootstrapper::new(engine)));
357
358 let weak1 = weak.clone();
359 msgbus::register_data_command_endpoint(
360 MessagingSwitchboard::data_engine_execute(),
361 TypedIntoHandler::from(move |cmd: DataCommand| {
362 if let Some(rc) = weak1.upgrade() {
363 rc.borrow_mut().execute(cmd);
364 }
365 }),
366 );
367
368 msgbus::register_data_command_endpoint(
369 MessagingSwitchboard::data_engine_queue_execute(),
370 TypedIntoHandler::from(move |cmd: DataCommand| {
371 get_data_cmd_sender().execute(cmd);
372 }),
373 );
374
375 let weak2 = weak.clone();
377 msgbus::register_any(
378 MessagingSwitchboard::data_engine_process(),
379 ShareableMessageHandler::from_any(move |data: &dyn Any| {
380 if let Some(rc) = weak2.upgrade() {
381 rc.borrow_mut().process(data);
382 }
383 }),
384 );
385
386 let weak3 = weak.clone();
388 msgbus::register_data_endpoint(
389 MessagingSwitchboard::data_engine_process_data(),
390 TypedIntoHandler::from(move |data: Data| {
391 if let Some(rc) = weak3.upgrade() {
392 rc.borrow_mut().process_data(data);
393 }
394 }),
395 );
396
397 let external_data_ingress = engine.borrow().cache.borrow().has_external_data_clients();
398 get_message_bus()
399 .borrow_mut()
400 .set_external_data_ingress(external_data_ingress);
401
402 #[cfg(feature = "defi")]
404 {
405 let weak4 = weak.clone();
406 msgbus::register_defi_data_endpoint(
407 MessagingSwitchboard::data_engine_process_defi_data(),
408 TypedIntoHandler::from(move |data: DefiData| {
409 if let Some(rc) = weak4.upgrade() {
410 rc.borrow_mut().process_defi_data(data);
411 }
412 }),
413 );
414 }
415
416 let weak5 = weak;
417 msgbus::register_data_response_endpoint(
418 MessagingSwitchboard::data_engine_response(),
419 TypedIntoHandler::from(move |resp: DataResponse| {
420 if let Some(rc) = weak5.upgrade() {
421 rc.borrow_mut().response(resp);
422 }
423 }),
424 );
425 }
426
427 #[must_use]
429 pub const fn command_count(&self) -> u64 {
430 self.command_count
431 }
432
433 #[must_use]
435 pub const fn data_count(&self) -> u64 {
436 self.data_count
437 }
438
439 #[cfg(feature = "defi")]
440 pub(crate) const fn increment_data_count(&mut self) {
441 self.data_count += 1;
442 }
443
444 #[must_use]
446 pub const fn request_count(&self) -> u64 {
447 self.request_count
448 }
449
450 #[must_use]
452 pub const fn response_count(&self) -> u64 {
453 self.response_count
454 }
455
456 #[must_use]
458 pub fn request_pipeline_count(&self) -> usize {
459 self.request_pipeline_parent_request.len()
460 }
461
462 #[must_use]
464 pub fn time_range_pipeline_count(&self) -> usize {
465 self.time_range_pipeline_requests.len()
466 }
467
468 #[must_use]
470 pub fn pending_join_request_count(&self) -> usize {
471 self.pending_join_requests.len()
472 }
473
474 pub fn start(&mut self) {
476 self.is_stopped = false;
477
478 for client in self.get_clients_mut() {
479 client.is_stopped = false;
480 if client.start().is_err() {
481 log::error!("Failed to start data client {}", client.client_id());
482 }
483 }
484
485 for ((_, request_id), aggregator) in &self.bar_aggregators {
486 let is_subscription = request_id.is_none() && !aggregator.borrow().is_historical();
489 if is_subscription && aggregator.borrow().bar_type().spec().is_time_aggregated() {
490 aggregator
491 .borrow_mut()
492 .start_timer(Some(Rc::clone(aggregator)));
493 }
494 }
495
496 for state in self.spread_quote_states.values() {
497 state
498 .aggregator
499 .borrow_mut()
500 .start_timer(Some(Rc::clone(&state.aggregator)));
501 }
502 }
503
504 pub fn stop(&mut self) {
510 self.is_stopped = true;
511
512 for client in self.get_clients_mut() {
513 client.is_stopped = true;
514 if client.stop().is_err() {
515 log::error!("Failed to stop data client {}", client.client_id());
516 }
517 }
518
519 for aggregator in self.bar_aggregators.values() {
520 aggregator.borrow_mut().stop();
521 }
522
523 for state in self.spread_quote_states.values() {
524 state.aggregator.borrow_mut().stop_timer();
525 }
526 }
527
528 pub fn reset(&mut self) {
530 self.is_stopped = false;
531
532 for client in self.get_clients_mut() {
533 client.is_stopped = false;
534 match client.reset() {
535 Ok(()) => client.clear_subscription_state(),
536 Err(_) => log::error!("Failed to reset data client {}", client.client_id()),
537 }
538 }
539
540 let keys: Vec<BarAggregatorKey> = self.bar_aggregators.keys().copied().collect();
541 for (bar_type, request_id) in keys {
542 if let Err(e) = self.stop_bar_aggregator(bar_type, request_id) {
543 log::error!("Error stopping bar aggregator during reset for {bar_type}: {e}");
544 }
545 }
546 self.subscriptions_bar_aggregation.clear();
547
548 self.request_bar_aggregations.clear();
549 self.inflight_requests.clear();
550 self.request_pipeline_parent_request.clear();
551 self.request_pipeline_n_components.clear();
552 self.request_pipeline_parent_request_id.clear();
553 self.request_pipeline_responses.clear();
554 self.time_range_pipeline_requests.clear();
555 self.time_range_pipeline_parent_request_id.clear();
556 self.parent_join_request_id.clear();
557 self.pending_join_requests.clear();
558 self.continuous_future_requests.clear();
559 self.continuous_future_parent_request_id.clear();
560
561 for state in self.continuous_future_subscriptions.values_mut() {
562 if let Some(name) = state.timer_name.take() {
563 self.clock.borrow_mut().cancel_timer(&name);
564 }
565 }
566 self.continuous_future_subscriptions.clear();
567
568 let spread_ids: Vec<InstrumentId> = self.spread_quote_states.keys().copied().collect();
569 for spread_id in spread_ids {
570 self.stop_spread_quote_aggregation(spread_id);
571 }
572
573 let managers: Vec<_> = self.option_chain_managers.drain().collect();
575 for (_, manager) in managers {
576 manager.borrow_mut().teardown(&self.clock);
577 }
578
579 self.option_chain_instrument_index.clear();
580 self.cancel_pending_option_chain_requests(None);
581 self.clear_option_chain_greeks_bootstraps();
582
583 let book_updaters: Vec<(InstrumentId, Rc<BookUpdater>)> =
588 self.book_updaters.drain().collect();
589 for (instrument_id, updater) in book_updaters {
590 let deltas_topic = switchboard::get_book_deltas_topic(instrument_id);
591 let depth_topic = switchboard::get_book_depth_topic(instrument_id);
592 let deltas_handler: TypedHandler<OrderBookDeltas> =
593 TypedHandler::new(Rc::clone(&updater));
594 let depth_handler: TypedHandler<OrderBookDepth> = TypedHandler::new(updater);
595 msgbus::unsubscribe_book_deltas(deltas_topic.into(), &deltas_handler);
596 msgbus::unsubscribe_book_depth(depth_topic.into(), &depth_handler);
597 }
598
599 self.book_subscriptions.clear();
600 self.book_subscription_owners.clear();
601
602 self.book_deltas_counts.clear();
603 self.book_depth_counts.clear();
604 self.book_intervals.clear();
605 self.book_snapshot_counts.clear();
606 self.book_snapshot_sources.clear();
607 self.book_snapshotters.clear();
608 self.buffered_deltas_map.clear();
609 self.deltas_frame.clear();
610
611 self.synthetic_quote_feeds.clear();
612 self.synthetic_trade_feeds.clear();
613 self.subscribed_synthetic_quotes.clear();
614 self.subscribed_synthetic_trades.clear();
615 self.subscriptions_external.clear();
616
617 self.deferred_cmd_queue.borrow_mut().clear();
618
619 #[cfg(feature = "defi")]
620 self.clear_pool_updaters();
621
622 self.clock.borrow_mut().cancel_timers();
623
624 self.command_count = 0;
625 self.data_count = 0;
626 self.request_count = 0;
627 self.response_count = 0;
628 }
629
630 pub fn dispose(&mut self) {
632 for client in self.get_clients_mut() {
633 if client.dispose().is_err() {
634 log::error!("Failed to dispose data client {}", client.client_id());
635 }
636 }
637
638 self.clear_option_chain_greeks_bootstraps();
639
640 let mut cf_sources = Vec::new();
643
644 for state in self.continuous_future_subscriptions.values_mut() {
645 if let Some(name) = state.timer_name.take() {
646 self.clock.borrow_mut().cancel_timer(&name);
647 }
648
649 if let Some(subscription) = state.active_source_subscription.take() {
650 cf_sources.push((state.target_bar_type, subscription));
651 }
652 }
653
654 for (target_bar_type, subscription) in cf_sources {
655 self.unsubscribe_continuous_future_source(target_bar_type, subscription);
656 }
657 self.continuous_future_subscriptions.clear();
658
659 let keys: Vec<BarAggregatorKey> = self.bar_aggregators.keys().copied().collect();
662 for (bar_type, request_id) in keys {
663 if let Err(e) = self.stop_bar_aggregator(bar_type, request_id) {
664 log::error!("Error stopping bar aggregator during dispose for {bar_type}: {e}");
665 }
666 }
667
668 self.subscriptions_external.clear();
669
670 self.clock.borrow_mut().cancel_timers();
671 }
672
673 pub async fn connect(&mut self) -> anyhow::Result<()> {
683 let futures: Vec<_> = self
684 .get_clients_mut()
685 .into_iter()
686 .map(|client| async move {
687 let client_id = client.client_id();
688 (client_id, client.connect().await)
689 })
690 .collect();
691
692 let results = join_all(futures).await;
693 let mut failed_clients = Vec::new();
694
695 for (client_id, result) in results {
696 if result.is_err() {
697 log::error!("Failed to connect data client {client_id}: connection failed");
698 failed_clients.push(client_id.to_string());
699 }
700 }
701
702 if failed_clients.is_empty() {
703 Ok(())
704 } else {
705 anyhow::bail!(
706 "Failed to connect data clients: {}",
707 failed_clients.join(", ")
708 )
709 }
710 }
711
712 pub async fn disconnect(&mut self) -> anyhow::Result<()> {
718 let futures: Vec<_> = self
719 .get_clients_mut()
720 .into_iter()
721 .map(|client| async move {
722 let client_id = client.client_id();
723 (client_id, client.disconnect().await)
724 })
725 .collect();
726
727 let results = join_all(futures).await;
728
729 #[cfg(feature = "defi")]
731 self.abandon_pool_snapshots();
732
733 let failed_clients: Vec<_> = results
734 .into_iter()
735 .filter_map(|(client_id, result)| result.err().map(|_| client_id.to_string()))
736 .collect();
737
738 if failed_clients.is_empty() {
739 Ok(())
740 } else {
741 anyhow::bail!(
742 "Failed to disconnect data clients: {}",
743 failed_clients.join(", ")
744 )
745 }
746 }
747
748 #[must_use]
750 pub fn check_connected(&self) -> bool {
751 self.get_clients()
752 .iter()
753 .all(|client| client.is_connected())
754 }
755
756 #[must_use]
758 pub fn check_disconnected(&self) -> bool {
759 self.get_clients()
760 .iter()
761 .all(|client| !client.is_connected())
762 }
763
764 #[must_use]
766 pub fn subscribed_custom_data(&self) -> Vec<DataType> {
767 self.collect_subscriptions(|client| &client.subscriptions_custom)
768 }
769
770 #[must_use]
772 pub fn subscribed_instruments(&self) -> Vec<InstrumentId> {
773 self.collect_subscriptions(|client| &client.subscriptions_instrument)
774 }
775
776 #[must_use]
778 pub fn subscribed_quotes(&self) -> Vec<InstrumentId> {
779 self.collect_subscriptions(|client| &client.subscriptions_quotes)
780 }
781
782 #[must_use]
784 pub fn subscribed_trades(&self) -> Vec<InstrumentId> {
785 self.collect_subscriptions(|client| &client.subscriptions_trades)
786 }
787
788 #[must_use]
791 pub fn subscribed_bars(&self) -> Vec<BarType> {
792 let mut subscribed = self.collect_subscriptions(|client| &client.subscriptions_bars);
793 subscribed.extend(
794 self.bar_aggregators
795 .keys()
796 .filter(|key| key.1.is_none() && !self.bar_aggregators_pending_stop.contains(*key))
797 .map(|(bar_type, _)| *bar_type),
798 );
799 subscribed
800 }
801
802 #[must_use]
804 pub fn subscribed_mark_prices(&self) -> Vec<InstrumentId> {
805 self.collect_subscriptions(|client| &client.subscriptions_mark_prices)
806 }
807
808 #[must_use]
810 pub fn subscribed_index_prices(&self) -> Vec<InstrumentId> {
811 self.collect_subscriptions(|client| &client.subscriptions_index_prices)
812 }
813
814 #[must_use]
816 pub fn subscribed_funding_rates(&self) -> Vec<InstrumentId> {
817 self.collect_subscriptions(|client| &client.subscriptions_funding_rates)
818 }
819
820 #[must_use]
822 pub fn subscribed_instrument_status(&self) -> Vec<InstrumentId> {
823 self.collect_subscriptions(|client| &client.subscriptions_instrument_status)
824 }
825
826 #[must_use]
828 pub fn subscribed_instrument_close(&self) -> Vec<InstrumentId> {
829 self.collect_subscriptions(|client| &client.subscriptions_instrument_close)
830 }
831
832 pub fn execute(&mut self, cmd: DataCommand) {
841 match &cmd {
842 DataCommand::Subscribe(_) | DataCommand::Unsubscribe(_) => self.command_count += 1,
843 DataCommand::Request(_) => self.request_count += 1,
844 #[cfg(feature = "defi")]
845 DataCommand::DefiRequest(_) => self.request_count += 1,
846 #[cfg(feature = "defi")]
847 DataCommand::DefiSubscribe(_) | DataCommand::DefiUnsubscribe(_) => {
848 self.command_count += 1;
849 }
850 _ => {}
851 }
852
853 let command_context = match &cmd {
854 DataCommand::Subscribe(c) => (
855 "Subscribe",
856 c.client_id().copied(),
857 c.venue().copied(),
858 Some(c.command_id()),
859 ),
860 DataCommand::Unsubscribe(c) => (
861 "Unsubscribe",
862 c.client_id().copied(),
863 c.venue().copied(),
864 Some(c.command_id()),
865 ),
866 DataCommand::Request(c) => (
867 "Request",
868 c.client_id().copied(),
869 c.venue().copied(),
870 Some(*c.request_id()),
871 ),
872 #[cfg(feature = "defi")]
873 DataCommand::DefiRequest(c) => (
874 "DefiRequest",
875 c.client_id().copied(),
876 c.venue().copied(),
877 Some(*c.request_id()),
878 ),
879 #[cfg(feature = "defi")]
880 DataCommand::DefiSubscribe(c) => (
881 "DefiSubscribe",
882 c.client_id().copied(),
883 c.venue().copied(),
884 Some(c.command_id()),
885 ),
886 #[cfg(feature = "defi")]
887 DataCommand::DefiUnsubscribe(c) => (
888 "DefiUnsubscribe",
889 c.client_id().copied(),
890 c.venue().copied(),
891 Some(c.command_id()),
892 ),
893 _ => ("Unknown", None, None, None),
894 };
895
896 let result = match cmd {
897 DataCommand::Subscribe(c) => self.execute_subscribe(c),
898 DataCommand::Unsubscribe(c) => self.execute_unsubscribe(&c),
899 DataCommand::Request(c) => self.execute_request(c),
900 #[cfg(feature = "defi")]
901 DataCommand::DefiRequest(c) => self.execute_defi_request(c),
902 #[cfg(feature = "defi")]
903 DataCommand::DefiSubscribe(c) => self.execute_defi_subscribe(c),
904 #[cfg(feature = "defi")]
905 DataCommand::DefiUnsubscribe(c) => self.execute_defi_unsubscribe(&c),
906 _ => {
907 log::warn!("Unhandled DataCommand variant");
908 Ok(())
909 }
910 };
911
912 if let Err(e) = result {
913 log::error!("Data command failed: {command_context:?}: {e}");
914 }
915 }
916
917 pub fn execute_subscribe(&mut self, cmd: SubscribeCommand) -> anyhow::Result<()> {
924 cmd.validate_routing(&self.cache.borrow())?;
925 if let Some(client_id) = cmd.client_id().copied()
926 && self.is_external_client(client_id)
927 {
928 return self.subscribe_external(client_id, &cmd);
929 }
930
931 self.retain_intent(&cmd)?;
932
933 let retained = cmd.clone();
934 let (groups, result) = self.prepare_subscribe_groups(cmd);
935 self.dispatch_subscribe_groups(&retained, groups);
936 result
937 }
938
939 fn retain_intent(&mut self, cmd: &SubscribeCommand) -> anyhow::Result<()> {
940 let retain_intent = match cmd {
941 SubscribeCommand::Quotes(command) => {
942 self.quote_routing(&command.instrument_ids, command.params.as_ref())?
943 == QuoteRouting::Client
944 }
945 SubscribeCommand::Trades(command) => !command.instrument_ids[0].is_synthetic(),
946 _ => false,
947 };
948
949 if let Some(client) = self.get_command_client(cmd.client_id(), cmd.venue()) {
950 if let SubscribeCommand::Bars(command) = cmd {
951 let external: Vec<_> = command
952 .bar_types
953 .iter()
954 .copied()
955 .filter(BarType::is_externally_aggregated)
956 .collect();
957
958 if !external.is_empty() {
959 let intent = SubscribeCommand::Bars(command.with_bar_types(external));
960 client.retain_intent(&intent);
961 }
962 } else if retain_intent {
963 client.retain_intent(cmd);
964 }
965 }
966
967 Ok(())
968 }
969
970 fn prepare_subscribe_groups(
971 &mut self,
972 cmd: SubscribeCommand,
973 ) -> (Vec<SubscribeCommand>, anyhow::Result<()>) {
974 let members = if matches!(
975 &cmd,
976 SubscribeCommand::Quotes(_) | SubscribeCommand::Trades(_)
977 ) {
978 vec![cmd]
979 } else {
980 split_subscribe(cmd)
981 };
982
983 let mut groups: IndexMap<Option<Value>, Vec<SubscribeCommand>> = IndexMap::new();
984 let mut result = Ok(());
985
986 for member in members {
987 let prepared = match self.prepare_subscribe_member(&member) {
988 Ok(Some(prepared)) => prepared,
989 Ok(None) => continue,
990 Err(e) => {
991 log::error!(
992 "Subscribe preparation failed: command={}, client={:?}, instrument_ids={:?}, bar_types={:?}: {e}",
993 member.command_id(),
994 member.client_id(),
995 member.instrument_ids(),
996 member.bar_types(),
997 );
998
999 if result.is_ok() {
1000 result = Err(e);
1001 }
1002
1003 continue;
1004 }
1005 };
1006
1007 let params = prepared.params().map(|params| {
1009 Value::Object(params.iter().map(|(k, v)| (k.clone(), v.clone())).collect())
1010 });
1011
1012 groups.entry(params).or_default().push(prepared);
1013 }
1014
1015 let groups = groups
1016 .into_values()
1017 .map(|mut members| {
1018 if members.len() == 1 {
1019 return members.pop().expect("nonempty group");
1020 }
1021
1022 let first = &members[0];
1023 if first.instrument_ids().is_some() {
1024 let ids: Vec<_> = members
1025 .iter()
1026 .flat_map(|member| member.instrument_ids().expect("same parent kind"))
1027 .copied()
1028 .collect();
1029 first.with_instrument_ids(ids).expect("instrument command")
1030 } else if first.bar_types().is_some() {
1031 let types: Vec<_> = members
1032 .iter()
1033 .flat_map(|member| member.bar_types().expect("same parent kind"))
1034 .copied()
1035 .collect();
1036 first.with_bar_types(types).expect("bar command")
1037 } else {
1038 first.clone()
1039 }
1040 })
1041 .collect();
1042
1043 (groups, result)
1044 }
1045
1046 fn dispatch_subscribe_groups(&mut self, cmd: &SubscribeCommand, groups: Vec<SubscribeCommand>) {
1047 for group in groups {
1048 let retained = if let Some(ids) = group.instrument_ids() {
1049 cmd.with_instrument_ids(ids).expect("instrument parent")
1050 } else if let Some(types) = group.bar_types() {
1051 cmd.with_bar_types(types).expect("bar parent")
1052 } else {
1053 cmd.clone()
1054 };
1055
1056 let retain_on_failure = !matches!(
1057 &group,
1058 SubscribeCommand::BookDeltas(_) | SubscribeCommand::BookDepth(_)
1059 );
1060
1061 if let Some(client) = self.get_command_client(group.client_id(), group.venue()) {
1062 client.execute_subscribe_with_retained(group, retained, retain_on_failure);
1063 } else {
1064 log::error!(
1065 "Cannot handle command: no client found for client_id={:?}, venue={:?}",
1066 group.client_id(),
1067 group.venue()
1068 );
1069 }
1070 }
1071 }
1072
1073 fn prepare_subscribe_member(
1074 &mut self,
1075 cmd: &SubscribeCommand,
1076 ) -> anyhow::Result<Option<SubscribeCommand>> {
1077 match cmd {
1079 SubscribeCommand::BookDeltas(book_cmd) => {
1080 if !self.subscribe_book_deltas(book_cmd)? && self.client_subscription_active(cmd) {
1081 return Ok(None);
1082 }
1083 }
1084 SubscribeCommand::BookDepth(book_cmd) => {
1085 if !self.subscribe_book_depth(book_cmd)? && self.client_subscription_active(cmd) {
1086 return Ok(None);
1087 }
1088 }
1089 SubscribeCommand::BookSnapshots(cmd) => {
1090 self.subscribe_book_snapshots(cmd)?;
1092 return Ok(None);
1093 }
1094 SubscribeCommand::Bars(cmd) if has_continuous_future_params(cmd.params.as_ref()) => {
1095 self.subscribe_continuous_future_bars(cmd)?;
1096 return Ok(None);
1097 }
1098 SubscribeCommand::Bars(cmd) => {
1099 self.subscribe_bars(cmd)?;
1100 if cmd
1101 .single_bar_type()
1102 .expect("member preparation receives one bar type")
1103 .is_internally_aggregated()
1104 {
1105 return Ok(None);
1106 }
1107 }
1108 SubscribeCommand::OptionChain(cmd) => {
1109 self.subscribe_option_chain(cmd);
1110 return Ok(None);
1111 }
1112 SubscribeCommand::Quotes(cmd) if self.subscribe_quotes(cmd)? => {
1113 return Ok(None);
1114 }
1115 SubscribeCommand::Trades(cmd) if self.subscribe_trades(cmd) => {
1116 return Ok(None);
1117 }
1118 SubscribeCommand::Instrument(cmd)
1119 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1120 {
1121 anyhow::bail!("Cannot subscribe for synthetic instrument `Instrument` data");
1122 }
1123 SubscribeCommand::InstrumentStatus(cmd)
1124 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1125 {
1126 anyhow::bail!("Cannot subscribe for synthetic instrument `InstrumentStatus` data");
1127 }
1128 SubscribeCommand::InstrumentClose(cmd)
1129 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1130 {
1131 anyhow::bail!("Cannot subscribe for synthetic instrument `InstrumentClose` data");
1132 }
1133 SubscribeCommand::OptionGreeks(cmd)
1134 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1135 {
1136 anyhow::bail!("Cannot subscribe for synthetic instrument `OptionGreeks` data");
1137 }
1138 _ => {} }
1140
1141 let cmd = cmd.clone();
1142 #[cfg(feature = "streaming")]
1143 let cmd = self.subscribe_command_with_prefilled_start_ns(cmd)?;
1144
1145 Ok(Some(cmd))
1146 }
1147
1148 fn client_subscription_active(&mut self, cmd: &SubscribeCommand) -> bool {
1149 self.get_command_client(cmd.client_id(), cmd.venue())
1150 .is_some_and(|client| client.has_active_subscription(cmd))
1151 }
1152
1153 pub fn execute_unsubscribe(&mut self, cmd: &UnsubscribeCommand) -> anyhow::Result<()> {
1159 if let Some(bar_types) = cmd.bar_types() {
1160 ensure_routable_bar_types(bar_types, cmd.client_id().is_some())?;
1161 }
1162
1163 if let Some(ids) = cmd.instrument_ids() {
1164 ensure_routable_instrument_ids(ids, cmd.client_id().is_some())?;
1165 }
1166
1167 if let UnsubscribeCommand::Trades(trade) = cmd
1168 && !trade
1169 .instrument_ids
1170 .iter()
1171 .all(|id| id.is_synthetic() == trade.instrument_ids[0].is_synthetic())
1172 {
1173 anyhow::bail!("trade identifier lists cannot mix client and synthetic routes");
1174 }
1175
1176 if let Some(client_id) = cmd.client_id().copied()
1177 && self.is_external_client(client_id)
1178 {
1179 self.unsubscribe_external(client_id, cmd);
1180 return Ok(());
1181 }
1182
1183 if matches!(
1184 &cmd,
1185 UnsubscribeCommand::Instrument(_)
1186 | UnsubscribeCommand::InstrumentStatus(_)
1187 | UnsubscribeCommand::InstrumentClose(_)
1188 | UnsubscribeCommand::OptionGreeks(_)
1189 | UnsubscribeCommand::BookDeltas(_)
1190 | UnsubscribeCommand::BookDepth(_)
1191 | UnsubscribeCommand::BookSnapshots(_)
1192 ) && cmd
1193 .instrument_ids()
1194 .is_some_and(|ids| ids.len() > 1 && ids.iter().any(InstrumentId::is_synthetic))
1195 {
1196 anyhow::bail!("Cannot unsubscribe synthetic instruments for this data type");
1197 }
1198
1199 let (prepared, result) = self.prepare_unsubscribe_members(cmd);
1200
1201 for command in UnsubscribeCommand::group(prepared) {
1202 if let Some(client) = self.get_command_client(command.client_id(), command.venue()) {
1203 client.execute_unsubscribe(&command);
1204 } else {
1205 log::error!(
1206 "Cannot handle command: no client found for client_id={:?}, venue={:?}",
1207 command.client_id(),
1208 command.venue()
1209 );
1210 }
1211 }
1212
1213 result
1214 }
1215
1216 fn prepare_unsubscribe_members(
1217 &mut self,
1218 cmd: &UnsubscribeCommand,
1219 ) -> (Vec<UnsubscribeCommand>, anyhow::Result<()>) {
1220 let members = if matches!(
1221 cmd,
1222 UnsubscribeCommand::Quotes(_) | UnsubscribeCommand::Trades(_)
1223 ) {
1224 vec![cmd.clone()]
1225 } else {
1226 split_unsubscribe(cmd)
1227 };
1228
1229 let mut prepared = Vec::new();
1230 let mut result = Ok(());
1231
1232 for member in members {
1233 match self.prepare_unsubscribe_member(&member) {
1234 Ok(true) => prepared.push(member),
1235 Ok(false) => {}
1236 Err(e) => {
1237 log::error!(
1238 "Unsubscribe preparation failed: command={}, client={:?}, instrument_ids={:?}, bar_types={:?}: {e}",
1239 member.command_id(),
1240 member.client_id(),
1241 member.instrument_ids(),
1242 member.bar_types(),
1243 );
1244
1245 if result.is_ok() {
1246 result = Err(e);
1247 }
1248 }
1249 }
1250 }
1251
1252 (prepared, result)
1253 }
1254
1255 fn prepare_unsubscribe_member(&mut self, cmd: &UnsubscribeCommand) -> anyhow::Result<bool> {
1256 if matches!(
1257 cmd,
1258 UnsubscribeCommand::BookDeltas(_)
1259 | UnsubscribeCommand::BookDepth(_)
1260 | UnsubscribeCommand::BookSnapshots(_)
1261 ) && !self.release_book_subscription(cmd)
1262 {
1263 return Ok(false);
1264 }
1265
1266 match &cmd {
1267 UnsubscribeCommand::BookDeltas(cmd) if !self.unsubscribe_book_deltas(cmd) => {
1268 return Ok(false);
1269 }
1270 UnsubscribeCommand::BookDepth(cmd) if !self.unsubscribe_book_depth(cmd) => {
1271 return Ok(false);
1272 }
1273 UnsubscribeCommand::BookSnapshots(cmd) => {
1274 self.unsubscribe_book_snapshots(cmd);
1276 return Ok(false);
1277 }
1278 UnsubscribeCommand::Bars(cmd)
1279 if self.continuous_future_subscriptions.contains_key(
1280 &cmd.single_bar_type()
1281 .expect("member preparation receives one bar type")
1282 .standard(),
1283 ) =>
1284 {
1285 let topic = switchboard::get_bars_topic(
1287 cmd.single_bar_type()
1288 .expect("member preparation receives one bar type")
1289 .standard(),
1290 );
1291
1292 if msgbus::exact_subscriber_count_bars(topic) == 0 {
1293 self.unsubscribe_continuous_future_bars(cmd);
1294 }
1295
1296 return Ok(false);
1297 }
1298 UnsubscribeCommand::Bars(cmd) => {
1299 self.unsubscribe_bars(cmd);
1300 if cmd
1301 .single_bar_type()
1302 .expect("member preparation receives one bar type")
1303 .is_internally_aggregated()
1304 {
1305 return Ok(false);
1306 }
1307 }
1308 UnsubscribeCommand::OptionChain(cmd) => {
1309 self.unsubscribe_option_chain(cmd);
1310 return Ok(false);
1311 }
1312 UnsubscribeCommand::Quotes(cmd) if self.unsubscribe_quotes(cmd)? => {
1313 return Ok(false);
1314 }
1315 UnsubscribeCommand::Trades(cmd) if self.unsubscribe_trades(cmd) => {
1316 return Ok(false);
1317 }
1318 UnsubscribeCommand::Instrument(cmd)
1319 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1320 {
1321 anyhow::bail!("Cannot unsubscribe from synthetic instrument `Instrument` data");
1322 }
1323 UnsubscribeCommand::InstrumentStatus(cmd)
1324 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1325 {
1326 anyhow::bail!(
1327 "Cannot unsubscribe from synthetic instrument `InstrumentStatus` data"
1328 );
1329 }
1330 UnsubscribeCommand::InstrumentClose(cmd)
1331 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1332 {
1333 anyhow::bail!(
1334 "Cannot unsubscribe from synthetic instrument `InstrumentClose` data"
1335 );
1336 }
1337 UnsubscribeCommand::OptionGreeks(cmd)
1338 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1339 {
1340 anyhow::bail!("Cannot unsubscribe from synthetic instrument `OptionGreeks` data");
1341 }
1342 _ => {}
1343 }
1344
1345 Ok(true)
1346 }
1347
1348 fn subscribe_external(
1349 &mut self,
1350 client_id: ClientId,
1351 cmd: &SubscribeCommand,
1352 ) -> anyhow::Result<()> {
1353 let (dispatch, result) = match cmd {
1354 SubscribeCommand::OptionChain(command) => {
1355 self.retain_external_option_chain(client_id, command, cmd);
1356 (Some(cmd.clone()), Ok(()))
1357 }
1358 _ => self.prepare_external_subscribe(client_id, cmd),
1359 };
1360
1361 let Some(dispatch) = dispatch else {
1362 return result;
1363 };
1364
1365 if self.is_stopped {
1366 log::debug!(
1367 "Skipped external subscribe after stop: client_id={client_id}, command_id={}",
1368 dispatch.command_id()
1369 );
1370 return result;
1371 }
1372
1373 register_external_streaming_type(&dispatch);
1374 publish_external_data_command(client_id, &dispatch);
1375
1376 if self.config.debug {
1377 log::debug!("Skipping subscribe command for external client {client_id}: {cmd:?}");
1378 }
1379
1380 result
1381 }
1382
1383 fn prepare_external_subscribe(
1384 &mut self,
1385 client_id: ClientId,
1386 cmd: &SubscribeCommand,
1387 ) -> (Option<SubscribeCommand>, anyhow::Result<()>) {
1388 let mut result = Ok(());
1389 let mut acquired = false;
1390 let mut fresh_ids = Vec::new();
1391 let mut fresh_bar_types = Vec::new();
1392
1393 for member in split_subscribe(cmd.clone()) {
1394 let key = SubscriptionKey::from_subscribe(&member);
1395 let instrument_id = member.instrument_ids().and_then(|ids| ids.first()).copied();
1396 let bar_type = member.bar_types().and_then(|types| types.first()).copied();
1397
1398 match self.retain_external_subscription(client_id, key.clone(), member) {
1399 Ok(false) => continue,
1400 Ok(true) => {}
1401 Err(e) => {
1402 log::error!(
1403 "External subscribe preparation failed: command={}, client={client_id}, subscription={}: {e}",
1404 cmd.command_id(),
1405 key.log_context(),
1406 );
1407
1408 if result.is_ok() {
1409 result = Err(e);
1410 }
1411
1412 continue;
1413 }
1414 }
1415
1416 acquired = true;
1417 fresh_ids.extend(instrument_id);
1418 fresh_bar_types.extend(bar_type);
1419 }
1420
1421 if !acquired {
1422 return (None, result);
1423 }
1424
1425 let dispatch = if cmd.bar_types().is_some() {
1426 cmd.with_bar_types(fresh_bar_types)
1427 .expect("bar command has types")
1428 } else {
1429 cmd.with_instrument_ids(fresh_ids)
1430 .unwrap_or_else(|| cmd.clone())
1431 };
1432
1433 (Some(dispatch), result)
1434 }
1435
1436 fn retain_external_subscription(
1437 &mut self,
1438 client_id: ClientId,
1439 key: SubscriptionKey,
1440 command: SubscribeCommand,
1441 ) -> anyhow::Result<bool> {
1442 let key = (client_id, key);
1443
1444 if self
1445 .subscriptions_external
1446 .get_mut(&key)
1447 .is_some_and(|active| active.acquisitions.contains(&command.command_id()))
1448 {
1449 return Ok(false);
1450 }
1451
1452 let params = self.subscriptions_external.get_mut(&key).map_or_else(
1454 || command.params().cloned(),
1455 |active| active.command.params().cloned(),
1456 );
1457
1458 match &command {
1459 SubscribeCommand::BookDeltas(command) => {
1460 let mut command = command.clone();
1461 command.params = params;
1462 self.subscribe_book_deltas(&command)?;
1463 }
1464 SubscribeCommand::BookDepth(command) => {
1465 let mut command = command.clone();
1466 command.params = params;
1467 self.subscribe_book_depth(&command)?;
1468 }
1469 _ => {}
1470 }
1471
1472 Ok(self
1473 .subscriptions_external
1474 .retain(key, command.command_id(), command))
1475 }
1476
1477 fn unsubscribe_external(&mut self, client_id: ClientId, cmd: &UnsubscribeCommand) {
1478 let mut releases = IndexMap::new();
1479
1480 for member in split_unsubscribe(cmd) {
1481 let key = (client_id, SubscriptionKey::from_unsubscribe(&member));
1482 if matches!(
1483 member,
1484 UnsubscribeCommand::BookDeltas(_) | UnsubscribeCommand::BookDepth(_)
1485 ) && self.subscriptions_external.contains(&key)
1486 {
1487 if !self.release_book_subscription(&member) {
1488 continue;
1489 }
1490
1491 match &member {
1492 UnsubscribeCommand::BookDeltas(command) => {
1493 self.unsubscribe_book_deltas(command);
1494 }
1495 UnsubscribeCommand::BookDepth(command) => {
1496 self.unsubscribe_book_depth(command);
1497 }
1498 _ => unreachable!("only delta and depth subscriptions retain external books"),
1499 }
1500 }
1501
1502 let (acquisition_id, command) = match self.subscriptions_external.release(&key) {
1503 SubscriptionRelease::Retained => continue,
1504 SubscriptionRelease::Final(subscribe) => (
1505 subscribe.command_id(),
1506 subscribe.into_unsubscribe(
1507 cmd.command_id(),
1508 cmd.ts_init(),
1509 cmd.correlation_id(),
1510 ),
1511 ),
1512 SubscriptionRelease::Untracked => (cmd.command_id(), member),
1513 };
1514
1515 self.subscriptions_external.remove(&key);
1516 releases
1517 .entry(acquisition_id)
1518 .or_insert_with(Vec::new)
1519 .push(command);
1520 }
1521
1522 for command in releases.into_values().flat_map(UnsubscribeCommand::group) {
1523 publish_external_data_command(client_id, &command);
1524 }
1525
1526 if self.config.debug {
1527 log::debug!("Skipping unsubscribe command for external client {client_id}: {cmd:?}");
1528 }
1529 }
1530
1531 fn subscribe_quotes(&mut self, cmd: &SubscribeQuotes) -> anyhow::Result<bool> {
1532 match self.quote_routing(&cmd.instrument_ids, cmd.params.as_ref())? {
1533 QuoteRouting::Client => Ok(false),
1534 QuoteRouting::Synthetic => {
1535 for id in &cmd.instrument_ids {
1536 self.subscribe_synthetic_quotes(*id);
1537 }
1538
1539 Ok(true)
1540 }
1541 QuoteRouting::Spread => {
1542 for id in &cmd.instrument_ids {
1543 let member = cmd.with_instrument_ids(*id);
1544 self.subscribe_spread_quotes(&member);
1545 }
1546
1547 Ok(true)
1548 }
1549 }
1550 }
1551
1552 fn unsubscribe_quotes(&mut self, cmd: &UnsubscribeQuotes) -> anyhow::Result<bool> {
1553 match self.quote_routing(&cmd.instrument_ids, cmd.params.as_ref())? {
1554 QuoteRouting::Client => Ok(false),
1555 QuoteRouting::Synthetic => {
1556 for id in &cmd.instrument_ids {
1557 self.unsubscribe_synthetic_quotes(*id);
1558 }
1559
1560 Ok(true)
1561 }
1562 QuoteRouting::Spread => {
1563 for id in &cmd.instrument_ids {
1564 let member = cmd.with_instrument_ids(*id);
1565 self.unsubscribe_spread_quotes(&member);
1566 }
1567
1568 Ok(true)
1569 }
1570 }
1571 }
1572
1573 fn subscribe_trades(&mut self, cmd: &SubscribeTrades) -> bool {
1574 if !cmd.instrument_ids[0].is_synthetic() {
1575 return false;
1576 }
1577
1578 for id in &cmd.instrument_ids {
1579 self.subscribe_synthetic_trades(*id);
1580 }
1581
1582 true
1583 }
1584
1585 fn unsubscribe_trades(&mut self, cmd: &UnsubscribeTrades) -> bool {
1586 if !cmd.instrument_ids[0].is_synthetic() {
1587 return false;
1588 }
1589
1590 for id in &cmd.instrument_ids {
1591 self.unsubscribe_synthetic_trades(*id);
1592 }
1593
1594 true
1595 }
1596
1597 fn quote_routing(
1598 &self,
1599 ids: &[InstrumentId],
1600 params: Option<&Params>,
1601 ) -> anyhow::Result<QuoteRouting> {
1602 quote_routing(ids, None, params, &self.cache.borrow())
1603 }
1604
1605 pub fn execute_request(&mut self, req: RequestCommand) -> anyhow::Result<()> {
1614 let request_id = *req.request_id();
1615 let result = self.execute_request_inner(req);
1616 if let Err(e) = &result {
1617 self.process_response_envelope(DataResponseEnvelope::terminal(
1618 request_id,
1619 Some(e.to_string()),
1620 ));
1621 }
1622
1623 result
1624 }
1625
1626 fn execute_request_inner(&mut self, req: RequestCommand) -> anyhow::Result<()> {
1627 Self::validate_request(&req)?;
1628 #[cfg(feature = "streaming")]
1629 self.validate_catalog_update(&req)?;
1630
1631 if let Some(cid) = req.client_id()
1633 && self.is_external_client(*cid)
1634 {
1635 if self.config.debug {
1636 log::debug!("Skipping data request for external client {cid}: {req:?}");
1637 }
1638 return Ok(());
1639 }
1640
1641 if let RequestCommand::Join(join) = req {
1642 return self.handle_request_join(join);
1643 }
1644
1645 if has_continuous_future_params(request_params(&req)) {
1646 return self.execute_continuous_future_request(req);
1647 }
1648
1649 let request_id = *req.request_id();
1650 self.prepare_request_bar_aggregators(&req)?;
1651
1652 if has_time_range_pipeline_params(request_params(&req))
1653 && is_time_range_pipeline_variant(&req)
1654 {
1655 let result = self.execute_time_range_pipeline_request(req);
1656 if result.is_err() {
1657 self.cleanup_request_bar_aggregators(&request_id);
1658 }
1659 return result;
1660 }
1661
1662 #[cfg(feature = "streaming")]
1663 if self.catalogs_registered() && streaming::is_date_range_variant(&req) {
1664 let result = self.dispatch_date_range_request(req);
1665 if result.is_err() {
1666 self.cleanup_request_bar_aggregators(&request_id);
1667 }
1668 return result;
1669 }
1670
1671 let result = self.dispatch_request_to_client(req);
1672
1673 if result.is_err() {
1674 self.cleanup_request_bar_aggregators(&request_id);
1675 }
1676
1677 result.map(|_| ())
1678 }
1679
1680 fn validate_request(req: &RequestCommand) -> anyhow::Result<()> {
1681 let (instrument_ids, requires_single, limit) = match req {
1682 RequestCommand::Instrument(cmd) => (&cmd.instrument_ids, false, None),
1683 RequestCommand::BookSnapshot(cmd) => (&cmd.instrument_ids, true, None),
1684 RequestCommand::BookDeltas(cmd) => (&cmd.instrument_ids, true, None),
1685 RequestCommand::BookDepth(cmd) => (&cmd.instrument_ids, true, None),
1686 RequestCommand::Quotes(cmd) => (&cmd.instrument_ids, false, cmd.limit),
1687 RequestCommand::Trades(cmd) => (&cmd.instrument_ids, false, cmd.limit),
1688 RequestCommand::FundingRates(cmd) => (&cmd.instrument_ids, true, None),
1689 RequestCommand::Bars(cmd) => {
1690 ensure_routable_bar_types(&cmd.bar_types, cmd.client_id.is_some())?;
1691 cmd.single_bar_type()?;
1692 return Ok(());
1693 }
1694 _ => return Ok(()),
1695 };
1696
1697 ensure_routable_instrument_ids(instrument_ids, req.client_id().is_some())?;
1698 anyhow::ensure!(
1699 !requires_single || instrument_ids.len() == 1,
1700 "request requires one instrument ID for this data type, received {}",
1701 instrument_ids.len(),
1702 );
1703 anyhow::ensure!(
1704 instrument_ids.len() == 1 || limit.is_none(),
1705 "A record limit is unsupported for multi-instrument requests",
1706 );
1707 Ok(())
1708 }
1709
1710 pub(super) fn dispatch_request_to_client(
1711 &mut self,
1712 req: RequestCommand,
1713 ) -> anyhow::Result<ClientId> {
1714 self.inflight_requests
1715 .insert(*req.request_id(), req.clone());
1716 let client_id = req.client_id().copied();
1717 let venue = req.venue().copied();
1718 let Some(client) = self.get_client(client_id.as_ref(), venue.as_ref()) else {
1719 anyhow::bail!("Cannot handle request: no client found for {client_id:?} {venue:?}");
1720 };
1721 let resolved_client_id = client.client_id();
1722
1723 #[rustfmt::skip]
1724 let result = match req {
1725 RequestCommand::Data(req) => client.request_data(req),
1726 RequestCommand::Instrument(req) => client.request_instrument(req),
1727 RequestCommand::Instruments(req) => client.request_instruments(req),
1728 RequestCommand::BookSnapshot(req) => client.request_book_snapshot(req),
1729 RequestCommand::BookDeltas(req) => client.request_book_deltas(req),
1730 RequestCommand::BookDepth(req) => client.request_book_depth(req),
1731 RequestCommand::Quotes(req) => client.request_quotes(req),
1732 RequestCommand::Trades(req) => client.request_trades(req),
1733 RequestCommand::FundingRates(req) => client.request_funding_rates(req),
1734 RequestCommand::OptionChainReferencePrice(req) => client.request_option_chain_reference_price(req),
1735 RequestCommand::Bars(req) => client.request_bars(req),
1736 RequestCommand::Join(_) => anyhow::bail!("RequestJoin must be handled by handle_request_join"),
1737 };
1738
1739 result.map_err(|e| {
1740 let context = format!("Data client {resolved_client_id} request failed");
1741
1742 let context = if let Some(rejection) = e.downcast_ref::<IdentifierCountError>() {
1743 format!("{context}: {rejection}")
1744 } else {
1745 context
1746 };
1747
1748 e.context(context)
1749 })?;
1750
1751 Ok(resolved_client_id)
1752 }
1753
1754 pub fn process(&mut self, data: &dyn Any) {
1759 if let Some(response) = data.downcast_ref::<DataResponseEnvelope>() {
1760 self.response_envelope(response.clone());
1761 return;
1762 }
1763
1764 if let Some(failure) = data.downcast_ref::<DataRequestFailed>() {
1765 self.request_failed(failure.clone());
1766 return;
1767 }
1768
1769 self.data_count += 1;
1770
1771 if let Some(instrument) = data.downcast_ref::<InstrumentAny>() {
1772 self.handle_instrument(instrument);
1773 } else if let Some(funding_rate) = data.downcast_ref::<FundingRateUpdate>() {
1774 self.handle_funding_rate(*funding_rate);
1775 } else if let Some(option_greeks) = data.downcast_ref::<OptionGreeks>() {
1776 self.cache.borrow_mut().add_option_greeks(*option_greeks);
1777 self.feed_option_greeks_to_pre_bootstrap_chain(option_greeks);
1778 let topic = switchboard::get_option_greeks_topic(option_greeks.instrument_id);
1779 msgbus::publish_option_greeks(topic, option_greeks);
1780 self.drain_deferred_commands();
1781 } else if let Some(status) = data.downcast_ref::<InstrumentStatus>() {
1782 self.handle_instrument_status(*status);
1783 } else if let Some(custom) = data.downcast_ref::<CustomData>() {
1784 self.handle_custom_data(custom);
1785 } else {
1786 #[cfg(feature = "defi")]
1787 if let Some(response) = data.downcast_ref::<PoolSnapshotResponse>() {
1788 self.handle_pool_snapshot_response(response);
1789 return;
1790 }
1791
1792 log::error!("Cannot process data {data:?}, type is unrecognized");
1793 }
1794 }
1795
1796 #[allow(
1798 clippy::needless_pass_by_value,
1799 reason = "callers hand over ownership; the payload is only moved when a DeFi handler consumes it"
1800 )]
1801 pub fn process_data(&mut self, data: Data) {
1802 #[cfg(feature = "defi")]
1803 let data = match data {
1804 Data::Defi(defi) => {
1805 self.process_defi_data(*defi);
1806 return;
1807 }
1808 data => data,
1809 };
1810
1811 self.process_data_ref(DataRef::from(&data));
1812 }
1813
1814 pub fn process_data_ref(&mut self, data: DataRef<'_>) {
1819 #[cfg(feature = "defi")]
1820 let data = match data {
1821 DataRef::Defi(defi) => {
1822 self.process_defi_data(defi.clone());
1823 return;
1824 }
1825 data => data,
1826 };
1827
1828 self.data_count += 1;
1829
1830 match data {
1831 DataRef::Instrument(instrument) => self.handle_instrument(instrument),
1832 DataRef::BookDelta(delta) => self.handle_delta(*delta),
1833 DataRef::BookDeltas(deltas) => self.handle_deltas(deltas),
1834 DataRef::BookDepth(depth) => self.handle_depth(depth),
1835 DataRef::Quote(quote) => {
1836 self.handle_quote(*quote);
1837 self.drain_deferred_commands();
1838 }
1839 DataRef::Trade(trade) => self.handle_trade(*trade),
1840 DataRef::Bar(bar) => self.handle_bar(*bar),
1841 DataRef::MarkPrice(mark_price) => {
1842 self.handle_mark_price(*mark_price);
1843 self.drain_deferred_commands();
1844 }
1845 DataRef::IndexPrice(index_price) => {
1846 self.handle_index_price(*index_price);
1847 self.drain_deferred_commands();
1848 }
1849 DataRef::FundingRate(funding_rate) => {
1850 self.handle_funding_rate(*funding_rate);
1851 self.drain_deferred_commands();
1852 }
1853 DataRef::OptionGreeks(greeks) => {
1854 self.cache.borrow_mut().add_option_greeks(*greeks);
1855 self.feed_option_greeks_to_pre_bootstrap_chain(greeks);
1856 let topic = switchboard::get_option_greeks_topic(greeks.instrument_id);
1857 msgbus::publish_option_greeks(topic, greeks);
1858 self.drain_deferred_commands();
1859 }
1860 DataRef::InstrumentStatus(status) => {
1861 self.handle_instrument_status(*status);
1862 self.drain_deferred_commands();
1863 }
1864 DataRef::InstrumentClose(close) => self.handle_instrument_close(*close),
1865 DataRef::Custom(custom) => self.handle_custom_data(custom),
1866 #[cfg(feature = "defi")]
1867 DataRef::Defi(_) => unreachable!("handled before market data dispatch"),
1868 #[cfg(not(feature = "defi"))]
1869 #[allow(
1870 unreachable_patterns,
1871 reason = "DeFi variants can exist without this crate's defi feature"
1872 )]
1873 other => log_defi_data_dropped(other),
1874 }
1875 }
1876
1877 pub fn process_pipeline(&mut self, data: Data) {
1884 #[cfg(feature = "defi")]
1885 let data = match data {
1886 Data::Defi(defi) => {
1887 self.process_defi_data(*defi);
1888 return;
1889 }
1890 data => data,
1891 };
1892
1893 self.data_count += 1;
1894
1895 match data {
1896 Data::Instrument(instrument) => self.handle_instrument(&instrument),
1897 Data::BookDelta(delta) => self.handle_delta_pipeline(delta),
1898 Data::BookDeltas(deltas) => self.handle_deltas_pipeline(&deltas),
1899 Data::BookDepth(depth) => self.handle_depth_pipeline(&depth),
1900 Data::Quote(quote) => self.handle_quote_pipeline(quote),
1901 Data::Trade(trade) => self.handle_trade_pipeline(trade),
1902 Data::Bar(bar) => self.handle_bar_pipeline(bar),
1903 Data::MarkPrice(mark_price) => self.handle_mark_price_pipeline(mark_price),
1904 Data::IndexPrice(index_price) => self.handle_index_price_pipeline(index_price),
1905 Data::FundingRate(funding_rate) => {
1906 self.handle_funding_rate_pipeline(funding_rate);
1907 }
1908 Data::OptionGreeks(greeks) => self.handle_option_greeks_pipeline(greeks),
1909 Data::InstrumentStatus(status) => self.handle_instrument_status_pipeline(status),
1910 Data::InstrumentClose(close) => self.handle_instrument_close_pipeline(close),
1911 Data::Custom(custom) => self.handle_custom_data_pipeline(&custom),
1912 #[cfg(feature = "defi")]
1913 Data::Defi(_) => unreachable!("handled before market data dispatch"),
1914 #[cfg(not(feature = "defi"))]
1915 #[allow(
1916 unreachable_patterns,
1917 reason = "DeFi variants can exist without this crate's defi feature"
1918 )]
1919 other => log_defi_data_dropped(DataRef::from(&other)),
1920 }
1921 }
1922
1923 pub fn response(&mut self, mut resp: DataResponse) {
1928 resp.trim_to_bounds();
1929 let request = self.inflight_requests.remove(resp.correlation_id());
1930 #[cfg(feature = "streaming")]
1931 if request
1932 .as_ref()
1933 .and_then(request_params)
1934 .is_some_and(|params| params.get_bool(streaming::PARAM_UPDATE_CATALOG) == Some(true))
1935 {
1936 log::warn!(
1937 "Catalog write-back requires an explicit completed response envelope, request={}",
1938 resp.correlation_id()
1939 );
1940 }
1941
1942 #[cfg(not(feature = "streaming"))]
1943 let _ = request;
1944 let envelope = DataResponseEnvelope::new(resp, true, None);
1945 self.process_response_envelope(envelope);
1946 }
1947
1948 pub fn request_failed(&mut self, failure: DataRequestFailed) {
1950 log::error!(
1951 "Data request {} failed: {}",
1952 failure.request_id,
1953 failure.reason
1954 );
1955
1956 let envelope = if let Some(mut response) = failure.response {
1957 response.trim_to_bounds();
1958 DataResponseEnvelope::new(response, true, None)
1959 .with_correlation_id(failure.request_id)
1960 .with_failure(failure.reason)
1961 } else {
1962 DataResponseEnvelope::terminal(failure.request_id, Some(failure.reason))
1963 };
1964
1965 self.process_response_envelope(envelope);
1966 }
1967
1968 pub fn response_envelope(&mut self, mut envelope: DataResponseEnvelope) {
1970 if let Some(response) = envelope.response_mut() {
1971 response.trim_to_bounds();
1972 }
1973
1974 self.process_response_envelope(envelope);
1975 }
1976
1977 fn process_response_envelope(&mut self, mut envelope: DataResponseEnvelope) {
1978 let accepted = match envelope.response() {
1979 Some(DataResponse::Instrument(response)) => self.cache_instrument(&response.data),
1980 Some(DataResponse::Instruments(response)) => {
1981 let mut accepted = true;
1982 for instrument in &response.data {
1983 accepted &= self.cache_instrument(instrument);
1984 }
1985
1986 accepted
1987 }
1988 _ => true,
1989 };
1990
1991 if !accepted {
1992 envelope = DataResponseEnvelope::terminal(
1993 *envelope.correlation_id(),
1994 Some("Instrument currency integrity check failed".into()),
1995 );
1996 }
1997
1998 let request = self.inflight_requests.remove(envelope.correlation_id());
1999 #[cfg(feature = "streaming")]
2000 let envelope = match (&request, envelope.response()) {
2001 (Some(request), Some(response))
2002 if envelope.completed() && envelope.failure_reason().is_none() =>
2003 {
2004 match self.update_catalog_from_response(response, request) {
2005 Ok(()) => envelope,
2006 Err(e) => envelope.with_failure(e.to_string()),
2007 }
2008 }
2009 _ => envelope,
2010 };
2011
2012 #[cfg(not(feature = "streaming"))]
2013 let _ = request;
2014
2015 let Some(envelope) = self.handle_request_pipeline_response(envelope) else {
2016 return;
2017 };
2018
2019 if self
2020 .request_pipeline_parent_request_id
2021 .contains_key(envelope.correlation_id())
2022 {
2023 self.process_response_envelope(envelope);
2024 return;
2025 }
2026
2027 if self.handle_continuous_future_child_response(&envelope)
2028 || self.handle_time_range_pipeline_child_response(&envelope)
2029 {
2030 return;
2031 }
2032
2033 let Some(resp) = envelope.response().cloned() else {
2034 self.finish_empty_response(&envelope);
2035 return;
2036 };
2037
2038 self.route_response_payload(resp, &envelope);
2039 }
2040
2041 fn finish_empty_response(&mut self, envelope: &DataResponseEnvelope) {
2042 if let Some(original_id) = self
2043 .parent_join_request_id
2044 .remove(envelope.correlation_id())
2045 {
2046 if let Some(original) = self.pending_join_requests.remove(&original_id) {
2047 for child_id in original.request_ids {
2048 msgbus::send_response_envelope(&DataResponseEnvelope::terminal(
2049 child_id,
2050 envelope.failure_reason().map(str::to_owned),
2051 ));
2052 }
2053 }
2054
2055 self.process_response_envelope(DataResponseEnvelope::terminal(
2056 original_id,
2057 envelope.failure_reason().map(str::to_owned),
2058 ));
2059 return;
2060 }
2061
2062 self.cleanup_request_bar_aggregators(envelope.correlation_id());
2063 msgbus::send_response_envelope(envelope);
2064 }
2065
2066 fn route_response_payload(&mut self, resp: DataResponse, envelope: &DataResponseEnvelope) {
2067 let resp = rebind_response_correlation(resp, *envelope.correlation_id());
2068 if log::log_enabled!(log::Level::Debug) {
2069 let correlation_id = resp.correlation_id();
2070 match resp.record_count() {
2071 Some(count) => log::debug!(
2072 "{RECV}{RES} {} correlation_id={correlation_id} records={count}",
2073 resp.kind(),
2074 ),
2075 None => log::debug!(
2076 "{RECV}{RES} {} correlation_id={correlation_id}",
2077 resp.kind(),
2078 ),
2079 }
2080 }
2081 log::trace!("{RECV}{RES} {resp:?}");
2082
2083 self.response_count += 1;
2084
2085 if self
2086 .parent_join_request_id
2087 .contains_key(resp.correlation_id())
2088 {
2089 self.finalize_request_join(resp, envelope.failure_reason().map(str::to_owned));
2090 return;
2091 }
2092
2093 if !self.handle_response_payload(&resp) {
2094 return;
2095 }
2096
2097 self.process_request_bar_aggregation_response(&resp, envelope.completed());
2098
2099 msgbus::send_response_envelope(envelope);
2100 }
2101
2102 fn handle_response_payload(&mut self, resp: &DataResponse) -> bool {
2103 let correlation_id = *resp.correlation_id();
2104
2105 match resp {
2106 DataResponse::Instrument(_) | DataResponse::Instruments(_) => {}
2107 DataResponse::Quotes(r) => {
2108 if !log_if_empty_response(
2109 &r.data,
2110 &format_args!("{:?}", r.instrument_ids),
2111 &correlation_id,
2112 ) {
2113 self.handle_quotes(&r.data);
2114 }
2115 }
2116 DataResponse::Trades(r) => {
2117 if !log_if_empty_response(
2118 &r.data,
2119 &format_args!("{:?}", r.instrument_ids),
2120 &correlation_id,
2121 ) {
2122 self.handle_trades(&r.data);
2123 }
2124 }
2125 DataResponse::FundingRates(r) => {
2126 if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2127 self.handle_funding_rates(&r.data);
2128 }
2129 }
2130 DataResponse::Bars(r) => {
2131 if !log_if_empty_response(&r.data, &r.bar_type, &correlation_id) {
2132 self.handle_bars(&r.data);
2133 }
2134 }
2135 DataResponse::Book(r) => self.handle_book_response(&r.data),
2136 DataResponse::BookDeltas(r) => {
2137 if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2138 self.handle_book_deltas_response(r);
2139 }
2140 }
2141 DataResponse::BookDepth(r) => {
2142 if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2143 self.handle_book_depth_response(r);
2144 }
2145 }
2146 DataResponse::OptionChainReferencePrice(r) => {
2147 self.process_request_bar_aggregation_response(resp, true);
2148 self.handle_option_chain_reference_price_response(&correlation_id, r);
2149 return false;
2150 }
2151 DataResponse::Data(_) => {}
2152 }
2153
2154 true
2155 }
2156
2157 pub fn new_request_pipeline(&mut self, parent: RequestCommand, n_components: usize) {
2159 let parent_id = *parent.request_id();
2160 self.request_pipeline_n_components
2161 .insert(parent_id, n_components);
2162 self.request_pipeline_parent_request
2163 .insert(parent_id, parent);
2164 self.request_pipeline_responses
2165 .insert(parent_id, Vec::with_capacity(n_components));
2166 }
2167
2168 pub fn register_request_pipeline_leg(&mut self, leg_id: UUID4, parent_id: UUID4) {
2174 let slots = self
2175 .request_pipeline_responses
2176 .get_mut(&parent_id)
2177 .expect("pipeline registered before its legs");
2178 let index = slots.len();
2179 slots.push(None);
2180 self.request_pipeline_parent_request_id
2181 .insert(leg_id, (parent_id, index));
2182 }
2183
2184 fn handle_request_pipeline_response(
2185 &mut self,
2186 envelope: DataResponseEnvelope,
2187 ) -> Option<DataResponseEnvelope> {
2188 let leg_id = *envelope.correlation_id();
2189
2190 let Some(&(parent_id, index)) = self.request_pipeline_parent_request_id.get(&leg_id) else {
2191 return Some(envelope);
2192 };
2193
2194 let completed = envelope.completed();
2195 let slots = self.request_pipeline_responses.get_mut(&parent_id)?;
2196 slots[index].get_or_insert_with(Vec::new).push(envelope);
2197
2198 if !completed {
2199 return None;
2200 }
2201
2202 self.request_pipeline_parent_request_id.remove(&leg_id);
2203 let expected = *self.request_pipeline_n_components.get(&parent_id)?;
2204
2205 if slots
2206 .iter()
2207 .flatten()
2208 .filter(|slot| slot.last().is_some_and(DataResponseEnvelope::completed))
2209 .count()
2210 < expected
2211 {
2212 return None;
2213 }
2214
2215 let slots = self.request_pipeline_responses.remove(&parent_id)?;
2216 self.request_pipeline_n_components.remove(&parent_id);
2217 let parent = self.request_pipeline_parent_request.remove(&parent_id);
2218 let failures = slots
2219 .iter()
2220 .flatten()
2221 .flatten()
2222 .filter_map(|value| value.failure_reason())
2223 .collect::<Vec<_>>();
2224 let failure_reason = (!failures.is_empty()).then(|| failures.join("; "));
2225 let legs = slots
2226 .into_iter()
2227 .flatten()
2228 .flatten()
2229 .filter_map(|value| value.response().cloned())
2230 .collect::<Vec<_>>();
2231
2232 if legs.is_empty() {
2233 return Some(DataResponseEnvelope::terminal(parent_id, failure_reason));
2234 }
2235
2236 let (parent_start, parent_end) = parent_request_window(parent.as_ref());
2237 if self.emit_partitioned_book_responses(parent_id, parent.as_ref(), &legs) {
2238 return Some(DataResponseEnvelope::terminal(parent_id, failure_reason));
2239 }
2240
2241 let rebuilt = rebuild_pipeline_response(parent_id, parent.as_ref(), legs);
2242
2243 let Some(mut rebuilt) = rebuilt else {
2244 let reason = failure_reason
2245 .unwrap_or_else(|| "Request pipeline could not rebuild its response".to_owned());
2246
2247 return Some(DataResponseEnvelope::terminal(parent_id, Some(reason)));
2248 };
2249
2250 if let DataResponse::BookDeltas(r) = &mut rebuilt {
2251 self.book_deltas_snapshot_replay(r);
2252 }
2253
2254 if !matches!(parent, Some(RequestCommand::Instrument(_)))
2256 && (parent_start.is_some() || parent_end.is_some())
2257 {
2258 rebuilt.trim_to_bounds();
2259 }
2260
2261 let result = DataResponseEnvelope::new(rebuilt, true, None);
2262 Some(if let Some(reason) = failure_reason {
2263 result.with_failure(reason)
2264 } else {
2265 result
2266 })
2267 }
2268
2269 fn emit_partitioned_book_responses(
2270 &mut self,
2271 parent_id: UUID4,
2272 parent: Option<&RequestCommand>,
2273 legs: &[DataResponse],
2274 ) -> bool {
2275 if !matches!(parent, Some(RequestCommand::BookDeltas(_))) {
2276 return false;
2277 }
2278
2279 let mut ids = legs.iter().filter_map(|r| match r {
2280 DataResponse::BookDeltas(r) => Some(r.instrument_id),
2281 _ => None,
2282 });
2283
2284 let Some(first) = ids.next() else {
2285 return false;
2286 };
2287
2288 if !ids.any(|id| !same_instrument(id, first)) {
2289 return false;
2290 }
2291
2292 let (start, end) = parent_request_window(parent);
2293
2294 for response in legs {
2295 let child_id = *response.correlation_id();
2296 let mut response = rebind_response_correlation(response.clone(), parent_id);
2297 if let DataResponse::BookDeltas(r) = &mut response {
2298 r.start = start.or(r.start);
2299 r.end = end.or(r.end);
2300 self.book_deltas_snapshot_replay(r);
2301 }
2302
2303 response.trim_to_bounds();
2304 self.process_response_envelope(DataResponseEnvelope::new(
2305 response,
2306 false,
2307 Some(child_id),
2308 ));
2309 }
2310
2311 true
2312 }
2313
2314 fn handle_request_join(&mut self, req: RequestJoin) -> anyhow::Result<()> {
2315 if has_time_range_pipeline_params(req.params.as_ref()) {
2316 return self.execute_time_range_pipeline_request(RequestCommand::Join(req));
2317 }
2318
2319 let now_ns = self.clock.borrow().timestamp_ns();
2320 let now_dt = now_ns.to_datetime_utc();
2321 let zero = jiff::Timestamp::UNIX_EPOCH;
2322 let start = req.start.unwrap_or(zero).min(now_dt);
2323 let end = req.end.unwrap_or(now_dt).min(now_dt);
2324 let dated = req.with_dates(Some(start), Some(end), now_ns);
2325
2326 let original_id = req.request_id;
2327 let dated_id = dated.request_id;
2328
2329 self.pending_join_requests.insert(original_id, req);
2330 self.parent_join_request_id.insert(dated_id, original_id);
2331
2332 let leg_ids: Vec<UUID4> = dated.request_ids.clone();
2333 self.new_request_pipeline(RequestCommand::Join(dated), leg_ids.len());
2334 for leg_id in leg_ids {
2335 self.register_request_pipeline_leg(leg_id, dated_id);
2336 }
2337
2338 Ok(())
2339 }
2340
2341 fn finalize_request_join(&mut self, resp: DataResponse, failure_reason: Option<String>) {
2342 let dated_id = *resp.correlation_id();
2343 let Some(original_id) = self.parent_join_request_id.remove(&dated_id) else {
2344 log::error!("parent_join_request_id missing for dated correlation {dated_id}");
2345 return;
2346 };
2347
2348 let Some(original) = self.pending_join_requests.remove(&original_id) else {
2349 log::error!("pending_join_requests missing for original {original_id}");
2350 return;
2351 };
2352
2353 for leg_request_id in &original.request_ids {
2354 let terminal = DataResponseEnvelope::terminal(*leg_request_id, failure_reason.clone());
2355 msgbus::send_response_envelope(&terminal);
2356 }
2357
2358 let final_resp = rebind_response_correlation(resp, original_id);
2364 let envelope = DataResponseEnvelope::new(final_resp, true, None);
2365 self.process_response_envelope(if let Some(reason) = failure_reason {
2366 envelope.with_failure(reason)
2367 } else {
2368 envelope
2369 });
2370 }
2371}
2372
2373fn register_external_streaming_type(cmd: &SubscribeCommand) {
2374 if let Some(payload_type) = streaming_payload_type(cmd) {
2375 msgbus::get_message_bus()
2376 .borrow_mut()
2377 .add_streaming_type(payload_type);
2378 }
2379}
2380
2381fn publish_external_data_command<T>(client_id: ClientId, command: &T)
2382where
2383 T: Any,
2384{
2385 let topic = format!("commands.data.{client_id}");
2386 msgbus::publish_any(topic.into(), command);
2387}
2388
2389#[rustfmt::skip]
2390fn streaming_payload_type(cmd: &SubscribeCommand) -> Option<BusPayloadType> {
2391 match cmd {
2392 SubscribeCommand::Data(cmd) => Some(BusPayloadType::Custom(Ustr::from(
2393 cmd.data_type.type_name(),
2394 ))),
2395 SubscribeCommand::Instrument(_) | SubscribeCommand::Instruments(_) => Some(BusPayloadType::Instrument),
2396 SubscribeCommand::BookDeltas(_) | SubscribeCommand::BookSnapshots(_) => Some(BusPayloadType::OrderBookDeltas),
2397 SubscribeCommand::BookDepth(_) => Some(BusPayloadType::OrderBookDepth),
2398 SubscribeCommand::Quotes(_) => Some(BusPayloadType::QuoteTick),
2399 SubscribeCommand::Trades(_) => Some(BusPayloadType::TradeTick),
2400 SubscribeCommand::Bars(_) => Some(BusPayloadType::Bar),
2401 SubscribeCommand::MarkPrices(_) => Some(BusPayloadType::MarkPriceUpdate),
2402 SubscribeCommand::IndexPrices(_) => Some(BusPayloadType::IndexPriceUpdate),
2403 SubscribeCommand::FundingRates(_) => Some(BusPayloadType::FundingRateUpdate),
2404 SubscribeCommand::OptionGreeks(_) => Some(BusPayloadType::OptionGreeks),
2405 SubscribeCommand::InstrumentStatus(_)
2406 | SubscribeCommand::InstrumentClose(_)
2407 | SubscribeCommand::OptionChain(_) => None,
2408 }
2409}
2410
2411#[cfg(not(feature = "defi"))]
2412fn log_defi_data_dropped(data: DataRef<'_>) {
2413 log::error!("Cannot process data {data:?}, nautilus-data built without its `defi` feature");
2414}