1pub mod bar;
32pub mod book;
33pub mod config;
34
35mod commands;
36mod dispatch;
37mod futures;
38mod handlers;
39mod option_chain;
40mod registry;
41mod requests;
42mod responses;
43mod spread_quote;
44mod synthetic;
45mod time_range;
46
47#[cfg(feature = "defi")]
48pub mod pool;
49
50#[cfg(feature = "streaming")]
51mod streaming;
52
53use std::{
54 any::{Any, type_name},
55 cell::RefCell,
56 collections::VecDeque,
57 fmt::{Debug, Display},
58 mem,
59 num::NonZeroUsize,
60 rc::Rc,
61 str::FromStr,
62};
63
64use ::futures::future::join_all;
65use ahash::{AHashMap, AHashSet};
66use anyhow::Context;
67pub use bar::BarAggregatorSubscription;
68use bar::{BarAggregationSubscription, BarAggregatorKey, BarAggregatorWarmup, bar_aggregator_key};
69use book::{
70 BookDeltasKey, BookSnapshotInfos, BookSnapshotKey, BookSnapshotSource, BookSnapshotter,
71 BookSubscription, BookSubscriptionOwner, BookUpdater,
72};
73pub(crate) use commands::{DeferredCommand, DeferredCommandQueue};
74use config::DataEngineConfig;
75use dispatch::{log_error_on_cache_insert, process_engine_bar};
76use futures::{ContinuousFutureRoller, ContinuousFutureSubscriptionState, datetime_to_unix_nanos};
77use handlers::{
78 BAR_AGGREGATOR_PRIORITY, BarBarHandler, BarQuoteHandler, BarTradeHandler, SpreadQuoteHandler,
79};
80use indexmap::IndexMap;
81#[cfg(feature = "defi")]
82use nautilus_common::messages::defi::PoolSnapshotResponse;
83use nautilus_common::{
84 cache::Cache,
85 clock::Clock,
86 logging::{RECV, RES},
87 messages::data::{
88 BarsResponse, BookDeltasResponse, BookDepthResponse, CustomDataResponse, DataCommand,
89 DataRequestFailed, DataResponse, DataResponseEnvelope, IdentifierCountError,
90 OptionChainReferencePriceResponse, PARAMS_IS_PARENT, QuoteRouting, RequestBars,
91 RequestCommand, RequestJoin, RequestOptionChainReferencePrice, RequestQuotes,
92 RequestTrades, SubscribeBars, SubscribeBookDeltas, SubscribeBookDepth, SubscribeCommand,
93 SubscribeOptionChain, SubscribeOptionGreeks, SubscribeQuotes, SubscribeTrades,
94 UnsubscribeBars, UnsubscribeBookDeltas, UnsubscribeBookDepth, UnsubscribeBookSnapshots,
95 UnsubscribeCommand, UnsubscribeInstrumentStatus, UnsubscribeOptionChain,
96 UnsubscribeOptionGreeks, UnsubscribeQuotes, UnsubscribeTrades, ensure_routable_bar_types,
97 ensure_routable_instrument_ids, is_parent_subscription, quote_routing,
98 },
99 msgbus::{
100 self, BusPayloadType, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
101 get_message_bus,
102 switchboard::{self, MessagingSwitchboard},
103 },
104 runner::get_data_cmd_sender,
105 timer::{TimeEvent, TimeEventCallback},
106};
107use nautilus_core::{
108 DurationNanos, Params, UUID4, UnixNanos, WeakCell,
109 correctness::{FAILED, check_key_in_map, check_key_not_in_map, check_predicate_true},
110 datetime::NANOSECONDS_IN_DAY,
111};
112#[cfg(feature = "defi")]
113use nautilus_model::defi::DefiData;
114use nautilus_model::{
115 data::{
116 Bar, BarType, CustomData, Data, DataRef, DataType, FundingRateUpdate, HasTsInit,
117 IndexPriceUpdate, InstrumentClose, InstrumentStatus, MarkPriceUpdate, OrderBookDelta,
118 OrderBookDeltas, OrderBookDepth, QuoteTick, TradeTick,
119 option_chain::{OptionGreeks, StrikeRange},
120 },
121 enums::{
122 AggregationSource, BarAggregation, BookType, InstrumentClass, MarketStatusAction,
123 PriceType, RecordFlag,
124 },
125 identifiers::{
126 ClientId, GENERIC_SPREAD_ID_SEPARATOR, InstrumentId, OptionSeriesId, Venue,
127 parse_generic_spread_id_legs,
128 },
129 instruments::{Instrument, InstrumentAny, SyntheticInstrument},
130 orderbook::OrderBook,
131 types::{Price, Quantity},
132};
133use option_chain::{
134 OptionChainBootstrapper, OptionChainGreeksBootstrap, PendingOptionChainRequest,
135};
136use requests::{
137 ContinuousFutureRequest, ContinuousFutureRequestState, ContinuousFutureSegment,
138 ContinuousFutureSource, RequestBarAggregation, continuous_future_request_from_bars,
139 continuous_future_subscription_from_bars, has_continuous_future_params,
140 request_bar_aggregation_from_params, request_params, response_params,
141};
142use responses::{
143 log_if_empty_response, parent_request_window, rebind_response_correlation,
144 rebuild_pipeline_response, same_instrument,
145};
146use serde_json::Value;
147use spread_quote::SpreadQuoteState;
148#[cfg(feature = "streaming")]
149use streaming::CatalogMap;
150use time_range::{
151 TimeRangePipelineState, has_time_range_pipeline_params, is_time_range_pipeline_variant,
152};
153use ustr::Ustr;
154
155#[cfg(feature = "defi")]
156#[allow(unused_imports)] use crate::defi::engine as _;
158#[cfg(feature = "defi")]
159use crate::engine::pool::PoolUpdater;
160use crate::{
161 aggregation::{
162 BarAggregator, RenkoBarAggregator, SpreadQuoteAggregator, TickBarAggregator,
163 TickImbalanceBarAggregator, TickRunsBarAggregator, TimeBarAggregator, ValueBarAggregator,
164 ValueImbalanceBarAggregator, ValueRunsBarAggregator, VolumeBarAggregator,
165 VolumeImbalanceBarAggregator, VolumeRunsBarAggregator,
166 },
167 client::DataClientAdapter,
168 option_chains::OptionChainManager,
169 subscription::{
170 SubscriptionKey, SubscriptionRegistry, SubscriptionRelease, split_subscribe,
171 split_unsubscribe,
172 },
173};
174
175#[derive(Debug)]
177pub struct DataEngine {
178 clock: Rc<RefCell<dyn Clock>>,
179 cache: Rc<RefCell<Cache>>,
180 config: DataEngineConfig,
181 is_stopped: bool,
182 msgbus_priority: u32,
183 subscriptions_external: SubscriptionRegistry<(ClientId, SubscriptionKey), SubscribeCommand>,
184 clients: IndexMap<ClientId, DataClientAdapter>,
185 default_client_id: Option<ClientId>,
186 routing_map: IndexMap<Venue, ClientId>,
187 book_intervals: AHashMap<NonZeroUsize, BookSnapshotInfos>,
188 book_snapshot_counts: IndexMap<BookSnapshotKey, usize>,
189 book_snapshot_sources: AHashMap<InstrumentId, BookSnapshotSource>,
190 book_deltas_counts: IndexMap<BookDeltasKey, usize>,
191 book_depth_counts: IndexMap<BookDeltasKey, usize>,
192 book_updaters: AHashMap<InstrumentId, Rc<BookUpdater>>,
193 book_subscriptions: AHashMap<InstrumentId, BookSubscription>,
194 book_subscription_owners: AHashMap<InstrumentId, Vec<Rc<BookSubscriptionOwner>>>,
195 book_snapshotters: AHashMap<NonZeroUsize, Rc<BookSnapshotter>>,
196 bar_aggregators: IndexMap<BarAggregatorKey, Rc<RefCell<Box<dyn BarAggregator>>>>,
197 bar_aggregator_handlers: AHashMap<BarAggregatorKey, Vec<BarAggregatorSubscription>>,
198 bar_aggregator_warmups: AHashMap<BarAggregatorKey, BarAggregatorWarmup>,
199 bar_aggregators_pending_stop: AHashSet<BarAggregatorKey>,
200 subscriptions_bar_aggregation: AHashMap<BarType, BarAggregationSubscription>,
201 request_bar_aggregations: AHashMap<UUID4, RequestBarAggregation>,
202 inflight_requests: AHashMap<UUID4, RequestCommand>,
203 request_pipeline_parent_request: AHashMap<UUID4, RequestCommand>,
204 request_pipeline_n_components: AHashMap<UUID4, usize>,
205 request_pipeline_parent_request_id: AHashMap<UUID4, (UUID4, usize)>,
206 request_pipeline_responses: AHashMap<UUID4, Vec<Option<Vec<DataResponseEnvelope>>>>,
207 time_range_pipeline_requests: AHashMap<UUID4, TimeRangePipelineState>,
208 time_range_pipeline_parent_request_id: AHashMap<UUID4, UUID4>,
209 parent_join_request_id: AHashMap<UUID4, UUID4>,
210 pending_join_requests: AHashMap<UUID4, RequestJoin>,
211 continuous_future_requests: AHashMap<UUID4, ContinuousFutureRequestState>,
212 continuous_future_parent_request_id: AHashMap<UUID4, UUID4>,
213 continuous_future_subscriptions: AHashMap<BarType, ContinuousFutureSubscriptionState>,
214 continuous_future_roller: Option<Rc<ContinuousFutureRoller>>,
215 spread_quote_states: AHashMap<InstrumentId, SpreadQuoteState>,
216 option_chain_managers: AHashMap<OptionSeriesId, Rc<RefCell<OptionChainManager>>>,
217 option_chain_instrument_index: AHashMap<InstrumentId, OptionSeriesId>,
218 deferred_cmd_queue: DeferredCommandQueue,
219 option_chain_bootstrapper: Option<Rc<OptionChainBootstrapper>>,
220 pending_option_chain_requests: AHashMap<UUID4, PendingOptionChainRequest>,
221 option_chain_greeks_bootstraps: AHashMap<OptionSeriesId, OptionChainGreeksBootstrap>,
222 synthetic_quote_feeds: AHashMap<InstrumentId, Vec<SyntheticInstrument>>,
223 synthetic_trade_feeds: AHashMap<InstrumentId, Vec<SyntheticInstrument>>,
224 subscribed_synthetic_quotes: AHashMap<InstrumentId, usize>,
225 subscribed_synthetic_trades: AHashMap<InstrumentId, usize>,
226 buffered_deltas_map: AHashMap<InstrumentId, OrderBookDeltas>,
227 deltas_frame: Vec<OrderBookDelta>,
228 command_count: u64,
229 data_count: u64,
230 request_count: u64,
231 response_count: u64,
232 #[cfg(feature = "streaming")]
233 catalogs: CatalogMap,
234 #[cfg(feature = "defi")]
235 pub(crate) pool_updaters: AHashMap<InstrumentId, Rc<PoolUpdater>>,
236 #[cfg(feature = "defi")]
237 pub(crate) pool_snapshot_pending: AHashMap<InstrumentId, UUID4>,
238 #[cfg(feature = "defi")]
239 pub(crate) pool_event_buffers: AHashMap<InstrumentId, Vec<DefiData>>,
240}
241
242impl DataEngine {
243 #[must_use]
245 pub fn new(
246 clock: Rc<RefCell<dyn Clock>>,
247 cache: Rc<RefCell<Cache>>,
248 config: Option<DataEngineConfig>,
249 ) -> Self {
250 let config = config.unwrap_or_default();
251 if let Some(client_ids) = &config.external_clients {
252 let mut cache = cache.borrow_mut();
253 for client_id in client_ids {
254 cache.add_external_data_client(*client_id);
255 }
256 }
257
258 Self {
259 clock,
260 cache,
261 config,
262 is_stopped: false,
263 msgbus_priority: 10, subscriptions_external: SubscriptionRegistry::default(),
265 clients: IndexMap::new(),
266 default_client_id: None,
267 routing_map: IndexMap::new(),
268 book_intervals: AHashMap::new(),
269 book_snapshot_counts: IndexMap::new(),
270 book_snapshot_sources: AHashMap::new(),
271 book_deltas_counts: IndexMap::new(),
272 book_depth_counts: IndexMap::new(),
273 book_updaters: AHashMap::new(),
274 book_subscriptions: AHashMap::new(),
275 book_subscription_owners: AHashMap::new(),
276 book_snapshotters: AHashMap::new(),
277 bar_aggregators: IndexMap::new(),
278 bar_aggregator_handlers: AHashMap::new(),
279 bar_aggregator_warmups: AHashMap::new(),
280 bar_aggregators_pending_stop: AHashSet::new(),
281 subscriptions_bar_aggregation: AHashMap::new(),
282 request_bar_aggregations: AHashMap::new(),
283 inflight_requests: AHashMap::new(),
284 request_pipeline_parent_request: AHashMap::new(),
285 request_pipeline_n_components: AHashMap::new(),
286 request_pipeline_parent_request_id: AHashMap::new(),
287 request_pipeline_responses: AHashMap::new(),
288 time_range_pipeline_requests: AHashMap::new(),
289 time_range_pipeline_parent_request_id: AHashMap::new(),
290 parent_join_request_id: AHashMap::new(),
291 pending_join_requests: AHashMap::new(),
292 continuous_future_requests: AHashMap::new(),
293 continuous_future_parent_request_id: AHashMap::new(),
294 continuous_future_subscriptions: AHashMap::new(),
295 continuous_future_roller: None,
296 spread_quote_states: AHashMap::new(),
297 option_chain_managers: AHashMap::new(),
298 option_chain_instrument_index: AHashMap::new(),
299 deferred_cmd_queue: Rc::new(RefCell::new(VecDeque::new())),
300 option_chain_bootstrapper: None,
301 pending_option_chain_requests: AHashMap::new(),
302 option_chain_greeks_bootstraps: AHashMap::new(),
303 synthetic_quote_feeds: AHashMap::new(),
304 synthetic_trade_feeds: AHashMap::new(),
305 subscribed_synthetic_quotes: AHashMap::new(),
306 subscribed_synthetic_trades: AHashMap::new(),
307 buffered_deltas_map: AHashMap::new(),
308 deltas_frame: Vec::new(),
309 command_count: 0,
310 data_count: 0,
311 request_count: 0,
312 response_count: 0,
313 #[cfg(feature = "streaming")]
314 catalogs: CatalogMap::new(),
315 #[cfg(feature = "defi")]
316 pool_updaters: AHashMap::new(),
317 #[cfg(feature = "defi")]
318 pool_snapshot_pending: AHashMap::new(),
319 #[cfg(feature = "defi")]
320 pool_event_buffers: AHashMap::new(),
321 }
322 }
323
324 #[must_use]
326 pub fn clock(&self) -> &Rc<RefCell<dyn Clock>> {
327 &self.clock
328 }
329
330 #[must_use]
332 pub fn cache(&self) -> &Rc<RefCell<Cache>> {
333 &self.cache
334 }
335
336 #[must_use]
338 pub const fn config(&self) -> &DataEngineConfig {
339 &self.config
340 }
341
342 #[cfg(feature = "defi")]
343 #[must_use]
344 pub(crate) const fn msgbus_priority(&self) -> u32 {
345 self.msgbus_priority
346 }
347
348 pub fn register_msgbus_handlers(engine: &Rc<RefCell<Self>>) {
352 let weak = WeakCell::from(Rc::downgrade(engine));
353 engine.borrow_mut().continuous_future_roller =
354 Some(Rc::new(ContinuousFutureRoller::new(engine)));
355 engine.borrow_mut().option_chain_bootstrapper =
356 Some(Rc::new(OptionChainBootstrapper::new(engine)));
357
358 let weak1 = weak.clone();
359 msgbus::register_data_command_endpoint(
360 MessagingSwitchboard::data_engine_execute(),
361 TypedIntoHandler::from(move |cmd: DataCommand| {
362 if let Some(rc) = weak1.upgrade() {
363 rc.borrow_mut().execute(cmd);
364 }
365 }),
366 );
367
368 msgbus::register_data_command_endpoint(
369 MessagingSwitchboard::data_engine_queue_execute(),
370 TypedIntoHandler::from(move |cmd: DataCommand| {
371 get_data_cmd_sender().execute(cmd);
372 }),
373 );
374
375 let weak2 = weak.clone();
377 msgbus::register_any(
378 MessagingSwitchboard::data_engine_process(),
379 ShareableMessageHandler::from_any(move |data: &dyn Any| {
380 if let Some(rc) = weak2.upgrade() {
381 rc.borrow_mut().process(data);
382 }
383 }),
384 );
385
386 let weak3 = weak.clone();
388 msgbus::register_data_endpoint(
389 MessagingSwitchboard::data_engine_process_data(),
390 TypedIntoHandler::from(move |data: Data| {
391 if let Some(rc) = weak3.upgrade() {
392 rc.borrow_mut().process_data(data);
393 }
394 }),
395 );
396
397 let external_data_ingress = engine.borrow().cache.borrow().has_external_data_clients();
398 get_message_bus()
399 .borrow_mut()
400 .set_external_data_ingress(external_data_ingress);
401
402 #[cfg(feature = "defi")]
404 {
405 let weak4 = weak.clone();
406 msgbus::register_defi_data_endpoint(
407 MessagingSwitchboard::data_engine_process_defi_data(),
408 TypedIntoHandler::from(move |data: DefiData| {
409 if let Some(rc) = weak4.upgrade() {
410 rc.borrow_mut().process_defi_data(data);
411 }
412 }),
413 );
414 }
415
416 let weak5 = weak;
417 msgbus::register_data_response_endpoint(
418 MessagingSwitchboard::data_engine_response(),
419 TypedIntoHandler::from(move |resp: DataResponse| {
420 if let Some(rc) = weak5.upgrade() {
421 rc.borrow_mut().response(resp);
422 }
423 }),
424 );
425 }
426
427 #[must_use]
429 pub const fn command_count(&self) -> u64 {
430 self.command_count
431 }
432
433 #[must_use]
435 pub const fn data_count(&self) -> u64 {
436 self.data_count
437 }
438
439 #[cfg(feature = "defi")]
440 pub(crate) const fn increment_data_count(&mut self) {
441 self.data_count += 1;
442 }
443
444 #[must_use]
446 pub const fn request_count(&self) -> u64 {
447 self.request_count
448 }
449
450 #[must_use]
452 pub const fn response_count(&self) -> u64 {
453 self.response_count
454 }
455
456 #[must_use]
458 pub fn request_pipeline_count(&self) -> usize {
459 self.request_pipeline_parent_request.len()
460 }
461
462 #[must_use]
464 pub fn time_range_pipeline_count(&self) -> usize {
465 self.time_range_pipeline_requests.len()
466 }
467
468 #[must_use]
470 pub fn pending_join_request_count(&self) -> usize {
471 self.pending_join_requests.len()
472 }
473
474 pub fn start(&mut self) {
476 self.is_stopped = false;
477
478 for client in self.get_clients_mut() {
479 client.is_stopped = false;
480 if client.start().is_err() {
481 log::error!("Failed to start data client {}", client.client_id());
482 }
483 }
484
485 for ((_, request_id), aggregator) in &self.bar_aggregators {
486 let is_subscription = request_id.is_none() && !aggregator.borrow().is_historical();
489 if is_subscription && aggregator.borrow().bar_type().spec().is_time_aggregated() {
490 aggregator
491 .borrow_mut()
492 .start_timer(Some(Rc::clone(aggregator)));
493 }
494 }
495
496 for state in self.spread_quote_states.values() {
497 state
498 .aggregator
499 .borrow_mut()
500 .start_timer(Some(Rc::clone(&state.aggregator)));
501 }
502 }
503
504 pub fn stop(&mut self) {
510 self.is_stopped = true;
511
512 for client in self.get_clients_mut() {
513 client.is_stopped = true;
514 if client.stop().is_err() {
515 log::error!("Failed to stop data client {}", client.client_id());
516 }
517 }
518
519 for aggregator in self.bar_aggregators.values() {
520 aggregator.borrow_mut().stop();
521 }
522
523 for state in self.spread_quote_states.values() {
524 state.aggregator.borrow_mut().stop_timer();
525 }
526 }
527
528 pub fn reset(&mut self) {
530 self.is_stopped = false;
531
532 for client in self.get_clients_mut() {
533 client.is_stopped = false;
534 match client.reset() {
535 Ok(()) => client.clear_subscription_state(),
536 Err(_) => log::error!("Failed to reset data client {}", client.client_id()),
537 }
538 }
539
540 self.clear_bar_aggregators();
541 self.clear_request_state();
542
543 for state in self.continuous_future_subscriptions.values_mut() {
544 if let Some(name) = state.timer_name.take() {
545 self.clock.borrow_mut().cancel_timer(&name);
546 }
547 }
548 self.continuous_future_subscriptions.clear();
549
550 let spread_ids: Vec<InstrumentId> = self.spread_quote_states.keys().copied().collect();
551 for spread_id in spread_ids {
552 self.stop_spread_quote_aggregation(spread_id);
553 }
554
555 self.clear_option_chain_managers();
556
557 self.clear_book_state();
558
559 self.synthetic_quote_feeds.clear();
560 self.synthetic_trade_feeds.clear();
561 self.subscribed_synthetic_quotes.clear();
562 self.subscribed_synthetic_trades.clear();
563 self.subscriptions_external.clear();
564
565 self.deferred_cmd_queue.borrow_mut().clear();
566
567 #[cfg(feature = "defi")]
568 self.clear_pool_updaters();
569
570 self.clock.borrow_mut().cancel_timers();
571
572 self.command_count = 0;
573 self.data_count = 0;
574 self.request_count = 0;
575 self.response_count = 0;
576 }
577
578 fn clear_request_state(&mut self) {
579 self.request_bar_aggregations.clear();
580 self.inflight_requests.clear();
581 self.request_pipeline_parent_request.clear();
582 self.request_pipeline_n_components.clear();
583 self.request_pipeline_parent_request_id.clear();
584 self.request_pipeline_responses.clear();
585 self.time_range_pipeline_requests.clear();
586 self.time_range_pipeline_parent_request_id.clear();
587 self.parent_join_request_id.clear();
588 self.pending_join_requests.clear();
589 self.continuous_future_requests.clear();
590 self.continuous_future_parent_request_id.clear();
591 }
592
593 pub fn dispose(&mut self) {
595 for client in self.get_clients_mut() {
596 if client.dispose().is_err() {
597 log::error!("Failed to dispose data client {}", client.client_id());
598 }
599 }
600
601 self.clear_option_chain_greeks_bootstraps();
602
603 let mut cf_sources = Vec::new();
606
607 for state in self.continuous_future_subscriptions.values_mut() {
608 if let Some(name) = state.timer_name.take() {
609 self.clock.borrow_mut().cancel_timer(&name);
610 }
611
612 if let Some(subscription) = state.active_source_subscription.take() {
613 cf_sources.push((state.target_bar_type, subscription));
614 }
615 }
616
617 for (target_bar_type, subscription) in cf_sources {
618 self.unsubscribe_continuous_future_source(target_bar_type, subscription);
619 }
620 self.continuous_future_subscriptions.clear();
621
622 let keys: Vec<BarAggregatorKey> = self.bar_aggregators.keys().copied().collect();
625 for (bar_type, request_id) in keys {
626 if let Err(e) = self.stop_bar_aggregator(bar_type, request_id) {
627 log::error!("Error stopping bar aggregator during dispose for {bar_type}: {e}");
628 }
629 }
630
631 self.subscriptions_external.clear();
632
633 self.clock.borrow_mut().cancel_timers();
634 }
635
636 pub async fn connect(&mut self) -> anyhow::Result<()> {
646 let futures: Vec<_> = self
647 .get_clients_mut()
648 .into_iter()
649 .map(|client| async move {
650 let client_id = client.client_id();
651 (client_id, client.connect().await)
652 })
653 .collect();
654
655 let results = join_all(futures).await;
656 let mut failed_clients = Vec::new();
657
658 for (client_id, result) in results {
659 if result.is_err() {
660 log::error!("Failed to connect data client {client_id}: connection failed");
661 failed_clients.push(client_id.to_string());
662 }
663 }
664
665 if failed_clients.is_empty() {
666 Ok(())
667 } else {
668 anyhow::bail!(
669 "Failed to connect data clients: {}",
670 failed_clients.join(", ")
671 )
672 }
673 }
674
675 pub async fn disconnect(&mut self) -> anyhow::Result<()> {
681 let futures: Vec<_> = self
682 .get_clients_mut()
683 .into_iter()
684 .map(|client| async move {
685 let client_id = client.client_id();
686 (client_id, client.disconnect().await)
687 })
688 .collect();
689
690 let results = join_all(futures).await;
691
692 #[cfg(feature = "defi")]
694 self.abandon_pool_snapshots();
695
696 let failed_clients: Vec<_> = results
697 .into_iter()
698 .filter_map(|(client_id, result)| result.err().map(|_| client_id.to_string()))
699 .collect();
700
701 if failed_clients.is_empty() {
702 Ok(())
703 } else {
704 anyhow::bail!(
705 "Failed to disconnect data clients: {}",
706 failed_clients.join(", ")
707 )
708 }
709 }
710
711 #[must_use]
713 pub fn check_connected(&self) -> bool {
714 self.get_clients()
715 .iter()
716 .all(|client| client.is_connected())
717 }
718
719 #[must_use]
721 pub fn check_disconnected(&self) -> bool {
722 self.get_clients()
723 .iter()
724 .all(|client| !client.is_connected())
725 }
726
727 #[must_use]
729 pub fn subscribed_custom_data(&self) -> Vec<DataType> {
730 self.collect_subscriptions(|client| &client.subscriptions_custom)
731 }
732
733 #[must_use]
735 pub fn subscribed_instruments(&self) -> Vec<InstrumentId> {
736 self.collect_subscriptions(|client| &client.subscriptions_instrument)
737 }
738
739 #[must_use]
741 pub fn subscribed_quotes(&self) -> Vec<InstrumentId> {
742 self.collect_subscriptions(|client| &client.subscriptions_quotes)
743 }
744
745 #[must_use]
747 pub fn subscribed_trades(&self) -> Vec<InstrumentId> {
748 self.collect_subscriptions(|client| &client.subscriptions_trades)
749 }
750
751 #[must_use]
754 pub fn subscribed_bars(&self) -> Vec<BarType> {
755 let mut subscribed = self.collect_subscriptions(|client| &client.subscriptions_bars);
756 subscribed.extend(
757 self.bar_aggregators
758 .keys()
759 .filter(|key| key.1.is_none() && !self.bar_aggregators_pending_stop.contains(*key))
760 .map(|(bar_type, _)| *bar_type),
761 );
762 subscribed
763 }
764
765 #[must_use]
767 pub fn subscribed_mark_prices(&self) -> Vec<InstrumentId> {
768 self.collect_subscriptions(|client| &client.subscriptions_mark_prices)
769 }
770
771 #[must_use]
773 pub fn subscribed_index_prices(&self) -> Vec<InstrumentId> {
774 self.collect_subscriptions(|client| &client.subscriptions_index_prices)
775 }
776
777 #[must_use]
779 pub fn subscribed_funding_rates(&self) -> Vec<InstrumentId> {
780 self.collect_subscriptions(|client| &client.subscriptions_funding_rates)
781 }
782
783 #[must_use]
785 pub fn subscribed_instrument_status(&self) -> Vec<InstrumentId> {
786 self.collect_subscriptions(|client| &client.subscriptions_instrument_status)
787 }
788
789 #[must_use]
791 pub fn subscribed_instrument_close(&self) -> Vec<InstrumentId> {
792 self.collect_subscriptions(|client| &client.subscriptions_instrument_close)
793 }
794
795 pub fn execute(&mut self, cmd: DataCommand) {
804 match &cmd {
805 DataCommand::Subscribe(_) | DataCommand::Unsubscribe(_) => self.command_count += 1,
806 DataCommand::Request(_) => self.request_count += 1,
807 #[cfg(feature = "defi")]
808 DataCommand::DefiRequest(_) => self.request_count += 1,
809 #[cfg(feature = "defi")]
810 DataCommand::DefiSubscribe(_) | DataCommand::DefiUnsubscribe(_) => {
811 self.command_count += 1;
812 }
813 _ => {}
814 }
815
816 let command_context = match &cmd {
817 DataCommand::Subscribe(c) => (
818 "Subscribe",
819 c.client_id().copied(),
820 c.venue().copied(),
821 Some(c.command_id()),
822 ),
823 DataCommand::Unsubscribe(c) => (
824 "Unsubscribe",
825 c.client_id().copied(),
826 c.venue().copied(),
827 Some(c.command_id()),
828 ),
829 DataCommand::Request(c) => (
830 "Request",
831 c.client_id().copied(),
832 c.venue().copied(),
833 Some(*c.request_id()),
834 ),
835 #[cfg(feature = "defi")]
836 DataCommand::DefiRequest(c) => (
837 "DefiRequest",
838 c.client_id().copied(),
839 c.venue().copied(),
840 Some(*c.request_id()),
841 ),
842 #[cfg(feature = "defi")]
843 DataCommand::DefiSubscribe(c) => (
844 "DefiSubscribe",
845 c.client_id().copied(),
846 c.venue().copied(),
847 Some(c.command_id()),
848 ),
849 #[cfg(feature = "defi")]
850 DataCommand::DefiUnsubscribe(c) => (
851 "DefiUnsubscribe",
852 c.client_id().copied(),
853 c.venue().copied(),
854 Some(c.command_id()),
855 ),
856 _ => ("Unknown", None, None, None),
857 };
858
859 let result = match cmd {
860 DataCommand::Subscribe(c) => self.execute_subscribe(c),
861 DataCommand::Unsubscribe(c) => self.execute_unsubscribe(&c),
862 DataCommand::Request(c) => self.execute_request(c),
863 #[cfg(feature = "defi")]
864 DataCommand::DefiRequest(c) => self.execute_defi_request(c),
865 #[cfg(feature = "defi")]
866 DataCommand::DefiSubscribe(c) => self.execute_defi_subscribe(c),
867 #[cfg(feature = "defi")]
868 DataCommand::DefiUnsubscribe(c) => self.execute_defi_unsubscribe(&c),
869 _ => {
870 log::warn!("Unhandled DataCommand variant");
871 Ok(())
872 }
873 };
874
875 if let Err(e) = result {
876 log::error!("Data command failed: {command_context:?}: {e}");
877 }
878 }
879
880 pub fn execute_subscribe(&mut self, cmd: SubscribeCommand) -> anyhow::Result<()> {
887 cmd.validate_routing(&self.cache.borrow())?;
888 if let Some(client_id) = cmd.client_id().copied()
889 && self.is_external_client(client_id)
890 {
891 return self.subscribe_external(client_id, &cmd);
892 }
893
894 self.retain_intent(&cmd)?;
895
896 let retained = cmd.clone();
897 let (groups, result) = self.prepare_subscribe_groups(cmd);
898 self.dispatch_subscribe_groups(&retained, groups);
899 result
900 }
901
902 fn retain_intent(&mut self, cmd: &SubscribeCommand) -> anyhow::Result<()> {
903 let retain_intent = match cmd {
904 SubscribeCommand::Quotes(command) => {
905 self.quote_routing(&command.instrument_ids, command.params.as_ref())?
906 == QuoteRouting::Client
907 }
908 SubscribeCommand::Trades(command) => !command.instrument_ids[0].is_synthetic(),
909 _ => false,
910 };
911
912 if let Some(client) = self.get_command_client(cmd.client_id(), cmd.venue()) {
913 if let SubscribeCommand::Bars(command) = cmd {
914 let external: Vec<_> = command
915 .bar_types
916 .iter()
917 .copied()
918 .filter(BarType::is_externally_aggregated)
919 .collect();
920
921 if !external.is_empty() {
922 let intent = SubscribeCommand::Bars(command.with_bar_types(external));
923 client.retain_intent(&intent);
924 }
925 } else if retain_intent {
926 client.retain_intent(cmd);
927 }
928 }
929
930 Ok(())
931 }
932
933 fn prepare_subscribe_groups(
934 &mut self,
935 cmd: SubscribeCommand,
936 ) -> (Vec<SubscribeCommand>, anyhow::Result<()>) {
937 let members = if matches!(
938 &cmd,
939 SubscribeCommand::Quotes(_) | SubscribeCommand::Trades(_)
940 ) {
941 vec![cmd]
942 } else {
943 split_subscribe(cmd)
944 };
945
946 let mut groups: IndexMap<Option<Value>, Vec<SubscribeCommand>> = IndexMap::new();
947 let mut result = Ok(());
948
949 for member in members {
950 let prepared = match self.prepare_subscribe_member(&member) {
951 Ok(Some(prepared)) => prepared,
952 Ok(None) => continue,
953 Err(e) => {
954 log::error!(
955 "Subscribe preparation failed: command={}, client={:?}, instrument_ids={:?}, bar_types={:?}: {e}",
956 member.command_id(),
957 member.client_id(),
958 member.instrument_ids(),
959 member.bar_types(),
960 );
961
962 if result.is_ok() {
963 result = Err(e);
964 }
965
966 continue;
967 }
968 };
969
970 let params = prepared.params().map(|params| {
972 Value::Object(params.iter().map(|(k, v)| (k.clone(), v.clone())).collect())
973 });
974
975 groups.entry(params).or_default().push(prepared);
976 }
977
978 let groups = groups
979 .into_values()
980 .map(|mut members| {
981 if members.len() == 1 {
982 return members.pop().expect("nonempty group");
983 }
984
985 let first = &members[0];
986 if first.instrument_ids().is_some() {
987 let ids: Vec<_> = members
988 .iter()
989 .flat_map(|member| member.instrument_ids().expect("same parent kind"))
990 .copied()
991 .collect();
992 first.with_instrument_ids(ids).expect("instrument command")
993 } else if first.bar_types().is_some() {
994 let types: Vec<_> = members
995 .iter()
996 .flat_map(|member| member.bar_types().expect("same parent kind"))
997 .copied()
998 .collect();
999 first.with_bar_types(types).expect("bar command")
1000 } else {
1001 first.clone()
1002 }
1003 })
1004 .collect();
1005
1006 (groups, result)
1007 }
1008
1009 fn dispatch_subscribe_groups(&mut self, cmd: &SubscribeCommand, groups: Vec<SubscribeCommand>) {
1010 for group in groups {
1011 let retained = if let Some(ids) = group.instrument_ids() {
1012 cmd.with_instrument_ids(ids).expect("instrument parent")
1013 } else if let Some(types) = group.bar_types() {
1014 cmd.with_bar_types(types).expect("bar parent")
1015 } else {
1016 cmd.clone()
1017 };
1018
1019 let retain_on_failure = !matches!(
1020 &group,
1021 SubscribeCommand::BookDeltas(_) | SubscribeCommand::BookDepth(_)
1022 );
1023
1024 if let Some(client) = self.get_command_client(group.client_id(), group.venue()) {
1025 client.execute_subscribe_with_retained(group, retained, retain_on_failure);
1026 } else {
1027 log::error!(
1028 "Cannot handle command: no client found for client_id={:?}, venue={:?}",
1029 group.client_id(),
1030 group.venue()
1031 );
1032 }
1033 }
1034 }
1035
1036 fn prepare_subscribe_member(
1037 &mut self,
1038 cmd: &SubscribeCommand,
1039 ) -> anyhow::Result<Option<SubscribeCommand>> {
1040 match cmd {
1042 SubscribeCommand::BookDeltas(book_cmd) => {
1043 if !self.subscribe_book_deltas(book_cmd)? && self.client_subscription_active(cmd) {
1044 return Ok(None);
1045 }
1046 }
1047 SubscribeCommand::BookDepth(book_cmd) => {
1048 if !self.subscribe_book_depth(book_cmd)? && self.client_subscription_active(cmd) {
1049 return Ok(None);
1050 }
1051 }
1052 SubscribeCommand::BookSnapshots(cmd) => {
1053 self.subscribe_book_snapshots(cmd)?;
1055 return Ok(None);
1056 }
1057 SubscribeCommand::Bars(cmd) if has_continuous_future_params(cmd.params.as_ref()) => {
1058 self.subscribe_continuous_future_bars(cmd)?;
1059 return Ok(None);
1060 }
1061 SubscribeCommand::Bars(cmd) => {
1062 self.subscribe_bars(cmd)?;
1063 if cmd
1064 .single_bar_type()
1065 .expect("member preparation receives one bar type")
1066 .is_internally_aggregated()
1067 {
1068 return Ok(None);
1069 }
1070 }
1071 SubscribeCommand::OptionChain(cmd) => {
1072 self.subscribe_option_chain(cmd);
1073 return Ok(None);
1074 }
1075 SubscribeCommand::Quotes(cmd) if self.subscribe_quotes(cmd)? => {
1076 return Ok(None);
1077 }
1078 SubscribeCommand::Trades(cmd) if self.subscribe_trades(cmd) => {
1079 return Ok(None);
1080 }
1081 SubscribeCommand::Instrument(cmd)
1082 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1083 {
1084 anyhow::bail!("Cannot subscribe for synthetic instrument `Instrument` data");
1085 }
1086 SubscribeCommand::InstrumentStatus(cmd)
1087 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1088 {
1089 anyhow::bail!("Cannot subscribe for synthetic instrument `InstrumentStatus` data");
1090 }
1091 SubscribeCommand::InstrumentClose(cmd)
1092 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1093 {
1094 anyhow::bail!("Cannot subscribe for synthetic instrument `InstrumentClose` data");
1095 }
1096 SubscribeCommand::OptionGreeks(cmd)
1097 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1098 {
1099 anyhow::bail!("Cannot subscribe for synthetic instrument `OptionGreeks` data");
1100 }
1101 _ => {} }
1103
1104 let cmd = cmd.clone();
1105 #[cfg(feature = "streaming")]
1106 let cmd = self.subscribe_command_with_prefilled_start_ns(cmd)?;
1107
1108 Ok(Some(cmd))
1109 }
1110
1111 fn client_subscription_active(&mut self, cmd: &SubscribeCommand) -> bool {
1112 self.get_command_client(cmd.client_id(), cmd.venue())
1113 .is_some_and(|client| client.has_active_subscription(cmd))
1114 }
1115
1116 pub fn execute_unsubscribe(&mut self, cmd: &UnsubscribeCommand) -> anyhow::Result<()> {
1122 if let Some(bar_types) = cmd.bar_types() {
1123 ensure_routable_bar_types(bar_types, cmd.client_id().is_some())?;
1124 }
1125
1126 if let Some(ids) = cmd.instrument_ids() {
1127 ensure_routable_instrument_ids(ids, cmd.client_id().is_some())?;
1128 }
1129
1130 if let UnsubscribeCommand::Trades(trade) = cmd
1131 && !trade
1132 .instrument_ids
1133 .iter()
1134 .all(|id| id.is_synthetic() == trade.instrument_ids[0].is_synthetic())
1135 {
1136 anyhow::bail!("trade identifier lists cannot mix client and synthetic routes");
1137 }
1138
1139 if let Some(client_id) = cmd.client_id().copied()
1140 && self.is_external_client(client_id)
1141 {
1142 self.unsubscribe_external(client_id, cmd);
1143 return Ok(());
1144 }
1145
1146 if matches!(
1147 &cmd,
1148 UnsubscribeCommand::Instrument(_)
1149 | UnsubscribeCommand::InstrumentStatus(_)
1150 | UnsubscribeCommand::InstrumentClose(_)
1151 | UnsubscribeCommand::OptionGreeks(_)
1152 | UnsubscribeCommand::BookDeltas(_)
1153 | UnsubscribeCommand::BookDepth(_)
1154 | UnsubscribeCommand::BookSnapshots(_)
1155 ) && cmd
1156 .instrument_ids()
1157 .is_some_and(|ids| ids.len() > 1 && ids.iter().any(InstrumentId::is_synthetic))
1158 {
1159 anyhow::bail!("Cannot unsubscribe synthetic instruments for this data type");
1160 }
1161
1162 let (prepared, result) = self.prepare_unsubscribe_members(cmd);
1163
1164 for command in UnsubscribeCommand::group(prepared) {
1165 if let Some(client) = self.get_command_client(command.client_id(), command.venue()) {
1166 client.execute_unsubscribe(&command);
1167 } else {
1168 log::error!(
1169 "Cannot handle command: no client found for client_id={:?}, venue={:?}",
1170 command.client_id(),
1171 command.venue()
1172 );
1173 }
1174 }
1175
1176 result
1177 }
1178
1179 fn prepare_unsubscribe_members(
1180 &mut self,
1181 cmd: &UnsubscribeCommand,
1182 ) -> (Vec<UnsubscribeCommand>, anyhow::Result<()>) {
1183 let members = if matches!(
1184 cmd,
1185 UnsubscribeCommand::Quotes(_) | UnsubscribeCommand::Trades(_)
1186 ) {
1187 vec![cmd.clone()]
1188 } else {
1189 split_unsubscribe(cmd)
1190 };
1191
1192 let mut prepared = Vec::new();
1193 let mut result = Ok(());
1194
1195 for member in members {
1196 match self.prepare_unsubscribe_member(&member) {
1197 Ok(true) => prepared.push(member),
1198 Ok(false) => {}
1199 Err(e) => {
1200 log::error!(
1201 "Unsubscribe preparation failed: command={}, client={:?}, instrument_ids={:?}, bar_types={:?}: {e}",
1202 member.command_id(),
1203 member.client_id(),
1204 member.instrument_ids(),
1205 member.bar_types(),
1206 );
1207
1208 if result.is_ok() {
1209 result = Err(e);
1210 }
1211 }
1212 }
1213 }
1214
1215 (prepared, result)
1216 }
1217
1218 fn prepare_unsubscribe_member(&mut self, cmd: &UnsubscribeCommand) -> anyhow::Result<bool> {
1219 if matches!(
1220 cmd,
1221 UnsubscribeCommand::BookDeltas(_)
1222 | UnsubscribeCommand::BookDepth(_)
1223 | UnsubscribeCommand::BookSnapshots(_)
1224 ) && !self.release_book_subscription(cmd)
1225 {
1226 return Ok(false);
1227 }
1228
1229 match &cmd {
1230 UnsubscribeCommand::BookDeltas(cmd) if !self.unsubscribe_book_deltas(cmd) => {
1231 return Ok(false);
1232 }
1233 UnsubscribeCommand::BookDepth(cmd) if !self.unsubscribe_book_depth(cmd) => {
1234 return Ok(false);
1235 }
1236 UnsubscribeCommand::BookSnapshots(cmd) => {
1237 self.unsubscribe_book_snapshots(cmd);
1239 return Ok(false);
1240 }
1241 UnsubscribeCommand::Bars(cmd)
1242 if self.continuous_future_subscriptions.contains_key(
1243 &cmd.single_bar_type()
1244 .expect("member preparation receives one bar type")
1245 .standard(),
1246 ) =>
1247 {
1248 let topic = switchboard::get_bars_topic(
1250 cmd.single_bar_type()
1251 .expect("member preparation receives one bar type")
1252 .standard(),
1253 );
1254
1255 if msgbus::exact_subscriber_count_bars(topic) == 0 {
1256 self.unsubscribe_continuous_future_bars(cmd);
1257 }
1258
1259 return Ok(false);
1260 }
1261 UnsubscribeCommand::Bars(cmd) => {
1262 self.unsubscribe_bars(cmd);
1263 if cmd
1264 .single_bar_type()
1265 .expect("member preparation receives one bar type")
1266 .is_internally_aggregated()
1267 {
1268 return Ok(false);
1269 }
1270 }
1271 UnsubscribeCommand::OptionChain(cmd) => {
1272 self.unsubscribe_option_chain(cmd);
1273 return Ok(false);
1274 }
1275 UnsubscribeCommand::Quotes(cmd) if self.unsubscribe_quotes(cmd)? => {
1276 return Ok(false);
1277 }
1278 UnsubscribeCommand::Trades(cmd) if self.unsubscribe_trades(cmd) => {
1279 return Ok(false);
1280 }
1281 UnsubscribeCommand::Instrument(cmd)
1282 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1283 {
1284 anyhow::bail!("Cannot unsubscribe from synthetic instrument `Instrument` data");
1285 }
1286 UnsubscribeCommand::InstrumentStatus(cmd)
1287 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1288 {
1289 anyhow::bail!(
1290 "Cannot unsubscribe from synthetic instrument `InstrumentStatus` data"
1291 );
1292 }
1293 UnsubscribeCommand::InstrumentClose(cmd)
1294 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1295 {
1296 anyhow::bail!(
1297 "Cannot unsubscribe from synthetic instrument `InstrumentClose` data"
1298 );
1299 }
1300 UnsubscribeCommand::OptionGreeks(cmd)
1301 if cmd.instrument_ids.iter().any(InstrumentId::is_synthetic) =>
1302 {
1303 anyhow::bail!("Cannot unsubscribe from synthetic instrument `OptionGreeks` data");
1304 }
1305 _ => {}
1306 }
1307
1308 Ok(true)
1309 }
1310
1311 fn subscribe_external(
1312 &mut self,
1313 client_id: ClientId,
1314 cmd: &SubscribeCommand,
1315 ) -> anyhow::Result<()> {
1316 let (dispatch, result) = match cmd {
1317 SubscribeCommand::OptionChain(command) => {
1318 self.retain_external_option_chain(client_id, command, cmd);
1319 (Some(cmd.clone()), Ok(()))
1320 }
1321 _ => self.prepare_external_subscribe(client_id, cmd),
1322 };
1323
1324 let Some(dispatch) = dispatch else {
1325 return result;
1326 };
1327
1328 if self.is_stopped {
1329 log::debug!(
1330 "Skipped external subscribe after stop: client_id={client_id}, command_id={}",
1331 dispatch.command_id()
1332 );
1333 return result;
1334 }
1335
1336 register_external_streaming_type(&dispatch);
1337 publish_external_data_command(client_id, &dispatch);
1338
1339 if self.config.debug {
1340 log::debug!("Skipping subscribe command for external client {client_id}: {cmd:?}");
1341 }
1342
1343 result
1344 }
1345
1346 fn prepare_external_subscribe(
1347 &mut self,
1348 client_id: ClientId,
1349 cmd: &SubscribeCommand,
1350 ) -> (Option<SubscribeCommand>, anyhow::Result<()>) {
1351 let mut result = Ok(());
1352 let mut acquired = false;
1353 let mut fresh_ids = Vec::new();
1354 let mut fresh_bar_types = Vec::new();
1355
1356 for member in split_subscribe(cmd.clone()) {
1357 let key = SubscriptionKey::from_subscribe(&member);
1358 let instrument_id = member.instrument_ids().and_then(|ids| ids.first()).copied();
1359 let bar_type = member.bar_types().and_then(|types| types.first()).copied();
1360
1361 match self.retain_external_subscription(client_id, key.clone(), member) {
1362 Ok(false) => continue,
1363 Ok(true) => {}
1364 Err(e) => {
1365 log::error!(
1366 "External subscribe preparation failed: command={}, client={client_id}, subscription={}: {e}",
1367 cmd.command_id(),
1368 key.log_context(),
1369 );
1370
1371 if result.is_ok() {
1372 result = Err(e);
1373 }
1374
1375 continue;
1376 }
1377 }
1378
1379 acquired = true;
1380 fresh_ids.extend(instrument_id);
1381 fresh_bar_types.extend(bar_type);
1382 }
1383
1384 if !acquired {
1385 return (None, result);
1386 }
1387
1388 let dispatch = if cmd.bar_types().is_some() {
1389 cmd.with_bar_types(fresh_bar_types)
1390 .expect("bar command has types")
1391 } else {
1392 cmd.with_instrument_ids(fresh_ids)
1393 .unwrap_or_else(|| cmd.clone())
1394 };
1395
1396 (Some(dispatch), result)
1397 }
1398
1399 fn retain_external_subscription(
1400 &mut self,
1401 client_id: ClientId,
1402 key: SubscriptionKey,
1403 command: SubscribeCommand,
1404 ) -> anyhow::Result<bool> {
1405 let key = (client_id, key);
1406
1407 if self
1408 .subscriptions_external
1409 .get_mut(&key)
1410 .is_some_and(|active| active.acquisitions.contains(&command.command_id()))
1411 {
1412 return Ok(false);
1413 }
1414
1415 let params = self.subscriptions_external.get_mut(&key).map_or_else(
1417 || command.params().cloned(),
1418 |active| active.command.params().cloned(),
1419 );
1420
1421 match &command {
1422 SubscribeCommand::BookDeltas(command) => {
1423 let mut command = command.clone();
1424 command.params = params;
1425 self.subscribe_book_deltas(&command)?;
1426 }
1427 SubscribeCommand::BookDepth(command) => {
1428 let mut command = command.clone();
1429 command.params = params;
1430 self.subscribe_book_depth(&command)?;
1431 }
1432 _ => {}
1433 }
1434
1435 Ok(self
1436 .subscriptions_external
1437 .retain(key, command.command_id(), command))
1438 }
1439
1440 fn unsubscribe_external(&mut self, client_id: ClientId, cmd: &UnsubscribeCommand) {
1441 let mut releases = IndexMap::new();
1442
1443 for member in split_unsubscribe(cmd) {
1444 let key = (client_id, SubscriptionKey::from_unsubscribe(&member));
1445 if matches!(
1446 member,
1447 UnsubscribeCommand::BookDeltas(_) | UnsubscribeCommand::BookDepth(_)
1448 ) && self.subscriptions_external.contains(&key)
1449 {
1450 if !self.release_book_subscription(&member) {
1451 continue;
1452 }
1453
1454 match &member {
1455 UnsubscribeCommand::BookDeltas(command) => {
1456 self.unsubscribe_book_deltas(command);
1457 }
1458 UnsubscribeCommand::BookDepth(command) => {
1459 self.unsubscribe_book_depth(command);
1460 }
1461 _ => unreachable!("only delta and depth subscriptions retain external books"),
1462 }
1463 }
1464
1465 let (acquisition_id, command) = match self.subscriptions_external.release(&key) {
1466 SubscriptionRelease::Retained => continue,
1467 SubscriptionRelease::Final(subscribe) => (
1468 subscribe.command_id(),
1469 subscribe.into_unsubscribe(
1470 cmd.command_id(),
1471 cmd.ts_init(),
1472 cmd.correlation_id(),
1473 ),
1474 ),
1475 SubscriptionRelease::Untracked => (cmd.command_id(), member),
1476 };
1477
1478 self.subscriptions_external.remove(&key);
1479 releases
1480 .entry(acquisition_id)
1481 .or_insert_with(Vec::new)
1482 .push(command);
1483 }
1484
1485 for command in releases.into_values().flat_map(UnsubscribeCommand::group) {
1486 publish_external_data_command(client_id, &command);
1487 }
1488
1489 if self.config.debug {
1490 log::debug!("Skipping unsubscribe command for external client {client_id}: {cmd:?}");
1491 }
1492 }
1493
1494 fn subscribe_quotes(&mut self, cmd: &SubscribeQuotes) -> anyhow::Result<bool> {
1495 match self.quote_routing(&cmd.instrument_ids, cmd.params.as_ref())? {
1496 QuoteRouting::Client => Ok(false),
1497 QuoteRouting::Synthetic => {
1498 for id in &cmd.instrument_ids {
1499 self.subscribe_synthetic_quotes(*id);
1500 }
1501
1502 Ok(true)
1503 }
1504 QuoteRouting::Spread => {
1505 for id in &cmd.instrument_ids {
1506 let member = cmd.with_instrument_ids(*id);
1507 self.subscribe_spread_quotes(&member);
1508 }
1509
1510 Ok(true)
1511 }
1512 }
1513 }
1514
1515 fn unsubscribe_quotes(&mut self, cmd: &UnsubscribeQuotes) -> anyhow::Result<bool> {
1516 match self.quote_routing(&cmd.instrument_ids, cmd.params.as_ref())? {
1517 QuoteRouting::Client => Ok(false),
1518 QuoteRouting::Synthetic => {
1519 for id in &cmd.instrument_ids {
1520 self.unsubscribe_synthetic_quotes(*id);
1521 }
1522
1523 Ok(true)
1524 }
1525 QuoteRouting::Spread => {
1526 for id in &cmd.instrument_ids {
1527 let member = cmd.with_instrument_ids(*id);
1528 self.unsubscribe_spread_quotes(&member);
1529 }
1530
1531 Ok(true)
1532 }
1533 }
1534 }
1535
1536 fn subscribe_trades(&mut self, cmd: &SubscribeTrades) -> bool {
1537 if !cmd.instrument_ids[0].is_synthetic() {
1538 return false;
1539 }
1540
1541 for id in &cmd.instrument_ids {
1542 self.subscribe_synthetic_trades(*id);
1543 }
1544
1545 true
1546 }
1547
1548 fn unsubscribe_trades(&mut self, cmd: &UnsubscribeTrades) -> bool {
1549 if !cmd.instrument_ids[0].is_synthetic() {
1550 return false;
1551 }
1552
1553 for id in &cmd.instrument_ids {
1554 self.unsubscribe_synthetic_trades(*id);
1555 }
1556
1557 true
1558 }
1559
1560 fn quote_routing(
1561 &self,
1562 ids: &[InstrumentId],
1563 params: Option<&Params>,
1564 ) -> anyhow::Result<QuoteRouting> {
1565 quote_routing(ids, None, params, &self.cache.borrow())
1566 }
1567
1568 pub fn execute_request(&mut self, req: RequestCommand) -> anyhow::Result<()> {
1577 let request_id = *req.request_id();
1578 let result = self.execute_request_inner(req);
1579 if let Err(e) = &result {
1580 self.process_response_envelope(DataResponseEnvelope::terminal(
1581 request_id,
1582 Some(e.to_string()),
1583 ));
1584 }
1585
1586 result
1587 }
1588
1589 fn execute_request_inner(&mut self, req: RequestCommand) -> anyhow::Result<()> {
1590 Self::validate_request(&req)?;
1591 #[cfg(feature = "streaming")]
1592 self.validate_catalog_update(&req)?;
1593
1594 if let Some(cid) = req.client_id()
1596 && self.is_external_client(*cid)
1597 {
1598 if self.config.debug {
1599 log::debug!("Skipping data request for external client {cid}: {req:?}");
1600 }
1601 return Ok(());
1602 }
1603
1604 if let RequestCommand::Join(join) = req {
1605 return self.handle_request_join(join);
1606 }
1607
1608 if has_continuous_future_params(request_params(&req)) {
1609 return self.execute_continuous_future_request(req);
1610 }
1611
1612 let request_id = *req.request_id();
1613 self.prepare_request_bar_aggregators(&req)?;
1614
1615 if has_time_range_pipeline_params(request_params(&req))
1616 && is_time_range_pipeline_variant(&req)
1617 {
1618 let result = self.execute_time_range_pipeline_request(req);
1619 if result.is_err() {
1620 self.cleanup_request_bar_aggregators(&request_id);
1621 }
1622 return result;
1623 }
1624
1625 #[cfg(feature = "streaming")]
1626 if self.catalogs_registered() && streaming::is_date_range_variant(&req) {
1627 let result = self.dispatch_date_range_request(req);
1628 if result.is_err() {
1629 self.cleanup_request_bar_aggregators(&request_id);
1630 }
1631 return result;
1632 }
1633
1634 let result = self.dispatch_request_to_client(req);
1635
1636 if result.is_err() {
1637 self.cleanup_request_bar_aggregators(&request_id);
1638 }
1639
1640 result.map(|_| ())
1641 }
1642
1643 fn validate_request(req: &RequestCommand) -> anyhow::Result<()> {
1644 let (instrument_ids, requires_single, limit) = match req {
1645 RequestCommand::Instrument(cmd) => (&cmd.instrument_ids, false, None),
1646 RequestCommand::BookSnapshot(cmd) => (&cmd.instrument_ids, true, None),
1647 RequestCommand::BookDeltas(cmd) => (&cmd.instrument_ids, true, None),
1648 RequestCommand::BookDepth(cmd) => (&cmd.instrument_ids, true, None),
1649 RequestCommand::Quotes(cmd) => (&cmd.instrument_ids, false, cmd.limit),
1650 RequestCommand::Trades(cmd) => (&cmd.instrument_ids, false, cmd.limit),
1651 RequestCommand::FundingRates(cmd) => (&cmd.instrument_ids, true, None),
1652 RequestCommand::Bars(cmd) => {
1653 ensure_routable_bar_types(&cmd.bar_types, cmd.client_id.is_some())?;
1654 cmd.single_bar_type()?;
1655 return Ok(());
1656 }
1657 _ => return Ok(()),
1658 };
1659
1660 ensure_routable_instrument_ids(instrument_ids, req.client_id().is_some())?;
1661 anyhow::ensure!(
1662 !requires_single || instrument_ids.len() == 1,
1663 "request requires one instrument ID for this data type, received {}",
1664 instrument_ids.len(),
1665 );
1666 anyhow::ensure!(
1667 instrument_ids.len() == 1 || limit.is_none(),
1668 "A record limit is unsupported for multi-instrument requests",
1669 );
1670 Ok(())
1671 }
1672
1673 pub(super) fn dispatch_request_to_client(
1674 &mut self,
1675 req: RequestCommand,
1676 ) -> anyhow::Result<ClientId> {
1677 self.inflight_requests
1678 .insert(*req.request_id(), req.clone());
1679 let client_id = req.client_id().copied();
1680 let venue = req.venue().copied();
1681 let Some(client) = self.get_client(client_id.as_ref(), venue.as_ref()) else {
1682 anyhow::bail!("Cannot handle request: no client found for {client_id:?} {venue:?}");
1683 };
1684 let resolved_client_id = client.client_id();
1685
1686 #[rustfmt::skip]
1687 let result = match req {
1688 RequestCommand::Data(req) => client.request_data(req),
1689 RequestCommand::Instrument(req) => client.request_instrument(req),
1690 RequestCommand::Instruments(req) => client.request_instruments(req),
1691 RequestCommand::BookSnapshot(req) => client.request_book_snapshot(req),
1692 RequestCommand::BookDeltas(req) => client.request_book_deltas(req),
1693 RequestCommand::BookDepth(req) => client.request_book_depth(req),
1694 RequestCommand::Quotes(req) => client.request_quotes(req),
1695 RequestCommand::Trades(req) => client.request_trades(req),
1696 RequestCommand::FundingRates(req) => client.request_funding_rates(req),
1697 RequestCommand::OptionChainReferencePrice(req) => client.request_option_chain_reference_price(req),
1698 RequestCommand::Bars(req) => client.request_bars(req),
1699 RequestCommand::Join(_) => anyhow::bail!("RequestJoin must be handled by handle_request_join"),
1700 };
1701
1702 result.map_err(|e| {
1703 let context = format!("Data client {resolved_client_id} request failed");
1704
1705 let context = if let Some(rejection) = e.downcast_ref::<IdentifierCountError>() {
1706 format!("{context}: {rejection}")
1707 } else {
1708 context
1709 };
1710
1711 e.context(context)
1712 })?;
1713
1714 Ok(resolved_client_id)
1715 }
1716
1717 pub fn process(&mut self, data: &dyn Any) {
1722 if let Some(response) = data.downcast_ref::<DataResponseEnvelope>() {
1723 self.response_envelope(response.clone());
1724 return;
1725 }
1726
1727 if let Some(failure) = data.downcast_ref::<DataRequestFailed>() {
1728 self.request_failed(failure.clone());
1729 return;
1730 }
1731
1732 self.data_count += 1;
1733
1734 if let Some(instrument) = data.downcast_ref::<InstrumentAny>() {
1735 self.handle_instrument(instrument);
1736 } else if let Some(funding_rate) = data.downcast_ref::<FundingRateUpdate>() {
1737 self.handle_funding_rate(*funding_rate);
1738 } else if let Some(option_greeks) = data.downcast_ref::<OptionGreeks>() {
1739 self.cache.borrow_mut().add_option_greeks(*option_greeks);
1740 self.feed_option_greeks_to_pre_bootstrap_chain(option_greeks);
1741 let topic = switchboard::get_option_greeks_topic(option_greeks.instrument_id);
1742 msgbus::publish_option_greeks(topic, option_greeks);
1743 self.drain_deferred_commands();
1744 } else if let Some(status) = data.downcast_ref::<InstrumentStatus>() {
1745 self.handle_instrument_status(*status);
1746 } else if let Some(custom) = data.downcast_ref::<CustomData>() {
1747 self.handle_custom_data(custom);
1748 } else {
1749 #[cfg(feature = "defi")]
1750 if let Some(response) = data.downcast_ref::<PoolSnapshotResponse>() {
1751 self.handle_pool_snapshot_response(response);
1752 return;
1753 }
1754
1755 log::error!("Cannot process data {data:?}, type is unrecognized");
1756 }
1757 }
1758
1759 #[allow(
1761 clippy::needless_pass_by_value,
1762 reason = "callers hand over ownership; the payload is only moved when a DeFi handler consumes it"
1763 )]
1764 pub fn process_data(&mut self, data: Data) {
1765 #[cfg(feature = "defi")]
1766 let data = match data {
1767 Data::Defi(defi) => {
1768 self.process_defi_data(*defi);
1769 return;
1770 }
1771 data => data,
1772 };
1773
1774 self.process_data_ref(DataRef::from(&data));
1775 }
1776
1777 pub fn process_data_ref(&mut self, data: DataRef<'_>) {
1782 #[cfg(feature = "defi")]
1783 let data = match data {
1784 DataRef::Defi(defi) => {
1785 self.process_defi_data(defi.clone());
1786 return;
1787 }
1788 data => data,
1789 };
1790
1791 self.data_count += 1;
1792
1793 match data {
1794 DataRef::Instrument(instrument) => self.handle_instrument(instrument),
1795 DataRef::BookDelta(delta) => self.handle_delta(*delta),
1796 DataRef::BookDeltas(deltas) => self.handle_deltas(deltas),
1797 DataRef::BookDepth(depth) => self.handle_depth(depth),
1798 DataRef::Quote(quote) => {
1799 self.handle_quote(*quote);
1800 self.drain_deferred_commands();
1801 }
1802 DataRef::Trade(trade) => self.handle_trade(*trade),
1803 DataRef::Bar(bar) => self.handle_bar(*bar),
1804 DataRef::MarkPrice(mark_price) => {
1805 self.handle_mark_price(*mark_price);
1806 self.drain_deferred_commands();
1807 }
1808 DataRef::IndexPrice(index_price) => {
1809 self.handle_index_price(*index_price);
1810 self.drain_deferred_commands();
1811 }
1812 DataRef::FundingRate(funding_rate) => {
1813 self.handle_funding_rate(*funding_rate);
1814 self.drain_deferred_commands();
1815 }
1816 DataRef::OptionGreeks(greeks) => {
1817 self.cache.borrow_mut().add_option_greeks(*greeks);
1818 self.feed_option_greeks_to_pre_bootstrap_chain(greeks);
1819 let topic = switchboard::get_option_greeks_topic(greeks.instrument_id);
1820 msgbus::publish_option_greeks(topic, greeks);
1821 self.drain_deferred_commands();
1822 }
1823 DataRef::InstrumentStatus(status) => {
1824 self.handle_instrument_status(*status);
1825 self.drain_deferred_commands();
1826 }
1827 DataRef::InstrumentClose(close) => self.handle_instrument_close(*close),
1828 DataRef::Custom(custom) => self.handle_custom_data(custom),
1829 #[cfg(feature = "defi")]
1830 DataRef::Defi(_) => unreachable!("handled before market data dispatch"),
1831 #[cfg(not(feature = "defi"))]
1832 #[allow(
1833 unreachable_patterns,
1834 reason = "DeFi variants can exist without this crate's defi feature"
1835 )]
1836 other => log_defi_data_dropped(other),
1837 }
1838 }
1839
1840 pub fn process_pipeline(&mut self, data: Data) {
1847 #[cfg(feature = "defi")]
1848 let data = match data {
1849 Data::Defi(defi) => {
1850 self.process_defi_data(*defi);
1851 return;
1852 }
1853 data => data,
1854 };
1855
1856 self.data_count += 1;
1857
1858 match data {
1859 Data::Instrument(instrument) => self.handle_instrument(&instrument),
1860 Data::BookDelta(delta) => self.handle_delta_pipeline(delta),
1861 Data::BookDeltas(deltas) => self.handle_deltas_pipeline(&deltas),
1862 Data::BookDepth(depth) => self.handle_depth_pipeline(&depth),
1863 Data::Quote(quote) => self.handle_quote_pipeline(quote),
1864 Data::Trade(trade) => self.handle_trade_pipeline(trade),
1865 Data::Bar(bar) => self.handle_bar_pipeline(bar),
1866 Data::MarkPrice(mark_price) => self.handle_mark_price_pipeline(mark_price),
1867 Data::IndexPrice(index_price) => self.handle_index_price_pipeline(index_price),
1868 Data::FundingRate(funding_rate) => {
1869 self.handle_funding_rate_pipeline(funding_rate);
1870 }
1871 Data::OptionGreeks(greeks) => self.handle_option_greeks_pipeline(greeks),
1872 Data::InstrumentStatus(status) => self.handle_instrument_status_pipeline(status),
1873 Data::InstrumentClose(close) => self.handle_instrument_close_pipeline(close),
1874 Data::Custom(custom) => self.handle_custom_data_pipeline(&custom),
1875 #[cfg(feature = "defi")]
1876 Data::Defi(_) => unreachable!("handled before market data dispatch"),
1877 #[cfg(not(feature = "defi"))]
1878 #[allow(
1879 unreachable_patterns,
1880 reason = "DeFi variants can exist without this crate's defi feature"
1881 )]
1882 other => log_defi_data_dropped(DataRef::from(&other)),
1883 }
1884 }
1885
1886 pub fn response(&mut self, mut resp: DataResponse) {
1891 resp.trim_to_bounds();
1892 let request = self.inflight_requests.remove(resp.correlation_id());
1893 #[cfg(feature = "streaming")]
1894 if request
1895 .as_ref()
1896 .and_then(request_params)
1897 .is_some_and(|params| params.get_bool(streaming::PARAM_UPDATE_CATALOG) == Some(true))
1898 {
1899 log::warn!(
1900 "Catalog write-back requires an explicit completed response envelope, request={}",
1901 resp.correlation_id()
1902 );
1903 }
1904
1905 #[cfg(not(feature = "streaming"))]
1906 let _ = request;
1907 let envelope = DataResponseEnvelope::new(resp, true, None);
1908 self.process_response_envelope(envelope);
1909 }
1910
1911 pub fn request_failed(&mut self, failure: DataRequestFailed) {
1913 log::error!(
1914 "Data request {} failed: {}",
1915 failure.request_id,
1916 failure.reason
1917 );
1918
1919 let envelope = if let Some(mut response) = failure.response {
1920 response.trim_to_bounds();
1921 DataResponseEnvelope::new(response, true, None)
1922 .with_correlation_id(failure.request_id)
1923 .with_failure(failure.reason)
1924 } else {
1925 DataResponseEnvelope::terminal(failure.request_id, Some(failure.reason))
1926 };
1927
1928 self.process_response_envelope(envelope);
1929 }
1930
1931 pub fn response_envelope(&mut self, mut envelope: DataResponseEnvelope) {
1933 if let Some(response) = envelope.response_mut() {
1934 response.trim_to_bounds();
1935 }
1936
1937 self.process_response_envelope(envelope);
1938 }
1939
1940 fn process_response_envelope(&mut self, mut envelope: DataResponseEnvelope) {
1941 let accepted = match envelope.response() {
1942 Some(DataResponse::Instrument(response)) => self.cache_instrument(&response.data),
1943 Some(DataResponse::Instruments(response)) => {
1944 let mut accepted = true;
1945 for instrument in &response.data {
1946 accepted &= self.cache_instrument(instrument);
1947 }
1948
1949 accepted
1950 }
1951 _ => true,
1952 };
1953
1954 if !accepted {
1955 envelope = DataResponseEnvelope::terminal(
1956 *envelope.correlation_id(),
1957 Some("Instrument currency integrity check failed".into()),
1958 );
1959 }
1960
1961 let request = self.inflight_requests.remove(envelope.correlation_id());
1962 #[cfg(feature = "streaming")]
1963 let envelope = match (&request, envelope.response()) {
1964 (Some(request), Some(response))
1965 if envelope.completed() && envelope.failure_reason().is_none() =>
1966 {
1967 match self.update_catalog_from_response(response, request) {
1968 Ok(()) => envelope,
1969 Err(e) => envelope.with_failure(e.to_string()),
1970 }
1971 }
1972 _ => envelope,
1973 };
1974
1975 #[cfg(not(feature = "streaming"))]
1976 let _ = request;
1977
1978 let Some(envelope) = self.handle_request_pipeline_response(envelope) else {
1979 return;
1980 };
1981
1982 if self
1983 .request_pipeline_parent_request_id
1984 .contains_key(envelope.correlation_id())
1985 {
1986 self.process_response_envelope(envelope);
1987 return;
1988 }
1989
1990 if self.handle_continuous_future_child_response(&envelope)
1991 || self.handle_time_range_pipeline_child_response(&envelope)
1992 {
1993 return;
1994 }
1995
1996 let Some(resp) = envelope.response().cloned() else {
1997 self.finish_empty_response(&envelope);
1998 return;
1999 };
2000
2001 self.route_response_payload(resp, &envelope);
2002 }
2003
2004 fn finish_empty_response(&mut self, envelope: &DataResponseEnvelope) {
2005 if let Some(original_id) = self
2006 .parent_join_request_id
2007 .remove(envelope.correlation_id())
2008 {
2009 if let Some(original) = self.pending_join_requests.remove(&original_id) {
2010 for child_id in original.request_ids {
2011 msgbus::send_response_envelope(&DataResponseEnvelope::terminal(
2012 child_id,
2013 envelope.failure_reason().map(str::to_owned),
2014 ));
2015 }
2016 }
2017
2018 self.process_response_envelope(DataResponseEnvelope::terminal(
2019 original_id,
2020 envelope.failure_reason().map(str::to_owned),
2021 ));
2022 return;
2023 }
2024
2025 self.cleanup_request_bar_aggregators(envelope.correlation_id());
2026 msgbus::send_response_envelope(envelope);
2027 }
2028
2029 fn route_response_payload(&mut self, resp: DataResponse, envelope: &DataResponseEnvelope) {
2030 let resp = rebind_response_correlation(resp, *envelope.correlation_id());
2031 if log::log_enabled!(log::Level::Debug) {
2032 let correlation_id = resp.correlation_id();
2033 match resp.record_count() {
2034 Some(count) => log::debug!(
2035 "{RECV}{RES} {} correlation_id={correlation_id} records={count}",
2036 resp.kind(),
2037 ),
2038 None => log::debug!(
2039 "{RECV}{RES} {} correlation_id={correlation_id}",
2040 resp.kind(),
2041 ),
2042 }
2043 }
2044 log::trace!("{RECV}{RES} {resp:?}");
2045
2046 self.response_count += 1;
2047
2048 if self
2049 .parent_join_request_id
2050 .contains_key(resp.correlation_id())
2051 {
2052 self.finalize_request_join(resp, envelope.failure_reason().map(str::to_owned));
2053 return;
2054 }
2055
2056 if !self.handle_response_payload(&resp) {
2057 return;
2058 }
2059
2060 self.process_request_bar_aggregation_response(&resp, envelope.completed());
2061
2062 msgbus::send_response_envelope(envelope);
2063 }
2064
2065 fn handle_response_payload(&mut self, resp: &DataResponse) -> bool {
2066 let correlation_id = *resp.correlation_id();
2067
2068 match resp {
2069 DataResponse::Instrument(_) | DataResponse::Instruments(_) => {}
2070 DataResponse::Quotes(r) => {
2071 if !log_if_empty_response(
2072 &r.data,
2073 &format_args!("{:?}", r.instrument_ids),
2074 &correlation_id,
2075 ) {
2076 self.handle_quotes(&r.data);
2077 }
2078 }
2079 DataResponse::Trades(r) => {
2080 if !log_if_empty_response(
2081 &r.data,
2082 &format_args!("{:?}", r.instrument_ids),
2083 &correlation_id,
2084 ) {
2085 self.handle_trades(&r.data);
2086 }
2087 }
2088 DataResponse::FundingRates(r) => {
2089 if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2090 self.handle_funding_rates(&r.data);
2091 }
2092 }
2093 DataResponse::Bars(r) => {
2094 if !log_if_empty_response(&r.data, &r.bar_type, &correlation_id) {
2095 self.handle_bars(&r.data);
2096 }
2097 }
2098 DataResponse::Book(r) => self.handle_book_response(&r.data),
2099 DataResponse::BookDeltas(r) => {
2100 if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2101 self.handle_book_deltas_response(r);
2102 }
2103 }
2104 DataResponse::BookDepth(r) => {
2105 if !log_if_empty_response(&r.data, &r.instrument_id, &correlation_id) {
2106 self.handle_book_depth_response(r);
2107 }
2108 }
2109 DataResponse::OptionChainReferencePrice(r) => {
2110 self.process_request_bar_aggregation_response(resp, true);
2111 self.handle_option_chain_reference_price_response(&correlation_id, r);
2112 return false;
2113 }
2114 DataResponse::Data(_) => {}
2115 }
2116
2117 true
2118 }
2119
2120 pub fn new_request_pipeline(&mut self, parent: RequestCommand, n_components: usize) {
2122 let parent_id = *parent.request_id();
2123 self.request_pipeline_n_components
2124 .insert(parent_id, n_components);
2125 self.request_pipeline_parent_request
2126 .insert(parent_id, parent);
2127 self.request_pipeline_responses
2128 .insert(parent_id, Vec::with_capacity(n_components));
2129 }
2130
2131 pub fn register_request_pipeline_leg(&mut self, leg_id: UUID4, parent_id: UUID4) {
2137 let slots = self
2138 .request_pipeline_responses
2139 .get_mut(&parent_id)
2140 .expect("pipeline registered before its legs");
2141 let index = slots.len();
2142 slots.push(None);
2143 self.request_pipeline_parent_request_id
2144 .insert(leg_id, (parent_id, index));
2145 }
2146
2147 fn handle_request_pipeline_response(
2148 &mut self,
2149 envelope: DataResponseEnvelope,
2150 ) -> Option<DataResponseEnvelope> {
2151 let leg_id = *envelope.correlation_id();
2152
2153 let Some(&(parent_id, index)) = self.request_pipeline_parent_request_id.get(&leg_id) else {
2154 return Some(envelope);
2155 };
2156
2157 let completed = envelope.completed();
2158 let slots = self.request_pipeline_responses.get_mut(&parent_id)?;
2159 slots[index].get_or_insert_with(Vec::new).push(envelope);
2160
2161 if !completed {
2162 return None;
2163 }
2164
2165 self.request_pipeline_parent_request_id.remove(&leg_id);
2166 let expected = *self.request_pipeline_n_components.get(&parent_id)?;
2167
2168 if slots
2169 .iter()
2170 .flatten()
2171 .filter(|slot| slot.last().is_some_and(DataResponseEnvelope::completed))
2172 .count()
2173 < expected
2174 {
2175 return None;
2176 }
2177
2178 let slots = self.request_pipeline_responses.remove(&parent_id)?;
2179 self.request_pipeline_n_components.remove(&parent_id);
2180 let parent = self.request_pipeline_parent_request.remove(&parent_id);
2181 let failures = slots
2182 .iter()
2183 .flatten()
2184 .flatten()
2185 .filter_map(|value| value.failure_reason())
2186 .collect::<Vec<_>>();
2187 let failure_reason = (!failures.is_empty()).then(|| failures.join("; "));
2188 let legs = slots
2189 .into_iter()
2190 .flatten()
2191 .flatten()
2192 .filter_map(|value| value.response().cloned())
2193 .collect::<Vec<_>>();
2194
2195 if legs.is_empty() {
2196 return Some(DataResponseEnvelope::terminal(parent_id, failure_reason));
2197 }
2198
2199 let (parent_start, parent_end) = parent_request_window(parent.as_ref());
2200 if self.emit_partitioned_book_responses(parent_id, parent.as_ref(), &legs) {
2201 return Some(DataResponseEnvelope::terminal(parent_id, failure_reason));
2202 }
2203
2204 let rebuilt = rebuild_pipeline_response(parent_id, parent.as_ref(), legs);
2205
2206 let Some(mut rebuilt) = rebuilt else {
2207 let reason = failure_reason
2208 .unwrap_or_else(|| "Request pipeline could not rebuild its response".to_owned());
2209
2210 return Some(DataResponseEnvelope::terminal(parent_id, Some(reason)));
2211 };
2212
2213 if let DataResponse::BookDeltas(r) = &mut rebuilt {
2214 self.book_deltas_snapshot_replay(r);
2215 }
2216
2217 if !matches!(parent, Some(RequestCommand::Instrument(_)))
2219 && (parent_start.is_some() || parent_end.is_some())
2220 {
2221 rebuilt.trim_to_bounds();
2222 }
2223
2224 let result = DataResponseEnvelope::new(rebuilt, true, None);
2225 Some(if let Some(reason) = failure_reason {
2226 result.with_failure(reason)
2227 } else {
2228 result
2229 })
2230 }
2231
2232 fn emit_partitioned_book_responses(
2233 &mut self,
2234 parent_id: UUID4,
2235 parent: Option<&RequestCommand>,
2236 legs: &[DataResponse],
2237 ) -> bool {
2238 if !matches!(parent, Some(RequestCommand::BookDeltas(_))) {
2239 return false;
2240 }
2241
2242 let mut ids = legs.iter().filter_map(|r| match r {
2243 DataResponse::BookDeltas(r) => Some(r.instrument_id),
2244 _ => None,
2245 });
2246
2247 let Some(first) = ids.next() else {
2248 return false;
2249 };
2250
2251 if !ids.any(|id| !same_instrument(id, first)) {
2252 return false;
2253 }
2254
2255 let (start, end) = parent_request_window(parent);
2256
2257 for response in legs {
2258 let child_id = *response.correlation_id();
2259 let mut response = rebind_response_correlation(response.clone(), parent_id);
2260 if let DataResponse::BookDeltas(r) = &mut response {
2261 r.start = start.or(r.start);
2262 r.end = end.or(r.end);
2263 self.book_deltas_snapshot_replay(r);
2264 }
2265
2266 response.trim_to_bounds();
2267 self.process_response_envelope(DataResponseEnvelope::new(
2268 response,
2269 false,
2270 Some(child_id),
2271 ));
2272 }
2273
2274 true
2275 }
2276
2277 fn handle_request_join(&mut self, req: RequestJoin) -> anyhow::Result<()> {
2278 if has_time_range_pipeline_params(req.params.as_ref()) {
2279 return self.execute_time_range_pipeline_request(RequestCommand::Join(req));
2280 }
2281
2282 let now_ns = self.clock.borrow().timestamp_ns();
2283 let now_dt = now_ns.to_datetime_utc();
2284 let zero = jiff::Timestamp::UNIX_EPOCH;
2285 let start = req.start.unwrap_or(zero).min(now_dt);
2286 let end = req.end.unwrap_or(now_dt).min(now_dt);
2287 let dated = req.with_dates(Some(start), Some(end), now_ns);
2288
2289 let original_id = req.request_id;
2290 let dated_id = dated.request_id;
2291
2292 self.pending_join_requests.insert(original_id, req);
2293 self.parent_join_request_id.insert(dated_id, original_id);
2294
2295 let leg_ids: Vec<UUID4> = dated.request_ids.clone();
2296 self.new_request_pipeline(RequestCommand::Join(dated), leg_ids.len());
2297 for leg_id in leg_ids {
2298 self.register_request_pipeline_leg(leg_id, dated_id);
2299 }
2300
2301 Ok(())
2302 }
2303
2304 fn finalize_request_join(&mut self, resp: DataResponse, failure_reason: Option<String>) {
2305 let dated_id = *resp.correlation_id();
2306 let Some(original_id) = self.parent_join_request_id.remove(&dated_id) else {
2307 log::error!("parent_join_request_id missing for dated correlation {dated_id}");
2308 return;
2309 };
2310
2311 let Some(original) = self.pending_join_requests.remove(&original_id) else {
2312 log::error!("pending_join_requests missing for original {original_id}");
2313 return;
2314 };
2315
2316 for leg_request_id in &original.request_ids {
2317 let terminal = DataResponseEnvelope::terminal(*leg_request_id, failure_reason.clone());
2318 msgbus::send_response_envelope(&terminal);
2319 }
2320
2321 let final_resp = rebind_response_correlation(resp, original_id);
2327 let envelope = DataResponseEnvelope::new(final_resp, true, None);
2328 self.process_response_envelope(if let Some(reason) = failure_reason {
2329 envelope.with_failure(reason)
2330 } else {
2331 envelope
2332 });
2333 }
2334}
2335
2336fn register_external_streaming_type(cmd: &SubscribeCommand) {
2337 if let Some(payload_type) = streaming_payload_type(cmd) {
2338 msgbus::get_message_bus()
2339 .borrow_mut()
2340 .add_streaming_type(payload_type);
2341 }
2342}
2343
2344fn publish_external_data_command<T>(client_id: ClientId, command: &T)
2345where
2346 T: Any,
2347{
2348 let topic = format!("commands.data.{client_id}");
2349 msgbus::publish_any(topic.into(), command);
2350}
2351
2352#[rustfmt::skip]
2353fn streaming_payload_type(cmd: &SubscribeCommand) -> Option<BusPayloadType> {
2354 match cmd {
2355 SubscribeCommand::Data(cmd) => Some(BusPayloadType::Custom(Ustr::from(
2356 cmd.data_type.type_name(),
2357 ))),
2358 SubscribeCommand::Instrument(_) | SubscribeCommand::Instruments(_) => Some(BusPayloadType::Instrument),
2359 SubscribeCommand::BookDeltas(_) | SubscribeCommand::BookSnapshots(_) => Some(BusPayloadType::OrderBookDeltas),
2360 SubscribeCommand::BookDepth(_) => Some(BusPayloadType::OrderBookDepth),
2361 SubscribeCommand::Quotes(_) => Some(BusPayloadType::QuoteTick),
2362 SubscribeCommand::Trades(_) => Some(BusPayloadType::TradeTick),
2363 SubscribeCommand::Bars(_) => Some(BusPayloadType::Bar),
2364 SubscribeCommand::MarkPrices(_) => Some(BusPayloadType::MarkPriceUpdate),
2365 SubscribeCommand::IndexPrices(_) => Some(BusPayloadType::IndexPriceUpdate),
2366 SubscribeCommand::FundingRates(_) => Some(BusPayloadType::FundingRateUpdate),
2367 SubscribeCommand::OptionGreeks(_) => Some(BusPayloadType::OptionGreeks),
2368 SubscribeCommand::InstrumentStatus(_)
2369 | SubscribeCommand::InstrumentClose(_)
2370 | SubscribeCommand::OptionChain(_) => None,
2371 }
2372}
2373
2374#[cfg(not(feature = "defi"))]
2375fn log_defi_data_dropped(data: DataRef<'_>) {
2376 log::error!("Cannot process data {data:?}, nautilus-data built without its `defi` feature");
2377}