1use std::{
22 path::PathBuf,
23 str::FromStr,
24 sync::{
25 Arc,
26 atomic::{AtomicBool, Ordering},
27 },
28 time::Duration,
29};
30
31use ahash::AHashMap;
32use databento::{dbn, live::Subscription};
33use indexmap::IndexMap;
34use nautilus_common::{
35 clients::{DataClient, execute_bar_subscriptions, execute_instrument_subscriptions},
36 live::{runner::get_data_event_sender, sender::EventSender},
37 messages::{
38 DataEvent, DataResponse,
39 data::{
40 BarsResponse, BookDeltasResponse, BookDepthResponse, DataRequestFailed,
41 DataResponseEnvelope, InstrumentResponse, InstrumentsResponse, IntoInstrumentIds,
42 QuotesResponse, RequestBars, RequestBookDeltas, RequestBookDepth, RequestInstrument,
43 RequestInstruments, RequestQuotes, RequestTrades, SubscribeBars, SubscribeBookDeltas,
44 SubscribeBookDepth, SubscribeCustomData, SubscribeInstrument,
45 SubscribeInstrumentStatus, SubscribeQuotes, SubscribeTrades, SubscriptionError,
46 TradesResponse, UnsubscribeBars, UnsubscribeBookDeltas, UnsubscribeBookDepth,
47 UnsubscribeCustomData, UnsubscribeInstrumentStatus, UnsubscribeQuotes,
48 UnsubscribeTrades,
49 },
50 },
51};
52use nautilus_core::{
53 AtomicMap, DurationNanos, Params, UnixNanos,
54 datetime::datetime_to_unix_nanos,
55 time::{AtomicTime, get_atomic_clock_realtime},
56};
57use nautilus_live::task::TaskGroup;
58use nautilus_model::{
59 data::{BarSpecification, BarType, CustomData, Data, DataType, custom::CustomDataTrait},
60 enums::{BarAggregation, PriceType},
61 identifiers::{ClientId, InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
62 instruments::{Instrument, InstrumentAny},
63};
64use parking_lot::Mutex;
65use tokio_util::sync::CancellationToken;
66
67use crate::{
68 common::{Credential, DATABENTO_VENUE, ohlcv_schema_from_aggregation},
69 historical::{DatabentoHistoricalClient, RangeQueryParams},
70 live::{DatabentoFeedHandler, DatabentoMessage, HandlerCommand},
71 loader::DatabentoDataLoader,
72 symbology::{
73 check_consistent_symbology, infer_symbology_type, instrument_id_to_symbol_string,
74 raw_symbol,
75 },
76 types::{DatabentoImbalance, DatabentoStatistics, Dataset, PublisherId},
77};
78
79const PRICE_PRECISION_PARAM: &str = "price_precision";
80const SCHEMA_PARAM: &str = "schema";
81const MBP10_DEPTH: usize = 10;
82const QUOTE_SCHEMAS: &[dbn::Schema] = &[
83 dbn::Schema::Mbp1,
84 dbn::Schema::Bbo1S,
85 dbn::Schema::Bbo1M,
86 dbn::Schema::Cmbp1,
87 dbn::Schema::Cbbo1S,
88 dbn::Schema::Cbbo1M,
89 dbn::Schema::Tbbo,
90 dbn::Schema::Tcbbo,
91];
92const TRADE_SCHEMAS: &[dbn::Schema] = &[
93 dbn::Schema::Trades,
94 dbn::Schema::Tbbo,
95 dbn::Schema::Tcbbo,
96 dbn::Schema::Mbp1,
97 dbn::Schema::Cmbp1,
98];
99
100#[derive(Debug, Clone)]
102#[cfg_attr(
103 feature = "python",
104 pyo3::pyclass(module = "nautilus_trader.adapters.databento", from_py_object)
105)]
106#[cfg_attr(
107 feature = "python",
108 pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
109)]
110pub struct DatabentoDataClientConfig {
111 pub(crate) credential: Credential,
113 pub publishers_filepath: PathBuf,
115 pub historical_base_url: Option<String>,
117 pub live_gateway_addr: Option<String>,
119 pub venue_dataset_map: IndexMap<String, String>,
121 pub use_exchange_as_venue: bool,
123 pub bars_timestamp_on_close: bool,
125 pub reconnect_timeout_mins: Option<u64>,
127 pub futures_year_digits: FuturesYearDigits,
129}
130
131#[cfg(feature = "python")]
132nautilus_core::impl_pyo3_config_getters!(DatabentoDataClientConfig {
133 publishers_filepath: PathBuf,
134 historical_base_url: Option<String>,
135 live_gateway_addr: Option<String>,
136 use_exchange_as_venue: bool,
137 bars_timestamp_on_close: bool,
138 venue_dataset_map: IndexMap<String, String>,
139});
140
141impl DatabentoDataClientConfig {
142 #[must_use]
144 pub fn new(
145 api_key: impl Into<String>,
146 publishers_filepath: PathBuf,
147 use_exchange_as_venue: bool,
148 bars_timestamp_on_close: bool,
149 ) -> Self {
150 Self {
151 futures_year_digits: FuturesYearDigits::default(),
152 credential: Credential::new(api_key),
153 publishers_filepath,
154 venue_dataset_map: IndexMap::new(),
155 use_exchange_as_venue,
156 bars_timestamp_on_close,
157 reconnect_timeout_mins: Some(10), historical_base_url: None,
159 live_gateway_addr: None,
160 }
161 }
162
163 #[must_use]
165 pub fn api_key(&self) -> &str {
166 self.credential.api_key()
167 }
168
169 #[must_use]
171 pub fn api_key_masked(&self) -> String {
172 self.credential.api_key_masked()
173 }
174}
175
176#[cfg_attr(feature = "python", pyo3::pyclass)]
182#[cfg_attr(
183 feature = "python",
184 pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
185)]
186#[derive(Debug)]
187pub struct DatabentoDataClient {
188 client_id: ClientId,
189 config: DatabentoDataClientConfig,
190 is_connected: AtomicBool,
191 historical: DatabentoHistoricalClient,
192 loader: DatabentoDataLoader,
193 cmd_channels: Arc<Mutex<AHashMap<String, tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>>,
194 task_handles: TaskGroup,
195 cancellation_token: CancellationToken,
196 publisher_venue_map: Arc<IndexMap<PublisherId, Venue>>,
197 symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
198 data_sender: EventSender<DataEvent>,
199}
200
201impl DatabentoDataClient {
202 pub fn new(
208 client_id: ClientId,
209 config: DatabentoDataClientConfig,
210 clock: &'static AtomicTime,
211 ) -> anyhow::Result<Self> {
212 let historical = match &config.historical_base_url {
213 Some(base_url) => DatabentoHistoricalClient::new_with_base_url(
214 config.credential.clone(),
215 config.publishers_filepath.clone(),
216 clock,
217 config.use_exchange_as_venue,
218 base_url,
219 )?,
220 None => DatabentoHistoricalClient::new(
221 config.credential.clone(),
222 config.publishers_filepath.clone(),
223 clock,
224 config.use_exchange_as_venue,
225 )?,
226 }
227 .with_futures_year_digits(config.futures_year_digits);
228
229 let mut loader = DatabentoDataLoader::new(Some(config.publishers_filepath.clone()))?
231 .with_futures_year_digits(config.futures_year_digits);
232 for (venue, dataset) in &config.venue_dataset_map {
233 loader.set_dataset_for_venue(
234 Dataset::from(dataset.as_str()),
235 Venue::from(venue.as_str()),
236 );
237 }
238
239 let file_content = std::fs::read_to_string(&config.publishers_filepath)?;
241 let publishers_vec: Vec<crate::types::DatabentoPublisher> =
242 serde_json::from_str(&file_content)?;
243
244 let publisher_venue_map = publishers_vec
245 .into_iter()
246 .map(|p| (p.publisher_id, Venue::from(p.venue.as_str())))
247 .collect::<IndexMap<u16, Venue>>();
248
249 let data_sender = get_data_event_sender();
250
251 let task_handles = TaskGroup::new();
252
253 Ok(Self {
254 client_id,
255 config,
256 is_connected: AtomicBool::new(false),
257 historical,
258 loader,
259 cmd_channels: Arc::new(Mutex::new(AHashMap::new())),
260 cancellation_token: task_handles.cancellation_token(),
261 task_handles,
262 publisher_venue_map: Arc::new(publisher_venue_map),
263 symbol_venue_map: Arc::new(AtomicMap::new()),
264 data_sender,
265 })
266 }
267
268 #[must_use]
270 pub fn api_key(&self) -> &str {
271 self.config.api_key()
272 }
273
274 #[must_use]
276 pub fn api_key_masked(&self) -> String {
277 self.config.api_key_masked()
278 }
279
280 fn get_dataset_for_venue(&self, venue: Venue) -> anyhow::Result<String> {
286 self.loader
287 .get_dataset_for_venue(&venue)
288 .map(ToString::to_string)
289 .ok_or_else(|| anyhow::anyhow!("No dataset found for venue: {venue}"))
290 }
291
292 fn get_or_create_feed_handler(&self, dataset: &str) -> bool {
294 let mut channels = self.cmd_channels.lock();
295
296 if !channels.contains_key(dataset) {
297 log::debug!("Creating new feed handler for dataset: {dataset}");
298 let cmd_tx = self.initialize_live_feed(dataset.to_string());
299 channels.insert(dataset.to_string(), cmd_tx);
300
301 log::debug!("Feed handler created for dataset: {dataset}, channel stored");
302 return true;
303 }
304
305 false
306 }
307
308 fn send_subscription_to_dataset(
309 &self,
310 dataset: &str,
311 price_precisions: impl IntoIterator<Item = (Symbol, u8)>,
312 mut subscription: Subscription,
313 start_after_subscribe: bool,
314 ) -> anyhow::Result<()> {
315 if subscription.stype_in == dbn::SType::RawSymbol
316 && let databento::Symbols::Symbols(symbols) = &mut subscription.symbols
317 {
318 for symbol in symbols {
319 let raw = raw_symbol(symbol, dataset, self.config.futures_year_digits);
320
321 *symbol = raw;
322 }
323 }
324
325 let price_precisions = price_precisions.into_iter().map(|(symbol, precision)| {
326 let raw = raw_symbol(symbol.as_str(), dataset, self.config.futures_year_digits);
327 (Symbol::from(raw.as_str()), precision)
328 });
329
330 let tx = {
331 let channels = self.cmd_channels.lock();
332 channels
333 .get(dataset)
334 .cloned()
335 .ok_or_else(|| anyhow::anyhow!("No feed handler found for dataset: {dataset}"))?
336 };
337
338 send_subscription_commands(
339 &tx,
340 dataset,
341 price_precisions,
342 subscription,
343 start_after_subscribe,
344 )
345 }
346
347 fn subscribe_schema(
348 &self,
349 instrument_id: InstrumentId,
350 schema: dbn::Schema,
351 ) -> anyhow::Result<()> {
352 let symbol = instrument_id.symbol.to_string();
353 anyhow::ensure!(
356 infer_symbology_type(&symbol) == dbn::SType::RawSymbol,
357 "Unsupported symbol {symbol} for a Databento {schema} subscription, only raw symbols \
358 are available (not continuous or parent symbols)",
359 );
360
361 let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
362 let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
363
364 self.symbol_venue_map
365 .insert(instrument_id.symbol, instrument_id.venue);
366
367 let subscription = Subscription::builder()
368 .schema(schema)
369 .symbols(symbol)
370 .build();
371
372 self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)
373 }
374
375 fn send_close_to_active_feeds(&self) {
376 let channels = self.cmd_channels.lock();
377 for (dataset, tx) in channels.iter() {
378 if let Err(e) = tx.send(HandlerCommand::Close) {
379 log::warn!("Failed to send close command to dataset {dataset}: {e}");
380 }
381 }
382 }
383
384 fn clear_feed_channels(&self) {
385 let mut channels = self.cmd_channels.lock();
386 channels.clear();
387 }
388
389 fn abort_active_tasks(&self) {
390 self.task_handles.begin_shutdown();
391 }
392
393 fn spawn_task<F>(&self, future: F)
394 where
395 F: std::future::Future<Output = ()> + Send + 'static,
396 {
397 if let Err(e) = self.task_handles.spawn(future) {
398 log::debug!("Skipping Databento task after shutdown began: {e}");
399 }
400 }
401
402 fn initialize_live_feed(
404 &self,
405 dataset: String,
406 ) -> tokio::sync::mpsc::UnboundedSender<HandlerCommand> {
407 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
408 let (msg_tx, msg_rx) = tokio::sync::mpsc::unbounded_channel();
409 let feed_dataset = dataset.clone();
410 let feed_channels = self.cmd_channels.clone();
411
412 let mut feed_handler = DatabentoFeedHandler::new(
413 self.config.credential.clone(),
414 dataset,
415 cmd_rx,
416 msg_tx,
417 (*self.publisher_venue_map).clone(),
418 self.symbol_venue_map.clone(),
419 self.config.use_exchange_as_venue,
420 self.config.bars_timestamp_on_close,
421 self.config.reconnect_timeout_mins,
422 )
423 .with_futures_year_digits(self.config.futures_year_digits);
424
425 if let Some(addr) = &self.config.live_gateway_addr {
426 feed_handler = feed_handler.with_gateway_addr(addr.clone());
427 }
428
429 let feed_future = async move {
430 let _ = feed_handler.run().await;
432 feed_channels.lock().remove(&feed_dataset);
433 };
434
435 let cancellation_token = self.cancellation_token.clone();
436 let data_sender = self.data_sender.clone();
437
438 let msg_future = async move {
440 let mut msg_rx = msg_rx;
441
442 loop {
443 tokio::select! {
444 msg = msg_rx.recv() => {
445 match msg {
446 Some(DatabentoMessage::Data(data)) => {
447 log::debug!("Received data: {data:?}");
448 if let Err(e) = data_sender.send(DataEvent::Data(data)) {
449 log::error!("Failed to send data event: {e}");
450 }
451 }
452 Some(DatabentoMessage::Instrument(instrument)) => {
453 log::debug!("Received instrument definition: {}", instrument.id());
454 if let Err(e) = data_sender.send(DataEvent::Data(Data::Instrument(instrument))) {
455 log::error!("Failed to send instrument: {e}");
456 }
457 }
458 Some(DatabentoMessage::Status(status)) => {
459 log::debug!("Received status: {status:?}");
460 if let Err(e) =
461 data_sender.send(DataEvent::Data(Data::InstrumentStatus(status)))
462 {
463 log::error!("Failed to send status data event: {e}");
464 }
465 }
466 Some(DatabentoMessage::Imbalance(imbalance)) => {
467 log::debug!("Received imbalance: {imbalance:?}");
468 let instrument_id = imbalance.instrument_id;
469 let data = Data::Custom(custom_data_for_instrument(Arc::new(imbalance), instrument_id));
470 if let Err(e) = data_sender.send(DataEvent::Data(data)) {
471 log::error!("Failed to send imbalance data event: {e}");
472 }
473 }
474 Some(DatabentoMessage::Statistics(statistics)) => {
475 log::debug!("Received statistics: {statistics:?}");
476 let instrument_id = statistics.instrument_id;
477 let data = Data::Custom(custom_data_for_instrument(Arc::new(statistics), instrument_id));
478 if let Err(e) = data_sender.send(DataEvent::Data(data)) {
479 log::error!("Failed to send statistics data event: {e}");
480 }
481 }
482 Some(DatabentoMessage::SubscriptionAck(ack)) => {
483 log::debug!("Received subscription ack: {}", ack.message);
484 }
485 Some(DatabentoMessage::Error(error)) => {
486 log::error!("Feed handler error: {error}");
487 }
488 Some(DatabentoMessage::Close) => {
489 log::debug!("Feed handler closed");
490 break;
491 }
492 None => {
493 log::debug!("Message channel closed");
494 break;
495 }
496 }
497 }
498 () = cancellation_token.cancelled() => {
499 log::debug!("Message processing cancelled");
500 break;
501 }
502 }
503 }
504 };
505
506 if let Err(e) = self.task_handles.spawn(feed_future) {
507 log::warn!("Skipping Databento feed task after shutdown began: {e}");
508 }
509
510 if let Err(e) = self.task_handles.spawn(msg_future) {
511 log::warn!("Skipping Databento message task after shutdown began: {e}");
512 }
513
514 cmd_tx
515 }
516}
517
518#[async_trait::async_trait(?Send)]
519impl DataClient for DatabentoDataClient {
520 fn client_id(&self) -> ClientId {
522 self.client_id
523 }
524
525 fn venue(&self) -> Option<Venue> {
527 None
528 }
529
530 fn start(&mut self) -> anyhow::Result<()> {
536 log::debug!("Starting");
537 Ok(())
538 }
539
540 fn stop(&mut self) -> anyhow::Result<()> {
546 log::debug!("Stopping");
547
548 self.send_close_to_active_feeds();
549 self.clear_feed_channels();
550 self.cancellation_token.cancel();
551 self.abort_active_tasks();
552 self.is_connected.store(false, Ordering::Relaxed);
553
554 Ok(())
555 }
556
557 fn reset(&mut self) -> anyhow::Result<()> {
558 log::debug!("Resetting");
559 self.send_close_to_active_feeds();
560 self.clear_feed_channels();
561 self.abort_active_tasks();
562 self.is_connected.store(false, Ordering::Relaxed);
563 Ok(())
564 }
565
566 fn dispose(&mut self) -> anyhow::Result<()> {
567 log::debug!("Disposing");
568 self.stop()
569 }
570
571 async fn connect(&mut self) -> anyhow::Result<()> {
572 log::debug!("Connecting...");
573
574 if !self.task_handles.is_open() {
575 self.task_handles
576 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
577 .await
578 .map_err(|e| anyhow::anyhow!("Failed to terminate Databento tasks: {e}"))?;
579 self.task_handles
580 .start_generation()
581 .map_err(|e| anyhow::anyhow!("Failed to start Databento task generation: {e}"))?;
582 self.cancellation_token = self.task_handles.cancellation_token();
583 }
584
585 self.is_connected.store(true, Ordering::Relaxed);
586
587 log::info!("Connected");
588 Ok(())
589 }
590
591 async fn disconnect(&mut self) -> anyhow::Result<()> {
592 log::debug!("Disconnecting...");
593
594 self.send_close_to_active_feeds();
595 self.clear_feed_channels();
596 self.task_handles.begin_shutdown();
597
598 let tasks_result = self
599 .task_handles
600 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
601 .await
602 .map_err(|e| anyhow::anyhow!("Failed to terminate Databento tasks: {e}"));
603
604 self.is_connected.store(false, Ordering::Relaxed);
605
606 log::info!("Disconnected");
607 tasks_result
608 }
609
610 fn is_connected(&self) -> bool {
612 self.is_connected.load(Ordering::Relaxed)
613 }
614
615 fn is_disconnected(&self) -> bool {
616 !self.is_connected()
617 }
618
619 fn subscribe_instrument(&mut self, cmd: SubscribeInstrument) -> anyhow::Result<()> {
625 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
626 let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
627 let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
628
629 self.symbol_venue_map
630 .insert(instrument_id.symbol, instrument_id.venue);
631 let symbol = instrument_id.symbol.to_string();
632
633 let subscription = Subscription::builder()
634 .schema(databento::dbn::Schema::Definition)
635 .symbols(symbol)
636 .build();
637
638 self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)?;
639
640 Ok(())
641 })
642 }
643
644 fn subscribe_quotes(&mut self, cmd: SubscribeQuotes) -> anyhow::Result<()> {
650 self.subscribe_batch(
651 cmd.instrument_ids,
652 cmd.params.as_ref(),
653 dbn::Schema::Mbp1,
654 QUOTE_SCHEMAS,
655 )
656 }
657
658 fn subscribe_trades(&mut self, cmd: SubscribeTrades) -> anyhow::Result<()> {
664 self.subscribe_batch(
665 cmd.instrument_ids,
666 cmd.params.as_ref(),
667 dbn::Schema::Trades,
668 TRADE_SCHEMAS,
669 )
670 }
671
672 fn subscribe_book_deltas(&mut self, cmd: SubscribeBookDeltas) -> anyhow::Result<()> {
678 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
679 let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
680 let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
681
682 self.symbol_venue_map
683 .insert(instrument_id.symbol, instrument_id.venue);
684 let symbol = instrument_id.symbol.to_string();
685
686 let subscription = Subscription::builder()
687 .schema(databento::dbn::Schema::Mbo) .symbols(symbol)
689 .build();
690
691 self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)?;
692
693 Ok(())
694 })
695 }
696
697 fn subscribe(&mut self, cmd: SubscribeCustomData) -> anyhow::Result<()> {
705 let schema = custom_data_schema(&cmd.data_type)?;
706 let instrument_id = custom_data_instrument_id(&cmd.data_type)?;
707 self.subscribe_schema(instrument_id, schema)
708 }
709
710 fn subscribe_book_depth(&mut self, cmd: SubscribeBookDepth) -> anyhow::Result<()> {
717 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
718 let cmd = cmd.with_instrument_ids(instrument_id);
719 if let Some(depth) = cmd.depth {
720 anyhow::ensure!(
721 depth.get() == MBP10_DEPTH,
722 "Unsupported book depth {depth} for Databento `mbp-10`, only a depth of {MBP10_DEPTH} is available",
723 );
724 }
725
726 self.subscribe_schema(instrument_id, dbn::Schema::Mbp10)
727 })
728 }
729
730 fn subscribe_bars(&mut self, cmd: SubscribeBars) -> anyhow::Result<()> {
737 execute_bar_subscriptions(&cmd.bar_types, |bar_type| {
738 let schema = ohlcv_schema(bar_type)?;
739 self.subscribe_schema(bar_type.instrument_id(), schema)
740 })
741 }
742
743 fn subscribe_instrument_status(
749 &mut self,
750 cmd: SubscribeInstrumentStatus,
751 ) -> anyhow::Result<()> {
752 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
753 let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
754 let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
755
756 self.symbol_venue_map
757 .insert(instrument_id.symbol, instrument_id.venue);
758 let symbol = instrument_id.symbol.to_string();
759
760 let subscription = Subscription::builder()
761 .schema(databento::dbn::Schema::Status)
762 .symbols(symbol)
763 .build();
764
765 self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)?;
766
767 Ok(())
768 })
769 }
770
771 fn unsubscribe_quotes(&mut self, cmd: &UnsubscribeQuotes) -> anyhow::Result<()> {
773 log::debug!(
774 "Databento does not support granular unsubscribing, ignoring quote unsubscribe request for {:?}",
775 cmd.instrument_ids
776 );
777 Ok(())
778 }
779
780 fn unsubscribe_trades(&mut self, cmd: &UnsubscribeTrades) -> anyhow::Result<()> {
781 log::debug!(
785 "Databento does not support granular unsubscribing - ignoring unsubscribe request for {:?}",
786 cmd.instrument_ids
787 );
788
789 Ok(())
790 }
791
792 fn unsubscribe_book_deltas(&mut self, cmd: &UnsubscribeBookDeltas) -> anyhow::Result<()> {
793 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
794 log::debug!(
798 "Databento does not support granular unsubscribing - ignoring unsubscribe request for {instrument_id}"
799 );
800
801 Ok(())
802 })
803 }
804
805 fn unsubscribe(&mut self, cmd: &UnsubscribeCustomData) -> anyhow::Result<()> {
806 log::debug!(
810 "Databento does not support granular unsubscribing - ignoring unsubscribe request for {}",
811 cmd.data_type
812 );
813
814 Ok(())
815 }
816
817 fn unsubscribe_book_depth(&mut self, cmd: &UnsubscribeBookDepth) -> anyhow::Result<()> {
818 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
819 log::debug!(
823 "Databento does not support granular unsubscribing - ignoring unsubscribe request for {instrument_id}"
824 );
825
826 Ok(())
827 })
828 }
829
830 fn unsubscribe_bars(&mut self, cmd: &UnsubscribeBars) -> anyhow::Result<()> {
831 execute_bar_subscriptions(&cmd.bar_types, |bar_type| {
832 log::debug!(
836 "Databento does not support granular unsubscribing - ignoring unsubscribe request for {bar_type}"
837 );
838
839 Ok(())
840 })
841 }
842
843 fn unsubscribe_instrument_status(
844 &mut self,
845 cmd: &UnsubscribeInstrumentStatus,
846 ) -> anyhow::Result<()> {
847 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
848 log::debug!(
852 "Databento does not support granular unsubscribing - ignoring unsubscribe request for {instrument_id}"
853 );
854
855 Ok(())
856 })
857 }
858
859 fn request_instruments(&self, request: RequestInstruments) -> anyhow::Result<()> {
860 log::debug!("Request instruments: {request:?}");
861
862 let historical_client = self.historical.clone();
863 let data_sender = self.data_sender.clone();
864 let dataset = request
865 .venue
866 .map(|venue| self.get_dataset_for_venue(venue))
867 .transpose()?
868 .unwrap_or_else(|| "GLBX.MDP3".to_string());
869 let request_id = request.request_id;
870 let client_id = request.client_id.unwrap_or(self.client_id);
871 let venue = request.venue.unwrap_or(*DATABENTO_VENUE);
872 let start_nanos = datetime_to_unix_nanos(request.start);
873 let end_nanos = datetime_to_unix_nanos(request.end);
874 let request_params = request.params;
875 let (query_start, query_end) = resolve_request_time_range(start_nanos, end_nanos);
876
877 self.spawn_task(async move {
878 let query_params = instruments_query_params(dataset, query_start, query_end);
879
880 match historical_client.get_range_instruments(query_params).await {
881 Ok(instruments) => {
882 log::debug!("Retrieved {} instruments", instruments.len());
883
884 let response = DataResponse::Instruments(InstrumentsResponse::new(
885 request_id,
886 client_id,
887 venue,
888 instruments,
889 start_nanos,
890 end_nanos,
891 get_atomic_clock_realtime().get_time_ns(),
892 request_params,
893 ));
894
895 send_data_response(&data_sender, response, "instruments");
896 }
897 Err(e) => {
898 send_request_result(&data_sender, request_id, None, &[e.to_string()]);
899 }
900 }
901 });
902
903 Ok(())
904 }
905
906 fn request_instrument(&self, request: RequestInstrument) -> anyhow::Result<()> {
907 let groups = self.group_request_instruments(&request.instrument_ids)?;
908 let historical_client = self.historical.clone();
909 let data_sender = self.data_sender.clone();
910 let client_id = request.client_id.unwrap_or(self.client_id);
911 let start = datetime_to_unix_nanos(request.start);
912 let end = datetime_to_unix_nanos(request.end);
913 let (query_start, query_end) = resolve_request_time_range(start, end);
914 self.spawn_task(async move {
915 let mut data = Vec::new();
916 let mut failures = Vec::new();
917
918 for (dataset, ids) in groups {
919 let symbols = historical_client.prepare_symbols_from_instrument_ids(&ids);
920
921 let params = RangeQueryParams {
922 dataset: dataset.clone(),
923 symbols,
924 start: query_start,
925 end: query_end,
926 limit: None,
927 price_precision: None,
928 };
929
930 match historical_client.get_range_instruments(params).await {
931 Ok(instruments) => {
932 for id in ids {
933 if let Some(instrument) = requested_instrument(&instruments, id) {
934 data.push(instrument);
935 } else {
936 failures.push(format!("Instrument not found: {id}"));
937 }
938 }
939 }
940 Err(e) => failures.push(format!("Dataset {dataset}: {e}")),
941 }
942 }
943
944 let ts_init = get_atomic_clock_realtime().get_time_ns();
945
946 let response = if request.instrument_ids.len() == 1 && data.len() == 1 {
947 let instrument = data.remove(0);
948 Some(DataResponse::Instrument(Box::new(InstrumentResponse::new(
949 request.request_id,
950 client_id,
951 instrument.id(),
952 instrument,
953 start,
954 end,
955 ts_init,
956 request.params,
957 ))))
958 } else if request.instrument_ids.len() > 1 {
959 Some(DataResponse::Instruments(InstrumentsResponse::new(
960 request.request_id,
961 client_id,
962 *DATABENTO_VENUE,
963 data,
964 start,
965 end,
966 ts_init,
967 request.params,
968 )))
969 } else {
970 None
971 };
972
973 send_request_result(&data_sender, request.request_id, response, &failures);
974 });
975
976 Ok(())
977 }
978
979 fn request_quotes(&self, request: RequestQuotes) -> anyhow::Result<()> {
980 anyhow::ensure!(
981 request.instrument_ids.len() == 1 || request.limit.is_none(),
982 "A record limit is unsupported for multi-instrument requests"
983 );
984 let groups = self.group_request_instruments(&request.instrument_ids)?;
985 let historical_client = self.historical.clone();
986 let data_sender = self.data_sender.clone();
987 let client_id = request.client_id.unwrap_or(self.client_id);
988 let start = datetime_to_unix_nanos(request.start);
989 let end = datetime_to_unix_nanos(request.end);
990 let limit = request.limit.map(|value| value.get() as u64);
991 let precision = price_precision_from_params(request.params.as_ref())?;
992 let schema = schema_from_params(request.params.as_ref(), dbn::Schema::Mbp1, QUOTE_SCHEMAS)?
993 .to_string();
994 let (query_start, query_end) = resolve_request_time_range(start, end);
995 self.spawn_task(async move {
996 let mut data = Vec::new();
997 let mut failures = Vec::new();
998
999 for (dataset, ids) in groups {
1000 for &id in &ids {
1001 seed_price_precision_if_needed(
1002 &historical_client,
1003 &dataset,
1004 id,
1005 query_start,
1006 query_end,
1007 precision,
1008 )
1009 .await;
1010 }
1011
1012 let symbols = historical_client.prepare_symbols_from_instrument_ids(&ids);
1013
1014 let params = RangeQueryParams {
1015 dataset: dataset.clone(),
1016 symbols,
1017 start: query_start,
1018 end: query_end,
1019 limit,
1020 price_precision: precision,
1021 };
1022
1023 match historical_client
1024 .get_range_quotes(params, Some(schema.clone()))
1025 .await
1026 {
1027 Ok(rows) => data.extend(rows),
1028 Err(e) => failures.push(format!("Dataset {dataset}: {e}")),
1029 }
1030 }
1031
1032 data.sort_by_key(|row| (row.ts_init, row.instrument_id));
1033 let response = DataResponse::Quotes(QuotesResponse::new(
1034 request.request_id,
1035 client_id,
1036 request.instrument_ids,
1037 data,
1038 start,
1039 end,
1040 get_atomic_clock_realtime().get_time_ns(),
1041 request.params,
1042 ));
1043 send_request_result(&data_sender, request.request_id, Some(response), &failures);
1044 });
1045
1046 Ok(())
1047 }
1048
1049 fn request_trades(&self, request: RequestTrades) -> anyhow::Result<()> {
1050 anyhow::ensure!(
1051 request.instrument_ids.len() == 1 || request.limit.is_none(),
1052 "A record limit is unsupported for multi-instrument requests"
1053 );
1054 let groups = self.group_request_instruments(&request.instrument_ids)?;
1055 let historical_client = self.historical.clone();
1056 let data_sender = self.data_sender.clone();
1057 let client_id = request.client_id.unwrap_or(self.client_id);
1058 let start = datetime_to_unix_nanos(request.start);
1059 let end = datetime_to_unix_nanos(request.end);
1060 let limit = request.limit.map(|value| value.get() as u64);
1061 let precision = price_precision_from_params(request.params.as_ref())?;
1062 let schema =
1063 schema_from_params(request.params.as_ref(), dbn::Schema::Trades, TRADE_SCHEMAS)?
1064 .to_string();
1065 let (query_start, query_end) = resolve_request_time_range(start, end);
1066 self.spawn_task(async move {
1067 let mut data = Vec::new();
1068 let mut failures = Vec::new();
1069
1070 for (dataset, ids) in groups {
1071 for &id in &ids {
1072 seed_price_precision_if_needed(
1073 &historical_client,
1074 &dataset,
1075 id,
1076 query_start,
1077 query_end,
1078 precision,
1079 )
1080 .await;
1081 }
1082
1083 let symbols = historical_client.prepare_symbols_from_instrument_ids(&ids);
1084
1085 let params = RangeQueryParams {
1086 dataset: dataset.clone(),
1087 symbols,
1088 start: query_start,
1089 end: query_end,
1090 limit,
1091 price_precision: precision,
1092 };
1093
1094 match historical_client
1095 .get_range_trades(params, Some(schema.clone()))
1096 .await
1097 {
1098 Ok(rows) => data.extend(rows),
1099 Err(e) => failures.push(format!("Dataset {dataset}: {e}")),
1100 }
1101 }
1102
1103 data.sort_by_key(|row| (row.ts_init, row.instrument_id));
1104 let response = DataResponse::Trades(TradesResponse::new(
1105 request.request_id,
1106 client_id,
1107 request.instrument_ids,
1108 data,
1109 start,
1110 end,
1111 get_atomic_clock_realtime().get_time_ns(),
1112 request.params,
1113 ));
1114 send_request_result(&data_sender, request.request_id, Some(response), &failures);
1115 });
1116
1117 Ok(())
1118 }
1119
1120 fn request_bars(&self, request: RequestBars) -> anyhow::Result<()> {
1121 let bar_type = request.single_bar_type()?;
1122 let aggregation = bar_type.spec().aggregation;
1123 anyhow::ensure!(
1124 matches!(
1125 aggregation,
1126 BarAggregation::Second
1127 | BarAggregation::Minute
1128 | BarAggregation::Hour
1129 | BarAggregation::Day
1130 ),
1131 "Unsupported bar aggregation: {aggregation:?}"
1132 );
1133
1134 log::debug!("Request bars: {request:?}");
1135
1136 let historical_client = self.historical.clone();
1137 let data_sender = self.data_sender.clone();
1138 let instrument_id = bar_type.instrument_id();
1139 let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
1140 let symbols = historical_client.prepare_symbols_from_instrument_ids(&[instrument_id]);
1141 let request_id = request.request_id;
1142 let client_id = request.client_id.unwrap_or(self.client_id);
1143 let start_nanos = datetime_to_unix_nanos(request.start);
1144 let end_nanos = datetime_to_unix_nanos(request.end);
1145 let limit = request.limit.map(|limit| limit.get() as u64);
1146 let request_params = request.params;
1147 let price_precision = price_precision_from_params(request_params.as_ref())?;
1148 let timestamp_on_close = self.config.bars_timestamp_on_close;
1149 let spec = BarSpecification::new(1, aggregation, PriceType::Last);
1150 let interval = DurationNanos::try_from(spec.timedelta()).expect("supported OHLCV interval");
1151 let (start, end) = resolve_request_bounds(start_nanos, end_nanos);
1152
1153 let (query_start, query_end) = resolve_request_time_range(
1155 Some(start.saturating_sub(interval)),
1156 Some(end.saturating_sub(interval)),
1157 );
1158
1159 self.spawn_task(async move {
1160 seed_price_precision_if_needed(
1161 &historical_client,
1162 dataset.as_str(),
1163 instrument_id,
1164 query_start,
1165 query_end,
1166 price_precision,
1167 )
1168 .await;
1169
1170 let params = RangeQueryParams {
1171 dataset,
1172 symbols,
1173 start: query_start,
1174 end: query_end,
1175 limit,
1176 price_precision,
1177 };
1178
1179 match historical_client
1180 .get_range_bars(params, aggregation, timestamp_on_close)
1181 .await
1182 {
1183 Ok(bars) => {
1184 log::debug!("Retrieved {} bars", bars.len());
1185 let response = DataResponse::Bars(BarsResponse::new(
1186 request_id,
1187 client_id,
1188 bar_type,
1189 bars,
1190 start_nanos,
1191 end_nanos,
1192 get_atomic_clock_realtime().get_time_ns(),
1193 request_params,
1194 ));
1195
1196 send_data_response(&data_sender, response, "bars");
1197 }
1198 Err(e) => {
1199 send_request_result(&data_sender, request_id, None, &[e.to_string()]);
1200 }
1201 }
1202 });
1203
1204 Ok(())
1205 }
1206
1207 fn request_book_depth(&self, request: RequestBookDepth) -> anyhow::Result<()> {
1208 let instrument_id = request.single_instrument_id()?;
1209
1210 log::debug!("Request book depth: {request:?}");
1211
1212 let historical_client = self.historical.clone();
1213 let data_sender = self.data_sender.clone();
1214 let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
1215 let symbols = historical_client.prepare_symbols_from_instrument_ids(&[instrument_id]);
1216 let request_id = request.request_id;
1217 let client_id = request.client_id.unwrap_or(self.client_id);
1218 let start_nanos = datetime_to_unix_nanos(request.start);
1219 let end_nanos = datetime_to_unix_nanos(request.end);
1220 let limit = request.limit.map(|limit| limit.get() as u64);
1221 let depth = request.depth.map(|depth| depth.get());
1222 let request_params = request.params;
1223 let price_precision = price_precision_from_params(request_params.as_ref())?;
1224 let (query_start, query_end) = resolve_request_time_range(start_nanos, end_nanos);
1225
1226 self.spawn_task(async move {
1227 seed_price_precision_if_needed(
1228 &historical_client,
1229 dataset.as_str(),
1230 instrument_id,
1231 query_start,
1232 query_end,
1233 price_precision,
1234 )
1235 .await;
1236
1237 let params = RangeQueryParams {
1238 dataset,
1239 symbols,
1240 start: query_start,
1241 end: query_end,
1242 limit,
1243 price_precision,
1244 };
1245
1246 match historical_client
1247 .get_range_order_book_depth(params, depth)
1248 .await
1249 {
1250 Ok(depths) => {
1251 log::debug!("Retrieved {} order book depths", depths.len());
1252 let response = DataResponse::BookDepth(BookDepthResponse::new(
1253 request_id,
1254 client_id,
1255 instrument_id,
1256 depths,
1257 start_nanos,
1258 end_nanos,
1259 get_atomic_clock_realtime().get_time_ns(),
1260 request_params,
1261 ));
1262
1263 send_data_response(&data_sender, response, "book depth");
1264 }
1265 Err(e) => {
1266 send_request_result(&data_sender, request_id, None, &[e.to_string()]);
1267 }
1268 }
1269 });
1270
1271 Ok(())
1272 }
1273
1274 fn request_book_deltas(&self, request: RequestBookDeltas) -> anyhow::Result<()> {
1275 let instrument_id = request.single_instrument_id()?;
1276
1277 log::debug!("Request book deltas: {request:?}");
1278
1279 let historical_client = self.historical.clone();
1280 let data_sender = self.data_sender.clone();
1281 let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
1282 let symbols = historical_client.prepare_symbols_from_instrument_ids(&[instrument_id]);
1283 let request_id = request.request_id;
1284 let client_id = request.client_id.unwrap_or(self.client_id);
1285 let start_nanos = datetime_to_unix_nanos(request.start);
1286 let end_nanos = datetime_to_unix_nanos(request.end);
1287 let limit = request.limit.map(|limit| limit.get() as u64);
1288 let request_params = request.params;
1289 let price_precision = price_precision_from_params(request_params.as_ref())?;
1290 let (query_start, query_end) = resolve_request_time_range(start_nanos, end_nanos);
1291
1292 self.spawn_task(async move {
1293 seed_price_precision_if_needed(
1294 &historical_client,
1295 dataset.as_str(),
1296 instrument_id,
1297 query_start,
1298 query_end,
1299 price_precision,
1300 )
1301 .await;
1302
1303 let params = RangeQueryParams {
1304 dataset,
1305 symbols,
1306 start: query_start,
1307 end: query_end,
1308 limit,
1309 price_precision,
1310 };
1311
1312 match historical_client.get_range_order_book_deltas(params).await {
1313 Ok(deltas) => {
1314 log::debug!("Retrieved {} order book deltas", deltas.len());
1315 let response = BookDeltasResponse::new(
1316 request_id,
1317 client_id,
1318 instrument_id,
1319 deltas,
1320 start_nanos,
1321 end_nanos,
1322 get_atomic_clock_realtime().get_time_ns(),
1323 request_params,
1324 );
1325
1326 send_book_deltas_response(&data_sender, response);
1327 }
1328 Err(e) => {
1329 send_request_result(&data_sender, request_id, None, &[e.to_string()]);
1330 }
1331 }
1332 });
1333
1334 Ok(())
1335 }
1336}
1337
1338impl DatabentoDataClient {
1339 fn subscribe_batch(
1340 &self,
1341 instrument_ids: Vec<InstrumentId>,
1342 params: Option<&Params>,
1343 default_schema: dbn::Schema,
1344 schemas: &[dbn::Schema],
1345 ) -> anyhow::Result<()> {
1346 let price_precision = price_precision_from_params(params)?;
1347 let schema = schema_from_params(params, default_schema, schemas)?;
1348 let ids = instrument_ids.into_instrument_ids()?;
1349 let groups = self.group_instruments(&ids)?;
1350
1351 let mut accepted = Vec::new();
1352 let mut failure = None;
1353
1354 for (dataset, ids) in &groups {
1355 match self.subscribe_dataset(dataset, ids, schema, price_precision) {
1356 Ok(()) => accepted.extend_from_slice(ids),
1357 Err(e) => {
1358 log::error!(
1359 "Databento subscription enqueue failed: dataset={dataset}, schema={schema}, instrument_ids={ids:?}"
1360 );
1361
1362 if failure.is_none() {
1363 failure = Some(e);
1364 }
1365 }
1366 }
1367 }
1368
1369 match failure {
1370 Some(e) => Err(SubscriptionError::instruments(accepted, e).into()),
1371 None => Ok(()),
1372 }
1373 }
1374
1375 fn group_request_instruments(
1376 &self,
1377 ids: &[InstrumentId],
1378 ) -> anyhow::Result<IndexMap<String, Vec<InstrumentId>>> {
1379 let groups = self.group_instruments(ids)?;
1380 for ids in groups.values() {
1381 let symbols = ids.iter().map(|id| id.symbol.as_str()).collect::<Vec<_>>();
1382 check_consistent_symbology(&symbols)?;
1383 }
1384
1385 Ok(groups)
1386 }
1387
1388 fn group_instruments(
1389 &self,
1390 ids: &[InstrumentId],
1391 ) -> anyhow::Result<IndexMap<String, Vec<InstrumentId>>> {
1392 let mut groups: IndexMap<String, Vec<InstrumentId>> = IndexMap::new();
1393 let mut symbol_venues = AHashMap::new();
1394
1395 for &id in ids {
1396 let dataset = self.get_dataset_for_venue(id.venue)?;
1397 if let Some(venue) = symbol_venues.insert(id.symbol, id.venue) {
1398 anyhow::ensure!(
1399 venue == id.venue,
1400 "Identifiers share raw symbol {} across different venues",
1401 id.symbol
1402 );
1403 }
1404
1405 groups.entry(dataset).or_default().push(id);
1406 }
1407
1408 Ok(groups)
1409 }
1410
1411 fn subscribe_dataset(
1412 &self,
1413 dataset: &str,
1414 ids: &[InstrumentId],
1415 schema: dbn::Schema,
1416 price_precision: Option<u8>,
1417 ) -> anyhow::Result<()> {
1418 let symbols: Vec<String> = ids.iter().map(|id| id.symbol.to_string()).collect();
1419 let precisions: Vec<(Symbol, u8)> = ids
1420 .iter()
1421 .filter_map(|id| price_precision.map(|precision| (id.symbol, precision)))
1422 .collect();
1423 let subscription = Subscription::builder()
1424 .schema(schema)
1425 .symbols(symbols)
1426 .build();
1427 let start_after_subscribe = self.get_or_create_feed_handler(dataset);
1428 let previous_venues: Vec<_> = ids
1429 .iter()
1430 .map(|id| (id.symbol, self.symbol_venue_map.get_cloned(&id.symbol)))
1431 .collect();
1432
1433 for id in ids {
1434 self.symbol_venue_map.insert(id.symbol, id.venue);
1435 }
1436
1437 if let Err(e) = self.send_subscription_to_dataset(
1438 dataset,
1439 precisions,
1440 subscription,
1441 start_after_subscribe,
1442 ) {
1443 for (symbol, previous) in previous_venues {
1444 if let Some(venue) = previous {
1445 self.symbol_venue_map.insert(symbol, venue);
1446 } else {
1447 self.symbol_venue_map.remove(&symbol);
1448 }
1449 }
1450
1451 return Err(e);
1452 }
1453
1454 Ok(())
1455 }
1456}
1457
1458fn instruments_query_params(
1459 dataset: String,
1460 start_nanos: UnixNanos,
1461 end_nanos: Option<UnixNanos>,
1462) -> RangeQueryParams {
1463 RangeQueryParams {
1464 dataset,
1465 symbols: vec!["ALL_SYMBOLS".to_string()],
1466 start: start_nanos,
1467 end: end_nanos,
1468 limit: None,
1469 price_precision: None,
1470 }
1471}
1472
1473fn instrument_query_params(
1474 dataset: String,
1475 instrument_id: InstrumentId,
1476 start_nanos: UnixNanos,
1477 end_nanos: Option<UnixNanos>,
1478) -> RangeQueryParams {
1479 RangeQueryParams {
1480 dataset,
1481 symbols: vec![instrument_id_to_symbol_string(
1482 instrument_id,
1483 &mut AHashMap::new(),
1484 )],
1485 start: start_nanos,
1486 end: end_nanos,
1487 limit: None,
1488 price_precision: None,
1489 }
1490}
1491
1492fn resolve_request_time_range(
1493 start_nanos: Option<UnixNanos>,
1494 end_nanos: Option<UnixNanos>,
1495) -> (UnixNanos, Option<UnixNanos>) {
1496 let (start, end) = resolve_request_bounds(start_nanos, end_nanos);
1497
1498 (start, Some(end.saturating_add(DurationNanos::new(1))))
1500}
1501
1502fn resolve_request_bounds(
1503 start_nanos: Option<UnixNanos>,
1504 end_nanos: Option<UnixNanos>,
1505) -> (UnixNanos, UnixNanos) {
1506 let end = end_nanos.unwrap_or_else(|| get_atomic_clock_realtime().get_time_ns());
1507 let mut start = start_nanos.unwrap_or_else(|| start_of_utc_day(end));
1508
1509 if start > end {
1510 start = end;
1511 }
1512
1513 (start, end)
1514}
1515
1516fn start_of_utc_day(timestamp: UnixNanos) -> UnixNanos {
1517 timestamp.floor(DurationNanos::from_days(1))
1518}
1519
1520async fn seed_price_precision_if_needed(
1521 historical_client: &DatabentoHistoricalClient,
1522 dataset: &str,
1523 instrument_id: InstrumentId,
1524 start_nanos: UnixNanos,
1525 end_nanos: Option<UnixNanos>,
1526 price_precision: Option<u8>,
1527) {
1528 if price_precision.is_some()
1529 || historical_client
1530 .price_precision(instrument_id.symbol)
1531 .is_some()
1532 {
1533 return;
1534 }
1535
1536 let query_start = start_of_utc_day(start_nanos);
1537 let query_params =
1538 instrument_query_params(dataset.to_string(), instrument_id, query_start, end_nanos);
1539
1540 if let Err(e) = historical_client.get_range_instruments(query_params).await {
1541 log::warn!("Failed to seed price precision for {instrument_id}: {e}");
1542 }
1543}
1544
1545fn send_request_result(
1546 data_sender: &EventSender<DataEvent>,
1547 request_id: nautilus_core::UUID4,
1548 response: Option<DataResponse>,
1549 failures: &[String],
1550) {
1551 let event = if failures.is_empty() {
1552 let Some(response) = response else {
1553 return;
1554 };
1555
1556 let envelope = DataResponseEnvelope::new(response, true, None);
1557 DataEvent::ResponseEnvelope(envelope)
1558 } else {
1559 DataEvent::RequestFailed(DataRequestFailed {
1560 request_id,
1561 reason: failures.join("; "),
1562 response,
1563 })
1564 };
1565
1566 if let Err(e) = data_sender.send(event) {
1567 log::error!("Failed to send request result: {e}");
1568 }
1569}
1570
1571fn send_data_response(data_sender: &EventSender<DataEvent>, response: DataResponse, label: &str) {
1572 let envelope = DataResponseEnvelope::new(response, true, None);
1573 if let Err(e) = data_sender.send(DataEvent::ResponseEnvelope(envelope)) {
1574 log::error!("Failed to send {label} response: {e}");
1575 }
1576}
1577
1578fn send_book_deltas_response(sender: &EventSender<DataEvent>, response: BookDeltasResponse) {
1579 let responses = partition_book_deltas_response(response);
1580 let last = responses.len() - 1;
1581 for (index, response) in responses.into_iter().enumerate() {
1582 let envelope =
1583 DataResponseEnvelope::new(DataResponse::BookDeltas(response), index == last, None);
1584
1585 if let Err(e) = sender.send(DataEvent::ResponseEnvelope(envelope)) {
1586 log::error!("Failed to send book deltas response: {e}");
1587 }
1588 }
1589}
1590
1591fn partition_book_deltas_response(response: BookDeltasResponse) -> Vec<BookDeltasResponse> {
1592 if response.data.is_empty() {
1593 return vec![response];
1594 }
1595
1596 let mut partitions = IndexMap::new();
1597 for delta in response.data.iter().copied() {
1598 partitions
1599 .entry(delta.instrument_id)
1600 .or_insert_with(Vec::new)
1601 .push(delta);
1602 }
1603
1604 partitions
1605 .into_iter()
1606 .map(|(instrument_id, data)| {
1607 let mut child = response.clone();
1608 child.instrument_id = instrument_id;
1609 child.data = data.into();
1610 child
1611 })
1612 .collect()
1613}
1614
1615fn requested_instrument(
1616 instruments: &[InstrumentAny],
1617 instrument_id: InstrumentId,
1618) -> Option<InstrumentAny> {
1619 instruments
1620 .iter()
1621 .rev()
1622 .find(|instrument| instrument.id() == instrument_id)
1623 .cloned()
1624}
1625
1626fn price_precision_from_params(params: Option<&Params>) -> anyhow::Result<Option<u8>> {
1627 let Some(price_precision) = params.and_then(|params| params.get_u64(PRICE_PRECISION_PARAM))
1628 else {
1629 return Ok(None);
1630 };
1631
1632 Ok(Some(u8::try_from(price_precision).map_err(|_| {
1633 anyhow::anyhow!(
1634 "`{PRICE_PRECISION_PARAM}` must be less than or equal to {}",
1635 u8::MAX
1636 )
1637 })?))
1638}
1639
1640fn custom_data_for_instrument(
1641 data: Arc<dyn CustomDataTrait>,
1642 instrument_id: InstrumentId,
1643) -> CustomData {
1644 let data_type = DataType::new(data.type_name(), None, Some(instrument_id.to_string()));
1645 CustomData::new(data, data_type)
1646}
1647
1648fn custom_data_schema(data_type: &DataType) -> anyhow::Result<dbn::Schema> {
1649 match data_type.type_name() {
1650 name if name == DatabentoStatistics::type_name_static() => Ok(dbn::Schema::Statistics),
1651 name if name == DatabentoImbalance::type_name_static() => Ok(dbn::Schema::Imbalance),
1652 name => anyhow::bail!(
1653 "Unsupported custom data type for Databento live subscription: {name}, expected {} or {}",
1654 DatabentoStatistics::type_name_static(),
1655 DatabentoImbalance::type_name_static(),
1656 ),
1657 }
1658}
1659
1660fn custom_data_instrument_id(data_type: &DataType) -> anyhow::Result<InstrumentId> {
1661 anyhow::ensure!(
1663 data_type
1664 .metadata()
1665 .is_none_or(|metadata| metadata.is_empty()),
1666 "{} subscriptions do not support metadata, received {}",
1667 data_type.type_name(),
1668 data_type.metadata_str(),
1669 );
1670
1671 let identifier = data_type.identifier().ok_or_else(|| {
1672 anyhow::anyhow!(
1673 "{} subscriptions require an instrument ID identifier",
1674 data_type.type_name()
1675 )
1676 })?;
1677
1678 InstrumentId::from_str(identifier)
1679 .map_err(|e| anyhow::anyhow!("Invalid instrument ID identifier `{identifier}`: {e}"))
1680}
1681
1682fn ohlcv_schema(bar_type: BarType) -> anyhow::Result<dbn::Schema> {
1683 let spec = bar_type.spec();
1684 anyhow::ensure!(
1685 spec.price_type == PriceType::Last,
1686 "Unsupported bar price type for Databento OHLCV: {spec}, only LAST is emitted",
1687 );
1688 anyhow::ensure!(
1689 spec.step.get() == 1,
1690 "Unsupported bar step for Databento OHLCV: {spec}, only a step of 1 is available",
1691 );
1692
1693 ohlcv_schema_from_aggregation(spec.aggregation)
1694}
1695
1696fn schema_from_params(
1697 params: Option<&Params>,
1698 default_schema: dbn::Schema,
1699 allowed_schemas: &[dbn::Schema],
1700) -> anyhow::Result<dbn::Schema> {
1701 let schema = if let Some(schema) = params.and_then(|params| params.get_str(SCHEMA_PARAM)) {
1702 dbn::Schema::from_str(schema)?
1703 } else {
1704 default_schema
1705 };
1706
1707 if allowed_schemas.contains(&schema) {
1708 return Ok(schema);
1709 }
1710
1711 let allowed = allowed_schemas
1712 .iter()
1713 .map(dbn::Schema::as_str)
1714 .collect::<Vec<_>>()
1715 .join(", ");
1716 anyhow::bail!(
1717 "Invalid `{SCHEMA_PARAM}` '{}'. Must be one of: {allowed}",
1718 schema.as_str()
1719 );
1720}
1721
1722fn send_subscription_commands(
1723 tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1724 dataset: &str,
1725 price_precisions: impl IntoIterator<Item = (Symbol, u8)>,
1726 subscription: Subscription,
1727 start_after_subscribe: bool,
1728) -> anyhow::Result<()> {
1729 for (symbol, precision) in price_precisions {
1730 tx.send(HandlerCommand::SetPricePrecision(symbol, precision))
1731 .map_err(|e| anyhow::anyhow!("Failed to send command to dataset {dataset}: {e}"))?;
1732 }
1733
1734 tx.send(HandlerCommand::Subscribe(subscription))
1735 .map_err(|e| anyhow::anyhow!("Failed to send command to dataset {dataset}: {e}"))?;
1736
1737 if start_after_subscribe {
1738 tx.send(HandlerCommand::Start)
1739 .map_err(|e| anyhow::anyhow!("Failed to send command to dataset {dataset}: {e}"))?;
1740 }
1741
1742 Ok(())
1743}
1744
1745#[cfg(test)]
1746mod tests {
1747 use std::{num::NonZeroUsize, path::PathBuf};
1748
1749 use databento::dbn::{
1750 MappingInterval, SymbolMapping,
1751 decode::{DbnMetadata, DecodeRecord, dbn::Decoder},
1752 encode::EncodeRecord,
1753 };
1754 use nautilus_common::{
1755 live::runner::replace_data_event_sender,
1756 msgbus::{self, switchboard::get_custom_topic},
1757 };
1758 use nautilus_core::UUID4;
1759 use nautilus_model::{
1760 data::{BatchView, NautilusDataType, OrderBookDelta, QuoteTick},
1761 enums::BookType,
1762 identifiers::{ClientId, InstrumentId},
1763 instruments::{CurrencyPair, InstrumentAny},
1764 types::{Currency, Price, Quantity},
1765 };
1766 use rstest::rstest;
1767 use serde_json::json;
1768 use time::macros::date;
1769 use tokio::{
1770 io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
1771 net::TcpListener,
1772 };
1773
1774 use super::*;
1775 use crate::enums::{DatabentoStatisticType, DatabentoStatisticUpdateAction};
1776
1777 #[derive(Clone, Copy)]
1778 enum SubscribeKind {
1779 Quotes,
1780 Trades,
1781 }
1782
1783 fn currency_pair(instrument_id: &str) -> InstrumentAny {
1784 currency_pair_with_ts_init(instrument_id, UnixNanos::default())
1785 }
1786
1787 fn test_data_client() -> DatabentoDataClient {
1788 let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
1789 replace_data_event_sender(sender);
1790
1791 let config = DatabentoDataClientConfig::new(
1792 "32-character-with-lots-of-filler",
1793 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
1794 true,
1795 true,
1796 );
1797 DatabentoDataClient::new(
1798 ClientId::from("DATABENTO-TEST"),
1799 config,
1800 get_atomic_clock_realtime(),
1801 )
1802 .expect("test client should initialize")
1803 }
1804
1805 #[rstest]
1806 #[case::quotes(NautilusDataType::QuoteTick, true)]
1807 #[case::trades(NautilusDataType::TradeTick, true)]
1808 #[case::bars_close(NautilusDataType::Bar, true)]
1809 #[case::bars_open(NautilusDataType::Bar, false)]
1810 #[tokio::test]
1811 #[ignore = "Requires DATABENTO_API_KEY and GLBX.MDP3 historical access"]
1812 async fn test_historical_request_list_live(
1813 #[case] kind: NautilusDataType,
1814 #[case] timestamp_on_close: bool,
1815 ) {
1816 let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel();
1817 replace_data_event_sender(sender);
1818 let api_key = std::env::var("DATABENTO_API_KEY").unwrap();
1819
1820 let config = DatabentoDataClientConfig::new(
1821 api_key,
1822 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
1823 true,
1824 timestamp_on_close,
1825 );
1826 let client_id = ClientId::from("DATABENTO-HISTORY");
1827 let client =
1828 DatabentoDataClient::new(client_id, config, get_atomic_clock_realtime()).unwrap();
1829 let mut ids = vec![
1830 InstrumentId::from("ESM4.GLBX"),
1831 InstrumentId::from("NQM4.GLBX"),
1832 ];
1833 let start = UnixNanos::from(1_717_423_200_000_000_000);
1834
1835 let end = if kind == NautilusDataType::Bar {
1836 ids.truncate(1);
1837 start
1838 } else {
1839 start + DurationNanos::from_secs(1)
1840 };
1841
1842 let request_id = UUID4::new();
1843 let params = Params::new([(PRICE_PRECISION_PARAM.to_owned(), json!(2))].into());
1844 let bar_type = "ESM4.GLBX-1-MINUTE-LAST-EXTERNAL"
1845 .parse::<BarType>()
1846 .unwrap();
1847
1848 match kind {
1849 NautilusDataType::QuoteTick => client
1850 .request_quotes(RequestQuotes::new(
1851 ids.clone(),
1852 Some(start.to_datetime_utc()),
1853 Some(end.to_datetime_utc()),
1854 None,
1855 Some(client_id),
1856 request_id,
1857 UnixNanos::from(37),
1858 Some(params.clone()),
1859 ))
1860 .unwrap(),
1861 NautilusDataType::TradeTick => client
1862 .request_trades(RequestTrades::new(
1863 ids.clone(),
1864 Some(start.to_datetime_utc()),
1865 Some(end.to_datetime_utc()),
1866 None,
1867 Some(client_id),
1868 request_id,
1869 UnixNanos::from(37),
1870 Some(params.clone()),
1871 ))
1872 .unwrap(),
1873 NautilusDataType::Bar => client
1874 .request_bars(RequestBars::new(
1875 [bar_type],
1876 Some(start.to_datetime_utc()),
1877 Some(end.to_datetime_utc()),
1878 None,
1879 Some(client_id),
1880 request_id,
1881 UnixNanos::from(37),
1882 Some(params.clone()),
1883 ))
1884 .unwrap(),
1885 _ => panic!("unexpected live request kind"),
1886 }
1887
1888 let event = tokio::time::timeout(Duration::from_secs(60), receiver.recv())
1889 .await
1890 .unwrap()
1891 .unwrap();
1892
1893 let DataEvent::ResponseEnvelope(envelope) = event else {
1894 panic!("expected successful historical envelope, received {event:?}");
1895 };
1896
1897 let response = envelope.response().unwrap();
1898
1899 let (response_ids, rows) = match (kind, response) {
1900 (NautilusDataType::QuoteTick, DataResponse::Quotes(response)) => (
1901 response.instrument_ids.clone(),
1902 response
1903 .data
1904 .iter()
1905 .map(|row| (row.ts_init, row.instrument_id))
1906 .collect::<Vec<_>>(),
1907 ),
1908 (NautilusDataType::TradeTick, DataResponse::Trades(response)) => (
1909 response.instrument_ids.clone(),
1910 response
1911 .data
1912 .iter()
1913 .map(|row| (row.ts_init, row.instrument_id))
1914 .collect::<Vec<_>>(),
1915 ),
1916 (NautilusDataType::Bar, DataResponse::Bars(response)) => {
1917 let expected_event = if timestamp_on_close {
1918 start
1919 } else {
1920 start - DurationNanos::from_mins(1)
1921 };
1922
1923 assert_eq!(response.bar_type, bar_type);
1924 assert_eq!(response.data.len(), 1);
1925 assert_eq!(response.data[0].bar_type, bar_type);
1926 assert_eq!(response.data[0].ts_init, start);
1927 assert_eq!(response.data[0].ts_event, expected_event);
1928 (
1929 vec![response.bar_type.instrument_id()],
1930 response
1931 .data
1932 .iter()
1933 .map(|row| (row.ts_init, row.bar_type.instrument_id()))
1934 .collect::<Vec<_>>(),
1935 )
1936 }
1937 _ => panic!("unexpected historical response variant"),
1938 };
1939
1940 let mut actual_ids = rows.iter().map(|(_, id)| *id).collect::<Vec<_>>();
1941 actual_ids.sort_unstable();
1942 actual_ids.dedup();
1943 let mut expected_ids = ids.clone();
1944 expected_ids.sort_unstable();
1945
1946 assert_eq!(envelope.correlation_id(), &request_id);
1947 assert!(envelope.completed());
1948 assert_eq!(envelope.sub_correlation_id(), None);
1949 assert_eq!(envelope.failure_reason(), None);
1950 assert_eq!(response.correlation_id(), &request_id);
1951 assert_eq!(response.client_id(), &client_id);
1952 assert_eq!(response.start(), Some(start));
1953 assert_eq!(response.end(), Some(end));
1954 assert_eq!(response.params(), Some(¶ms));
1955 assert_eq!(response_ids, ids);
1956 assert_eq!(actual_ids, expected_ids);
1957 assert!(rows.is_sorted());
1958 assert!(rows.iter().all(|(ts, _)| *ts >= start && *ts <= end));
1959 assert_eq!(
1960 receiver.try_recv().unwrap_err(),
1961 tokio::sync::mpsc::error::TryRecvError::Empty
1962 );
1963 }
1964
1965 #[rstest]
1966 #[case::raw("ESM4.GLBX", "NQM4.GLBX")]
1967 #[case::parent("ES.FUT.GLBX", "NQ.FUT.GLBX")]
1968 fn test_historical_request_groups_keep_dataset_and_input_order(
1969 #[case] first: &str,
1970 #[case] third: &str,
1971 ) {
1972 let client = test_data_client();
1973 let first = InstrumentId::from(first);
1974 let second = InstrumentId::from("AAPL.EQUS");
1975 let third = InstrumentId::from(third);
1976 let groups = client
1977 .group_request_instruments(&[first, second, third])
1978 .unwrap();
1979 assert_eq!(
1980 groups.into_iter().collect::<Vec<_>>(),
1981 vec![
1982 ("GLBX.MDP3".to_owned(), vec![first, third]),
1983 ("EQUS.MINI".to_owned(), vec![second])
1984 ]
1985 );
1986 }
1987
1988 #[rstest]
1989 #[case::instruments(NautilusDataType::Instrument)]
1990 #[case::quotes(NautilusDataType::QuoteTick)]
1991 #[case::trades(NautilusDataType::TradeTick)]
1992 #[tokio::test]
1993 async fn test_historical_requests_reject_mixed_symbology_before_dispatch(
1994 #[case] kind: NautilusDataType,
1995 #[values(false, true)] earlier_dataset: bool,
1996 ) {
1997 let client = test_data_client();
1998 let mut ids = Vec::new();
1999 if earlier_dataset {
2000 ids.push(InstrumentId::from("AAPL.EQUS"));
2001 }
2002
2003 ids.extend([
2004 InstrumentId::from("ES.FUT.GLBX"),
2005 InstrumentId::from("ESM4.GLBX"),
2006 ]);
2007 let request_id = UUID4::new();
2008
2009 let result = match kind {
2010 NautilusDataType::Instrument => client.request_instrument(RequestInstrument::new(
2011 ids,
2012 None,
2013 None,
2014 None,
2015 request_id,
2016 UnixNanos::from(37),
2017 None,
2018 )),
2019 NautilusDataType::QuoteTick => client.request_quotes(RequestQuotes::new(
2020 ids,
2021 None,
2022 None,
2023 None,
2024 None,
2025 request_id,
2026 UnixNanos::from(37),
2027 None,
2028 )),
2029 NautilusDataType::TradeTick => client.request_trades(RequestTrades::new(
2030 ids,
2031 None,
2032 None,
2033 None,
2034 None,
2035 request_id,
2036 UnixNanos::from(37),
2037 None,
2038 )),
2039 _ => panic!("unexpected historical request kind"),
2040 };
2041
2042 assert_eq!(
2043 result.unwrap_err().to_string(),
2044 "Inconsistent symbology types: 'parent' for ES.FUT vs 'raw_symbol' for ESM4"
2045 );
2046 assert!(client.task_handles.is_empty());
2047 }
2048
2049 #[rstest]
2050 #[case::quotes(true)]
2051 #[case::trades(false)]
2052 fn test_historical_request_lists_reject_limit_before_dispatch(#[case] quotes: bool) {
2053 let client = test_data_client();
2054 let ids = [
2055 InstrumentId::from("ESM4.GLBX"),
2056 InstrumentId::from("AAPL.EQUS"),
2057 ];
2058
2059 let result = if quotes {
2060 client.request_quotes(RequestQuotes::new(
2061 ids,
2062 None,
2063 None,
2064 NonZeroUsize::new(5),
2065 None,
2066 UUID4::new(),
2067 UnixNanos::from(37),
2068 None,
2069 ))
2070 } else {
2071 client.request_trades(RequestTrades::new(
2072 ids,
2073 None,
2074 None,
2075 NonZeroUsize::new(5),
2076 None,
2077 UUID4::new(),
2078 UnixNanos::from(37),
2079 None,
2080 ))
2081 };
2082
2083 assert_eq!(
2084 result.unwrap_err().to_string(),
2085 "A record limit is unsupported for multi-instrument requests"
2086 );
2087 }
2088
2089 #[rstest]
2090 fn test_historical_bars_reject_unsupported_aggregation_before_dispatch() {
2091 let client = test_data_client();
2092
2093 let request = RequestBars::new(
2094 BarType::from("ESM4.GLBX-100-TICK-LAST-EXTERNAL"),
2095 None,
2096 None,
2097 None,
2098 None,
2099 UUID4::new(),
2100 UnixNanos::from(37),
2101 None,
2102 );
2103 assert_eq!(
2104 client.request_bars(request).unwrap_err().to_string(),
2105 "Unsupported bar aggregation: Tick"
2106 );
2107 assert!(client.task_handles.is_empty());
2108 }
2109
2110 #[rstest]
2111 fn test_dataset_failure_preserves_successful_group_rows() {
2112 let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2113 let sender: EventSender<DataEvent> = sender.into();
2114 let root = UUID4::new();
2115 let first = InstrumentId::from("ESM6.GLBX");
2116 let second = InstrumentId::from("AAPL.EQUS");
2117
2118 let quote = QuoteTick {
2119 instrument_id: first,
2120 ts_init: UnixNanos::from(17),
2121 ..Default::default()
2122 };
2123
2124 let response = QuotesResponse::new(
2125 root,
2126 ClientId::from("HISTORY"),
2127 [first, second],
2128 vec![quote],
2129 Some(UnixNanos::from(11)),
2130 Some(UnixNanos::from(29)),
2131 UnixNanos::from(31),
2132 None,
2133 );
2134
2135 send_request_result(
2136 &sender,
2137 root,
2138 Some(DataResponse::Quotes(response)),
2139 &["Dataset EQUS: source unavailable".to_owned()],
2140 );
2141
2142 let DataEvent::RequestFailed(failure) = receiver.try_recv().unwrap() else {
2143 panic!("Expected failed dataset result");
2144 };
2145
2146 assert_eq!(failure.request_id, root);
2147 assert_eq!(failure.reason, "Dataset EQUS: source unavailable");
2148
2149 let Some(DataResponse::Quotes(response)) = failure.response else {
2150 panic!("Expected successful sibling rows");
2151 };
2152
2153 assert_eq!(response.correlation_id, root);
2154 assert_eq!(response.client_id, ClientId::from("HISTORY"));
2155 assert_eq!(response.instrument_ids, vec![first, second]);
2156 assert_eq!(response.data.as_ref(), &[quote]);
2157 assert_eq!(response.start, Some(UnixNanos::from(11)));
2158 assert_eq!(response.end, Some(UnixNanos::from(29)));
2159 assert_eq!(response.ts_init, UnixNanos::from(31));
2160 assert_eq!(response.params, None);
2161 assert_eq!(
2162 receiver.try_recv().unwrap_err(),
2163 tokio::sync::mpsc::error::TryRecvError::Empty
2164 );
2165 }
2166
2167 #[rstest]
2168 fn test_book_reply_completes_after_all_partitioned_payloads() {
2169 let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2170 let sender: EventSender<DataEvent> = sender.into();
2171 let root = UUID4::new();
2172 let first = InstrumentId::from("ESM6.GLBX");
2173 let second = InstrumentId::from("ESU6.GLBX");
2174 let a = OrderBookDelta::clear(first, 11, UnixNanos::from(13), UnixNanos::from(17));
2175 let b = OrderBookDelta::clear(second, 19, UnixNanos::from(23), UnixNanos::from(29));
2176 let c = OrderBookDelta::clear(first, 31, UnixNanos::from(37), UnixNanos::from(41));
2177
2178 let response = BookDeltasResponse::new(
2179 root,
2180 ClientId::from("HISTORY"),
2181 InstrumentId::from("ES.FUT.GLBX"),
2182 vec![a, b, c],
2183 None,
2184 None,
2185 UnixNanos::from(43),
2186 None,
2187 );
2188 send_book_deltas_response(&sender, response);
2189
2190 let DataEvent::ResponseEnvelope(left) = receiver.try_recv().unwrap() else {
2191 panic!("expected first envelope");
2192 };
2193
2194 let DataEvent::ResponseEnvelope(right) = receiver.try_recv().unwrap() else {
2195 panic!("expected final envelope");
2196 };
2197
2198 let Some(DataResponse::BookDeltas(left_payload)) = left.response() else {
2199 panic!("expected first book");
2200 };
2201
2202 let Some(DataResponse::BookDeltas(right_payload)) = right.response() else {
2203 panic!("expected second book");
2204 };
2205
2206 assert_eq!(
2207 (left.correlation_id(), right.correlation_id()),
2208 (&root, &root)
2209 );
2210 assert!(!left.completed());
2211 assert!(right.completed());
2212 assert_eq!(left_payload.instrument_id, first);
2213 assert_eq!(right_payload.instrument_id, second);
2214 assert_eq!(left_payload.data.as_ref(), &vec![a, c]);
2215 assert_eq!(right_payload.data.as_ref(), &vec![b]);
2216 assert_eq!(
2217 receiver.try_recv().unwrap_err(),
2218 tokio::sync::mpsc::error::TryRecvError::Empty
2219 );
2220 }
2221
2222 #[rstest]
2223 #[tokio::test]
2224 async fn test_stop_closes_task_admission_until_disconnect_drains() {
2225 let mut client = test_data_client();
2226
2227 client
2228 .task_handles
2229 .spawn(async { std::future::pending::<()>().await })
2230 .unwrap();
2231 client.is_connected.store(true, Ordering::Relaxed);
2232
2233 client.stop().unwrap();
2234
2235 assert!(!client.task_handles.is_open());
2236 assert!(client.is_disconnected());
2237
2238 client.disconnect().await.unwrap();
2239 assert!(client.task_handles.is_empty());
2240 }
2241
2242 #[rstest]
2243 #[case("EQUS", "EQUS.PLUS")] #[case("GLBX", "EQUS.MINI")] fn test_venue_dataset_map_overrides_default(#[case] venue: &str, #[case] dataset: &str) {
2246 let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2247 replace_data_event_sender(sender);
2248
2249 let mut config = DatabentoDataClientConfig::new(
2250 "32-character-with-lots-of-filler",
2251 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2252 true,
2253 true,
2254 );
2255 config.venue_dataset_map = IndexMap::from([(venue.to_string(), dataset.to_string())]);
2256
2257 let client = DatabentoDataClient::new(
2258 ClientId::from("DATABENTO-TEST"),
2259 config,
2260 get_atomic_clock_realtime(),
2261 )
2262 .expect("test client should initialize");
2263
2264 assert_eq!(
2265 client.get_dataset_for_venue(Venue::from(venue)).unwrap(),
2266 dataset
2267 );
2268
2269 assert_eq!(
2271 client.get_dataset_for_venue(Venue::from("XCBO")).unwrap(),
2272 "OPRA.PILLAR"
2273 );
2274 }
2275
2276 #[rstest]
2277 fn test_config_base_url_overrides_default_to_none() {
2278 let config = DatabentoDataClientConfig::new(
2279 "32-character-with-lots-of-filler",
2280 PathBuf::from("test_publishers.json"),
2281 true,
2282 true,
2283 );
2284
2285 assert!(config.historical_base_url.is_none());
2286 assert!(config.live_gateway_addr.is_none());
2287 }
2288
2289 #[rstest]
2290 fn test_historical_base_url_override_invalid_errors() {
2291 let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2292 replace_data_event_sender(sender);
2293
2294 let mut config = DatabentoDataClientConfig::new(
2295 "32-character-with-lots-of-filler",
2296 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2297 true,
2298 true,
2299 );
2300 config.historical_base_url = Some("not a url".to_string());
2301
2302 let result = DatabentoDataClient::new(
2303 ClientId::from("DATABENTO-TEST"),
2304 config,
2305 get_atomic_clock_realtime(),
2306 );
2307
2308 assert!(result.is_err());
2309 }
2310
2311 fn subscribe_quotes_cmd(params: Option<Params>) -> SubscribeQuotes {
2312 SubscribeQuotes::new(
2313 InstrumentId::from("ESM4.GLBX"),
2314 Some(ClientId::from("DATABENTO-TEST")),
2315 None,
2316 UUID4::new(),
2317 UnixNanos::default(),
2318 None,
2319 params,
2320 )
2321 }
2322
2323 fn subscribe_trades_cmd(params: Option<Params>) -> SubscribeTrades {
2324 SubscribeTrades::new(
2325 InstrumentId::from("ESM4.GLBX"),
2326 Some(ClientId::from("DATABENTO-TEST")),
2327 None,
2328 UUID4::new(),
2329 UnixNanos::default(),
2330 None,
2331 params,
2332 )
2333 }
2334
2335 fn subscribe_book_depth_cmd(depth: Option<NonZeroUsize>) -> SubscribeBookDepth {
2336 SubscribeBookDepth::new(
2337 InstrumentId::from("ESM4.GLBX"),
2338 BookType::L2_MBP,
2339 Some(ClientId::from("DATABENTO-TEST")),
2340 None,
2341 UUID4::new(),
2342 UnixNanos::default(),
2343 depth,
2344 true,
2345 None,
2346 None,
2347 )
2348 }
2349
2350 fn currency_pair_with_ts_init(instrument_id: &str, ts_init: UnixNanos) -> InstrumentAny {
2351 let instrument_id = InstrumentId::from(instrument_id);
2352 InstrumentAny::CurrencyPair(
2353 CurrencyPair::builder()
2354 .instrument_id(instrument_id)
2355 .raw_symbol(instrument_id.symbol)
2356 .base_currency(Currency::from("BTC"))
2357 .quote_currency(Currency::from("USDT"))
2358 .price_precision(2)
2359 .size_precision(6)
2360 .price_increment(Price::from("0.01"))
2361 .size_increment(Quantity::from("0.000001"))
2362 .ts_event(UnixNanos::default())
2363 .ts_init(ts_init)
2364 .build()
2365 .unwrap(),
2366 )
2367 }
2368
2369 #[rstest]
2370 fn test_instruments_query_params_requests_all_symbols() {
2371 let start = UnixNanos::from(1_000_000_000);
2372 let end = UnixNanos::from(2_000_000_000);
2373
2374 let params = instruments_query_params("GLBX.MDP3".to_string(), start, Some(end));
2375
2376 assert_eq!(params.dataset, "GLBX.MDP3");
2377 assert_eq!(params.symbols, vec!["ALL_SYMBOLS"]);
2378 assert_eq!(params.start, start);
2379 assert_eq!(params.end, Some(end));
2380 assert_eq!(params.limit, None);
2381 assert_eq!(params.price_precision, None);
2382 }
2383
2384 #[rstest]
2385 fn test_instrument_query_params_requests_single_symbol() {
2386 let instrument_id = InstrumentId::from("ESM4.GLBX");
2387
2388 let start = UnixNanos::from(1_000_000_000);
2389 let end = UnixNanos::from(2_000_000_000);
2390
2391 let params =
2392 instrument_query_params("GLBX.MDP3".to_string(), instrument_id, start, Some(end));
2393
2394 assert_eq!(params.dataset, "GLBX.MDP3");
2395 assert_eq!(params.symbols, vec!["ESM4"]);
2396 assert_eq!(params.start, start);
2397 assert_eq!(params.end, Some(end));
2398 assert_eq!(params.limit, None);
2399 assert_eq!(params.price_precision, None);
2400 }
2401
2402 #[rstest]
2403 #[case::intraday(1_782_302_400_000_000_001)]
2404 #[case::midnight(1_782_259_200_000_000_000)]
2405 #[case::day_end(1_782_345_599_999_999_999)]
2406 #[tokio::test]
2407 async fn test_seed_price_precision_includes_midnight(#[case] start: u64) {
2408 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2409 .join("test_data/test_data.definition.futures_contract.dbn.zst");
2410 let mut decoder = Decoder::from_zstd_file(path).unwrap();
2411 let mut metadata = decoder.metadata().clone();
2412 let record = decoder
2413 .decode_record::<dbn::InstrumentDefMsg>()
2414 .unwrap()
2415 .unwrap();
2416 metadata.symbols = vec!["ESU6".to_string()];
2417
2418 metadata.mappings = vec![SymbolMapping {
2419 raw_symbol: "ESU6".to_string(),
2420 intervals: vec![MappingInterval {
2421 start_date: date!(2026 - 06 - 24),
2422 end_date: date!(2026 - 06 - 25),
2423 symbol: record.hd.instrument_id.to_string(),
2424 }],
2425 }];
2426
2427 let mut body = Vec::new();
2428 {
2429 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
2430 encoder.encode_record(record).unwrap();
2431 encoder.flush().unwrap();
2432 }
2433
2434 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2435 let base_url = format!("http://{}/", listener.local_addr().unwrap());
2436
2437 let server = tokio::spawn(serve_historical_response(listener, body));
2438
2439 let client = DatabentoHistoricalClient::new_with_base_url(
2440 Credential::new("32-character-with-lots-of-filler"),
2441 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2442 get_atomic_clock_realtime(),
2443 false,
2444 &base_url,
2445 )
2446 .unwrap();
2447 let instrument_id = InstrumentId::from("ESU26.GLBX");
2448 let end = UnixNanos::from(1_782_345_600_000_000_123);
2449
2450 tokio::time::timeout(
2451 std::time::Duration::from_secs(5),
2452 seed_price_precision_if_needed(
2453 &client,
2454 "GLBX.MDP3",
2455 instrument_id,
2456 start.into(),
2457 Some(end),
2458 None,
2459 ),
2460 )
2461 .await
2462 .unwrap();
2463 let request = server.await.unwrap();
2464 let fields: AHashMap<_, _> = request
2465 .split('&')
2466 .map(|field| field.split_once('=').unwrap())
2467 .collect();
2468
2469 assert_eq!(fields["dataset"], "GLBX.MDP3");
2470 assert_eq!(fields["schema"], "definition");
2471 assert_eq!(fields["symbols"], "ESU6");
2472 assert_eq!(fields["start"], "1782259200000000000");
2473 assert_eq!(fields["end"], end.to_string());
2474 assert_eq!(client.price_precision(instrument_id.symbol), Some(2));
2475 }
2476
2477 async fn serve_historical_response(listener: TcpListener, body: Vec<u8>) -> String {
2478 let (stream, _) = listener.accept().await.unwrap();
2479 let mut stream = BufReader::new(stream);
2480 let mut line = String::new();
2481 stream.read_line(&mut line).await.unwrap();
2482 assert_eq!(line, "POST /v0/timeseries.get_range HTTP/1.1\r\n");
2483 let mut content_length = None;
2484
2485 loop {
2486 line.clear();
2487 stream.read_line(&mut line).await.unwrap();
2488
2489 if line == "\r\n" {
2490 break;
2491 }
2492
2493 if let Some((name, value)) = line.split_once(':')
2494 && name.eq_ignore_ascii_case("content-length")
2495 {
2496 content_length = Some(value.trim().parse::<usize>().unwrap());
2497 }
2498 }
2499
2500 let mut request = vec![0; content_length.unwrap()];
2501 stream.read_exact(&mut request).await.unwrap();
2502 let headers = format!(
2503 "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
2504 body.len(),
2505 );
2506 stream.write_all(headers.as_bytes()).await.unwrap();
2507 stream.write_all(&body).await.unwrap();
2508 stream.shutdown().await.unwrap();
2509 String::from_utf8(request).unwrap()
2510 }
2511
2512 #[rstest]
2513 #[case::explicit(Some(5), None)]
2514 #[case::cached(None, Some(3))]
2515 #[tokio::test]
2516 async fn test_seed_price_precision_skips_known_precision(
2517 #[case] price_precision: Option<u8>,
2518 #[case] cached_precision: Option<u8>,
2519 ) {
2520 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2521 let base_url = format!("http://{}/", listener.local_addr().unwrap());
2522
2523 let client = DatabentoHistoricalClient::new_with_base_url(
2524 Credential::new("32-character-with-lots-of-filler"),
2525 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2526 get_atomic_clock_realtime(),
2527 false,
2528 &base_url,
2529 )
2530 .unwrap();
2531 let instrument_id = InstrumentId::from("ESU6.GLBX");
2532 if let Some(precision) = cached_precision {
2533 client.set_price_precision(instrument_id.symbol, precision);
2534 }
2535
2536 tokio::select! {
2537 () = seed_price_precision_if_needed(
2538 &client,
2539 "GLBX.MDP3",
2540 instrument_id,
2541 1_706_443_200_000_000_001.into(),
2542 Some(1_706_486_400_000_000_123.into()),
2543 price_precision,
2544 ) => {}
2545 result = listener.accept() => panic!("Unexpected definition request: {result:?}"),
2546 () = tokio::time::sleep(std::time::Duration::from_secs(5)) => {
2547 panic!("Precision seeding did not finish");
2548 }
2549 }
2550
2551 assert_eq!(
2552 client.price_precision(instrument_id.symbol),
2553 cached_precision
2554 );
2555 }
2556
2557 #[rstest]
2558 fn test_resolve_request_time_range_defaults_to_end_day() {
2559 let end = UnixNanos::from(1_706_443_200_000_000_001);
2560
2561 let (start, resolved_end) = resolve_request_time_range(None, Some(end));
2562
2563 assert_eq!(start, UnixNanos::from(1_706_400_000_000_000_000));
2564 assert_eq!(resolved_end, Some(end + DurationNanos::new(1)));
2565 }
2566
2567 #[rstest]
2568 #[case::range(17, 23, 17, 24)]
2569 #[case::point(17, 17, 17, 18)]
2570 #[case::zero(0, 0, 0, 1)]
2571 #[case::inverted(23, 17, 17, 18)]
2572 fn test_resolve_request_time_range_converts_inclusive_bounds(
2573 #[case] start: u64,
2574 #[case] end: u64,
2575 #[case] expected_start: u64,
2576 #[case] expected_end: u64,
2577 ) {
2578 let (start, end) = resolve_request_time_range(Some(start.into()), Some(end.into()));
2579
2580 assert_eq!(start, UnixNanos::from(expected_start));
2581 assert_eq!(end, Some(UnixNanos::from(expected_end)));
2582 }
2583
2584 #[rstest]
2585 #[case::quotes(false, true, false)]
2586 #[case::bars_close(true, true, false)]
2587 #[case::bars_open(true, false, false)]
2588 #[case::bars_default_start(true, true, true)]
2589 #[tokio::test]
2590 async fn test_historical_request_sends_inclusive_bounds(
2591 #[case] bars: bool,
2592 #[case] timestamp_on_close: bool,
2593 #[case] missing_start: bool,
2594 ) {
2595 let filename = if bars {
2596 "test_data.ohlcv-1m.dbn.zst"
2597 } else {
2598 "test_data.mbp-1.dbn.zst"
2599 };
2600
2601 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2602 .join("test_data")
2603 .join(filename);
2604 let decoder = Decoder::from_zstd_file(path).unwrap();
2605 let mut metadata = decoder.metadata().clone();
2606 metadata.dataset = "GLBX.MDP3".to_owned();
2607 let mut body = Vec::new();
2608 {
2609 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
2610 encoder.flush().unwrap();
2611 }
2612
2613 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2614 let base_url = format!("http://{}/", listener.local_addr().unwrap());
2615
2616 let server = tokio::spawn(serve_historical_response(listener, body));
2617 let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel();
2618 replace_data_event_sender(sender);
2619
2620 let mut config = DatabentoDataClientConfig::new(
2621 "32-character-with-lots-of-filler",
2622 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2623 true,
2624 timestamp_on_close,
2625 );
2626 config.historical_base_url = Some(base_url);
2627 let client_id = ClientId::from("DATABENTO-BOUNDS");
2628 let client =
2629 DatabentoDataClient::new(client_id, config, get_atomic_clock_realtime()).unwrap();
2630 let id = InstrumentId::from("ESM4.GLBX");
2631 let start = UnixNanos::from(1_717_423_200_000_000_000);
2632 let request_start = (!missing_start).then_some(start);
2633 let end = start + DurationNanos::from_secs(60) + DurationNanos::new(123);
2634 let request_id = UUID4::new();
2635 let params = Params::new([(PRICE_PRECISION_PARAM.to_owned(), json!(2))].into());
2636
2637 if bars {
2638 let bar_type = "ESM4.GLBX-1-MINUTE-LAST-EXTERNAL"
2639 .parse::<BarType>()
2640 .unwrap();
2641 client
2642 .request_bars(RequestBars::new(
2643 bar_type,
2644 request_start.map(|timestamp| timestamp.to_datetime_utc()),
2645 Some(end.to_datetime_utc()),
2646 None,
2647 Some(client_id),
2648 request_id,
2649 UnixNanos::from(37),
2650 Some(params.clone()),
2651 ))
2652 .unwrap();
2653 } else {
2654 client
2655 .request_quotes(RequestQuotes::new(
2656 id,
2657 request_start.map(|timestamp| timestamp.to_datetime_utc()),
2658 Some(end.to_datetime_utc()),
2659 None,
2660 Some(client_id),
2661 request_id,
2662 UnixNanos::from(37),
2663 Some(params.clone()),
2664 ))
2665 .unwrap();
2666 }
2667
2668 let event = tokio::time::timeout(Duration::from_secs(5), receiver.recv())
2669 .await
2670 .unwrap()
2671 .unwrap();
2672
2673 let DataEvent::ResponseEnvelope(envelope) = event else {
2674 panic!("expected completed response, received {event:?}");
2675 };
2676
2677 let response = envelope.response().unwrap();
2678 let request = server.await.unwrap();
2679 let fields: AHashMap<_, _> = request
2680 .split('&')
2681 .map(|field| field.split_once('=').unwrap())
2682 .collect();
2683
2684 let offset = if bars {
2685 DurationNanos::from_secs(60)
2686 } else {
2687 DurationNanos::new(0)
2688 };
2689
2690 let expected_start = if missing_start {
2691 UnixNanos::from(1_717_372_800_000_000_000)
2692 } else {
2693 start
2694 };
2695
2696 assert_eq!(fields["dataset"], "GLBX.MDP3");
2697 assert_eq!(fields["schema"], if bars { "ohlcv-1m" } else { "mbp-1" });
2698 assert_eq!(fields["symbols"], "ESM4");
2699 assert_eq!(fields["start"], (expected_start - offset).to_string());
2700 assert_eq!(
2701 fields["end"],
2702 (end - offset + DurationNanos::new(1)).to_string()
2703 );
2704 assert_eq!(envelope.correlation_id(), &request_id);
2705 assert!(envelope.completed());
2706 assert_eq!(envelope.failure_reason(), None);
2707 assert_eq!(response.start(), request_start);
2708 assert_eq!(response.end(), Some(end));
2709 assert_eq!(response.client_id(), &client_id);
2710 assert_eq!(response.params(), Some(¶ms));
2711 assert_eq!(response.record_count(), Some(0));
2712 }
2713
2714 #[rstest]
2715 fn test_requested_instrument_filters_exact_id() {
2716 let requested_id = InstrumentId::from("BTCUSDT.BINANCE");
2717 let instruments = vec![
2718 currency_pair("ETHUSDT.BINANCE"),
2719 currency_pair("BTCUSDT.BINANCE"),
2720 ];
2721
2722 let instrument = requested_instrument(&instruments, requested_id).expect("instrument");
2723
2724 assert_eq!(instrument.id(), requested_id);
2725 }
2726
2727 #[rstest]
2728 fn test_requested_instrument_returns_latest_matching_id() {
2729 let requested_id = InstrumentId::from("BTCUSDT.BINANCE");
2730 let instruments = vec![
2731 currency_pair_with_ts_init("BTCUSDT.BINANCE", UnixNanos::from(1)),
2732 currency_pair_with_ts_init("BTCUSDT.BINANCE", UnixNanos::from(2)),
2733 ];
2734
2735 let instrument = requested_instrument(&instruments, requested_id).expect("instrument");
2736
2737 assert_eq!(instrument.ts_init(), UnixNanos::from(2));
2738 }
2739
2740 #[rstest]
2741 fn test_requested_instrument_returns_none_on_miss() {
2742 let instruments = vec![currency_pair("ETHUSDT.BINANCE")];
2743
2744 let instrument = requested_instrument(&instruments, InstrumentId::from("BTCUSDT.BINANCE"));
2745
2746 assert!(instrument.is_none());
2747 }
2748
2749 #[rstest]
2750 fn test_price_precision_from_params() {
2751 let params = Params::new([(PRICE_PRECISION_PARAM.to_string(), json!(5))].into());
2752
2753 let price_precision = price_precision_from_params(Some(¶ms)).unwrap();
2754
2755 assert_eq!(price_precision, Some(5));
2756 }
2757
2758 #[rstest]
2759 fn test_price_precision_from_params_rejects_out_of_range_value() {
2760 let params = Params::new(
2761 [(
2762 PRICE_PRECISION_PARAM.to_string(),
2763 json!(u64::from(u8::MAX) + 1),
2764 )]
2765 .into(),
2766 );
2767
2768 let result = price_precision_from_params(Some(¶ms));
2769
2770 assert!(result.is_err());
2771 }
2772
2773 #[rstest]
2774 fn test_partition_book_deltas_response_by_child_instrument(
2775 #[values(false, true)] narrowed: bool,
2776 ) {
2777 let correlation_id = UUID4::new();
2778 let client_id = ClientId::from("DATABENTO-TEST");
2779 let parent = InstrumentId::from("ES.FUT.GLBX");
2780 let child_a = InstrumentId::from("ESM6.GLBX");
2781 let child_b = InstrumentId::from("ESU6.GLBX");
2782 let deltas = vec![
2783 OrderBookDelta::clear(child_a, 1, UnixNanos::from(1_000), UnixNanos::from(1_000)),
2784 OrderBookDelta::clear(child_b, 2, UnixNanos::from(2_000), UnixNanos::from(2_000)),
2785 OrderBookDelta::clear(child_a, 3, UnixNanos::from(3_000), UnixNanos::from(3_000)),
2786 ];
2787 let params = Params::new([(PRICE_PRECISION_PARAM.to_string(), json!(5))].into());
2788
2789 let rows = if narrowed {
2790 let hidden = OrderBookDelta::clear(
2791 InstrumentId::from("HIDDEN.GLBX"),
2792 4,
2793 UnixNanos::from(5_000),
2794 UnixNanos::from(5_000),
2795 );
2796 let mut backing = vec![hidden];
2797 backing.extend_from_slice(&deltas);
2798 backing.push(hidden);
2799 BatchView::new(Arc::new(backing), 1..4)
2800 } else {
2801 deltas.clone().into()
2802 };
2803
2804 let response = BookDeltasResponse::new(
2805 correlation_id,
2806 client_id,
2807 parent,
2808 rows,
2809 Some(UnixNanos::from(500)),
2810 Some(UnixNanos::from(4_000)),
2811 UnixNanos::from(5_000),
2812 Some(params.clone()),
2813 );
2814
2815 let responses = partition_book_deltas_response(response);
2816
2817 assert_eq!(responses.len(), 2);
2818 assert_eq!(responses[0].correlation_id, correlation_id);
2819 assert_eq!(responses[1].correlation_id, correlation_id);
2820 assert_eq!(responses[0].client_id, client_id);
2821 assert_eq!(responses[1].client_id, client_id);
2822 assert_eq!(responses[0].instrument_id, child_a);
2823 assert_eq!(responses[1].instrument_id, child_b);
2824 assert_eq!(responses[0].data.as_ref(), &vec![deltas[0], deltas[2]]);
2825 assert_eq!(responses[1].data.as_ref(), &vec![deltas[1]]);
2826 assert_eq!(responses[0].start, Some(UnixNanos::from(500)));
2827 assert_eq!(responses[1].start, Some(UnixNanos::from(500)));
2828 assert_eq!(responses[0].end, Some(UnixNanos::from(4_000)));
2829 assert_eq!(responses[1].end, Some(UnixNanos::from(4_000)));
2830 assert_eq!(responses[0].ts_init, UnixNanos::from(5_000));
2831 assert_eq!(responses[1].ts_init, UnixNanos::from(5_000));
2832 assert_eq!(responses[0].params, Some(params.clone()));
2833 assert_eq!(responses[1].params, Some(params));
2834 }
2835
2836 #[rstest]
2837 fn test_partition_book_deltas_response_preserves_homogeneous_response() {
2838 let correlation_id = UUID4::new();
2839 let client_id = ClientId::from("DATABENTO-TEST");
2840 let instrument_id = InstrumentId::from("ESM6.GLBX");
2841 let delta = OrderBookDelta::clear(
2842 instrument_id,
2843 1,
2844 UnixNanos::from(1_000),
2845 UnixNanos::from(1_000),
2846 );
2847 let response = BookDeltasResponse::new(
2848 correlation_id,
2849 client_id,
2850 instrument_id,
2851 vec![delta],
2852 Some(UnixNanos::from(500)),
2853 Some(UnixNanos::from(1_500)),
2854 UnixNanos::from(2_000),
2855 None,
2856 );
2857
2858 let responses = partition_book_deltas_response(response);
2859
2860 assert_eq!(responses.len(), 1);
2861 assert_eq!(responses[0].correlation_id, correlation_id);
2862 assert_eq!(responses[0].client_id, client_id);
2863 assert_eq!(responses[0].instrument_id, instrument_id);
2864 assert_eq!(responses[0].data.as_ref(), &vec![delta]);
2865 assert_eq!(responses[0].start, Some(UnixNanos::from(500)));
2866 assert_eq!(responses[0].end, Some(UnixNanos::from(1_500)));
2867 assert_eq!(responses[0].ts_init, UnixNanos::from(2_000));
2868 assert_eq!(responses[0].params, None);
2869 }
2870
2871 #[rstest]
2872 fn test_partition_book_deltas_response_preserves_empty_parent_response() {
2873 let correlation_id = UUID4::new();
2874 let parent = InstrumentId::from("ES.FUT.GLBX");
2875 let response = BookDeltasResponse::new(
2876 correlation_id,
2877 ClientId::from("DATABENTO-TEST"),
2878 parent,
2879 Vec::new(),
2880 None,
2881 None,
2882 UnixNanos::from(1_000),
2883 None,
2884 );
2885
2886 let responses = partition_book_deltas_response(response);
2887
2888 assert_eq!(responses.len(), 1);
2889 let response = &responses[0];
2890 assert_eq!(response.correlation_id, correlation_id);
2891 assert_eq!(response.instrument_id, parent);
2892 assert!(response.data.is_empty());
2893 }
2894
2895 #[rstest]
2896 fn test_schema_from_params_returns_default() {
2897 let schema = schema_from_params(None, dbn::Schema::Mbp1, QUOTE_SCHEMAS).unwrap();
2898
2899 assert_eq!(schema, dbn::Schema::Mbp1);
2900 }
2901
2902 #[rstest]
2903 fn test_schema_from_params_accepts_allowed_value() {
2904 let params = Params::new([(SCHEMA_PARAM.to_string(), json!("tbbo"))].into());
2905
2906 let schema = schema_from_params(Some(¶ms), dbn::Schema::Mbp1, QUOTE_SCHEMAS).unwrap();
2907
2908 assert_eq!(schema, dbn::Schema::Tbbo);
2909 }
2910
2911 #[rstest]
2912 fn test_schema_from_params_rejects_disallowed_value() {
2913 let params = Params::new([(SCHEMA_PARAM.to_string(), json!("mbo"))].into());
2914
2915 let result = schema_from_params(Some(¶ms), dbn::Schema::Mbp1, QUOTE_SCHEMAS);
2916
2917 assert!(result.is_err());
2918 }
2919
2920 #[rstest]
2921 #[case::quotes(SubscribeKind::Quotes)]
2922 #[case::trades(SubscribeKind::Trades)]
2923 fn test_invalid_subscribe_params_do_not_create_feed_handler(#[case] kind: SubscribeKind) {
2924 let mut client = test_data_client();
2925 let params = Params::new([(SCHEMA_PARAM.to_string(), json!("definition"))].into());
2926 let result = match kind {
2927 SubscribeKind::Quotes => client.subscribe_quotes(subscribe_quotes_cmd(Some(params))),
2928 SubscribeKind::Trades => client.subscribe_trades(subscribe_trades_cmd(Some(params))),
2929 };
2930
2931 assert!(result.is_err());
2932 assert!(client.cmd_channels.lock().is_empty());
2933 }
2934
2935 #[rstest]
2936 fn test_bar_list_reports_exact_full_types_and_continues_after_invalid_member() {
2937 let mut client = test_data_client();
2938 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
2939 client
2940 .cmd_channels
2941 .lock()
2942 .insert("GLBX.MDP3".to_string(), tx);
2943 let first = BarType::from("ESM4.GLBX-1-SECOND-LAST-EXTERNAL");
2944 let invalid = BarType::from("ESM4.GLBX-5-MINUTE-LAST-EXTERNAL");
2945 let last = BarType::from("ESM4.GLBX-1-MINUTE-LAST-EXTERNAL");
2946
2947 let command = SubscribeBars::new(
2948 [first, invalid, last],
2949 Some(client.client_id),
2950 None,
2951 UUID4::new(),
2952 UnixNanos::from(17),
2953 Some(UUID4::new()),
2954 None,
2955 );
2956 let error = client.subscribe_bars(command).unwrap_err();
2957 let partial = error.downcast_ref::<SubscriptionError>().unwrap();
2958 assert_eq!(partial.bar_types_accepted, vec![first, last]);
2959 assert_eq!(partial.instrument_ids_accepted, Vec::<InstrumentId>::new());
2960
2961 for schema in [dbn::Schema::Ohlcv1S, dbn::Schema::Ohlcv1M] {
2962 let HandlerCommand::Subscribe(actual) = rx.try_recv().unwrap() else {
2963 panic!("expected bar schema subscription");
2964 };
2965
2966 let expected = Subscription::builder()
2967 .schema(schema)
2968 .symbols("ESM4")
2969 .build();
2970 assert_eq!(actual, expected);
2971 }
2972
2973 assert_eq!(
2974 rx.try_recv().unwrap_err(),
2975 tokio::sync::mpsc::error::TryRecvError::Empty
2976 );
2977 }
2978
2979 macro_rules! batch_tests {
2980 ($batch_test:ident, $failure_test:ident, $validation_test:ident, $command:ident, $subscribe:ident, $schema:ident) => {
2981 #[rstest]
2982 fn $batch_test() {
2983 let mut client = test_data_client();
2984 let (glbx_tx, mut glbx_rx) = tokio::sync::mpsc::unbounded_channel();
2985 let (opra_tx, mut opra_rx) = tokio::sync::mpsc::unbounded_channel();
2986 client
2987 .cmd_channels
2988 .lock()
2989 .insert("GLBX.MDP3".to_string(), glbx_tx);
2990 client
2991 .cmd_channels
2992 .lock()
2993 .insert("OPRA.PILLAR".to_string(), opra_tx);
2994 let first = InstrumentId::from("ESM4.GLBX");
2995 let second = InstrumentId::from("NQM4.GLBX");
2996 let third = InstrumentId::from("SPY_OPTION.XCBO");
2997 let params = Params::new([(PRICE_PRECISION_PARAM.to_string(), json!(5))].into());
2998 let command = $command(Some(params))
2999 .with_instrument_ids([second, third, first, second]);
3000 let original = serde_json::to_value(&command).unwrap();
3001 client.$subscribe(command.clone()).unwrap();
3002 assert!(
3003 matches!(glbx_rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 5) if symbol == second.symbol)
3004 );
3005 assert!(
3006 matches!(glbx_rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 5) if symbol == first.symbol)
3007 );
3008
3009 let HandlerCommand::Subscribe(glbx_subscription) = glbx_rx.try_recv().unwrap()
3010 else {
3011 panic!("expected GLBX subscription")
3012 };
3013
3014 assert!(
3015 matches!(opra_rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 5) if symbol == third.symbol)
3016 );
3017
3018 let HandlerCommand::Subscribe(opra_subscription) = opra_rx.try_recv().unwrap()
3019 else {
3020 panic!("expected OPRA subscription")
3021 };
3022
3023 let expected_glbx = Subscription::builder()
3024 .schema(dbn::Schema::$schema)
3025 .symbols(vec![second.symbol.to_string(), first.symbol.to_string()])
3026 .build();
3027 let expected_opra = Subscription::builder()
3028 .schema(dbn::Schema::$schema)
3029 .symbols(vec![third.symbol.to_string()])
3030 .build();
3031 assert_eq!(glbx_subscription, expected_glbx);
3032 assert_eq!(opra_subscription, expected_opra);
3033 assert_eq!(
3034 glbx_rx.try_recv().unwrap_err(),
3035 tokio::sync::mpsc::error::TryRecvError::Empty
3036 );
3037 assert_eq!(
3038 opra_rx.try_recv().unwrap_err(),
3039 tokio::sync::mpsc::error::TryRecvError::Empty
3040 );
3041 assert_eq!(client.symbol_venue_map.len(), 3);
3042
3043 for id in [first, second, third] {
3044 assert_eq!(
3045 client.symbol_venue_map.get_cloned(&id.symbol),
3046 Some(id.venue)
3047 );
3048 }
3049
3050 assert_eq!(serde_json::to_value(command).unwrap(), original);
3051 }
3052
3053 #[rstest]
3054 #[case::failure_first(true)]
3055 #[case::failure_middle(false)]
3056 fn $failure_test(
3057 #[case] failure_first: bool,
3058 #[values(false, true)] previous_mapping: bool,
3059 ) {
3060 let mut client = test_data_client();
3061 let (glbx_tx, mut glbx_rx) = tokio::sync::mpsc::unbounded_channel();
3062 let (opra_tx, opra_rx) = tokio::sync::mpsc::unbounded_channel();
3063 let (equs_tx, mut equs_rx) = tokio::sync::mpsc::unbounded_channel();
3064 drop(opra_rx);
3065 client
3066 .cmd_channels
3067 .lock()
3068 .insert("GLBX.MDP3".to_string(), glbx_tx);
3069 client
3070 .cmd_channels
3071 .lock()
3072 .insert("OPRA.PILLAR".to_string(), opra_tx);
3073 client
3074 .cmd_channels
3075 .lock()
3076 .insert("EQUS.MINI".to_string(), equs_tx);
3077 let first = InstrumentId::from("ESM4.GLBX");
3078 let second = InstrumentId::from("SPY_OPTION.XCBO");
3079 let third = InstrumentId::from("AAPL.EQUS");
3080 let previous = previous_mapping.then_some(Venue::from("PREVIOUS"));
3081 if let Some(venue) = previous {
3082 client.symbol_venue_map.insert(second.symbol, venue);
3083 }
3084 let ids = if failure_first {
3085 [second, first, third]
3086 } else {
3087 [first, second, third]
3088 };
3089 let command = $command(None).with_instrument_ids(ids);
3090 let result = client.$subscribe(command);
3091 let error = result.unwrap_err();
3092 let partial = error
3093 .downcast_ref::<SubscriptionError>()
3094 .expect("partial subscription error");
3095
3096 assert_eq!(partial.instrument_ids_accepted, vec![first, third]);
3097 assert_eq!(partial.bar_types_accepted, Vec::<BarType>::new());
3098 assert_eq!(
3099 partial.source.to_string(),
3100 "Failed to send command to dataset OPRA.PILLAR: channel closed"
3101 );
3102 assert_eq!(client.symbol_venue_map.get_cloned(&second.symbol), previous);
3103
3104 for (id, receiver) in [(first, &mut glbx_rx), (third, &mut equs_rx)] {
3105 let HandlerCommand::Subscribe(accepted) = receiver.try_recv().unwrap() else {
3106 panic!("expected accepted subscription");
3107 };
3108 let expected = Subscription::builder()
3109 .schema(dbn::Schema::$schema)
3110 .symbols(vec![id.symbol.to_string()])
3111 .build();
3112 assert_eq!(accepted, expected);
3113 assert_eq!(client.symbol_venue_map.get_cloned(&id.symbol), Some(id.venue));
3114 assert_eq!(
3115 receiver.try_recv().unwrap_err(),
3116 tokio::sync::mpsc::error::TryRecvError::Empty
3117 );
3118 }
3119 }
3120
3121 #[rstest]
3122 #[case::unknown_dataset("UNKNOWN.UNKNOWN", "No dataset found for venue: UNKNOWN")]
3123 #[case::ambiguous_symbol(
3124 "ESM4.XCBO",
3125 "Identifiers share raw symbol ESM4 across different venues"
3126 )]
3127 fn $validation_test(#[case] invalid_id: &str, #[case] error: &str) {
3128 let mut client = test_data_client();
3129 let command = $command(None).with_instrument_ids([
3130 InstrumentId::from("ESM4.GLBX"),
3131 InstrumentId::from(invalid_id),
3132 ]);
3133 let result = client.$subscribe(command);
3134 assert_eq!(result.unwrap_err().to_string(), error);
3135 assert!(client.cmd_channels.lock().is_empty());
3136 assert!(client.symbol_venue_map.is_empty());
3137 }
3138 };
3139 }
3140
3141 batch_tests!(
3142 test_quote_batches_send_one_subscription_per_dataset,
3143 test_quote_batch_enqueue_failure_can_follow_accepted_dataset,
3144 test_quote_batches_validate_all_members_before_dispatch,
3145 subscribe_quotes_cmd,
3146 subscribe_quotes,
3147 Mbp1
3148 );
3149
3150 batch_tests!(
3151 test_trade_batches_send_one_subscription_per_dataset,
3152 test_trade_batch_enqueue_failure_can_follow_accepted_dataset,
3153 test_trade_batches_validate_all_members_before_dispatch,
3154 subscribe_trades_cmd,
3155 subscribe_trades,
3156 Trades
3157 );
3158
3159 #[rstest]
3160 fn test_trade_batch_unsubscribe_is_noop() {
3161 let mut client = test_data_client();
3162
3163 let command = UnsubscribeTrades::new(
3164 [
3165 InstrumentId::from("ESM4.GLBX"),
3166 InstrumentId::from("SPY_OPTION.XCBO"),
3167 ],
3168 Some(ClientId::from("DATABENTO")),
3169 None,
3170 UUID4::new(),
3171 UnixNanos::from(37),
3172 None,
3173 None,
3174 );
3175 client.unsubscribe_trades(&command).unwrap();
3176 assert!(client.cmd_channels.lock().is_empty());
3177 assert!(client.symbol_venue_map.is_empty());
3178 }
3179
3180 #[rstest]
3181 fn test_send_subscription_commands_starts_after_subscribe() {
3182 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3183 let subscription = Subscription::builder()
3184 .schema(dbn::Schema::Mbp1)
3185 .symbols(vec!["ESM4"])
3186 .build();
3187
3188 send_subscription_commands(
3189 &tx,
3190 "GLBX.MDP3",
3191 Some((Symbol::from("ESM4"), 2)),
3192 subscription,
3193 true,
3194 )
3195 .unwrap();
3196
3197 assert!(matches!(
3198 rx.try_recv().unwrap(),
3199 HandlerCommand::SetPricePrecision(symbol, 2) if symbol == Symbol::from("ESM4")
3200 ));
3201 assert!(matches!(
3202 rx.try_recv().unwrap(),
3203 HandlerCommand::Subscribe(sub) if sub.schema == dbn::Schema::Mbp1
3204 ));
3205 assert!(matches!(rx.try_recv().unwrap(), HandlerCommand::Start));
3206 }
3207
3208 #[rstest]
3209 #[case::second("ESM4.GLBX-1-SECOND-LAST-EXTERNAL", dbn::Schema::Ohlcv1S)]
3210 #[case::minute("ESM4.GLBX-1-MINUTE-LAST-EXTERNAL", dbn::Schema::Ohlcv1M)]
3211 #[case::hour("ESM4.GLBX-1-HOUR-LAST-EXTERNAL", dbn::Schema::Ohlcv1H)]
3212 #[case::day("ESM4.GLBX-1-DAY-LAST-EXTERNAL", dbn::Schema::Ohlcv1D)]
3213 fn test_ohlcv_schema_maps_aggregation(#[case] bar_type: &str, #[case] expected: dbn::Schema) {
3214 let schema = ohlcv_schema(BarType::from(bar_type)).unwrap();
3215
3216 assert_eq!(schema, expected);
3217 }
3218
3219 #[rstest]
3220 #[case::step("ESM4.GLBX-5-MINUTE-LAST-EXTERNAL")]
3221 #[case::aggregation("ESM4.GLBX-100-TICK-LAST-EXTERNAL")]
3222 #[case::bid("ESM4.GLBX-1-MINUTE-BID-EXTERNAL")]
3223 #[case::mid("ESM4.GLBX-1-MINUTE-MID-EXTERNAL")]
3224 fn test_ohlcv_schema_rejects_unsupported_bar_type(#[case] bar_type: &str) {
3225 assert!(ohlcv_schema(BarType::from(bar_type)).is_err());
3226 }
3227
3228 #[rstest]
3229 #[case::statistics("DatabentoStatistics", dbn::Schema::Statistics)]
3230 #[case::imbalance("DatabentoImbalance", dbn::Schema::Imbalance)]
3231 fn test_subscribe_custom_data_sends_schema_subscription(
3232 #[case] type_name: &str,
3233 #[case] expected: dbn::Schema,
3234 ) {
3235 let mut client = test_data_client();
3236 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3237 client
3238 .cmd_channels
3239 .lock()
3240 .insert("GLBX.MDP3".to_string(), tx);
3241 let command = SubscribeCustomData::new(
3242 Some(ClientId::from("DATABENTO-TEST")),
3243 None,
3244 DataType::new(type_name, None, Some("ESM4.GLBX".to_string())),
3245 UUID4::new(),
3246 UnixNanos::default(),
3247 None,
3248 None,
3249 );
3250
3251 client.subscribe(command).unwrap();
3252
3253 let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3254 panic!("expected subscription");
3255 };
3256 assert_eq!(subscription.schema, expected);
3257 assert_eq!(
3258 subscription.symbols,
3259 databento::Symbols::Symbols(vec!["ESM4".to_string()])
3260 );
3261 assert!(rx.try_recv().is_err());
3262 }
3263
3264 #[rstest]
3265 #[case::unsupported_type("DatabentoOther", Some("ESM4.GLBX"))]
3266 #[case::missing_identifier("DatabentoStatistics", None)]
3267 #[case::invalid_identifier("DatabentoStatistics", Some("ESM4"))]
3268 fn test_subscribe_custom_data_rejects_invalid_data_type(
3269 #[case] type_name: &str,
3270 #[case] identifier: Option<&str>,
3271 ) {
3272 let mut client = test_data_client();
3273 let command = SubscribeCustomData::new(
3274 Some(ClientId::from("DATABENTO-TEST")),
3275 None,
3276 DataType::new(type_name, None, identifier.map(str::to_string)),
3277 UUID4::new(),
3278 UnixNanos::default(),
3279 None,
3280 None,
3281 );
3282
3283 assert!(client.subscribe(command).is_err());
3284 assert!(client.cmd_channels.lock().is_empty());
3285 }
3286
3287 #[rstest]
3288 fn test_custom_data_for_instrument_matches_subscribed_topic() {
3289 let instrument_id = InstrumentId::from("ESM4.GLBX");
3290 let statistics = DatabentoStatistics::new(
3291 instrument_id,
3292 DatabentoStatisticType::SettlementPrice,
3293 DatabentoStatisticUpdateAction::Added,
3294 Some(Price::from("5000.25")),
3295 None,
3296 1,
3297 2,
3298 3,
3299 UnixNanos::from(4),
3300 5,
3301 UnixNanos::from(6),
3302 UnixNanos::from(7),
3303 UnixNanos::from(8),
3304 );
3305 let subscribed = DataType::new("DatabentoStatistics", None, Some("ESM4.GLBX".to_string()));
3306
3307 let custom = custom_data_for_instrument(Arc::new(statistics), instrument_id);
3308
3309 assert_eq!(custom.data_type, subscribed);
3310 assert_eq!(
3311 get_custom_topic(&custom.data_type),
3312 get_custom_topic(&subscribed)
3313 );
3314 assert_eq!(custom.data_type.identifier(), Some("ESM4.GLBX"));
3315 }
3316
3317 #[rstest]
3318 fn test_subscribe_schema_sends_raw_symbol() {
3319 let client = test_data_client();
3320 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3321 client
3322 .cmd_channels
3323 .lock()
3324 .insert("GLBX.MDP3".to_string(), tx);
3325
3326 client
3327 .subscribe_schema(InstrumentId::from("ESM4.GLBX"), dbn::Schema::Ohlcv1M)
3328 .unwrap();
3329
3330 let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3331 panic!("expected subscription");
3332 };
3333 assert_eq!(subscription.stype_in, dbn::SType::RawSymbol);
3334 assert_eq!(
3335 subscription.symbols,
3336 databento::Symbols::Symbols(vec!["ESM4".to_string()])
3337 );
3338 }
3339
3340 #[rstest]
3341 #[case::continuous("ES.c.0.GLBX")]
3342 #[case::parent("ES.FUT.GLBX")]
3343 #[case::instrument_id("12345.GLBX")]
3344 fn test_subscribe_schema_rejects_non_raw_symbols(#[case] instrument_id: &str) {
3345 let client = test_data_client();
3346
3347 let result = client.subscribe_schema(InstrumentId::from(instrument_id), dbn::Schema::Mbp10);
3348
3349 assert!(result.is_err());
3350 assert!(client.cmd_channels.lock().is_empty());
3351 }
3352
3353 #[rstest]
3354 fn test_subscribe_custom_data_rejects_metadata() {
3355 let mut client = test_data_client();
3356 let metadata = Params::new([("venue".to_string(), json!("GLBX"))].into());
3357
3358 let command = SubscribeCustomData::new(
3359 Some(ClientId::from("DATABENTO-TEST")),
3360 None,
3361 DataType::new(
3362 "DatabentoStatistics",
3363 Some(metadata),
3364 Some("ESM4.GLBX".to_string()),
3365 ),
3366 UUID4::new(),
3367 UnixNanos::default(),
3368 None,
3369 None,
3370 );
3371
3372 assert!(client.subscribe(command).is_err());
3373 assert!(client.cmd_channels.lock().is_empty());
3374 }
3375
3376 #[rstest]
3377 fn test_subscribed_custom_data_reaches_subscriber() {
3378 let instrument_id = InstrumentId::from("ESM4.GLBX");
3379 let subscribed = DataType::new("DatabentoStatistics", None, Some("ESM4.GLBX".to_string()));
3380 let other = DataType::new("DatabentoStatistics", None, Some("ESU4.GLBX".to_string()));
3381 let handler = msgbus::stubs::get_message_saving_handler::<CustomData>(None);
3382 msgbus::subscribe_any(get_custom_topic(&subscribed).into(), handler.clone(), None);
3383 let statistics = DatabentoStatistics::new(
3384 instrument_id,
3385 DatabentoStatisticType::SettlementPrice,
3386 DatabentoStatisticUpdateAction::Added,
3387 Some(Price::from("5000.25")),
3388 None,
3389 1,
3390 2,
3391 3,
3392 UnixNanos::from(4),
3393 5,
3394 UnixNanos::from(6),
3395 UnixNanos::from(7),
3396 UnixNanos::from(8),
3397 );
3398 let custom = custom_data_for_instrument(Arc::new(statistics), instrument_id);
3399
3400 msgbus::publish_any(get_custom_topic(&custom.data_type), &custom);
3401 msgbus::publish_any(get_custom_topic(&other), &custom);
3402
3403 let received = msgbus::stubs::get_saved_messages::<CustomData>(&handler);
3404 assert_eq!(received, vec![custom]);
3405 }
3406
3407 #[rstest]
3408 fn test_subscribe_book_depth_accepts_depth_ten() {
3409 let mut client = test_data_client();
3410 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3411 client
3412 .cmd_channels
3413 .lock()
3414 .insert("GLBX.MDP3".to_string(), tx);
3415
3416 client
3417 .subscribe_book_depth(subscribe_book_depth_cmd(NonZeroUsize::new(10)))
3418 .unwrap();
3419
3420 let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3421 panic!("expected subscription");
3422 };
3423 assert_eq!(subscription.schema, dbn::Schema::Mbp10);
3424 }
3425
3426 #[rstest]
3427 #[case::five(5)]
3428 #[case::twenty(20)]
3429 fn test_subscribe_book_depth_rejects_other_depths(#[case] depth: usize) {
3430 let mut client = test_data_client();
3431
3432 let result =
3433 client.subscribe_book_depth(subscribe_book_depth_cmd(NonZeroUsize::new(depth)));
3434
3435 assert!(result.is_err());
3436 assert!(client.cmd_channels.lock().is_empty());
3437 }
3438
3439 #[rstest]
3440 fn test_subscribe_bars_sends_ohlcv_subscription() {
3441 let mut client = test_data_client();
3442 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3443 client
3444 .cmd_channels
3445 .lock()
3446 .insert("GLBX.MDP3".to_string(), tx);
3447 let command = SubscribeBars::new(
3448 BarType::from("ESM4.GLBX-1-MINUTE-LAST-EXTERNAL"),
3449 Some(ClientId::from("DATABENTO-TEST")),
3450 None,
3451 UUID4::new(),
3452 UnixNanos::default(),
3453 None,
3454 None,
3455 );
3456
3457 client.subscribe_bars(command).unwrap();
3458
3459 let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3460 panic!("expected subscription");
3461 };
3462 assert_eq!(subscription.schema, dbn::Schema::Ohlcv1M);
3463 assert_eq!(
3464 subscription.symbols,
3465 databento::Symbols::Symbols(vec!["ESM4".to_string()])
3466 );
3467 assert!(rx.try_recv().is_err());
3468 }
3469
3470 #[rstest]
3471 fn test_subscribe_book_depth_sends_mbp10_subscription() {
3472 let mut client = test_data_client();
3473 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3474 client
3475 .cmd_channels
3476 .lock()
3477 .insert("GLBX.MDP3".to_string(), tx);
3478 client
3479 .subscribe_book_depth(subscribe_book_depth_cmd(None))
3480 .unwrap();
3481
3482 let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3483 panic!("expected subscription");
3484 };
3485 assert_eq!(subscription.schema, dbn::Schema::Mbp10);
3486 assert_eq!(
3487 subscription.symbols,
3488 databento::Symbols::Symbols(vec!["ESM4".to_string()])
3489 );
3490 assert!(rx.try_recv().is_err());
3491 }
3492
3493 #[rstest]
3494 fn test_send_subscription_commands_without_precision_or_start() {
3495 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3496 let subscription = Subscription::builder()
3497 .schema(dbn::Schema::Mbp1)
3498 .symbols(vec!["ESM4"])
3499 .build();
3500
3501 send_subscription_commands(&tx, "GLBX.MDP3", None, subscription, false).unwrap();
3502
3503 assert!(matches!(
3504 rx.try_recv().unwrap(),
3505 HandlerCommand::Subscribe(sub) if sub.schema == dbn::Schema::Mbp1
3506 ));
3507 assert!(matches!(
3508 rx.try_recv(),
3509 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
3510 ));
3511 }
3512}