1use std::{fmt::Debug, fs, num::NonZeroU64, path::PathBuf, str::FromStr, sync::Arc};
19
20use ahash::{AHashMap, AHashSet};
21use databento::{
22 dbn::{self, decode::DbnMetadata},
23 historical::timeseries::GetRangeParams,
24};
25use indexmap::IndexMap;
26use nautilus_core::{AtomicMap, UnixNanos, consts::NAUTILUS_USER_AGENT, time::AtomicTime};
27use nautilus_model::{
28 data::{Bar, Data, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
29 enums::BarAggregation,
30 identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
31 instruments::{Instrument, InstrumentAny},
32};
33
34use crate::{
35 common::{Credential, get_date_time_range, ohlcv_schema_from_aggregation},
36 decode::{
37 MboDeltaBuffer, decode_imbalance_msg, decode_instrument_def_msg, decode_mbo_msg,
38 decode_mbp10_msg, decode_record, decode_statistics_msg, decode_status_msg,
39 is_supported_stat_type,
40 },
41 symbology::{
42 MetadataCache, check_consistent_symbology, decode_nautilus_instrument_id,
43 infer_symbology_type, nautilus_symbol, raw_symbol,
44 },
45 types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, PublisherId},
46};
47
48#[derive(Clone)]
53pub struct DatabentoHistoricalClient {
54 futures_year_digits: FuturesYearDigits,
55 share_class_space: bool,
56 credential: Credential,
57 clock: &'static AtomicTime,
58 inner: Arc<databento::HistoricalClient>,
59 publisher_venue_map: Arc<IndexMap<PublisherId, Venue>>,
60 symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
61 price_precisions: Arc<AtomicMap<Symbol, u8>>,
62 use_exchange_as_venue: bool,
63 futures_symbols: Arc<AtomicMap<(PublisherId, u32, time::Date), Symbol>>,
64}
65
66#[derive(Debug)]
68pub struct RangeQueryParams {
69 pub dataset: String,
70 pub symbols: Vec<String>,
71 pub start: UnixNanos,
72 pub end: Option<UnixNanos>,
73 pub limit: Option<u64>,
74 pub price_precision: Option<u8>,
75}
76
77#[derive(Debug, Clone)]
79pub struct DatasetRange {
80 pub start: String,
81 pub end: String,
82}
83
84impl Debug for DatabentoHistoricalClient {
85 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86 f.debug_struct(stringify!(DatabentoHistoricalClient))
87 .field("credential", &self.credential)
88 .finish()
89 }
90}
91
92impl DatabentoHistoricalClient {
93 #[must_use]
95 pub fn api_key(&self) -> &str {
96 self.credential.api_key()
97 }
98
99 pub fn new(
105 credential: Credential,
106 publishers_filepath: PathBuf,
107 clock: &'static AtomicTime,
108 use_exchange_as_venue: bool,
109 ) -> anyhow::Result<Self> {
110 let client = databento::HistoricalClient::builder()
111 .user_agent_extension(NAUTILUS_USER_AGENT.into())
112 .key(credential.api_key())
113 .map_err(|e| anyhow::anyhow!("Failed to create client builder: {e}"))?
114 .build()
115 .map_err(|e| anyhow::anyhow!("Failed to build client: {e}"))?;
116
117 Self::from_client(
118 credential,
119 publishers_filepath,
120 clock,
121 use_exchange_as_venue,
122 client,
123 )
124 }
125
126 #[must_use]
128 pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
129 self.futures_year_digits = futures_year_digits;
130 self
131 }
132
133 #[must_use]
135 pub fn with_share_class_space(mut self, share_class_space: bool) -> Self {
136 self.share_class_space = share_class_space;
137 self
138 }
139
140 pub fn new_with_base_url(
149 credential: Credential,
150 publishers_filepath: PathBuf,
151 clock: &'static AtomicTime,
152 use_exchange_as_venue: bool,
153 base_url: &str,
154 ) -> anyhow::Result<Self> {
155 let client = databento::HistoricalClient::builder()
156 .user_agent_extension(NAUTILUS_USER_AGENT.into())
157 .base_url(base_url.parse().map_err(|e| {
158 anyhow::anyhow!("Failed to parse Databento Historical API base URL: {e}")
159 })?)
160 .key(credential.api_key())
161 .map_err(|e| anyhow::anyhow!("Failed to create client builder: {e}"))?
162 .build()
163 .map_err(|e| anyhow::anyhow!("Failed to build client: {e}"))?;
164
165 Self::from_client(
166 credential,
167 publishers_filepath,
168 clock,
169 use_exchange_as_venue,
170 client,
171 )
172 }
173
174 fn from_client(
175 credential: Credential,
176 publishers_filepath: PathBuf,
177 clock: &'static AtomicTime,
178 use_exchange_as_venue: bool,
179 client: databento::HistoricalClient,
180 ) -> anyhow::Result<Self> {
181 let file_content = fs::read_to_string(publishers_filepath)?;
182 let publishers_vec: Vec<DatabentoPublisher> = serde_json::from_str(&file_content)?;
183
184 let publisher_venue_map = publishers_vec
185 .into_iter()
186 .map(|p| (p.publisher_id, Venue::from(p.venue.as_str())))
187 .collect::<IndexMap<u16, Venue>>();
188
189 Ok(Self {
190 clock,
191 futures_year_digits: FuturesYearDigits::default(),
192 share_class_space: true,
193 inner: Arc::new(client),
194 publisher_venue_map: Arc::new(publisher_venue_map),
195 symbol_venue_map: Arc::new(AtomicMap::new()),
196 price_precisions: Arc::new(AtomicMap::new()),
197 credential,
198 use_exchange_as_venue,
199 futures_symbols: Arc::new(AtomicMap::new()),
200 })
201 }
202
203 pub fn set_price_precision(&self, symbol: Symbol, price_precision: u8) {
209 self.price_precisions.insert(symbol, price_precision);
210 }
211
212 #[must_use]
214 pub fn price_precision(&self, symbol: Symbol) -> Option<u8> {
215 self.price_precisions.load().get(&symbol).copied()
216 }
217
218 fn resolve_price_precision(
228 &self,
229 instrument_id: &InstrumentId,
230 price_precision: Option<u8>,
231 ) -> anyhow::Result<u8> {
232 if let Some(precision) = price_precision {
233 return Ok(precision);
234 }
235
236 let precisions = self.price_precisions.load();
237 precisions
238 .get(&instrument_id.symbol)
239 .copied()
240 .ok_or_else(|| {
241 anyhow::anyhow!(
242 "Could not resolve `price_precision` for {instrument_id}: \
243 pass `price_precision` explicitly, call `set_price_precision`, \
244 or fetch the instrument definitions first via `get_range_instruments`"
245 )
246 })
247 }
248
249 fn resolve_cached_price_precision(
250 &self,
251 instrument_id: &InstrumentId,
252 price_precision: Option<u8>,
253 precision_cache: &mut AHashMap<InstrumentId, u8>,
254 ) -> anyhow::Result<u8> {
255 if let Some(precision) = price_precision {
256 return Ok(precision);
257 }
258
259 if let Some(precision) = precision_cache.get(instrument_id) {
260 return Ok(*precision);
261 }
262
263 let precision = self.resolve_price_precision(instrument_id, None)?;
264 precision_cache.insert(*instrument_id, precision);
265 Ok(precision)
266 }
267
268 pub async fn get_dataset_range(&self, dataset: &str) -> anyhow::Result<DatasetRange> {
274 let mut client = (*self.inner).clone();
275 let response = client
276 .metadata()
277 .get_dataset_range(dataset)
278 .await
279 .map_err(|e| anyhow::anyhow!("Failed to get dataset range: {e}"))?;
280
281 Ok(DatasetRange {
282 start: response.start.to_string(),
283 end: response.end.to_string(),
284 })
285 }
286
287 pub async fn get_range_instruments(
293 &self,
294 params: RangeQueryParams,
295 ) -> anyhow::Result<Vec<InstrumentAny>> {
296 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
297 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
298 check_consistent_symbology(&symbols)?;
299
300 let first_symbol = params
301 .symbols
302 .first()
303 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
304 let stype_in = infer_symbology_type(first_symbol);
305 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
306 let time_range = get_date_time_range(params.start, end)?;
307
308 let range_params = GetRangeParams::builder()
309 .dataset(params.dataset)
310 .date_time_range(time_range)
311 .symbols(symbols)
312 .stype_in(stype_in)
313 .schema(dbn::Schema::Definition)
314 .maybe_limit(params.limit.and_then(NonZeroU64::new))
315 .build();
316
317 let mut client = (*self.inner).clone();
318 let mut decoder = client
319 .timeseries()
320 .get_range(&range_params)
321 .await
322 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
323
324 let metadata = decoder.metadata().clone();
325 let mut metadata_cache = MetadataCache::new(metadata);
326 let mut instruments = Vec::new();
327 let mut futures_symbols = Vec::new();
328
329 while let Some(msg) = decoder.decode_record::<dbn::InstrumentDefMsg>().await? {
330 let record = dbn::RecordRef::from(msg);
331 let sym_map = self.symbol_venue_map.load();
332 let mut instrument_id = decode_nautilus_instrument_id(
333 &record,
334 &mut metadata_cache,
335 &self.publisher_venue_map,
336 &sym_map,
337 self.futures_year_digits,
338 self.share_class_space,
339 &self.futures_symbols.load(),
340 Some(&requested_symbols),
341 )?;
342
343 if self.use_exchange_as_venue && instrument_id.venue == Venue::GLBX() {
344 let exchange = msg
345 .exchange()
346 .map_err(|e| anyhow::anyhow!("Missing exchange in record: {e}"))?;
347 let venue = Venue::from_code(exchange)
348 .map_err(|e| anyhow::anyhow!("Venue not found for exchange {exchange}: {e}"))?;
349 instrument_id.venue = venue;
350 }
351
352 match decode_instrument_def_msg(msg, instrument_id, None, None) {
353 Ok(Some(instrument)) => {
354 if nautilus_symbol(instrument.raw_symbol().as_str(), self.share_class_space)
355 != instrument.id().symbol
356 {
357 let date = time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(
358 msg.ts_recv,
359 ))?
360 .date();
361 futures_symbols.push((
362 (msg.hd.publisher_id, msg.hd.instrument_id, date),
363 instrument.id().symbol,
364 ));
365 }
366
367 instruments.push(instrument);
368 }
369 Ok(None) => {} Err(e) => anyhow::bail!("Failed to decode instrument {instrument_id}: {e}"),
371 }
372 }
373
374 if !futures_symbols.is_empty() {
375 self.futures_symbols
376 .rcu(|symbols| symbols.extend(futures_symbols.iter().copied()));
377 }
378
379 self.cache_price_precisions(&instruments);
380
381 Ok(instruments)
382 }
383
384 fn cache_price_precisions(&self, instruments: &[InstrumentAny]) {
385 if instruments.is_empty() {
386 return;
387 }
388
389 self.price_precisions.rcu(|precisions| {
390 for instrument in instruments {
391 precisions.insert(instrument.id().symbol, instrument.price_precision());
392 }
393 });
394 }
395
396 pub async fn get_range_quotes(
402 &self,
403 params: RangeQueryParams,
404 schema: Option<String>,
405 ) -> anyhow::Result<Vec<QuoteTick>> {
406 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
407 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
408 check_consistent_symbology(&symbols)?;
409
410 let first_symbol = params
411 .symbols
412 .first()
413 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
414 let stype_in = infer_symbology_type(first_symbol);
415 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
416 let time_range = get_date_time_range(params.start, end)?;
417 let schema = schema.unwrap_or_else(|| "mbp-1".to_string());
418 let dbn_schema = dbn::Schema::from_str(&schema)?;
419
420 match dbn_schema {
421 dbn::Schema::Mbp1
422 | dbn::Schema::Tbbo
423 | dbn::Schema::Bbo1S
424 | dbn::Schema::Bbo1M
425 | dbn::Schema::Cmbp1
426 | dbn::Schema::Tcbbo
427 | dbn::Schema::Cbbo1S
428 | dbn::Schema::Cbbo1M => (),
429 _ => anyhow::bail!(
430 "Invalid schema. Must be one of: mbp-1, tbbo, bbo-1s, bbo-1m, cmbp-1, tcbbo, cbbo-1s, cbbo-1m"
431 ),
432 }
433
434 let range_params = GetRangeParams::builder()
435 .dataset(params.dataset)
436 .date_time_range(time_range)
437 .symbols(symbols)
438 .stype_in(stype_in)
439 .schema(dbn_schema)
440 .maybe_limit(params.limit.and_then(NonZeroU64::new))
441 .build();
442
443 let price_precision_arg = params.price_precision;
444
445 let mut client = (*self.inner).clone();
446 let mut decoder = client
447 .timeseries()
448 .get_range(&range_params)
449 .await
450 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
451
452 let metadata = decoder.metadata().clone();
453 let mut metadata_cache = MetadataCache::new(metadata);
454 let mut precision_cache = AHashMap::new();
455 let mut result: Vec<QuoteTick> = Vec::new();
456
457 let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
458 let instrument_id =
459 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
460 let price_precision = self.resolve_cached_price_precision(
461 &instrument_id,
462 price_precision_arg,
463 &mut precision_cache,
464 )?;
465
466 let (data, _) = decode_record(
467 &record,
468 instrument_id,
469 price_precision,
470 None,
471 false, true,
473 )?;
474
475 match data {
476 Some(Data::Quote(quote)) => result.push(quote),
477 None => {} _ => anyhow::bail!("Invalid data element not `QuoteTick`, was {data:?}"),
479 }
480 Ok(())
481 };
482
483 match dbn_schema {
484 dbn::Schema::Mbp1 => {
485 while let Some(msg) = decoder.decode_record::<dbn::Mbp1Msg>().await? {
486 process_record(dbn::RecordRef::from(msg))?;
487 }
488 }
489 dbn::Schema::Tbbo => {
490 while let Some(msg) = decoder.decode_record::<dbn::TbboMsg>().await? {
491 process_record(dbn::RecordRef::from(msg))?;
492 }
493 }
494 dbn::Schema::Cmbp1 => {
495 while let Some(msg) = decoder.decode_record::<dbn::Cmbp1Msg>().await? {
496 process_record(dbn::RecordRef::from(msg))?;
497 }
498 }
499 dbn::Schema::Tcbbo => {
500 while let Some(msg) = decoder.decode_record::<dbn::TcbboMsg>().await? {
501 process_record(dbn::RecordRef::from(msg))?;
502 }
503 }
504 dbn::Schema::Bbo1M => {
505 while let Some(msg) = decoder.decode_record::<dbn::Bbo1MMsg>().await? {
506 process_record(dbn::RecordRef::from(msg))?;
507 }
508 }
509 dbn::Schema::Bbo1S => {
510 while let Some(msg) = decoder.decode_record::<dbn::Bbo1SMsg>().await? {
511 process_record(dbn::RecordRef::from(msg))?;
512 }
513 }
514 dbn::Schema::Cbbo1S | dbn::Schema::Cbbo1M => {
515 while let Some(msg) = decoder.decode_record::<dbn::CbboMsg>().await? {
516 process_record(dbn::RecordRef::from(msg))?;
517 }
518 }
519 _ => anyhow::bail!("Invalid schema {dbn_schema}"),
520 }
521
522 Ok(result)
523 }
524
525 pub async fn get_range_order_book_depth(
531 &self,
532 params: RangeQueryParams,
533 depth: Option<usize>,
534 ) -> anyhow::Result<Vec<OrderBookDepth>> {
535 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
536 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
537 check_consistent_symbology(&symbols)?;
538
539 let first_symbol = params
540 .symbols
541 .first()
542 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
543 let stype_in = infer_symbology_type(first_symbol);
544 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
545 let time_range = get_date_time_range(params.start, end)?;
546
547 let _depth = depth.unwrap_or(10);
549 if _depth != 10 {
550 anyhow::bail!("Only depth=10 is currently supported for order book depths");
551 }
552
553 let range_params = GetRangeParams::builder()
554 .dataset(params.dataset)
555 .date_time_range(time_range)
556 .symbols(symbols)
557 .stype_in(stype_in)
558 .schema(dbn::Schema::Mbp10)
559 .maybe_limit(params.limit.and_then(NonZeroU64::new))
560 .build();
561
562 let price_precision_arg = params.price_precision;
563
564 let mut client = (*self.inner).clone();
565 let mut decoder = client
566 .timeseries()
567 .get_range(&range_params)
568 .await
569 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
570
571 let metadata = decoder.metadata().clone();
572 let mut metadata_cache = MetadataCache::new(metadata);
573 let mut precision_cache = AHashMap::new();
574 let mut result: Vec<OrderBookDepth> = Vec::new();
575
576 let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
577 let instrument_id =
578 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
579 let price_precision = self.resolve_cached_price_precision(
580 &instrument_id,
581 price_precision_arg,
582 &mut precision_cache,
583 )?;
584
585 if let Some(msg) = record.get::<dbn::Mbp10Msg>() {
586 let depth = decode_mbp10_msg(msg, instrument_id, price_precision, None)?;
587 result.push(depth);
588 }
589
590 Ok(())
591 };
592
593 while let Some(msg) = decoder.decode_record::<dbn::Mbp10Msg>().await? {
594 process_record(dbn::RecordRef::from(msg))?;
595 }
596
597 Ok(result)
598 }
599
600 pub async fn get_range_order_book_deltas(
606 &self,
607 params: RangeQueryParams,
608 ) -> anyhow::Result<Vec<OrderBookDelta>> {
609 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
610 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
611 check_consistent_symbology(&symbols)?;
612
613 let first_symbol = params
614 .symbols
615 .first()
616 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
617 let stype_in = infer_symbology_type(first_symbol);
618 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
619 let time_range = get_date_time_range(params.start, end)?;
620
621 let range_params = GetRangeParams::builder()
622 .dataset(params.dataset)
623 .date_time_range(time_range)
624 .symbols(symbols)
625 .stype_in(stype_in)
626 .schema(dbn::Schema::Mbo)
627 .maybe_limit(params.limit.and_then(NonZeroU64::new))
628 .build();
629
630 let price_precision_arg = params.price_precision;
631
632 let mut client = (*self.inner).clone();
633 let mut decoder = client
634 .timeseries()
635 .get_range(&range_params)
636 .await
637 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
638
639 let metadata = decoder.metadata().clone();
640 let mut metadata_cache = MetadataCache::new(metadata);
641 let mut precision_cache = AHashMap::new();
642 let mut result: Vec<OrderBookDelta> = Vec::new();
643 let mut delta_buffer = MboDeltaBuffer::default();
644
645 let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
646 let instrument_id =
647 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
648 let price_precision = self.resolve_cached_price_precision(
649 &instrument_id,
650 price_precision_arg,
651 &mut precision_cache,
652 )?;
653
654 if let Some(msg) = record.get::<dbn::MboMsg>() {
655 let (delta, _trade) =
656 decode_mbo_msg(msg, instrument_id, price_precision, None, false)?;
657 delta_buffer.push(msg, instrument_id, delta);
658 while let Some(delta) = delta_buffer.pop_ready() {
659 result.push(delta);
660 }
661 }
662
663 Ok(())
664 };
665
666 while let Some(msg) = decoder.decode_record::<dbn::MboMsg>().await? {
667 process_record(dbn::RecordRef::from(msg))?;
668 }
669
670 delta_buffer.finish();
671 while let Some(delta) = delta_buffer.pop_ready() {
672 result.push(delta);
673 }
674
675 Ok(result)
676 }
677
678 pub async fn get_range_trades(
684 &self,
685 params: RangeQueryParams,
686 schema: Option<String>,
687 ) -> anyhow::Result<Vec<TradeTick>> {
688 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
689 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
690 check_consistent_symbology(&symbols)?;
691
692 let first_symbol = params
693 .symbols
694 .first()
695 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
696 let stype_in = infer_symbology_type(first_symbol);
697 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
698 let time_range = get_date_time_range(params.start, end)?;
699 let schema = schema.unwrap_or_else(|| "trades".to_string());
700 let dbn_schema = dbn::Schema::from_str(&schema)?;
701
702 match dbn_schema {
703 dbn::Schema::Trades
704 | dbn::Schema::Tbbo
705 | dbn::Schema::Tcbbo
706 | dbn::Schema::Mbp1
707 | dbn::Schema::Cmbp1 => (),
708 _ => {
709 anyhow::bail!("Invalid schema. Must be one of: trades, tbbo, tcbbo, mbp-1, cmbp-1")
710 }
711 }
712
713 let range_params = GetRangeParams::builder()
714 .dataset(params.dataset)
715 .date_time_range(time_range)
716 .symbols(symbols)
717 .stype_in(stype_in)
718 .schema(dbn_schema)
719 .maybe_limit(params.limit.and_then(NonZeroU64::new))
720 .build();
721
722 let price_precision_arg = params.price_precision;
723
724 let mut client = (*self.inner).clone();
725 let mut decoder = client
726 .timeseries()
727 .get_range(&range_params)
728 .await
729 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
730
731 let metadata = decoder.metadata().clone();
732 let mut metadata_cache = MetadataCache::new(metadata);
733 let mut precision_cache = AHashMap::new();
734 let mut result: Vec<TradeTick> = Vec::new();
735
736 let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
737 let instrument_id =
738 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
739 let price_precision = self.resolve_cached_price_precision(
740 &instrument_id,
741 price_precision_arg,
742 &mut precision_cache,
743 )?;
744
745 let (data, data2) =
746 decode_record(&record, instrument_id, price_precision, None, true, true)?;
747
748 match (data, data2) {
749 (Some(Data::Trade(trade)), _) | (_, Some(Data::Trade(trade))) => result.push(trade),
750 (Some(_) | None, None) => {}
751 (None, Some(data)) => {
752 anyhow::bail!("Invalid data element not `TradeTick`, was {data:?}")
753 }
754 (Some(data), Some(_)) => {
755 anyhow::bail!("Invalid data element not `TradeTick`, was {data:?}")
756 }
757 }
758 Ok(())
759 };
760
761 match dbn_schema {
762 dbn::Schema::Trades => {
763 while let Some(msg) = decoder.decode_record::<dbn::TradeMsg>().await? {
764 process_record(dbn::RecordRef::from(msg))?;
765 }
766 }
767 dbn::Schema::Mbp1 => {
768 while let Some(msg) = decoder.decode_record::<dbn::Mbp1Msg>().await? {
769 process_record(dbn::RecordRef::from(msg))?;
770 }
771 }
772 dbn::Schema::Tbbo => {
773 while let Some(msg) = decoder.decode_record::<dbn::TbboMsg>().await? {
774 process_record(dbn::RecordRef::from(msg))?;
775 }
776 }
777 dbn::Schema::Cmbp1 => {
778 while let Some(msg) = decoder.decode_record::<dbn::Cmbp1Msg>().await? {
779 process_record(dbn::RecordRef::from(msg))?;
780 }
781 }
782 dbn::Schema::Tcbbo => {
783 while let Some(msg) = decoder.decode_record::<dbn::TcbboMsg>().await? {
784 process_record(dbn::RecordRef::from(msg))?;
785 }
786 }
787 _ => anyhow::bail!("Invalid schema {dbn_schema}"),
788 }
789
790 Ok(result)
791 }
792
793 pub async fn get_range_bars(
799 &self,
800 params: RangeQueryParams,
801 aggregation: BarAggregation,
802 timestamp_on_close: bool,
803 ) -> anyhow::Result<Vec<Bar>> {
804 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
805 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
806 check_consistent_symbology(&symbols)?;
807
808 let first_symbol = params
809 .symbols
810 .first()
811 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
812 let stype_in = infer_symbology_type(first_symbol);
813 let schema = ohlcv_schema_from_aggregation(aggregation)?;
814
815 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
816 let time_range = get_date_time_range(params.start, end)?;
817
818 let range_params = GetRangeParams::builder()
819 .dataset(params.dataset)
820 .date_time_range(time_range)
821 .symbols(symbols)
822 .stype_in(stype_in)
823 .schema(schema)
824 .maybe_limit(params.limit.and_then(NonZeroU64::new))
825 .build();
826
827 let price_precision_arg = params.price_precision;
828
829 let mut client = (*self.inner).clone();
830 let mut decoder = client
831 .timeseries()
832 .get_range(&range_params)
833 .await
834 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
835
836 let metadata = decoder.metadata().clone();
837 let mut metadata_cache = MetadataCache::new(metadata);
838 let mut precision_cache = AHashMap::new();
839 let mut result: Vec<Bar> = Vec::new();
840
841 while let Some(msg) = decoder.decode_record::<dbn::OhlcvMsg>().await? {
842 let record = dbn::RecordRef::from(msg);
843 let instrument_id =
844 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
845 let price_precision = self.resolve_cached_price_precision(
846 &instrument_id,
847 price_precision_arg,
848 &mut precision_cache,
849 )?;
850
851 let (data, _) = decode_record(
852 &record,
853 instrument_id,
854 price_precision,
855 None,
856 false, timestamp_on_close,
858 )?;
859
860 match data {
861 Some(Data::Bar(bar)) => {
862 result.push(bar);
863 }
864 _ => anyhow::bail!("Invalid data element not `Bar`, was {data:?}"),
865 }
866 }
867
868 Ok(result)
869 }
870
871 pub async fn get_range_imbalance(
877 &self,
878 params: RangeQueryParams,
879 ) -> anyhow::Result<Vec<DatabentoImbalance>> {
880 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
881 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
882 check_consistent_symbology(&symbols)?;
883
884 let first_symbol = params
885 .symbols
886 .first()
887 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
888 let stype_in = infer_symbology_type(first_symbol);
889 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
890 let time_range = get_date_time_range(params.start, end)?;
891
892 let range_params = GetRangeParams::builder()
893 .dataset(params.dataset)
894 .date_time_range(time_range)
895 .symbols(symbols)
896 .stype_in(stype_in)
897 .schema(dbn::Schema::Imbalance)
898 .maybe_limit(params.limit.and_then(NonZeroU64::new))
899 .build();
900
901 let price_precision_arg = params.price_precision;
902
903 let mut client = (*self.inner).clone();
904 let mut decoder = client
905 .timeseries()
906 .get_range(&range_params)
907 .await
908 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
909
910 let metadata = decoder.metadata().clone();
911 let mut metadata_cache = MetadataCache::new(metadata);
912 let mut precision_cache = AHashMap::new();
913 let mut result: Vec<DatabentoImbalance> = Vec::new();
914
915 while let Some(msg) = decoder.decode_record::<dbn::ImbalanceMsg>().await? {
916 let record = dbn::RecordRef::from(msg);
917 let instrument_id =
918 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
919 let price_precision = self.resolve_cached_price_precision(
920 &instrument_id,
921 price_precision_arg,
922 &mut precision_cache,
923 )?;
924
925 let imbalance = decode_imbalance_msg(msg, instrument_id, price_precision, None)?;
926 result.push(imbalance);
927 }
928
929 Ok(result)
930 }
931
932 pub async fn get_range_statistics(
938 &self,
939 params: RangeQueryParams,
940 ) -> anyhow::Result<Vec<DatabentoStatistics>> {
941 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
942 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
943 check_consistent_symbology(&symbols)?;
944
945 let first_symbol = params
946 .symbols
947 .first()
948 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
949 let stype_in = infer_symbology_type(first_symbol);
950 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
951 let time_range = get_date_time_range(params.start, end)?;
952
953 let range_params = GetRangeParams::builder()
954 .dataset(params.dataset)
955 .date_time_range(time_range)
956 .symbols(symbols)
957 .stype_in(stype_in)
958 .schema(dbn::Schema::Statistics)
959 .maybe_limit(params.limit.and_then(NonZeroU64::new))
960 .build();
961
962 let price_precision_arg = params.price_precision;
963
964 let mut client = (*self.inner).clone();
965 let mut decoder = client
966 .timeseries()
967 .get_range(&range_params)
968 .await
969 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
970
971 let metadata = decoder.metadata().clone();
972 let mut metadata_cache = MetadataCache::new(metadata);
973 let mut precision_cache = AHashMap::new();
974 let mut result: Vec<DatabentoStatistics> = Vec::new();
975
976 while let Some(msg) = decoder.decode_record::<dbn::StatMsg>().await? {
977 if !is_supported_stat_type(msg.stat_type) {
979 log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
980 continue;
981 }
982
983 let record = dbn::RecordRef::from(msg);
984 let instrument_id =
985 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
986 let price_precision = self.resolve_cached_price_precision(
987 &instrument_id,
988 price_precision_arg,
989 &mut precision_cache,
990 )?;
991
992 if let Some(statistics) =
993 decode_statistics_msg(msg, instrument_id, price_precision, None)?
994 {
995 result.push(statistics);
996 }
997 }
998
999 Ok(result)
1000 }
1001
1002 pub async fn get_range_status(
1008 &self,
1009 params: RangeQueryParams,
1010 ) -> anyhow::Result<Vec<InstrumentStatus>> {
1011 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
1012 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
1013 check_consistent_symbology(&symbols)?;
1014
1015 let first_symbol = params
1016 .symbols
1017 .first()
1018 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
1019 let stype_in = infer_symbology_type(first_symbol);
1020 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
1021 let time_range = get_date_time_range(params.start, end)?;
1022
1023 let range_params = GetRangeParams::builder()
1024 .dataset(params.dataset)
1025 .date_time_range(time_range)
1026 .symbols(symbols)
1027 .stype_in(stype_in)
1028 .schema(dbn::Schema::Status)
1029 .maybe_limit(params.limit.and_then(NonZeroU64::new))
1030 .build();
1031
1032 let mut client = (*self.inner).clone();
1033 let mut decoder = client
1034 .timeseries()
1035 .get_range(&range_params)
1036 .await
1037 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
1038
1039 let metadata = decoder.metadata().clone();
1040 let mut metadata_cache = MetadataCache::new(metadata);
1041 let mut result: Vec<InstrumentStatus> = Vec::new();
1042
1043 while let Some(msg) = decoder.decode_record::<dbn::StatusMsg>().await? {
1044 let record = dbn::RecordRef::from(msg);
1045 let instrument_id =
1046 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
1047
1048 let status = decode_status_msg(msg, instrument_id, None)?;
1049 result.push(status);
1050 }
1051
1052 Ok(result)
1053 }
1054
1055 fn instrument_id(
1056 &self,
1057 record: &dbn::RecordRef,
1058 metadata: &mut MetadataCache,
1059 requested_symbols: &AHashSet<Symbol>,
1060 ) -> anyhow::Result<InstrumentId> {
1061 decode_nautilus_instrument_id(
1062 record,
1063 metadata,
1064 &self.publisher_venue_map,
1065 &self.symbol_venue_map.load(),
1066 self.futures_year_digits,
1067 self.share_class_space,
1068 &self.futures_symbols.load(),
1069 Some(requested_symbols),
1070 )
1071 }
1072
1073 fn prepare_symbols(&self, params: &RangeQueryParams) -> (Vec<String>, AHashSet<Symbol>) {
1074 let mut requested_symbols = AHashSet::new();
1075
1076 let raw_symbols = params
1077 .symbols
1078 .iter()
1079 .map(|symbol| {
1080 let raw = raw_symbol(
1081 symbol,
1082 ¶ms.dataset,
1083 self.futures_year_digits,
1084 self.share_class_space,
1085 );
1086
1087 if raw != *symbol {
1088 requested_symbols.insert(Symbol::from(symbol.as_str()));
1089 }
1090
1091 raw
1092 })
1093 .collect();
1094
1095 (raw_symbols, requested_symbols)
1096 }
1097
1098 pub fn prepare_symbols_from_instrument_ids(
1100 &self,
1101 instrument_ids: &[InstrumentId],
1102 ) -> Vec<String> {
1103 self.symbol_venue_map.rcu(|m| {
1104 for id in instrument_ids {
1105 m.entry(id.symbol).or_insert(id.venue);
1106 }
1107 });
1108
1109 instrument_ids
1110 .iter()
1111 .map(|id| id.symbol.to_string())
1112 .collect()
1113 }
1114}
1115
1116#[cfg(test)]
1117mod tests {
1118 use std::ffi::c_char;
1119
1120 use databento::dbn::{
1121 MappingInterval, SymbolMapping, decode::DecodeRecord, encode::EncodeRecord,
1122 };
1123 use nautilus_core::time::get_atomic_clock_realtime;
1124 use nautilus_model::types::{Price, Quantity};
1125 use rstest::{fixture, rstest};
1126 use time::macros::date;
1127 use tokio::{
1128 io::{AsyncReadExt, AsyncWriteExt},
1129 net::TcpListener,
1130 };
1131
1132 use super::*;
1133
1134 fn test_api_key() -> String {
1135 "test-000000000000000000000000000".to_string()
1136 }
1137
1138 fn publishers_path() -> PathBuf {
1139 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json")
1140 }
1141
1142 fn mbo_record(action: c_char, flags: u8, sequence: u32) -> dbn::MboMsg {
1143 let ts_event = 1_609_160_400_000_000_000;
1144 dbn::MboMsg {
1145 hd: dbn::RecordHeader::new::<dbn::MboMsg>(dbn::rtype::MBO, 1, 1, ts_event),
1146 order_id: u64::from(sequence),
1147 price: 4_800_250_000_000,
1148 size: 2,
1149 flags: dbn::FlagSet::new(flags),
1150 channel_id: 1,
1151 action,
1152 side: 'A' as c_char,
1153 ts_recv: ts_event,
1154 ts_in_delta: 0,
1155 sequence,
1156 }
1157 }
1158
1159 fn encode_mbo_response(records: &[dbn::MboMsg]) -> Vec<u8> {
1160 let metadata = dbn::Metadata::builder()
1161 .dataset("GLBX.MDP3")
1162 .schema(Some(dbn::Schema::Mbo))
1163 .start(1_609_160_400_000_000_000)
1164 .stype_in(Some(dbn::SType::RawSymbol))
1165 .stype_out(dbn::SType::InstrumentId)
1166 .symbols(vec!["ESM4".to_string()])
1167 .mappings(vec![SymbolMapping {
1168 raw_symbol: "ESM4".to_string(),
1169 intervals: vec![MappingInterval {
1170 start_date: date!(2020 - 12 - 28),
1171 end_date: date!(2020 - 12 - 29),
1172 symbol: "1".to_string(),
1173 }],
1174 }])
1175 .build();
1176 let mut body = Vec::new();
1177 {
1178 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1179 encoder.encode_records(records).unwrap();
1180 encoder.flush().unwrap();
1181 }
1182 body
1183 }
1184
1185 async fn serve_response(listener: TcpListener, body: Vec<u8>) {
1186 let (mut stream, _) = listener.accept().await.unwrap();
1187 let mut request = Vec::new();
1188 let mut buffer = [0_u8; 1024];
1189 loop {
1190 let read = stream.read(&mut buffer).await.unwrap();
1191 if read == 0 {
1192 break;
1193 }
1194 request.extend_from_slice(&buffer[..read]);
1195 if request.windows(4).any(|window| window == b"\r\n\r\n") {
1196 break;
1197 }
1198 }
1199
1200 let headers = format!(
1201 "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
1202 body.len(),
1203 );
1204 stream.write_all(headers.as_bytes()).await.unwrap();
1205 stream.write_all(&body).await.unwrap();
1206 stream.shutdown().await.unwrap();
1207 }
1208
1209 #[fixture]
1210 fn historical_client() -> DatabentoHistoricalClient {
1211 DatabentoHistoricalClient::new(
1212 Credential::new(test_api_key()),
1213 publishers_path(),
1214 get_atomic_clock_realtime(),
1215 false,
1216 )
1217 .unwrap()
1218 }
1219
1220 #[rstest]
1221 fn test_new_with_base_url_rejects_invalid_url() {
1222 let err = DatabentoHistoricalClient::new_with_base_url(
1223 Credential::new(test_api_key()),
1224 publishers_path(),
1225 get_atomic_clock_realtime(),
1226 false,
1227 "://invalid",
1228 )
1229 .expect_err("expected invalid base URL to fail");
1230 let err_msg = format!("{err}");
1231
1232 assert!(
1233 err_msg.contains("Failed to parse Databento Historical API base URL"),
1234 "unexpected error message: {err_msg}",
1235 );
1236 }
1237
1238 #[rstest]
1239 #[tokio::test]
1240 async fn test_get_range_order_book_deltas_preserves_boundaries_and_drains_end() {
1241 let last = dbn::flags::LAST;
1242 let body = encode_mbo_response(&[
1243 mbo_record('C' as c_char, 0, 1),
1244 mbo_record('N' as c_char, last, 2),
1245 mbo_record('C' as c_char, 0, 3),
1246 ]);
1247 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1248 let base_url = format!("http://{}/", listener.local_addr().unwrap());
1249 let server = tokio::spawn(serve_response(listener, body));
1250 let client = DatabentoHistoricalClient::new_with_base_url(
1251 Credential::new(test_api_key()),
1252 publishers_path(),
1253 get_atomic_clock_realtime(),
1254 false,
1255 &base_url,
1256 )
1257 .unwrap();
1258
1259 let deltas = client
1260 .get_range_order_book_deltas(RangeQueryParams {
1261 dataset: "GLBX.MDP3".to_string(),
1262 symbols: vec!["ESM4".to_string()],
1263 start: 1_609_160_000_000_000_000.into(),
1264 end: Some(1_609_161_000_000_000_000.into()),
1265 limit: None,
1266 price_precision: Some(2),
1267 })
1268 .await
1269 .unwrap();
1270 server.await.unwrap();
1271
1272 assert_eq!(deltas.len(), 2);
1273 assert_eq!(deltas[0].instrument_id, InstrumentId::from("ESM24.GLBX"));
1274 assert_eq!(deltas[0].order.order_id, 1);
1275 assert_eq!(deltas[0].flags, last);
1276 assert_eq!(deltas[0].sequence, 1);
1277 assert_eq!(deltas[1].instrument_id, InstrumentId::from("ESM24.GLBX"));
1278 assert_eq!(deltas[1].order.order_id, 3);
1279 assert_eq!(deltas[1].flags, 0);
1280 assert_eq!(deltas[1].sequence, 3);
1281 }
1282
1283 #[rstest]
1284 fn test_definition_precision_batch_merges_and_last_symbol_wins(
1285 historical_client: DatabentoHistoricalClient,
1286 ) {
1287 let mut first = nautilus_model::instruments::stubs::crypto_perpetual_ethusdt();
1288 first.id = "FIRST.GLBX".into();
1289 first.price_precision = 3;
1290 let mut second = first.clone();
1291 second.id = "SECOND.GLBX".into();
1292 second.price_precision = 5;
1293 let mut latest = first.clone();
1294 latest.id = "FIRST.XNAS".into();
1295 latest.price_precision = 7;
1296 historical_client.set_price_precision("RETAINED".into(), 9);
1297 historical_client.set_price_precision("FIRST".into(), 1);
1298
1299 historical_client.cache_price_precisions(&[
1300 InstrumentAny::CryptoPerpetual(first),
1301 InstrumentAny::CryptoPerpetual(second),
1302 InstrumentAny::CryptoPerpetual(latest),
1303 ]);
1304
1305 assert_eq!(historical_client.price_precisions.len(), 3);
1306
1307 for (symbol, precision) in [("RETAINED", 9), ("FIRST", 7), ("SECOND", 5)] {
1308 assert_eq!(
1309 historical_client
1310 .resolve_price_precision(&InstrumentId::new(symbol.into(), "GLBX".into()), None)
1311 .unwrap(),
1312 precision
1313 );
1314 }
1315 }
1316
1317 #[rstest]
1318 fn test_set_price_precision_inserts_into_cache(historical_client: DatabentoHistoricalClient) {
1319 let symbol = Symbol::from("ESM4");
1320
1321 assert_eq!(historical_client.price_precision(symbol), None);
1322
1323 historical_client.set_price_precision(symbol, 2);
1324
1325 assert_eq!(historical_client.price_precision(symbol), Some(2));
1326 }
1327
1328 #[rstest]
1329 fn test_resolve_price_precision_explicit_arg(historical_client: DatabentoHistoricalClient) {
1330 let instrument_id = InstrumentId::from("ESM4.GLBX");
1331 historical_client.set_price_precision(Symbol::from("ESM4"), 9);
1333
1334 let precision = historical_client
1335 .resolve_price_precision(&instrument_id, Some(2))
1336 .unwrap();
1337 assert_eq!(precision, 2);
1338 }
1339
1340 #[rstest]
1341 fn test_resolve_price_precision_cache_hit(historical_client: DatabentoHistoricalClient) {
1342 let instrument_id = InstrumentId::from("ESM4.GLBX");
1343 historical_client.set_price_precision(Symbol::from("ESM4"), 2);
1344
1345 let precision = historical_client
1346 .resolve_price_precision(&instrument_id, None)
1347 .unwrap();
1348 assert_eq!(precision, 2);
1349 }
1350
1351 #[rstest]
1352 fn test_resolve_price_precision_cache_miss_errors(
1353 historical_client: DatabentoHistoricalClient,
1354 ) {
1355 let instrument_id = InstrumentId::from("ESM4.GLBX");
1356
1357 let err = historical_client
1358 .resolve_price_precision(&instrument_id, None)
1359 .expect_err("expected cache-miss error");
1360 let err_msg = format!("{err}");
1361 assert!(
1362 err_msg.contains("Could not resolve `price_precision`"),
1363 "unexpected error message: {err_msg}",
1364 );
1365 assert!(
1366 err_msg.contains("ESM4.GLBX"),
1367 "error should name the instrument: {err_msg}",
1368 );
1369 }
1370
1371 #[rstest]
1372 #[case(1_000)]
1373 #[case(10_000)]
1374 #[tokio::test]
1375 async fn test_historical_definition_batch_preserves_all_symbols(#[case] count: u32) {
1376 let path = publishers_path()
1377 .parent()
1378 .unwrap()
1379 .join("test_data/test_data.definition.futures_contract.dbn.zst");
1380 let mut decoder = dbn::decode::dbn::Decoder::from_zstd_file(path).unwrap();
1381 let mut definition = decoder
1382 .decode_record::<dbn::InstrumentDefMsg>()
1383 .unwrap()
1384 .unwrap()
1385 .clone();
1386 definition.ts_recv = "2026-01-01T00:00:00Z"
1387 .parse::<jiff::Timestamp>()
1388 .unwrap()
1389 .as_nanosecond() as u64;
1390 definition.maturity_year = 2026;
1391 let mut definitions = Vec::new();
1392 let mut mappings = Vec::new();
1393
1394 for id in 1..=count {
1395 let raw = format!("T{id}U6");
1396 definition.hd.instrument_id = id;
1397 definition.raw_symbol.fill(0);
1398
1399 for (target, byte) in definition.raw_symbol.iter_mut().zip(raw.bytes()) {
1400 *target = byte as c_char;
1401 }
1402
1403 definitions.push(definition.clone());
1404 mappings.push(SymbolMapping {
1405 raw_symbol: raw,
1406 intervals: vec![MappingInterval {
1407 start_date: date!(2026 - 01 - 01),
1408 end_date: date!(2026 - 01 - 02),
1409 symbol: id.to_string(),
1410 }],
1411 });
1412 }
1413
1414 let mut replacement = definitions[0].clone();
1415 replacement.maturity_year = 2036;
1416 definitions.push(replacement);
1417 let metadata = dbn::Metadata::builder()
1418 .dataset("GLBX.MDP3")
1419 .schema(Some(dbn::Schema::Definition))
1420 .start(definition.ts_recv)
1421 .stype_in(Some(dbn::SType::Parent))
1422 .stype_out(dbn::SType::InstrumentId)
1423 .mappings(mappings)
1424 .build();
1425 let mut body = Vec::new();
1426 {
1427 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1428 encoder.encode_records(&definitions).unwrap();
1429 encoder.flush().unwrap();
1430 }
1431
1432 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1433 let base_url = format!("http://{}/", listener.local_addr().unwrap());
1434
1435 let server = tokio::spawn(serve_response(listener, body));
1436 let client = DatabentoHistoricalClient::new_with_base_url(
1437 Credential::new(test_api_key()),
1438 publishers_path(),
1439 get_atomic_clock_realtime(),
1440 false,
1441 &base_url,
1442 )
1443 .unwrap();
1444 let publisher = definition.hd.publisher_id;
1445 let old_key = (publisher, 1, date!(2025 - 12 - 31));
1446 client
1447 .futures_symbols
1448 .insert(old_key, Symbol::from("T1U16"));
1449 let started = std::time::Instant::now();
1450
1451 let instruments = client
1452 .get_range_instruments(RangeQueryParams {
1453 dataset: "GLBX.MDP3".to_string(),
1454 symbols: vec!["T.FUT".to_string()],
1455 start: definition.ts_recv.into(),
1456 end: Some((definition.ts_recv + 1).into()),
1457 limit: None,
1458 price_precision: None,
1459 })
1460 .await
1461 .unwrap();
1462
1463 let elapsed = started.elapsed();
1464 server.await.unwrap();
1465 let symbols = client.futures_symbols.load();
1466 let mut expected = AHashMap::new();
1467 expected.insert(old_key, Symbol::from("T1U16"));
1468
1469 for id in 1..=count {
1470 let symbol = Symbol::from(format!("T{id}U26").as_str());
1471 expected.insert((publisher, id, date!(2026 - 01 - 01)), symbol);
1472 assert_eq!(
1473 instruments[(id - 1) as usize].id(),
1474 InstrumentId::new(symbol, Venue::GLBX())
1475 );
1476 assert_eq!(
1477 instruments[(id - 1) as usize].raw_symbol(),
1478 Symbol::from(format!("T{id}U6").as_str())
1479 );
1480 }
1481
1482 expected.insert((publisher, 1, date!(2026 - 01 - 01)), Symbol::from("T1U36"));
1483 assert_eq!(instruments.len(), count as usize + 1);
1484 assert_eq!(
1485 instruments.last().unwrap().id(),
1486 InstrumentId::from("T1U36.GLBX")
1487 );
1488 assert_eq!(symbols.as_ref(), &expected);
1489 eprintln!("Decoded {count} definitions plus replacement in {elapsed:?}");
1490 }
1491
1492 #[tokio::test]
1493 async fn test_historical_records_use_definition_year_for_the_same_date() {
1494 let path = publishers_path()
1495 .parent()
1496 .unwrap()
1497 .join("test_data/test_data.definition.futures_contract.dbn.zst");
1498 let mut decoder = dbn::decode::dbn::Decoder::from_zstd_file(path).unwrap();
1499 let mut definition = decoder
1500 .decode_record::<dbn::InstrumentDefMsg>()
1501 .unwrap()
1502 .unwrap()
1503 .clone();
1504 definition.ts_recv = "2028-01-01T00:00:00Z"
1505 .parse::<jiff::Timestamp>()
1506 .unwrap()
1507 .as_nanosecond() as u64;
1508 let metadata = dbn::Metadata::builder()
1509 .dataset("GLBX.MDP3")
1510 .schema(Some(dbn::Schema::Definition))
1511 .start(definition.ts_recv)
1512 .stype_in(Some(dbn::SType::RawSymbol))
1513 .stype_out(dbn::SType::InstrumentId)
1514 .mappings(vec![SymbolMapping {
1515 raw_symbol: "ESU6".to_string(),
1516 intervals: vec![MappingInterval {
1517 start_date: date!(2028 - 01 - 01),
1518 end_date: date!(2028 - 01 - 02),
1519 symbol: definition.hd.instrument_id.to_string(),
1520 }],
1521 }])
1522 .build();
1523 let mut body = Vec::new();
1524 {
1525 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1526 encoder.encode_record(&definition).unwrap();
1527 encoder.flush().unwrap();
1528 }
1529
1530 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1531 let base_url = format!("http://{}/", listener.local_addr().unwrap());
1532
1533 let server = tokio::spawn(serve_response(listener, body));
1534 let client = DatabentoHistoricalClient::new_with_base_url(
1535 Credential::new(test_api_key()),
1536 publishers_path(),
1537 get_atomic_clock_realtime(),
1538 false,
1539 &base_url,
1540 )
1541 .unwrap();
1542
1543 let instruments = client
1544 .get_range_instruments(RangeQueryParams {
1545 dataset: "GLBX.MDP3".to_string(),
1546 symbols: vec!["ESU26".to_string()],
1547 start: definition.ts_recv.into(),
1548 end: Some((definition.ts_recv + 1).into()),
1549 limit: Some(1),
1550 price_precision: None,
1551 })
1552 .await
1553 .unwrap();
1554
1555 server.await.unwrap();
1556 let mut record = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1557 record.hd.publisher_id = definition.hd.publisher_id;
1558 record.hd.instrument_id = definition.hd.instrument_id;
1559 record.ts_recv = definition.ts_recv;
1560 let id = client
1561 .instrument_id(
1562 &dbn::RecordRef::from(&record),
1563 &mut MetadataCache::new(metadata),
1564 &AHashSet::from_iter([Symbol::from("ESU36")]),
1565 )
1566 .unwrap();
1567 assert_eq!(instruments.len(), 1);
1568 assert_eq!(instruments[0].id(), InstrumentId::from("ESU26.GLBX"));
1569 assert_eq!(instruments[0].raw_symbol(), Symbol::from("ESU6"));
1570 assert_eq!(id, InstrumentId::from("ESU26.GLBX"));
1571 }
1572
1573 #[rstest]
1574 #[case(None, "SR3Z26.XCME", true)]
1575 #[case(Some("SR3Z36.XNYM"), "SR3Z26.XCME", true)]
1576 #[case(Some("SR3Z26.XNYM"), "SR3Z36.XCME", true)]
1577 #[case(Some("SR3Z36.XNYM"), "SR3Z26.GLBX", false)]
1578 #[tokio::test]
1579 async fn test_requested_post_year_quotes_use_cached_precision_without_definitions(
1580 #[case] previous: Option<&str>,
1581 #[case] requested: &str,
1582 #[case] register_venue: bool,
1583 ) {
1584 let ts = "2027-02-01T00:00:00Z"
1585 .parse::<jiff::Timestamp>()
1586 .unwrap()
1587 .as_nanosecond() as u64;
1588 let mut msg = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1589 msg.hd.publisher_id = 1;
1590 msg.hd.instrument_id = 42;
1591 msg.hd.ts_event = ts - 1;
1592 msg.ts_recv = ts;
1593 msg.levels[0].bid_px = 95_125_000_000;
1594 msg.levels[0].ask_px = 95_250_000_000;
1595 msg.levels[0].bid_sz = 7;
1596 msg.levels[0].ask_sz = 11;
1597 let metadata = dbn::Metadata::builder()
1598 .dataset("GLBX.MDP3")
1599 .schema(Some(dbn::Schema::Bbo1S))
1600 .start(ts)
1601 .stype_in(Some(dbn::SType::RawSymbol))
1602 .stype_out(dbn::SType::InstrumentId)
1603 .mappings(vec![SymbolMapping {
1604 raw_symbol: "SR3Z6".to_string(),
1605 intervals: vec![MappingInterval {
1606 start_date: date!(2027 - 02 - 01),
1607 end_date: date!(2027 - 02 - 02),
1608 symbol: "42".to_string(),
1609 }],
1610 }])
1611 .build();
1612 let mut body = Vec::new();
1613 {
1614 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1615 encoder.encode_record(&msg).unwrap();
1616 encoder.flush().unwrap();
1617 }
1618
1619 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1620 let base_url = format!("http://{}/", listener.local_addr().unwrap());
1621
1622 let server = tokio::spawn(serve_response(listener, body));
1623 let client = DatabentoHistoricalClient::new_with_base_url(
1624 Credential::new(test_api_key()),
1625 publishers_path(),
1626 get_atomic_clock_realtime(),
1627 false,
1628 &base_url,
1629 )
1630 .unwrap();
1631
1632 if let Some(previous) = previous {
1633 let previous = InstrumentId::from(previous);
1634 client.prepare_symbols_from_instrument_ids(&[previous]);
1635 client.set_price_precision(previous.symbol, 3);
1636 }
1637
1638 let id = InstrumentId::from(requested);
1639 let symbols = if register_venue {
1640 client.prepare_symbols_from_instrument_ids(&[id])
1641 } else {
1642 vec![id.symbol.to_string()]
1643 };
1644
1645 client.set_price_precision(id.symbol, 3);
1646
1647 let quotes = client
1648 .get_range_quotes(
1649 RangeQueryParams {
1650 dataset: "GLBX.MDP3".to_string(),
1651 symbols,
1652 start: ts.into(),
1653 end: Some((ts + 1).into()),
1654 limit: Some(1),
1655 price_precision: None,
1656 },
1657 Some("bbo-1s".to_string()),
1658 )
1659 .await
1660 .unwrap();
1661
1662 server.await.unwrap();
1663
1664 assert_eq!(
1665 quotes,
1666 vec![QuoteTick::new(
1667 id,
1668 Price::from("95.125"),
1669 Price::from("95.250"),
1670 Quantity::from(7),
1671 Quantity::from(11),
1672 ts.into(),
1673 ts.into(),
1674 )]
1675 );
1676 }
1677}