1use std::{fmt::Debug, fs, num::NonZeroU64, path::PathBuf, str::FromStr, sync::Arc};
19
20use ahash::{AHashMap, AHashSet};
21use databento::{
22 dbn::{self, decode::DbnMetadata},
23 historical::timeseries::GetRangeParams,
24};
25use indexmap::IndexMap;
26use nautilus_core::{AtomicMap, UnixNanos, consts::NAUTILUS_USER_AGENT, time::AtomicTime};
27use nautilus_model::{
28 data::{Bar, Data, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
29 enums::BarAggregation,
30 identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
31 instruments::{Instrument, InstrumentAny},
32};
33
34use crate::{
35 common::{Credential, get_date_time_range, ohlcv_schema_from_aggregation},
36 decode::{
37 MboDeltaBuffer, decode_imbalance_msg, decode_instrument_def_msg, decode_mbo_msg,
38 decode_mbp10_msg, decode_record, decode_statistics_msg, decode_status_msg,
39 is_supported_stat_type,
40 },
41 symbology::{
42 MetadataCache, check_consistent_symbology, decode_nautilus_instrument_id,
43 infer_symbology_type, raw_symbol,
44 },
45 types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, PublisherId},
46};
47
48#[derive(Clone)]
53pub struct DatabentoHistoricalClient {
54 futures_year_digits: FuturesYearDigits,
55 credential: Credential,
56 clock: &'static AtomicTime,
57 inner: Arc<databento::HistoricalClient>,
58 publisher_venue_map: Arc<IndexMap<PublisherId, Venue>>,
59 symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
60 price_precisions: Arc<AtomicMap<Symbol, u8>>,
61 use_exchange_as_venue: bool,
62 futures_symbols: Arc<AtomicMap<(PublisherId, u32, time::Date), Symbol>>,
63}
64
65#[derive(Debug)]
67pub struct RangeQueryParams {
68 pub dataset: String,
69 pub symbols: Vec<String>,
70 pub start: UnixNanos,
71 pub end: Option<UnixNanos>,
72 pub limit: Option<u64>,
73 pub price_precision: Option<u8>,
74}
75
76#[derive(Debug, Clone)]
78pub struct DatasetRange {
79 pub start: String,
80 pub end: String,
81}
82
83impl Debug for DatabentoHistoricalClient {
84 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
85 f.debug_struct(stringify!(DatabentoHistoricalClient))
86 .field("credential", &self.credential)
87 .finish()
88 }
89}
90
91impl DatabentoHistoricalClient {
92 #[must_use]
94 pub fn api_key(&self) -> &str {
95 self.credential.api_key()
96 }
97
98 pub fn new(
104 credential: Credential,
105 publishers_filepath: PathBuf,
106 clock: &'static AtomicTime,
107 use_exchange_as_venue: bool,
108 ) -> anyhow::Result<Self> {
109 let client = databento::HistoricalClient::builder()
110 .user_agent_extension(NAUTILUS_USER_AGENT.into())
111 .key(credential.api_key())
112 .map_err(|e| anyhow::anyhow!("Failed to create client builder: {e}"))?
113 .build()
114 .map_err(|e| anyhow::anyhow!("Failed to build client: {e}"))?;
115
116 Self::from_client(
117 credential,
118 publishers_filepath,
119 clock,
120 use_exchange_as_venue,
121 client,
122 )
123 }
124
125 #[must_use]
127 pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
128 self.futures_year_digits = futures_year_digits;
129 self
130 }
131
132 pub fn new_with_base_url(
141 credential: Credential,
142 publishers_filepath: PathBuf,
143 clock: &'static AtomicTime,
144 use_exchange_as_venue: bool,
145 base_url: &str,
146 ) -> anyhow::Result<Self> {
147 let client = databento::HistoricalClient::builder()
148 .user_agent_extension(NAUTILUS_USER_AGENT.into())
149 .base_url(base_url.parse().map_err(|e| {
150 anyhow::anyhow!("Failed to parse Databento Historical API base URL: {e}")
151 })?)
152 .key(credential.api_key())
153 .map_err(|e| anyhow::anyhow!("Failed to create client builder: {e}"))?
154 .build()
155 .map_err(|e| anyhow::anyhow!("Failed to build client: {e}"))?;
156
157 Self::from_client(
158 credential,
159 publishers_filepath,
160 clock,
161 use_exchange_as_venue,
162 client,
163 )
164 }
165
166 fn from_client(
167 credential: Credential,
168 publishers_filepath: PathBuf,
169 clock: &'static AtomicTime,
170 use_exchange_as_venue: bool,
171 client: databento::HistoricalClient,
172 ) -> anyhow::Result<Self> {
173 let file_content = fs::read_to_string(publishers_filepath)?;
174 let publishers_vec: Vec<DatabentoPublisher> = serde_json::from_str(&file_content)?;
175
176 let publisher_venue_map = publishers_vec
177 .into_iter()
178 .map(|p| (p.publisher_id, Venue::from(p.venue.as_str())))
179 .collect::<IndexMap<u16, Venue>>();
180
181 Ok(Self {
182 clock,
183 futures_year_digits: FuturesYearDigits::default(),
184 inner: Arc::new(client),
185 publisher_venue_map: Arc::new(publisher_venue_map),
186 symbol_venue_map: Arc::new(AtomicMap::new()),
187 price_precisions: Arc::new(AtomicMap::new()),
188 credential,
189 use_exchange_as_venue,
190 futures_symbols: Arc::new(AtomicMap::new()),
191 })
192 }
193
194 pub fn set_price_precision(&self, symbol: Symbol, price_precision: u8) {
200 self.price_precisions.insert(symbol, price_precision);
201 }
202
203 #[must_use]
205 pub fn price_precision(&self, symbol: Symbol) -> Option<u8> {
206 self.price_precisions.load().get(&symbol).copied()
207 }
208
209 fn resolve_price_precision(
219 &self,
220 instrument_id: &InstrumentId,
221 price_precision: Option<u8>,
222 ) -> anyhow::Result<u8> {
223 if let Some(precision) = price_precision {
224 return Ok(precision);
225 }
226
227 let precisions = self.price_precisions.load();
228 precisions
229 .get(&instrument_id.symbol)
230 .copied()
231 .ok_or_else(|| {
232 anyhow::anyhow!(
233 "Could not resolve `price_precision` for {instrument_id}: \
234 pass `price_precision` explicitly, call `set_price_precision`, \
235 or fetch the instrument definitions first via `get_range_instruments`"
236 )
237 })
238 }
239
240 fn resolve_cached_price_precision(
241 &self,
242 instrument_id: &InstrumentId,
243 price_precision: Option<u8>,
244 precision_cache: &mut AHashMap<InstrumentId, u8>,
245 ) -> anyhow::Result<u8> {
246 if let Some(precision) = price_precision {
247 return Ok(precision);
248 }
249
250 if let Some(precision) = precision_cache.get(instrument_id) {
251 return Ok(*precision);
252 }
253
254 let precision = self.resolve_price_precision(instrument_id, None)?;
255 precision_cache.insert(*instrument_id, precision);
256 Ok(precision)
257 }
258
259 pub async fn get_dataset_range(&self, dataset: &str) -> anyhow::Result<DatasetRange> {
265 let mut client = (*self.inner).clone();
266 let response = client
267 .metadata()
268 .get_dataset_range(dataset)
269 .await
270 .map_err(|e| anyhow::anyhow!("Failed to get dataset range: {e}"))?;
271
272 Ok(DatasetRange {
273 start: response.start.to_string(),
274 end: response.end.to_string(),
275 })
276 }
277
278 pub async fn get_range_instruments(
284 &self,
285 params: RangeQueryParams,
286 ) -> anyhow::Result<Vec<InstrumentAny>> {
287 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
288 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
289 check_consistent_symbology(&symbols)?;
290
291 let first_symbol = params
292 .symbols
293 .first()
294 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
295 let stype_in = infer_symbology_type(first_symbol);
296 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
297 let time_range = get_date_time_range(params.start, end)?;
298
299 let range_params = GetRangeParams::builder()
300 .dataset(params.dataset)
301 .date_time_range(time_range)
302 .symbols(symbols)
303 .stype_in(stype_in)
304 .schema(dbn::Schema::Definition)
305 .maybe_limit(params.limit.and_then(NonZeroU64::new))
306 .build();
307
308 let mut client = (*self.inner).clone();
309 let mut decoder = client
310 .timeseries()
311 .get_range(&range_params)
312 .await
313 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
314
315 let metadata = decoder.metadata().clone();
316 let mut metadata_cache = MetadataCache::new(metadata);
317 let mut instruments = Vec::new();
318 let mut futures_symbols = Vec::new();
319
320 while let Some(msg) = decoder.decode_record::<dbn::InstrumentDefMsg>().await? {
321 let record = dbn::RecordRef::from(msg);
322 let sym_map = self.symbol_venue_map.load();
323 let mut instrument_id = decode_nautilus_instrument_id(
324 &record,
325 &mut metadata_cache,
326 &self.publisher_venue_map,
327 &sym_map,
328 self.futures_year_digits,
329 &self.futures_symbols.load(),
330 Some(&requested_symbols),
331 )?;
332
333 if self.use_exchange_as_venue && instrument_id.venue == Venue::GLBX() {
334 let exchange = msg
335 .exchange()
336 .map_err(|e| anyhow::anyhow!("Missing exchange in record: {e}"))?;
337 let venue = Venue::from_code(exchange)
338 .map_err(|e| anyhow::anyhow!("Venue not found for exchange {exchange}: {e}"))?;
339 instrument_id.venue = venue;
340 }
341
342 match decode_instrument_def_msg(msg, instrument_id, None, None) {
343 Ok(Some(instrument)) => {
344 if instrument.raw_symbol() != instrument.id().symbol {
345 let date = time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(
346 msg.ts_recv,
347 ))?
348 .date();
349 futures_symbols.push((
350 (msg.hd.publisher_id, msg.hd.instrument_id, date),
351 instrument.id().symbol,
352 ));
353 }
354
355 instruments.push(instrument);
356 }
357 Ok(None) => {} Err(e) => anyhow::bail!("Failed to decode instrument {instrument_id}: {e}"),
359 }
360 }
361
362 if !futures_symbols.is_empty() {
363 self.futures_symbols
364 .rcu(|symbols| symbols.extend(futures_symbols.iter().copied()));
365 }
366
367 self.cache_price_precisions(&instruments);
368
369 Ok(instruments)
370 }
371
372 fn cache_price_precisions(&self, instruments: &[InstrumentAny]) {
373 if instruments.is_empty() {
374 return;
375 }
376
377 self.price_precisions.rcu(|precisions| {
378 for instrument in instruments {
379 precisions.insert(instrument.id().symbol, instrument.price_precision());
380 }
381 });
382 }
383
384 pub async fn get_range_quotes(
390 &self,
391 params: RangeQueryParams,
392 schema: Option<String>,
393 ) -> anyhow::Result<Vec<QuoteTick>> {
394 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
395 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
396 check_consistent_symbology(&symbols)?;
397
398 let first_symbol = params
399 .symbols
400 .first()
401 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
402 let stype_in = infer_symbology_type(first_symbol);
403 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
404 let time_range = get_date_time_range(params.start, end)?;
405 let schema = schema.unwrap_or_else(|| "mbp-1".to_string());
406 let dbn_schema = dbn::Schema::from_str(&schema)?;
407
408 match dbn_schema {
409 dbn::Schema::Mbp1
410 | dbn::Schema::Tbbo
411 | dbn::Schema::Bbo1S
412 | dbn::Schema::Bbo1M
413 | dbn::Schema::Cmbp1
414 | dbn::Schema::Tcbbo
415 | dbn::Schema::Cbbo1S
416 | dbn::Schema::Cbbo1M => (),
417 _ => anyhow::bail!(
418 "Invalid schema. Must be one of: mbp-1, tbbo, bbo-1s, bbo-1m, cmbp-1, tcbbo, cbbo-1s, cbbo-1m"
419 ),
420 }
421
422 let range_params = GetRangeParams::builder()
423 .dataset(params.dataset)
424 .date_time_range(time_range)
425 .symbols(symbols)
426 .stype_in(stype_in)
427 .schema(dbn_schema)
428 .maybe_limit(params.limit.and_then(NonZeroU64::new))
429 .build();
430
431 let price_precision_arg = params.price_precision;
432
433 let mut client = (*self.inner).clone();
434 let mut decoder = client
435 .timeseries()
436 .get_range(&range_params)
437 .await
438 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
439
440 let metadata = decoder.metadata().clone();
441 let mut metadata_cache = MetadataCache::new(metadata);
442 let mut precision_cache = AHashMap::new();
443 let mut result: Vec<QuoteTick> = Vec::new();
444
445 let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
446 let instrument_id =
447 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
448 let price_precision = self.resolve_cached_price_precision(
449 &instrument_id,
450 price_precision_arg,
451 &mut precision_cache,
452 )?;
453
454 let (data, _) = decode_record(
455 &record,
456 instrument_id,
457 price_precision,
458 None,
459 false, true,
461 )?;
462
463 match data {
464 Some(Data::Quote(quote)) => result.push(quote),
465 None => {} _ => anyhow::bail!("Invalid data element not `QuoteTick`, was {data:?}"),
467 }
468 Ok(())
469 };
470
471 match dbn_schema {
472 dbn::Schema::Mbp1 => {
473 while let Some(msg) = decoder.decode_record::<dbn::Mbp1Msg>().await? {
474 process_record(dbn::RecordRef::from(msg))?;
475 }
476 }
477 dbn::Schema::Tbbo => {
478 while let Some(msg) = decoder.decode_record::<dbn::TbboMsg>().await? {
479 process_record(dbn::RecordRef::from(msg))?;
480 }
481 }
482 dbn::Schema::Cmbp1 => {
483 while let Some(msg) = decoder.decode_record::<dbn::Cmbp1Msg>().await? {
484 process_record(dbn::RecordRef::from(msg))?;
485 }
486 }
487 dbn::Schema::Tcbbo => {
488 while let Some(msg) = decoder.decode_record::<dbn::TcbboMsg>().await? {
489 process_record(dbn::RecordRef::from(msg))?;
490 }
491 }
492 dbn::Schema::Bbo1M => {
493 while let Some(msg) = decoder.decode_record::<dbn::Bbo1MMsg>().await? {
494 process_record(dbn::RecordRef::from(msg))?;
495 }
496 }
497 dbn::Schema::Bbo1S => {
498 while let Some(msg) = decoder.decode_record::<dbn::Bbo1SMsg>().await? {
499 process_record(dbn::RecordRef::from(msg))?;
500 }
501 }
502 dbn::Schema::Cbbo1S | dbn::Schema::Cbbo1M => {
503 while let Some(msg) = decoder.decode_record::<dbn::CbboMsg>().await? {
504 process_record(dbn::RecordRef::from(msg))?;
505 }
506 }
507 _ => anyhow::bail!("Invalid schema {dbn_schema}"),
508 }
509
510 Ok(result)
511 }
512
513 pub async fn get_range_order_book_depth(
519 &self,
520 params: RangeQueryParams,
521 depth: Option<usize>,
522 ) -> anyhow::Result<Vec<OrderBookDepth>> {
523 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
524 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
525 check_consistent_symbology(&symbols)?;
526
527 let first_symbol = params
528 .symbols
529 .first()
530 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
531 let stype_in = infer_symbology_type(first_symbol);
532 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
533 let time_range = get_date_time_range(params.start, end)?;
534
535 let _depth = depth.unwrap_or(10);
537 if _depth != 10 {
538 anyhow::bail!("Only depth=10 is currently supported for order book depths");
539 }
540
541 let range_params = GetRangeParams::builder()
542 .dataset(params.dataset)
543 .date_time_range(time_range)
544 .symbols(symbols)
545 .stype_in(stype_in)
546 .schema(dbn::Schema::Mbp10)
547 .maybe_limit(params.limit.and_then(NonZeroU64::new))
548 .build();
549
550 let price_precision_arg = params.price_precision;
551
552 let mut client = (*self.inner).clone();
553 let mut decoder = client
554 .timeseries()
555 .get_range(&range_params)
556 .await
557 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
558
559 let metadata = decoder.metadata().clone();
560 let mut metadata_cache = MetadataCache::new(metadata);
561 let mut precision_cache = AHashMap::new();
562 let mut result: Vec<OrderBookDepth> = Vec::new();
563
564 let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
565 let instrument_id =
566 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
567 let price_precision = self.resolve_cached_price_precision(
568 &instrument_id,
569 price_precision_arg,
570 &mut precision_cache,
571 )?;
572
573 if let Some(msg) = record.get::<dbn::Mbp10Msg>() {
574 let depth = decode_mbp10_msg(msg, instrument_id, price_precision, None)?;
575 result.push(depth);
576 }
577
578 Ok(())
579 };
580
581 while let Some(msg) = decoder.decode_record::<dbn::Mbp10Msg>().await? {
582 process_record(dbn::RecordRef::from(msg))?;
583 }
584
585 Ok(result)
586 }
587
588 pub async fn get_range_order_book_deltas(
594 &self,
595 params: RangeQueryParams,
596 ) -> anyhow::Result<Vec<OrderBookDelta>> {
597 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
598 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
599 check_consistent_symbology(&symbols)?;
600
601 let first_symbol = params
602 .symbols
603 .first()
604 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
605 let stype_in = infer_symbology_type(first_symbol);
606 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
607 let time_range = get_date_time_range(params.start, end)?;
608
609 let range_params = GetRangeParams::builder()
610 .dataset(params.dataset)
611 .date_time_range(time_range)
612 .symbols(symbols)
613 .stype_in(stype_in)
614 .schema(dbn::Schema::Mbo)
615 .maybe_limit(params.limit.and_then(NonZeroU64::new))
616 .build();
617
618 let price_precision_arg = params.price_precision;
619
620 let mut client = (*self.inner).clone();
621 let mut decoder = client
622 .timeseries()
623 .get_range(&range_params)
624 .await
625 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
626
627 let metadata = decoder.metadata().clone();
628 let mut metadata_cache = MetadataCache::new(metadata);
629 let mut precision_cache = AHashMap::new();
630 let mut result: Vec<OrderBookDelta> = Vec::new();
631 let mut delta_buffer = MboDeltaBuffer::default();
632
633 let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
634 let instrument_id =
635 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
636 let price_precision = self.resolve_cached_price_precision(
637 &instrument_id,
638 price_precision_arg,
639 &mut precision_cache,
640 )?;
641
642 if let Some(msg) = record.get::<dbn::MboMsg>() {
643 let (delta, _trade) =
644 decode_mbo_msg(msg, instrument_id, price_precision, None, false)?;
645 delta_buffer.push(msg, instrument_id, delta);
646 while let Some(delta) = delta_buffer.pop_ready() {
647 result.push(delta);
648 }
649 }
650
651 Ok(())
652 };
653
654 while let Some(msg) = decoder.decode_record::<dbn::MboMsg>().await? {
655 process_record(dbn::RecordRef::from(msg))?;
656 }
657
658 delta_buffer.finish();
659 while let Some(delta) = delta_buffer.pop_ready() {
660 result.push(delta);
661 }
662
663 Ok(result)
664 }
665
666 pub async fn get_range_trades(
672 &self,
673 params: RangeQueryParams,
674 schema: Option<String>,
675 ) -> anyhow::Result<Vec<TradeTick>> {
676 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
677 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
678 check_consistent_symbology(&symbols)?;
679
680 let first_symbol = params
681 .symbols
682 .first()
683 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
684 let stype_in = infer_symbology_type(first_symbol);
685 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
686 let time_range = get_date_time_range(params.start, end)?;
687 let schema = schema.unwrap_or_else(|| "trades".to_string());
688 let dbn_schema = dbn::Schema::from_str(&schema)?;
689
690 match dbn_schema {
691 dbn::Schema::Trades
692 | dbn::Schema::Tbbo
693 | dbn::Schema::Tcbbo
694 | dbn::Schema::Mbp1
695 | dbn::Schema::Cmbp1 => (),
696 _ => {
697 anyhow::bail!("Invalid schema. Must be one of: trades, tbbo, tcbbo, mbp-1, cmbp-1")
698 }
699 }
700
701 let range_params = GetRangeParams::builder()
702 .dataset(params.dataset)
703 .date_time_range(time_range)
704 .symbols(symbols)
705 .stype_in(stype_in)
706 .schema(dbn_schema)
707 .maybe_limit(params.limit.and_then(NonZeroU64::new))
708 .build();
709
710 let price_precision_arg = params.price_precision;
711
712 let mut client = (*self.inner).clone();
713 let mut decoder = client
714 .timeseries()
715 .get_range(&range_params)
716 .await
717 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
718
719 let metadata = decoder.metadata().clone();
720 let mut metadata_cache = MetadataCache::new(metadata);
721 let mut precision_cache = AHashMap::new();
722 let mut result: Vec<TradeTick> = Vec::new();
723
724 let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
725 let instrument_id =
726 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
727 let price_precision = self.resolve_cached_price_precision(
728 &instrument_id,
729 price_precision_arg,
730 &mut precision_cache,
731 )?;
732
733 let (data, data2) =
734 decode_record(&record, instrument_id, price_precision, None, true, true)?;
735
736 match (data, data2) {
737 (Some(Data::Trade(trade)), _) | (_, Some(Data::Trade(trade))) => result.push(trade),
738 (Some(_) | None, None) => {}
739 (None, Some(data)) => {
740 anyhow::bail!("Invalid data element not `TradeTick`, was {data:?}")
741 }
742 (Some(data), Some(_)) => {
743 anyhow::bail!("Invalid data element not `TradeTick`, was {data:?}")
744 }
745 }
746 Ok(())
747 };
748
749 match dbn_schema {
750 dbn::Schema::Trades => {
751 while let Some(msg) = decoder.decode_record::<dbn::TradeMsg>().await? {
752 process_record(dbn::RecordRef::from(msg))?;
753 }
754 }
755 dbn::Schema::Mbp1 => {
756 while let Some(msg) = decoder.decode_record::<dbn::Mbp1Msg>().await? {
757 process_record(dbn::RecordRef::from(msg))?;
758 }
759 }
760 dbn::Schema::Tbbo => {
761 while let Some(msg) = decoder.decode_record::<dbn::TbboMsg>().await? {
762 process_record(dbn::RecordRef::from(msg))?;
763 }
764 }
765 dbn::Schema::Cmbp1 => {
766 while let Some(msg) = decoder.decode_record::<dbn::Cmbp1Msg>().await? {
767 process_record(dbn::RecordRef::from(msg))?;
768 }
769 }
770 dbn::Schema::Tcbbo => {
771 while let Some(msg) = decoder.decode_record::<dbn::TcbboMsg>().await? {
772 process_record(dbn::RecordRef::from(msg))?;
773 }
774 }
775 _ => anyhow::bail!("Invalid schema {dbn_schema}"),
776 }
777
778 Ok(result)
779 }
780
781 pub async fn get_range_bars(
787 &self,
788 params: RangeQueryParams,
789 aggregation: BarAggregation,
790 timestamp_on_close: bool,
791 ) -> anyhow::Result<Vec<Bar>> {
792 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
793 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
794 check_consistent_symbology(&symbols)?;
795
796 let first_symbol = params
797 .symbols
798 .first()
799 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
800 let stype_in = infer_symbology_type(first_symbol);
801 let schema = ohlcv_schema_from_aggregation(aggregation)?;
802
803 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
804 let time_range = get_date_time_range(params.start, end)?;
805
806 let range_params = GetRangeParams::builder()
807 .dataset(params.dataset)
808 .date_time_range(time_range)
809 .symbols(symbols)
810 .stype_in(stype_in)
811 .schema(schema)
812 .maybe_limit(params.limit.and_then(NonZeroU64::new))
813 .build();
814
815 let price_precision_arg = params.price_precision;
816
817 let mut client = (*self.inner).clone();
818 let mut decoder = client
819 .timeseries()
820 .get_range(&range_params)
821 .await
822 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
823
824 let metadata = decoder.metadata().clone();
825 let mut metadata_cache = MetadataCache::new(metadata);
826 let mut precision_cache = AHashMap::new();
827 let mut result: Vec<Bar> = Vec::new();
828
829 while let Some(msg) = decoder.decode_record::<dbn::OhlcvMsg>().await? {
830 let record = dbn::RecordRef::from(msg);
831 let instrument_id =
832 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
833 let price_precision = self.resolve_cached_price_precision(
834 &instrument_id,
835 price_precision_arg,
836 &mut precision_cache,
837 )?;
838
839 let (data, _) = decode_record(
840 &record,
841 instrument_id,
842 price_precision,
843 None,
844 false, timestamp_on_close,
846 )?;
847
848 match data {
849 Some(Data::Bar(bar)) => {
850 result.push(bar);
851 }
852 _ => anyhow::bail!("Invalid data element not `Bar`, was {data:?}"),
853 }
854 }
855
856 Ok(result)
857 }
858
859 pub async fn get_range_imbalance(
865 &self,
866 params: RangeQueryParams,
867 ) -> anyhow::Result<Vec<DatabentoImbalance>> {
868 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
869 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
870 check_consistent_symbology(&symbols)?;
871
872 let first_symbol = params
873 .symbols
874 .first()
875 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
876 let stype_in = infer_symbology_type(first_symbol);
877 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
878 let time_range = get_date_time_range(params.start, end)?;
879
880 let range_params = GetRangeParams::builder()
881 .dataset(params.dataset)
882 .date_time_range(time_range)
883 .symbols(symbols)
884 .stype_in(stype_in)
885 .schema(dbn::Schema::Imbalance)
886 .maybe_limit(params.limit.and_then(NonZeroU64::new))
887 .build();
888
889 let price_precision_arg = params.price_precision;
890
891 let mut client = (*self.inner).clone();
892 let mut decoder = client
893 .timeseries()
894 .get_range(&range_params)
895 .await
896 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
897
898 let metadata = decoder.metadata().clone();
899 let mut metadata_cache = MetadataCache::new(metadata);
900 let mut precision_cache = AHashMap::new();
901 let mut result: Vec<DatabentoImbalance> = Vec::new();
902
903 while let Some(msg) = decoder.decode_record::<dbn::ImbalanceMsg>().await? {
904 let record = dbn::RecordRef::from(msg);
905 let instrument_id =
906 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
907 let price_precision = self.resolve_cached_price_precision(
908 &instrument_id,
909 price_precision_arg,
910 &mut precision_cache,
911 )?;
912
913 let imbalance = decode_imbalance_msg(msg, instrument_id, price_precision, None)?;
914 result.push(imbalance);
915 }
916
917 Ok(result)
918 }
919
920 pub async fn get_range_statistics(
926 &self,
927 params: RangeQueryParams,
928 ) -> anyhow::Result<Vec<DatabentoStatistics>> {
929 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
930 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
931 check_consistent_symbology(&symbols)?;
932
933 let first_symbol = params
934 .symbols
935 .first()
936 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
937 let stype_in = infer_symbology_type(first_symbol);
938 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
939 let time_range = get_date_time_range(params.start, end)?;
940
941 let range_params = GetRangeParams::builder()
942 .dataset(params.dataset)
943 .date_time_range(time_range)
944 .symbols(symbols)
945 .stype_in(stype_in)
946 .schema(dbn::Schema::Statistics)
947 .maybe_limit(params.limit.and_then(NonZeroU64::new))
948 .build();
949
950 let price_precision_arg = params.price_precision;
951
952 let mut client = (*self.inner).clone();
953 let mut decoder = client
954 .timeseries()
955 .get_range(&range_params)
956 .await
957 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
958
959 let metadata = decoder.metadata().clone();
960 let mut metadata_cache = MetadataCache::new(metadata);
961 let mut precision_cache = AHashMap::new();
962 let mut result: Vec<DatabentoStatistics> = Vec::new();
963
964 while let Some(msg) = decoder.decode_record::<dbn::StatMsg>().await? {
965 if !is_supported_stat_type(msg.stat_type) {
967 log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
968 continue;
969 }
970
971 let record = dbn::RecordRef::from(msg);
972 let instrument_id =
973 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
974 let price_precision = self.resolve_cached_price_precision(
975 &instrument_id,
976 price_precision_arg,
977 &mut precision_cache,
978 )?;
979
980 if let Some(statistics) =
981 decode_statistics_msg(msg, instrument_id, price_precision, None)?
982 {
983 result.push(statistics);
984 }
985 }
986
987 Ok(result)
988 }
989
990 pub async fn get_range_status(
996 &self,
997 params: RangeQueryParams,
998 ) -> anyhow::Result<Vec<InstrumentStatus>> {
999 let (raw_symbols, requested_symbols) = self.prepare_symbols(¶ms);
1000 let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
1001 check_consistent_symbology(&symbols)?;
1002
1003 let first_symbol = params
1004 .symbols
1005 .first()
1006 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
1007 let stype_in = infer_symbology_type(first_symbol);
1008 let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
1009 let time_range = get_date_time_range(params.start, end)?;
1010
1011 let range_params = GetRangeParams::builder()
1012 .dataset(params.dataset)
1013 .date_time_range(time_range)
1014 .symbols(symbols)
1015 .stype_in(stype_in)
1016 .schema(dbn::Schema::Status)
1017 .maybe_limit(params.limit.and_then(NonZeroU64::new))
1018 .build();
1019
1020 let mut client = (*self.inner).clone();
1021 let mut decoder = client
1022 .timeseries()
1023 .get_range(&range_params)
1024 .await
1025 .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
1026
1027 let metadata = decoder.metadata().clone();
1028 let mut metadata_cache = MetadataCache::new(metadata);
1029 let mut result: Vec<InstrumentStatus> = Vec::new();
1030
1031 while let Some(msg) = decoder.decode_record::<dbn::StatusMsg>().await? {
1032 let record = dbn::RecordRef::from(msg);
1033 let instrument_id =
1034 self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
1035
1036 let status = decode_status_msg(msg, instrument_id, None)?;
1037 result.push(status);
1038 }
1039
1040 Ok(result)
1041 }
1042
1043 fn instrument_id(
1044 &self,
1045 record: &dbn::RecordRef,
1046 metadata: &mut MetadataCache,
1047 requested_symbols: &AHashSet<Symbol>,
1048 ) -> anyhow::Result<InstrumentId> {
1049 decode_nautilus_instrument_id(
1050 record,
1051 metadata,
1052 &self.publisher_venue_map,
1053 &self.symbol_venue_map.load(),
1054 self.futures_year_digits,
1055 &self.futures_symbols.load(),
1056 Some(requested_symbols),
1057 )
1058 }
1059
1060 fn prepare_symbols(&self, params: &RangeQueryParams) -> (Vec<String>, AHashSet<Symbol>) {
1061 let mut requested_symbols = AHashSet::new();
1062
1063 let raw_symbols = params
1064 .symbols
1065 .iter()
1066 .map(|symbol| {
1067 let raw = raw_symbol(symbol, ¶ms.dataset, self.futures_year_digits);
1068 if raw != *symbol {
1069 requested_symbols.insert(Symbol::from(symbol.as_str()));
1070 }
1071
1072 raw
1073 })
1074 .collect();
1075
1076 (raw_symbols, requested_symbols)
1077 }
1078
1079 pub fn prepare_symbols_from_instrument_ids(
1081 &self,
1082 instrument_ids: &[InstrumentId],
1083 ) -> Vec<String> {
1084 self.symbol_venue_map.rcu(|m| {
1085 for id in instrument_ids {
1086 m.entry(id.symbol).or_insert(id.venue);
1087 }
1088 });
1089
1090 instrument_ids
1091 .iter()
1092 .map(|id| id.symbol.to_string())
1093 .collect()
1094 }
1095}
1096
1097#[cfg(test)]
1098mod tests {
1099 use std::ffi::c_char;
1100
1101 use databento::dbn::{
1102 MappingInterval, SymbolMapping, decode::DecodeRecord, encode::EncodeRecord,
1103 };
1104 use nautilus_core::time::get_atomic_clock_realtime;
1105 use nautilus_model::types::{Price, Quantity};
1106 use rstest::{fixture, rstest};
1107 use time::macros::date;
1108 use tokio::{
1109 io::{AsyncReadExt, AsyncWriteExt},
1110 net::TcpListener,
1111 };
1112
1113 use super::*;
1114
1115 fn test_api_key() -> String {
1116 "test-000000000000000000000000000".to_string()
1117 }
1118
1119 fn publishers_path() -> PathBuf {
1120 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json")
1121 }
1122
1123 fn mbo_record(action: c_char, flags: u8, sequence: u32) -> dbn::MboMsg {
1124 let ts_event = 1_609_160_400_000_000_000;
1125 dbn::MboMsg {
1126 hd: dbn::RecordHeader::new::<dbn::MboMsg>(dbn::rtype::MBO, 1, 1, ts_event),
1127 order_id: u64::from(sequence),
1128 price: 4_800_250_000_000,
1129 size: 2,
1130 flags: dbn::FlagSet::new(flags),
1131 channel_id: 1,
1132 action,
1133 side: 'A' as c_char,
1134 ts_recv: ts_event,
1135 ts_in_delta: 0,
1136 sequence,
1137 }
1138 }
1139
1140 fn encode_mbo_response(records: &[dbn::MboMsg]) -> Vec<u8> {
1141 let metadata = dbn::Metadata::builder()
1142 .dataset("GLBX.MDP3")
1143 .schema(Some(dbn::Schema::Mbo))
1144 .start(1_609_160_400_000_000_000)
1145 .stype_in(Some(dbn::SType::RawSymbol))
1146 .stype_out(dbn::SType::InstrumentId)
1147 .symbols(vec!["ESM4".to_string()])
1148 .mappings(vec![SymbolMapping {
1149 raw_symbol: "ESM4".to_string(),
1150 intervals: vec![MappingInterval {
1151 start_date: date!(2020 - 12 - 28),
1152 end_date: date!(2020 - 12 - 29),
1153 symbol: "1".to_string(),
1154 }],
1155 }])
1156 .build();
1157 let mut body = Vec::new();
1158 {
1159 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1160 encoder.encode_records(records).unwrap();
1161 encoder.flush().unwrap();
1162 }
1163 body
1164 }
1165
1166 async fn serve_response(listener: TcpListener, body: Vec<u8>) {
1167 let (mut stream, _) = listener.accept().await.unwrap();
1168 let mut request = Vec::new();
1169 let mut buffer = [0_u8; 1024];
1170 loop {
1171 let read = stream.read(&mut buffer).await.unwrap();
1172 if read == 0 {
1173 break;
1174 }
1175 request.extend_from_slice(&buffer[..read]);
1176 if request.windows(4).any(|window| window == b"\r\n\r\n") {
1177 break;
1178 }
1179 }
1180
1181 let headers = format!(
1182 "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
1183 body.len(),
1184 );
1185 stream.write_all(headers.as_bytes()).await.unwrap();
1186 stream.write_all(&body).await.unwrap();
1187 stream.shutdown().await.unwrap();
1188 }
1189
1190 #[fixture]
1191 fn historical_client() -> DatabentoHistoricalClient {
1192 DatabentoHistoricalClient::new(
1193 Credential::new(test_api_key()),
1194 publishers_path(),
1195 get_atomic_clock_realtime(),
1196 false,
1197 )
1198 .unwrap()
1199 }
1200
1201 #[rstest]
1202 fn test_new_with_base_url_rejects_invalid_url() {
1203 let err = DatabentoHistoricalClient::new_with_base_url(
1204 Credential::new(test_api_key()),
1205 publishers_path(),
1206 get_atomic_clock_realtime(),
1207 false,
1208 "://invalid",
1209 )
1210 .expect_err("expected invalid base URL to fail");
1211 let err_msg = format!("{err}");
1212
1213 assert!(
1214 err_msg.contains("Failed to parse Databento Historical API base URL"),
1215 "unexpected error message: {err_msg}",
1216 );
1217 }
1218
1219 #[rstest]
1220 #[tokio::test]
1221 async fn test_get_range_order_book_deltas_preserves_boundaries_and_drains_end() {
1222 let last = dbn::flags::LAST;
1223 let body = encode_mbo_response(&[
1224 mbo_record('C' as c_char, 0, 1),
1225 mbo_record('N' as c_char, last, 2),
1226 mbo_record('C' as c_char, 0, 3),
1227 ]);
1228 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1229 let base_url = format!("http://{}/", listener.local_addr().unwrap());
1230 let server = tokio::spawn(serve_response(listener, body));
1231 let client = DatabentoHistoricalClient::new_with_base_url(
1232 Credential::new(test_api_key()),
1233 publishers_path(),
1234 get_atomic_clock_realtime(),
1235 false,
1236 &base_url,
1237 )
1238 .unwrap();
1239
1240 let deltas = client
1241 .get_range_order_book_deltas(RangeQueryParams {
1242 dataset: "GLBX.MDP3".to_string(),
1243 symbols: vec!["ESM4".to_string()],
1244 start: 1_609_160_000_000_000_000.into(),
1245 end: Some(1_609_161_000_000_000_000.into()),
1246 limit: None,
1247 price_precision: Some(2),
1248 })
1249 .await
1250 .unwrap();
1251 server.await.unwrap();
1252
1253 assert_eq!(deltas.len(), 2);
1254 assert_eq!(deltas[0].instrument_id, InstrumentId::from("ESM24.GLBX"));
1255 assert_eq!(deltas[0].order.order_id, 1);
1256 assert_eq!(deltas[0].flags, last);
1257 assert_eq!(deltas[0].sequence, 1);
1258 assert_eq!(deltas[1].instrument_id, InstrumentId::from("ESM24.GLBX"));
1259 assert_eq!(deltas[1].order.order_id, 3);
1260 assert_eq!(deltas[1].flags, 0);
1261 assert_eq!(deltas[1].sequence, 3);
1262 }
1263
1264 #[rstest]
1265 fn test_definition_precision_batch_merges_and_last_symbol_wins(
1266 historical_client: DatabentoHistoricalClient,
1267 ) {
1268 let mut first = nautilus_model::instruments::stubs::crypto_perpetual_ethusdt();
1269 first.id = "FIRST.GLBX".into();
1270 first.price_precision = 3;
1271 let mut second = first.clone();
1272 second.id = "SECOND.GLBX".into();
1273 second.price_precision = 5;
1274 let mut latest = first.clone();
1275 latest.id = "FIRST.XNAS".into();
1276 latest.price_precision = 7;
1277 historical_client.set_price_precision("RETAINED".into(), 9);
1278 historical_client.set_price_precision("FIRST".into(), 1);
1279
1280 historical_client.cache_price_precisions(&[
1281 InstrumentAny::CryptoPerpetual(first),
1282 InstrumentAny::CryptoPerpetual(second),
1283 InstrumentAny::CryptoPerpetual(latest),
1284 ]);
1285
1286 assert_eq!(historical_client.price_precisions.len(), 3);
1287
1288 for (symbol, precision) in [("RETAINED", 9), ("FIRST", 7), ("SECOND", 5)] {
1289 assert_eq!(
1290 historical_client
1291 .resolve_price_precision(&InstrumentId::new(symbol.into(), "GLBX".into()), None)
1292 .unwrap(),
1293 precision
1294 );
1295 }
1296 }
1297
1298 #[rstest]
1299 fn test_set_price_precision_inserts_into_cache(historical_client: DatabentoHistoricalClient) {
1300 let symbol = Symbol::from("ESM4");
1301
1302 assert_eq!(historical_client.price_precision(symbol), None);
1303
1304 historical_client.set_price_precision(symbol, 2);
1305
1306 assert_eq!(historical_client.price_precision(symbol), Some(2));
1307 }
1308
1309 #[rstest]
1310 fn test_resolve_price_precision_explicit_arg(historical_client: DatabentoHistoricalClient) {
1311 let instrument_id = InstrumentId::from("ESM4.GLBX");
1312 historical_client.set_price_precision(Symbol::from("ESM4"), 9);
1314
1315 let precision = historical_client
1316 .resolve_price_precision(&instrument_id, Some(2))
1317 .unwrap();
1318 assert_eq!(precision, 2);
1319 }
1320
1321 #[rstest]
1322 fn test_resolve_price_precision_cache_hit(historical_client: DatabentoHistoricalClient) {
1323 let instrument_id = InstrumentId::from("ESM4.GLBX");
1324 historical_client.set_price_precision(Symbol::from("ESM4"), 2);
1325
1326 let precision = historical_client
1327 .resolve_price_precision(&instrument_id, None)
1328 .unwrap();
1329 assert_eq!(precision, 2);
1330 }
1331
1332 #[rstest]
1333 fn test_resolve_price_precision_cache_miss_errors(
1334 historical_client: DatabentoHistoricalClient,
1335 ) {
1336 let instrument_id = InstrumentId::from("ESM4.GLBX");
1337
1338 let err = historical_client
1339 .resolve_price_precision(&instrument_id, None)
1340 .expect_err("expected cache-miss error");
1341 let err_msg = format!("{err}");
1342 assert!(
1343 err_msg.contains("Could not resolve `price_precision`"),
1344 "unexpected error message: {err_msg}",
1345 );
1346 assert!(
1347 err_msg.contains("ESM4.GLBX"),
1348 "error should name the instrument: {err_msg}",
1349 );
1350 }
1351
1352 #[rstest]
1353 #[case(1_000)]
1354 #[case(10_000)]
1355 #[tokio::test]
1356 async fn test_historical_definition_batch_preserves_all_symbols(#[case] count: u32) {
1357 let path = publishers_path()
1358 .parent()
1359 .unwrap()
1360 .join("test_data/test_data.definition.futures_contract.dbn.zst");
1361 let mut decoder = dbn::decode::dbn::Decoder::from_zstd_file(path).unwrap();
1362 let mut definition = decoder
1363 .decode_record::<dbn::InstrumentDefMsg>()
1364 .unwrap()
1365 .unwrap()
1366 .clone();
1367 definition.ts_recv = "2026-01-01T00:00:00Z"
1368 .parse::<jiff::Timestamp>()
1369 .unwrap()
1370 .as_nanosecond() as u64;
1371 definition.maturity_year = 2026;
1372 let mut definitions = Vec::new();
1373 let mut mappings = Vec::new();
1374
1375 for id in 1..=count {
1376 let raw = format!("T{id}U6");
1377 definition.hd.instrument_id = id;
1378 definition.raw_symbol.fill(0);
1379
1380 for (target, byte) in definition.raw_symbol.iter_mut().zip(raw.bytes()) {
1381 *target = byte as c_char;
1382 }
1383
1384 definitions.push(definition.clone());
1385 mappings.push(SymbolMapping {
1386 raw_symbol: raw,
1387 intervals: vec![MappingInterval {
1388 start_date: date!(2026 - 01 - 01),
1389 end_date: date!(2026 - 01 - 02),
1390 symbol: id.to_string(),
1391 }],
1392 });
1393 }
1394
1395 let mut replacement = definitions[0].clone();
1396 replacement.maturity_year = 2036;
1397 definitions.push(replacement);
1398 let metadata = dbn::Metadata::builder()
1399 .dataset("GLBX.MDP3")
1400 .schema(Some(dbn::Schema::Definition))
1401 .start(definition.ts_recv)
1402 .stype_in(Some(dbn::SType::Parent))
1403 .stype_out(dbn::SType::InstrumentId)
1404 .mappings(mappings)
1405 .build();
1406 let mut body = Vec::new();
1407 {
1408 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1409 encoder.encode_records(&definitions).unwrap();
1410 encoder.flush().unwrap();
1411 }
1412
1413 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1414 let base_url = format!("http://{}/", listener.local_addr().unwrap());
1415
1416 let server = tokio::spawn(serve_response(listener, body));
1417 let client = DatabentoHistoricalClient::new_with_base_url(
1418 Credential::new(test_api_key()),
1419 publishers_path(),
1420 get_atomic_clock_realtime(),
1421 false,
1422 &base_url,
1423 )
1424 .unwrap();
1425 let publisher = definition.hd.publisher_id;
1426 let old_key = (publisher, 1, date!(2025 - 12 - 31));
1427 client
1428 .futures_symbols
1429 .insert(old_key, Symbol::from("T1U16"));
1430 let started = std::time::Instant::now();
1431
1432 let instruments = client
1433 .get_range_instruments(RangeQueryParams {
1434 dataset: "GLBX.MDP3".to_string(),
1435 symbols: vec!["T.FUT".to_string()],
1436 start: definition.ts_recv.into(),
1437 end: Some((definition.ts_recv + 1).into()),
1438 limit: None,
1439 price_precision: None,
1440 })
1441 .await
1442 .unwrap();
1443
1444 let elapsed = started.elapsed();
1445 server.await.unwrap();
1446 let symbols = client.futures_symbols.load();
1447 let mut expected = AHashMap::new();
1448 expected.insert(old_key, Symbol::from("T1U16"));
1449
1450 for id in 1..=count {
1451 let symbol = Symbol::from(format!("T{id}U26").as_str());
1452 expected.insert((publisher, id, date!(2026 - 01 - 01)), symbol);
1453 assert_eq!(
1454 instruments[(id - 1) as usize].id(),
1455 InstrumentId::new(symbol, Venue::GLBX())
1456 );
1457 assert_eq!(
1458 instruments[(id - 1) as usize].raw_symbol(),
1459 Symbol::from(format!("T{id}U6").as_str())
1460 );
1461 }
1462
1463 expected.insert((publisher, 1, date!(2026 - 01 - 01)), Symbol::from("T1U36"));
1464 assert_eq!(instruments.len(), count as usize + 1);
1465 assert_eq!(
1466 instruments.last().unwrap().id(),
1467 InstrumentId::from("T1U36.GLBX")
1468 );
1469 assert_eq!(symbols.as_ref(), &expected);
1470 eprintln!("Decoded {count} definitions plus replacement in {elapsed:?}");
1471 }
1472
1473 #[tokio::test]
1474 async fn test_historical_records_use_definition_year_for_the_same_date() {
1475 let path = publishers_path()
1476 .parent()
1477 .unwrap()
1478 .join("test_data/test_data.definition.futures_contract.dbn.zst");
1479 let mut decoder = dbn::decode::dbn::Decoder::from_zstd_file(path).unwrap();
1480 let mut definition = decoder
1481 .decode_record::<dbn::InstrumentDefMsg>()
1482 .unwrap()
1483 .unwrap()
1484 .clone();
1485 definition.ts_recv = "2028-01-01T00:00:00Z"
1486 .parse::<jiff::Timestamp>()
1487 .unwrap()
1488 .as_nanosecond() as u64;
1489 let metadata = dbn::Metadata::builder()
1490 .dataset("GLBX.MDP3")
1491 .schema(Some(dbn::Schema::Definition))
1492 .start(definition.ts_recv)
1493 .stype_in(Some(dbn::SType::RawSymbol))
1494 .stype_out(dbn::SType::InstrumentId)
1495 .mappings(vec![SymbolMapping {
1496 raw_symbol: "ESU6".to_string(),
1497 intervals: vec![MappingInterval {
1498 start_date: date!(2028 - 01 - 01),
1499 end_date: date!(2028 - 01 - 02),
1500 symbol: definition.hd.instrument_id.to_string(),
1501 }],
1502 }])
1503 .build();
1504 let mut body = Vec::new();
1505 {
1506 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1507 encoder.encode_record(&definition).unwrap();
1508 encoder.flush().unwrap();
1509 }
1510
1511 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1512 let base_url = format!("http://{}/", listener.local_addr().unwrap());
1513
1514 let server = tokio::spawn(serve_response(listener, body));
1515 let client = DatabentoHistoricalClient::new_with_base_url(
1516 Credential::new(test_api_key()),
1517 publishers_path(),
1518 get_atomic_clock_realtime(),
1519 false,
1520 &base_url,
1521 )
1522 .unwrap();
1523
1524 let instruments = client
1525 .get_range_instruments(RangeQueryParams {
1526 dataset: "GLBX.MDP3".to_string(),
1527 symbols: vec!["ESU26".to_string()],
1528 start: definition.ts_recv.into(),
1529 end: Some((definition.ts_recv + 1).into()),
1530 limit: Some(1),
1531 price_precision: None,
1532 })
1533 .await
1534 .unwrap();
1535
1536 server.await.unwrap();
1537 let mut record = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1538 record.hd.publisher_id = definition.hd.publisher_id;
1539 record.hd.instrument_id = definition.hd.instrument_id;
1540 record.ts_recv = definition.ts_recv;
1541 let id = client
1542 .instrument_id(
1543 &dbn::RecordRef::from(&record),
1544 &mut MetadataCache::new(metadata),
1545 &AHashSet::from_iter([Symbol::from("ESU36")]),
1546 )
1547 .unwrap();
1548 assert_eq!(instruments.len(), 1);
1549 assert_eq!(instruments[0].id(), InstrumentId::from("ESU26.GLBX"));
1550 assert_eq!(instruments[0].raw_symbol(), Symbol::from("ESU6"));
1551 assert_eq!(id, InstrumentId::from("ESU26.GLBX"));
1552 }
1553
1554 #[rstest]
1555 #[case(None, "SR3Z26.XCME", true)]
1556 #[case(Some("SR3Z36.XNYM"), "SR3Z26.XCME", true)]
1557 #[case(Some("SR3Z26.XNYM"), "SR3Z36.XCME", true)]
1558 #[case(Some("SR3Z36.XNYM"), "SR3Z26.GLBX", false)]
1559 #[tokio::test]
1560 async fn test_requested_post_year_quotes_use_cached_precision_without_definitions(
1561 #[case] previous: Option<&str>,
1562 #[case] requested: &str,
1563 #[case] register_venue: bool,
1564 ) {
1565 let ts = "2027-02-01T00:00:00Z"
1566 .parse::<jiff::Timestamp>()
1567 .unwrap()
1568 .as_nanosecond() as u64;
1569 let mut msg = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1570 msg.hd.publisher_id = 1;
1571 msg.hd.instrument_id = 42;
1572 msg.hd.ts_event = ts - 1;
1573 msg.ts_recv = ts;
1574 msg.levels[0].bid_px = 95_125_000_000;
1575 msg.levels[0].ask_px = 95_250_000_000;
1576 msg.levels[0].bid_sz = 7;
1577 msg.levels[0].ask_sz = 11;
1578 let metadata = dbn::Metadata::builder()
1579 .dataset("GLBX.MDP3")
1580 .schema(Some(dbn::Schema::Bbo1S))
1581 .start(ts)
1582 .stype_in(Some(dbn::SType::RawSymbol))
1583 .stype_out(dbn::SType::InstrumentId)
1584 .mappings(vec![SymbolMapping {
1585 raw_symbol: "SR3Z6".to_string(),
1586 intervals: vec![MappingInterval {
1587 start_date: date!(2027 - 02 - 01),
1588 end_date: date!(2027 - 02 - 02),
1589 symbol: "42".to_string(),
1590 }],
1591 }])
1592 .build();
1593 let mut body = Vec::new();
1594 {
1595 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1596 encoder.encode_record(&msg).unwrap();
1597 encoder.flush().unwrap();
1598 }
1599
1600 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1601 let base_url = format!("http://{}/", listener.local_addr().unwrap());
1602
1603 let server = tokio::spawn(serve_response(listener, body));
1604 let client = DatabentoHistoricalClient::new_with_base_url(
1605 Credential::new(test_api_key()),
1606 publishers_path(),
1607 get_atomic_clock_realtime(),
1608 false,
1609 &base_url,
1610 )
1611 .unwrap();
1612
1613 if let Some(previous) = previous {
1614 let previous = InstrumentId::from(previous);
1615 client.prepare_symbols_from_instrument_ids(&[previous]);
1616 client.set_price_precision(previous.symbol, 3);
1617 }
1618
1619 let id = InstrumentId::from(requested);
1620 let symbols = if register_venue {
1621 client.prepare_symbols_from_instrument_ids(&[id])
1622 } else {
1623 vec![id.symbol.to_string()]
1624 };
1625
1626 client.set_price_precision(id.symbol, 3);
1627
1628 let quotes = client
1629 .get_range_quotes(
1630 RangeQueryParams {
1631 dataset: "GLBX.MDP3".to_string(),
1632 symbols,
1633 start: ts.into(),
1634 end: Some((ts + 1).into()),
1635 limit: Some(1),
1636 price_precision: None,
1637 },
1638 Some("bbo-1s".to_string()),
1639 )
1640 .await
1641 .unwrap();
1642
1643 server.await.unwrap();
1644
1645 assert_eq!(
1646 quotes,
1647 vec![QuoteTick::new(
1648 id,
1649 Price::from("95.125"),
1650 Price::from("95.250"),
1651 Quantity::from(7),
1652 Quantity::from(11),
1653 ts.into(),
1654 ts.into(),
1655 )]
1656 );
1657 }
1658}