1use std::{
17 env,
18 path::{Path, PathBuf},
19};
20
21use ahash::AHashMap;
22use anyhow::Context;
23use databento::dbn::{self, InstrumentDefMsg};
24use dbn::{
25 Publisher,
26 decode::{DbnMetadata, DecodeStream, dbn::Decoder},
27};
28use fallible_streaming_iterator::FallibleStreamingIterator;
29use indexmap::IndexMap;
30use nautilus_model::{
31 data::{Bar, Data, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
32 identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
33 instruments::{Instrument, InstrumentAny},
34};
35
36use super::{
37 decode::{
38 MboDeltaBuffer, decode_imbalance_msg, decode_mbo_msg, decode_record, decode_statistics_msg,
39 decode_status_msg, is_supported_stat_type,
40 },
41 symbology::{decode_nautilus_instrument_id, nautilus_symbol, resolve_futures_instrument_id},
42 types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, Dataset, PublisherId},
43};
44use crate::{
45 common::{build_publisher_venue_map, load_publishers},
46 decode::{DatabentoDecodeConfig, decode_instrument_def_msg},
47 symbology::MetadataCache,
48};
49
50#[cfg_attr(
78 feature = "python",
79 pyo3::pyclass(module = "nautilus_trader.adapters.databento")
80)]
81#[cfg_attr(
82 feature = "python",
83 pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
84)]
85#[derive(Debug)]
86pub struct DatabentoDataLoader {
87 futures_year_digits: FuturesYearDigits,
88 share_class_space: bool,
89 publishers_map: IndexMap<PublisherId, DatabentoPublisher>,
90 venue_dataset_map: IndexMap<Venue, Dataset>,
91 publisher_venue_map: IndexMap<PublisherId, Venue>,
92 symbol_venue_map: AHashMap<Symbol, Venue>,
93 price_precisions: AHashMap<Symbol, u8>,
94 futures_symbols: AHashMap<(PublisherId, u32, time::Date), Symbol>,
95}
96
97impl DatabentoDataLoader {
98 pub fn new(publishers_filepath: Option<PathBuf>) -> anyhow::Result<Self> {
104 let mut loader = Self {
105 futures_year_digits: FuturesYearDigits::default(),
106 share_class_space: true,
107 publishers_map: IndexMap::new(),
108 venue_dataset_map: IndexMap::new(),
109 publisher_venue_map: IndexMap::new(),
110 symbol_venue_map: AHashMap::new(),
111 price_precisions: AHashMap::new(),
112 futures_symbols: AHashMap::new(),
113 };
114
115 let publishers_filepath = if let Some(p) = publishers_filepath {
117 p
118 } else {
119 let mut exe_path = env::current_exe()?;
121 exe_path.pop();
122 exe_path.push("publishers.json");
123 exe_path
124 };
125
126 loader
127 .load_publishers(publishers_filepath)
128 .context("error loading publishers.json")?;
129
130 Ok(loader)
131 }
132
133 #[must_use]
135 pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
136 self.futures_year_digits = futures_year_digits;
137 self
138 }
139
140 #[must_use]
142 pub fn with_share_class_space(mut self, share_class_space: bool) -> Self {
143 self.share_class_space = share_class_space;
144 self
145 }
146
147 pub fn load_publishers(&mut self, filepath: PathBuf) -> anyhow::Result<()> {
153 let publishers = load_publishers(filepath)?;
154
155 self.publishers_map = publishers
156 .iter()
157 .cloned()
158 .map(|p| (p.publisher_id, p))
159 .collect();
160
161 let mut venue_dataset_map = IndexMap::new();
162
163 for publisher in &publishers {
165 let venue = Venue::from(publisher.venue.as_str());
166 let dataset = Dataset::from(publisher.dataset.as_str());
167 venue_dataset_map.entry(venue).or_insert(dataset);
168 }
169
170 self.venue_dataset_map = venue_dataset_map;
171 apply_default_venue_dataset_mappings(&mut self.venue_dataset_map);
172
173 self.publisher_venue_map = build_publisher_venue_map(&publishers);
174
175 Ok(())
176 }
177
178 #[must_use]
180 pub const fn get_publishers(&self) -> &IndexMap<u16, DatabentoPublisher> {
181 &self.publishers_map
182 }
183
184 pub fn set_dataset_for_venue(&mut self, dataset: Dataset, venue: Venue) {
186 _ = self.venue_dataset_map.insert(venue, dataset);
187 }
188
189 #[must_use]
191 pub fn get_dataset_for_venue(&self, venue: &Venue) -> Option<&Dataset> {
192 self.venue_dataset_map.get(venue)
193 }
194
195 #[must_use]
197 pub fn get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option<&Venue> {
198 self.publisher_venue_map.get(&publisher_id)
199 }
200
201 pub fn set_price_precision(&mut self, symbol: Symbol, price_precision: u8) {
207 self.price_precisions.insert(symbol, price_precision);
208 }
209
210 #[must_use]
212 pub const fn get_price_precisions(&self) -> &AHashMap<Symbol, u8> {
213 &self.price_precisions
214 }
215
216 fn resolve_price_precision(
226 &self,
227 instrument_id: &InstrumentId,
228 price_precision: Option<u8>,
229 ) -> anyhow::Result<u8> {
230 if let Some(precision) = price_precision {
231 return Ok(precision);
232 }
233
234 self.price_precisions
235 .get(&instrument_id.symbol)
236 .copied()
237 .ok_or_else(|| {
238 anyhow::anyhow!(
239 "Could not resolve `price_precision` for {instrument_id}: \
240 pass `price_precision` explicitly, call `set_price_precision`, \
241 or load the instrument definitions first via `load_instruments`"
242 )
243 })
244 }
245
246 pub fn schema_from_file(&self, filepath: &Path) -> anyhow::Result<Option<String>> {
252 let decoder = Decoder::from_zstd_file(filepath)?;
253 let metadata = decoder.metadata();
254 Ok(metadata.schema.map(|schema| schema.to_string()))
255 }
256
257 pub fn read_definition_records<'a>(
263 &'a mut self,
264 filepath: &Path,
265 use_exchange_as_venue: bool,
266 decode_config: Option<&'a DatabentoDecodeConfig>,
267 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<InstrumentAny>> + 'a> {
268 let decoder = Decoder::from_zstd_file(filepath)?;
269 let mut dbn_stream = decoder.decode_stream::<InstrumentDefMsg>();
270
271 Ok(std::iter::from_fn(move || {
274 loop {
275 let advance = dbn_stream
276 .advance()
277 .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"));
278 if let Err(e) = advance {
279 return Some(Err(e));
280 }
281
282 let rec = dbn_stream.get()?;
283
284 let result: anyhow::Result<Option<InstrumentAny>> = (|| {
285 let record = dbn::RecordRef::from(rec);
286 let msg = record
287 .get::<InstrumentDefMsg>()
288 .ok_or_else(|| anyhow::anyhow!("Failed to decode InstrumentDefMsg"))?;
289
290 let raw_symbol = rec
291 .raw_symbol()
292 .map_err(|e| anyhow::anyhow!("Error decoding `raw_symbol`: {e}"))?;
293 let symbol = nautilus_symbol(raw_symbol, self.share_class_space);
294
295 let publisher = rec
296 .hd
297 .publisher()
298 .map_err(|e| anyhow::anyhow!("Invalid `publisher` for record: {e}"))?;
299 let venue = match publisher {
300 Publisher::GlbxMdp3Glbx if use_exchange_as_venue => {
301 let exchange = rec.exchange().map_err(|e| {
302 anyhow::anyhow!("Missing `exchange` for record: {e}")
303 })?;
304 let venue = Venue::from_code(exchange).map_err(|e| {
305 anyhow::anyhow!("Venue not found for exchange {exchange}: {e}")
306 })?;
307 self.symbol_venue_map.insert(symbol, venue);
308 venue
309 }
310 _ => *self
311 .publisher_venue_map
312 .get(&msg.hd.publisher_id)
313 .ok_or_else(|| {
314 anyhow::anyhow!(
315 "Venue not found for publisher_id {}",
316 msg.hd.publisher_id
317 )
318 })?,
319 };
320 let instrument_id = InstrumentId::new(symbol, venue);
321 let instrument_id = resolve_futures_instrument_id(
322 &record,
323 instrument_id,
324 self.futures_year_digits,
325 &self.symbol_venue_map,
326 None,
327 )?;
328 self.symbol_venue_map.insert(instrument_id.symbol, venue);
329 let ts_init = msg.ts_recv.into();
330
331 let instrument = decode_instrument_def_msg(
332 rec,
333 instrument_id,
334 Some(ts_init),
335 decode_config,
336 )?;
337
338 if instrument.is_some() && instrument_id.symbol != symbol {
339 let date = time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(
340 msg.ts_recv,
341 ))?
342 .date();
343 self.futures_symbols.insert(
344 (msg.hd.publisher_id, msg.hd.instrument_id, date),
345 instrument_id.symbol,
346 );
347 }
348
349 Ok(instrument)
350 })();
351
352 match result {
353 Ok(Some(item)) => return Some(Ok(item)),
354 Ok(None) => {}
355 Err(e) => return Some(Err(e)),
356 }
357 }
358 }))
359 }
360
361 pub fn read_records<T>(
367 &self,
368 filepath: &Path,
369 instrument_id: Option<InstrumentId>,
370 price_precision: Option<u8>,
371 include_trades: bool,
372 bars_timestamp_on_close: Option<bool>,
373 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<(Option<Data>, Option<Data>)>> + '_>
374 where
375 T: dbn::Record + dbn::HasRType + 'static,
376 {
377 let decoder = Decoder::from_zstd_file(filepath)?;
378 let mut metadata_cache = if instrument_id.is_none() {
379 Some(MetadataCache::new(decoder.metadata().clone()))
380 } else {
381 None
382 };
383 let mut dbn_stream = decoder.decode_stream::<T>();
384 let fixed_instrument_id = instrument_id.is_some();
385 let mut fixed_price_precision = price_precision;
386
387 Ok(std::iter::from_fn(move || {
388 let result: anyhow::Result<Option<(Option<Data>, Option<Data>)>> = (|| {
389 dbn_stream
390 .advance()
391 .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"))?;
392
393 if let Some(rec) = dbn_stream.get() {
394 let record = dbn::RecordRef::from(rec);
395 let instrument_id = self
396 .resolve_record_instrument_id(&record, instrument_id, &mut metadata_cache)
397 .context("failed to decode instrument id")?;
398 let resolved_precision = self.resolve_stream_price_precision(
399 &instrument_id,
400 fixed_instrument_id,
401 &mut fixed_price_precision,
402 )?;
403 let (item1, item2) = decode_record(
404 &record,
405 instrument_id,
406 resolved_precision,
407 None,
408 include_trades,
409 bars_timestamp_on_close.unwrap_or(true),
410 )?;
411 Ok(Some((item1, item2)))
412 } else {
413 Ok(None)
414 }
415 })();
416
417 match result {
418 Ok(Some(v)) => Some(Ok(v)),
419 Ok(None) => None,
420 Err(e) => Some(Err(e)),
421 }
422 }))
423 }
424
425 pub fn load_instruments(
434 &mut self,
435 filepath: &Path,
436 use_exchange_as_venue: bool,
437 skip_on_error: bool,
438 decode_config: Option<&DatabentoDecodeConfig>,
439 ) -> anyhow::Result<Vec<InstrumentAny>> {
440 let instruments = if skip_on_error {
441 let mut collected = Vec::new();
442
443 for result in
444 self.read_definition_records(filepath, use_exchange_as_venue, decode_config)?
445 {
446 match result {
447 Ok(instrument) => collected.push(instrument),
448 Err(e) => log::warn!("Skipping instrument: {e}"),
449 }
450 }
451 collected
452 } else {
453 self.read_definition_records(filepath, use_exchange_as_venue, decode_config)?
454 .collect::<Result<Vec<_>, _>>()?
455 };
456
457 for instrument in &instruments {
458 self.price_precisions
459 .insert(instrument.id().symbol, instrument.price_precision());
460 }
461
462 Ok(instruments)
463 }
464
465 pub fn load_order_book_deltas(
473 &self,
474 filepath: &Path,
475 instrument_id: Option<InstrumentId>,
476 price_precision: Option<u8>,
477 ) -> anyhow::Result<Vec<OrderBookDelta>> {
478 self.read_order_book_deltas(filepath, instrument_id, price_precision)?
479 .collect()
480 }
481
482 pub fn read_order_book_deltas(
490 &self,
491 filepath: &Path,
492 instrument_id: Option<InstrumentId>,
493 price_precision: Option<u8>,
494 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<OrderBookDelta>> + '_> {
495 let decoder = Decoder::from_zstd_file(filepath)?;
496 let mut metadata_cache = if instrument_id.is_none() {
497 Some(MetadataCache::new(decoder.metadata().clone()))
498 } else {
499 None
500 };
501 let mut dbn_stream = decoder.decode_stream::<dbn::MboMsg>();
502 let fixed_instrument_id = instrument_id.is_some();
503 let mut fixed_price_precision = price_precision;
504 let mut delta_buffer = MboDeltaBuffer::default();
505 let mut terminal_error = None;
506 let mut finished = false;
507
508 Ok(std::iter::from_fn(move || {
509 loop {
510 if let Some(delta) = delta_buffer.pop_ready() {
511 return Some(Ok(delta));
512 }
513
514 if finished {
515 return terminal_error.take().map(Err);
516 }
517
518 let result: anyhow::Result<bool> = (|| {
519 dbn_stream
520 .advance()
521 .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"))?;
522
523 let Some(rec) = dbn_stream.get() else {
524 return Ok(false);
525 };
526 let record = dbn::RecordRef::from(rec);
527 let instrument_id = self
528 .resolve_record_instrument_id(&record, instrument_id, &mut metadata_cache)
529 .context("failed to decode instrument id")?;
530 let resolved_precision = self.resolve_stream_price_precision(
531 &instrument_id,
532 fixed_instrument_id,
533 &mut fixed_price_precision,
534 )?;
535 let msg = record
536 .get::<dbn::MboMsg>()
537 .ok_or_else(|| anyhow::anyhow!("Failed to decode MboMsg"))?;
538 let (delta, _trade) =
539 decode_mbo_msg(msg, instrument_id, resolved_precision, None, false)?;
540 delta_buffer.push(msg, instrument_id, delta);
541 Ok(true)
542 })();
543
544 match result {
545 Ok(true) => {}
546 Ok(false) => {
547 delta_buffer.finish();
548 finished = true;
549 }
550 Err(e) => {
551 delta_buffer.finish();
552 terminal_error = Some(e);
553 finished = true;
554 }
555 }
556 }
557 }))
558 }
559
560 pub fn load_order_book_depth(
566 &self,
567 filepath: &Path,
568 instrument_id: Option<InstrumentId>,
569 price_precision: Option<u8>,
570 ) -> anyhow::Result<Vec<OrderBookDepth>> {
571 self.read_records::<dbn::Mbp10Msg>(filepath, instrument_id, price_precision, false, None)?
572 .filter_map(|result| match result {
573 Ok((Some(item1), _)) => {
574 if let Data::BookDepth(depth) = item1 {
575 Some(Ok(*depth))
576 } else {
577 None
578 }
579 }
580 Ok((None, _)) => None,
581 Err(e) => Some(Err(e)),
582 })
583 .collect()
584 }
585
586 pub fn load_quotes(
592 &self,
593 filepath: &Path,
594 instrument_id: Option<InstrumentId>,
595 price_precision: Option<u8>,
596 ) -> anyhow::Result<Vec<QuoteTick>> {
597 self.read_records::<dbn::Mbp1Msg>(filepath, instrument_id, price_precision, false, None)?
598 .filter_map(|result| match result {
599 Ok((Some(item1), _)) => {
600 if let Data::Quote(quote) = item1 {
601 Some(Ok(quote))
602 } else {
603 None
604 }
605 }
606 Ok((None, _)) => None,
607 Err(e) => Some(Err(e)),
608 })
609 .collect()
610 }
611
612 pub fn load_bbo_quotes(
618 &self,
619 filepath: &Path,
620 instrument_id: Option<InstrumentId>,
621 price_precision: Option<u8>,
622 ) -> anyhow::Result<Vec<QuoteTick>> {
623 self.read_records::<dbn::BboMsg>(filepath, instrument_id, price_precision, false, None)?
624 .filter_map(|result| match result {
625 Ok((Some(item1), _)) => {
626 if let Data::Quote(quote) = item1 {
627 Some(Ok(quote))
628 } else {
629 None
630 }
631 }
632 Ok((None, _)) => None,
633 Err(e) => Some(Err(e)),
634 })
635 .collect()
636 }
637
638 pub fn load_cmbp_quotes(
644 &self,
645 filepath: &Path,
646 instrument_id: Option<InstrumentId>,
647 price_precision: Option<u8>,
648 ) -> anyhow::Result<Vec<QuoteTick>> {
649 self.read_records::<dbn::Cmbp1Msg>(filepath, instrument_id, price_precision, false, None)?
650 .filter_map(|result| match result {
651 Ok((Some(item1), _)) => {
652 if let Data::Quote(quote) = item1 {
653 Some(Ok(quote))
654 } else {
655 None
656 }
657 }
658 Ok((None, _)) => None,
659 Err(e) => Some(Err(e)),
660 })
661 .collect()
662 }
663
664 pub fn load_cbbo_quotes(
670 &self,
671 filepath: &Path,
672 instrument_id: Option<InstrumentId>,
673 price_precision: Option<u8>,
674 ) -> anyhow::Result<Vec<QuoteTick>> {
675 self.read_records::<dbn::CbboMsg>(filepath, instrument_id, price_precision, false, None)?
676 .filter_map(|result| match result {
677 Ok((Some(item1), _)) => {
678 if let Data::Quote(quote) = item1 {
679 Some(Ok(quote))
680 } else {
681 None
682 }
683 }
684 Ok((None, _)) => None,
685 Err(e) => Some(Err(e)),
686 })
687 .collect()
688 }
689
690 pub fn load_tbbo_trades(
696 &self,
697 filepath: &Path,
698 instrument_id: Option<InstrumentId>,
699 price_precision: Option<u8>,
700 ) -> anyhow::Result<Vec<TradeTick>> {
701 self.read_records::<dbn::TbboMsg>(filepath, instrument_id, price_precision, true, None)?
702 .filter_map(|result| match result {
703 Ok((_, maybe_item2)) => {
704 if let Some(Data::Trade(trade)) = maybe_item2 {
705 Some(Ok(trade))
706 } else {
707 None
708 }
709 }
710 Err(e) => Some(Err(e)),
711 })
712 .collect()
713 }
714
715 pub fn load_tcbbo_trades(
721 &self,
722 filepath: &Path,
723 instrument_id: Option<InstrumentId>,
724 price_precision: Option<u8>,
725 ) -> anyhow::Result<Vec<TradeTick>> {
726 self.read_records::<dbn::TcbboMsg>(filepath, instrument_id, price_precision, true, None)?
727 .filter_map(|result| match result {
728 Ok((_, maybe_item2)) => {
729 if let Some(Data::Trade(trade)) = maybe_item2 {
730 Some(Ok(trade))
731 } else {
732 None
733 }
734 }
735 Err(e) => Some(Err(e)),
736 })
737 .collect()
738 }
739
740 pub fn load_trades(
746 &self,
747 filepath: &Path,
748 instrument_id: Option<InstrumentId>,
749 price_precision: Option<u8>,
750 ) -> anyhow::Result<Vec<TradeTick>> {
751 self.read_records::<dbn::TradeMsg>(filepath, instrument_id, price_precision, false, None)?
752 .filter_map(|result| match result {
753 Ok((Some(item1), _)) => {
754 if let Data::Trade(trade) = item1 {
755 Some(Ok(trade))
756 } else {
757 None
758 }
759 }
760 Ok((None, _)) => None,
761 Err(e) => Some(Err(e)),
762 })
763 .collect()
764 }
765
766 pub fn load_bars(
772 &self,
773 filepath: &Path,
774 instrument_id: Option<InstrumentId>,
775 price_precision: Option<u8>,
776 timestamp_on_close: Option<bool>,
777 ) -> anyhow::Result<Vec<Bar>> {
778 self.read_records::<dbn::OhlcvMsg>(
779 filepath,
780 instrument_id,
781 price_precision,
782 false,
783 timestamp_on_close,
784 )?
785 .filter_map(|result| match result {
786 Ok((Some(item1), _)) => {
787 if let Data::Bar(bar) = item1 {
788 Some(Ok(bar))
789 } else {
790 None
791 }
792 }
793 Ok((None, _)) => None,
794 Err(e) => Some(Err(e)),
795 })
796 .collect()
797 }
798
799 pub fn load_status_records<T>(
805 &self,
806 filepath: &Path,
807 instrument_id: Option<InstrumentId>,
808 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<InstrumentStatus>> + '_>
809 where
810 T: dbn::Record + dbn::HasRType + 'static,
811 {
812 let decoder = Decoder::from_zstd_file(filepath)?;
813 let mut metadata_cache = if instrument_id.is_none() {
814 Some(MetadataCache::new(decoder.metadata().clone()))
815 } else {
816 None
817 };
818 let mut dbn_stream = decoder.decode_stream::<T>();
819
820 Ok(std::iter::from_fn(move || {
821 if let Err(e) = dbn_stream.advance() {
822 return Some(Err(e.into()));
823 }
824
825 match dbn_stream.get() {
826 Some(rec) => {
827 let record = dbn::RecordRef::from(rec);
828 let instrument_id = match self.resolve_record_instrument_id(
829 &record,
830 instrument_id,
831 &mut metadata_cache,
832 ) {
833 Ok(id) => id,
834 Err(e) => return Some(Err(e)),
835 };
836
837 let msg = match record.get::<dbn::StatusMsg>() {
838 Some(m) => m,
839 None => return Some(Err(anyhow::anyhow!("Invalid `StatusMsg`"))),
840 };
841 let ts_init = msg.ts_recv.into();
842
843 match decode_status_msg(msg, instrument_id, Some(ts_init)) {
844 Ok(data) => Some(Ok(data)),
845 Err(e) => Some(Err(e)),
846 }
847 }
848 None => None,
849 }
850 }))
851 }
852
853 pub fn read_imbalance_records<T>(
859 &self,
860 filepath: &Path,
861 instrument_id: Option<InstrumentId>,
862 price_precision: Option<u8>,
863 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<DatabentoImbalance>> + '_>
864 where
865 T: dbn::Record + dbn::HasRType + 'static,
866 {
867 let decoder = Decoder::from_zstd_file(filepath)?;
868 let mut metadata_cache = if instrument_id.is_none() {
869 Some(MetadataCache::new(decoder.metadata().clone()))
870 } else {
871 None
872 };
873 let mut dbn_stream = decoder.decode_stream::<T>();
874 let fixed_instrument_id = instrument_id.is_some();
875 let mut fixed_price_precision = price_precision;
876
877 Ok(std::iter::from_fn(move || {
878 if let Err(e) = dbn_stream.advance() {
879 return Some(Err(e.into()));
880 }
881
882 match dbn_stream.get() {
883 Some(rec) => {
884 let record = dbn::RecordRef::from(rec);
885 let instrument_id = match self.resolve_record_instrument_id(
886 &record,
887 instrument_id,
888 &mut metadata_cache,
889 ) {
890 Ok(id) => id,
891 Err(e) => return Some(Err(e)),
892 };
893 let resolved_precision = match self.resolve_stream_price_precision(
894 &instrument_id,
895 fixed_instrument_id,
896 &mut fixed_price_precision,
897 ) {
898 Ok(p) => p,
899 Err(e) => return Some(Err(e)),
900 };
901
902 let msg = match record.get::<dbn::ImbalanceMsg>() {
903 Some(m) => m,
904 None => return Some(Err(anyhow::anyhow!("Invalid `ImbalanceMsg`"))),
905 };
906 let ts_init = msg.ts_recv.into();
907
908 match decode_imbalance_msg(
909 msg,
910 instrument_id,
911 resolved_precision,
912 Some(ts_init),
913 ) {
914 Ok(data) => Some(Ok(data)),
915 Err(e) => Some(Err(e)),
916 }
917 }
918 None => None,
919 }
920 }))
921 }
922
923 pub fn read_statistics_records<T>(
929 &self,
930 filepath: &Path,
931 instrument_id: Option<InstrumentId>,
932 price_precision: Option<u8>,
933 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<DatabentoStatistics>> + '_>
934 where
935 T: dbn::Record + dbn::HasRType + 'static,
936 {
937 let decoder = Decoder::from_zstd_file(filepath)?;
938 let mut metadata_cache = if instrument_id.is_none() {
939 Some(MetadataCache::new(decoder.metadata().clone()))
940 } else {
941 None
942 };
943 let mut dbn_stream = decoder.decode_stream::<T>();
944 let fixed_instrument_id = instrument_id.is_some();
945 let mut fixed_price_precision = price_precision;
946
947 Ok(std::iter::from_fn(move || {
950 loop {
951 if let Err(e) = dbn_stream.advance() {
952 return Some(Err(e.into()));
953 }
954
955 let rec = dbn_stream.get()?;
956 let record = dbn::RecordRef::from(rec);
957 let msg = match record.get::<dbn::StatMsg>() {
958 Some(m) => m,
959 None => return Some(Err(anyhow::anyhow!("Invalid `StatMsg`"))),
960 };
961
962 if !is_supported_stat_type(msg.stat_type) {
963 log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
964 continue;
965 }
966
967 let instrument_id = match self.resolve_record_instrument_id(
968 &record,
969 instrument_id,
970 &mut metadata_cache,
971 ) {
972 Ok(id) => id,
973 Err(e) => return Some(Err(e)),
974 };
975 let resolved_precision = match self.resolve_stream_price_precision(
976 &instrument_id,
977 fixed_instrument_id,
978 &mut fixed_price_precision,
979 ) {
980 Ok(p) => p,
981 Err(e) => return Some(Err(e)),
982 };
983 let ts_init = msg.ts_recv.into();
984
985 match decode_statistics_msg(msg, instrument_id, resolved_precision, Some(ts_init)) {
986 Ok(Some(data)) => return Some(Ok(data)),
987 Ok(None) => {}
988 Err(e) => return Some(Err(e)),
989 }
990 }
991 }))
992 }
993
994 fn resolve_record_instrument_id(
995 &self,
996 record: &dbn::RecordRef,
997 instrument_id: Option<InstrumentId>,
998 metadata_cache: &mut Option<MetadataCache>,
999 ) -> anyhow::Result<InstrumentId> {
1000 if let Some(instrument_id) = instrument_id {
1001 return Ok(instrument_id);
1002 }
1003
1004 let metadata_cache = metadata_cache
1005 .as_mut()
1006 .ok_or_else(|| anyhow::anyhow!("missing metadata cache for dynamic instrument id"))?;
1007
1008 decode_nautilus_instrument_id(
1009 record,
1010 metadata_cache,
1011 &self.publisher_venue_map,
1012 &self.symbol_venue_map,
1013 self.futures_year_digits,
1014 self.share_class_space,
1015 &self.futures_symbols,
1016 None,
1017 )
1018 }
1019
1020 fn resolve_stream_price_precision(
1021 &self,
1022 instrument_id: &InstrumentId,
1023 fixed_instrument_id: bool,
1024 fixed_price_precision: &mut Option<u8>,
1025 ) -> anyhow::Result<u8> {
1026 if let Some(precision) = *fixed_price_precision {
1027 return Ok(precision);
1028 }
1029
1030 let precision = self.resolve_price_precision(instrument_id, None)?;
1031 if fixed_instrument_id {
1032 *fixed_price_precision = Some(precision);
1033 }
1034
1035 Ok(precision)
1036 }
1037}
1038
1039fn apply_default_venue_dataset_mappings(venue_dataset_map: &mut IndexMap<Venue, Dataset>) {
1043 let glbx = Dataset::from("GLBX.MDP3");
1044
1045 for venue in [
1046 Venue::CBCM(),
1047 Venue::GLBX(),
1048 Venue::NYUM(),
1049 Venue::XCBT(),
1050 Venue::XCEC(),
1051 Venue::XCME(),
1052 Venue::XFXS(),
1053 Venue::XNYM(),
1054 ] {
1055 _ = venue_dataset_map.insert(venue, glbx);
1056 }
1057
1058 _ = venue_dataset_map.insert(Venue::from("EQUS"), Dataset::from("EQUS.MINI"));
1061
1062 let opra = Dataset::from("OPRA.PILLAR");
1063 for venue_code in [
1064 "AMXO", "XBOX", "XCBO", "EMLD", "EDGO", "GMNI", "XISX", "MCRY", "XMIO", "ARCO", "OPRA",
1065 "MPRL", "XNDQ", "XBXO", "C2OX", "XPHL", "BATO", "MXOP", "SPHR",
1066 ] {
1067 _ = venue_dataset_map.insert(Venue::from(venue_code), opra);
1068 }
1069}
1070
1071#[cfg(test)]
1072mod tests {
1073 use std::{
1074 ffi::c_char,
1075 fs::{File, OpenOptions},
1076 io::Write,
1077 path::{Path, PathBuf},
1078 process,
1079 sync::atomic::{AtomicU64, Ordering},
1080 };
1081
1082 use databento::dbn::{decode::DecodeRecord, encode::EncodeRecord};
1083 use nautilus_model::{
1084 enums::BookAction,
1085 types::{Price, Quantity},
1086 };
1087 use rstest::{fixture, rstest};
1088 use ustr::Ustr;
1089
1090 use super::*;
1091
1092 fn test_data_path() -> PathBuf {
1093 Path::new(env!("CARGO_MANIFEST_DIR")).join("test_data")
1094 }
1095
1096 fn mbo_record(action: c_char, flags: u8, sequence: u32) -> dbn::MboMsg {
1097 let ts_event = 1_609_160_400_000_000_000;
1098 dbn::MboMsg {
1099 hd: dbn::RecordHeader::new::<dbn::MboMsg>(dbn::rtype::MBO, 1, 1, ts_event),
1100 order_id: 42,
1101 price: 4_800_250_000_000,
1102 size: 2,
1103 flags: dbn::FlagSet::new(flags),
1104 channel_id: 1,
1105 action,
1106 side: 'A' as c_char,
1107 ts_recv: ts_event,
1108 ts_in_delta: 0,
1109 sequence,
1110 }
1111 }
1112
1113 fn write_mbo_records(records: &[dbn::MboMsg]) -> PathBuf {
1114 static FILE_ID: AtomicU64 = AtomicU64::new(0);
1115
1116 let id = FILE_ID.fetch_add(1, Ordering::Relaxed);
1117 let path = std::env::temp_dir().join(format!(
1118 "nautilus-databento-mbo-{}-{id}.dbn.zst",
1119 process::id(),
1120 ));
1121 let metadata = dbn::Metadata::builder()
1122 .dataset("GLBX.MDP3")
1123 .schema(Some(dbn::Schema::Mbo))
1124 .start(0)
1125 .stype_in(Some(dbn::SType::InstrumentId))
1126 .stype_out(dbn::SType::InstrumentId)
1127 .build();
1128 let file = File::create(&path).unwrap();
1129 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(file, &metadata).unwrap();
1130 encoder.encode_records(records).unwrap();
1131 encoder.flush().unwrap();
1132 drop(encoder);
1133
1134 path
1135 }
1136
1137 #[fixture]
1138 fn loader() -> DatabentoDataLoader {
1139 let publishers_filepath = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1140 let mut loader = DatabentoDataLoader::new(Some(publishers_filepath)).unwrap();
1141 loader.set_price_precision(Symbol::from("ESM4"), 2);
1143 loader
1144 }
1145
1146 #[fixture]
1147 fn loader_without_seed() -> DatabentoDataLoader {
1148 let publishers_filepath = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1149 DatabentoDataLoader::new(Some(publishers_filepath)).unwrap()
1150 }
1151
1152 #[rstest]
1155 fn test_set_dataset_venue_mapping(mut loader: DatabentoDataLoader) {
1156 let dataset = Ustr::from("EQUS.PLUS");
1157 let venue = Venue::from("XNAS");
1158 loader.set_dataset_for_venue(dataset, venue);
1159
1160 let result = loader.get_dataset_for_venue(&venue).unwrap();
1161 assert_eq!(*result, dataset);
1162 }
1163
1164 #[rstest]
1165 fn test_default_venue_dataset_mappings(loader: DatabentoDataLoader) {
1166 let xcme = Venue::XCME();
1167 let result = loader.get_dataset_for_venue(&xcme).unwrap();
1168 assert_eq!(*result, Ustr::from("GLBX.MDP3"));
1169
1170 let xcbo = Venue::from("XCBO");
1171 let result = loader.get_dataset_for_venue(&xcbo).unwrap();
1172 assert_eq!(*result, Ustr::from("OPRA.PILLAR"));
1173
1174 let equs = Venue::from("EQUS");
1175 let result = loader.get_dataset_for_venue(&equs).unwrap();
1176 assert_eq!(*result, Ustr::from("EQUS.MINI"));
1177 }
1178
1179 #[rstest]
1180 #[case(test_data_path().join("test_data.definition.equity.dbn.zst"))]
1181 fn test_load_instruments(mut loader: DatabentoDataLoader, #[case] path: PathBuf) {
1182 let instruments = loader.load_instruments(&path, false, false, None).unwrap();
1183
1184 assert_eq!(instruments.len(), 2);
1185 assert_eq!(
1187 loader.get_price_precisions().get(&Symbol::from("ESM4")),
1188 Some(&2)
1189 );
1190 }
1191
1192 #[rstest]
1193 fn test_load_instruments_populates_price_precisions_cache(
1194 mut loader_without_seed: DatabentoDataLoader,
1195 ) {
1196 let path = test_data_path().join("test_data.definition.equity.dbn.zst");
1197 assert!(loader_without_seed.get_price_precisions().is_empty());
1198
1199 let instruments = loader_without_seed
1200 .load_instruments(&path, false, false, None)
1201 .unwrap();
1202
1203 assert_eq!(instruments.len(), 2);
1204 for instrument in &instruments {
1205 let symbol = instrument.id().symbol;
1206 assert_eq!(
1207 loader_without_seed.get_price_precisions().get(&symbol),
1208 Some(&instrument.price_precision()),
1209 "cache missing or mismatched entry for {symbol}",
1210 );
1211 }
1212 }
1213
1214 #[rstest]
1215 fn test_read_records_errors_when_precision_unresolvable(
1216 loader_without_seed: DatabentoDataLoader,
1217 ) {
1218 let path = test_data_path().join("test_data.mbo.dbn.zst");
1219 let instrument_id = InstrumentId::from("ESM4.GLBX");
1220
1221 let result = loader_without_seed.load_order_book_deltas(&path, Some(instrument_id), None);
1222
1223 let err = result.expect_err("expected precision-resolution error");
1224 let err_msg = format!("{err}");
1225 assert!(
1226 err_msg.contains("Could not resolve `price_precision`"),
1227 "unexpected error message: {err_msg}",
1228 );
1229 assert!(
1230 err_msg.contains("ESM4.GLBX"),
1231 "error should name the instrument: {err_msg}",
1232 );
1233 }
1234
1235 #[rstest]
1236 fn test_set_price_precision_unblocks_reads(mut loader_without_seed: DatabentoDataLoader) {
1237 let path = test_data_path().join("test_data.mbo.dbn.zst");
1238 let instrument_id = InstrumentId::from("ESM4.GLBX");
1239
1240 assert!(
1242 loader_without_seed
1243 .load_order_book_deltas(&path, Some(instrument_id), None)
1244 .is_err()
1245 );
1246
1247 loader_without_seed.set_price_precision(Symbol::from("ESM4"), 2);
1248
1249 let deltas = loader_without_seed
1250 .load_order_book_deltas(&path, Some(instrument_id), None)
1251 .unwrap();
1252 assert_eq!(deltas.len(), 2);
1253 }
1254
1255 #[rstest]
1256 fn test_resolve_price_precision_explicit_arg_overrides_cache(
1257 mut loader_without_seed: DatabentoDataLoader,
1258 ) {
1259 let instrument_id = InstrumentId::from("ESM4.GLBX");
1260 loader_without_seed.set_price_precision(Symbol::from("ESM4"), 9);
1262
1263 let explicit = loader_without_seed
1264 .resolve_price_precision(&instrument_id, Some(2))
1265 .unwrap();
1266 assert_eq!(explicit, 2);
1267
1268 let cached = loader_without_seed
1269 .resolve_price_precision(&instrument_id, None)
1270 .unwrap();
1271 assert_eq!(cached, 9);
1272 }
1273
1274 #[rstest]
1275 fn test_resolve_price_precision_cache_miss_errors(loader_without_seed: DatabentoDataLoader) {
1276 let instrument_id = InstrumentId::from("ESM4.GLBX");
1277
1278 let err = loader_without_seed
1279 .resolve_price_precision(&instrument_id, None)
1280 .expect_err("expected cache-miss error");
1281 assert!(format!("{err}").contains("Could not resolve `price_precision`"));
1282 }
1283
1284 #[rstest]
1285 fn test_load_order_book_deltas(loader: DatabentoDataLoader) {
1286 let path = test_data_path().join("test_data.mbo.dbn.zst");
1287 let instrument_id = InstrumentId::from("ESM4.GLBX");
1288
1289 let deltas = loader
1290 .load_order_book_deltas(&path, Some(instrument_id), None)
1291 .unwrap();
1292
1293 assert_eq!(deltas.len(), 2);
1294 }
1295
1296 #[rstest]
1297 fn test_read_order_book_deltas_streams_without_collecting(loader: DatabentoDataLoader) {
1298 let path = test_data_path().join("test_data.mbo.dbn.zst");
1299 let instrument_id = InstrumentId::from("ESM4.GLBX");
1300
1301 let count = loader
1302 .read_order_book_deltas(&path, Some(instrument_id), None)
1303 .unwrap()
1304 .map(|result| result.map(|_| 1usize))
1305 .sum::<anyhow::Result<usize>>()
1306 .unwrap();
1307
1308 assert_eq!(count, 2);
1309 }
1310
1311 #[rstest]
1312 #[case::standalone(false)]
1313 #[case::legacy_inline(true)]
1314 fn test_load_order_book_deltas_preserves_event_boundary(
1315 loader: DatabentoDataLoader,
1316 #[case] inline_last: bool,
1317 ) {
1318 let instrument_id = InstrumentId::from("ESM4.GLBX");
1319 let last = dbn::flags::LAST;
1320 let mut records = vec![mbo_record(
1321 'C' as c_char,
1322 if inline_last { last } else { 0 },
1323 1,
1324 )];
1325
1326 if !inline_last {
1327 records.push(mbo_record('N' as c_char, last, 2));
1328 }
1329 let path = write_mbo_records(&records);
1330
1331 let deltas = loader
1332 .load_order_book_deltas(&path, Some(instrument_id), Some(2))
1333 .unwrap();
1334 std::fs::remove_file(path).unwrap();
1335
1336 assert_eq!(deltas.len(), 1);
1337 assert_eq!(deltas[0].instrument_id, instrument_id);
1338 assert_eq!(deltas[0].action, BookAction::Delete);
1339 assert_eq!(deltas[0].order.order_id, 42);
1340 assert_eq!(deltas[0].flags, last);
1341 assert_eq!(deltas[0].sequence, 1);
1342 }
1343
1344 #[rstest]
1345 fn test_read_order_book_deltas_drains_before_terminal_error(loader: DatabentoDataLoader) {
1346 let instrument_id = InstrumentId::from("ESM4.GLBX");
1347 let path = write_mbo_records(&[mbo_record('C' as c_char, 0, 1)]);
1348 OpenOptions::new()
1349 .append(true)
1350 .open(&path)
1351 .unwrap()
1352 .write_all(&[0; 8])
1353 .unwrap();
1354 let mut deltas = loader
1355 .read_order_book_deltas(&path, Some(instrument_id), Some(2))
1356 .unwrap();
1357
1358 let delta = deltas.next().unwrap().unwrap();
1359 let error = deltas
1360 .next()
1361 .unwrap()
1362 .expect_err("expected trailing decode error");
1363 let end = deltas.next();
1364 std::fs::remove_file(path).unwrap();
1365
1366 assert_eq!(delta.instrument_id, instrument_id);
1367 assert_eq!(delta.order.order_id, 42);
1368 assert_eq!(delta.flags, 0);
1369 assert!(format!("{error}").contains("Stream advance error"));
1370 assert!(end.is_none());
1371 }
1372
1373 #[rstest]
1374 fn test_load_order_book_depth(loader: DatabentoDataLoader) {
1375 let path = test_data_path().join("test_data.mbp-10.dbn.zst");
1376 let instrument_id = InstrumentId::from("ESM4.GLBX");
1377
1378 let depths = loader
1379 .load_order_book_depth(&path, Some(instrument_id), None)
1380 .unwrap();
1381
1382 assert_eq!(depths.len(), 2);
1383 }
1384
1385 #[rstest]
1386 fn test_load_quotes(loader: DatabentoDataLoader) {
1387 let path = test_data_path().join("test_data.mbp-1.dbn.zst");
1388 let instrument_id = InstrumentId::from("ESM4.GLBX");
1389
1390 let quotes = loader
1391 .load_quotes(&path, Some(instrument_id), None)
1392 .unwrap();
1393
1394 assert_eq!(quotes.len(), 2);
1395 }
1396
1397 #[rstest]
1398 #[case(test_data_path().join("test_data.bbo-1s.dbn.zst"))]
1399 #[case(test_data_path().join("test_data.bbo-1m.dbn.zst"))]
1400 fn test_load_bbo_quotes(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1401 let instrument_id = InstrumentId::from("ESM4.GLBX");
1402
1403 let quotes = loader
1404 .load_bbo_quotes(&path, Some(instrument_id), None)
1405 .unwrap();
1406
1407 assert_eq!(quotes.len(), 4);
1408 }
1409
1410 #[rstest]
1411 fn test_load_cmbp_quotes(loader: DatabentoDataLoader) {
1412 let path = test_data_path().join("test_data.cmbp-1.dbn.zst");
1413 let instrument_id = InstrumentId::from("ESM4.GLBX");
1414
1415 let quotes = loader
1416 .load_cmbp_quotes(&path, Some(instrument_id), None)
1417 .unwrap();
1418
1419 assert_eq!(quotes.len(), 2);
1421
1422 let first_quote = "es[0];
1424 assert_eq!(first_quote.instrument_id, instrument_id);
1425 assert_eq!(first_quote.bid_price, Price::from("3720.25"));
1426 assert_eq!(first_quote.ask_price, Price::from("3720.50"));
1427 assert_eq!(first_quote.bid_size, Quantity::from(24));
1428 assert_eq!(first_quote.ask_size, Quantity::from(11));
1429 assert_eq!(first_quote.ts_event, 1609160400006136329);
1430 assert_eq!(first_quote.ts_init, 1609160400006136329);
1431 }
1432
1433 #[rstest]
1434 fn test_load_cbbo_quotes(loader: DatabentoDataLoader) {
1435 let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1436 let instrument_id = InstrumentId::from("ESM4.GLBX");
1437
1438 let quotes = loader
1439 .load_cbbo_quotes(&path, Some(instrument_id), None)
1440 .unwrap();
1441
1442 assert_eq!(quotes.len(), 2);
1444
1445 let first_quote = "es[0];
1447 assert_eq!(first_quote.instrument_id, instrument_id);
1448 assert_eq!(first_quote.bid_price, Price::from("3720.25"));
1449 assert_eq!(first_quote.ask_price, Price::from("3720.50"));
1450 assert_eq!(first_quote.bid_size, Quantity::from(24));
1451 assert_eq!(first_quote.ask_size, Quantity::from(11));
1452 assert_eq!(first_quote.ts_event, 1609160400006136329);
1453 assert_eq!(first_quote.ts_init, 1609160400006136329);
1454 }
1455
1456 #[rstest]
1457 #[case(1, "ESH1.GLBX")]
1458 #[case(2, "ESH21.GLBX")]
1459 fn test_load_cbbo_quotes_resolves_instrument_id(
1460 loader: DatabentoDataLoader,
1461 #[case] digits: u8,
1462 #[case] expected: InstrumentId,
1463 ) {
1464 let loader = loader.with_futures_year_digits(digits.try_into().unwrap());
1465 let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1466 let quotes = loader.load_cbbo_quotes(&path, None, Some(2)).unwrap();
1467
1468 assert_eq!(quotes.len(), 2);
1469 assert_eq!(quotes[0].instrument_id, expected);
1470 assert_eq!(quotes[1].instrument_id, expected);
1471 assert_eq!(quotes[0].bid_price, Price::from("3720.25"));
1472 assert_eq!(quotes[0].ask_price, Price::from("3720.50"));
1473 assert_eq!(quotes[0].bid_size, Quantity::from(24));
1474 assert_eq!(quotes[0].ask_size, Quantity::from(11));
1475 assert_eq!(quotes[0].ts_event, 1609160400006136329);
1476 assert_eq!(quotes[0].ts_init, 1609160400006136329);
1477 }
1478
1479 #[rstest]
1480 fn test_load_tbbo_trades(loader: DatabentoDataLoader) {
1481 let path = test_data_path().join("test_data.tbbo.dbn.zst");
1482 let instrument_id = InstrumentId::from("ESM4.GLBX");
1483
1484 let trades = loader
1485 .load_tbbo_trades(&path, Some(instrument_id), None)
1486 .unwrap();
1487
1488 assert_eq!(trades.len(), 2);
1489 assert_eq!(trades[0].instrument_id, instrument_id);
1490 assert_eq!(trades[0].price, Price::from("3720.25"));
1491 assert_eq!(trades[0].size, Quantity::from("5"));
1492 }
1493
1494 #[rstest]
1495 fn test_load_tcbbo_trades_rejects_cbbo_fixture(loader: DatabentoDataLoader) {
1496 let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1497 let instrument_id = InstrumentId::from("ESM4.GLBX");
1498
1499 let result = loader.load_tcbbo_trades(&path, Some(instrument_id), None);
1500
1501 assert!(result.is_err());
1502 }
1503
1504 #[rstest]
1505 fn test_load_trades(loader: DatabentoDataLoader) {
1506 let path = test_data_path().join("test_data.trades.dbn.zst");
1507 let instrument_id = InstrumentId::from("ESM4.GLBX");
1508 let trades = loader
1509 .load_trades(&path, Some(instrument_id), None)
1510 .unwrap();
1511
1512 assert_eq!(trades.len(), 2);
1513 }
1514
1515 #[rstest]
1516 #[case(test_data_path().join("test_data.ohlcv-1h.dbn.zst"))]
1518 #[case(test_data_path().join("test_data.ohlcv-1m.dbn.zst"))]
1519 #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1520 fn test_load_bars(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1521 let instrument_id = InstrumentId::from("ESM4.GLBX");
1522 let bars = loader
1523 .load_bars(&path, Some(instrument_id), None, None)
1524 .unwrap();
1525
1526 assert_eq!(bars.len(), 2);
1527 }
1528
1529 #[rstest]
1530 #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1531 fn test_load_bars_timestamp_on_close_true(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1532 let instrument_id = InstrumentId::from("ESM4.GLBX");
1533 let bars = loader
1534 .load_bars(&path, Some(instrument_id), None, Some(true))
1535 .unwrap();
1536
1537 assert_eq!(bars.len(), 2);
1538
1539 for bar in &bars {
1541 assert_eq!(
1542 bar.ts_event, bar.ts_init,
1543 "ts_event and ts_init should both be close time when bars_timestamp_on_close=true"
1544 );
1545 }
1546 }
1547
1548 #[rstest]
1549 #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1550 fn test_load_bars_timestamp_on_close_false(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1551 let instrument_id = InstrumentId::from("ESM4.GLBX");
1552 let bars = loader
1553 .load_bars(&path, Some(instrument_id), None, Some(false))
1554 .unwrap();
1555
1556 assert_eq!(bars.len(), 2);
1557
1558 for bar in &bars {
1560 assert_ne!(
1561 bar.ts_event, bar.ts_init,
1562 "ts_event should be open time and ts_init should be close time when bars_timestamp_on_close=false"
1563 );
1564 assert_eq!(bar.ts_init.as_u64(), bar.ts_event.as_u64() + 1_000_000_000);
1566 }
1567 }
1568
1569 #[rstest]
1570 #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"), 0)]
1571 #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"), 1)]
1572 fn test_load_bars_timestamp_comparison(
1573 loader: DatabentoDataLoader,
1574 #[case] path: PathBuf,
1575 #[case] bar_index: usize,
1576 ) {
1577 const ONE_SECOND_NS: u64 = 1_000_000_000;
1578
1579 let instrument_id = InstrumentId::from("ESM4.GLBX");
1580
1581 let bars_close = loader
1582 .load_bars(&path, Some(instrument_id), None, Some(true))
1583 .unwrap();
1584
1585 let bars_open = loader
1586 .load_bars(&path, Some(instrument_id), None, Some(false))
1587 .unwrap();
1588
1589 assert_eq!(bars_close.len(), bars_open.len());
1590 assert_eq!(bars_close.len(), 2);
1591
1592 let bar_close = &bars_close[bar_index];
1593 let bar_open = &bars_open[bar_index];
1594
1595 assert_eq!(bar_close.open, bar_open.open);
1597 assert_eq!(bar_close.high, bar_open.high);
1598 assert_eq!(bar_close.low, bar_open.low);
1599 assert_eq!(bar_close.close, bar_open.close);
1600 assert_eq!(bar_close.volume, bar_open.volume);
1601
1602 assert!(
1605 bar_close.ts_event > bar_open.ts_event,
1606 "Close-timestamped bar should have later timestamp than open-timestamped bar"
1607 );
1608
1609 assert_eq!(
1611 bar_close.ts_event.as_u64() - bar_open.ts_event.as_u64(),
1612 ONE_SECOND_NS,
1613 "Timestamp difference should be exactly 1 second for 1s bars"
1614 );
1615 }
1616
1617 #[rstest]
1618 fn test_load_status_records(loader: DatabentoDataLoader) {
1619 let path = test_data_path().join("test_data.status.dbn.zst");
1620 let instrument_id = InstrumentId::from("ESM4.GLBX");
1621
1622 let statuses = loader
1623 .load_status_records::<dbn::StatusMsg>(&path, Some(instrument_id))
1624 .unwrap()
1625 .collect::<anyhow::Result<Vec<_>>>()
1626 .unwrap();
1627
1628 assert_eq!(statuses.len(), 4, "Should load exactly 4 status records");
1630
1631 let first = &statuses[0];
1633 assert_eq!(first.instrument_id, instrument_id);
1634 assert_eq!(first.ts_event.as_u64(), 1609110000000000000);
1635 assert_eq!(first.ts_init.as_u64(), 1609113600000000000);
1636 }
1637
1638 #[rstest]
1639 fn test_read_imbalance_records(loader: DatabentoDataLoader) {
1640 let path = test_data_path().join("test_data.imbalance.dbn.zst");
1641 let instrument_id = InstrumentId::from("ESM4.GLBX");
1642
1643 let imbalances = loader
1644 .read_imbalance_records::<dbn::ImbalanceMsg>(&path, Some(instrument_id), None)
1645 .unwrap()
1646 .collect::<anyhow::Result<Vec<_>>>()
1647 .unwrap();
1648
1649 assert_eq!(
1651 imbalances.len(),
1652 2,
1653 "Should load exactly 2 imbalance records"
1654 );
1655
1656 let first = &imbalances[0];
1658 assert_eq!(first.instrument_id, instrument_id);
1659 assert!(
1660 first.ref_price.as_f64() > 0.0,
1661 "ref_price should be positive"
1662 );
1663 assert!(first.ts_event.as_u64() > 0, "ts_event should be set");
1664 assert!(first.ts_recv.as_u64() > 0, "ts_recv should be set");
1665 assert!(first.ts_init.as_u64() > 0, "ts_init should be set");
1666 }
1667
1668 #[rstest]
1669 fn test_read_statistics_records(loader: DatabentoDataLoader) {
1670 let path = test_data_path().join("test_data.statistics.dbn.zst");
1671 let instrument_id = InstrumentId::from("ESM4.GLBX");
1672
1673 let statistics = loader
1674 .read_statistics_records::<dbn::StatMsg>(&path, Some(instrument_id), None)
1675 .unwrap()
1676 .collect::<anyhow::Result<Vec<_>>>()
1677 .unwrap();
1678
1679 assert_eq!(
1681 statistics.len(),
1682 2,
1683 "Should load exactly 2 statistics records"
1684 );
1685
1686 let first = &statistics[0];
1688 assert_eq!(first.instrument_id, instrument_id);
1689 assert!(first.ts_event.as_u64() > 0, "ts_event should be set");
1690 assert!(first.ts_recv.as_u64() > 0, "ts_recv should be set");
1691 assert!(first.ts_init.as_u64() > 0, "ts_init should be set");
1692 assert!(first.sequence > 0, "sequence should be positive");
1693 }
1694
1695 #[rstest]
1696 #[case("futures_contract", "ESU6", "ESU26.GLBX")]
1697 #[case("futures_spread", "ESU6-ESM7", "ESU26-ESM27.GLBX")]
1698 fn test_loader_defaults_to_two_digit_years(
1699 #[case] fixture: &str,
1700 #[case] raw: &str,
1701 #[case] expected: InstrumentId,
1702 ) {
1703 let publishers = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1704 let path = Path::new(env!("CARGO_MANIFEST_DIR"))
1705 .join(format!("test_data/test_data.definition.{fixture}.dbn.zst"));
1706 let mut loader = DatabentoDataLoader::new(Some(publishers.clone())).unwrap();
1707 let mut legacy = DatabentoDataLoader::new(Some(publishers))
1708 .unwrap()
1709 .with_futures_year_digits(FuturesYearDigits::One);
1710 let instruments = loader.load_instruments(&path, false, false, None).unwrap();
1711 let legacy_instruments = legacy.load_instruments(&path, false, false, None).unwrap();
1712 assert_eq!(instruments.len(), 1);
1713 assert_eq!(legacy_instruments.len(), 1);
1714 assert_eq!(instruments[0].id(), expected);
1715 assert_eq!(instruments[0].raw_symbol(), Symbol::from(raw));
1716 assert_eq!(
1717 legacy_instruments[0].id(),
1718 InstrumentId::new(Symbol::from(raw), Venue::from("GLBX"))
1719 );
1720 assert_eq!(legacy_instruments[0].raw_symbol(), Symbol::from(raw));
1721 }
1722
1723 #[rstest]
1724 #[case(true, "BRK B.XNAS")]
1725 #[case(false, "BRK.B.XNAS")]
1726 fn test_loader_share_class_keeps_native_raw_symbol(
1727 #[case] share_class_space: bool,
1728 #[case] expected: InstrumentId,
1729 ) {
1730 let publishers = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1731 let path = test_data_path().join("test_data.definition.equity.dbn.zst");
1732 let mut decoder = Decoder::from_zstd_file(&path).unwrap();
1733 let metadata = decoder.metadata().clone();
1734 let mut definition = decoder
1735 .decode_record::<dbn::InstrumentDefMsg>()
1736 .unwrap()
1737 .unwrap()
1738 .clone();
1739 definition.raw_symbol = dbn::record::str_to_c_chars("BRK.B").unwrap();
1740 let replay_path =
1741 std::env::temp_dir().join(format!("equity-{}.dbn.zst", nautilus_core::UUID4::new()));
1742 let mut encoded = Vec::new();
1743 {
1744 let mut encoder =
1745 dbn::encode::dbn::Encoder::with_zstd(&mut encoded, &metadata).unwrap();
1746 encoder.encode_record(&definition).unwrap();
1747 encoder.flush().unwrap();
1748 }
1749 std::fs::write(&replay_path, encoded).unwrap();
1750 let mut loader = DatabentoDataLoader::new(Some(publishers))
1751 .unwrap()
1752 .with_share_class_space(share_class_space);
1753
1754 let instruments = loader
1755 .load_instruments(&replay_path, false, false, None)
1756 .unwrap();
1757
1758 std::fs::remove_file(replay_path).unwrap();
1759 assert_eq!(instruments.len(), 1);
1760 assert_eq!(instruments[0].id(), expected);
1761 assert_eq!(instruments[0].raw_symbol(), Symbol::from("BRK.B"));
1762 }
1763
1764 #[rstest]
1765 #[case("futures_contract", "ESU6", "ESU26.GLBX", "ESU36.GLBX")]
1766 #[case("futures_spread", "ESU6-ESM7", "ESU26-ESM27.GLBX", "ESU36-ESM37.GLBX")]
1767 fn test_loader_definition_year_is_scoped_to_record_date(
1768 mut loader_without_seed: DatabentoDataLoader,
1769 #[case] fixture: &str,
1770 #[case] raw: &str,
1771 #[case] expected: InstrumentId,
1772 #[case] inferred: InstrumentId,
1773 ) {
1774 let loader = &mut loader_without_seed;
1775 let path = test_data_path().join(format!("test_data.definition.{fixture}.dbn.zst"));
1776 let mut decoder = Decoder::from_zstd_file(&path).unwrap();
1777 let metadata_source = decoder.metadata().clone();
1778 let mut definition = decoder
1779 .decode_record::<dbn::InstrumentDefMsg>()
1780 .unwrap()
1781 .unwrap()
1782 .clone();
1783 definition.ts_recv = "2028-01-01T00:00:00Z"
1784 .parse::<jiff::Timestamp>()
1785 .unwrap()
1786 .as_nanosecond() as u64;
1787 let replay_path =
1788 std::env::temp_dir().join(format!("futures-{}.dbn.zst", nautilus_core::UUID4::new()));
1789 let mut encoded = Vec::new();
1790 {
1791 let mut encoder =
1792 dbn::encode::dbn::Encoder::with_zstd(&mut encoded, &metadata_source).unwrap();
1793 encoder.encode_record(&definition).unwrap();
1794 encoder.flush().unwrap();
1795 }
1796
1797 std::fs::write(&replay_path, encoded).unwrap();
1798 let replayed = loader
1799 .load_instruments(&replay_path, false, false, None)
1800 .unwrap();
1801 std::fs::remove_file(replay_path).unwrap();
1802 assert_eq!(replayed[0].id(), expected);
1803 let mut record = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1804 record.hd.publisher_id = definition.hd.publisher_id;
1805 record.hd.instrument_id = definition.hd.instrument_id;
1806 record.ts_recv = definition.ts_recv;
1807 let metadata = dbn::Metadata::builder()
1808 .dataset("GLBX.MDP3")
1809 .schema(Some(dbn::Schema::Bbo1S))
1810 .start(record.ts_recv)
1811 .stype_in(Some(dbn::SType::RawSymbol))
1812 .stype_out(dbn::SType::InstrumentId)
1813 .mappings(vec![dbn::SymbolMapping {
1814 raw_symbol: raw.to_string(),
1815 intervals: vec![dbn::MappingInterval {
1816 start_date: time::macros::date!(2028 - 01 - 01),
1817 end_date: time::macros::date!(2028 - 01 - 03),
1818 symbol: record.hd.instrument_id.to_string(),
1819 }],
1820 }])
1821 .build();
1822 let mut metadata = Some(MetadataCache::new(metadata));
1823 let id = loader
1824 .resolve_record_instrument_id(&dbn::RecordRef::from(&record), None, &mut metadata)
1825 .unwrap();
1826 assert_eq!(id, expected);
1827 record.ts_recv += 86_400_000_000_000;
1828 let next_day = loader
1829 .resolve_record_instrument_id(&dbn::RecordRef::from(&record), None, &mut metadata)
1830 .unwrap();
1831 assert_eq!(next_day, inferred);
1832 }
1833}