1use std::{
17 env,
18 path::{Path, PathBuf},
19};
20
21use ahash::AHashMap;
22use anyhow::Context;
23use databento::dbn::{self, InstrumentDefMsg};
24use dbn::{
25 Publisher,
26 decode::{DbnMetadata, DecodeStream, dbn::Decoder},
27};
28use fallible_streaming_iterator::FallibleStreamingIterator;
29use indexmap::IndexMap;
30use nautilus_model::{
31 data::{Bar, Data, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
32 identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
33 instruments::{Instrument, InstrumentAny},
34};
35
36use super::{
37 decode::{
38 MboDeltaBuffer, decode_imbalance_msg, decode_mbo_msg, decode_record, decode_statistics_msg,
39 decode_status_msg, is_supported_stat_type,
40 },
41 symbology::{decode_nautilus_instrument_id, resolve_futures_instrument_id},
42 types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, Dataset, PublisherId},
43};
44use crate::{
45 common::{build_publisher_venue_map, load_publishers},
46 decode::{DatabentoDecodeConfig, decode_instrument_def_msg},
47 symbology::MetadataCache,
48};
49
50#[cfg_attr(
78 feature = "python",
79 pyo3::pyclass(module = "nautilus_trader.adapters.databento")
80)]
81#[cfg_attr(
82 feature = "python",
83 pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
84)]
85#[derive(Debug)]
86pub struct DatabentoDataLoader {
87 futures_year_digits: FuturesYearDigits,
88 publishers_map: IndexMap<PublisherId, DatabentoPublisher>,
89 venue_dataset_map: IndexMap<Venue, Dataset>,
90 publisher_venue_map: IndexMap<PublisherId, Venue>,
91 symbol_venue_map: AHashMap<Symbol, Venue>,
92 price_precisions: AHashMap<Symbol, u8>,
93 futures_symbols: AHashMap<(PublisherId, u32, time::Date), Symbol>,
94}
95
96impl DatabentoDataLoader {
97 pub fn new(publishers_filepath: Option<PathBuf>) -> anyhow::Result<Self> {
103 let mut loader = Self {
104 futures_year_digits: FuturesYearDigits::default(),
105 publishers_map: IndexMap::new(),
106 venue_dataset_map: IndexMap::new(),
107 publisher_venue_map: IndexMap::new(),
108 symbol_venue_map: AHashMap::new(),
109 price_precisions: AHashMap::new(),
110 futures_symbols: AHashMap::new(),
111 };
112
113 let publishers_filepath = if let Some(p) = publishers_filepath {
115 p
116 } else {
117 let mut exe_path = env::current_exe()?;
119 exe_path.pop();
120 exe_path.push("publishers.json");
121 exe_path
122 };
123
124 loader
125 .load_publishers(publishers_filepath)
126 .context("error loading publishers.json")?;
127
128 Ok(loader)
129 }
130
131 #[must_use]
133 pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
134 self.futures_year_digits = futures_year_digits;
135 self
136 }
137
138 pub fn load_publishers(&mut self, filepath: PathBuf) -> anyhow::Result<()> {
144 let publishers = load_publishers(filepath)?;
145
146 self.publishers_map = publishers
147 .iter()
148 .cloned()
149 .map(|p| (p.publisher_id, p))
150 .collect();
151
152 let mut venue_dataset_map = IndexMap::new();
153
154 for publisher in &publishers {
156 let venue = Venue::from(publisher.venue.as_str());
157 let dataset = Dataset::from(publisher.dataset.as_str());
158 venue_dataset_map.entry(venue).or_insert(dataset);
159 }
160
161 self.venue_dataset_map = venue_dataset_map;
162 apply_default_venue_dataset_mappings(&mut self.venue_dataset_map);
163
164 self.publisher_venue_map = build_publisher_venue_map(&publishers);
165
166 Ok(())
167 }
168
169 #[must_use]
171 pub const fn get_publishers(&self) -> &IndexMap<u16, DatabentoPublisher> {
172 &self.publishers_map
173 }
174
175 pub fn set_dataset_for_venue(&mut self, dataset: Dataset, venue: Venue) {
177 _ = self.venue_dataset_map.insert(venue, dataset);
178 }
179
180 #[must_use]
182 pub fn get_dataset_for_venue(&self, venue: &Venue) -> Option<&Dataset> {
183 self.venue_dataset_map.get(venue)
184 }
185
186 #[must_use]
188 pub fn get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option<&Venue> {
189 self.publisher_venue_map.get(&publisher_id)
190 }
191
192 pub fn set_price_precision(&mut self, symbol: Symbol, price_precision: u8) {
198 self.price_precisions.insert(symbol, price_precision);
199 }
200
201 #[must_use]
203 pub const fn get_price_precisions(&self) -> &AHashMap<Symbol, u8> {
204 &self.price_precisions
205 }
206
207 fn resolve_price_precision(
217 &self,
218 instrument_id: &InstrumentId,
219 price_precision: Option<u8>,
220 ) -> anyhow::Result<u8> {
221 if let Some(precision) = price_precision {
222 return Ok(precision);
223 }
224
225 self.price_precisions
226 .get(&instrument_id.symbol)
227 .copied()
228 .ok_or_else(|| {
229 anyhow::anyhow!(
230 "Could not resolve `price_precision` for {instrument_id}: \
231 pass `price_precision` explicitly, call `set_price_precision`, \
232 or load the instrument definitions first via `load_instruments`"
233 )
234 })
235 }
236
237 pub fn schema_from_file(&self, filepath: &Path) -> anyhow::Result<Option<String>> {
243 let decoder = Decoder::from_zstd_file(filepath)?;
244 let metadata = decoder.metadata();
245 Ok(metadata.schema.map(|schema| schema.to_string()))
246 }
247
248 pub fn read_definition_records<'a>(
254 &'a mut self,
255 filepath: &Path,
256 use_exchange_as_venue: bool,
257 decode_config: Option<&'a DatabentoDecodeConfig>,
258 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<InstrumentAny>> + 'a> {
259 let decoder = Decoder::from_zstd_file(filepath)?;
260 let mut dbn_stream = decoder.decode_stream::<InstrumentDefMsg>();
261
262 Ok(std::iter::from_fn(move || {
265 loop {
266 let advance = dbn_stream
267 .advance()
268 .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"));
269 if let Err(e) = advance {
270 return Some(Err(e));
271 }
272
273 let rec = dbn_stream.get()?;
274
275 let result: anyhow::Result<Option<InstrumentAny>> = (|| {
276 let record = dbn::RecordRef::from(rec);
277 let msg = record
278 .get::<InstrumentDefMsg>()
279 .ok_or_else(|| anyhow::anyhow!("Failed to decode InstrumentDefMsg"))?;
280
281 let raw_symbol = rec
282 .raw_symbol()
283 .map_err(|e| anyhow::anyhow!("Error decoding `raw_symbol`: {e}"))?;
284 let symbol = Symbol::from(raw_symbol);
285
286 let publisher = rec
287 .hd
288 .publisher()
289 .map_err(|e| anyhow::anyhow!("Invalid `publisher` for record: {e}"))?;
290 let venue = match publisher {
291 Publisher::GlbxMdp3Glbx if use_exchange_as_venue => {
292 let exchange = rec.exchange().map_err(|e| {
293 anyhow::anyhow!("Missing `exchange` for record: {e}")
294 })?;
295 let venue = Venue::from_code(exchange).map_err(|e| {
296 anyhow::anyhow!("Venue not found for exchange {exchange}: {e}")
297 })?;
298 self.symbol_venue_map.insert(symbol, venue);
299 venue
300 }
301 _ => *self
302 .publisher_venue_map
303 .get(&msg.hd.publisher_id)
304 .ok_or_else(|| {
305 anyhow::anyhow!(
306 "Venue not found for publisher_id {}",
307 msg.hd.publisher_id
308 )
309 })?,
310 };
311 let instrument_id = InstrumentId::new(symbol, venue);
312 let instrument_id = resolve_futures_instrument_id(
313 &record,
314 instrument_id,
315 self.futures_year_digits,
316 &self.symbol_venue_map,
317 None,
318 )?;
319 self.symbol_venue_map.insert(instrument_id.symbol, venue);
320 let ts_init = msg.ts_recv.into();
321
322 let instrument = decode_instrument_def_msg(
323 rec,
324 instrument_id,
325 Some(ts_init),
326 decode_config,
327 )?;
328
329 if instrument.is_some() && instrument_id.symbol != symbol {
330 let date = time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(
331 msg.ts_recv,
332 ))?
333 .date();
334 self.futures_symbols.insert(
335 (msg.hd.publisher_id, msg.hd.instrument_id, date),
336 instrument_id.symbol,
337 );
338 }
339
340 Ok(instrument)
341 })();
342
343 match result {
344 Ok(Some(item)) => return Some(Ok(item)),
345 Ok(None) => {}
346 Err(e) => return Some(Err(e)),
347 }
348 }
349 }))
350 }
351
352 pub fn read_records<T>(
358 &self,
359 filepath: &Path,
360 instrument_id: Option<InstrumentId>,
361 price_precision: Option<u8>,
362 include_trades: bool,
363 bars_timestamp_on_close: Option<bool>,
364 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<(Option<Data>, Option<Data>)>> + '_>
365 where
366 T: dbn::Record + dbn::HasRType + 'static,
367 {
368 let decoder = Decoder::from_zstd_file(filepath)?;
369 let mut metadata_cache = if instrument_id.is_none() {
370 Some(MetadataCache::new(decoder.metadata().clone()))
371 } else {
372 None
373 };
374 let mut dbn_stream = decoder.decode_stream::<T>();
375 let fixed_instrument_id = instrument_id.is_some();
376 let mut fixed_price_precision = price_precision;
377
378 Ok(std::iter::from_fn(move || {
379 let result: anyhow::Result<Option<(Option<Data>, Option<Data>)>> = (|| {
380 dbn_stream
381 .advance()
382 .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"))?;
383
384 if let Some(rec) = dbn_stream.get() {
385 let record = dbn::RecordRef::from(rec);
386 let instrument_id = self
387 .resolve_record_instrument_id(&record, instrument_id, &mut metadata_cache)
388 .context("failed to decode instrument id")?;
389 let resolved_precision = self.resolve_stream_price_precision(
390 &instrument_id,
391 fixed_instrument_id,
392 &mut fixed_price_precision,
393 )?;
394 let (item1, item2) = decode_record(
395 &record,
396 instrument_id,
397 resolved_precision,
398 None,
399 include_trades,
400 bars_timestamp_on_close.unwrap_or(true),
401 )?;
402 Ok(Some((item1, item2)))
403 } else {
404 Ok(None)
405 }
406 })();
407
408 match result {
409 Ok(Some(v)) => Some(Ok(v)),
410 Ok(None) => None,
411 Err(e) => Some(Err(e)),
412 }
413 }))
414 }
415
416 pub fn load_instruments(
425 &mut self,
426 filepath: &Path,
427 use_exchange_as_venue: bool,
428 skip_on_error: bool,
429 decode_config: Option<&DatabentoDecodeConfig>,
430 ) -> anyhow::Result<Vec<InstrumentAny>> {
431 let instruments = if skip_on_error {
432 let mut collected = Vec::new();
433
434 for result in
435 self.read_definition_records(filepath, use_exchange_as_venue, decode_config)?
436 {
437 match result {
438 Ok(instrument) => collected.push(instrument),
439 Err(e) => log::warn!("Skipping instrument: {e}"),
440 }
441 }
442 collected
443 } else {
444 self.read_definition_records(filepath, use_exchange_as_venue, decode_config)?
445 .collect::<Result<Vec<_>, _>>()?
446 };
447
448 for instrument in &instruments {
449 self.price_precisions
450 .insert(instrument.id().symbol, instrument.price_precision());
451 }
452
453 Ok(instruments)
454 }
455
456 pub fn load_order_book_deltas(
464 &self,
465 filepath: &Path,
466 instrument_id: Option<InstrumentId>,
467 price_precision: Option<u8>,
468 ) -> anyhow::Result<Vec<OrderBookDelta>> {
469 self.read_order_book_deltas(filepath, instrument_id, price_precision)?
470 .collect()
471 }
472
473 pub fn read_order_book_deltas(
481 &self,
482 filepath: &Path,
483 instrument_id: Option<InstrumentId>,
484 price_precision: Option<u8>,
485 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<OrderBookDelta>> + '_> {
486 let decoder = Decoder::from_zstd_file(filepath)?;
487 let mut metadata_cache = if instrument_id.is_none() {
488 Some(MetadataCache::new(decoder.metadata().clone()))
489 } else {
490 None
491 };
492 let mut dbn_stream = decoder.decode_stream::<dbn::MboMsg>();
493 let fixed_instrument_id = instrument_id.is_some();
494 let mut fixed_price_precision = price_precision;
495 let mut delta_buffer = MboDeltaBuffer::default();
496 let mut terminal_error = None;
497 let mut finished = false;
498
499 Ok(std::iter::from_fn(move || {
500 loop {
501 if let Some(delta) = delta_buffer.pop_ready() {
502 return Some(Ok(delta));
503 }
504
505 if finished {
506 return terminal_error.take().map(Err);
507 }
508
509 let result: anyhow::Result<bool> = (|| {
510 dbn_stream
511 .advance()
512 .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"))?;
513
514 let Some(rec) = dbn_stream.get() else {
515 return Ok(false);
516 };
517 let record = dbn::RecordRef::from(rec);
518 let instrument_id = self
519 .resolve_record_instrument_id(&record, instrument_id, &mut metadata_cache)
520 .context("failed to decode instrument id")?;
521 let resolved_precision = self.resolve_stream_price_precision(
522 &instrument_id,
523 fixed_instrument_id,
524 &mut fixed_price_precision,
525 )?;
526 let msg = record
527 .get::<dbn::MboMsg>()
528 .ok_or_else(|| anyhow::anyhow!("Failed to decode MboMsg"))?;
529 let (delta, _trade) =
530 decode_mbo_msg(msg, instrument_id, resolved_precision, None, false)?;
531 delta_buffer.push(msg, instrument_id, delta);
532 Ok(true)
533 })();
534
535 match result {
536 Ok(true) => {}
537 Ok(false) => {
538 delta_buffer.finish();
539 finished = true;
540 }
541 Err(e) => {
542 delta_buffer.finish();
543 terminal_error = Some(e);
544 finished = true;
545 }
546 }
547 }
548 }))
549 }
550
551 pub fn load_order_book_depth(
557 &self,
558 filepath: &Path,
559 instrument_id: Option<InstrumentId>,
560 price_precision: Option<u8>,
561 ) -> anyhow::Result<Vec<OrderBookDepth>> {
562 self.read_records::<dbn::Mbp10Msg>(filepath, instrument_id, price_precision, false, None)?
563 .filter_map(|result| match result {
564 Ok((Some(item1), _)) => {
565 if let Data::BookDepth(depth) = item1 {
566 Some(Ok(*depth))
567 } else {
568 None
569 }
570 }
571 Ok((None, _)) => None,
572 Err(e) => Some(Err(e)),
573 })
574 .collect()
575 }
576
577 pub fn load_quotes(
583 &self,
584 filepath: &Path,
585 instrument_id: Option<InstrumentId>,
586 price_precision: Option<u8>,
587 ) -> anyhow::Result<Vec<QuoteTick>> {
588 self.read_records::<dbn::Mbp1Msg>(filepath, instrument_id, price_precision, false, None)?
589 .filter_map(|result| match result {
590 Ok((Some(item1), _)) => {
591 if let Data::Quote(quote) = item1 {
592 Some(Ok(quote))
593 } else {
594 None
595 }
596 }
597 Ok((None, _)) => None,
598 Err(e) => Some(Err(e)),
599 })
600 .collect()
601 }
602
603 pub fn load_bbo_quotes(
609 &self,
610 filepath: &Path,
611 instrument_id: Option<InstrumentId>,
612 price_precision: Option<u8>,
613 ) -> anyhow::Result<Vec<QuoteTick>> {
614 self.read_records::<dbn::BboMsg>(filepath, instrument_id, price_precision, false, None)?
615 .filter_map(|result| match result {
616 Ok((Some(item1), _)) => {
617 if let Data::Quote(quote) = item1 {
618 Some(Ok(quote))
619 } else {
620 None
621 }
622 }
623 Ok((None, _)) => None,
624 Err(e) => Some(Err(e)),
625 })
626 .collect()
627 }
628
629 pub fn load_cmbp_quotes(
635 &self,
636 filepath: &Path,
637 instrument_id: Option<InstrumentId>,
638 price_precision: Option<u8>,
639 ) -> anyhow::Result<Vec<QuoteTick>> {
640 self.read_records::<dbn::Cmbp1Msg>(filepath, instrument_id, price_precision, false, None)?
641 .filter_map(|result| match result {
642 Ok((Some(item1), _)) => {
643 if let Data::Quote(quote) = item1 {
644 Some(Ok(quote))
645 } else {
646 None
647 }
648 }
649 Ok((None, _)) => None,
650 Err(e) => Some(Err(e)),
651 })
652 .collect()
653 }
654
655 pub fn load_cbbo_quotes(
661 &self,
662 filepath: &Path,
663 instrument_id: Option<InstrumentId>,
664 price_precision: Option<u8>,
665 ) -> anyhow::Result<Vec<QuoteTick>> {
666 self.read_records::<dbn::CbboMsg>(filepath, instrument_id, price_precision, false, None)?
667 .filter_map(|result| match result {
668 Ok((Some(item1), _)) => {
669 if let Data::Quote(quote) = item1 {
670 Some(Ok(quote))
671 } else {
672 None
673 }
674 }
675 Ok((None, _)) => None,
676 Err(e) => Some(Err(e)),
677 })
678 .collect()
679 }
680
681 pub fn load_tbbo_trades(
687 &self,
688 filepath: &Path,
689 instrument_id: Option<InstrumentId>,
690 price_precision: Option<u8>,
691 ) -> anyhow::Result<Vec<TradeTick>> {
692 self.read_records::<dbn::TbboMsg>(filepath, instrument_id, price_precision, true, None)?
693 .filter_map(|result| match result {
694 Ok((_, maybe_item2)) => {
695 if let Some(Data::Trade(trade)) = maybe_item2 {
696 Some(Ok(trade))
697 } else {
698 None
699 }
700 }
701 Err(e) => Some(Err(e)),
702 })
703 .collect()
704 }
705
706 pub fn load_tcbbo_trades(
712 &self,
713 filepath: &Path,
714 instrument_id: Option<InstrumentId>,
715 price_precision: Option<u8>,
716 ) -> anyhow::Result<Vec<TradeTick>> {
717 self.read_records::<dbn::TcbboMsg>(filepath, instrument_id, price_precision, true, None)?
718 .filter_map(|result| match result {
719 Ok((_, maybe_item2)) => {
720 if let Some(Data::Trade(trade)) = maybe_item2 {
721 Some(Ok(trade))
722 } else {
723 None
724 }
725 }
726 Err(e) => Some(Err(e)),
727 })
728 .collect()
729 }
730
731 pub fn load_trades(
737 &self,
738 filepath: &Path,
739 instrument_id: Option<InstrumentId>,
740 price_precision: Option<u8>,
741 ) -> anyhow::Result<Vec<TradeTick>> {
742 self.read_records::<dbn::TradeMsg>(filepath, instrument_id, price_precision, false, None)?
743 .filter_map(|result| match result {
744 Ok((Some(item1), _)) => {
745 if let Data::Trade(trade) = item1 {
746 Some(Ok(trade))
747 } else {
748 None
749 }
750 }
751 Ok((None, _)) => None,
752 Err(e) => Some(Err(e)),
753 })
754 .collect()
755 }
756
757 pub fn load_bars(
763 &self,
764 filepath: &Path,
765 instrument_id: Option<InstrumentId>,
766 price_precision: Option<u8>,
767 timestamp_on_close: Option<bool>,
768 ) -> anyhow::Result<Vec<Bar>> {
769 self.read_records::<dbn::OhlcvMsg>(
770 filepath,
771 instrument_id,
772 price_precision,
773 false,
774 timestamp_on_close,
775 )?
776 .filter_map(|result| match result {
777 Ok((Some(item1), _)) => {
778 if let Data::Bar(bar) = item1 {
779 Some(Ok(bar))
780 } else {
781 None
782 }
783 }
784 Ok((None, _)) => None,
785 Err(e) => Some(Err(e)),
786 })
787 .collect()
788 }
789
790 pub fn load_status_records<T>(
796 &self,
797 filepath: &Path,
798 instrument_id: Option<InstrumentId>,
799 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<InstrumentStatus>> + '_>
800 where
801 T: dbn::Record + dbn::HasRType + 'static,
802 {
803 let decoder = Decoder::from_zstd_file(filepath)?;
804 let mut metadata_cache = if instrument_id.is_none() {
805 Some(MetadataCache::new(decoder.metadata().clone()))
806 } else {
807 None
808 };
809 let mut dbn_stream = decoder.decode_stream::<T>();
810
811 Ok(std::iter::from_fn(move || {
812 if let Err(e) = dbn_stream.advance() {
813 return Some(Err(e.into()));
814 }
815
816 match dbn_stream.get() {
817 Some(rec) => {
818 let record = dbn::RecordRef::from(rec);
819 let instrument_id = match self.resolve_record_instrument_id(
820 &record,
821 instrument_id,
822 &mut metadata_cache,
823 ) {
824 Ok(id) => id,
825 Err(e) => return Some(Err(e)),
826 };
827
828 let msg = match record.get::<dbn::StatusMsg>() {
829 Some(m) => m,
830 None => return Some(Err(anyhow::anyhow!("Invalid `StatusMsg`"))),
831 };
832 let ts_init = msg.ts_recv.into();
833
834 match decode_status_msg(msg, instrument_id, Some(ts_init)) {
835 Ok(data) => Some(Ok(data)),
836 Err(e) => Some(Err(e)),
837 }
838 }
839 None => None,
840 }
841 }))
842 }
843
844 pub fn read_imbalance_records<T>(
850 &self,
851 filepath: &Path,
852 instrument_id: Option<InstrumentId>,
853 price_precision: Option<u8>,
854 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<DatabentoImbalance>> + '_>
855 where
856 T: dbn::Record + dbn::HasRType + 'static,
857 {
858 let decoder = Decoder::from_zstd_file(filepath)?;
859 let mut metadata_cache = if instrument_id.is_none() {
860 Some(MetadataCache::new(decoder.metadata().clone()))
861 } else {
862 None
863 };
864 let mut dbn_stream = decoder.decode_stream::<T>();
865 let fixed_instrument_id = instrument_id.is_some();
866 let mut fixed_price_precision = price_precision;
867
868 Ok(std::iter::from_fn(move || {
869 if let Err(e) = dbn_stream.advance() {
870 return Some(Err(e.into()));
871 }
872
873 match dbn_stream.get() {
874 Some(rec) => {
875 let record = dbn::RecordRef::from(rec);
876 let instrument_id = match self.resolve_record_instrument_id(
877 &record,
878 instrument_id,
879 &mut metadata_cache,
880 ) {
881 Ok(id) => id,
882 Err(e) => return Some(Err(e)),
883 };
884 let resolved_precision = match self.resolve_stream_price_precision(
885 &instrument_id,
886 fixed_instrument_id,
887 &mut fixed_price_precision,
888 ) {
889 Ok(p) => p,
890 Err(e) => return Some(Err(e)),
891 };
892
893 let msg = match record.get::<dbn::ImbalanceMsg>() {
894 Some(m) => m,
895 None => return Some(Err(anyhow::anyhow!("Invalid `ImbalanceMsg`"))),
896 };
897 let ts_init = msg.ts_recv.into();
898
899 match decode_imbalance_msg(
900 msg,
901 instrument_id,
902 resolved_precision,
903 Some(ts_init),
904 ) {
905 Ok(data) => Some(Ok(data)),
906 Err(e) => Some(Err(e)),
907 }
908 }
909 None => None,
910 }
911 }))
912 }
913
914 pub fn read_statistics_records<T>(
920 &self,
921 filepath: &Path,
922 instrument_id: Option<InstrumentId>,
923 price_precision: Option<u8>,
924 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<DatabentoStatistics>> + '_>
925 where
926 T: dbn::Record + dbn::HasRType + 'static,
927 {
928 let decoder = Decoder::from_zstd_file(filepath)?;
929 let mut metadata_cache = if instrument_id.is_none() {
930 Some(MetadataCache::new(decoder.metadata().clone()))
931 } else {
932 None
933 };
934 let mut dbn_stream = decoder.decode_stream::<T>();
935 let fixed_instrument_id = instrument_id.is_some();
936 let mut fixed_price_precision = price_precision;
937
938 Ok(std::iter::from_fn(move || {
941 loop {
942 if let Err(e) = dbn_stream.advance() {
943 return Some(Err(e.into()));
944 }
945
946 let rec = dbn_stream.get()?;
947 let record = dbn::RecordRef::from(rec);
948 let msg = match record.get::<dbn::StatMsg>() {
949 Some(m) => m,
950 None => return Some(Err(anyhow::anyhow!("Invalid `StatMsg`"))),
951 };
952
953 if !is_supported_stat_type(msg.stat_type) {
954 log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
955 continue;
956 }
957
958 let instrument_id = match self.resolve_record_instrument_id(
959 &record,
960 instrument_id,
961 &mut metadata_cache,
962 ) {
963 Ok(id) => id,
964 Err(e) => return Some(Err(e)),
965 };
966 let resolved_precision = match self.resolve_stream_price_precision(
967 &instrument_id,
968 fixed_instrument_id,
969 &mut fixed_price_precision,
970 ) {
971 Ok(p) => p,
972 Err(e) => return Some(Err(e)),
973 };
974 let ts_init = msg.ts_recv.into();
975
976 match decode_statistics_msg(msg, instrument_id, resolved_precision, Some(ts_init)) {
977 Ok(Some(data)) => return Some(Ok(data)),
978 Ok(None) => {}
979 Err(e) => return Some(Err(e)),
980 }
981 }
982 }))
983 }
984
985 fn resolve_record_instrument_id(
986 &self,
987 record: &dbn::RecordRef,
988 instrument_id: Option<InstrumentId>,
989 metadata_cache: &mut Option<MetadataCache>,
990 ) -> anyhow::Result<InstrumentId> {
991 if let Some(instrument_id) = instrument_id {
992 return Ok(instrument_id);
993 }
994
995 let metadata_cache = metadata_cache
996 .as_mut()
997 .ok_or_else(|| anyhow::anyhow!("missing metadata cache for dynamic instrument id"))?;
998
999 decode_nautilus_instrument_id(
1000 record,
1001 metadata_cache,
1002 &self.publisher_venue_map,
1003 &self.symbol_venue_map,
1004 self.futures_year_digits,
1005 &self.futures_symbols,
1006 None,
1007 )
1008 }
1009
1010 fn resolve_stream_price_precision(
1011 &self,
1012 instrument_id: &InstrumentId,
1013 fixed_instrument_id: bool,
1014 fixed_price_precision: &mut Option<u8>,
1015 ) -> anyhow::Result<u8> {
1016 if let Some(precision) = *fixed_price_precision {
1017 return Ok(precision);
1018 }
1019
1020 let precision = self.resolve_price_precision(instrument_id, None)?;
1021 if fixed_instrument_id {
1022 *fixed_price_precision = Some(precision);
1023 }
1024
1025 Ok(precision)
1026 }
1027}
1028
1029fn apply_default_venue_dataset_mappings(venue_dataset_map: &mut IndexMap<Venue, Dataset>) {
1033 let glbx = Dataset::from("GLBX.MDP3");
1034
1035 for venue in [
1036 Venue::CBCM(),
1037 Venue::GLBX(),
1038 Venue::NYUM(),
1039 Venue::XCBT(),
1040 Venue::XCEC(),
1041 Venue::XCME(),
1042 Venue::XFXS(),
1043 Venue::XNYM(),
1044 ] {
1045 _ = venue_dataset_map.insert(venue, glbx);
1046 }
1047
1048 _ = venue_dataset_map.insert(Venue::from("EQUS"), Dataset::from("EQUS.MINI"));
1051
1052 let opra = Dataset::from("OPRA.PILLAR");
1053 for venue_code in [
1054 "AMXO", "XBOX", "XCBO", "EMLD", "EDGO", "GMNI", "XISX", "MCRY", "XMIO", "ARCO", "OPRA",
1055 "MPRL", "XNDQ", "XBXO", "C2OX", "XPHL", "BATO", "MXOP", "SPHR",
1056 ] {
1057 _ = venue_dataset_map.insert(Venue::from(venue_code), opra);
1058 }
1059}
1060
1061#[cfg(test)]
1062mod tests {
1063 use std::{
1064 ffi::c_char,
1065 fs::{File, OpenOptions},
1066 io::Write,
1067 path::{Path, PathBuf},
1068 process,
1069 sync::atomic::{AtomicU64, Ordering},
1070 };
1071
1072 use databento::dbn::{decode::DecodeRecord, encode::EncodeRecord};
1073 use nautilus_model::{
1074 enums::BookAction,
1075 types::{Price, Quantity},
1076 };
1077 use rstest::{fixture, rstest};
1078 use ustr::Ustr;
1079
1080 use super::*;
1081
1082 fn test_data_path() -> PathBuf {
1083 Path::new(env!("CARGO_MANIFEST_DIR")).join("test_data")
1084 }
1085
1086 fn mbo_record(action: c_char, flags: u8, sequence: u32) -> dbn::MboMsg {
1087 let ts_event = 1_609_160_400_000_000_000;
1088 dbn::MboMsg {
1089 hd: dbn::RecordHeader::new::<dbn::MboMsg>(dbn::rtype::MBO, 1, 1, ts_event),
1090 order_id: 42,
1091 price: 4_800_250_000_000,
1092 size: 2,
1093 flags: dbn::FlagSet::new(flags),
1094 channel_id: 1,
1095 action,
1096 side: 'A' as c_char,
1097 ts_recv: ts_event,
1098 ts_in_delta: 0,
1099 sequence,
1100 }
1101 }
1102
1103 fn write_mbo_records(records: &[dbn::MboMsg]) -> PathBuf {
1104 static FILE_ID: AtomicU64 = AtomicU64::new(0);
1105
1106 let id = FILE_ID.fetch_add(1, Ordering::Relaxed);
1107 let path = std::env::temp_dir().join(format!(
1108 "nautilus-databento-mbo-{}-{id}.dbn.zst",
1109 process::id(),
1110 ));
1111 let metadata = dbn::Metadata::builder()
1112 .dataset("GLBX.MDP3")
1113 .schema(Some(dbn::Schema::Mbo))
1114 .start(0)
1115 .stype_in(Some(dbn::SType::InstrumentId))
1116 .stype_out(dbn::SType::InstrumentId)
1117 .build();
1118 let file = File::create(&path).unwrap();
1119 let mut encoder = dbn::encode::dbn::Encoder::with_zstd(file, &metadata).unwrap();
1120 encoder.encode_records(records).unwrap();
1121 encoder.flush().unwrap();
1122 drop(encoder);
1123
1124 path
1125 }
1126
1127 #[fixture]
1128 fn loader() -> DatabentoDataLoader {
1129 let publishers_filepath = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1130 let mut loader = DatabentoDataLoader::new(Some(publishers_filepath)).unwrap();
1131 loader.set_price_precision(Symbol::from("ESM4"), 2);
1133 loader
1134 }
1135
1136 #[fixture]
1137 fn loader_without_seed() -> DatabentoDataLoader {
1138 let publishers_filepath = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1139 DatabentoDataLoader::new(Some(publishers_filepath)).unwrap()
1140 }
1141
1142 #[rstest]
1145 fn test_set_dataset_venue_mapping(mut loader: DatabentoDataLoader) {
1146 let dataset = Ustr::from("EQUS.PLUS");
1147 let venue = Venue::from("XNAS");
1148 loader.set_dataset_for_venue(dataset, venue);
1149
1150 let result = loader.get_dataset_for_venue(&venue).unwrap();
1151 assert_eq!(*result, dataset);
1152 }
1153
1154 #[rstest]
1155 fn test_default_venue_dataset_mappings(loader: DatabentoDataLoader) {
1156 let xcme = Venue::XCME();
1157 let result = loader.get_dataset_for_venue(&xcme).unwrap();
1158 assert_eq!(*result, Ustr::from("GLBX.MDP3"));
1159
1160 let xcbo = Venue::from("XCBO");
1161 let result = loader.get_dataset_for_venue(&xcbo).unwrap();
1162 assert_eq!(*result, Ustr::from("OPRA.PILLAR"));
1163
1164 let equs = Venue::from("EQUS");
1165 let result = loader.get_dataset_for_venue(&equs).unwrap();
1166 assert_eq!(*result, Ustr::from("EQUS.MINI"));
1167 }
1168
1169 #[rstest]
1170 #[case(test_data_path().join("test_data.definition.equity.dbn.zst"))]
1171 fn test_load_instruments(mut loader: DatabentoDataLoader, #[case] path: PathBuf) {
1172 let instruments = loader.load_instruments(&path, false, false, None).unwrap();
1173
1174 assert_eq!(instruments.len(), 2);
1175 assert_eq!(
1177 loader.get_price_precisions().get(&Symbol::from("ESM4")),
1178 Some(&2)
1179 );
1180 }
1181
1182 #[rstest]
1183 fn test_load_instruments_populates_price_precisions_cache(
1184 mut loader_without_seed: DatabentoDataLoader,
1185 ) {
1186 let path = test_data_path().join("test_data.definition.equity.dbn.zst");
1187 assert!(loader_without_seed.get_price_precisions().is_empty());
1188
1189 let instruments = loader_without_seed
1190 .load_instruments(&path, false, false, None)
1191 .unwrap();
1192
1193 assert_eq!(instruments.len(), 2);
1194 for instrument in &instruments {
1195 let symbol = instrument.id().symbol;
1196 assert_eq!(
1197 loader_without_seed.get_price_precisions().get(&symbol),
1198 Some(&instrument.price_precision()),
1199 "cache missing or mismatched entry for {symbol}",
1200 );
1201 }
1202 }
1203
1204 #[rstest]
1205 fn test_read_records_errors_when_precision_unresolvable(
1206 loader_without_seed: DatabentoDataLoader,
1207 ) {
1208 let path = test_data_path().join("test_data.mbo.dbn.zst");
1209 let instrument_id = InstrumentId::from("ESM4.GLBX");
1210
1211 let result = loader_without_seed.load_order_book_deltas(&path, Some(instrument_id), None);
1212
1213 let err = result.expect_err("expected precision-resolution error");
1214 let err_msg = format!("{err}");
1215 assert!(
1216 err_msg.contains("Could not resolve `price_precision`"),
1217 "unexpected error message: {err_msg}",
1218 );
1219 assert!(
1220 err_msg.contains("ESM4.GLBX"),
1221 "error should name the instrument: {err_msg}",
1222 );
1223 }
1224
1225 #[rstest]
1226 fn test_set_price_precision_unblocks_reads(mut loader_without_seed: DatabentoDataLoader) {
1227 let path = test_data_path().join("test_data.mbo.dbn.zst");
1228 let instrument_id = InstrumentId::from("ESM4.GLBX");
1229
1230 assert!(
1232 loader_without_seed
1233 .load_order_book_deltas(&path, Some(instrument_id), None)
1234 .is_err()
1235 );
1236
1237 loader_without_seed.set_price_precision(Symbol::from("ESM4"), 2);
1238
1239 let deltas = loader_without_seed
1240 .load_order_book_deltas(&path, Some(instrument_id), None)
1241 .unwrap();
1242 assert_eq!(deltas.len(), 2);
1243 }
1244
1245 #[rstest]
1246 fn test_resolve_price_precision_explicit_arg_overrides_cache(
1247 mut loader_without_seed: DatabentoDataLoader,
1248 ) {
1249 let instrument_id = InstrumentId::from("ESM4.GLBX");
1250 loader_without_seed.set_price_precision(Symbol::from("ESM4"), 9);
1252
1253 let explicit = loader_without_seed
1254 .resolve_price_precision(&instrument_id, Some(2))
1255 .unwrap();
1256 assert_eq!(explicit, 2);
1257
1258 let cached = loader_without_seed
1259 .resolve_price_precision(&instrument_id, None)
1260 .unwrap();
1261 assert_eq!(cached, 9);
1262 }
1263
1264 #[rstest]
1265 fn test_resolve_price_precision_cache_miss_errors(loader_without_seed: DatabentoDataLoader) {
1266 let instrument_id = InstrumentId::from("ESM4.GLBX");
1267
1268 let err = loader_without_seed
1269 .resolve_price_precision(&instrument_id, None)
1270 .expect_err("expected cache-miss error");
1271 assert!(format!("{err}").contains("Could not resolve `price_precision`"));
1272 }
1273
1274 #[rstest]
1275 fn test_load_order_book_deltas(loader: DatabentoDataLoader) {
1276 let path = test_data_path().join("test_data.mbo.dbn.zst");
1277 let instrument_id = InstrumentId::from("ESM4.GLBX");
1278
1279 let deltas = loader
1280 .load_order_book_deltas(&path, Some(instrument_id), None)
1281 .unwrap();
1282
1283 assert_eq!(deltas.len(), 2);
1284 }
1285
1286 #[rstest]
1287 fn test_read_order_book_deltas_streams_without_collecting(loader: DatabentoDataLoader) {
1288 let path = test_data_path().join("test_data.mbo.dbn.zst");
1289 let instrument_id = InstrumentId::from("ESM4.GLBX");
1290
1291 let count = loader
1292 .read_order_book_deltas(&path, Some(instrument_id), None)
1293 .unwrap()
1294 .map(|result| result.map(|_| 1usize))
1295 .sum::<anyhow::Result<usize>>()
1296 .unwrap();
1297
1298 assert_eq!(count, 2);
1299 }
1300
1301 #[rstest]
1302 #[case::standalone(false)]
1303 #[case::legacy_inline(true)]
1304 fn test_load_order_book_deltas_preserves_event_boundary(
1305 loader: DatabentoDataLoader,
1306 #[case] inline_last: bool,
1307 ) {
1308 let instrument_id = InstrumentId::from("ESM4.GLBX");
1309 let last = dbn::flags::LAST;
1310 let mut records = vec![mbo_record(
1311 'C' as c_char,
1312 if inline_last { last } else { 0 },
1313 1,
1314 )];
1315
1316 if !inline_last {
1317 records.push(mbo_record('N' as c_char, last, 2));
1318 }
1319 let path = write_mbo_records(&records);
1320
1321 let deltas = loader
1322 .load_order_book_deltas(&path, Some(instrument_id), Some(2))
1323 .unwrap();
1324 std::fs::remove_file(path).unwrap();
1325
1326 assert_eq!(deltas.len(), 1);
1327 assert_eq!(deltas[0].instrument_id, instrument_id);
1328 assert_eq!(deltas[0].action, BookAction::Delete);
1329 assert_eq!(deltas[0].order.order_id, 42);
1330 assert_eq!(deltas[0].flags, last);
1331 assert_eq!(deltas[0].sequence, 1);
1332 }
1333
1334 #[rstest]
1335 fn test_read_order_book_deltas_drains_before_terminal_error(loader: DatabentoDataLoader) {
1336 let instrument_id = InstrumentId::from("ESM4.GLBX");
1337 let path = write_mbo_records(&[mbo_record('C' as c_char, 0, 1)]);
1338 OpenOptions::new()
1339 .append(true)
1340 .open(&path)
1341 .unwrap()
1342 .write_all(&[0; 8])
1343 .unwrap();
1344 let mut deltas = loader
1345 .read_order_book_deltas(&path, Some(instrument_id), Some(2))
1346 .unwrap();
1347
1348 let delta = deltas.next().unwrap().unwrap();
1349 let error = deltas
1350 .next()
1351 .unwrap()
1352 .expect_err("expected trailing decode error");
1353 let end = deltas.next();
1354 std::fs::remove_file(path).unwrap();
1355
1356 assert_eq!(delta.instrument_id, instrument_id);
1357 assert_eq!(delta.order.order_id, 42);
1358 assert_eq!(delta.flags, 0);
1359 assert!(format!("{error}").contains("Stream advance error"));
1360 assert!(end.is_none());
1361 }
1362
1363 #[rstest]
1364 fn test_load_order_book_depth(loader: DatabentoDataLoader) {
1365 let path = test_data_path().join("test_data.mbp-10.dbn.zst");
1366 let instrument_id = InstrumentId::from("ESM4.GLBX");
1367
1368 let depths = loader
1369 .load_order_book_depth(&path, Some(instrument_id), None)
1370 .unwrap();
1371
1372 assert_eq!(depths.len(), 2);
1373 }
1374
1375 #[rstest]
1376 fn test_load_quotes(loader: DatabentoDataLoader) {
1377 let path = test_data_path().join("test_data.mbp-1.dbn.zst");
1378 let instrument_id = InstrumentId::from("ESM4.GLBX");
1379
1380 let quotes = loader
1381 .load_quotes(&path, Some(instrument_id), None)
1382 .unwrap();
1383
1384 assert_eq!(quotes.len(), 2);
1385 }
1386
1387 #[rstest]
1388 #[case(test_data_path().join("test_data.bbo-1s.dbn.zst"))]
1389 #[case(test_data_path().join("test_data.bbo-1m.dbn.zst"))]
1390 fn test_load_bbo_quotes(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1391 let instrument_id = InstrumentId::from("ESM4.GLBX");
1392
1393 let quotes = loader
1394 .load_bbo_quotes(&path, Some(instrument_id), None)
1395 .unwrap();
1396
1397 assert_eq!(quotes.len(), 4);
1398 }
1399
1400 #[rstest]
1401 fn test_load_cmbp_quotes(loader: DatabentoDataLoader) {
1402 let path = test_data_path().join("test_data.cmbp-1.dbn.zst");
1403 let instrument_id = InstrumentId::from("ESM4.GLBX");
1404
1405 let quotes = loader
1406 .load_cmbp_quotes(&path, Some(instrument_id), None)
1407 .unwrap();
1408
1409 assert_eq!(quotes.len(), 2);
1411
1412 let first_quote = "es[0];
1414 assert_eq!(first_quote.instrument_id, instrument_id);
1415 assert_eq!(first_quote.bid_price, Price::from("3720.25"));
1416 assert_eq!(first_quote.ask_price, Price::from("3720.50"));
1417 assert_eq!(first_quote.bid_size, Quantity::from(24));
1418 assert_eq!(first_quote.ask_size, Quantity::from(11));
1419 assert_eq!(first_quote.ts_event, 1609160400006136329);
1420 assert_eq!(first_quote.ts_init, 1609160400006136329);
1421 }
1422
1423 #[rstest]
1424 fn test_load_cbbo_quotes(loader: DatabentoDataLoader) {
1425 let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1426 let instrument_id = InstrumentId::from("ESM4.GLBX");
1427
1428 let quotes = loader
1429 .load_cbbo_quotes(&path, Some(instrument_id), None)
1430 .unwrap();
1431
1432 assert_eq!(quotes.len(), 2);
1434
1435 let first_quote = "es[0];
1437 assert_eq!(first_quote.instrument_id, instrument_id);
1438 assert_eq!(first_quote.bid_price, Price::from("3720.25"));
1439 assert_eq!(first_quote.ask_price, Price::from("3720.50"));
1440 assert_eq!(first_quote.bid_size, Quantity::from(24));
1441 assert_eq!(first_quote.ask_size, Quantity::from(11));
1442 assert_eq!(first_quote.ts_event, 1609160400006136329);
1443 assert_eq!(first_quote.ts_init, 1609160400006136329);
1444 }
1445
1446 #[rstest]
1447 #[case(1, "ESH1.GLBX")]
1448 #[case(2, "ESH21.GLBX")]
1449 fn test_load_cbbo_quotes_resolves_instrument_id(
1450 loader: DatabentoDataLoader,
1451 #[case] digits: u8,
1452 #[case] expected: InstrumentId,
1453 ) {
1454 let loader = loader.with_futures_year_digits(digits.try_into().unwrap());
1455 let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1456 let quotes = loader.load_cbbo_quotes(&path, None, Some(2)).unwrap();
1457
1458 assert_eq!(quotes.len(), 2);
1459 assert_eq!(quotes[0].instrument_id, expected);
1460 assert_eq!(quotes[1].instrument_id, expected);
1461 assert_eq!(quotes[0].bid_price, Price::from("3720.25"));
1462 assert_eq!(quotes[0].ask_price, Price::from("3720.50"));
1463 assert_eq!(quotes[0].bid_size, Quantity::from(24));
1464 assert_eq!(quotes[0].ask_size, Quantity::from(11));
1465 assert_eq!(quotes[0].ts_event, 1609160400006136329);
1466 assert_eq!(quotes[0].ts_init, 1609160400006136329);
1467 }
1468
1469 #[rstest]
1470 fn test_load_tbbo_trades(loader: DatabentoDataLoader) {
1471 let path = test_data_path().join("test_data.tbbo.dbn.zst");
1472 let instrument_id = InstrumentId::from("ESM4.GLBX");
1473
1474 let trades = loader
1475 .load_tbbo_trades(&path, Some(instrument_id), None)
1476 .unwrap();
1477
1478 assert_eq!(trades.len(), 2);
1479 assert_eq!(trades[0].instrument_id, instrument_id);
1480 assert_eq!(trades[0].price, Price::from("3720.25"));
1481 assert_eq!(trades[0].size, Quantity::from("5"));
1482 }
1483
1484 #[rstest]
1485 fn test_load_tcbbo_trades_rejects_cbbo_fixture(loader: DatabentoDataLoader) {
1486 let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1487 let instrument_id = InstrumentId::from("ESM4.GLBX");
1488
1489 let result = loader.load_tcbbo_trades(&path, Some(instrument_id), None);
1490
1491 assert!(result.is_err());
1492 }
1493
1494 #[rstest]
1495 fn test_load_trades(loader: DatabentoDataLoader) {
1496 let path = test_data_path().join("test_data.trades.dbn.zst");
1497 let instrument_id = InstrumentId::from("ESM4.GLBX");
1498 let trades = loader
1499 .load_trades(&path, Some(instrument_id), None)
1500 .unwrap();
1501
1502 assert_eq!(trades.len(), 2);
1503 }
1504
1505 #[rstest]
1506 #[case(test_data_path().join("test_data.ohlcv-1h.dbn.zst"))]
1508 #[case(test_data_path().join("test_data.ohlcv-1m.dbn.zst"))]
1509 #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1510 fn test_load_bars(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1511 let instrument_id = InstrumentId::from("ESM4.GLBX");
1512 let bars = loader
1513 .load_bars(&path, Some(instrument_id), None, None)
1514 .unwrap();
1515
1516 assert_eq!(bars.len(), 2);
1517 }
1518
1519 #[rstest]
1520 #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1521 fn test_load_bars_timestamp_on_close_true(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1522 let instrument_id = InstrumentId::from("ESM4.GLBX");
1523 let bars = loader
1524 .load_bars(&path, Some(instrument_id), None, Some(true))
1525 .unwrap();
1526
1527 assert_eq!(bars.len(), 2);
1528
1529 for bar in &bars {
1531 assert_eq!(
1532 bar.ts_event, bar.ts_init,
1533 "ts_event and ts_init should both be close time when bars_timestamp_on_close=true"
1534 );
1535 }
1536 }
1537
1538 #[rstest]
1539 #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1540 fn test_load_bars_timestamp_on_close_false(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1541 let instrument_id = InstrumentId::from("ESM4.GLBX");
1542 let bars = loader
1543 .load_bars(&path, Some(instrument_id), None, Some(false))
1544 .unwrap();
1545
1546 assert_eq!(bars.len(), 2);
1547
1548 for bar in &bars {
1550 assert_ne!(
1551 bar.ts_event, bar.ts_init,
1552 "ts_event should be open time and ts_init should be close time when bars_timestamp_on_close=false"
1553 );
1554 assert_eq!(bar.ts_init.as_u64(), bar.ts_event.as_u64() + 1_000_000_000);
1556 }
1557 }
1558
1559 #[rstest]
1560 #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"), 0)]
1561 #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"), 1)]
1562 fn test_load_bars_timestamp_comparison(
1563 loader: DatabentoDataLoader,
1564 #[case] path: PathBuf,
1565 #[case] bar_index: usize,
1566 ) {
1567 const ONE_SECOND_NS: u64 = 1_000_000_000;
1568
1569 let instrument_id = InstrumentId::from("ESM4.GLBX");
1570
1571 let bars_close = loader
1572 .load_bars(&path, Some(instrument_id), None, Some(true))
1573 .unwrap();
1574
1575 let bars_open = loader
1576 .load_bars(&path, Some(instrument_id), None, Some(false))
1577 .unwrap();
1578
1579 assert_eq!(bars_close.len(), bars_open.len());
1580 assert_eq!(bars_close.len(), 2);
1581
1582 let bar_close = &bars_close[bar_index];
1583 let bar_open = &bars_open[bar_index];
1584
1585 assert_eq!(bar_close.open, bar_open.open);
1587 assert_eq!(bar_close.high, bar_open.high);
1588 assert_eq!(bar_close.low, bar_open.low);
1589 assert_eq!(bar_close.close, bar_open.close);
1590 assert_eq!(bar_close.volume, bar_open.volume);
1591
1592 assert!(
1595 bar_close.ts_event > bar_open.ts_event,
1596 "Close-timestamped bar should have later timestamp than open-timestamped bar"
1597 );
1598
1599 assert_eq!(
1601 bar_close.ts_event.as_u64() - bar_open.ts_event.as_u64(),
1602 ONE_SECOND_NS,
1603 "Timestamp difference should be exactly 1 second for 1s bars"
1604 );
1605 }
1606
1607 #[rstest]
1608 fn test_load_status_records(loader: DatabentoDataLoader) {
1609 let path = test_data_path().join("test_data.status.dbn.zst");
1610 let instrument_id = InstrumentId::from("ESM4.GLBX");
1611
1612 let statuses = loader
1613 .load_status_records::<dbn::StatusMsg>(&path, Some(instrument_id))
1614 .unwrap()
1615 .collect::<anyhow::Result<Vec<_>>>()
1616 .unwrap();
1617
1618 assert_eq!(statuses.len(), 4, "Should load exactly 4 status records");
1620
1621 let first = &statuses[0];
1623 assert_eq!(first.instrument_id, instrument_id);
1624 assert_eq!(first.ts_event.as_u64(), 1609110000000000000);
1625 assert_eq!(first.ts_init.as_u64(), 1609113600000000000);
1626 }
1627
1628 #[rstest]
1629 fn test_read_imbalance_records(loader: DatabentoDataLoader) {
1630 let path = test_data_path().join("test_data.imbalance.dbn.zst");
1631 let instrument_id = InstrumentId::from("ESM4.GLBX");
1632
1633 let imbalances = loader
1634 .read_imbalance_records::<dbn::ImbalanceMsg>(&path, Some(instrument_id), None)
1635 .unwrap()
1636 .collect::<anyhow::Result<Vec<_>>>()
1637 .unwrap();
1638
1639 assert_eq!(
1641 imbalances.len(),
1642 2,
1643 "Should load exactly 2 imbalance records"
1644 );
1645
1646 let first = &imbalances[0];
1648 assert_eq!(first.instrument_id, instrument_id);
1649 assert!(
1650 first.ref_price.as_f64() > 0.0,
1651 "ref_price should be positive"
1652 );
1653 assert!(first.ts_event.as_u64() > 0, "ts_event should be set");
1654 assert!(first.ts_recv.as_u64() > 0, "ts_recv should be set");
1655 assert!(first.ts_init.as_u64() > 0, "ts_init should be set");
1656 }
1657
1658 #[rstest]
1659 fn test_read_statistics_records(loader: DatabentoDataLoader) {
1660 let path = test_data_path().join("test_data.statistics.dbn.zst");
1661 let instrument_id = InstrumentId::from("ESM4.GLBX");
1662
1663 let statistics = loader
1664 .read_statistics_records::<dbn::StatMsg>(&path, Some(instrument_id), None)
1665 .unwrap()
1666 .collect::<anyhow::Result<Vec<_>>>()
1667 .unwrap();
1668
1669 assert_eq!(
1671 statistics.len(),
1672 2,
1673 "Should load exactly 2 statistics records"
1674 );
1675
1676 let first = &statistics[0];
1678 assert_eq!(first.instrument_id, instrument_id);
1679 assert!(first.ts_event.as_u64() > 0, "ts_event should be set");
1680 assert!(first.ts_recv.as_u64() > 0, "ts_recv should be set");
1681 assert!(first.ts_init.as_u64() > 0, "ts_init should be set");
1682 assert!(first.sequence > 0, "sequence should be positive");
1683 }
1684
1685 #[rstest]
1686 #[case("futures_contract", "ESU6", "ESU26.GLBX")]
1687 #[case("futures_spread", "ESU6-ESM7", "ESU26-ESM27.GLBX")]
1688 fn test_loader_defaults_to_two_digit_years(
1689 #[case] fixture: &str,
1690 #[case] raw: &str,
1691 #[case] expected: InstrumentId,
1692 ) {
1693 let publishers = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1694 let path = Path::new(env!("CARGO_MANIFEST_DIR"))
1695 .join(format!("test_data/test_data.definition.{fixture}.dbn.zst"));
1696 let mut loader = DatabentoDataLoader::new(Some(publishers.clone())).unwrap();
1697 let mut legacy = DatabentoDataLoader::new(Some(publishers))
1698 .unwrap()
1699 .with_futures_year_digits(FuturesYearDigits::One);
1700 let instruments = loader.load_instruments(&path, false, false, None).unwrap();
1701 let legacy_instruments = legacy.load_instruments(&path, false, false, None).unwrap();
1702 assert_eq!(instruments.len(), 1);
1703 assert_eq!(legacy_instruments.len(), 1);
1704 assert_eq!(instruments[0].id(), expected);
1705 assert_eq!(instruments[0].raw_symbol(), Symbol::from(raw));
1706 assert_eq!(
1707 legacy_instruments[0].id(),
1708 InstrumentId::new(Symbol::from(raw), Venue::from("GLBX"))
1709 );
1710 assert_eq!(legacy_instruments[0].raw_symbol(), Symbol::from(raw));
1711 }
1712
1713 #[rstest]
1714 #[case("futures_contract", "ESU6", "ESU26.GLBX", "ESU36.GLBX")]
1715 #[case("futures_spread", "ESU6-ESM7", "ESU26-ESM27.GLBX", "ESU36-ESM37.GLBX")]
1716 fn test_loader_definition_year_is_scoped_to_record_date(
1717 mut loader_without_seed: DatabentoDataLoader,
1718 #[case] fixture: &str,
1719 #[case] raw: &str,
1720 #[case] expected: InstrumentId,
1721 #[case] inferred: InstrumentId,
1722 ) {
1723 let loader = &mut loader_without_seed;
1724 let path = test_data_path().join(format!("test_data.definition.{fixture}.dbn.zst"));
1725 let mut decoder = Decoder::from_zstd_file(&path).unwrap();
1726 let metadata_source = decoder.metadata().clone();
1727 let mut definition = decoder
1728 .decode_record::<dbn::InstrumentDefMsg>()
1729 .unwrap()
1730 .unwrap()
1731 .clone();
1732 definition.ts_recv = "2028-01-01T00:00:00Z"
1733 .parse::<jiff::Timestamp>()
1734 .unwrap()
1735 .as_nanosecond() as u64;
1736 let replay_path =
1737 std::env::temp_dir().join(format!("futures-{}.dbn.zst", nautilus_core::UUID4::new()));
1738 let mut encoded = Vec::new();
1739 {
1740 let mut encoder =
1741 dbn::encode::dbn::Encoder::with_zstd(&mut encoded, &metadata_source).unwrap();
1742 encoder.encode_record(&definition).unwrap();
1743 encoder.flush().unwrap();
1744 }
1745
1746 std::fs::write(&replay_path, encoded).unwrap();
1747 let replayed = loader
1748 .load_instruments(&replay_path, false, false, None)
1749 .unwrap();
1750 std::fs::remove_file(replay_path).unwrap();
1751 assert_eq!(replayed[0].id(), expected);
1752 let mut record = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1753 record.hd.publisher_id = definition.hd.publisher_id;
1754 record.hd.instrument_id = definition.hd.instrument_id;
1755 record.ts_recv = definition.ts_recv;
1756 let metadata = dbn::Metadata::builder()
1757 .dataset("GLBX.MDP3")
1758 .schema(Some(dbn::Schema::Bbo1S))
1759 .start(record.ts_recv)
1760 .stype_in(Some(dbn::SType::RawSymbol))
1761 .stype_out(dbn::SType::InstrumentId)
1762 .mappings(vec![dbn::SymbolMapping {
1763 raw_symbol: raw.to_string(),
1764 intervals: vec![dbn::MappingInterval {
1765 start_date: time::macros::date!(2028 - 01 - 01),
1766 end_date: time::macros::date!(2028 - 01 - 03),
1767 symbol: record.hd.instrument_id.to_string(),
1768 }],
1769 }])
1770 .build();
1771 let mut metadata = Some(MetadataCache::new(metadata));
1772 let id = loader
1773 .resolve_record_instrument_id(&dbn::RecordRef::from(&record), None, &mut metadata)
1774 .unwrap();
1775 assert_eq!(id, expected);
1776 record.ts_recv += 86_400_000_000_000;
1777 let next_day = loader
1778 .resolve_record_instrument_id(&dbn::RecordRef::from(&record), None, &mut metadata)
1779 .unwrap();
1780 assert_eq!(next_day, inferred);
1781 }
1782}