Skip to main content

nautilus_databento/
loader.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    env,
18    path::{Path, PathBuf},
19};
20
21use ahash::AHashMap;
22use anyhow::Context;
23use databento::dbn::{self, InstrumentDefMsg};
24use dbn::{
25    Publisher,
26    decode::{DbnMetadata, DecodeStream, dbn::Decoder},
27};
28use fallible_streaming_iterator::FallibleStreamingIterator;
29use indexmap::IndexMap;
30use nautilus_model::{
31    data::{Bar, Data, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
32    identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
33    instruments::{Instrument, InstrumentAny},
34};
35
36use super::{
37    decode::{
38        MboDeltaBuffer, decode_imbalance_msg, decode_mbo_msg, decode_record, decode_statistics_msg,
39        decode_status_msg, is_supported_stat_type,
40    },
41    symbology::{decode_nautilus_instrument_id, resolve_futures_instrument_id},
42    types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, Dataset, PublisherId},
43};
44use crate::{
45    common::{build_publisher_venue_map, load_publishers},
46    decode::{DatabentoDecodeConfig, decode_instrument_def_msg},
47    symbology::MetadataCache,
48};
49
50/// A Nautilus data loader for Databento Binary Encoding (DBN) format data.
51///
52/// # Supported Schemas
53///  - `MBO` -> `OrderBookDelta`
54///  - `MBP_1` -> `(QuoteTick, Option<TradeTick>)`
55///  - `MBP_10` -> `OrderBookDepth`
56///  - `BBO_1S` -> `QuoteTick`
57///  - `BBO_1M` -> `QuoteTick`
58///  - `CMBP_1` -> `(QuoteTick, Option<TradeTick>)`
59///  - `CBBO_1S` -> `QuoteTick`
60///  - `CBBO_1M` -> `QuoteTick`
61///  - `TCBBO` -> `(QuoteTick, TradeTick)`
62///  - `TBBO` -> `(QuoteTick, TradeTick)`
63///  - `TRADES` -> `TradeTick`
64///  - `OHLCV_1S` -> `Bar`
65///  - `OHLCV_1M` -> `Bar`
66///  - `OHLCV_1H` -> `Bar`
67///  - `OHLCV_1D` -> `Bar`
68///  - `OHLCV_EOD` -> `Bar`
69///  - `DEFINITION` -> `Instrument`
70///  - `IMBALANCE` -> `DatabentoImbalance`
71///  - `STATISTICS` -> `DatabentoStatistics`
72///  - `STATUS` -> `InstrumentStatus`
73///
74/// # References
75///
76/// <https://databento.com/docs/schemas-and-data-formats>
77#[cfg_attr(
78    feature = "python",
79    pyo3::pyclass(module = "nautilus_trader.adapters.databento")
80)]
81#[cfg_attr(
82    feature = "python",
83    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
84)]
85#[derive(Debug)]
86pub struct DatabentoDataLoader {
87    futures_year_digits: FuturesYearDigits,
88    publishers_map: IndexMap<PublisherId, DatabentoPublisher>,
89    venue_dataset_map: IndexMap<Venue, Dataset>,
90    publisher_venue_map: IndexMap<PublisherId, Venue>,
91    symbol_venue_map: AHashMap<Symbol, Venue>,
92    price_precisions: AHashMap<Symbol, u8>,
93    futures_symbols: AHashMap<(PublisherId, u32, time::Date), Symbol>,
94}
95
96impl DatabentoDataLoader {
97    /// Creates a new [`DatabentoDataLoader`] instance.
98    ///
99    /// # Errors
100    ///
101    /// Returns an error if locating or loading publishers data fails.
102    pub fn new(publishers_filepath: Option<PathBuf>) -> anyhow::Result<Self> {
103        let mut loader = Self {
104            futures_year_digits: FuturesYearDigits::default(),
105            publishers_map: IndexMap::new(),
106            venue_dataset_map: IndexMap::new(),
107            publisher_venue_map: IndexMap::new(),
108            symbol_venue_map: AHashMap::new(),
109            price_precisions: AHashMap::new(),
110            futures_symbols: AHashMap::new(),
111        };
112
113        // Load publishers
114        let publishers_filepath = if let Some(p) = publishers_filepath {
115            p
116        } else {
117            // Use built-in publishers path
118            let mut exe_path = env::current_exe()?;
119            exe_path.pop();
120            exe_path.push("publishers.json");
121            exe_path
122        };
123
124        loader
125            .load_publishers(publishers_filepath)
126            .context("error loading publishers.json")?;
127
128        Ok(loader)
129    }
130
131    /// Selects one- or two-digit years for futures instrument IDs.
132    #[must_use]
133    pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
134        self.futures_year_digits = futures_year_digits;
135        self
136    }
137
138    /// Load the publishers data from the file at the given `filepath`.
139    ///
140    /// # Errors
141    ///
142    /// Returns an error if the file cannot be read or parsed as JSON.
143    pub fn load_publishers(&mut self, filepath: PathBuf) -> anyhow::Result<()> {
144        let publishers = load_publishers(filepath)?;
145
146        self.publishers_map = publishers
147            .iter()
148            .cloned()
149            .map(|p| (p.publisher_id, p))
150            .collect();
151
152        let mut venue_dataset_map = IndexMap::new();
153
154        // Only insert a dataset if the venue key is not already in the map
155        for publisher in &publishers {
156            let venue = Venue::from(publisher.venue.as_str());
157            let dataset = Dataset::from(publisher.dataset.as_str());
158            venue_dataset_map.entry(venue).or_insert(dataset);
159        }
160
161        self.venue_dataset_map = venue_dataset_map;
162        apply_default_venue_dataset_mappings(&mut self.venue_dataset_map);
163
164        self.publisher_venue_map = build_publisher_venue_map(&publishers);
165
166        Ok(())
167    }
168
169    /// Returns the internal Databento publishers currently held by the loader.
170    #[must_use]
171    pub const fn get_publishers(&self) -> &IndexMap<u16, DatabentoPublisher> {
172        &self.publishers_map
173    }
174
175    /// Sets the `venue` to map to the given `dataset`.
176    pub fn set_dataset_for_venue(&mut self, dataset: Dataset, venue: Venue) {
177        _ = self.venue_dataset_map.insert(venue, dataset);
178    }
179
180    /// Returns the dataset which matches the given `venue` (if found).
181    #[must_use]
182    pub fn get_dataset_for_venue(&self, venue: &Venue) -> Option<&Dataset> {
183        self.venue_dataset_map.get(venue)
184    }
185
186    /// Returns the venue which matches the given `publisher_id` (if found).
187    #[must_use]
188    pub fn get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option<&Venue> {
189        self.publisher_venue_map.get(&publisher_id)
190    }
191
192    /// Caches a `price_precision` for the given `symbol`.
193    ///
194    /// When market data is read without an explicit `price_precision` argument,
195    /// the loader resolves precision per record from this cache. Definitions
196    /// loaded via [`Self::load_instruments`] are inserted automatically.
197    pub fn set_price_precision(&mut self, symbol: Symbol, price_precision: u8) {
198        self.price_precisions.insert(symbol, price_precision);
199    }
200
201    /// Returns the cached price precisions keyed by symbol.
202    #[must_use]
203    pub const fn get_price_precisions(&self) -> &AHashMap<Symbol, u8> {
204        &self.price_precisions
205    }
206
207    /// Resolves a price precision for the given `instrument_id`.
208    ///
209    /// Resolution order:
210    /// 1. The explicit `price_precision` argument (if `Some`).
211    /// 2. The cached precision for the instrument's symbol.
212    ///
213    /// # Errors
214    ///
215    /// Returns an error if no precision is available.
216    fn resolve_price_precision(
217        &self,
218        instrument_id: &InstrumentId,
219        price_precision: Option<u8>,
220    ) -> anyhow::Result<u8> {
221        if let Some(precision) = price_precision {
222            return Ok(precision);
223        }
224
225        self.price_precisions
226            .get(&instrument_id.symbol)
227            .copied()
228            .ok_or_else(|| {
229                anyhow::anyhow!(
230                    "Could not resolve `price_precision` for {instrument_id}: \
231                     pass `price_precision` explicitly, call `set_price_precision`, \
232                     or load the instrument definitions first via `load_instruments`"
233                )
234            })
235    }
236
237    /// Returns the schema for the given `filepath`.
238    ///
239    /// # Errors
240    ///
241    /// Returns an error if the file cannot be decoded or metadata retrieval fails.
242    pub fn schema_from_file(&self, filepath: &Path) -> anyhow::Result<Option<String>> {
243        let decoder = Decoder::from_zstd_file(filepath)?;
244        let metadata = decoder.metadata();
245        Ok(metadata.schema.map(|schema| schema.to_string()))
246    }
247
248    /// Reads instrument definition records from a DBN file.
249    ///
250    /// # Errors
251    ///
252    /// Returns an error if decoding the definition records fails.
253    pub fn read_definition_records<'a>(
254        &'a mut self,
255        filepath: &Path,
256        use_exchange_as_venue: bool,
257        decode_config: Option<&'a DatabentoDecodeConfig>,
258    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<InstrumentAny>> + 'a> {
259        let decoder = Decoder::from_zstd_file(filepath)?;
260        let mut dbn_stream = decoder.decode_stream::<InstrumentDefMsg>();
261
262        // Loop over skipped records (Ok(None)) so one unsupported class does not
263        // terminate the stream
264        Ok(std::iter::from_fn(move || {
265            loop {
266                let advance = dbn_stream
267                    .advance()
268                    .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"));
269                if let Err(e) = advance {
270                    return Some(Err(e));
271                }
272
273                let rec = dbn_stream.get()?;
274
275                let result: anyhow::Result<Option<InstrumentAny>> = (|| {
276                    let record = dbn::RecordRef::from(rec);
277                    let msg = record
278                        .get::<InstrumentDefMsg>()
279                        .ok_or_else(|| anyhow::anyhow!("Failed to decode InstrumentDefMsg"))?;
280
281                    let raw_symbol = rec
282                        .raw_symbol()
283                        .map_err(|e| anyhow::anyhow!("Error decoding `raw_symbol`: {e}"))?;
284                    let symbol = Symbol::from(raw_symbol);
285
286                    let publisher = rec
287                        .hd
288                        .publisher()
289                        .map_err(|e| anyhow::anyhow!("Invalid `publisher` for record: {e}"))?;
290                    let venue = match publisher {
291                        Publisher::GlbxMdp3Glbx if use_exchange_as_venue => {
292                            let exchange = rec.exchange().map_err(|e| {
293                                anyhow::anyhow!("Missing `exchange` for record: {e}")
294                            })?;
295                            let venue = Venue::from_code(exchange).map_err(|e| {
296                                anyhow::anyhow!("Venue not found for exchange {exchange}: {e}")
297                            })?;
298                            self.symbol_venue_map.insert(symbol, venue);
299                            venue
300                        }
301                        _ => *self
302                            .publisher_venue_map
303                            .get(&msg.hd.publisher_id)
304                            .ok_or_else(|| {
305                                anyhow::anyhow!(
306                                    "Venue not found for publisher_id {}",
307                                    msg.hd.publisher_id
308                                )
309                            })?,
310                    };
311                    let instrument_id = InstrumentId::new(symbol, venue);
312                    let instrument_id = resolve_futures_instrument_id(
313                        &record,
314                        instrument_id,
315                        self.futures_year_digits,
316                        &self.symbol_venue_map,
317                        None,
318                    )?;
319                    self.symbol_venue_map.insert(instrument_id.symbol, venue);
320                    let ts_init = msg.ts_recv.into();
321
322                    let instrument = decode_instrument_def_msg(
323                        rec,
324                        instrument_id,
325                        Some(ts_init),
326                        decode_config,
327                    )?;
328
329                    if instrument.is_some() && instrument_id.symbol != symbol {
330                        let date = time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(
331                            msg.ts_recv,
332                        ))?
333                        .date();
334                        self.futures_symbols.insert(
335                            (msg.hd.publisher_id, msg.hd.instrument_id, date),
336                            instrument_id.symbol,
337                        );
338                    }
339
340                    Ok(instrument)
341                })();
342
343                match result {
344                    Ok(Some(item)) => return Some(Ok(item)),
345                    Ok(None) => {}
346                    Err(e) => return Some(Err(e)),
347                }
348            }
349        }))
350    }
351
352    /// Reads and decodes market data records from a DBN file.
353    ///
354    /// # Errors
355    ///
356    /// Returns an error if reading records fails.
357    pub fn read_records<T>(
358        &self,
359        filepath: &Path,
360        instrument_id: Option<InstrumentId>,
361        price_precision: Option<u8>,
362        include_trades: bool,
363        bars_timestamp_on_close: Option<bool>,
364    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<(Option<Data>, Option<Data>)>> + '_>
365    where
366        T: dbn::Record + dbn::HasRType + 'static,
367    {
368        let decoder = Decoder::from_zstd_file(filepath)?;
369        let mut metadata_cache = if instrument_id.is_none() {
370            Some(MetadataCache::new(decoder.metadata().clone()))
371        } else {
372            None
373        };
374        let mut dbn_stream = decoder.decode_stream::<T>();
375        let fixed_instrument_id = instrument_id.is_some();
376        let mut fixed_price_precision = price_precision;
377
378        Ok(std::iter::from_fn(move || {
379            let result: anyhow::Result<Option<(Option<Data>, Option<Data>)>> = (|| {
380                dbn_stream
381                    .advance()
382                    .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"))?;
383
384                if let Some(rec) = dbn_stream.get() {
385                    let record = dbn::RecordRef::from(rec);
386                    let instrument_id = self
387                        .resolve_record_instrument_id(&record, instrument_id, &mut metadata_cache)
388                        .context("failed to decode instrument id")?;
389                    let resolved_precision = self.resolve_stream_price_precision(
390                        &instrument_id,
391                        fixed_instrument_id,
392                        &mut fixed_price_precision,
393                    )?;
394                    let (item1, item2) = decode_record(
395                        &record,
396                        instrument_id,
397                        resolved_precision,
398                        None,
399                        include_trades,
400                        bars_timestamp_on_close.unwrap_or(true),
401                    )?;
402                    Ok(Some((item1, item2)))
403                } else {
404                    Ok(None)
405                }
406            })();
407
408            match result {
409                Ok(Some(v)) => Some(Ok(v)),
410                Ok(None) => None,
411                Err(e) => Some(Err(e)),
412            }
413        }))
414    }
415
416    /// Loads all instrument definitions from a DBN file.
417    ///
418    /// When `skip_on_error` is true, instruments that fail to decode are logged
419    /// as warnings and skipped. When false (default), any decode error is propagated.
420    ///
421    /// # Errors
422    ///
423    /// Returns an error if loading instruments fails.
424    pub fn load_instruments(
425        &mut self,
426        filepath: &Path,
427        use_exchange_as_venue: bool,
428        skip_on_error: bool,
429        decode_config: Option<&DatabentoDecodeConfig>,
430    ) -> anyhow::Result<Vec<InstrumentAny>> {
431        let instruments = if skip_on_error {
432            let mut collected = Vec::new();
433
434            for result in
435                self.read_definition_records(filepath, use_exchange_as_venue, decode_config)?
436            {
437                match result {
438                    Ok(instrument) => collected.push(instrument),
439                    Err(e) => log::warn!("Skipping instrument: {e}"),
440                }
441            }
442            collected
443        } else {
444            self.read_definition_records(filepath, use_exchange_as_venue, decode_config)?
445                .collect::<Result<Vec<_>, _>>()?
446        };
447
448        for instrument in &instruments {
449            self.price_precisions
450                .insert(instrument.id().symbol, instrument.price_precision());
451        }
452
453        Ok(instruments)
454    }
455
456    /// Loads order book delta messages from a DBN MBO schema file.
457    ///
458    /// Cannot include trades.
459    ///
460    /// # Errors
461    ///
462    /// Returns an error if loading order book deltas fails.
463    pub fn load_order_book_deltas(
464        &self,
465        filepath: &Path,
466        instrument_id: Option<InstrumentId>,
467        price_precision: Option<u8>,
468    ) -> anyhow::Result<Vec<OrderBookDelta>> {
469        self.read_order_book_deltas(filepath, instrument_id, price_precision)?
470            .collect()
471    }
472
473    /// Reads order book delta messages from a DBN MBO schema file without collecting them.
474    ///
475    /// Cannot include trades.
476    ///
477    /// # Errors
478    ///
479    /// Returns an error if opening or decoding order book deltas fails.
480    pub fn read_order_book_deltas(
481        &self,
482        filepath: &Path,
483        instrument_id: Option<InstrumentId>,
484        price_precision: Option<u8>,
485    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<OrderBookDelta>> + '_> {
486        let decoder = Decoder::from_zstd_file(filepath)?;
487        let mut metadata_cache = if instrument_id.is_none() {
488            Some(MetadataCache::new(decoder.metadata().clone()))
489        } else {
490            None
491        };
492        let mut dbn_stream = decoder.decode_stream::<dbn::MboMsg>();
493        let fixed_instrument_id = instrument_id.is_some();
494        let mut fixed_price_precision = price_precision;
495        let mut delta_buffer = MboDeltaBuffer::default();
496        let mut terminal_error = None;
497        let mut finished = false;
498
499        Ok(std::iter::from_fn(move || {
500            loop {
501                if let Some(delta) = delta_buffer.pop_ready() {
502                    return Some(Ok(delta));
503                }
504
505                if finished {
506                    return terminal_error.take().map(Err);
507                }
508
509                let result: anyhow::Result<bool> = (|| {
510                    dbn_stream
511                        .advance()
512                        .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"))?;
513
514                    let Some(rec) = dbn_stream.get() else {
515                        return Ok(false);
516                    };
517                    let record = dbn::RecordRef::from(rec);
518                    let instrument_id = self
519                        .resolve_record_instrument_id(&record, instrument_id, &mut metadata_cache)
520                        .context("failed to decode instrument id")?;
521                    let resolved_precision = self.resolve_stream_price_precision(
522                        &instrument_id,
523                        fixed_instrument_id,
524                        &mut fixed_price_precision,
525                    )?;
526                    let msg = record
527                        .get::<dbn::MboMsg>()
528                        .ok_or_else(|| anyhow::anyhow!("Failed to decode MboMsg"))?;
529                    let (delta, _trade) =
530                        decode_mbo_msg(msg, instrument_id, resolved_precision, None, false)?;
531                    delta_buffer.push(msg, instrument_id, delta);
532                    Ok(true)
533                })();
534
535                match result {
536                    Ok(true) => {}
537                    Ok(false) => {
538                        delta_buffer.finish();
539                        finished = true;
540                    }
541                    Err(e) => {
542                        delta_buffer.finish();
543                        terminal_error = Some(e);
544                        finished = true;
545                    }
546                }
547            }
548        }))
549    }
550
551    /// Loads order book depth snapshots from a DBN MBP-10 schema file.
552    ///
553    /// # Errors
554    ///
555    /// Returns an error if loading order book depth fails.
556    pub fn load_order_book_depth(
557        &self,
558        filepath: &Path,
559        instrument_id: Option<InstrumentId>,
560        price_precision: Option<u8>,
561    ) -> anyhow::Result<Vec<OrderBookDepth>> {
562        self.read_records::<dbn::Mbp10Msg>(filepath, instrument_id, price_precision, false, None)?
563            .filter_map(|result| match result {
564                Ok((Some(item1), _)) => {
565                    if let Data::BookDepth(depth) = item1 {
566                        Some(Ok(*depth))
567                    } else {
568                        None
569                    }
570                }
571                Ok((None, _)) => None,
572                Err(e) => Some(Err(e)),
573            })
574            .collect()
575    }
576
577    /// Loads quote tick messages from a DBN MBP-1 or TBBO schema file.
578    ///
579    /// # Errors
580    ///
581    /// Returns an error if loading quotes fails.
582    pub fn load_quotes(
583        &self,
584        filepath: &Path,
585        instrument_id: Option<InstrumentId>,
586        price_precision: Option<u8>,
587    ) -> anyhow::Result<Vec<QuoteTick>> {
588        self.read_records::<dbn::Mbp1Msg>(filepath, instrument_id, price_precision, false, None)?
589            .filter_map(|result| match result {
590                Ok((Some(item1), _)) => {
591                    if let Data::Quote(quote) = item1 {
592                        Some(Ok(quote))
593                    } else {
594                        None
595                    }
596                }
597                Ok((None, _)) => None,
598                Err(e) => Some(Err(e)),
599            })
600            .collect()
601    }
602
603    /// Loads best bid/offer quote messages from a DBN BBO schema file.
604    ///
605    /// # Errors
606    ///
607    /// Returns an error if loading BBO quotes fails.
608    pub fn load_bbo_quotes(
609        &self,
610        filepath: &Path,
611        instrument_id: Option<InstrumentId>,
612        price_precision: Option<u8>,
613    ) -> anyhow::Result<Vec<QuoteTick>> {
614        self.read_records::<dbn::BboMsg>(filepath, instrument_id, price_precision, false, None)?
615            .filter_map(|result| match result {
616                Ok((Some(item1), _)) => {
617                    if let Data::Quote(quote) = item1 {
618                        Some(Ok(quote))
619                    } else {
620                        None
621                    }
622                }
623                Ok((None, _)) => None,
624                Err(e) => Some(Err(e)),
625            })
626            .collect()
627    }
628
629    /// Loads consolidated MBP-1 quote messages from a DBN CMBP-1 schema file.
630    ///
631    /// # Errors
632    ///
633    /// Returns an error if loading consolidated MBP-1 quotes fails.
634    pub fn load_cmbp_quotes(
635        &self,
636        filepath: &Path,
637        instrument_id: Option<InstrumentId>,
638        price_precision: Option<u8>,
639    ) -> anyhow::Result<Vec<QuoteTick>> {
640        self.read_records::<dbn::Cmbp1Msg>(filepath, instrument_id, price_precision, false, None)?
641            .filter_map(|result| match result {
642                Ok((Some(item1), _)) => {
643                    if let Data::Quote(quote) = item1 {
644                        Some(Ok(quote))
645                    } else {
646                        None
647                    }
648                }
649                Ok((None, _)) => None,
650                Err(e) => Some(Err(e)),
651            })
652            .collect()
653    }
654
655    /// Loads consolidated best bid/offer quote messages from a DBN CBBO schema file.
656    ///
657    /// # Errors
658    ///
659    /// Returns an error if loading consolidated BBO quotes fails.
660    pub fn load_cbbo_quotes(
661        &self,
662        filepath: &Path,
663        instrument_id: Option<InstrumentId>,
664        price_precision: Option<u8>,
665    ) -> anyhow::Result<Vec<QuoteTick>> {
666        self.read_records::<dbn::CbboMsg>(filepath, instrument_id, price_precision, false, None)?
667            .filter_map(|result| match result {
668                Ok((Some(item1), _)) => {
669                    if let Data::Quote(quote) = item1 {
670                        Some(Ok(quote))
671                    } else {
672                        None
673                    }
674                }
675                Ok((None, _)) => None,
676                Err(e) => Some(Err(e)),
677            })
678            .collect()
679    }
680
681    /// Loads trade messages from a DBN TBBO schema file.
682    ///
683    /// # Errors
684    ///
685    /// Returns an error if loading TBBO trades fails.
686    pub fn load_tbbo_trades(
687        &self,
688        filepath: &Path,
689        instrument_id: Option<InstrumentId>,
690        price_precision: Option<u8>,
691    ) -> anyhow::Result<Vec<TradeTick>> {
692        self.read_records::<dbn::TbboMsg>(filepath, instrument_id, price_precision, true, None)?
693            .filter_map(|result| match result {
694                Ok((_, maybe_item2)) => {
695                    if let Some(Data::Trade(trade)) = maybe_item2 {
696                        Some(Ok(trade))
697                    } else {
698                        None
699                    }
700                }
701                Err(e) => Some(Err(e)),
702            })
703            .collect()
704    }
705
706    /// Loads trade messages from a DBN TCBBO schema file.
707    ///
708    /// # Errors
709    ///
710    /// Returns an error if loading TCBBO trades fails.
711    pub fn load_tcbbo_trades(
712        &self,
713        filepath: &Path,
714        instrument_id: Option<InstrumentId>,
715        price_precision: Option<u8>,
716    ) -> anyhow::Result<Vec<TradeTick>> {
717        self.read_records::<dbn::TcbboMsg>(filepath, instrument_id, price_precision, true, None)?
718            .filter_map(|result| match result {
719                Ok((_, maybe_item2)) => {
720                    if let Some(Data::Trade(trade)) = maybe_item2 {
721                        Some(Ok(trade))
722                    } else {
723                        None
724                    }
725                }
726                Err(e) => Some(Err(e)),
727            })
728            .collect()
729    }
730
731    /// Loads trade messages from a DBN TRADES schema file.
732    ///
733    /// # Errors
734    ///
735    /// Returns an error if loading trades fails.
736    pub fn load_trades(
737        &self,
738        filepath: &Path,
739        instrument_id: Option<InstrumentId>,
740        price_precision: Option<u8>,
741    ) -> anyhow::Result<Vec<TradeTick>> {
742        self.read_records::<dbn::TradeMsg>(filepath, instrument_id, price_precision, false, None)?
743            .filter_map(|result| match result {
744                Ok((Some(item1), _)) => {
745                    if let Data::Trade(trade) = item1 {
746                        Some(Ok(trade))
747                    } else {
748                        None
749                    }
750                }
751                Ok((None, _)) => None,
752                Err(e) => Some(Err(e)),
753            })
754            .collect()
755    }
756
757    /// Loads OHLCV bar messages from a DBN OHLCV schema file.
758    ///
759    /// # Errors
760    ///
761    /// Returns an error if loading bars fails.
762    pub fn load_bars(
763        &self,
764        filepath: &Path,
765        instrument_id: Option<InstrumentId>,
766        price_precision: Option<u8>,
767        timestamp_on_close: Option<bool>,
768    ) -> anyhow::Result<Vec<Bar>> {
769        self.read_records::<dbn::OhlcvMsg>(
770            filepath,
771            instrument_id,
772            price_precision,
773            false,
774            timestamp_on_close,
775        )?
776        .filter_map(|result| match result {
777            Ok((Some(item1), _)) => {
778                if let Data::Bar(bar) = item1 {
779                    Some(Ok(bar))
780                } else {
781                    None
782                }
783            }
784            Ok((None, _)) => None,
785            Err(e) => Some(Err(e)),
786        })
787        .collect()
788    }
789
790    /// Loads instrument status messages from a DBN STATUS schema file.
791    ///
792    /// # Errors
793    ///
794    /// Returns an error if loading status records fails.
795    pub fn load_status_records<T>(
796        &self,
797        filepath: &Path,
798        instrument_id: Option<InstrumentId>,
799    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<InstrumentStatus>> + '_>
800    where
801        T: dbn::Record + dbn::HasRType + 'static,
802    {
803        let decoder = Decoder::from_zstd_file(filepath)?;
804        let mut metadata_cache = if instrument_id.is_none() {
805            Some(MetadataCache::new(decoder.metadata().clone()))
806        } else {
807            None
808        };
809        let mut dbn_stream = decoder.decode_stream::<T>();
810
811        Ok(std::iter::from_fn(move || {
812            if let Err(e) = dbn_stream.advance() {
813                return Some(Err(e.into()));
814            }
815
816            match dbn_stream.get() {
817                Some(rec) => {
818                    let record = dbn::RecordRef::from(rec);
819                    let instrument_id = match self.resolve_record_instrument_id(
820                        &record,
821                        instrument_id,
822                        &mut metadata_cache,
823                    ) {
824                        Ok(id) => id,
825                        Err(e) => return Some(Err(e)),
826                    };
827
828                    let msg = match record.get::<dbn::StatusMsg>() {
829                        Some(m) => m,
830                        None => return Some(Err(anyhow::anyhow!("Invalid `StatusMsg`"))),
831                    };
832                    let ts_init = msg.ts_recv.into();
833
834                    match decode_status_msg(msg, instrument_id, Some(ts_init)) {
835                        Ok(data) => Some(Ok(data)),
836                        Err(e) => Some(Err(e)),
837                    }
838                }
839                None => None,
840            }
841        }))
842    }
843
844    /// Reads imbalance messages from a DBN IMBALANCE schema file.
845    ///
846    /// # Errors
847    ///
848    /// Returns an error if reading imbalance records fails.
849    pub fn read_imbalance_records<T>(
850        &self,
851        filepath: &Path,
852        instrument_id: Option<InstrumentId>,
853        price_precision: Option<u8>,
854    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<DatabentoImbalance>> + '_>
855    where
856        T: dbn::Record + dbn::HasRType + 'static,
857    {
858        let decoder = Decoder::from_zstd_file(filepath)?;
859        let mut metadata_cache = if instrument_id.is_none() {
860            Some(MetadataCache::new(decoder.metadata().clone()))
861        } else {
862            None
863        };
864        let mut dbn_stream = decoder.decode_stream::<T>();
865        let fixed_instrument_id = instrument_id.is_some();
866        let mut fixed_price_precision = price_precision;
867
868        Ok(std::iter::from_fn(move || {
869            if let Err(e) = dbn_stream.advance() {
870                return Some(Err(e.into()));
871            }
872
873            match dbn_stream.get() {
874                Some(rec) => {
875                    let record = dbn::RecordRef::from(rec);
876                    let instrument_id = match self.resolve_record_instrument_id(
877                        &record,
878                        instrument_id,
879                        &mut metadata_cache,
880                    ) {
881                        Ok(id) => id,
882                        Err(e) => return Some(Err(e)),
883                    };
884                    let resolved_precision = match self.resolve_stream_price_precision(
885                        &instrument_id,
886                        fixed_instrument_id,
887                        &mut fixed_price_precision,
888                    ) {
889                        Ok(p) => p,
890                        Err(e) => return Some(Err(e)),
891                    };
892
893                    let msg = match record.get::<dbn::ImbalanceMsg>() {
894                        Some(m) => m,
895                        None => return Some(Err(anyhow::anyhow!("Invalid `ImbalanceMsg`"))),
896                    };
897                    let ts_init = msg.ts_recv.into();
898
899                    match decode_imbalance_msg(
900                        msg,
901                        instrument_id,
902                        resolved_precision,
903                        Some(ts_init),
904                    ) {
905                        Ok(data) => Some(Ok(data)),
906                        Err(e) => Some(Err(e)),
907                    }
908                }
909                None => None,
910            }
911        }))
912    }
913
914    /// Reads statistics messages from a DBN STATISTICS schema file.
915    ///
916    /// # Errors
917    ///
918    /// Returns an error if reading statistics records fails.
919    pub fn read_statistics_records<T>(
920        &self,
921        filepath: &Path,
922        instrument_id: Option<InstrumentId>,
923        price_precision: Option<u8>,
924    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<DatabentoStatistics>> + '_>
925    where
926        T: dbn::Record + dbn::HasRType + 'static,
927    {
928        let decoder = Decoder::from_zstd_file(filepath)?;
929        let mut metadata_cache = if instrument_id.is_none() {
930            Some(MetadataCache::new(decoder.metadata().clone()))
931        } else {
932            None
933        };
934        let mut dbn_stream = decoder.decode_stream::<T>();
935        let fixed_instrument_id = instrument_id.is_some();
936        let mut fixed_price_precision = price_precision;
937
938        // Loop over skipped records so one unsupported stat_type does not terminate
939        // the stream; precheck before precision resolution.
940        Ok(std::iter::from_fn(move || {
941            loop {
942                if let Err(e) = dbn_stream.advance() {
943                    return Some(Err(e.into()));
944                }
945
946                let rec = dbn_stream.get()?;
947                let record = dbn::RecordRef::from(rec);
948                let msg = match record.get::<dbn::StatMsg>() {
949                    Some(m) => m,
950                    None => return Some(Err(anyhow::anyhow!("Invalid `StatMsg`"))),
951                };
952
953                if !is_supported_stat_type(msg.stat_type) {
954                    log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
955                    continue;
956                }
957
958                let instrument_id = match self.resolve_record_instrument_id(
959                    &record,
960                    instrument_id,
961                    &mut metadata_cache,
962                ) {
963                    Ok(id) => id,
964                    Err(e) => return Some(Err(e)),
965                };
966                let resolved_precision = match self.resolve_stream_price_precision(
967                    &instrument_id,
968                    fixed_instrument_id,
969                    &mut fixed_price_precision,
970                ) {
971                    Ok(p) => p,
972                    Err(e) => return Some(Err(e)),
973                };
974                let ts_init = msg.ts_recv.into();
975
976                match decode_statistics_msg(msg, instrument_id, resolved_precision, Some(ts_init)) {
977                    Ok(Some(data)) => return Some(Ok(data)),
978                    Ok(None) => {}
979                    Err(e) => return Some(Err(e)),
980                }
981            }
982        }))
983    }
984
985    fn resolve_record_instrument_id(
986        &self,
987        record: &dbn::RecordRef,
988        instrument_id: Option<InstrumentId>,
989        metadata_cache: &mut Option<MetadataCache>,
990    ) -> anyhow::Result<InstrumentId> {
991        if let Some(instrument_id) = instrument_id {
992            return Ok(instrument_id);
993        }
994
995        let metadata_cache = metadata_cache
996            .as_mut()
997            .ok_or_else(|| anyhow::anyhow!("missing metadata cache for dynamic instrument id"))?;
998
999        decode_nautilus_instrument_id(
1000            record,
1001            metadata_cache,
1002            &self.publisher_venue_map,
1003            &self.symbol_venue_map,
1004            self.futures_year_digits,
1005            &self.futures_symbols,
1006            None,
1007        )
1008    }
1009
1010    fn resolve_stream_price_precision(
1011        &self,
1012        instrument_id: &InstrumentId,
1013        fixed_instrument_id: bool,
1014        fixed_price_precision: &mut Option<u8>,
1015    ) -> anyhow::Result<u8> {
1016        if let Some(precision) = *fixed_price_precision {
1017            return Ok(precision);
1018        }
1019
1020        let precision = self.resolve_price_precision(instrument_id, None)?;
1021        if fixed_instrument_id {
1022            *fixed_price_precision = Some(precision);
1023        }
1024
1025        Ok(precision)
1026    }
1027}
1028
1029/// Applies default venue-to-dataset mappings for consolidated Databento feeds.
1030/// GLBX.MDP3 covers CME Globex exchange MICs; OPRA.PILLAR covers OPRA option venues;
1031/// EQUS.MINI is the consolidated US equities default.
1032fn apply_default_venue_dataset_mappings(venue_dataset_map: &mut IndexMap<Venue, Dataset>) {
1033    let glbx = Dataset::from("GLBX.MDP3");
1034
1035    for venue in [
1036        Venue::CBCM(),
1037        Venue::GLBX(),
1038        Venue::NYUM(),
1039        Venue::XCBT(),
1040        Venue::XCEC(),
1041        Venue::XCME(),
1042        Venue::XFXS(),
1043        Venue::XNYM(),
1044    ] {
1045        _ = venue_dataset_map.insert(venue, glbx);
1046    }
1047
1048    // publishers.json seeds the consolidated EQUS venue with the unreleased EQUS.PLUS,
1049    // so pin it to EQUS.MINI, the cheapest released US equities feed.
1050    _ = venue_dataset_map.insert(Venue::from("EQUS"), Dataset::from("EQUS.MINI"));
1051
1052    let opra = Dataset::from("OPRA.PILLAR");
1053    for venue_code in [
1054        "AMXO", "XBOX", "XCBO", "EMLD", "EDGO", "GMNI", "XISX", "MCRY", "XMIO", "ARCO", "OPRA",
1055        "MPRL", "XNDQ", "XBXO", "C2OX", "XPHL", "BATO", "MXOP", "SPHR",
1056    ] {
1057        _ = venue_dataset_map.insert(Venue::from(venue_code), opra);
1058    }
1059}
1060
1061#[cfg(test)]
1062mod tests {
1063    use std::{
1064        ffi::c_char,
1065        fs::{File, OpenOptions},
1066        io::Write,
1067        path::{Path, PathBuf},
1068        process,
1069        sync::atomic::{AtomicU64, Ordering},
1070    };
1071
1072    use databento::dbn::{decode::DecodeRecord, encode::EncodeRecord};
1073    use nautilus_model::{
1074        enums::BookAction,
1075        types::{Price, Quantity},
1076    };
1077    use rstest::{fixture, rstest};
1078    use ustr::Ustr;
1079
1080    use super::*;
1081
1082    fn test_data_path() -> PathBuf {
1083        Path::new(env!("CARGO_MANIFEST_DIR")).join("test_data")
1084    }
1085
1086    fn mbo_record(action: c_char, flags: u8, sequence: u32) -> dbn::MboMsg {
1087        let ts_event = 1_609_160_400_000_000_000;
1088        dbn::MboMsg {
1089            hd: dbn::RecordHeader::new::<dbn::MboMsg>(dbn::rtype::MBO, 1, 1, ts_event),
1090            order_id: 42,
1091            price: 4_800_250_000_000,
1092            size: 2,
1093            flags: dbn::FlagSet::new(flags),
1094            channel_id: 1,
1095            action,
1096            side: 'A' as c_char,
1097            ts_recv: ts_event,
1098            ts_in_delta: 0,
1099            sequence,
1100        }
1101    }
1102
1103    fn write_mbo_records(records: &[dbn::MboMsg]) -> PathBuf {
1104        static FILE_ID: AtomicU64 = AtomicU64::new(0);
1105
1106        let id = FILE_ID.fetch_add(1, Ordering::Relaxed);
1107        let path = std::env::temp_dir().join(format!(
1108            "nautilus-databento-mbo-{}-{id}.dbn.zst",
1109            process::id(),
1110        ));
1111        let metadata = dbn::Metadata::builder()
1112            .dataset("GLBX.MDP3")
1113            .schema(Some(dbn::Schema::Mbo))
1114            .start(0)
1115            .stype_in(Some(dbn::SType::InstrumentId))
1116            .stype_out(dbn::SType::InstrumentId)
1117            .build();
1118        let file = File::create(&path).unwrap();
1119        let mut encoder = dbn::encode::dbn::Encoder::with_zstd(file, &metadata).unwrap();
1120        encoder.encode_records(records).unwrap();
1121        encoder.flush().unwrap();
1122        drop(encoder);
1123
1124        path
1125    }
1126
1127    #[fixture]
1128    fn loader() -> DatabentoDataLoader {
1129        let publishers_filepath = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1130        let mut loader = DatabentoDataLoader::new(Some(publishers_filepath)).unwrap();
1131        // ES futures test data uses precision 2 (USD cents)
1132        loader.set_price_precision(Symbol::from("ESM4"), 2);
1133        loader
1134    }
1135
1136    #[fixture]
1137    fn loader_without_seed() -> DatabentoDataLoader {
1138        let publishers_filepath = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1139        DatabentoDataLoader::new(Some(publishers_filepath)).unwrap()
1140    }
1141
1142    // TODO: Improve the below assertions that we've actually read the records we expected
1143
1144    #[rstest]
1145    fn test_set_dataset_venue_mapping(mut loader: DatabentoDataLoader) {
1146        let dataset = Ustr::from("EQUS.PLUS");
1147        let venue = Venue::from("XNAS");
1148        loader.set_dataset_for_venue(dataset, venue);
1149
1150        let result = loader.get_dataset_for_venue(&venue).unwrap();
1151        assert_eq!(*result, dataset);
1152    }
1153
1154    #[rstest]
1155    fn test_default_venue_dataset_mappings(loader: DatabentoDataLoader) {
1156        let xcme = Venue::XCME();
1157        let result = loader.get_dataset_for_venue(&xcme).unwrap();
1158        assert_eq!(*result, Ustr::from("GLBX.MDP3"));
1159
1160        let xcbo = Venue::from("XCBO");
1161        let result = loader.get_dataset_for_venue(&xcbo).unwrap();
1162        assert_eq!(*result, Ustr::from("OPRA.PILLAR"));
1163
1164        let equs = Venue::from("EQUS");
1165        let result = loader.get_dataset_for_venue(&equs).unwrap();
1166        assert_eq!(*result, Ustr::from("EQUS.MINI"));
1167    }
1168
1169    #[rstest]
1170    #[case(test_data_path().join("test_data.definition.equity.dbn.zst"))]
1171    fn test_load_instruments(mut loader: DatabentoDataLoader, #[case] path: PathBuf) {
1172        let instruments = loader.load_instruments(&path, false, false, None).unwrap();
1173
1174        assert_eq!(instruments.len(), 2);
1175        // Definition records auto-populate the precision cache
1176        assert_eq!(
1177            loader.get_price_precisions().get(&Symbol::from("ESM4")),
1178            Some(&2)
1179        );
1180    }
1181
1182    #[rstest]
1183    fn test_load_instruments_populates_price_precisions_cache(
1184        mut loader_without_seed: DatabentoDataLoader,
1185    ) {
1186        let path = test_data_path().join("test_data.definition.equity.dbn.zst");
1187        assert!(loader_without_seed.get_price_precisions().is_empty());
1188
1189        let instruments = loader_without_seed
1190            .load_instruments(&path, false, false, None)
1191            .unwrap();
1192
1193        assert_eq!(instruments.len(), 2);
1194        for instrument in &instruments {
1195            let symbol = instrument.id().symbol;
1196            assert_eq!(
1197                loader_without_seed.get_price_precisions().get(&symbol),
1198                Some(&instrument.price_precision()),
1199                "cache missing or mismatched entry for {symbol}",
1200            );
1201        }
1202    }
1203
1204    #[rstest]
1205    fn test_read_records_errors_when_precision_unresolvable(
1206        loader_without_seed: DatabentoDataLoader,
1207    ) {
1208        let path = test_data_path().join("test_data.mbo.dbn.zst");
1209        let instrument_id = InstrumentId::from("ESM4.GLBX");
1210
1211        let result = loader_without_seed.load_order_book_deltas(&path, Some(instrument_id), None);
1212
1213        let err = result.expect_err("expected precision-resolution error");
1214        let err_msg = format!("{err}");
1215        assert!(
1216            err_msg.contains("Could not resolve `price_precision`"),
1217            "unexpected error message: {err_msg}",
1218        );
1219        assert!(
1220            err_msg.contains("ESM4.GLBX"),
1221            "error should name the instrument: {err_msg}",
1222        );
1223    }
1224
1225    #[rstest]
1226    fn test_set_price_precision_unblocks_reads(mut loader_without_seed: DatabentoDataLoader) {
1227        let path = test_data_path().join("test_data.mbo.dbn.zst");
1228        let instrument_id = InstrumentId::from("ESM4.GLBX");
1229
1230        // Without a seeded precision the read errors
1231        assert!(
1232            loader_without_seed
1233                .load_order_book_deltas(&path, Some(instrument_id), None)
1234                .is_err()
1235        );
1236
1237        loader_without_seed.set_price_precision(Symbol::from("ESM4"), 2);
1238
1239        let deltas = loader_without_seed
1240            .load_order_book_deltas(&path, Some(instrument_id), None)
1241            .unwrap();
1242        assert_eq!(deltas.len(), 2);
1243    }
1244
1245    #[rstest]
1246    fn test_resolve_price_precision_explicit_arg_overrides_cache(
1247        mut loader_without_seed: DatabentoDataLoader,
1248    ) {
1249        let instrument_id = InstrumentId::from("ESM4.GLBX");
1250        // Seed a deliberately wrong cache value so we can detect which path is taken
1251        loader_without_seed.set_price_precision(Symbol::from("ESM4"), 9);
1252
1253        let explicit = loader_without_seed
1254            .resolve_price_precision(&instrument_id, Some(2))
1255            .unwrap();
1256        assert_eq!(explicit, 2);
1257
1258        let cached = loader_without_seed
1259            .resolve_price_precision(&instrument_id, None)
1260            .unwrap();
1261        assert_eq!(cached, 9);
1262    }
1263
1264    #[rstest]
1265    fn test_resolve_price_precision_cache_miss_errors(loader_without_seed: DatabentoDataLoader) {
1266        let instrument_id = InstrumentId::from("ESM4.GLBX");
1267
1268        let err = loader_without_seed
1269            .resolve_price_precision(&instrument_id, None)
1270            .expect_err("expected cache-miss error");
1271        assert!(format!("{err}").contains("Could not resolve `price_precision`"));
1272    }
1273
1274    #[rstest]
1275    fn test_load_order_book_deltas(loader: DatabentoDataLoader) {
1276        let path = test_data_path().join("test_data.mbo.dbn.zst");
1277        let instrument_id = InstrumentId::from("ESM4.GLBX");
1278
1279        let deltas = loader
1280            .load_order_book_deltas(&path, Some(instrument_id), None)
1281            .unwrap();
1282
1283        assert_eq!(deltas.len(), 2);
1284    }
1285
1286    #[rstest]
1287    fn test_read_order_book_deltas_streams_without_collecting(loader: DatabentoDataLoader) {
1288        let path = test_data_path().join("test_data.mbo.dbn.zst");
1289        let instrument_id = InstrumentId::from("ESM4.GLBX");
1290
1291        let count = loader
1292            .read_order_book_deltas(&path, Some(instrument_id), None)
1293            .unwrap()
1294            .map(|result| result.map(|_| 1usize))
1295            .sum::<anyhow::Result<usize>>()
1296            .unwrap();
1297
1298        assert_eq!(count, 2);
1299    }
1300
1301    #[rstest]
1302    #[case::standalone(false)]
1303    #[case::legacy_inline(true)]
1304    fn test_load_order_book_deltas_preserves_event_boundary(
1305        loader: DatabentoDataLoader,
1306        #[case] inline_last: bool,
1307    ) {
1308        let instrument_id = InstrumentId::from("ESM4.GLBX");
1309        let last = dbn::flags::LAST;
1310        let mut records = vec![mbo_record(
1311            'C' as c_char,
1312            if inline_last { last } else { 0 },
1313            1,
1314        )];
1315
1316        if !inline_last {
1317            records.push(mbo_record('N' as c_char, last, 2));
1318        }
1319        let path = write_mbo_records(&records);
1320
1321        let deltas = loader
1322            .load_order_book_deltas(&path, Some(instrument_id), Some(2))
1323            .unwrap();
1324        std::fs::remove_file(path).unwrap();
1325
1326        assert_eq!(deltas.len(), 1);
1327        assert_eq!(deltas[0].instrument_id, instrument_id);
1328        assert_eq!(deltas[0].action, BookAction::Delete);
1329        assert_eq!(deltas[0].order.order_id, 42);
1330        assert_eq!(deltas[0].flags, last);
1331        assert_eq!(deltas[0].sequence, 1);
1332    }
1333
1334    #[rstest]
1335    fn test_read_order_book_deltas_drains_before_terminal_error(loader: DatabentoDataLoader) {
1336        let instrument_id = InstrumentId::from("ESM4.GLBX");
1337        let path = write_mbo_records(&[mbo_record('C' as c_char, 0, 1)]);
1338        OpenOptions::new()
1339            .append(true)
1340            .open(&path)
1341            .unwrap()
1342            .write_all(&[0; 8])
1343            .unwrap();
1344        let mut deltas = loader
1345            .read_order_book_deltas(&path, Some(instrument_id), Some(2))
1346            .unwrap();
1347
1348        let delta = deltas.next().unwrap().unwrap();
1349        let error = deltas
1350            .next()
1351            .unwrap()
1352            .expect_err("expected trailing decode error");
1353        let end = deltas.next();
1354        std::fs::remove_file(path).unwrap();
1355
1356        assert_eq!(delta.instrument_id, instrument_id);
1357        assert_eq!(delta.order.order_id, 42);
1358        assert_eq!(delta.flags, 0);
1359        assert!(format!("{error}").contains("Stream advance error"));
1360        assert!(end.is_none());
1361    }
1362
1363    #[rstest]
1364    fn test_load_order_book_depth(loader: DatabentoDataLoader) {
1365        let path = test_data_path().join("test_data.mbp-10.dbn.zst");
1366        let instrument_id = InstrumentId::from("ESM4.GLBX");
1367
1368        let depths = loader
1369            .load_order_book_depth(&path, Some(instrument_id), None)
1370            .unwrap();
1371
1372        assert_eq!(depths.len(), 2);
1373    }
1374
1375    #[rstest]
1376    fn test_load_quotes(loader: DatabentoDataLoader) {
1377        let path = test_data_path().join("test_data.mbp-1.dbn.zst");
1378        let instrument_id = InstrumentId::from("ESM4.GLBX");
1379
1380        let quotes = loader
1381            .load_quotes(&path, Some(instrument_id), None)
1382            .unwrap();
1383
1384        assert_eq!(quotes.len(), 2);
1385    }
1386
1387    #[rstest]
1388    #[case(test_data_path().join("test_data.bbo-1s.dbn.zst"))]
1389    #[case(test_data_path().join("test_data.bbo-1m.dbn.zst"))]
1390    fn test_load_bbo_quotes(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1391        let instrument_id = InstrumentId::from("ESM4.GLBX");
1392
1393        let quotes = loader
1394            .load_bbo_quotes(&path, Some(instrument_id), None)
1395            .unwrap();
1396
1397        assert_eq!(quotes.len(), 4);
1398    }
1399
1400    #[rstest]
1401    fn test_load_cmbp_quotes(loader: DatabentoDataLoader) {
1402        let path = test_data_path().join("test_data.cmbp-1.dbn.zst");
1403        let instrument_id = InstrumentId::from("ESM4.GLBX");
1404
1405        let quotes = loader
1406            .load_cmbp_quotes(&path, Some(instrument_id), None)
1407            .unwrap();
1408
1409        // Verify exact data count
1410        assert_eq!(quotes.len(), 2);
1411
1412        // Verify first quote fields
1413        let first_quote = &quotes[0];
1414        assert_eq!(first_quote.instrument_id, instrument_id);
1415        assert_eq!(first_quote.bid_price, Price::from("3720.25"));
1416        assert_eq!(first_quote.ask_price, Price::from("3720.50"));
1417        assert_eq!(first_quote.bid_size, Quantity::from(24));
1418        assert_eq!(first_quote.ask_size, Quantity::from(11));
1419        assert_eq!(first_quote.ts_event, 1609160400006136329);
1420        assert_eq!(first_quote.ts_init, 1609160400006136329);
1421    }
1422
1423    #[rstest]
1424    fn test_load_cbbo_quotes(loader: DatabentoDataLoader) {
1425        let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1426        let instrument_id = InstrumentId::from("ESM4.GLBX");
1427
1428        let quotes = loader
1429            .load_cbbo_quotes(&path, Some(instrument_id), None)
1430            .unwrap();
1431
1432        // Verify exact data count
1433        assert_eq!(quotes.len(), 2);
1434
1435        // Verify first quote fields
1436        let first_quote = &quotes[0];
1437        assert_eq!(first_quote.instrument_id, instrument_id);
1438        assert_eq!(first_quote.bid_price, Price::from("3720.25"));
1439        assert_eq!(first_quote.ask_price, Price::from("3720.50"));
1440        assert_eq!(first_quote.bid_size, Quantity::from(24));
1441        assert_eq!(first_quote.ask_size, Quantity::from(11));
1442        assert_eq!(first_quote.ts_event, 1609160400006136329);
1443        assert_eq!(first_quote.ts_init, 1609160400006136329);
1444    }
1445
1446    #[rstest]
1447    #[case(1, "ESH1.GLBX")]
1448    #[case(2, "ESH21.GLBX")]
1449    fn test_load_cbbo_quotes_resolves_instrument_id(
1450        loader: DatabentoDataLoader,
1451        #[case] digits: u8,
1452        #[case] expected: InstrumentId,
1453    ) {
1454        let loader = loader.with_futures_year_digits(digits.try_into().unwrap());
1455        let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1456        let quotes = loader.load_cbbo_quotes(&path, None, Some(2)).unwrap();
1457
1458        assert_eq!(quotes.len(), 2);
1459        assert_eq!(quotes[0].instrument_id, expected);
1460        assert_eq!(quotes[1].instrument_id, expected);
1461        assert_eq!(quotes[0].bid_price, Price::from("3720.25"));
1462        assert_eq!(quotes[0].ask_price, Price::from("3720.50"));
1463        assert_eq!(quotes[0].bid_size, Quantity::from(24));
1464        assert_eq!(quotes[0].ask_size, Quantity::from(11));
1465        assert_eq!(quotes[0].ts_event, 1609160400006136329);
1466        assert_eq!(quotes[0].ts_init, 1609160400006136329);
1467    }
1468
1469    #[rstest]
1470    fn test_load_tbbo_trades(loader: DatabentoDataLoader) {
1471        let path = test_data_path().join("test_data.tbbo.dbn.zst");
1472        let instrument_id = InstrumentId::from("ESM4.GLBX");
1473
1474        let trades = loader
1475            .load_tbbo_trades(&path, Some(instrument_id), None)
1476            .unwrap();
1477
1478        assert_eq!(trades.len(), 2);
1479        assert_eq!(trades[0].instrument_id, instrument_id);
1480        assert_eq!(trades[0].price, Price::from("3720.25"));
1481        assert_eq!(trades[0].size, Quantity::from("5"));
1482    }
1483
1484    #[rstest]
1485    fn test_load_tcbbo_trades_rejects_cbbo_fixture(loader: DatabentoDataLoader) {
1486        let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1487        let instrument_id = InstrumentId::from("ESM4.GLBX");
1488
1489        let result = loader.load_tcbbo_trades(&path, Some(instrument_id), None);
1490
1491        assert!(result.is_err());
1492    }
1493
1494    #[rstest]
1495    fn test_load_trades(loader: DatabentoDataLoader) {
1496        let path = test_data_path().join("test_data.trades.dbn.zst");
1497        let instrument_id = InstrumentId::from("ESM4.GLBX");
1498        let trades = loader
1499            .load_trades(&path, Some(instrument_id), None)
1500            .unwrap();
1501
1502        assert_eq!(trades.len(), 2);
1503    }
1504
1505    #[rstest]
1506    // #[case(test_data_path().join("test_data.ohlcv-1d.dbn.zst"))]  // TODO: Empty file (0 records)
1507    #[case(test_data_path().join("test_data.ohlcv-1h.dbn.zst"))]
1508    #[case(test_data_path().join("test_data.ohlcv-1m.dbn.zst"))]
1509    #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1510    fn test_load_bars(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1511        let instrument_id = InstrumentId::from("ESM4.GLBX");
1512        let bars = loader
1513            .load_bars(&path, Some(instrument_id), None, None)
1514            .unwrap();
1515
1516        assert_eq!(bars.len(), 2);
1517    }
1518
1519    #[rstest]
1520    #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1521    fn test_load_bars_timestamp_on_close_true(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1522        let instrument_id = InstrumentId::from("ESM4.GLBX");
1523        let bars = loader
1524            .load_bars(&path, Some(instrument_id), None, Some(true))
1525            .unwrap();
1526
1527        assert_eq!(bars.len(), 2);
1528
1529        // When bars_timestamp_on_close is true, both ts_event and ts_init should be close time
1530        for bar in &bars {
1531            assert_eq!(
1532                bar.ts_event, bar.ts_init,
1533                "ts_event and ts_init should both be close time when bars_timestamp_on_close=true"
1534            );
1535        }
1536    }
1537
1538    #[rstest]
1539    #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1540    fn test_load_bars_timestamp_on_close_false(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1541        let instrument_id = InstrumentId::from("ESM4.GLBX");
1542        let bars = loader
1543            .load_bars(&path, Some(instrument_id), None, Some(false))
1544            .unwrap();
1545
1546        assert_eq!(bars.len(), 2);
1547
1548        // When bars_timestamp_on_close is false, ts_event is open time and ts_init is close time
1549        for bar in &bars {
1550            assert_ne!(
1551                bar.ts_event, bar.ts_init,
1552                "ts_event should be open time and ts_init should be close time when bars_timestamp_on_close=false"
1553            );
1554            // For 1-second bars, ts_init (close) should be 1 second after ts_event (open)
1555            assert_eq!(bar.ts_init.as_u64(), bar.ts_event.as_u64() + 1_000_000_000);
1556        }
1557    }
1558
1559    #[rstest]
1560    #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"), 0)]
1561    #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"), 1)]
1562    fn test_load_bars_timestamp_comparison(
1563        loader: DatabentoDataLoader,
1564        #[case] path: PathBuf,
1565        #[case] bar_index: usize,
1566    ) {
1567        const ONE_SECOND_NS: u64 = 1_000_000_000;
1568
1569        let instrument_id = InstrumentId::from("ESM4.GLBX");
1570
1571        let bars_close = loader
1572            .load_bars(&path, Some(instrument_id), None, Some(true))
1573            .unwrap();
1574
1575        let bars_open = loader
1576            .load_bars(&path, Some(instrument_id), None, Some(false))
1577            .unwrap();
1578
1579        assert_eq!(bars_close.len(), bars_open.len());
1580        assert_eq!(bars_close.len(), 2);
1581
1582        let bar_close = &bars_close[bar_index];
1583        let bar_open = &bars_open[bar_index];
1584
1585        // Bars should have the same OHLCV data
1586        assert_eq!(bar_close.open, bar_open.open);
1587        assert_eq!(bar_close.high, bar_open.high);
1588        assert_eq!(bar_close.low, bar_open.low);
1589        assert_eq!(bar_close.close, bar_open.close);
1590        assert_eq!(bar_close.volume, bar_open.volume);
1591
1592        // The close-timestamped bar should have later timestamp than open-timestamped bar
1593        // For 1-second bars, this should be exactly 1 second difference
1594        assert!(
1595            bar_close.ts_event > bar_open.ts_event,
1596            "Close-timestamped bar should have later timestamp than open-timestamped bar"
1597        );
1598
1599        // The difference should be exactly 1 second (1_000_000_000 nanoseconds) for 1s bars
1600        assert_eq!(
1601            bar_close.ts_event.as_u64() - bar_open.ts_event.as_u64(),
1602            ONE_SECOND_NS,
1603            "Timestamp difference should be exactly 1 second for 1s bars"
1604        );
1605    }
1606
1607    #[rstest]
1608    fn test_load_status_records(loader: DatabentoDataLoader) {
1609        let path = test_data_path().join("test_data.status.dbn.zst");
1610        let instrument_id = InstrumentId::from("ESM4.GLBX");
1611
1612        let statuses = loader
1613            .load_status_records::<dbn::StatusMsg>(&path, Some(instrument_id))
1614            .unwrap()
1615            .collect::<anyhow::Result<Vec<_>>>()
1616            .unwrap();
1617
1618        // Assert total count matches Python test expectations
1619        assert_eq!(statuses.len(), 4, "Should load exactly 4 status records");
1620
1621        // Assert first record fields match Python test expectations
1622        let first = &statuses[0];
1623        assert_eq!(first.instrument_id, instrument_id);
1624        assert_eq!(first.ts_event.as_u64(), 1609110000000000000);
1625        assert_eq!(first.ts_init.as_u64(), 1609113600000000000);
1626    }
1627
1628    #[rstest]
1629    fn test_read_imbalance_records(loader: DatabentoDataLoader) {
1630        let path = test_data_path().join("test_data.imbalance.dbn.zst");
1631        let instrument_id = InstrumentId::from("ESM4.GLBX");
1632
1633        let imbalances = loader
1634            .read_imbalance_records::<dbn::ImbalanceMsg>(&path, Some(instrument_id), None)
1635            .unwrap()
1636            .collect::<anyhow::Result<Vec<_>>>()
1637            .unwrap();
1638
1639        // Assert total count
1640        assert_eq!(
1641            imbalances.len(),
1642            2,
1643            "Should load exactly 2 imbalance records"
1644        );
1645
1646        // Assert first record has required fields
1647        let first = &imbalances[0];
1648        assert_eq!(first.instrument_id, instrument_id);
1649        assert!(
1650            first.ref_price.as_f64() > 0.0,
1651            "ref_price should be positive"
1652        );
1653        assert!(first.ts_event.as_u64() > 0, "ts_event should be set");
1654        assert!(first.ts_recv.as_u64() > 0, "ts_recv should be set");
1655        assert!(first.ts_init.as_u64() > 0, "ts_init should be set");
1656    }
1657
1658    #[rstest]
1659    fn test_read_statistics_records(loader: DatabentoDataLoader) {
1660        let path = test_data_path().join("test_data.statistics.dbn.zst");
1661        let instrument_id = InstrumentId::from("ESM4.GLBX");
1662
1663        let statistics = loader
1664            .read_statistics_records::<dbn::StatMsg>(&path, Some(instrument_id), None)
1665            .unwrap()
1666            .collect::<anyhow::Result<Vec<_>>>()
1667            .unwrap();
1668
1669        // Assert total count
1670        assert_eq!(
1671            statistics.len(),
1672            2,
1673            "Should load exactly 2 statistics records"
1674        );
1675
1676        // Assert first record has required fields
1677        let first = &statistics[0];
1678        assert_eq!(first.instrument_id, instrument_id);
1679        assert!(first.ts_event.as_u64() > 0, "ts_event should be set");
1680        assert!(first.ts_recv.as_u64() > 0, "ts_recv should be set");
1681        assert!(first.ts_init.as_u64() > 0, "ts_init should be set");
1682        assert!(first.sequence > 0, "sequence should be positive");
1683    }
1684
1685    #[rstest]
1686    #[case("futures_contract", "ESU6", "ESU26.GLBX")]
1687    #[case("futures_spread", "ESU6-ESM7", "ESU26-ESM27.GLBX")]
1688    fn test_loader_defaults_to_two_digit_years(
1689        #[case] fixture: &str,
1690        #[case] raw: &str,
1691        #[case] expected: InstrumentId,
1692    ) {
1693        let publishers = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1694        let path = Path::new(env!("CARGO_MANIFEST_DIR"))
1695            .join(format!("test_data/test_data.definition.{fixture}.dbn.zst"));
1696        let mut loader = DatabentoDataLoader::new(Some(publishers.clone())).unwrap();
1697        let mut legacy = DatabentoDataLoader::new(Some(publishers))
1698            .unwrap()
1699            .with_futures_year_digits(FuturesYearDigits::One);
1700        let instruments = loader.load_instruments(&path, false, false, None).unwrap();
1701        let legacy_instruments = legacy.load_instruments(&path, false, false, None).unwrap();
1702        assert_eq!(instruments.len(), 1);
1703        assert_eq!(legacy_instruments.len(), 1);
1704        assert_eq!(instruments[0].id(), expected);
1705        assert_eq!(instruments[0].raw_symbol(), Symbol::from(raw));
1706        assert_eq!(
1707            legacy_instruments[0].id(),
1708            InstrumentId::new(Symbol::from(raw), Venue::from("GLBX"))
1709        );
1710        assert_eq!(legacy_instruments[0].raw_symbol(), Symbol::from(raw));
1711    }
1712
1713    #[rstest]
1714    #[case("futures_contract", "ESU6", "ESU26.GLBX", "ESU36.GLBX")]
1715    #[case("futures_spread", "ESU6-ESM7", "ESU26-ESM27.GLBX", "ESU36-ESM37.GLBX")]
1716    fn test_loader_definition_year_is_scoped_to_record_date(
1717        mut loader_without_seed: DatabentoDataLoader,
1718        #[case] fixture: &str,
1719        #[case] raw: &str,
1720        #[case] expected: InstrumentId,
1721        #[case] inferred: InstrumentId,
1722    ) {
1723        let loader = &mut loader_without_seed;
1724        let path = test_data_path().join(format!("test_data.definition.{fixture}.dbn.zst"));
1725        let mut decoder = Decoder::from_zstd_file(&path).unwrap();
1726        let metadata_source = decoder.metadata().clone();
1727        let mut definition = decoder
1728            .decode_record::<dbn::InstrumentDefMsg>()
1729            .unwrap()
1730            .unwrap()
1731            .clone();
1732        definition.ts_recv = "2028-01-01T00:00:00Z"
1733            .parse::<jiff::Timestamp>()
1734            .unwrap()
1735            .as_nanosecond() as u64;
1736        let replay_path =
1737            std::env::temp_dir().join(format!("futures-{}.dbn.zst", nautilus_core::UUID4::new()));
1738        let mut encoded = Vec::new();
1739        {
1740            let mut encoder =
1741                dbn::encode::dbn::Encoder::with_zstd(&mut encoded, &metadata_source).unwrap();
1742            encoder.encode_record(&definition).unwrap();
1743            encoder.flush().unwrap();
1744        }
1745
1746        std::fs::write(&replay_path, encoded).unwrap();
1747        let replayed = loader
1748            .load_instruments(&replay_path, false, false, None)
1749            .unwrap();
1750        std::fs::remove_file(replay_path).unwrap();
1751        assert_eq!(replayed[0].id(), expected);
1752        let mut record = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1753        record.hd.publisher_id = definition.hd.publisher_id;
1754        record.hd.instrument_id = definition.hd.instrument_id;
1755        record.ts_recv = definition.ts_recv;
1756        let metadata = dbn::Metadata::builder()
1757            .dataset("GLBX.MDP3")
1758            .schema(Some(dbn::Schema::Bbo1S))
1759            .start(record.ts_recv)
1760            .stype_in(Some(dbn::SType::RawSymbol))
1761            .stype_out(dbn::SType::InstrumentId)
1762            .mappings(vec![dbn::SymbolMapping {
1763                raw_symbol: raw.to_string(),
1764                intervals: vec![dbn::MappingInterval {
1765                    start_date: time::macros::date!(2028 - 01 - 01),
1766                    end_date: time::macros::date!(2028 - 01 - 03),
1767                    symbol: record.hd.instrument_id.to_string(),
1768                }],
1769            }])
1770            .build();
1771        let mut metadata = Some(MetadataCache::new(metadata));
1772        let id = loader
1773            .resolve_record_instrument_id(&dbn::RecordRef::from(&record), None, &mut metadata)
1774            .unwrap();
1775        assert_eq!(id, expected);
1776        record.ts_recv += 86_400_000_000_000;
1777        let next_day = loader
1778            .resolve_record_instrument_id(&dbn::RecordRef::from(&record), None, &mut metadata)
1779            .unwrap();
1780        assert_eq!(next_day, inferred);
1781    }
1782}