Skip to main content

nautilus_databento/
loader.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    env,
18    path::{Path, PathBuf},
19};
20
21use ahash::AHashMap;
22use anyhow::Context;
23use databento::dbn::{self, InstrumentDefMsg};
24use dbn::{
25    Publisher,
26    decode::{DbnMetadata, DecodeStream, dbn::Decoder},
27};
28use fallible_streaming_iterator::FallibleStreamingIterator;
29use indexmap::IndexMap;
30use nautilus_model::{
31    data::{Bar, Data, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
32    identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
33    instruments::{Instrument, InstrumentAny},
34};
35
36use super::{
37    decode::{
38        MboDeltaBuffer, decode_imbalance_msg, decode_mbo_msg, decode_record, decode_statistics_msg,
39        decode_status_msg, is_supported_stat_type,
40    },
41    symbology::{decode_nautilus_instrument_id, nautilus_symbol, resolve_futures_instrument_id},
42    types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, Dataset, PublisherId},
43};
44use crate::{
45    common::{build_publisher_venue_map, load_publishers},
46    decode::{DatabentoDecodeConfig, decode_instrument_def_msg},
47    symbology::MetadataCache,
48};
49
50/// A Nautilus data loader for Databento Binary Encoding (DBN) format data.
51///
52/// # Supported Schemas
53///  - `MBO` -> `OrderBookDelta`
54///  - `MBP_1` -> `(QuoteTick, Option<TradeTick>)`
55///  - `MBP_10` -> `OrderBookDepth`
56///  - `BBO_1S` -> `QuoteTick`
57///  - `BBO_1M` -> `QuoteTick`
58///  - `CMBP_1` -> `(QuoteTick, Option<TradeTick>)`
59///  - `CBBO_1S` -> `QuoteTick`
60///  - `CBBO_1M` -> `QuoteTick`
61///  - `TCBBO` -> `(QuoteTick, TradeTick)`
62///  - `TBBO` -> `(QuoteTick, TradeTick)`
63///  - `TRADES` -> `TradeTick`
64///  - `OHLCV_1S` -> `Bar`
65///  - `OHLCV_1M` -> `Bar`
66///  - `OHLCV_1H` -> `Bar`
67///  - `OHLCV_1D` -> `Bar`
68///  - `OHLCV_EOD` -> `Bar`
69///  - `DEFINITION` -> `Instrument`
70///  - `IMBALANCE` -> `DatabentoImbalance`
71///  - `STATISTICS` -> `DatabentoStatistics`
72///  - `STATUS` -> `InstrumentStatus`
73///
74/// # References
75///
76/// <https://databento.com/docs/schemas-and-data-formats>
77#[cfg_attr(
78    feature = "python",
79    pyo3::pyclass(module = "nautilus_trader.adapters.databento")
80)]
81#[cfg_attr(
82    feature = "python",
83    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
84)]
85#[derive(Debug)]
86pub struct DatabentoDataLoader {
87    futures_year_digits: FuturesYearDigits,
88    share_class_space: bool,
89    publishers_map: IndexMap<PublisherId, DatabentoPublisher>,
90    venue_dataset_map: IndexMap<Venue, Dataset>,
91    publisher_venue_map: IndexMap<PublisherId, Venue>,
92    symbol_venue_map: AHashMap<Symbol, Venue>,
93    price_precisions: AHashMap<Symbol, u8>,
94    futures_symbols: AHashMap<(PublisherId, u32, time::Date), Symbol>,
95}
96
97impl DatabentoDataLoader {
98    /// Creates a new [`DatabentoDataLoader`] instance.
99    ///
100    /// # Errors
101    ///
102    /// Returns an error if locating or loading publishers data fails.
103    pub fn new(publishers_filepath: Option<PathBuf>) -> anyhow::Result<Self> {
104        let mut loader = Self {
105            futures_year_digits: FuturesYearDigits::default(),
106            share_class_space: true,
107            publishers_map: IndexMap::new(),
108            venue_dataset_map: IndexMap::new(),
109            publisher_venue_map: IndexMap::new(),
110            symbol_venue_map: AHashMap::new(),
111            price_precisions: AHashMap::new(),
112            futures_symbols: AHashMap::new(),
113        };
114
115        // Load publishers
116        let publishers_filepath = if let Some(p) = publishers_filepath {
117            p
118        } else {
119            // Use built-in publishers path
120            let mut exe_path = env::current_exe()?;
121            exe_path.pop();
122            exe_path.push("publishers.json");
123            exe_path
124        };
125
126        loader
127            .load_publishers(publishers_filepath)
128            .context("error loading publishers.json")?;
129
130        Ok(loader)
131    }
132
133    /// Selects one- or two-digit years for futures instrument IDs.
134    #[must_use]
135    pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
136        self.futures_year_digits = futures_year_digits;
137        self
138    }
139
140    /// Spells a share class with a space in instrument IDs (`BRK.B` -> `BRK B`).
141    #[must_use]
142    pub fn with_share_class_space(mut self, share_class_space: bool) -> Self {
143        self.share_class_space = share_class_space;
144        self
145    }
146
147    /// Load the publishers data from the file at the given `filepath`.
148    ///
149    /// # Errors
150    ///
151    /// Returns an error if the file cannot be read or parsed as JSON.
152    pub fn load_publishers(&mut self, filepath: PathBuf) -> anyhow::Result<()> {
153        let publishers = load_publishers(filepath)?;
154
155        self.publishers_map = publishers
156            .iter()
157            .cloned()
158            .map(|p| (p.publisher_id, p))
159            .collect();
160
161        let mut venue_dataset_map = IndexMap::new();
162
163        // Only insert a dataset if the venue key is not already in the map
164        for publisher in &publishers {
165            let venue = Venue::from(publisher.venue.as_str());
166            let dataset = Dataset::from(publisher.dataset.as_str());
167            venue_dataset_map.entry(venue).or_insert(dataset);
168        }
169
170        self.venue_dataset_map = venue_dataset_map;
171        apply_default_venue_dataset_mappings(&mut self.venue_dataset_map);
172
173        self.publisher_venue_map = build_publisher_venue_map(&publishers);
174
175        Ok(())
176    }
177
178    /// Returns the internal Databento publishers currently held by the loader.
179    #[must_use]
180    pub const fn get_publishers(&self) -> &IndexMap<u16, DatabentoPublisher> {
181        &self.publishers_map
182    }
183
184    /// Sets the `venue` to map to the given `dataset`.
185    pub fn set_dataset_for_venue(&mut self, dataset: Dataset, venue: Venue) {
186        _ = self.venue_dataset_map.insert(venue, dataset);
187    }
188
189    /// Returns the dataset which matches the given `venue` (if found).
190    #[must_use]
191    pub fn get_dataset_for_venue(&self, venue: &Venue) -> Option<&Dataset> {
192        self.venue_dataset_map.get(venue)
193    }
194
195    /// Returns the venue which matches the given `publisher_id` (if found).
196    #[must_use]
197    pub fn get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option<&Venue> {
198        self.publisher_venue_map.get(&publisher_id)
199    }
200
201    /// Caches a `price_precision` for the given `symbol`.
202    ///
203    /// When market data is read without an explicit `price_precision` argument,
204    /// the loader resolves precision per record from this cache. Definitions
205    /// loaded via [`Self::load_instruments`] are inserted automatically.
206    pub fn set_price_precision(&mut self, symbol: Symbol, price_precision: u8) {
207        self.price_precisions.insert(symbol, price_precision);
208    }
209
210    /// Returns the cached price precisions keyed by symbol.
211    #[must_use]
212    pub const fn get_price_precisions(&self) -> &AHashMap<Symbol, u8> {
213        &self.price_precisions
214    }
215
216    /// Resolves a price precision for the given `instrument_id`.
217    ///
218    /// Resolution order:
219    /// 1. The explicit `price_precision` argument (if `Some`).
220    /// 2. The cached precision for the instrument's symbol.
221    ///
222    /// # Errors
223    ///
224    /// Returns an error if no precision is available.
225    fn resolve_price_precision(
226        &self,
227        instrument_id: &InstrumentId,
228        price_precision: Option<u8>,
229    ) -> anyhow::Result<u8> {
230        if let Some(precision) = price_precision {
231            return Ok(precision);
232        }
233
234        self.price_precisions
235            .get(&instrument_id.symbol)
236            .copied()
237            .ok_or_else(|| {
238                anyhow::anyhow!(
239                    "Could not resolve `price_precision` for {instrument_id}: \
240                     pass `price_precision` explicitly, call `set_price_precision`, \
241                     or load the instrument definitions first via `load_instruments`"
242                )
243            })
244    }
245
246    /// Returns the schema for the given `filepath`.
247    ///
248    /// # Errors
249    ///
250    /// Returns an error if the file cannot be decoded or metadata retrieval fails.
251    pub fn schema_from_file(&self, filepath: &Path) -> anyhow::Result<Option<String>> {
252        let decoder = Decoder::from_zstd_file(filepath)?;
253        let metadata = decoder.metadata();
254        Ok(metadata.schema.map(|schema| schema.to_string()))
255    }
256
257    /// Reads instrument definition records from a DBN file.
258    ///
259    /// # Errors
260    ///
261    /// Returns an error if decoding the definition records fails.
262    pub fn read_definition_records<'a>(
263        &'a mut self,
264        filepath: &Path,
265        use_exchange_as_venue: bool,
266        decode_config: Option<&'a DatabentoDecodeConfig>,
267    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<InstrumentAny>> + 'a> {
268        let decoder = Decoder::from_zstd_file(filepath)?;
269        let mut dbn_stream = decoder.decode_stream::<InstrumentDefMsg>();
270
271        // Loop over skipped records (Ok(None)) so one unsupported class does not
272        // terminate the stream
273        Ok(std::iter::from_fn(move || {
274            loop {
275                let advance = dbn_stream
276                    .advance()
277                    .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"));
278                if let Err(e) = advance {
279                    return Some(Err(e));
280                }
281
282                let rec = dbn_stream.get()?;
283
284                let result: anyhow::Result<Option<InstrumentAny>> = (|| {
285                    let record = dbn::RecordRef::from(rec);
286                    let msg = record
287                        .get::<InstrumentDefMsg>()
288                        .ok_or_else(|| anyhow::anyhow!("Failed to decode InstrumentDefMsg"))?;
289
290                    let raw_symbol = rec
291                        .raw_symbol()
292                        .map_err(|e| anyhow::anyhow!("Error decoding `raw_symbol`: {e}"))?;
293                    let symbol = nautilus_symbol(raw_symbol, self.share_class_space);
294
295                    let publisher = rec
296                        .hd
297                        .publisher()
298                        .map_err(|e| anyhow::anyhow!("Invalid `publisher` for record: {e}"))?;
299                    let venue = match publisher {
300                        Publisher::GlbxMdp3Glbx if use_exchange_as_venue => {
301                            let exchange = rec.exchange().map_err(|e| {
302                                anyhow::anyhow!("Missing `exchange` for record: {e}")
303                            })?;
304                            let venue = Venue::from_code(exchange).map_err(|e| {
305                                anyhow::anyhow!("Venue not found for exchange {exchange}: {e}")
306                            })?;
307                            self.symbol_venue_map.insert(symbol, venue);
308                            venue
309                        }
310                        _ => *self
311                            .publisher_venue_map
312                            .get(&msg.hd.publisher_id)
313                            .ok_or_else(|| {
314                                anyhow::anyhow!(
315                                    "Venue not found for publisher_id {}",
316                                    msg.hd.publisher_id
317                                )
318                            })?,
319                    };
320                    let instrument_id = InstrumentId::new(symbol, venue);
321                    let instrument_id = resolve_futures_instrument_id(
322                        &record,
323                        instrument_id,
324                        self.futures_year_digits,
325                        &self.symbol_venue_map,
326                        None,
327                    )?;
328                    self.symbol_venue_map.insert(instrument_id.symbol, venue);
329                    let ts_init = msg.ts_recv.into();
330
331                    let instrument = decode_instrument_def_msg(
332                        rec,
333                        instrument_id,
334                        Some(ts_init),
335                        decode_config,
336                    )?;
337
338                    if instrument.is_some() && instrument_id.symbol != symbol {
339                        let date = time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(
340                            msg.ts_recv,
341                        ))?
342                        .date();
343                        self.futures_symbols.insert(
344                            (msg.hd.publisher_id, msg.hd.instrument_id, date),
345                            instrument_id.symbol,
346                        );
347                    }
348
349                    Ok(instrument)
350                })();
351
352                match result {
353                    Ok(Some(item)) => return Some(Ok(item)),
354                    Ok(None) => {}
355                    Err(e) => return Some(Err(e)),
356                }
357            }
358        }))
359    }
360
361    /// Reads and decodes market data records from a DBN file.
362    ///
363    /// # Errors
364    ///
365    /// Returns an error if reading records fails.
366    pub fn read_records<T>(
367        &self,
368        filepath: &Path,
369        instrument_id: Option<InstrumentId>,
370        price_precision: Option<u8>,
371        include_trades: bool,
372        bars_timestamp_on_close: Option<bool>,
373    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<(Option<Data>, Option<Data>)>> + '_>
374    where
375        T: dbn::Record + dbn::HasRType + 'static,
376    {
377        let decoder = Decoder::from_zstd_file(filepath)?;
378        let mut metadata_cache = if instrument_id.is_none() {
379            Some(MetadataCache::new(decoder.metadata().clone()))
380        } else {
381            None
382        };
383        let mut dbn_stream = decoder.decode_stream::<T>();
384        let fixed_instrument_id = instrument_id.is_some();
385        let mut fixed_price_precision = price_precision;
386
387        Ok(std::iter::from_fn(move || {
388            let result: anyhow::Result<Option<(Option<Data>, Option<Data>)>> = (|| {
389                dbn_stream
390                    .advance()
391                    .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"))?;
392
393                if let Some(rec) = dbn_stream.get() {
394                    let record = dbn::RecordRef::from(rec);
395                    let instrument_id = self
396                        .resolve_record_instrument_id(&record, instrument_id, &mut metadata_cache)
397                        .context("failed to decode instrument id")?;
398                    let resolved_precision = self.resolve_stream_price_precision(
399                        &instrument_id,
400                        fixed_instrument_id,
401                        &mut fixed_price_precision,
402                    )?;
403                    let (item1, item2) = decode_record(
404                        &record,
405                        instrument_id,
406                        resolved_precision,
407                        None,
408                        include_trades,
409                        bars_timestamp_on_close.unwrap_or(true),
410                    )?;
411                    Ok(Some((item1, item2)))
412                } else {
413                    Ok(None)
414                }
415            })();
416
417            match result {
418                Ok(Some(v)) => Some(Ok(v)),
419                Ok(None) => None,
420                Err(e) => Some(Err(e)),
421            }
422        }))
423    }
424
425    /// Loads all instrument definitions from a DBN file.
426    ///
427    /// When `skip_on_error` is true, instruments that fail to decode are logged
428    /// as warnings and skipped. When false (default), any decode error is propagated.
429    ///
430    /// # Errors
431    ///
432    /// Returns an error if loading instruments fails.
433    pub fn load_instruments(
434        &mut self,
435        filepath: &Path,
436        use_exchange_as_venue: bool,
437        skip_on_error: bool,
438        decode_config: Option<&DatabentoDecodeConfig>,
439    ) -> anyhow::Result<Vec<InstrumentAny>> {
440        let instruments = if skip_on_error {
441            let mut collected = Vec::new();
442
443            for result in
444                self.read_definition_records(filepath, use_exchange_as_venue, decode_config)?
445            {
446                match result {
447                    Ok(instrument) => collected.push(instrument),
448                    Err(e) => log::warn!("Skipping instrument: {e}"),
449                }
450            }
451            collected
452        } else {
453            self.read_definition_records(filepath, use_exchange_as_venue, decode_config)?
454                .collect::<Result<Vec<_>, _>>()?
455        };
456
457        for instrument in &instruments {
458            self.price_precisions
459                .insert(instrument.id().symbol, instrument.price_precision());
460        }
461
462        Ok(instruments)
463    }
464
465    /// Loads order book delta messages from a DBN MBO schema file.
466    ///
467    /// Cannot include trades.
468    ///
469    /// # Errors
470    ///
471    /// Returns an error if loading order book deltas fails.
472    pub fn load_order_book_deltas(
473        &self,
474        filepath: &Path,
475        instrument_id: Option<InstrumentId>,
476        price_precision: Option<u8>,
477    ) -> anyhow::Result<Vec<OrderBookDelta>> {
478        self.read_order_book_deltas(filepath, instrument_id, price_precision)?
479            .collect()
480    }
481
482    /// Reads order book delta messages from a DBN MBO schema file without collecting them.
483    ///
484    /// Cannot include trades.
485    ///
486    /// # Errors
487    ///
488    /// Returns an error if opening or decoding order book deltas fails.
489    pub fn read_order_book_deltas(
490        &self,
491        filepath: &Path,
492        instrument_id: Option<InstrumentId>,
493        price_precision: Option<u8>,
494    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<OrderBookDelta>> + '_> {
495        let decoder = Decoder::from_zstd_file(filepath)?;
496        let mut metadata_cache = if instrument_id.is_none() {
497            Some(MetadataCache::new(decoder.metadata().clone()))
498        } else {
499            None
500        };
501        let mut dbn_stream = decoder.decode_stream::<dbn::MboMsg>();
502        let fixed_instrument_id = instrument_id.is_some();
503        let mut fixed_price_precision = price_precision;
504        let mut delta_buffer = MboDeltaBuffer::default();
505        let mut terminal_error = None;
506        let mut finished = false;
507
508        Ok(std::iter::from_fn(move || {
509            loop {
510                if let Some(delta) = delta_buffer.pop_ready() {
511                    return Some(Ok(delta));
512                }
513
514                if finished {
515                    return terminal_error.take().map(Err);
516                }
517
518                let result: anyhow::Result<bool> = (|| {
519                    dbn_stream
520                        .advance()
521                        .map_err(|e| anyhow::anyhow!("Stream advance error: {e}"))?;
522
523                    let Some(rec) = dbn_stream.get() else {
524                        return Ok(false);
525                    };
526                    let record = dbn::RecordRef::from(rec);
527                    let instrument_id = self
528                        .resolve_record_instrument_id(&record, instrument_id, &mut metadata_cache)
529                        .context("failed to decode instrument id")?;
530                    let resolved_precision = self.resolve_stream_price_precision(
531                        &instrument_id,
532                        fixed_instrument_id,
533                        &mut fixed_price_precision,
534                    )?;
535                    let msg = record
536                        .get::<dbn::MboMsg>()
537                        .ok_or_else(|| anyhow::anyhow!("Failed to decode MboMsg"))?;
538                    let (delta, _trade) =
539                        decode_mbo_msg(msg, instrument_id, resolved_precision, None, false)?;
540                    delta_buffer.push(msg, instrument_id, delta);
541                    Ok(true)
542                })();
543
544                match result {
545                    Ok(true) => {}
546                    Ok(false) => {
547                        delta_buffer.finish();
548                        finished = true;
549                    }
550                    Err(e) => {
551                        delta_buffer.finish();
552                        terminal_error = Some(e);
553                        finished = true;
554                    }
555                }
556            }
557        }))
558    }
559
560    /// Loads order book depth snapshots from a DBN MBP-10 schema file.
561    ///
562    /// # Errors
563    ///
564    /// Returns an error if loading order book depth fails.
565    pub fn load_order_book_depth(
566        &self,
567        filepath: &Path,
568        instrument_id: Option<InstrumentId>,
569        price_precision: Option<u8>,
570    ) -> anyhow::Result<Vec<OrderBookDepth>> {
571        self.read_records::<dbn::Mbp10Msg>(filepath, instrument_id, price_precision, false, None)?
572            .filter_map(|result| match result {
573                Ok((Some(item1), _)) => {
574                    if let Data::BookDepth(depth) = item1 {
575                        Some(Ok(*depth))
576                    } else {
577                        None
578                    }
579                }
580                Ok((None, _)) => None,
581                Err(e) => Some(Err(e)),
582            })
583            .collect()
584    }
585
586    /// Loads quote tick messages from a DBN MBP-1 or TBBO schema file.
587    ///
588    /// # Errors
589    ///
590    /// Returns an error if loading quotes fails.
591    pub fn load_quotes(
592        &self,
593        filepath: &Path,
594        instrument_id: Option<InstrumentId>,
595        price_precision: Option<u8>,
596    ) -> anyhow::Result<Vec<QuoteTick>> {
597        self.read_records::<dbn::Mbp1Msg>(filepath, instrument_id, price_precision, false, None)?
598            .filter_map(|result| match result {
599                Ok((Some(item1), _)) => {
600                    if let Data::Quote(quote) = item1 {
601                        Some(Ok(quote))
602                    } else {
603                        None
604                    }
605                }
606                Ok((None, _)) => None,
607                Err(e) => Some(Err(e)),
608            })
609            .collect()
610    }
611
612    /// Loads best bid/offer quote messages from a DBN BBO schema file.
613    ///
614    /// # Errors
615    ///
616    /// Returns an error if loading BBO quotes fails.
617    pub fn load_bbo_quotes(
618        &self,
619        filepath: &Path,
620        instrument_id: Option<InstrumentId>,
621        price_precision: Option<u8>,
622    ) -> anyhow::Result<Vec<QuoteTick>> {
623        self.read_records::<dbn::BboMsg>(filepath, instrument_id, price_precision, false, None)?
624            .filter_map(|result| match result {
625                Ok((Some(item1), _)) => {
626                    if let Data::Quote(quote) = item1 {
627                        Some(Ok(quote))
628                    } else {
629                        None
630                    }
631                }
632                Ok((None, _)) => None,
633                Err(e) => Some(Err(e)),
634            })
635            .collect()
636    }
637
638    /// Loads consolidated MBP-1 quote messages from a DBN CMBP-1 schema file.
639    ///
640    /// # Errors
641    ///
642    /// Returns an error if loading consolidated MBP-1 quotes fails.
643    pub fn load_cmbp_quotes(
644        &self,
645        filepath: &Path,
646        instrument_id: Option<InstrumentId>,
647        price_precision: Option<u8>,
648    ) -> anyhow::Result<Vec<QuoteTick>> {
649        self.read_records::<dbn::Cmbp1Msg>(filepath, instrument_id, price_precision, false, None)?
650            .filter_map(|result| match result {
651                Ok((Some(item1), _)) => {
652                    if let Data::Quote(quote) = item1 {
653                        Some(Ok(quote))
654                    } else {
655                        None
656                    }
657                }
658                Ok((None, _)) => None,
659                Err(e) => Some(Err(e)),
660            })
661            .collect()
662    }
663
664    /// Loads consolidated best bid/offer quote messages from a DBN CBBO schema file.
665    ///
666    /// # Errors
667    ///
668    /// Returns an error if loading consolidated BBO quotes fails.
669    pub fn load_cbbo_quotes(
670        &self,
671        filepath: &Path,
672        instrument_id: Option<InstrumentId>,
673        price_precision: Option<u8>,
674    ) -> anyhow::Result<Vec<QuoteTick>> {
675        self.read_records::<dbn::CbboMsg>(filepath, instrument_id, price_precision, false, None)?
676            .filter_map(|result| match result {
677                Ok((Some(item1), _)) => {
678                    if let Data::Quote(quote) = item1 {
679                        Some(Ok(quote))
680                    } else {
681                        None
682                    }
683                }
684                Ok((None, _)) => None,
685                Err(e) => Some(Err(e)),
686            })
687            .collect()
688    }
689
690    /// Loads trade messages from a DBN TBBO schema file.
691    ///
692    /// # Errors
693    ///
694    /// Returns an error if loading TBBO trades fails.
695    pub fn load_tbbo_trades(
696        &self,
697        filepath: &Path,
698        instrument_id: Option<InstrumentId>,
699        price_precision: Option<u8>,
700    ) -> anyhow::Result<Vec<TradeTick>> {
701        self.read_records::<dbn::TbboMsg>(filepath, instrument_id, price_precision, true, None)?
702            .filter_map(|result| match result {
703                Ok((_, maybe_item2)) => {
704                    if let Some(Data::Trade(trade)) = maybe_item2 {
705                        Some(Ok(trade))
706                    } else {
707                        None
708                    }
709                }
710                Err(e) => Some(Err(e)),
711            })
712            .collect()
713    }
714
715    /// Loads trade messages from a DBN TCBBO schema file.
716    ///
717    /// # Errors
718    ///
719    /// Returns an error if loading TCBBO trades fails.
720    pub fn load_tcbbo_trades(
721        &self,
722        filepath: &Path,
723        instrument_id: Option<InstrumentId>,
724        price_precision: Option<u8>,
725    ) -> anyhow::Result<Vec<TradeTick>> {
726        self.read_records::<dbn::TcbboMsg>(filepath, instrument_id, price_precision, true, None)?
727            .filter_map(|result| match result {
728                Ok((_, maybe_item2)) => {
729                    if let Some(Data::Trade(trade)) = maybe_item2 {
730                        Some(Ok(trade))
731                    } else {
732                        None
733                    }
734                }
735                Err(e) => Some(Err(e)),
736            })
737            .collect()
738    }
739
740    /// Loads trade messages from a DBN TRADES schema file.
741    ///
742    /// # Errors
743    ///
744    /// Returns an error if loading trades fails.
745    pub fn load_trades(
746        &self,
747        filepath: &Path,
748        instrument_id: Option<InstrumentId>,
749        price_precision: Option<u8>,
750    ) -> anyhow::Result<Vec<TradeTick>> {
751        self.read_records::<dbn::TradeMsg>(filepath, instrument_id, price_precision, false, None)?
752            .filter_map(|result| match result {
753                Ok((Some(item1), _)) => {
754                    if let Data::Trade(trade) = item1 {
755                        Some(Ok(trade))
756                    } else {
757                        None
758                    }
759                }
760                Ok((None, _)) => None,
761                Err(e) => Some(Err(e)),
762            })
763            .collect()
764    }
765
766    /// Loads OHLCV bar messages from a DBN OHLCV schema file.
767    ///
768    /// # Errors
769    ///
770    /// Returns an error if loading bars fails.
771    pub fn load_bars(
772        &self,
773        filepath: &Path,
774        instrument_id: Option<InstrumentId>,
775        price_precision: Option<u8>,
776        timestamp_on_close: Option<bool>,
777    ) -> anyhow::Result<Vec<Bar>> {
778        self.read_records::<dbn::OhlcvMsg>(
779            filepath,
780            instrument_id,
781            price_precision,
782            false,
783            timestamp_on_close,
784        )?
785        .filter_map(|result| match result {
786            Ok((Some(item1), _)) => {
787                if let Data::Bar(bar) = item1 {
788                    Some(Ok(bar))
789                } else {
790                    None
791                }
792            }
793            Ok((None, _)) => None,
794            Err(e) => Some(Err(e)),
795        })
796        .collect()
797    }
798
799    /// Loads instrument status messages from a DBN STATUS schema file.
800    ///
801    /// # Errors
802    ///
803    /// Returns an error if loading status records fails.
804    pub fn load_status_records<T>(
805        &self,
806        filepath: &Path,
807        instrument_id: Option<InstrumentId>,
808    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<InstrumentStatus>> + '_>
809    where
810        T: dbn::Record + dbn::HasRType + 'static,
811    {
812        let decoder = Decoder::from_zstd_file(filepath)?;
813        let mut metadata_cache = if instrument_id.is_none() {
814            Some(MetadataCache::new(decoder.metadata().clone()))
815        } else {
816            None
817        };
818        let mut dbn_stream = decoder.decode_stream::<T>();
819
820        Ok(std::iter::from_fn(move || {
821            if let Err(e) = dbn_stream.advance() {
822                return Some(Err(e.into()));
823            }
824
825            match dbn_stream.get() {
826                Some(rec) => {
827                    let record = dbn::RecordRef::from(rec);
828                    let instrument_id = match self.resolve_record_instrument_id(
829                        &record,
830                        instrument_id,
831                        &mut metadata_cache,
832                    ) {
833                        Ok(id) => id,
834                        Err(e) => return Some(Err(e)),
835                    };
836
837                    let msg = match record.get::<dbn::StatusMsg>() {
838                        Some(m) => m,
839                        None => return Some(Err(anyhow::anyhow!("Invalid `StatusMsg`"))),
840                    };
841                    let ts_init = msg.ts_recv.into();
842
843                    match decode_status_msg(msg, instrument_id, Some(ts_init)) {
844                        Ok(data) => Some(Ok(data)),
845                        Err(e) => Some(Err(e)),
846                    }
847                }
848                None => None,
849            }
850        }))
851    }
852
853    /// Reads imbalance messages from a DBN IMBALANCE schema file.
854    ///
855    /// # Errors
856    ///
857    /// Returns an error if reading imbalance records fails.
858    pub fn read_imbalance_records<T>(
859        &self,
860        filepath: &Path,
861        instrument_id: Option<InstrumentId>,
862        price_precision: Option<u8>,
863    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<DatabentoImbalance>> + '_>
864    where
865        T: dbn::Record + dbn::HasRType + 'static,
866    {
867        let decoder = Decoder::from_zstd_file(filepath)?;
868        let mut metadata_cache = if instrument_id.is_none() {
869            Some(MetadataCache::new(decoder.metadata().clone()))
870        } else {
871            None
872        };
873        let mut dbn_stream = decoder.decode_stream::<T>();
874        let fixed_instrument_id = instrument_id.is_some();
875        let mut fixed_price_precision = price_precision;
876
877        Ok(std::iter::from_fn(move || {
878            if let Err(e) = dbn_stream.advance() {
879                return Some(Err(e.into()));
880            }
881
882            match dbn_stream.get() {
883                Some(rec) => {
884                    let record = dbn::RecordRef::from(rec);
885                    let instrument_id = match self.resolve_record_instrument_id(
886                        &record,
887                        instrument_id,
888                        &mut metadata_cache,
889                    ) {
890                        Ok(id) => id,
891                        Err(e) => return Some(Err(e)),
892                    };
893                    let resolved_precision = match self.resolve_stream_price_precision(
894                        &instrument_id,
895                        fixed_instrument_id,
896                        &mut fixed_price_precision,
897                    ) {
898                        Ok(p) => p,
899                        Err(e) => return Some(Err(e)),
900                    };
901
902                    let msg = match record.get::<dbn::ImbalanceMsg>() {
903                        Some(m) => m,
904                        None => return Some(Err(anyhow::anyhow!("Invalid `ImbalanceMsg`"))),
905                    };
906                    let ts_init = msg.ts_recv.into();
907
908                    match decode_imbalance_msg(
909                        msg,
910                        instrument_id,
911                        resolved_precision,
912                        Some(ts_init),
913                    ) {
914                        Ok(data) => Some(Ok(data)),
915                        Err(e) => Some(Err(e)),
916                    }
917                }
918                None => None,
919            }
920        }))
921    }
922
923    /// Reads statistics messages from a DBN STATISTICS schema file.
924    ///
925    /// # Errors
926    ///
927    /// Returns an error if reading statistics records fails.
928    pub fn read_statistics_records<T>(
929        &self,
930        filepath: &Path,
931        instrument_id: Option<InstrumentId>,
932        price_precision: Option<u8>,
933    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<DatabentoStatistics>> + '_>
934    where
935        T: dbn::Record + dbn::HasRType + 'static,
936    {
937        let decoder = Decoder::from_zstd_file(filepath)?;
938        let mut metadata_cache = if instrument_id.is_none() {
939            Some(MetadataCache::new(decoder.metadata().clone()))
940        } else {
941            None
942        };
943        let mut dbn_stream = decoder.decode_stream::<T>();
944        let fixed_instrument_id = instrument_id.is_some();
945        let mut fixed_price_precision = price_precision;
946
947        // Loop over skipped records so one unsupported stat_type does not terminate
948        // the stream; precheck before precision resolution.
949        Ok(std::iter::from_fn(move || {
950            loop {
951                if let Err(e) = dbn_stream.advance() {
952                    return Some(Err(e.into()));
953                }
954
955                let rec = dbn_stream.get()?;
956                let record = dbn::RecordRef::from(rec);
957                let msg = match record.get::<dbn::StatMsg>() {
958                    Some(m) => m,
959                    None => return Some(Err(anyhow::anyhow!("Invalid `StatMsg`"))),
960                };
961
962                if !is_supported_stat_type(msg.stat_type) {
963                    log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
964                    continue;
965                }
966
967                let instrument_id = match self.resolve_record_instrument_id(
968                    &record,
969                    instrument_id,
970                    &mut metadata_cache,
971                ) {
972                    Ok(id) => id,
973                    Err(e) => return Some(Err(e)),
974                };
975                let resolved_precision = match self.resolve_stream_price_precision(
976                    &instrument_id,
977                    fixed_instrument_id,
978                    &mut fixed_price_precision,
979                ) {
980                    Ok(p) => p,
981                    Err(e) => return Some(Err(e)),
982                };
983                let ts_init = msg.ts_recv.into();
984
985                match decode_statistics_msg(msg, instrument_id, resolved_precision, Some(ts_init)) {
986                    Ok(Some(data)) => return Some(Ok(data)),
987                    Ok(None) => {}
988                    Err(e) => return Some(Err(e)),
989                }
990            }
991        }))
992    }
993
994    fn resolve_record_instrument_id(
995        &self,
996        record: &dbn::RecordRef,
997        instrument_id: Option<InstrumentId>,
998        metadata_cache: &mut Option<MetadataCache>,
999    ) -> anyhow::Result<InstrumentId> {
1000        if let Some(instrument_id) = instrument_id {
1001            return Ok(instrument_id);
1002        }
1003
1004        let metadata_cache = metadata_cache
1005            .as_mut()
1006            .ok_or_else(|| anyhow::anyhow!("missing metadata cache for dynamic instrument id"))?;
1007
1008        decode_nautilus_instrument_id(
1009            record,
1010            metadata_cache,
1011            &self.publisher_venue_map,
1012            &self.symbol_venue_map,
1013            self.futures_year_digits,
1014            self.share_class_space,
1015            &self.futures_symbols,
1016            None,
1017        )
1018    }
1019
1020    fn resolve_stream_price_precision(
1021        &self,
1022        instrument_id: &InstrumentId,
1023        fixed_instrument_id: bool,
1024        fixed_price_precision: &mut Option<u8>,
1025    ) -> anyhow::Result<u8> {
1026        if let Some(precision) = *fixed_price_precision {
1027            return Ok(precision);
1028        }
1029
1030        let precision = self.resolve_price_precision(instrument_id, None)?;
1031        if fixed_instrument_id {
1032            *fixed_price_precision = Some(precision);
1033        }
1034
1035        Ok(precision)
1036    }
1037}
1038
1039/// Applies default venue-to-dataset mappings for consolidated Databento feeds.
1040/// GLBX.MDP3 covers CME Globex exchange MICs; OPRA.PILLAR covers OPRA option venues;
1041/// EQUS.MINI is the consolidated US equities default.
1042fn apply_default_venue_dataset_mappings(venue_dataset_map: &mut IndexMap<Venue, Dataset>) {
1043    let glbx = Dataset::from("GLBX.MDP3");
1044
1045    for venue in [
1046        Venue::CBCM(),
1047        Venue::GLBX(),
1048        Venue::NYUM(),
1049        Venue::XCBT(),
1050        Venue::XCEC(),
1051        Venue::XCME(),
1052        Venue::XFXS(),
1053        Venue::XNYM(),
1054    ] {
1055        _ = venue_dataset_map.insert(venue, glbx);
1056    }
1057
1058    // publishers.json seeds the consolidated EQUS venue with the unreleased EQUS.PLUS,
1059    // so pin it to EQUS.MINI, the cheapest released US equities feed.
1060    _ = venue_dataset_map.insert(Venue::from("EQUS"), Dataset::from("EQUS.MINI"));
1061
1062    let opra = Dataset::from("OPRA.PILLAR");
1063    for venue_code in [
1064        "AMXO", "XBOX", "XCBO", "EMLD", "EDGO", "GMNI", "XISX", "MCRY", "XMIO", "ARCO", "OPRA",
1065        "MPRL", "XNDQ", "XBXO", "C2OX", "XPHL", "BATO", "MXOP", "SPHR",
1066    ] {
1067        _ = venue_dataset_map.insert(Venue::from(venue_code), opra);
1068    }
1069}
1070
1071#[cfg(test)]
1072mod tests {
1073    use std::{
1074        ffi::c_char,
1075        fs::{File, OpenOptions},
1076        io::Write,
1077        path::{Path, PathBuf},
1078        process,
1079        sync::atomic::{AtomicU64, Ordering},
1080    };
1081
1082    use databento::dbn::{decode::DecodeRecord, encode::EncodeRecord};
1083    use nautilus_model::{
1084        enums::BookAction,
1085        types::{Price, Quantity},
1086    };
1087    use rstest::{fixture, rstest};
1088    use ustr::Ustr;
1089
1090    use super::*;
1091
1092    fn test_data_path() -> PathBuf {
1093        Path::new(env!("CARGO_MANIFEST_DIR")).join("test_data")
1094    }
1095
1096    fn mbo_record(action: c_char, flags: u8, sequence: u32) -> dbn::MboMsg {
1097        let ts_event = 1_609_160_400_000_000_000;
1098        dbn::MboMsg {
1099            hd: dbn::RecordHeader::new::<dbn::MboMsg>(dbn::rtype::MBO, 1, 1, ts_event),
1100            order_id: 42,
1101            price: 4_800_250_000_000,
1102            size: 2,
1103            flags: dbn::FlagSet::new(flags),
1104            channel_id: 1,
1105            action,
1106            side: 'A' as c_char,
1107            ts_recv: ts_event,
1108            ts_in_delta: 0,
1109            sequence,
1110        }
1111    }
1112
1113    fn write_mbo_records(records: &[dbn::MboMsg]) -> PathBuf {
1114        static FILE_ID: AtomicU64 = AtomicU64::new(0);
1115
1116        let id = FILE_ID.fetch_add(1, Ordering::Relaxed);
1117        let path = std::env::temp_dir().join(format!(
1118            "nautilus-databento-mbo-{}-{id}.dbn.zst",
1119            process::id(),
1120        ));
1121        let metadata = dbn::Metadata::builder()
1122            .dataset("GLBX.MDP3")
1123            .schema(Some(dbn::Schema::Mbo))
1124            .start(0)
1125            .stype_in(Some(dbn::SType::InstrumentId))
1126            .stype_out(dbn::SType::InstrumentId)
1127            .build();
1128        let file = File::create(&path).unwrap();
1129        let mut encoder = dbn::encode::dbn::Encoder::with_zstd(file, &metadata).unwrap();
1130        encoder.encode_records(records).unwrap();
1131        encoder.flush().unwrap();
1132        drop(encoder);
1133
1134        path
1135    }
1136
1137    #[fixture]
1138    fn loader() -> DatabentoDataLoader {
1139        let publishers_filepath = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1140        let mut loader = DatabentoDataLoader::new(Some(publishers_filepath)).unwrap();
1141        // ES futures test data uses precision 2 (USD cents)
1142        loader.set_price_precision(Symbol::from("ESM4"), 2);
1143        loader
1144    }
1145
1146    #[fixture]
1147    fn loader_without_seed() -> DatabentoDataLoader {
1148        let publishers_filepath = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1149        DatabentoDataLoader::new(Some(publishers_filepath)).unwrap()
1150    }
1151
1152    // TODO: Improve the below assertions that we've actually read the records we expected
1153
1154    #[rstest]
1155    fn test_set_dataset_venue_mapping(mut loader: DatabentoDataLoader) {
1156        let dataset = Ustr::from("EQUS.PLUS");
1157        let venue = Venue::from("XNAS");
1158        loader.set_dataset_for_venue(dataset, venue);
1159
1160        let result = loader.get_dataset_for_venue(&venue).unwrap();
1161        assert_eq!(*result, dataset);
1162    }
1163
1164    #[rstest]
1165    fn test_default_venue_dataset_mappings(loader: DatabentoDataLoader) {
1166        let xcme = Venue::XCME();
1167        let result = loader.get_dataset_for_venue(&xcme).unwrap();
1168        assert_eq!(*result, Ustr::from("GLBX.MDP3"));
1169
1170        let xcbo = Venue::from("XCBO");
1171        let result = loader.get_dataset_for_venue(&xcbo).unwrap();
1172        assert_eq!(*result, Ustr::from("OPRA.PILLAR"));
1173
1174        let equs = Venue::from("EQUS");
1175        let result = loader.get_dataset_for_venue(&equs).unwrap();
1176        assert_eq!(*result, Ustr::from("EQUS.MINI"));
1177    }
1178
1179    #[rstest]
1180    #[case(test_data_path().join("test_data.definition.equity.dbn.zst"))]
1181    fn test_load_instruments(mut loader: DatabentoDataLoader, #[case] path: PathBuf) {
1182        let instruments = loader.load_instruments(&path, false, false, None).unwrap();
1183
1184        assert_eq!(instruments.len(), 2);
1185        // Definition records auto-populate the precision cache
1186        assert_eq!(
1187            loader.get_price_precisions().get(&Symbol::from("ESM4")),
1188            Some(&2)
1189        );
1190    }
1191
1192    #[rstest]
1193    fn test_load_instruments_populates_price_precisions_cache(
1194        mut loader_without_seed: DatabentoDataLoader,
1195    ) {
1196        let path = test_data_path().join("test_data.definition.equity.dbn.zst");
1197        assert!(loader_without_seed.get_price_precisions().is_empty());
1198
1199        let instruments = loader_without_seed
1200            .load_instruments(&path, false, false, None)
1201            .unwrap();
1202
1203        assert_eq!(instruments.len(), 2);
1204        for instrument in &instruments {
1205            let symbol = instrument.id().symbol;
1206            assert_eq!(
1207                loader_without_seed.get_price_precisions().get(&symbol),
1208                Some(&instrument.price_precision()),
1209                "cache missing or mismatched entry for {symbol}",
1210            );
1211        }
1212    }
1213
1214    #[rstest]
1215    fn test_read_records_errors_when_precision_unresolvable(
1216        loader_without_seed: DatabentoDataLoader,
1217    ) {
1218        let path = test_data_path().join("test_data.mbo.dbn.zst");
1219        let instrument_id = InstrumentId::from("ESM4.GLBX");
1220
1221        let result = loader_without_seed.load_order_book_deltas(&path, Some(instrument_id), None);
1222
1223        let err = result.expect_err("expected precision-resolution error");
1224        let err_msg = format!("{err}");
1225        assert!(
1226            err_msg.contains("Could not resolve `price_precision`"),
1227            "unexpected error message: {err_msg}",
1228        );
1229        assert!(
1230            err_msg.contains("ESM4.GLBX"),
1231            "error should name the instrument: {err_msg}",
1232        );
1233    }
1234
1235    #[rstest]
1236    fn test_set_price_precision_unblocks_reads(mut loader_without_seed: DatabentoDataLoader) {
1237        let path = test_data_path().join("test_data.mbo.dbn.zst");
1238        let instrument_id = InstrumentId::from("ESM4.GLBX");
1239
1240        // Without a seeded precision the read errors
1241        assert!(
1242            loader_without_seed
1243                .load_order_book_deltas(&path, Some(instrument_id), None)
1244                .is_err()
1245        );
1246
1247        loader_without_seed.set_price_precision(Symbol::from("ESM4"), 2);
1248
1249        let deltas = loader_without_seed
1250            .load_order_book_deltas(&path, Some(instrument_id), None)
1251            .unwrap();
1252        assert_eq!(deltas.len(), 2);
1253    }
1254
1255    #[rstest]
1256    fn test_resolve_price_precision_explicit_arg_overrides_cache(
1257        mut loader_without_seed: DatabentoDataLoader,
1258    ) {
1259        let instrument_id = InstrumentId::from("ESM4.GLBX");
1260        // Seed a deliberately wrong cache value so we can detect which path is taken
1261        loader_without_seed.set_price_precision(Symbol::from("ESM4"), 9);
1262
1263        let explicit = loader_without_seed
1264            .resolve_price_precision(&instrument_id, Some(2))
1265            .unwrap();
1266        assert_eq!(explicit, 2);
1267
1268        let cached = loader_without_seed
1269            .resolve_price_precision(&instrument_id, None)
1270            .unwrap();
1271        assert_eq!(cached, 9);
1272    }
1273
1274    #[rstest]
1275    fn test_resolve_price_precision_cache_miss_errors(loader_without_seed: DatabentoDataLoader) {
1276        let instrument_id = InstrumentId::from("ESM4.GLBX");
1277
1278        let err = loader_without_seed
1279            .resolve_price_precision(&instrument_id, None)
1280            .expect_err("expected cache-miss error");
1281        assert!(format!("{err}").contains("Could not resolve `price_precision`"));
1282    }
1283
1284    #[rstest]
1285    fn test_load_order_book_deltas(loader: DatabentoDataLoader) {
1286        let path = test_data_path().join("test_data.mbo.dbn.zst");
1287        let instrument_id = InstrumentId::from("ESM4.GLBX");
1288
1289        let deltas = loader
1290            .load_order_book_deltas(&path, Some(instrument_id), None)
1291            .unwrap();
1292
1293        assert_eq!(deltas.len(), 2);
1294    }
1295
1296    #[rstest]
1297    fn test_read_order_book_deltas_streams_without_collecting(loader: DatabentoDataLoader) {
1298        let path = test_data_path().join("test_data.mbo.dbn.zst");
1299        let instrument_id = InstrumentId::from("ESM4.GLBX");
1300
1301        let count = loader
1302            .read_order_book_deltas(&path, Some(instrument_id), None)
1303            .unwrap()
1304            .map(|result| result.map(|_| 1usize))
1305            .sum::<anyhow::Result<usize>>()
1306            .unwrap();
1307
1308        assert_eq!(count, 2);
1309    }
1310
1311    #[rstest]
1312    #[case::standalone(false)]
1313    #[case::legacy_inline(true)]
1314    fn test_load_order_book_deltas_preserves_event_boundary(
1315        loader: DatabentoDataLoader,
1316        #[case] inline_last: bool,
1317    ) {
1318        let instrument_id = InstrumentId::from("ESM4.GLBX");
1319        let last = dbn::flags::LAST;
1320        let mut records = vec![mbo_record(
1321            'C' as c_char,
1322            if inline_last { last } else { 0 },
1323            1,
1324        )];
1325
1326        if !inline_last {
1327            records.push(mbo_record('N' as c_char, last, 2));
1328        }
1329        let path = write_mbo_records(&records);
1330
1331        let deltas = loader
1332            .load_order_book_deltas(&path, Some(instrument_id), Some(2))
1333            .unwrap();
1334        std::fs::remove_file(path).unwrap();
1335
1336        assert_eq!(deltas.len(), 1);
1337        assert_eq!(deltas[0].instrument_id, instrument_id);
1338        assert_eq!(deltas[0].action, BookAction::Delete);
1339        assert_eq!(deltas[0].order.order_id, 42);
1340        assert_eq!(deltas[0].flags, last);
1341        assert_eq!(deltas[0].sequence, 1);
1342    }
1343
1344    #[rstest]
1345    fn test_read_order_book_deltas_drains_before_terminal_error(loader: DatabentoDataLoader) {
1346        let instrument_id = InstrumentId::from("ESM4.GLBX");
1347        let path = write_mbo_records(&[mbo_record('C' as c_char, 0, 1)]);
1348        OpenOptions::new()
1349            .append(true)
1350            .open(&path)
1351            .unwrap()
1352            .write_all(&[0; 8])
1353            .unwrap();
1354        let mut deltas = loader
1355            .read_order_book_deltas(&path, Some(instrument_id), Some(2))
1356            .unwrap();
1357
1358        let delta = deltas.next().unwrap().unwrap();
1359        let error = deltas
1360            .next()
1361            .unwrap()
1362            .expect_err("expected trailing decode error");
1363        let end = deltas.next();
1364        std::fs::remove_file(path).unwrap();
1365
1366        assert_eq!(delta.instrument_id, instrument_id);
1367        assert_eq!(delta.order.order_id, 42);
1368        assert_eq!(delta.flags, 0);
1369        assert!(format!("{error}").contains("Stream advance error"));
1370        assert!(end.is_none());
1371    }
1372
1373    #[rstest]
1374    fn test_load_order_book_depth(loader: DatabentoDataLoader) {
1375        let path = test_data_path().join("test_data.mbp-10.dbn.zst");
1376        let instrument_id = InstrumentId::from("ESM4.GLBX");
1377
1378        let depths = loader
1379            .load_order_book_depth(&path, Some(instrument_id), None)
1380            .unwrap();
1381
1382        assert_eq!(depths.len(), 2);
1383    }
1384
1385    #[rstest]
1386    fn test_load_quotes(loader: DatabentoDataLoader) {
1387        let path = test_data_path().join("test_data.mbp-1.dbn.zst");
1388        let instrument_id = InstrumentId::from("ESM4.GLBX");
1389
1390        let quotes = loader
1391            .load_quotes(&path, Some(instrument_id), None)
1392            .unwrap();
1393
1394        assert_eq!(quotes.len(), 2);
1395    }
1396
1397    #[rstest]
1398    #[case(test_data_path().join("test_data.bbo-1s.dbn.zst"))]
1399    #[case(test_data_path().join("test_data.bbo-1m.dbn.zst"))]
1400    fn test_load_bbo_quotes(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1401        let instrument_id = InstrumentId::from("ESM4.GLBX");
1402
1403        let quotes = loader
1404            .load_bbo_quotes(&path, Some(instrument_id), None)
1405            .unwrap();
1406
1407        assert_eq!(quotes.len(), 4);
1408    }
1409
1410    #[rstest]
1411    fn test_load_cmbp_quotes(loader: DatabentoDataLoader) {
1412        let path = test_data_path().join("test_data.cmbp-1.dbn.zst");
1413        let instrument_id = InstrumentId::from("ESM4.GLBX");
1414
1415        let quotes = loader
1416            .load_cmbp_quotes(&path, Some(instrument_id), None)
1417            .unwrap();
1418
1419        // Verify exact data count
1420        assert_eq!(quotes.len(), 2);
1421
1422        // Verify first quote fields
1423        let first_quote = &quotes[0];
1424        assert_eq!(first_quote.instrument_id, instrument_id);
1425        assert_eq!(first_quote.bid_price, Price::from("3720.25"));
1426        assert_eq!(first_quote.ask_price, Price::from("3720.50"));
1427        assert_eq!(first_quote.bid_size, Quantity::from(24));
1428        assert_eq!(first_quote.ask_size, Quantity::from(11));
1429        assert_eq!(first_quote.ts_event, 1609160400006136329);
1430        assert_eq!(first_quote.ts_init, 1609160400006136329);
1431    }
1432
1433    #[rstest]
1434    fn test_load_cbbo_quotes(loader: DatabentoDataLoader) {
1435        let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1436        let instrument_id = InstrumentId::from("ESM4.GLBX");
1437
1438        let quotes = loader
1439            .load_cbbo_quotes(&path, Some(instrument_id), None)
1440            .unwrap();
1441
1442        // Verify exact data count
1443        assert_eq!(quotes.len(), 2);
1444
1445        // Verify first quote fields
1446        let first_quote = &quotes[0];
1447        assert_eq!(first_quote.instrument_id, instrument_id);
1448        assert_eq!(first_quote.bid_price, Price::from("3720.25"));
1449        assert_eq!(first_quote.ask_price, Price::from("3720.50"));
1450        assert_eq!(first_quote.bid_size, Quantity::from(24));
1451        assert_eq!(first_quote.ask_size, Quantity::from(11));
1452        assert_eq!(first_quote.ts_event, 1609160400006136329);
1453        assert_eq!(first_quote.ts_init, 1609160400006136329);
1454    }
1455
1456    #[rstest]
1457    #[case(1, "ESH1.GLBX")]
1458    #[case(2, "ESH21.GLBX")]
1459    fn test_load_cbbo_quotes_resolves_instrument_id(
1460        loader: DatabentoDataLoader,
1461        #[case] digits: u8,
1462        #[case] expected: InstrumentId,
1463    ) {
1464        let loader = loader.with_futures_year_digits(digits.try_into().unwrap());
1465        let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1466        let quotes = loader.load_cbbo_quotes(&path, None, Some(2)).unwrap();
1467
1468        assert_eq!(quotes.len(), 2);
1469        assert_eq!(quotes[0].instrument_id, expected);
1470        assert_eq!(quotes[1].instrument_id, expected);
1471        assert_eq!(quotes[0].bid_price, Price::from("3720.25"));
1472        assert_eq!(quotes[0].ask_price, Price::from("3720.50"));
1473        assert_eq!(quotes[0].bid_size, Quantity::from(24));
1474        assert_eq!(quotes[0].ask_size, Quantity::from(11));
1475        assert_eq!(quotes[0].ts_event, 1609160400006136329);
1476        assert_eq!(quotes[0].ts_init, 1609160400006136329);
1477    }
1478
1479    #[rstest]
1480    fn test_load_tbbo_trades(loader: DatabentoDataLoader) {
1481        let path = test_data_path().join("test_data.tbbo.dbn.zst");
1482        let instrument_id = InstrumentId::from("ESM4.GLBX");
1483
1484        let trades = loader
1485            .load_tbbo_trades(&path, Some(instrument_id), None)
1486            .unwrap();
1487
1488        assert_eq!(trades.len(), 2);
1489        assert_eq!(trades[0].instrument_id, instrument_id);
1490        assert_eq!(trades[0].price, Price::from("3720.25"));
1491        assert_eq!(trades[0].size, Quantity::from("5"));
1492    }
1493
1494    #[rstest]
1495    fn test_load_tcbbo_trades_rejects_cbbo_fixture(loader: DatabentoDataLoader) {
1496        let path = test_data_path().join("test_data.cbbo-1s.dbn.zst");
1497        let instrument_id = InstrumentId::from("ESM4.GLBX");
1498
1499        let result = loader.load_tcbbo_trades(&path, Some(instrument_id), None);
1500
1501        assert!(result.is_err());
1502    }
1503
1504    #[rstest]
1505    fn test_load_trades(loader: DatabentoDataLoader) {
1506        let path = test_data_path().join("test_data.trades.dbn.zst");
1507        let instrument_id = InstrumentId::from("ESM4.GLBX");
1508        let trades = loader
1509            .load_trades(&path, Some(instrument_id), None)
1510            .unwrap();
1511
1512        assert_eq!(trades.len(), 2);
1513    }
1514
1515    #[rstest]
1516    // #[case(test_data_path().join("test_data.ohlcv-1d.dbn.zst"))]  // TODO: Empty file (0 records)
1517    #[case(test_data_path().join("test_data.ohlcv-1h.dbn.zst"))]
1518    #[case(test_data_path().join("test_data.ohlcv-1m.dbn.zst"))]
1519    #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1520    fn test_load_bars(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1521        let instrument_id = InstrumentId::from("ESM4.GLBX");
1522        let bars = loader
1523            .load_bars(&path, Some(instrument_id), None, None)
1524            .unwrap();
1525
1526        assert_eq!(bars.len(), 2);
1527    }
1528
1529    #[rstest]
1530    #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1531    fn test_load_bars_timestamp_on_close_true(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1532        let instrument_id = InstrumentId::from("ESM4.GLBX");
1533        let bars = loader
1534            .load_bars(&path, Some(instrument_id), None, Some(true))
1535            .unwrap();
1536
1537        assert_eq!(bars.len(), 2);
1538
1539        // When bars_timestamp_on_close is true, both ts_event and ts_init should be close time
1540        for bar in &bars {
1541            assert_eq!(
1542                bar.ts_event, bar.ts_init,
1543                "ts_event and ts_init should both be close time when bars_timestamp_on_close=true"
1544            );
1545        }
1546    }
1547
1548    #[rstest]
1549    #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"))]
1550    fn test_load_bars_timestamp_on_close_false(loader: DatabentoDataLoader, #[case] path: PathBuf) {
1551        let instrument_id = InstrumentId::from("ESM4.GLBX");
1552        let bars = loader
1553            .load_bars(&path, Some(instrument_id), None, Some(false))
1554            .unwrap();
1555
1556        assert_eq!(bars.len(), 2);
1557
1558        // When bars_timestamp_on_close is false, ts_event is open time and ts_init is close time
1559        for bar in &bars {
1560            assert_ne!(
1561                bar.ts_event, bar.ts_init,
1562                "ts_event should be open time and ts_init should be close time when bars_timestamp_on_close=false"
1563            );
1564            // For 1-second bars, ts_init (close) should be 1 second after ts_event (open)
1565            assert_eq!(bar.ts_init.as_u64(), bar.ts_event.as_u64() + 1_000_000_000);
1566        }
1567    }
1568
1569    #[rstest]
1570    #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"), 0)]
1571    #[case(test_data_path().join("test_data.ohlcv-1s.dbn.zst"), 1)]
1572    fn test_load_bars_timestamp_comparison(
1573        loader: DatabentoDataLoader,
1574        #[case] path: PathBuf,
1575        #[case] bar_index: usize,
1576    ) {
1577        const ONE_SECOND_NS: u64 = 1_000_000_000;
1578
1579        let instrument_id = InstrumentId::from("ESM4.GLBX");
1580
1581        let bars_close = loader
1582            .load_bars(&path, Some(instrument_id), None, Some(true))
1583            .unwrap();
1584
1585        let bars_open = loader
1586            .load_bars(&path, Some(instrument_id), None, Some(false))
1587            .unwrap();
1588
1589        assert_eq!(bars_close.len(), bars_open.len());
1590        assert_eq!(bars_close.len(), 2);
1591
1592        let bar_close = &bars_close[bar_index];
1593        let bar_open = &bars_open[bar_index];
1594
1595        // Bars should have the same OHLCV data
1596        assert_eq!(bar_close.open, bar_open.open);
1597        assert_eq!(bar_close.high, bar_open.high);
1598        assert_eq!(bar_close.low, bar_open.low);
1599        assert_eq!(bar_close.close, bar_open.close);
1600        assert_eq!(bar_close.volume, bar_open.volume);
1601
1602        // The close-timestamped bar should have later timestamp than open-timestamped bar
1603        // For 1-second bars, this should be exactly 1 second difference
1604        assert!(
1605            bar_close.ts_event > bar_open.ts_event,
1606            "Close-timestamped bar should have later timestamp than open-timestamped bar"
1607        );
1608
1609        // The difference should be exactly 1 second (1_000_000_000 nanoseconds) for 1s bars
1610        assert_eq!(
1611            bar_close.ts_event.as_u64() - bar_open.ts_event.as_u64(),
1612            ONE_SECOND_NS,
1613            "Timestamp difference should be exactly 1 second for 1s bars"
1614        );
1615    }
1616
1617    #[rstest]
1618    fn test_load_status_records(loader: DatabentoDataLoader) {
1619        let path = test_data_path().join("test_data.status.dbn.zst");
1620        let instrument_id = InstrumentId::from("ESM4.GLBX");
1621
1622        let statuses = loader
1623            .load_status_records::<dbn::StatusMsg>(&path, Some(instrument_id))
1624            .unwrap()
1625            .collect::<anyhow::Result<Vec<_>>>()
1626            .unwrap();
1627
1628        // Assert total count matches Python test expectations
1629        assert_eq!(statuses.len(), 4, "Should load exactly 4 status records");
1630
1631        // Assert first record fields match Python test expectations
1632        let first = &statuses[0];
1633        assert_eq!(first.instrument_id, instrument_id);
1634        assert_eq!(first.ts_event.as_u64(), 1609110000000000000);
1635        assert_eq!(first.ts_init.as_u64(), 1609113600000000000);
1636    }
1637
1638    #[rstest]
1639    fn test_read_imbalance_records(loader: DatabentoDataLoader) {
1640        let path = test_data_path().join("test_data.imbalance.dbn.zst");
1641        let instrument_id = InstrumentId::from("ESM4.GLBX");
1642
1643        let imbalances = loader
1644            .read_imbalance_records::<dbn::ImbalanceMsg>(&path, Some(instrument_id), None)
1645            .unwrap()
1646            .collect::<anyhow::Result<Vec<_>>>()
1647            .unwrap();
1648
1649        // Assert total count
1650        assert_eq!(
1651            imbalances.len(),
1652            2,
1653            "Should load exactly 2 imbalance records"
1654        );
1655
1656        // Assert first record has required fields
1657        let first = &imbalances[0];
1658        assert_eq!(first.instrument_id, instrument_id);
1659        assert!(
1660            first.ref_price.as_f64() > 0.0,
1661            "ref_price should be positive"
1662        );
1663        assert!(first.ts_event.as_u64() > 0, "ts_event should be set");
1664        assert!(first.ts_recv.as_u64() > 0, "ts_recv should be set");
1665        assert!(first.ts_init.as_u64() > 0, "ts_init should be set");
1666    }
1667
1668    #[rstest]
1669    fn test_read_statistics_records(loader: DatabentoDataLoader) {
1670        let path = test_data_path().join("test_data.statistics.dbn.zst");
1671        let instrument_id = InstrumentId::from("ESM4.GLBX");
1672
1673        let statistics = loader
1674            .read_statistics_records::<dbn::StatMsg>(&path, Some(instrument_id), None)
1675            .unwrap()
1676            .collect::<anyhow::Result<Vec<_>>>()
1677            .unwrap();
1678
1679        // Assert total count
1680        assert_eq!(
1681            statistics.len(),
1682            2,
1683            "Should load exactly 2 statistics records"
1684        );
1685
1686        // Assert first record has required fields
1687        let first = &statistics[0];
1688        assert_eq!(first.instrument_id, instrument_id);
1689        assert!(first.ts_event.as_u64() > 0, "ts_event should be set");
1690        assert!(first.ts_recv.as_u64() > 0, "ts_recv should be set");
1691        assert!(first.ts_init.as_u64() > 0, "ts_init should be set");
1692        assert!(first.sequence > 0, "sequence should be positive");
1693    }
1694
1695    #[rstest]
1696    #[case("futures_contract", "ESU6", "ESU26.GLBX")]
1697    #[case("futures_spread", "ESU6-ESM7", "ESU26-ESM27.GLBX")]
1698    fn test_loader_defaults_to_two_digit_years(
1699        #[case] fixture: &str,
1700        #[case] raw: &str,
1701        #[case] expected: InstrumentId,
1702    ) {
1703        let publishers = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1704        let path = Path::new(env!("CARGO_MANIFEST_DIR"))
1705            .join(format!("test_data/test_data.definition.{fixture}.dbn.zst"));
1706        let mut loader = DatabentoDataLoader::new(Some(publishers.clone())).unwrap();
1707        let mut legacy = DatabentoDataLoader::new(Some(publishers))
1708            .unwrap()
1709            .with_futures_year_digits(FuturesYearDigits::One);
1710        let instruments = loader.load_instruments(&path, false, false, None).unwrap();
1711        let legacy_instruments = legacy.load_instruments(&path, false, false, None).unwrap();
1712        assert_eq!(instruments.len(), 1);
1713        assert_eq!(legacy_instruments.len(), 1);
1714        assert_eq!(instruments[0].id(), expected);
1715        assert_eq!(instruments[0].raw_symbol(), Symbol::from(raw));
1716        assert_eq!(
1717            legacy_instruments[0].id(),
1718            InstrumentId::new(Symbol::from(raw), Venue::from("GLBX"))
1719        );
1720        assert_eq!(legacy_instruments[0].raw_symbol(), Symbol::from(raw));
1721    }
1722
1723    #[rstest]
1724    #[case(true, "BRK B.XNAS")]
1725    #[case(false, "BRK.B.XNAS")]
1726    fn test_loader_share_class_keeps_native_raw_symbol(
1727        #[case] share_class_space: bool,
1728        #[case] expected: InstrumentId,
1729    ) {
1730        let publishers = Path::new(env!("CARGO_MANIFEST_DIR")).join("publishers.json");
1731        let path = test_data_path().join("test_data.definition.equity.dbn.zst");
1732        let mut decoder = Decoder::from_zstd_file(&path).unwrap();
1733        let metadata = decoder.metadata().clone();
1734        let mut definition = decoder
1735            .decode_record::<dbn::InstrumentDefMsg>()
1736            .unwrap()
1737            .unwrap()
1738            .clone();
1739        definition.raw_symbol = dbn::record::str_to_c_chars("BRK.B").unwrap();
1740        let replay_path =
1741            std::env::temp_dir().join(format!("equity-{}.dbn.zst", nautilus_core::UUID4::new()));
1742        let mut encoded = Vec::new();
1743        {
1744            let mut encoder =
1745                dbn::encode::dbn::Encoder::with_zstd(&mut encoded, &metadata).unwrap();
1746            encoder.encode_record(&definition).unwrap();
1747            encoder.flush().unwrap();
1748        }
1749        std::fs::write(&replay_path, encoded).unwrap();
1750        let mut loader = DatabentoDataLoader::new(Some(publishers))
1751            .unwrap()
1752            .with_share_class_space(share_class_space);
1753
1754        let instruments = loader
1755            .load_instruments(&replay_path, false, false, None)
1756            .unwrap();
1757
1758        std::fs::remove_file(replay_path).unwrap();
1759        assert_eq!(instruments.len(), 1);
1760        assert_eq!(instruments[0].id(), expected);
1761        assert_eq!(instruments[0].raw_symbol(), Symbol::from("BRK.B"));
1762    }
1763
1764    #[rstest]
1765    #[case("futures_contract", "ESU6", "ESU26.GLBX", "ESU36.GLBX")]
1766    #[case("futures_spread", "ESU6-ESM7", "ESU26-ESM27.GLBX", "ESU36-ESM37.GLBX")]
1767    fn test_loader_definition_year_is_scoped_to_record_date(
1768        mut loader_without_seed: DatabentoDataLoader,
1769        #[case] fixture: &str,
1770        #[case] raw: &str,
1771        #[case] expected: InstrumentId,
1772        #[case] inferred: InstrumentId,
1773    ) {
1774        let loader = &mut loader_without_seed;
1775        let path = test_data_path().join(format!("test_data.definition.{fixture}.dbn.zst"));
1776        let mut decoder = Decoder::from_zstd_file(&path).unwrap();
1777        let metadata_source = decoder.metadata().clone();
1778        let mut definition = decoder
1779            .decode_record::<dbn::InstrumentDefMsg>()
1780            .unwrap()
1781            .unwrap()
1782            .clone();
1783        definition.ts_recv = "2028-01-01T00:00:00Z"
1784            .parse::<jiff::Timestamp>()
1785            .unwrap()
1786            .as_nanosecond() as u64;
1787        let replay_path =
1788            std::env::temp_dir().join(format!("futures-{}.dbn.zst", nautilus_core::UUID4::new()));
1789        let mut encoded = Vec::new();
1790        {
1791            let mut encoder =
1792                dbn::encode::dbn::Encoder::with_zstd(&mut encoded, &metadata_source).unwrap();
1793            encoder.encode_record(&definition).unwrap();
1794            encoder.flush().unwrap();
1795        }
1796
1797        std::fs::write(&replay_path, encoded).unwrap();
1798        let replayed = loader
1799            .load_instruments(&replay_path, false, false, None)
1800            .unwrap();
1801        std::fs::remove_file(replay_path).unwrap();
1802        assert_eq!(replayed[0].id(), expected);
1803        let mut record = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1804        record.hd.publisher_id = definition.hd.publisher_id;
1805        record.hd.instrument_id = definition.hd.instrument_id;
1806        record.ts_recv = definition.ts_recv;
1807        let metadata = dbn::Metadata::builder()
1808            .dataset("GLBX.MDP3")
1809            .schema(Some(dbn::Schema::Bbo1S))
1810            .start(record.ts_recv)
1811            .stype_in(Some(dbn::SType::RawSymbol))
1812            .stype_out(dbn::SType::InstrumentId)
1813            .mappings(vec![dbn::SymbolMapping {
1814                raw_symbol: raw.to_string(),
1815                intervals: vec![dbn::MappingInterval {
1816                    start_date: time::macros::date!(2028 - 01 - 01),
1817                    end_date: time::macros::date!(2028 - 01 - 03),
1818                    symbol: record.hd.instrument_id.to_string(),
1819                }],
1820            }])
1821            .build();
1822        let mut metadata = Some(MetadataCache::new(metadata));
1823        let id = loader
1824            .resolve_record_instrument_id(&dbn::RecordRef::from(&record), None, &mut metadata)
1825            .unwrap();
1826        assert_eq!(id, expected);
1827        record.ts_recv += 86_400_000_000_000;
1828        let next_day = loader
1829            .resolve_record_instrument_id(&dbn::RecordRef::from(&record), None, &mut metadata)
1830            .unwrap();
1831        assert_eq!(next_day, inferred);
1832    }
1833}