Skip to main content

nautilus_databento/python/
loader.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Python bindings for the Databento data loader.
17
18use std::{collections::HashMap, path::PathBuf};
19
20use databento::dbn;
21use jiff::civil::Time;
22use nautilus_core::python::{IntoPyObjectNautilusExt, to_pyvalue_err};
23use nautilus_model::{
24    data::{Bar, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
25    identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
26    python::instruments::instrument_any_to_pyobject,
27};
28use pyo3::{prelude::*, types::PyList};
29use ustr::Ustr;
30
31use crate::{
32    decode::DatabentoDecodeConfig,
33    loader::DatabentoDataLoader,
34    types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, PublisherId},
35};
36
37#[expect(clippy::needless_pass_by_value)]
38#[pymethods]
39#[pyo3_stub_gen::derive::gen_stub_pymethods]
40impl DatabentoDataLoader {
41    /// A Nautilus data loader for Databento Binary Encoding (DBN) format data.
42    ///
43    /// # Supported Schemas
44    ///  - `MBO` -> `OrderBookDelta`
45    ///  - `MBP_1` -> `(QuoteTick, Option<TradeTick>)`
46    ///  - `MBP_10` -> `OrderBookDepth`
47    ///  - `BBO_1S` -> `QuoteTick`
48    ///  - `BBO_1M` -> `QuoteTick`
49    ///  - `CMBP_1` -> `(QuoteTick, Option<TradeTick>)`
50    ///  - `CBBO_1S` -> `QuoteTick`
51    ///  - `CBBO_1M` -> `QuoteTick`
52    ///  - `TCBBO` -> `(QuoteTick, TradeTick)`
53    ///  - `TBBO` -> `(QuoteTick, TradeTick)`
54    ///  - `TRADES` -> `TradeTick`
55    ///  - `OHLCV_1S` -> `Bar`
56    ///  - `OHLCV_1M` -> `Bar`
57    ///  - `OHLCV_1H` -> `Bar`
58    ///  - `OHLCV_1D` -> `Bar`
59    ///  - `OHLCV_EOD` -> `Bar`
60    ///  - `DEFINITION` -> `Instrument`
61    ///  - `IMBALANCE` -> `DatabentoImbalance`
62    ///  - `STATISTICS` -> `DatabentoStatistics`
63    ///  - `STATUS` -> `InstrumentStatus`
64    ///
65    /// # References
66    ///
67    /// <https://databento.com/docs/schemas-and-data-formats>
68    #[new]
69    #[pyo3(signature = (publishers_filepath=None, futures_year_digits=None))]
70    fn py_new(
71        publishers_filepath: Option<PathBuf>,
72        futures_year_digits: Option<u8>,
73    ) -> PyResult<Self> {
74        let futures_year_digits = futures_year_digits
75            .map(FuturesYearDigits::try_from)
76            .transpose()
77            .map_err(to_pyvalue_err)?
78            .unwrap_or_default();
79
80        Self::new(publishers_filepath)
81            .map(|loader| loader.with_futures_year_digits(futures_year_digits))
82            .map_err(to_pyvalue_err)
83    }
84
85    /// Load the publishers data from the file at the given `filepath`.
86    ///
87    /// # Errors
88    ///
89    /// Returns an error if the file cannot be read or parsed as JSON.
90    #[pyo3(name = "load_publishers")]
91    fn py_load_publishers(&mut self, publishers_filepath: PathBuf) -> PyResult<()> {
92        self.load_publishers(publishers_filepath)
93            .map_err(to_pyvalue_err)
94    }
95
96    /// Returns the internal Databento publishers currently held by the loader.
97    #[must_use]
98    #[pyo3(name = "get_publishers")]
99    fn py_get_publishers(&self) -> HashMap<u16, DatabentoPublisher> {
100        self.get_publishers()
101            .iter()
102            .map(|(&key, value)| (key, value.clone()))
103            .collect::<HashMap<u16, DatabentoPublisher>>()
104    }
105
106    /// Sets the `venue` to map to the given `dataset`.
107    #[pyo3(name = "set_dataset_for_venue")]
108    fn py_set_dataset_for_venue(&mut self, dataset: String, venue: Venue) {
109        self.set_dataset_for_venue(Ustr::from(&dataset), venue);
110    }
111
112    /// Returns the dataset which matches the given `venue` (if found).
113    #[must_use]
114    #[pyo3(name = "get_dataset_for_venue")]
115    fn py_get_dataset_for_venue(&self, venue: &Venue) -> Option<String> {
116        self.get_dataset_for_venue(venue).map(ToString::to_string)
117    }
118
119    /// Returns the venue which matches the given `publisher_id` (if found).
120    #[must_use]
121    #[pyo3(name = "get_venue_for_publisher")]
122    fn py_get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option<String> {
123        self.get_venue_for_publisher(publisher_id)
124            .map(ToString::to_string)
125    }
126
127    /// Caches a `price_precision` for the given `symbol`.
128    ///
129    /// When market data is read without an explicit `price_precision` argument,
130    /// the loader resolves precision per record from this cache. Definitions
131    /// loaded via `Self.load_instruments` are inserted automatically.
132    #[pyo3(name = "set_price_precision")]
133    fn py_set_price_precision(&mut self, symbol: &str, price_precision: u8) {
134        self.set_price_precision(Symbol::from(symbol), price_precision);
135    }
136
137    /// Returns the cached price precisions keyed by symbol.
138    #[must_use]
139    #[pyo3(name = "get_price_precisions")]
140    fn py_get_price_precisions(&self) -> HashMap<String, u8> {
141        self.get_price_precisions()
142            .iter()
143            .map(|(symbol, precision)| (symbol.to_string(), *precision))
144            .collect()
145    }
146
147    #[pyo3(name = "schema_for_file")]
148    fn py_schema_for_file(&self, filepath: PathBuf) -> PyResult<Option<String>> {
149        self.schema_from_file(&filepath).map_err(to_pyvalue_err)
150    }
151
152    /// Loads all instrument definitions from a DBN file.
153    ///
154    /// When `skip_on_error` is true, instruments that fail to decode are logged
155    /// as warnings and skipped. When false (default), any decode error is propagated.
156    ///
157    /// # Errors
158    ///
159    /// Returns an error if loading instruments fails.
160    #[pyo3(name = "load_instruments")]
161    #[pyo3(signature = (filepath, use_exchange_as_venue, skip_on_error=false, expiration_overrides=None))]
162    fn py_load_instruments(
163        &mut self,
164        py: Python,
165        filepath: PathBuf,
166        use_exchange_as_venue: bool,
167        skip_on_error: bool,
168        expiration_overrides: Option<HashMap<String, HashMap<String, String>>>,
169    ) -> PyResult<Py<PyAny>> {
170        let decode_config = build_decode_config(expiration_overrides)?;
171        let iter = self
172            .load_instruments(
173                &filepath,
174                use_exchange_as_venue,
175                skip_on_error,
176                decode_config.as_ref(),
177            )
178            .map_err(to_pyvalue_err)?;
179
180        let mut data = Vec::new();
181
182        for instrument in iter {
183            let py_object = instrument_any_to_pyobject(py, instrument)?;
184            data.push(py_object);
185        }
186
187        let list = PyList::new(py, &data)?;
188
189        Ok(list.into_py_any_unwrap(py))
190    }
191
192    // Cannot include trades
193    /// Loads order book delta messages from a DBN MBO schema file.
194    ///
195    /// Cannot include trades.
196    ///
197    /// # Errors
198    ///
199    /// Returns an error if loading order book deltas fails.
200    #[pyo3(name = "load_order_book_deltas")]
201    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
202    fn py_load_order_book_deltas(
203        &self,
204        filepath: PathBuf,
205        instrument_id: Option<InstrumentId>,
206        price_precision: Option<u8>,
207    ) -> PyResult<Vec<OrderBookDelta>> {
208        self.load_order_book_deltas(&filepath, instrument_id, price_precision)
209            .map_err(to_pyvalue_err)
210    }
211
212    /// Loads order book depth snapshots from a DBN MBP-10 schema file.
213    ///
214    /// # Errors
215    ///
216    /// Returns an error if loading order book depth fails.
217    #[pyo3(name = "load_order_book_depth")]
218    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
219    fn py_load_order_book_depth(
220        &self,
221        filepath: PathBuf,
222        instrument_id: Option<InstrumentId>,
223        price_precision: Option<u8>,
224    ) -> PyResult<Vec<OrderBookDepth>> {
225        self.load_order_book_depth(&filepath, instrument_id, price_precision)
226            .map_err(to_pyvalue_err)
227    }
228
229    /// Loads quote tick messages from a DBN MBP-1 or TBBO schema file.
230    ///
231    /// # Errors
232    ///
233    /// Returns an error if loading quotes fails.
234    #[pyo3(name = "load_quotes")]
235    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
236    fn py_load_quotes(
237        &self,
238        filepath: PathBuf,
239        instrument_id: Option<InstrumentId>,
240        price_precision: Option<u8>,
241    ) -> PyResult<Vec<QuoteTick>> {
242        self.load_quotes(&filepath, instrument_id, price_precision)
243            .map_err(to_pyvalue_err)
244    }
245
246    /// Loads best bid/offer quote messages from a DBN BBO schema file.
247    ///
248    /// # Errors
249    ///
250    /// Returns an error if loading BBO quotes fails.
251    #[pyo3(name = "load_bbo_quotes")]
252    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
253    fn py_load_bbo_quotes(
254        &self,
255        filepath: PathBuf,
256        instrument_id: Option<InstrumentId>,
257        price_precision: Option<u8>,
258    ) -> PyResult<Vec<QuoteTick>> {
259        self.load_bbo_quotes(&filepath, instrument_id, price_precision)
260            .map_err(to_pyvalue_err)
261    }
262
263    /// Loads consolidated MBP-1 quote messages from a DBN CMBP-1 schema file.
264    ///
265    /// # Errors
266    ///
267    /// Returns an error if loading consolidated MBP-1 quotes fails.
268    #[pyo3(name = "load_cmbp_quotes")]
269    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
270    fn py_load_cmbp_quotes(
271        &self,
272        filepath: PathBuf,
273        instrument_id: Option<InstrumentId>,
274        price_precision: Option<u8>,
275    ) -> PyResult<Vec<QuoteTick>> {
276        self.load_cmbp_quotes(&filepath, instrument_id, price_precision)
277            .map_err(to_pyvalue_err)
278    }
279
280    /// Loads consolidated best bid/offer quote messages from a DBN CBBO schema file.
281    ///
282    /// # Errors
283    ///
284    /// Returns an error if loading consolidated BBO quotes fails.
285    #[pyo3(name = "load_cbbo_quotes")]
286    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
287    fn py_load_cbbo_quotes(
288        &self,
289        filepath: PathBuf,
290        instrument_id: Option<InstrumentId>,
291        price_precision: Option<u8>,
292    ) -> PyResult<Vec<QuoteTick>> {
293        self.load_cbbo_quotes(&filepath, instrument_id, price_precision)
294            .map_err(to_pyvalue_err)
295    }
296
297    /// Loads trade messages from a DBN TBBO schema file.
298    ///
299    /// # Errors
300    ///
301    /// Returns an error if loading TBBO trades fails.
302    #[pyo3(name = "load_tbbo_trades")]
303    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
304    fn py_load_tbbo_trades(
305        &self,
306        filepath: PathBuf,
307        instrument_id: Option<InstrumentId>,
308        price_precision: Option<u8>,
309    ) -> PyResult<Vec<TradeTick>> {
310        self.load_tbbo_trades(&filepath, instrument_id, price_precision)
311            .map_err(to_pyvalue_err)
312    }
313
314    /// Loads trade messages from a DBN TCBBO schema file.
315    ///
316    /// # Errors
317    ///
318    /// Returns an error if loading TCBBO trades fails.
319    #[pyo3(name = "load_tcbbo_trades")]
320    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
321    fn py_load_tcbbo_trades(
322        &self,
323        filepath: PathBuf,
324        instrument_id: Option<InstrumentId>,
325        price_precision: Option<u8>,
326    ) -> PyResult<Vec<TradeTick>> {
327        self.load_tcbbo_trades(&filepath, instrument_id, price_precision)
328            .map_err(to_pyvalue_err)
329    }
330
331    /// Loads trade messages from a DBN TRADES schema file.
332    ///
333    /// # Errors
334    ///
335    /// Returns an error if loading trades fails.
336    #[pyo3(name = "load_trades")]
337    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
338    fn py_load_trades(
339        &self,
340        filepath: PathBuf,
341        instrument_id: Option<InstrumentId>,
342        price_precision: Option<u8>,
343    ) -> PyResult<Vec<TradeTick>> {
344        self.load_trades(&filepath, instrument_id, price_precision)
345            .map_err(to_pyvalue_err)
346    }
347
348    /// Loads OHLCV bar messages from a DBN OHLCV schema file.
349    ///
350    /// # Errors
351    ///
352    /// Returns an error if loading bars fails.
353    #[pyo3(name = "load_bars")]
354    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None, timestamp_on_close=true))]
355    fn py_load_bars(
356        &self,
357        filepath: PathBuf,
358        instrument_id: Option<InstrumentId>,
359        price_precision: Option<u8>,
360        timestamp_on_close: bool,
361    ) -> PyResult<Vec<Bar>> {
362        self.load_bars(
363            &filepath,
364            instrument_id,
365            price_precision,
366            Some(timestamp_on_close),
367        )
368        .map_err(to_pyvalue_err)
369    }
370
371    #[pyo3(name = "load_status")]
372    #[pyo3(signature = (filepath, instrument_id=None))]
373    fn py_load_status(
374        &self,
375        filepath: PathBuf,
376        instrument_id: Option<InstrumentId>,
377    ) -> PyResult<Vec<InstrumentStatus>> {
378        let iter = self
379            .load_status_records::<dbn::StatusMsg>(&filepath, instrument_id)
380            .map_err(to_pyvalue_err)?;
381
382        let mut data = Vec::new();
383
384        for result in iter {
385            match result {
386                Ok(item) => data.push(item),
387                Err(e) => return Err(to_pyvalue_err(e)),
388            }
389        }
390
391        Ok(data)
392    }
393
394    #[pyo3(name = "load_imbalance")]
395    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
396    fn py_load_imbalance(
397        &self,
398        filepath: PathBuf,
399        instrument_id: Option<InstrumentId>,
400        price_precision: Option<u8>,
401    ) -> PyResult<Vec<DatabentoImbalance>> {
402        let iter = self
403            .read_imbalance_records::<dbn::ImbalanceMsg>(&filepath, instrument_id, price_precision)
404            .map_err(to_pyvalue_err)?;
405
406        let mut data = Vec::new();
407
408        for result in iter {
409            match result {
410                Ok(item) => data.push(item),
411                Err(e) => return Err(to_pyvalue_err(e)),
412            }
413        }
414
415        Ok(data)
416    }
417
418    #[pyo3(name = "load_statistics")]
419    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
420    fn py_load_statistics(
421        &self,
422        filepath: PathBuf,
423        instrument_id: Option<InstrumentId>,
424        price_precision: Option<u8>,
425    ) -> PyResult<Vec<DatabentoStatistics>> {
426        let iter = self
427            .read_statistics_records::<dbn::StatMsg>(&filepath, instrument_id, price_precision)
428            .map_err(to_pyvalue_err)?;
429
430        let mut data = Vec::new();
431
432        for result in iter {
433            match result {
434                Ok(item) => data.push(item),
435                Err(e) => return Err(to_pyvalue_err(e)),
436            }
437        }
438
439        Ok(data)
440    }
441}
442
443// Returns `None` when no overrides are supplied, so the loader applies its built-in defaults
444fn build_decode_config(
445    expiration_overrides: Option<HashMap<String, HashMap<String, String>>>,
446) -> PyResult<Option<DatabentoDecodeConfig>> {
447    expiration_overrides
448        .map(|overrides| decode_config_from_overrides(overrides).map_err(to_pyvalue_err))
449        .transpose()
450}
451
452// Builds a decode config from a dataset -> (underlying -> wall-clock time) mapping. The reserved
453// underlying key "default" sets a dataset's default time; other keys are per-underlying overrides.
454fn decode_config_from_overrides(
455    overrides: HashMap<String, HashMap<String, String>>,
456) -> Result<DatabentoDecodeConfig, String> {
457    let mut config = DatabentoDecodeConfig::default();
458
459    for (dataset_name, times) in overrides {
460        let dataset = dataset_name
461            .parse::<dbn::Dataset>()
462            .map_err(|_| format!("Unknown dataset '{dataset_name}'"))?;
463        let rule = config
464            .option_expiration
465            .get_mut(&dataset)
466            .ok_or_else(|| format!("No expiration correction rule for dataset '{dataset_name}'"))?;
467
468        for (underlying, value) in times {
469            let time = parse_expiration_time(&value)?;
470            if underlying == "default" {
471                rule.default_time = time;
472            } else {
473                rule.overrides.insert(Ustr::from(&underlying), time);
474            }
475        }
476    }
477
478    Ok(config)
479}
480
481// Parses a `HH:MM` or `HH:MM:SS` wall-clock time
482fn parse_expiration_time(value: &str) -> Result<Time, String> {
483    value
484        .parse::<Time>()
485        .map_err(|_| format!("Invalid expiration time '{value}', expected 'HH:MM' or 'HH:MM:SS'"))
486}
487
488#[cfg(test)]
489mod tests {
490    use std::collections::HashMap;
491
492    use databento::dbn;
493    use jiff::civil::Time;
494    use rstest::rstest;
495    use ustr::Ustr;
496
497    use super::decode_config_from_overrides;
498
499    fn overrides(
500        dataset: &str,
501        entries: &[(&str, &str)],
502    ) -> HashMap<String, HashMap<String, String>> {
503        let inner = entries
504            .iter()
505            .map(|(k, v)| ((*k).to_string(), (*v).to_string()))
506            .collect();
507        HashMap::from([(dataset.to_string(), inner)])
508    }
509
510    #[rstest]
511    fn test_default_key_sets_dataset_default_time() {
512        let config =
513            decode_config_from_overrides(overrides("OPRA.PILLAR", &[("default", "15:30")]))
514                .unwrap();
515        let rule = config
516            .option_expiration
517            .get(&dbn::Dataset::OpraPillar)
518            .unwrap();
519        assert_eq!(rule.default_time, Time::constant(15, 30, 0, 0));
520        assert!(rule.overrides.is_empty());
521    }
522
523    #[rstest]
524    fn test_underlying_key_sets_override_and_keeps_default() {
525        let config =
526            decode_config_from_overrides(overrides("OPRA.PILLAR", &[("SPX", "09:30:00")])).unwrap();
527        let rule = config
528            .option_expiration
529            .get(&dbn::Dataset::OpraPillar)
530            .unwrap();
531        assert_eq!(
532            rule.overrides.get(&Ustr::from("SPX")).copied(),
533            Some(Time::constant(9, 30, 0, 0))
534        );
535        assert_eq!(rule.default_time, Time::constant(16, 0, 0, 0));
536    }
537
538    #[rstest]
539    fn test_unknown_dataset_errors() {
540        let err = decode_config_from_overrides(overrides("NOT.ADATASET", &[("default", "16:00")]))
541            .unwrap_err();
542        assert!(err.contains("Unknown dataset"), "was: {err}");
543    }
544
545    #[rstest]
546    fn test_dataset_without_rule_errors() {
547        // GLBX is a valid dataset but has no built-in expiration rule, so it cannot be tuned
548        let err = decode_config_from_overrides(overrides("GLBX.MDP3", &[("default", "16:00")]))
549            .unwrap_err();
550        assert!(err.contains("No expiration correction rule"), "was: {err}");
551    }
552
553    #[rstest]
554    fn test_invalid_time_errors() {
555        let err = decode_config_from_overrides(overrides("OPRA.PILLAR", &[("default", "nope")]))
556            .unwrap_err();
557        assert!(err.contains("Invalid expiration time"), "was: {err}");
558    }
559}