Skip to main content

nautilus_databento/python/
loader.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Python bindings for the Databento data loader.
17
18use std::{collections::HashMap, path::PathBuf};
19
20use databento::dbn;
21use jiff::civil::Time;
22use nautilus_core::python::{IntoPyObjectNautilusExt, to_pyvalue_err};
23use nautilus_model::{
24    data::{Bar, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
25    identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
26    python::instruments::instrument_any_to_pyobject,
27};
28use pyo3::{prelude::*, types::PyList};
29use ustr::Ustr;
30
31use crate::{
32    decode::DatabentoDecodeConfig,
33    loader::DatabentoDataLoader,
34    types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, PublisherId},
35};
36
37#[expect(clippy::needless_pass_by_value)]
38#[pymethods]
39#[pyo3_stub_gen::derive::gen_stub_pymethods]
40impl DatabentoDataLoader {
41    /// A Nautilus data loader for Databento Binary Encoding (DBN) format data.
42    ///
43    /// # Supported Schemas
44    ///  - `MBO` -> `OrderBookDelta`
45    ///  - `MBP_1` -> `(QuoteTick, Option<TradeTick>)`
46    ///  - `MBP_10` -> `OrderBookDepth`
47    ///  - `BBO_1S` -> `QuoteTick`
48    ///  - `BBO_1M` -> `QuoteTick`
49    ///  - `CMBP_1` -> `(QuoteTick, Option<TradeTick>)`
50    ///  - `CBBO_1S` -> `QuoteTick`
51    ///  - `CBBO_1M` -> `QuoteTick`
52    ///  - `TCBBO` -> `(QuoteTick, TradeTick)`
53    ///  - `TBBO` -> `(QuoteTick, TradeTick)`
54    ///  - `TRADES` -> `TradeTick`
55    ///  - `OHLCV_1S` -> `Bar`
56    ///  - `OHLCV_1M` -> `Bar`
57    ///  - `OHLCV_1H` -> `Bar`
58    ///  - `OHLCV_1D` -> `Bar`
59    ///  - `OHLCV_EOD` -> `Bar`
60    ///  - `DEFINITION` -> `Instrument`
61    ///  - `IMBALANCE` -> `DatabentoImbalance`
62    ///  - `STATISTICS` -> `DatabentoStatistics`
63    ///  - `STATUS` -> `InstrumentStatus`
64    ///
65    /// # References
66    ///
67    /// <https://databento.com/docs/schemas-and-data-formats>
68    #[new]
69    #[pyo3(signature = (publishers_filepath=None, futures_year_digits=None, share_class_space=true))]
70    fn py_new(
71        publishers_filepath: Option<PathBuf>,
72        futures_year_digits: Option<u8>,
73        share_class_space: bool,
74    ) -> PyResult<Self> {
75        let futures_year_digits = futures_year_digits
76            .map(FuturesYearDigits::try_from)
77            .transpose()
78            .map_err(to_pyvalue_err)?
79            .unwrap_or_default();
80
81        Self::new(publishers_filepath)
82            .map(|loader| {
83                loader
84                    .with_futures_year_digits(futures_year_digits)
85                    .with_share_class_space(share_class_space)
86            })
87            .map_err(to_pyvalue_err)
88    }
89
90    /// Load the publishers data from the file at the given `filepath`.
91    ///
92    /// # Errors
93    ///
94    /// Returns an error if the file cannot be read or parsed as JSON.
95    #[pyo3(name = "load_publishers")]
96    fn py_load_publishers(&mut self, publishers_filepath: PathBuf) -> PyResult<()> {
97        self.load_publishers(publishers_filepath)
98            .map_err(to_pyvalue_err)
99    }
100
101    /// Returns the internal Databento publishers currently held by the loader.
102    #[must_use]
103    #[pyo3(name = "get_publishers")]
104    fn py_get_publishers(&self) -> HashMap<u16, DatabentoPublisher> {
105        self.get_publishers()
106            .iter()
107            .map(|(&key, value)| (key, value.clone()))
108            .collect::<HashMap<u16, DatabentoPublisher>>()
109    }
110
111    /// Sets the `venue` to map to the given `dataset`.
112    #[pyo3(name = "set_dataset_for_venue")]
113    fn py_set_dataset_for_venue(&mut self, dataset: String, venue: Venue) {
114        self.set_dataset_for_venue(Ustr::from(&dataset), venue);
115    }
116
117    /// Returns the dataset which matches the given `venue` (if found).
118    #[must_use]
119    #[pyo3(name = "get_dataset_for_venue")]
120    fn py_get_dataset_for_venue(&self, venue: &Venue) -> Option<String> {
121        self.get_dataset_for_venue(venue).map(ToString::to_string)
122    }
123
124    /// Returns the venue which matches the given `publisher_id` (if found).
125    #[must_use]
126    #[pyo3(name = "get_venue_for_publisher")]
127    fn py_get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option<String> {
128        self.get_venue_for_publisher(publisher_id)
129            .map(ToString::to_string)
130    }
131
132    /// Caches a `price_precision` for the given `symbol`.
133    ///
134    /// When market data is read without an explicit `price_precision` argument,
135    /// the loader resolves precision per record from this cache. Definitions
136    /// loaded via `Self.load_instruments` are inserted automatically.
137    #[pyo3(name = "set_price_precision")]
138    fn py_set_price_precision(&mut self, symbol: &str, price_precision: u8) {
139        self.set_price_precision(Symbol::from(symbol), price_precision);
140    }
141
142    /// Returns the cached price precisions keyed by symbol.
143    #[must_use]
144    #[pyo3(name = "get_price_precisions")]
145    fn py_get_price_precisions(&self) -> HashMap<String, u8> {
146        self.get_price_precisions()
147            .iter()
148            .map(|(symbol, precision)| (symbol.to_string(), *precision))
149            .collect()
150    }
151
152    #[pyo3(name = "schema_for_file")]
153    fn py_schema_for_file(&self, filepath: PathBuf) -> PyResult<Option<String>> {
154        self.schema_from_file(&filepath).map_err(to_pyvalue_err)
155    }
156
157    /// Loads all instrument definitions from a DBN file.
158    ///
159    /// When `skip_on_error` is true, instruments that fail to decode are logged
160    /// as warnings and skipped. When false (default), any decode error is propagated.
161    ///
162    /// # Errors
163    ///
164    /// Returns an error if loading instruments fails.
165    #[pyo3(name = "load_instruments")]
166    #[pyo3(signature = (filepath, use_exchange_as_venue, skip_on_error=false, expiration_overrides=None))]
167    fn py_load_instruments(
168        &mut self,
169        py: Python,
170        filepath: PathBuf,
171        use_exchange_as_venue: bool,
172        skip_on_error: bool,
173        expiration_overrides: Option<HashMap<String, HashMap<String, String>>>,
174    ) -> PyResult<Py<PyAny>> {
175        let decode_config = build_decode_config(expiration_overrides)?;
176        let iter = self
177            .load_instruments(
178                &filepath,
179                use_exchange_as_venue,
180                skip_on_error,
181                decode_config.as_ref(),
182            )
183            .map_err(to_pyvalue_err)?;
184
185        let mut data = Vec::new();
186
187        for instrument in iter {
188            let py_object = instrument_any_to_pyobject(py, instrument)?;
189            data.push(py_object);
190        }
191
192        let list = PyList::new(py, &data)?;
193
194        Ok(list.into_py_any_unwrap(py))
195    }
196
197    // Cannot include trades
198    /// Loads order book delta messages from a DBN MBO schema file.
199    ///
200    /// Cannot include trades.
201    ///
202    /// # Errors
203    ///
204    /// Returns an error if loading order book deltas fails.
205    #[pyo3(name = "load_order_book_deltas")]
206    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
207    fn py_load_order_book_deltas(
208        &self,
209        filepath: PathBuf,
210        instrument_id: Option<InstrumentId>,
211        price_precision: Option<u8>,
212    ) -> PyResult<Vec<OrderBookDelta>> {
213        self.load_order_book_deltas(&filepath, instrument_id, price_precision)
214            .map_err(to_pyvalue_err)
215    }
216
217    /// Loads order book depth snapshots from a DBN MBP-10 schema file.
218    ///
219    /// # Errors
220    ///
221    /// Returns an error if loading order book depth fails.
222    #[pyo3(name = "load_order_book_depth")]
223    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
224    fn py_load_order_book_depth(
225        &self,
226        filepath: PathBuf,
227        instrument_id: Option<InstrumentId>,
228        price_precision: Option<u8>,
229    ) -> PyResult<Vec<OrderBookDepth>> {
230        self.load_order_book_depth(&filepath, instrument_id, price_precision)
231            .map_err(to_pyvalue_err)
232    }
233
234    /// Loads quote tick messages from a DBN MBP-1 or TBBO schema file.
235    ///
236    /// # Errors
237    ///
238    /// Returns an error if loading quotes fails.
239    #[pyo3(name = "load_quotes")]
240    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
241    fn py_load_quotes(
242        &self,
243        filepath: PathBuf,
244        instrument_id: Option<InstrumentId>,
245        price_precision: Option<u8>,
246    ) -> PyResult<Vec<QuoteTick>> {
247        self.load_quotes(&filepath, instrument_id, price_precision)
248            .map_err(to_pyvalue_err)
249    }
250
251    /// Loads best bid/offer quote messages from a DBN BBO schema file.
252    ///
253    /// # Errors
254    ///
255    /// Returns an error if loading BBO quotes fails.
256    #[pyo3(name = "load_bbo_quotes")]
257    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
258    fn py_load_bbo_quotes(
259        &self,
260        filepath: PathBuf,
261        instrument_id: Option<InstrumentId>,
262        price_precision: Option<u8>,
263    ) -> PyResult<Vec<QuoteTick>> {
264        self.load_bbo_quotes(&filepath, instrument_id, price_precision)
265            .map_err(to_pyvalue_err)
266    }
267
268    /// Loads consolidated MBP-1 quote messages from a DBN CMBP-1 schema file.
269    ///
270    /// # Errors
271    ///
272    /// Returns an error if loading consolidated MBP-1 quotes fails.
273    #[pyo3(name = "load_cmbp_quotes")]
274    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
275    fn py_load_cmbp_quotes(
276        &self,
277        filepath: PathBuf,
278        instrument_id: Option<InstrumentId>,
279        price_precision: Option<u8>,
280    ) -> PyResult<Vec<QuoteTick>> {
281        self.load_cmbp_quotes(&filepath, instrument_id, price_precision)
282            .map_err(to_pyvalue_err)
283    }
284
285    /// Loads consolidated best bid/offer quote messages from a DBN CBBO schema file.
286    ///
287    /// # Errors
288    ///
289    /// Returns an error if loading consolidated BBO quotes fails.
290    #[pyo3(name = "load_cbbo_quotes")]
291    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
292    fn py_load_cbbo_quotes(
293        &self,
294        filepath: PathBuf,
295        instrument_id: Option<InstrumentId>,
296        price_precision: Option<u8>,
297    ) -> PyResult<Vec<QuoteTick>> {
298        self.load_cbbo_quotes(&filepath, instrument_id, price_precision)
299            .map_err(to_pyvalue_err)
300    }
301
302    /// Loads trade messages from a DBN TBBO schema file.
303    ///
304    /// # Errors
305    ///
306    /// Returns an error if loading TBBO trades fails.
307    #[pyo3(name = "load_tbbo_trades")]
308    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
309    fn py_load_tbbo_trades(
310        &self,
311        filepath: PathBuf,
312        instrument_id: Option<InstrumentId>,
313        price_precision: Option<u8>,
314    ) -> PyResult<Vec<TradeTick>> {
315        self.load_tbbo_trades(&filepath, instrument_id, price_precision)
316            .map_err(to_pyvalue_err)
317    }
318
319    /// Loads trade messages from a DBN TCBBO schema file.
320    ///
321    /// # Errors
322    ///
323    /// Returns an error if loading TCBBO trades fails.
324    #[pyo3(name = "load_tcbbo_trades")]
325    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
326    fn py_load_tcbbo_trades(
327        &self,
328        filepath: PathBuf,
329        instrument_id: Option<InstrumentId>,
330        price_precision: Option<u8>,
331    ) -> PyResult<Vec<TradeTick>> {
332        self.load_tcbbo_trades(&filepath, instrument_id, price_precision)
333            .map_err(to_pyvalue_err)
334    }
335
336    /// Loads trade messages from a DBN TRADES schema file.
337    ///
338    /// # Errors
339    ///
340    /// Returns an error if loading trades fails.
341    #[pyo3(name = "load_trades")]
342    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
343    fn py_load_trades(
344        &self,
345        filepath: PathBuf,
346        instrument_id: Option<InstrumentId>,
347        price_precision: Option<u8>,
348    ) -> PyResult<Vec<TradeTick>> {
349        self.load_trades(&filepath, instrument_id, price_precision)
350            .map_err(to_pyvalue_err)
351    }
352
353    /// Loads OHLCV bar messages from a DBN OHLCV schema file.
354    ///
355    /// # Errors
356    ///
357    /// Returns an error if loading bars fails.
358    #[pyo3(name = "load_bars")]
359    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None, timestamp_on_close=true))]
360    fn py_load_bars(
361        &self,
362        filepath: PathBuf,
363        instrument_id: Option<InstrumentId>,
364        price_precision: Option<u8>,
365        timestamp_on_close: bool,
366    ) -> PyResult<Vec<Bar>> {
367        self.load_bars(
368            &filepath,
369            instrument_id,
370            price_precision,
371            Some(timestamp_on_close),
372        )
373        .map_err(to_pyvalue_err)
374    }
375
376    #[pyo3(name = "load_status")]
377    #[pyo3(signature = (filepath, instrument_id=None))]
378    fn py_load_status(
379        &self,
380        filepath: PathBuf,
381        instrument_id: Option<InstrumentId>,
382    ) -> PyResult<Vec<InstrumentStatus>> {
383        let iter = self
384            .load_status_records::<dbn::StatusMsg>(&filepath, instrument_id)
385            .map_err(to_pyvalue_err)?;
386
387        let mut data = Vec::new();
388
389        for result in iter {
390            match result {
391                Ok(item) => data.push(item),
392                Err(e) => return Err(to_pyvalue_err(e)),
393            }
394        }
395
396        Ok(data)
397    }
398
399    #[pyo3(name = "load_imbalance")]
400    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
401    fn py_load_imbalance(
402        &self,
403        filepath: PathBuf,
404        instrument_id: Option<InstrumentId>,
405        price_precision: Option<u8>,
406    ) -> PyResult<Vec<DatabentoImbalance>> {
407        let iter = self
408            .read_imbalance_records::<dbn::ImbalanceMsg>(&filepath, instrument_id, price_precision)
409            .map_err(to_pyvalue_err)?;
410
411        let mut data = Vec::new();
412
413        for result in iter {
414            match result {
415                Ok(item) => data.push(item),
416                Err(e) => return Err(to_pyvalue_err(e)),
417            }
418        }
419
420        Ok(data)
421    }
422
423    #[pyo3(name = "load_statistics")]
424    #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
425    fn py_load_statistics(
426        &self,
427        filepath: PathBuf,
428        instrument_id: Option<InstrumentId>,
429        price_precision: Option<u8>,
430    ) -> PyResult<Vec<DatabentoStatistics>> {
431        let iter = self
432            .read_statistics_records::<dbn::StatMsg>(&filepath, instrument_id, price_precision)
433            .map_err(to_pyvalue_err)?;
434
435        let mut data = Vec::new();
436
437        for result in iter {
438            match result {
439                Ok(item) => data.push(item),
440                Err(e) => return Err(to_pyvalue_err(e)),
441            }
442        }
443
444        Ok(data)
445    }
446}
447
448// Returns `None` when no overrides are supplied, so the loader applies its built-in defaults
449fn build_decode_config(
450    expiration_overrides: Option<HashMap<String, HashMap<String, String>>>,
451) -> PyResult<Option<DatabentoDecodeConfig>> {
452    expiration_overrides
453        .map(|overrides| decode_config_from_overrides(overrides).map_err(to_pyvalue_err))
454        .transpose()
455}
456
457// Builds a decode config from a dataset -> (underlying -> wall-clock time) mapping. The reserved
458// underlying key "default" sets a dataset's default time; other keys are per-underlying overrides.
459fn decode_config_from_overrides(
460    overrides: HashMap<String, HashMap<String, String>>,
461) -> Result<DatabentoDecodeConfig, String> {
462    let mut config = DatabentoDecodeConfig::default();
463
464    for (dataset_name, times) in overrides {
465        let dataset = dataset_name
466            .parse::<dbn::Dataset>()
467            .map_err(|_| format!("Unknown dataset '{dataset_name}'"))?;
468        let rule = config
469            .option_expiration
470            .get_mut(&dataset)
471            .ok_or_else(|| format!("No expiration correction rule for dataset '{dataset_name}'"))?;
472
473        for (underlying, value) in times {
474            let time = parse_expiration_time(&value)?;
475            if underlying == "default" {
476                rule.default_time = time;
477            } else {
478                rule.overrides.insert(Ustr::from(&underlying), time);
479            }
480        }
481    }
482
483    Ok(config)
484}
485
486// Parses a `HH:MM` or `HH:MM:SS` wall-clock time
487fn parse_expiration_time(value: &str) -> Result<Time, String> {
488    value
489        .parse::<Time>()
490        .map_err(|_| format!("Invalid expiration time '{value}', expected 'HH:MM' or 'HH:MM:SS'"))
491}
492
493#[cfg(test)]
494mod tests {
495    use std::collections::HashMap;
496
497    use databento::dbn;
498    use jiff::civil::Time;
499    use rstest::rstest;
500    use ustr::Ustr;
501
502    use super::decode_config_from_overrides;
503
504    fn overrides(
505        dataset: &str,
506        entries: &[(&str, &str)],
507    ) -> HashMap<String, HashMap<String, String>> {
508        let inner = entries
509            .iter()
510            .map(|(k, v)| ((*k).to_string(), (*v).to_string()))
511            .collect();
512        HashMap::from([(dataset.to_string(), inner)])
513    }
514
515    #[rstest]
516    fn test_default_key_sets_dataset_default_time() {
517        let config =
518            decode_config_from_overrides(overrides("OPRA.PILLAR", &[("default", "15:30")]))
519                .unwrap();
520        let rule = config
521            .option_expiration
522            .get(&dbn::Dataset::OpraPillar)
523            .unwrap();
524        assert_eq!(rule.default_time, Time::constant(15, 30, 0, 0));
525        assert!(rule.overrides.is_empty());
526    }
527
528    #[rstest]
529    fn test_underlying_key_sets_override_and_keeps_default() {
530        let config =
531            decode_config_from_overrides(overrides("OPRA.PILLAR", &[("SPX", "09:30:00")])).unwrap();
532        let rule = config
533            .option_expiration
534            .get(&dbn::Dataset::OpraPillar)
535            .unwrap();
536        assert_eq!(
537            rule.overrides.get(&Ustr::from("SPX")).copied(),
538            Some(Time::constant(9, 30, 0, 0))
539        );
540        assert_eq!(rule.default_time, Time::constant(16, 0, 0, 0));
541    }
542
543    #[rstest]
544    fn test_unknown_dataset_errors() {
545        let err = decode_config_from_overrides(overrides("NOT.ADATASET", &[("default", "16:00")]))
546            .unwrap_err();
547        assert!(err.contains("Unknown dataset"), "was: {err}");
548    }
549
550    #[rstest]
551    fn test_dataset_without_rule_errors() {
552        // GLBX is a valid dataset but has no built-in expiration rule, so it cannot be tuned
553        let err = decode_config_from_overrides(overrides("GLBX.MDP3", &[("default", "16:00")]))
554            .unwrap_err();
555        assert!(err.contains("No expiration correction rule"), "was: {err}");
556    }
557
558    #[rstest]
559    fn test_invalid_time_errors() {
560        let err = decode_config_from_overrides(overrides("OPRA.PILLAR", &[("default", "nope")]))
561            .unwrap_err();
562        assert!(err.contains("Invalid expiration time"), "was: {err}");
563    }
564}