1use std::{collections::HashMap, path::PathBuf};
19
20use databento::dbn;
21use jiff::civil::Time;
22use nautilus_core::python::{IntoPyObjectNautilusExt, to_pyvalue_err};
23use nautilus_model::{
24 data::{Bar, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
25 identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
26 python::instruments::instrument_any_to_pyobject,
27};
28use pyo3::{prelude::*, types::PyList};
29use ustr::Ustr;
30
31use crate::{
32 decode::DatabentoDecodeConfig,
33 loader::DatabentoDataLoader,
34 types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, PublisherId},
35};
36
37#[expect(clippy::needless_pass_by_value)]
38#[pymethods]
39#[pyo3_stub_gen::derive::gen_stub_pymethods]
40impl DatabentoDataLoader {
41 #[new]
69 #[pyo3(signature = (publishers_filepath=None, futures_year_digits=None))]
70 fn py_new(
71 publishers_filepath: Option<PathBuf>,
72 futures_year_digits: Option<u8>,
73 ) -> PyResult<Self> {
74 let futures_year_digits = futures_year_digits
75 .map(FuturesYearDigits::try_from)
76 .transpose()
77 .map_err(to_pyvalue_err)?
78 .unwrap_or_default();
79
80 Self::new(publishers_filepath)
81 .map(|loader| loader.with_futures_year_digits(futures_year_digits))
82 .map_err(to_pyvalue_err)
83 }
84
85 #[pyo3(name = "load_publishers")]
91 fn py_load_publishers(&mut self, publishers_filepath: PathBuf) -> PyResult<()> {
92 self.load_publishers(publishers_filepath)
93 .map_err(to_pyvalue_err)
94 }
95
96 #[must_use]
98 #[pyo3(name = "get_publishers")]
99 fn py_get_publishers(&self) -> HashMap<u16, DatabentoPublisher> {
100 self.get_publishers()
101 .iter()
102 .map(|(&key, value)| (key, value.clone()))
103 .collect::<HashMap<u16, DatabentoPublisher>>()
104 }
105
106 #[pyo3(name = "set_dataset_for_venue")]
108 fn py_set_dataset_for_venue(&mut self, dataset: String, venue: Venue) {
109 self.set_dataset_for_venue(Ustr::from(&dataset), venue);
110 }
111
112 #[must_use]
114 #[pyo3(name = "get_dataset_for_venue")]
115 fn py_get_dataset_for_venue(&self, venue: &Venue) -> Option<String> {
116 self.get_dataset_for_venue(venue).map(ToString::to_string)
117 }
118
119 #[must_use]
121 #[pyo3(name = "get_venue_for_publisher")]
122 fn py_get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option<String> {
123 self.get_venue_for_publisher(publisher_id)
124 .map(ToString::to_string)
125 }
126
127 #[pyo3(name = "set_price_precision")]
133 fn py_set_price_precision(&mut self, symbol: &str, price_precision: u8) {
134 self.set_price_precision(Symbol::from(symbol), price_precision);
135 }
136
137 #[must_use]
139 #[pyo3(name = "get_price_precisions")]
140 fn py_get_price_precisions(&self) -> HashMap<String, u8> {
141 self.get_price_precisions()
142 .iter()
143 .map(|(symbol, precision)| (symbol.to_string(), *precision))
144 .collect()
145 }
146
147 #[pyo3(name = "schema_for_file")]
148 fn py_schema_for_file(&self, filepath: PathBuf) -> PyResult<Option<String>> {
149 self.schema_from_file(&filepath).map_err(to_pyvalue_err)
150 }
151
152 #[pyo3(name = "load_instruments")]
161 #[pyo3(signature = (filepath, use_exchange_as_venue, skip_on_error=false, expiration_overrides=None))]
162 fn py_load_instruments(
163 &mut self,
164 py: Python,
165 filepath: PathBuf,
166 use_exchange_as_venue: bool,
167 skip_on_error: bool,
168 expiration_overrides: Option<HashMap<String, HashMap<String, String>>>,
169 ) -> PyResult<Py<PyAny>> {
170 let decode_config = build_decode_config(expiration_overrides)?;
171 let iter = self
172 .load_instruments(
173 &filepath,
174 use_exchange_as_venue,
175 skip_on_error,
176 decode_config.as_ref(),
177 )
178 .map_err(to_pyvalue_err)?;
179
180 let mut data = Vec::new();
181
182 for instrument in iter {
183 let py_object = instrument_any_to_pyobject(py, instrument)?;
184 data.push(py_object);
185 }
186
187 let list = PyList::new(py, &data)?;
188
189 Ok(list.into_py_any_unwrap(py))
190 }
191
192 #[pyo3(name = "load_order_book_deltas")]
201 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
202 fn py_load_order_book_deltas(
203 &self,
204 filepath: PathBuf,
205 instrument_id: Option<InstrumentId>,
206 price_precision: Option<u8>,
207 ) -> PyResult<Vec<OrderBookDelta>> {
208 self.load_order_book_deltas(&filepath, instrument_id, price_precision)
209 .map_err(to_pyvalue_err)
210 }
211
212 #[pyo3(name = "load_order_book_depth")]
218 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
219 fn py_load_order_book_depth(
220 &self,
221 filepath: PathBuf,
222 instrument_id: Option<InstrumentId>,
223 price_precision: Option<u8>,
224 ) -> PyResult<Vec<OrderBookDepth>> {
225 self.load_order_book_depth(&filepath, instrument_id, price_precision)
226 .map_err(to_pyvalue_err)
227 }
228
229 #[pyo3(name = "load_quotes")]
235 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
236 fn py_load_quotes(
237 &self,
238 filepath: PathBuf,
239 instrument_id: Option<InstrumentId>,
240 price_precision: Option<u8>,
241 ) -> PyResult<Vec<QuoteTick>> {
242 self.load_quotes(&filepath, instrument_id, price_precision)
243 .map_err(to_pyvalue_err)
244 }
245
246 #[pyo3(name = "load_bbo_quotes")]
252 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
253 fn py_load_bbo_quotes(
254 &self,
255 filepath: PathBuf,
256 instrument_id: Option<InstrumentId>,
257 price_precision: Option<u8>,
258 ) -> PyResult<Vec<QuoteTick>> {
259 self.load_bbo_quotes(&filepath, instrument_id, price_precision)
260 .map_err(to_pyvalue_err)
261 }
262
263 #[pyo3(name = "load_cmbp_quotes")]
269 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
270 fn py_load_cmbp_quotes(
271 &self,
272 filepath: PathBuf,
273 instrument_id: Option<InstrumentId>,
274 price_precision: Option<u8>,
275 ) -> PyResult<Vec<QuoteTick>> {
276 self.load_cmbp_quotes(&filepath, instrument_id, price_precision)
277 .map_err(to_pyvalue_err)
278 }
279
280 #[pyo3(name = "load_cbbo_quotes")]
286 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
287 fn py_load_cbbo_quotes(
288 &self,
289 filepath: PathBuf,
290 instrument_id: Option<InstrumentId>,
291 price_precision: Option<u8>,
292 ) -> PyResult<Vec<QuoteTick>> {
293 self.load_cbbo_quotes(&filepath, instrument_id, price_precision)
294 .map_err(to_pyvalue_err)
295 }
296
297 #[pyo3(name = "load_tbbo_trades")]
303 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
304 fn py_load_tbbo_trades(
305 &self,
306 filepath: PathBuf,
307 instrument_id: Option<InstrumentId>,
308 price_precision: Option<u8>,
309 ) -> PyResult<Vec<TradeTick>> {
310 self.load_tbbo_trades(&filepath, instrument_id, price_precision)
311 .map_err(to_pyvalue_err)
312 }
313
314 #[pyo3(name = "load_tcbbo_trades")]
320 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
321 fn py_load_tcbbo_trades(
322 &self,
323 filepath: PathBuf,
324 instrument_id: Option<InstrumentId>,
325 price_precision: Option<u8>,
326 ) -> PyResult<Vec<TradeTick>> {
327 self.load_tcbbo_trades(&filepath, instrument_id, price_precision)
328 .map_err(to_pyvalue_err)
329 }
330
331 #[pyo3(name = "load_trades")]
337 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
338 fn py_load_trades(
339 &self,
340 filepath: PathBuf,
341 instrument_id: Option<InstrumentId>,
342 price_precision: Option<u8>,
343 ) -> PyResult<Vec<TradeTick>> {
344 self.load_trades(&filepath, instrument_id, price_precision)
345 .map_err(to_pyvalue_err)
346 }
347
348 #[pyo3(name = "load_bars")]
354 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None, timestamp_on_close=true))]
355 fn py_load_bars(
356 &self,
357 filepath: PathBuf,
358 instrument_id: Option<InstrumentId>,
359 price_precision: Option<u8>,
360 timestamp_on_close: bool,
361 ) -> PyResult<Vec<Bar>> {
362 self.load_bars(
363 &filepath,
364 instrument_id,
365 price_precision,
366 Some(timestamp_on_close),
367 )
368 .map_err(to_pyvalue_err)
369 }
370
371 #[pyo3(name = "load_status")]
372 #[pyo3(signature = (filepath, instrument_id=None))]
373 fn py_load_status(
374 &self,
375 filepath: PathBuf,
376 instrument_id: Option<InstrumentId>,
377 ) -> PyResult<Vec<InstrumentStatus>> {
378 let iter = self
379 .load_status_records::<dbn::StatusMsg>(&filepath, instrument_id)
380 .map_err(to_pyvalue_err)?;
381
382 let mut data = Vec::new();
383
384 for result in iter {
385 match result {
386 Ok(item) => data.push(item),
387 Err(e) => return Err(to_pyvalue_err(e)),
388 }
389 }
390
391 Ok(data)
392 }
393
394 #[pyo3(name = "load_imbalance")]
395 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
396 fn py_load_imbalance(
397 &self,
398 filepath: PathBuf,
399 instrument_id: Option<InstrumentId>,
400 price_precision: Option<u8>,
401 ) -> PyResult<Vec<DatabentoImbalance>> {
402 let iter = self
403 .read_imbalance_records::<dbn::ImbalanceMsg>(&filepath, instrument_id, price_precision)
404 .map_err(to_pyvalue_err)?;
405
406 let mut data = Vec::new();
407
408 for result in iter {
409 match result {
410 Ok(item) => data.push(item),
411 Err(e) => return Err(to_pyvalue_err(e)),
412 }
413 }
414
415 Ok(data)
416 }
417
418 #[pyo3(name = "load_statistics")]
419 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
420 fn py_load_statistics(
421 &self,
422 filepath: PathBuf,
423 instrument_id: Option<InstrumentId>,
424 price_precision: Option<u8>,
425 ) -> PyResult<Vec<DatabentoStatistics>> {
426 let iter = self
427 .read_statistics_records::<dbn::StatMsg>(&filepath, instrument_id, price_precision)
428 .map_err(to_pyvalue_err)?;
429
430 let mut data = Vec::new();
431
432 for result in iter {
433 match result {
434 Ok(item) => data.push(item),
435 Err(e) => return Err(to_pyvalue_err(e)),
436 }
437 }
438
439 Ok(data)
440 }
441}
442
443fn build_decode_config(
445 expiration_overrides: Option<HashMap<String, HashMap<String, String>>>,
446) -> PyResult<Option<DatabentoDecodeConfig>> {
447 expiration_overrides
448 .map(|overrides| decode_config_from_overrides(overrides).map_err(to_pyvalue_err))
449 .transpose()
450}
451
452fn decode_config_from_overrides(
455 overrides: HashMap<String, HashMap<String, String>>,
456) -> Result<DatabentoDecodeConfig, String> {
457 let mut config = DatabentoDecodeConfig::default();
458
459 for (dataset_name, times) in overrides {
460 let dataset = dataset_name
461 .parse::<dbn::Dataset>()
462 .map_err(|_| format!("Unknown dataset '{dataset_name}'"))?;
463 let rule = config
464 .option_expiration
465 .get_mut(&dataset)
466 .ok_or_else(|| format!("No expiration correction rule for dataset '{dataset_name}'"))?;
467
468 for (underlying, value) in times {
469 let time = parse_expiration_time(&value)?;
470 if underlying == "default" {
471 rule.default_time = time;
472 } else {
473 rule.overrides.insert(Ustr::from(&underlying), time);
474 }
475 }
476 }
477
478 Ok(config)
479}
480
481fn parse_expiration_time(value: &str) -> Result<Time, String> {
483 value
484 .parse::<Time>()
485 .map_err(|_| format!("Invalid expiration time '{value}', expected 'HH:MM' or 'HH:MM:SS'"))
486}
487
488#[cfg(test)]
489mod tests {
490 use std::collections::HashMap;
491
492 use databento::dbn;
493 use jiff::civil::Time;
494 use rstest::rstest;
495 use ustr::Ustr;
496
497 use super::decode_config_from_overrides;
498
499 fn overrides(
500 dataset: &str,
501 entries: &[(&str, &str)],
502 ) -> HashMap<String, HashMap<String, String>> {
503 let inner = entries
504 .iter()
505 .map(|(k, v)| ((*k).to_string(), (*v).to_string()))
506 .collect();
507 HashMap::from([(dataset.to_string(), inner)])
508 }
509
510 #[rstest]
511 fn test_default_key_sets_dataset_default_time() {
512 let config =
513 decode_config_from_overrides(overrides("OPRA.PILLAR", &[("default", "15:30")]))
514 .unwrap();
515 let rule = config
516 .option_expiration
517 .get(&dbn::Dataset::OpraPillar)
518 .unwrap();
519 assert_eq!(rule.default_time, Time::constant(15, 30, 0, 0));
520 assert!(rule.overrides.is_empty());
521 }
522
523 #[rstest]
524 fn test_underlying_key_sets_override_and_keeps_default() {
525 let config =
526 decode_config_from_overrides(overrides("OPRA.PILLAR", &[("SPX", "09:30:00")])).unwrap();
527 let rule = config
528 .option_expiration
529 .get(&dbn::Dataset::OpraPillar)
530 .unwrap();
531 assert_eq!(
532 rule.overrides.get(&Ustr::from("SPX")).copied(),
533 Some(Time::constant(9, 30, 0, 0))
534 );
535 assert_eq!(rule.default_time, Time::constant(16, 0, 0, 0));
536 }
537
538 #[rstest]
539 fn test_unknown_dataset_errors() {
540 let err = decode_config_from_overrides(overrides("NOT.ADATASET", &[("default", "16:00")]))
541 .unwrap_err();
542 assert!(err.contains("Unknown dataset"), "was: {err}");
543 }
544
545 #[rstest]
546 fn test_dataset_without_rule_errors() {
547 let err = decode_config_from_overrides(overrides("GLBX.MDP3", &[("default", "16:00")]))
549 .unwrap_err();
550 assert!(err.contains("No expiration correction rule"), "was: {err}");
551 }
552
553 #[rstest]
554 fn test_invalid_time_errors() {
555 let err = decode_config_from_overrides(overrides("OPRA.PILLAR", &[("default", "nope")]))
556 .unwrap_err();
557 assert!(err.contains("Invalid expiration time"), "was: {err}");
558 }
559}