1use std::{collections::HashMap, path::PathBuf};
19
20use databento::dbn;
21use jiff::civil::Time;
22use nautilus_core::python::{IntoPyObjectNautilusExt, to_pyvalue_err};
23use nautilus_model::{
24 data::{Bar, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
25 identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
26 python::instruments::instrument_any_to_pyobject,
27};
28use pyo3::{prelude::*, types::PyList};
29use ustr::Ustr;
30
31use crate::{
32 decode::DatabentoDecodeConfig,
33 loader::DatabentoDataLoader,
34 types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, PublisherId},
35};
36
37#[expect(clippy::needless_pass_by_value)]
38#[pymethods]
39#[pyo3_stub_gen::derive::gen_stub_pymethods]
40impl DatabentoDataLoader {
41 #[new]
69 #[pyo3(signature = (publishers_filepath=None, futures_year_digits=None, share_class_space=true))]
70 fn py_new(
71 publishers_filepath: Option<PathBuf>,
72 futures_year_digits: Option<u8>,
73 share_class_space: bool,
74 ) -> PyResult<Self> {
75 let futures_year_digits = futures_year_digits
76 .map(FuturesYearDigits::try_from)
77 .transpose()
78 .map_err(to_pyvalue_err)?
79 .unwrap_or_default();
80
81 Self::new(publishers_filepath)
82 .map(|loader| {
83 loader
84 .with_futures_year_digits(futures_year_digits)
85 .with_share_class_space(share_class_space)
86 })
87 .map_err(to_pyvalue_err)
88 }
89
90 #[pyo3(name = "load_publishers")]
96 fn py_load_publishers(&mut self, publishers_filepath: PathBuf) -> PyResult<()> {
97 self.load_publishers(publishers_filepath)
98 .map_err(to_pyvalue_err)
99 }
100
101 #[must_use]
103 #[pyo3(name = "get_publishers")]
104 fn py_get_publishers(&self) -> HashMap<u16, DatabentoPublisher> {
105 self.get_publishers()
106 .iter()
107 .map(|(&key, value)| (key, value.clone()))
108 .collect::<HashMap<u16, DatabentoPublisher>>()
109 }
110
111 #[pyo3(name = "set_dataset_for_venue")]
113 fn py_set_dataset_for_venue(&mut self, dataset: String, venue: Venue) {
114 self.set_dataset_for_venue(Ustr::from(&dataset), venue);
115 }
116
117 #[must_use]
119 #[pyo3(name = "get_dataset_for_venue")]
120 fn py_get_dataset_for_venue(&self, venue: &Venue) -> Option<String> {
121 self.get_dataset_for_venue(venue).map(ToString::to_string)
122 }
123
124 #[must_use]
126 #[pyo3(name = "get_venue_for_publisher")]
127 fn py_get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option<String> {
128 self.get_venue_for_publisher(publisher_id)
129 .map(ToString::to_string)
130 }
131
132 #[pyo3(name = "set_price_precision")]
138 fn py_set_price_precision(&mut self, symbol: &str, price_precision: u8) {
139 self.set_price_precision(Symbol::from(symbol), price_precision);
140 }
141
142 #[must_use]
144 #[pyo3(name = "get_price_precisions")]
145 fn py_get_price_precisions(&self) -> HashMap<String, u8> {
146 self.get_price_precisions()
147 .iter()
148 .map(|(symbol, precision)| (symbol.to_string(), *precision))
149 .collect()
150 }
151
152 #[pyo3(name = "schema_for_file")]
153 fn py_schema_for_file(&self, filepath: PathBuf) -> PyResult<Option<String>> {
154 self.schema_from_file(&filepath).map_err(to_pyvalue_err)
155 }
156
157 #[pyo3(name = "load_instruments")]
166 #[pyo3(signature = (filepath, use_exchange_as_venue, skip_on_error=false, expiration_overrides=None))]
167 fn py_load_instruments(
168 &mut self,
169 py: Python,
170 filepath: PathBuf,
171 use_exchange_as_venue: bool,
172 skip_on_error: bool,
173 expiration_overrides: Option<HashMap<String, HashMap<String, String>>>,
174 ) -> PyResult<Py<PyAny>> {
175 let decode_config = build_decode_config(expiration_overrides)?;
176 let iter = self
177 .load_instruments(
178 &filepath,
179 use_exchange_as_venue,
180 skip_on_error,
181 decode_config.as_ref(),
182 )
183 .map_err(to_pyvalue_err)?;
184
185 let mut data = Vec::new();
186
187 for instrument in iter {
188 let py_object = instrument_any_to_pyobject(py, instrument)?;
189 data.push(py_object);
190 }
191
192 let list = PyList::new(py, &data)?;
193
194 Ok(list.into_py_any_unwrap(py))
195 }
196
197 #[pyo3(name = "load_order_book_deltas")]
206 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
207 fn py_load_order_book_deltas(
208 &self,
209 filepath: PathBuf,
210 instrument_id: Option<InstrumentId>,
211 price_precision: Option<u8>,
212 ) -> PyResult<Vec<OrderBookDelta>> {
213 self.load_order_book_deltas(&filepath, instrument_id, price_precision)
214 .map_err(to_pyvalue_err)
215 }
216
217 #[pyo3(name = "load_order_book_depth")]
223 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
224 fn py_load_order_book_depth(
225 &self,
226 filepath: PathBuf,
227 instrument_id: Option<InstrumentId>,
228 price_precision: Option<u8>,
229 ) -> PyResult<Vec<OrderBookDepth>> {
230 self.load_order_book_depth(&filepath, instrument_id, price_precision)
231 .map_err(to_pyvalue_err)
232 }
233
234 #[pyo3(name = "load_quotes")]
240 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
241 fn py_load_quotes(
242 &self,
243 filepath: PathBuf,
244 instrument_id: Option<InstrumentId>,
245 price_precision: Option<u8>,
246 ) -> PyResult<Vec<QuoteTick>> {
247 self.load_quotes(&filepath, instrument_id, price_precision)
248 .map_err(to_pyvalue_err)
249 }
250
251 #[pyo3(name = "load_bbo_quotes")]
257 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
258 fn py_load_bbo_quotes(
259 &self,
260 filepath: PathBuf,
261 instrument_id: Option<InstrumentId>,
262 price_precision: Option<u8>,
263 ) -> PyResult<Vec<QuoteTick>> {
264 self.load_bbo_quotes(&filepath, instrument_id, price_precision)
265 .map_err(to_pyvalue_err)
266 }
267
268 #[pyo3(name = "load_cmbp_quotes")]
274 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
275 fn py_load_cmbp_quotes(
276 &self,
277 filepath: PathBuf,
278 instrument_id: Option<InstrumentId>,
279 price_precision: Option<u8>,
280 ) -> PyResult<Vec<QuoteTick>> {
281 self.load_cmbp_quotes(&filepath, instrument_id, price_precision)
282 .map_err(to_pyvalue_err)
283 }
284
285 #[pyo3(name = "load_cbbo_quotes")]
291 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
292 fn py_load_cbbo_quotes(
293 &self,
294 filepath: PathBuf,
295 instrument_id: Option<InstrumentId>,
296 price_precision: Option<u8>,
297 ) -> PyResult<Vec<QuoteTick>> {
298 self.load_cbbo_quotes(&filepath, instrument_id, price_precision)
299 .map_err(to_pyvalue_err)
300 }
301
302 #[pyo3(name = "load_tbbo_trades")]
308 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
309 fn py_load_tbbo_trades(
310 &self,
311 filepath: PathBuf,
312 instrument_id: Option<InstrumentId>,
313 price_precision: Option<u8>,
314 ) -> PyResult<Vec<TradeTick>> {
315 self.load_tbbo_trades(&filepath, instrument_id, price_precision)
316 .map_err(to_pyvalue_err)
317 }
318
319 #[pyo3(name = "load_tcbbo_trades")]
325 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
326 fn py_load_tcbbo_trades(
327 &self,
328 filepath: PathBuf,
329 instrument_id: Option<InstrumentId>,
330 price_precision: Option<u8>,
331 ) -> PyResult<Vec<TradeTick>> {
332 self.load_tcbbo_trades(&filepath, instrument_id, price_precision)
333 .map_err(to_pyvalue_err)
334 }
335
336 #[pyo3(name = "load_trades")]
342 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
343 fn py_load_trades(
344 &self,
345 filepath: PathBuf,
346 instrument_id: Option<InstrumentId>,
347 price_precision: Option<u8>,
348 ) -> PyResult<Vec<TradeTick>> {
349 self.load_trades(&filepath, instrument_id, price_precision)
350 .map_err(to_pyvalue_err)
351 }
352
353 #[pyo3(name = "load_bars")]
359 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None, timestamp_on_close=true))]
360 fn py_load_bars(
361 &self,
362 filepath: PathBuf,
363 instrument_id: Option<InstrumentId>,
364 price_precision: Option<u8>,
365 timestamp_on_close: bool,
366 ) -> PyResult<Vec<Bar>> {
367 self.load_bars(
368 &filepath,
369 instrument_id,
370 price_precision,
371 Some(timestamp_on_close),
372 )
373 .map_err(to_pyvalue_err)
374 }
375
376 #[pyo3(name = "load_status")]
377 #[pyo3(signature = (filepath, instrument_id=None))]
378 fn py_load_status(
379 &self,
380 filepath: PathBuf,
381 instrument_id: Option<InstrumentId>,
382 ) -> PyResult<Vec<InstrumentStatus>> {
383 let iter = self
384 .load_status_records::<dbn::StatusMsg>(&filepath, instrument_id)
385 .map_err(to_pyvalue_err)?;
386
387 let mut data = Vec::new();
388
389 for result in iter {
390 match result {
391 Ok(item) => data.push(item),
392 Err(e) => return Err(to_pyvalue_err(e)),
393 }
394 }
395
396 Ok(data)
397 }
398
399 #[pyo3(name = "load_imbalance")]
400 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
401 fn py_load_imbalance(
402 &self,
403 filepath: PathBuf,
404 instrument_id: Option<InstrumentId>,
405 price_precision: Option<u8>,
406 ) -> PyResult<Vec<DatabentoImbalance>> {
407 let iter = self
408 .read_imbalance_records::<dbn::ImbalanceMsg>(&filepath, instrument_id, price_precision)
409 .map_err(to_pyvalue_err)?;
410
411 let mut data = Vec::new();
412
413 for result in iter {
414 match result {
415 Ok(item) => data.push(item),
416 Err(e) => return Err(to_pyvalue_err(e)),
417 }
418 }
419
420 Ok(data)
421 }
422
423 #[pyo3(name = "load_statistics")]
424 #[pyo3(signature = (filepath, instrument_id=None, price_precision=None))]
425 fn py_load_statistics(
426 &self,
427 filepath: PathBuf,
428 instrument_id: Option<InstrumentId>,
429 price_precision: Option<u8>,
430 ) -> PyResult<Vec<DatabentoStatistics>> {
431 let iter = self
432 .read_statistics_records::<dbn::StatMsg>(&filepath, instrument_id, price_precision)
433 .map_err(to_pyvalue_err)?;
434
435 let mut data = Vec::new();
436
437 for result in iter {
438 match result {
439 Ok(item) => data.push(item),
440 Err(e) => return Err(to_pyvalue_err(e)),
441 }
442 }
443
444 Ok(data)
445 }
446}
447
448fn build_decode_config(
450 expiration_overrides: Option<HashMap<String, HashMap<String, String>>>,
451) -> PyResult<Option<DatabentoDecodeConfig>> {
452 expiration_overrides
453 .map(|overrides| decode_config_from_overrides(overrides).map_err(to_pyvalue_err))
454 .transpose()
455}
456
457fn decode_config_from_overrides(
460 overrides: HashMap<String, HashMap<String, String>>,
461) -> Result<DatabentoDecodeConfig, String> {
462 let mut config = DatabentoDecodeConfig::default();
463
464 for (dataset_name, times) in overrides {
465 let dataset = dataset_name
466 .parse::<dbn::Dataset>()
467 .map_err(|_| format!("Unknown dataset '{dataset_name}'"))?;
468 let rule = config
469 .option_expiration
470 .get_mut(&dataset)
471 .ok_or_else(|| format!("No expiration correction rule for dataset '{dataset_name}'"))?;
472
473 for (underlying, value) in times {
474 let time = parse_expiration_time(&value)?;
475 if underlying == "default" {
476 rule.default_time = time;
477 } else {
478 rule.overrides.insert(Ustr::from(&underlying), time);
479 }
480 }
481 }
482
483 Ok(config)
484}
485
486fn parse_expiration_time(value: &str) -> Result<Time, String> {
488 value
489 .parse::<Time>()
490 .map_err(|_| format!("Invalid expiration time '{value}', expected 'HH:MM' or 'HH:MM:SS'"))
491}
492
493#[cfg(test)]
494mod tests {
495 use std::collections::HashMap;
496
497 use databento::dbn;
498 use jiff::civil::Time;
499 use rstest::rstest;
500 use ustr::Ustr;
501
502 use super::decode_config_from_overrides;
503
504 fn overrides(
505 dataset: &str,
506 entries: &[(&str, &str)],
507 ) -> HashMap<String, HashMap<String, String>> {
508 let inner = entries
509 .iter()
510 .map(|(k, v)| ((*k).to_string(), (*v).to_string()))
511 .collect();
512 HashMap::from([(dataset.to_string(), inner)])
513 }
514
515 #[rstest]
516 fn test_default_key_sets_dataset_default_time() {
517 let config =
518 decode_config_from_overrides(overrides("OPRA.PILLAR", &[("default", "15:30")]))
519 .unwrap();
520 let rule = config
521 .option_expiration
522 .get(&dbn::Dataset::OpraPillar)
523 .unwrap();
524 assert_eq!(rule.default_time, Time::constant(15, 30, 0, 0));
525 assert!(rule.overrides.is_empty());
526 }
527
528 #[rstest]
529 fn test_underlying_key_sets_override_and_keeps_default() {
530 let config =
531 decode_config_from_overrides(overrides("OPRA.PILLAR", &[("SPX", "09:30:00")])).unwrap();
532 let rule = config
533 .option_expiration
534 .get(&dbn::Dataset::OpraPillar)
535 .unwrap();
536 assert_eq!(
537 rule.overrides.get(&Ustr::from("SPX")).copied(),
538 Some(Time::constant(9, 30, 0, 0))
539 );
540 assert_eq!(rule.default_time, Time::constant(16, 0, 0, 0));
541 }
542
543 #[rstest]
544 fn test_unknown_dataset_errors() {
545 let err = decode_config_from_overrides(overrides("NOT.ADATASET", &[("default", "16:00")]))
546 .unwrap_err();
547 assert!(err.contains("Unknown dataset"), "was: {err}");
548 }
549
550 #[rstest]
551 fn test_dataset_without_rule_errors() {
552 let err = decode_config_from_overrides(overrides("GLBX.MDP3", &[("default", "16:00")]))
554 .unwrap_err();
555 assert!(err.contains("No expiration correction rule"), "was: {err}");
556 }
557
558 #[rstest]
559 fn test_invalid_time_errors() {
560 let err = decode_config_from_overrides(overrides("OPRA.PILLAR", &[("default", "nope")]))
561 .unwrap_err();
562 assert!(err.contains("Invalid expiration time"), "was: {err}");
563 }
564}