1#![expect(
17 clippy::too_many_arguments,
18 reason = "PyO3 catalog wrapper mirrors Python API dispatch surface"
19)]
20
21use std::{collections::HashMap, sync::Mutex};
22
23use nautilus_common::{
24 config::ConfigError, enums::Environment, python::config_error_to_pyvalue_err,
25};
26use nautilus_core::{
27 UnixNanos,
28 python::{to_pytype_err, to_pyvalue_err},
29};
30use nautilus_model::{
31 data::{
32 Bar, Data, FundingRateUpdate, HasTsInit, IndexPriceUpdate, InstrumentStatus,
33 MarkPriceUpdate, NautilusDataType, OptionGreeks, OrderBookDelta, OrderBookDepth, QuoteTick,
34 TradeTick, close::InstrumentClose,
35 },
36 python::{
37 data::data_to_pyobject,
38 instruments::{
39 PyNautilusInstrumentType, instrument_any_to_pyobject, pyobject_to_instrument_any,
40 },
41 },
42};
43use nautilus_serialization::{
44 arrow::{
45 DecodeTypedFromRecordBatch, EncodeToRecordBatch, display::instrument::encode_instruments,
46 },
47 python::arrow::{arrow_record_batches_to_pyarrow_stream, arrow_record_batches_to_pybytes},
48};
49use pyo3::{
50 exceptions::PyIOError,
51 prelude::*,
52 types::{PyBytes, PyDict, PyList},
53};
54
55use crate::{
56 backend::{migration::build_catalog_migration_plan, parquet::catalog::ParquetDataCatalog},
57 catalog::{
58 traits::{CatalogQuery, CatalogReader, CatalogRecordQuery, CatalogWriter},
59 types::{CatalogDataType, HasCatalogDataType, parquet_catalog_data_type_path_prefixes},
60 },
61 config::CatalogCompression,
62 python::backend::{
63 PyCatalogDataType, arrow_ipc_batches, arrow_ipc_data_schema, arrow_ipc_record_schema,
64 arrow_record_batches_from_pybytes, catalog_metadata_to_pydict, catalog_record_type_from_py,
65 nautilus_data_type_from_py, to_pyio_err, write_record_params_from_py,
66 },
67};
68
69fn write_parquet_data<T>(
70 py: Python<'_>,
71 catalog: &PyParquetDataCatalog,
72 data: Vec<T>,
73 start: Option<u64>,
74 end: Option<u64>,
75 skip_disjoint_check: bool,
76 label: &str,
77) -> PyResult<String>
78where
79 T: HasTsInit + EncodeToRecordBatch + HasCatalogDataType + Send,
80{
81 catalog
82 .mutate(py, move || {
83 catalog.inner.write_to_parquet(
84 &data,
85 start.map(UnixNanos::from),
86 end.map(UnixNanos::from),
87 Some(skip_disjoint_check),
88 )
89 })
90 .map(|path| path.to_string_lossy().to_string())
91 .map_err(|e| PyIOError::new_err(format!("Failed to write {label}: {e}")))
92}
93
94fn query_parquet_data<T>(
95 py: Python<'_>,
96 catalog: &mut ParquetDataCatalog,
97 identifiers: Option<Vec<String>>,
98 start: Option<u64>,
99 end: Option<u64>,
100 where_clause: Option<&str>,
101 files: Option<Vec<String>>,
102 optimize_file_loading: bool,
103 error_context: &str,
104) -> PyResult<Vec<T>>
105where
106 T: DecodeTypedFromRecordBatch + HasCatalogDataType + HasTsInit + Send,
107{
108 py.detach(move || {
109 catalog.query::<T>(
110 identifiers,
111 start.map(UnixNanos::from),
112 end.map(UnixNanos::from),
113 where_clause,
114 files,
115 optimize_file_loading,
116 )
117 })
118 .map_err(|e| PyIOError::new_err(format!("{error_context}: {e}")))
119}
120
121fn reject_parquet_as_of(as_of: Option<&Bound<'_, PyAny>>) -> PyResult<()> {
122 if as_of.is_some() {
123 return Err(to_pyvalue_err("ParquetDataCatalog does not support as_of"));
124 }
125
126 Ok(())
127}
128
129#[pyclass(name = "ParquetDataCatalog", module = "nautilus_trader.persistence")]
131#[pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.persistence")]
132pub struct PyParquetDataCatalog {
133 inner: ParquetDataCatalog,
134 mutation_lock: Mutex<()>,
135}
136
137#[pymethods]
138#[pyo3_stub_gen::derive::gen_stub_pymethods]
139impl PyParquetDataCatalog {
140 #[new]
155 #[pyo3(signature = (base_path, storage_options=None, batch_size=None, compression=None, max_row_group_size=None))]
156 pub fn py_new(
157 base_path: &str,
158 storage_options: Option<HashMap<String, String>>,
159 batch_size: Option<usize>,
160 compression: Option<u8>,
161 max_row_group_size: Option<usize>,
162 ) -> PyResult<Self> {
163 let compression = compression
164 .map(catalog_compression_from_code)
165 .transpose()?
166 .map(parquet::basic::Compression::from);
167
168 let storage_options = storage_options.map(|m| m.into_iter().collect());
170
171 let inner = ParquetDataCatalog::from_uri(
172 base_path,
173 storage_options,
174 batch_size,
175 compression,
176 max_row_group_size,
177 )
178 .map_err(|e| match e.downcast::<ConfigError>() {
179 Ok(e) => config_error_to_pyvalue_err(e),
180 Err(e) => PyIOError::new_err(format!("Failed to create ParquetDataCatalog: {e}")),
181 })?;
182
183 Ok(Self {
184 inner,
185 mutation_lock: Mutex::new(()),
186 })
187 }
188
189 #[pyo3(signature = (parquet_path, storage_options=None, dry_run=false))]
191 pub fn migrate_from_legacy_parquet_path(
192 mut slf: PyRefMut<'_, Self>,
193 parquet_path: &str,
194 storage_options: Option<HashMap<String, String>>,
195 dry_run: bool,
196 ) -> PyResult<usize> {
197 let storage_options = storage_options.map(|m| m.into_iter().collect());
198 let source = ParquetDataCatalog::from_uri(parquet_path, storage_options, None, None, None)
199 .map_err(to_pyio_err)?;
200 let py = slf.py();
201 if dry_run {
202 return py
203 .detach(|| {
204 let plan = build_catalog_migration_plan(&source)?;
205 plan.ensure_ready()?;
206 Ok::<usize, anyhow::Error>(0)
207 })
208 .map_err(to_pyio_err);
209 }
210
211 let inner = &mut slf.inner;
212 py.detach(|| {
213 inner
214 .migrate_from_legacy_parquet_catalog(&source)
215 .map(|report| report.migrated_rows)
216 })
217 .map_err(to_pyio_err)
218 }
219
220 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
232 pub fn write_order_book_deltas(
233 &self,
234 py: Python<'_>,
235 data: Vec<OrderBookDelta>,
236 start: Option<u64>,
237 end: Option<u64>,
238 skip_disjoint_check: bool,
239 ) -> PyResult<String> {
240 write_parquet_data(
241 py,
242 self,
243 data,
244 start,
245 end,
246 skip_disjoint_check,
247 "order book deltas",
248 )
249 }
250
251 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
263 pub fn write_order_book_depths(
264 &self,
265 py: Python<'_>,
266 data: Vec<OrderBookDepth>,
267 start: Option<u64>,
268 end: Option<u64>,
269 skip_disjoint_check: bool,
270 ) -> PyResult<String> {
271 write_parquet_data(
272 py,
273 self,
274 data,
275 start,
276 end,
277 skip_disjoint_check,
278 "order book depths",
279 )
280 }
281
282 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
294 pub fn write_quote_ticks(
295 &self,
296 py: Python<'_>,
297 data: Vec<QuoteTick>,
298 start: Option<u64>,
299 end: Option<u64>,
300 skip_disjoint_check: bool,
301 ) -> PyResult<String> {
302 write_parquet_data(
303 py,
304 self,
305 data,
306 start,
307 end,
308 skip_disjoint_check,
309 "quote ticks",
310 )
311 }
312
313 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
325 pub fn write_trade_ticks(
326 &self,
327 py: Python<'_>,
328 data: Vec<TradeTick>,
329 start: Option<u64>,
330 end: Option<u64>,
331 skip_disjoint_check: bool,
332 ) -> PyResult<String> {
333 write_parquet_data(
334 py,
335 self,
336 data,
337 start,
338 end,
339 skip_disjoint_check,
340 "trade ticks",
341 )
342 }
343
344 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
356 pub fn write_bars(
357 &self,
358 py: Python<'_>,
359 data: Vec<Bar>,
360 start: Option<u64>,
361 end: Option<u64>,
362 skip_disjoint_check: bool,
363 ) -> PyResult<String> {
364 write_parquet_data(py, self, data, start, end, skip_disjoint_check, "bars")
365 }
366
367 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
379 pub fn write_mark_price_updates(
380 &self,
381 py: Python<'_>,
382 data: Vec<MarkPriceUpdate>,
383 start: Option<u64>,
384 end: Option<u64>,
385 skip_disjoint_check: bool,
386 ) -> PyResult<String> {
387 write_parquet_data(
388 py,
389 self,
390 data,
391 start,
392 end,
393 skip_disjoint_check,
394 "mark price updates",
395 )
396 }
397
398 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
410 pub fn write_index_price_updates(
411 &self,
412 py: Python<'_>,
413 data: Vec<IndexPriceUpdate>,
414 start: Option<u64>,
415 end: Option<u64>,
416 skip_disjoint_check: bool,
417 ) -> PyResult<String> {
418 write_parquet_data(
419 py,
420 self,
421 data,
422 start,
423 end,
424 skip_disjoint_check,
425 "index price updates",
426 )
427 }
428
429 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
441 pub fn write_funding_rate_updates(
442 &self,
443 py: Python<'_>,
444 data: Vec<FundingRateUpdate>,
445 start: Option<u64>,
446 end: Option<u64>,
447 skip_disjoint_check: bool,
448 ) -> PyResult<String> {
449 write_parquet_data(
450 py,
451 self,
452 data,
453 start,
454 end,
455 skip_disjoint_check,
456 "funding rate updates",
457 )
458 }
459
460 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
472 pub fn write_option_greeks(
473 &self,
474 py: Python<'_>,
475 data: Vec<OptionGreeks>,
476 start: Option<u64>,
477 end: Option<u64>,
478 skip_disjoint_check: bool,
479 ) -> PyResult<String> {
480 write_parquet_data(
481 py,
482 self,
483 data,
484 start,
485 end,
486 skip_disjoint_check,
487 "option greeks",
488 )
489 }
490
491 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
493 pub fn write_instrument_statuses(
494 &self,
495 py: Python<'_>,
496 data: Vec<InstrumentStatus>,
497 start: Option<u64>,
498 end: Option<u64>,
499 skip_disjoint_check: bool,
500 ) -> PyResult<String> {
501 write_parquet_data(
502 py,
503 self,
504 data,
505 start,
506 end,
507 skip_disjoint_check,
508 "instrument statuses",
509 )
510 }
511
512 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
514 pub fn write_instrument_closes(
515 &self,
516 py: Python<'_>,
517 data: Vec<InstrumentClose>,
518 start: Option<u64>,
519 end: Option<u64>,
520 skip_disjoint_check: bool,
521 ) -> PyResult<String> {
522 write_parquet_data(
523 py,
524 self,
525 data,
526 start,
527 end,
528 skip_disjoint_check,
529 "instrument closes",
530 )
531 }
532
533 pub fn write_instruments(&self, instruments: &Bound<'_, PyAny>) -> PyResult<Vec<String>> {
547 let data = instruments;
548 let py = data.py();
549 let list = data.cast::<PyList>()?;
550 let mut instruments = Vec::with_capacity(list.len());
551 for item in list.iter() {
552 let py_item: Py<PyAny> = item.unbind();
553 let instrument = pyobject_to_instrument_any(py, py_item)?;
554 instruments.push(instrument);
555 }
556
557 self.mutate(py, || self.inner.write_instruments(instruments))
558 .map(|paths| {
559 paths
560 .into_iter()
561 .map(|p| p.to_string_lossy().to_string())
562 .collect()
563 })
564 .map_err(|e| PyIOError::new_err(format!("Failed to write instruments: {e}")))
565 }
566
567 #[pyo3(signature = (
580 instrument_ids=None,
581 start=None,
582 end=None,
583 where_clause=None,
584 instrument_type=None,
585 ))]
586 #[expect(clippy::needless_pass_by_value)]
587 pub fn instruments(
588 &mut self,
589 py: Python<'_>,
590 instrument_ids: Option<Vec<String>>,
591 start: Option<u64>,
592 end: Option<u64>,
593 where_clause: Option<&str>,
594 instrument_type: Option<PyRef<'_, PyNautilusInstrumentType>>,
595 ) -> PyResult<Vec<Py<PyAny>>> {
596 let instrument_type = instrument_type.map(|instrument_type| instrument_type.inner());
597
598 let rust_instruments = py
599 .detach(|| {
600 self.inner.query_instruments_filtered_with_where_and_type(
601 instrument_ids.as_deref(),
602 start.map(UnixNanos::from),
603 end.map(UnixNanos::from),
604 where_clause,
605 instrument_type.as_ref(),
606 )
607 })
608 .map_err(|e| PyIOError::new_err(format!("Failed to query instruments: {e}")))?;
609
610 rust_instruments
611 .into_iter()
612 .map(|inst| instrument_any_to_pyobject(py, inst))
613 .collect()
614 }
615
616 #[pyo3(signature = (
618 instrument_ids=None,
619 start=None,
620 end=None,
621 where_clause=None,
622 instrument_type=None,
623 ))]
624 #[expect(clippy::needless_pass_by_value)]
625 pub fn query_instrument_arrow_bytes(
626 &mut self,
627 py: Python<'_>,
628 instrument_ids: Option<Vec<String>>,
629 start: Option<u64>,
630 end: Option<u64>,
631 where_clause: Option<&str>,
632 instrument_type: Option<PyRef<'_, PyNautilusInstrumentType>>,
633 ) -> PyResult<Py<PyBytes>> {
634 let instrument_type = instrument_type.map(|instrument_type| instrument_type.inner());
635
636 let instruments = py
637 .detach(|| {
638 self.inner.query_instruments_filtered_with_where_and_type(
639 instrument_ids.as_deref(),
640 start.map(UnixNanos::from),
641 end.map(UnixNanos::from),
642 where_clause,
643 instrument_type.as_ref(),
644 )
645 })
646 .map_err(|e| PyIOError::new_err(format!("Failed query instruments: {e}")))?;
647
648 let batch = encode_instruments(&instruments)
649 .map_err(|e| PyIOError::new_err(format!("Failed encode instruments: {e}")))?;
650 let schema = batch.schema().as_ref().clone();
651 arrow_record_batches_to_pybytes(py, &schema, &[batch])
652 }
653
654 #[pyo3(signature = (
656 instrument_ids=None,
657 start=None,
658 end=None,
659 where_clause=None,
660 instrument_type=None,
661 ))]
662 #[expect(clippy::needless_pass_by_value)]
663 pub fn query_instrument_arrow_stream(
664 &mut self,
665 py: Python<'_>,
666 instrument_ids: Option<Vec<String>>,
667 start: Option<u64>,
668 end: Option<u64>,
669 where_clause: Option<&str>,
670 instrument_type: Option<PyRef<'_, PyNautilusInstrumentType>>,
671 ) -> PyResult<Py<PyAny>> {
672 let instrument_type = instrument_type.map(|instrument_type| instrument_type.inner());
673
674 let instruments = py
675 .detach(|| {
676 self.inner.query_instruments_filtered_with_where_and_type(
677 instrument_ids.as_deref(),
678 start.map(UnixNanos::from),
679 end.map(UnixNanos::from),
680 where_clause,
681 instrument_type.as_ref(),
682 )
683 })
684 .map_err(|e| PyIOError::new_err(format!("Failed query instruments: {e}")))?;
685
686 let batch = encode_instruments(&instruments)
687 .map_err(|e| PyIOError::new_err(format!("Failed encode instruments: {e}")))?;
688 let schema = batch.schema().as_ref().clone();
689 arrow_record_batches_to_pyarrow_stream(py, &schema, vec![batch])
690 }
691
692 #[pyo3(signature = (data_type, identifier=None, *, start, end))]
701 #[expect(clippy::needless_pass_by_value)]
702 pub fn extend_file_name(
703 &self,
704 py: Python<'_>,
705 data_type: PyCatalogDataType,
706 identifier: Option<String>,
707 start: u64,
708 end: u64,
709 ) -> PyResult<()> {
710 let data_type = data_type.into_inner();
711 let start_nanos = UnixNanos::from(start);
712 let end_nanos = UnixNanos::from(end);
713
714 self.mutate(py, || {
715 self.inner
716 .extend_file_name(&data_type, identifier.as_deref(), start_nanos, end_nanos)
717 })
718 .map_err(|e| PyIOError::new_err(format!("Failed to extend file name: {e}")))
719 }
720
721 #[pyo3(signature = (start=None, end=None, ensure_contiguous_files=None, deduplicate=None))]
730 pub fn consolidate_catalog(
731 &self,
732 py: Python<'_>,
733 start: Option<u64>,
734 end: Option<u64>,
735 ensure_contiguous_files: Option<bool>,
736 deduplicate: Option<bool>,
737 ) -> PyResult<()> {
738 let start_nanos = start.map(UnixNanos::from);
739 let end_nanos = end.map(UnixNanos::from);
740
741 self.mutate(py, || {
742 self.inner.consolidate_catalog(
743 start_nanos,
744 end_nanos,
745 ensure_contiguous_files,
746 deduplicate,
747 )
748 })
749 .map_err(|e| PyIOError::new_err(format!("Failed to consolidate catalog: {e}")))
750 }
751
752 #[pyo3(signature = (data_type, identifier=None, start=None, end=None, ensure_contiguous_files=None, deduplicate=None))]
763 #[expect(clippy::needless_pass_by_value)]
764 pub fn consolidate_data(
765 &mut self,
766 py: Python<'_>,
767 data_type: PyCatalogDataType,
768 identifier: Option<String>,
769 start: Option<u64>,
770 end: Option<u64>,
771 ensure_contiguous_files: Option<bool>,
772 deduplicate: Option<bool>,
773 ) -> PyResult<()> {
774 let data_type = data_type.into_inner();
775 let start_nanos = start.map(UnixNanos::from);
776 let end_nanos = end.map(UnixNanos::from);
777
778 py.detach(|| {
779 self.inner.consolidate_data(
780 &data_type,
781 identifier.as_deref(),
782 start_nanos,
783 end_nanos,
784 ensure_contiguous_files,
785 deduplicate,
786 )
787 })
788 .map_err(|e| PyIOError::new_err(format!("Failed to consolidate data: {e}")))
789 }
790
791 #[pyo3(signature = (period_nanos=None, start=None, end=None, ensure_contiguous_files=None))]
806 pub fn consolidate_catalog_by_period(
807 &mut self,
808 period_nanos: Option<u64>,
809 start: Option<u64>,
810 end: Option<u64>,
811 ensure_contiguous_files: Option<bool>,
812 ) -> PyResult<()> {
813 let start_nanos = start.map(UnixNanos::from);
814 let end_nanos = end.map(UnixNanos::from);
815
816 self.inner
817 .consolidate_catalog_by_period(
818 period_nanos,
819 start_nanos,
820 end_nanos,
821 ensure_contiguous_files,
822 )
823 .map_err(|e| {
824 PyIOError::new_err(format!("Failed to consolidate catalog by period: {e}"))
825 })
826 }
827
828 #[pyo3(signature = (data_type, identifier=None, period_nanos=None, start=None, end=None, ensure_contiguous_files=None))]
844 #[expect(clippy::needless_pass_by_value)]
845 pub fn consolidate_data_by_period(
846 &mut self,
847 py: Python<'_>,
848 data_type: PyCatalogDataType,
849 identifier: Option<String>,
850 period_nanos: Option<u64>,
851 start: Option<u64>,
852 end: Option<u64>,
853 ensure_contiguous_files: Option<bool>,
854 ) -> PyResult<()> {
855 let data_type = data_type.into_inner();
856 let start_nanos = start.map(UnixNanos::from);
857 let end_nanos = end.map(UnixNanos::from);
858
859 let mut consolidate = || {
860 self.inner.consolidate_data_by_period(
861 &data_type,
862 identifier.as_deref(),
863 period_nanos,
864 start_nanos,
865 end_nanos,
866 ensure_contiguous_files,
867 )
868 };
869
870 let result = if matches!(
871 &data_type,
872 CatalogDataType::Data(NautilusDataType::Custom { .. })
873 ) {
874 consolidate()
875 } else {
876 py.detach(consolidate)
877 };
878
879 result.map_err(|e| PyIOError::new_err(format!("Failed to consolidate data by period: {e}")))
880 }
881
882 pub fn reset_all_file_names(&self, py: Python<'_>) -> PyResult<()> {
884 self.mutate(py, || self.inner.reset_all_file_names())
885 .map_err(|e| PyIOError::new_err(format!("Failed to reset catalog file names: {e}")))
886 }
887
888 #[pyo3(signature = (data_type, identifier=None))]
895 #[expect(clippy::needless_pass_by_value)]
896 pub fn reset_data_file_names(
897 &self,
898 py: Python<'_>,
899 data_type: PyCatalogDataType,
900 identifier: Option<String>,
901 ) -> PyResult<()> {
902 let data_type = data_type.into_inner();
903 self.mutate(py, || {
904 self.inner
905 .reset_data_file_names(&data_type, identifier.as_deref())
906 })
907 .map_err(|e| PyIOError::new_err(format!("Failed to reset data file names: {e}")))
908 }
909
910 #[pyo3(signature = (start=None, end=None))]
932 pub fn delete_catalog_range(&mut self, start: Option<u64>, end: Option<u64>) -> PyResult<()> {
933 let start_nanos = start.map(UnixNanos::from);
934 let end_nanos = end.map(UnixNanos::from);
935
936 self.inner
937 .delete_catalog_range(start_nanos, end_nanos)
938 .map_err(|e| PyIOError::new_err(format!("Failed to delete catalog range: {e}")))
939 }
940
941 #[pyo3(signature = (data_type, identifier=None, start=None, end=None))]
963 #[expect(clippy::needless_pass_by_value)]
964 pub fn delete_data_range(
965 &mut self,
966 py: Python<'_>,
967 #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
968 '_,
969 PyAny,
970 >,
971 identifier: Option<String>,
972 start: Option<u64>,
973 end: Option<u64>,
974 ) -> PyResult<()> {
975 let data_type = nautilus_data_type_from_py(data_type)?;
976 let start_nanos = start.map(UnixNanos::from);
977 let end_nanos = end.map(UnixNanos::from);
978
979 let mut delete = || {
980 self.inner
981 .delete_data_range(&data_type, identifier.as_deref(), start_nanos, end_nanos)
982 };
983
984 let result = if matches!(&data_type, NautilusDataType::Custom { .. }) {
985 delete()
986 } else {
987 py.detach(delete)
988 };
989
990 result.map_err(|e| PyIOError::new_err(format!("Failed to delete data range: {e}")))
991 }
992
993 #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
1001 pub fn write_custom_data(
1002 &self,
1003 py: Python<'_>,
1004 data: Vec<Bound<'_, PyAny>>,
1005 start: Option<u64>,
1006 end: Option<u64>,
1007 skip_disjoint_check: bool,
1008 ) -> PyResult<String> {
1009 use nautilus_model::data::CustomData;
1010
1011 let mut custom_items: Vec<CustomData> = Vec::with_capacity(data.len());
1012 for obj in data {
1013 let custom = obj.extract::<CustomData>().map_err(|_| {
1014 to_pytype_err(
1015 "write_custom_data requires CustomData wrappers; wrap with CustomData(data_type=DataType(cls, metadata=...), data=...)",
1016 )
1017 })?;
1018
1019 custom_items.push(custom);
1020 }
1021
1022 if custom_items.is_empty() {
1023 return Ok(String::new());
1024 }
1025
1026 let start_nanos = start.map(UnixNanos::from);
1027 let end_nanos = end.map(UnixNanos::from);
1028 let write_error =
1029 |e: anyhow::Error| PyIOError::new_err(format!("Failed to write custom data: {e}"));
1030 let custom_refs = custom_items.iter().collect::<Vec<_>>();
1031 let write = self
1032 .inner
1033 .prepare_custom_data_write(&custom_refs, start_nanos, end_nanos)
1034 .map_err(write_error)?;
1035
1036 self.mutate(py, || {
1037 self.inner
1038 .write_custom_data_prepared(&write, Some(skip_disjoint_check))
1039 })
1040 .map(|path| path.to_string_lossy().to_string())
1041 .map_err(write_error)
1042 }
1043
1044 pub fn list_instruments(
1046 &self,
1047 py: Python<'_>,
1048 data_type: PyCatalogDataType,
1049 ) -> PyResult<Vec<String>> {
1050 let data_type = data_type.into_inner();
1051 py.detach(|| self.inner.list_instruments(&data_type))
1052 .map_err(|e| PyIOError::new_err(format!("Failed to list instruments: {e}")))
1053 }
1054
1055 pub fn list_parquet_files(
1057 &self,
1058 py: Python<'_>,
1059 data_type: PyCatalogDataType,
1060 identifier: &str,
1061 ) -> PyResult<Vec<String>> {
1062 let data_type = data_type.into_inner();
1063 let list_files = |prefix: &str| -> PyResult<Vec<String>> {
1064 let directory = self
1065 .inner
1066 .make_path(prefix, Some(identifier))
1067 .map_err(|e| PyIOError::new_err(format!("Failed to list parquet files: {e}")))?;
1068 py.detach(|| self.inner.list_parquet_files(&directory))
1069 .map_err(|e| PyIOError::new_err(format!("Failed to list parquet files: {e}")))
1070 };
1071 let mut files = Vec::new();
1072
1073 for prefix in parquet_catalog_data_type_path_prefixes(&data_type) {
1074 files.extend(list_files(prefix.as_ref())?);
1075 }
1076
1077 Ok(files)
1078 }
1079
1080 #[pyo3(signature = (data_type, identifiers=None, start=None, end=None))]
1095 pub fn query_files(
1096 &self,
1097 py: Python<'_>,
1098 data_type: PyCatalogDataType,
1099 identifiers: Option<Vec<String>>,
1100 start: Option<u64>,
1101 end: Option<u64>,
1102 ) -> PyResult<Vec<String>> {
1103 let data_type = data_type.into_inner();
1104 let start_nanos = start.map(UnixNanos::from);
1105 let end_nanos = end.map(UnixNanos::from);
1106
1107 py.detach(|| {
1108 self.inner
1109 .query_files(&data_type, identifiers, start_nanos, end_nanos)
1110 })
1111 .map_err(|e| PyIOError::new_err(format!("Failed to query files list: {e}")))
1112 }
1113
1114 #[pyo3(signature = (data_type, identifiers=None, start=None, end=None, where_clause=None))]
1116 pub fn query_metadata(
1117 &mut self,
1118 py: Python<'_>,
1119 #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
1120 '_,
1121 PyAny,
1122 >,
1123 identifiers: Option<Vec<String>>,
1124 start: Option<u64>,
1125 end: Option<u64>,
1126 where_clause: Option<&str>,
1127 ) -> PyResult<Py<PyDict>> {
1128 let data_type = nautilus_data_type_from_py(data_type)?;
1129
1130 let metadata = py
1131 .detach(|| {
1132 CatalogReader::query_metadata(
1133 &mut self.inner,
1134 &CatalogQuery::new(data_type)
1135 .with_identifiers(identifiers)
1136 .with_range(start.map(UnixNanos::from), end.map(UnixNanos::from))
1137 .with_where_clause(where_clause.map(str::to_string)),
1138 )
1139 })
1140 .map_err(|e| PyIOError::new_err(format!("Metadata query failed: {e}")))?;
1141
1142 catalog_metadata_to_pydict(py, metadata)
1143 }
1144
1145 #[pyo3(signature = (
1147 data_type,
1148 identifiers=None,
1149 start=None,
1150 end=None,
1151 where_clause=None,
1152 display=true,
1153 as_of=None,
1154 ))]
1155 pub fn query_data_arrow_bytes(
1156 &mut self,
1157 py: Python<'_>,
1158 #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
1159 '_,
1160 PyAny,
1161 >,
1162 identifiers: Option<Vec<String>>,
1163 start: Option<u64>,
1164 end: Option<u64>,
1165 where_clause: Option<&str>,
1166 display: bool,
1167 as_of: Option<&Bound<'_, PyAny>>,
1168 ) -> PyResult<Py<PyBytes>> {
1169 let data_type = nautilus_data_type_from_py(data_type)?;
1170 reject_parquet_as_of(as_of)?;
1171 let query = CatalogQuery::new(data_type.clone())
1172 .with_identifiers(identifiers)
1173 .with_range(start.map(UnixNanos::from), end.map(UnixNanos::from))
1174 .with_where_clause(where_clause.map(str::to_string));
1175
1176 let batches = py
1177 .detach(|| {
1178 if display {
1179 CatalogReader::query_display_record_batches(&mut self.inner, &query)
1180 } else {
1181 let data = CatalogReader::query_batch(&mut self.inner, &query)?
1182 .to_data_vec_for_compat();
1183 crate::common::arrow::data_to_arrow_batches(&data_type, data)
1184 }
1185 })
1186 .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?;
1187
1188 let schema = arrow_ipc_data_schema(&data_type, &batches, display)?;
1189 let batches = arrow_ipc_batches(&schema, batches)?;
1190 arrow_record_batches_to_pybytes(py, &schema, &batches)
1191 }
1192
1193 #[pyo3(signature = (
1195 data_type,
1196 identifiers=None,
1197 start=None,
1198 end=None,
1199 where_clause=None,
1200 display=true,
1201 as_of=None,
1202 ))]
1203 pub fn query_data_arrow_stream(
1204 &mut self,
1205 py: Python<'_>,
1206 #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
1207 '_,
1208 PyAny,
1209 >,
1210 identifiers: Option<Vec<String>>,
1211 start: Option<u64>,
1212 end: Option<u64>,
1213 where_clause: Option<&str>,
1214 display: bool,
1215 as_of: Option<&Bound<'_, PyAny>>,
1216 ) -> PyResult<Py<PyAny>> {
1217 let data_type = nautilus_data_type_from_py(data_type)?;
1218 reject_parquet_as_of(as_of)?;
1219 let query = CatalogQuery::new(data_type.clone())
1220 .with_identifiers(identifiers)
1221 .with_range(start.map(UnixNanos::from), end.map(UnixNanos::from))
1222 .with_where_clause(where_clause.map(str::to_string));
1223
1224 let batches = py
1225 .detach(|| {
1226 if display {
1227 CatalogReader::query_display_record_batches(&mut self.inner, &query)
1228 } else {
1229 let data = CatalogReader::query_batch(&mut self.inner, &query)?
1230 .to_data_vec_for_compat();
1231 crate::common::arrow::data_to_arrow_batches(&data_type, data)
1232 }
1233 })
1234 .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?;
1235
1236 let schema = arrow_ipc_data_schema(&data_type, &batches, display)?;
1237 let batches = arrow_ipc_batches(&schema, batches)?;
1238 arrow_record_batches_to_pyarrow_stream(py, &schema, batches)
1239 }
1240
1241 #[pyo3(signature = (record_type, data, identifier=None, params=None))]
1243 pub fn write_record_arrow_bytes(
1244 &mut self,
1245 py: Python<'_>,
1246 #[gen_stub(override_type(type_repr = "model.NautilusRecordType"))] record_type: &Bound<
1247 '_,
1248 PyAny,
1249 >,
1250 data: Vec<u8>,
1251 identifier: Option<String>,
1252 params: Option<Py<PyDict>>,
1253 ) -> PyResult<()> {
1254 let record_type = catalog_record_type_from_py(record_type)?;
1255 let batches = arrow_record_batches_from_pybytes(data)?;
1256 let params = write_record_params_from_py(py, identifier, params)?;
1257
1258 py.detach(|| CatalogWriter::write_records(&mut self.inner, record_type, &batches, params))
1259 .map_err(|e| PyIOError::new_err(format!("Failed write records: {e}")))
1260 }
1261
1262 #[pyo3(signature = (
1264 record_type,
1265 identifier=None,
1266 start=None,
1267 end=None,
1268 where_clause=None,
1269 display=true,
1270 as_of=None,
1271 ))]
1272 pub fn query_record_arrow_bytes(
1273 &mut self,
1274 py: Python<'_>,
1275 #[gen_stub(override_type(type_repr = "model.NautilusRecordType"))] record_type: &Bound<
1276 '_,
1277 PyAny,
1278 >,
1279 identifier: Option<String>,
1280 start: Option<u64>,
1281 end: Option<u64>,
1282 where_clause: Option<&str>,
1283 display: bool,
1284 as_of: Option<&Bound<'_, PyAny>>,
1285 ) -> PyResult<Py<PyBytes>> {
1286 let record_type = catalog_record_type_from_py(record_type)?;
1287 reject_parquet_as_of(as_of)?;
1288 let start_nanos = start.map(UnixNanos::from);
1289 let end_nanos = end.map(UnixNanos::from);
1290
1291 let batches = py
1292 .detach(|| {
1293 if display {
1294 CatalogReader::query_record_display_batches(
1295 &mut self.inner,
1296 &CatalogRecordQuery::new(record_type)
1297 .with_identifier(identifier)
1298 .with_range(start_nanos, end_nanos)
1299 .with_where_clause(where_clause.map(str::to_string)),
1300 )
1301 } else {
1302 CatalogReader::query_record_batches(
1303 &mut self.inner,
1304 &CatalogRecordQuery::new(record_type)
1305 .with_identifier(identifier)
1306 .with_range(start_nanos, end_nanos)
1307 .with_where_clause(where_clause.map(str::to_string)),
1308 )
1309 }
1310 })
1311 .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?;
1312
1313 let schema = arrow_ipc_record_schema(record_type, &batches)?;
1314 let batches = arrow_ipc_batches(&schema, batches)?;
1315
1316 arrow_record_batches_to_pybytes(py, &schema, &batches)
1317 }
1318
1319 #[pyo3(signature = (
1321 record_type,
1322 identifier=None,
1323 start=None,
1324 end=None,
1325 where_clause=None,
1326 display=true,
1327 as_of=None,
1328 ))]
1329 pub fn query_record_arrow_stream(
1330 &mut self,
1331 py: Python<'_>,
1332 #[gen_stub(override_type(type_repr = "model.NautilusRecordType"))] record_type: &Bound<
1333 '_,
1334 PyAny,
1335 >,
1336 identifier: Option<String>,
1337 start: Option<u64>,
1338 end: Option<u64>,
1339 where_clause: Option<&str>,
1340 display: bool,
1341 as_of: Option<&Bound<'_, PyAny>>,
1342 ) -> PyResult<Py<PyAny>> {
1343 let record_type = catalog_record_type_from_py(record_type)?;
1344 reject_parquet_as_of(as_of)?;
1345 let start_nanos = start.map(UnixNanos::from);
1346 let end_nanos = end.map(UnixNanos::from);
1347
1348 let batches = py
1349 .detach(|| {
1350 if display {
1351 CatalogReader::query_record_display_batches(
1352 &mut self.inner,
1353 &CatalogRecordQuery::new(record_type)
1354 .with_identifier(identifier)
1355 .with_range(start_nanos, end_nanos)
1356 .with_where_clause(where_clause.map(str::to_string)),
1357 )
1358 } else {
1359 CatalogReader::query_record_batches(
1360 &mut self.inner,
1361 &CatalogRecordQuery::new(record_type)
1362 .with_identifier(identifier)
1363 .with_range(start_nanos, end_nanos)
1364 .with_where_clause(where_clause.map(str::to_string)),
1365 )
1366 }
1367 })
1368 .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?;
1369
1370 let schema = arrow_ipc_record_schema(record_type, &batches)?;
1371 let batches = arrow_ipc_batches(&schema, batches)?;
1372
1373 arrow_record_batches_to_pyarrow_stream(py, &schema, batches)
1374 }
1375
1376 #[pyo3(signature = (start, end, data_type, identifier=None))]
1389 #[expect(clippy::needless_pass_by_value)]
1390 pub fn get_missing_intervals_for_request(
1391 &self,
1392 py: Python<'_>,
1393 start: u64,
1394 end: u64,
1395 data_type: PyCatalogDataType,
1396 identifier: Option<String>,
1397 ) -> PyResult<Vec<(u64, u64)>> {
1398 let data_type = data_type.into_inner();
1399 py.detach(|| {
1400 self.inner.get_missing_intervals_for_request(
1401 start,
1402 end,
1403 &data_type,
1404 identifier.as_deref(),
1405 )
1406 })
1407 .map_err(|e| PyIOError::new_err(format!("Failed to get missing intervals: {e}")))
1408 }
1409
1410 #[pyo3(signature = (data_type, identifier=None))]
1421 #[expect(clippy::needless_pass_by_value)]
1422 pub fn query_first_timestamp(
1423 &self,
1424 py: Python<'_>,
1425 data_type: PyCatalogDataType,
1426 identifier: Option<String>,
1427 ) -> PyResult<Option<u64>> {
1428 let data_type = data_type.into_inner();
1429 py.detach(|| {
1430 self.inner
1431 .query_first_timestamp(&data_type, identifier.as_deref())
1432 })
1433 .map_err(|e| PyIOError::new_err(format!("Failed to query first timestamp: {e}")))
1434 }
1435
1436 #[pyo3(signature = (data_type, identifier=None))]
1447 #[expect(clippy::needless_pass_by_value)]
1448 pub fn query_last_timestamp(
1449 &self,
1450 py: Python<'_>,
1451 data_type: PyCatalogDataType,
1452 identifier: Option<String>,
1453 ) -> PyResult<Option<u64>> {
1454 let data_type = data_type.into_inner();
1455 py.detach(|| {
1456 self.inner
1457 .query_last_timestamp(&data_type, identifier.as_deref())
1458 })
1459 .map_err(|e| PyIOError::new_err(format!("Failed to query last timestamp: {e}")))
1460 }
1461
1462 #[pyo3(signature = (data_type, identifier=None))]
1473 #[expect(clippy::needless_pass_by_value)]
1474 pub fn get_intervals(
1475 &self,
1476 py: Python<'_>,
1477 data_type: PyCatalogDataType,
1478 identifier: Option<String>,
1479 ) -> PyResult<Vec<(u64, u64)>> {
1480 let data_type = data_type.into_inner();
1481 py.detach(|| self.inner.get_intervals(&data_type, identifier.as_deref()))
1482 .map_err(|e| PyIOError::new_err(format!("Failed to get intervals: {e}")))
1483 }
1484
1485 #[pyo3(signature = (data_type, identifiers=None, start=None, end=None, where_clause=None, files=None, optimize_file_loading=true))]
1487 #[expect(
1488 clippy::too_many_arguments,
1489 reason = "PyO3 signature mirrors the catalog query filters"
1490 )]
1491 pub fn query(
1492 &mut self,
1493 py: Python<'_>,
1494 #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
1495 '_,
1496 PyAny,
1497 >,
1498 identifiers: Option<Vec<String>>,
1499 start: Option<u64>,
1500 end: Option<u64>,
1501 where_clause: Option<&str>,
1502 files: Option<Vec<String>>,
1503 optimize_file_loading: bool,
1504 ) -> PyResult<Vec<Py<PyAny>>> {
1505 let data_type = nautilus_data_type_from_py(data_type)?;
1506 let start_nanos = start.map(UnixNanos::from);
1507 let end_nanos = end.map(UnixNanos::from);
1508
1509 macro_rules! typed {
1510 ($type:ty) => {
1511 query_parquet_data::<$type>(
1512 py,
1513 &mut self.inner,
1514 identifiers,
1515 start,
1516 end,
1517 where_clause,
1518 files,
1519 optimize_file_loading,
1520 "Query failed",
1521 )?
1522 .into_iter()
1523 .map(Data::from)
1524 .collect::<Vec<Data>>()
1525 };
1526 }
1527
1528 let data = match data_type {
1529 NautilusDataType::Custom { type_name } => py
1530 .detach(|| {
1531 self.inner.query_custom_data_dynamic(
1532 &type_name,
1533 identifiers.as_deref(),
1534 start_nanos,
1535 end_nanos,
1536 where_clause,
1537 files.clone(),
1538 optimize_file_loading,
1539 )
1540 })
1541 .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?,
1542 NautilusDataType::Instrument => {
1543 let instruments = py
1544 .detach(|| {
1545 self.inner.query_instruments_filtered_with_where(
1546 identifiers.as_deref(),
1547 start_nanos,
1548 end_nanos,
1549 where_clause,
1550 )
1551 })
1552 .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?;
1553
1554 return instruments
1555 .into_iter()
1556 .map(|instrument| instrument_any_to_pyobject(py, instrument))
1557 .collect();
1558 }
1559 NautilusDataType::OrderBookDelta => typed!(OrderBookDelta),
1560 NautilusDataType::OrderBookDepth => typed!(OrderBookDepth),
1561 NautilusDataType::QuoteTick => typed!(QuoteTick),
1562 NautilusDataType::TradeTick => typed!(TradeTick),
1563 NautilusDataType::Bar => typed!(Bar),
1564 NautilusDataType::MarkPriceUpdate => typed!(MarkPriceUpdate),
1565 NautilusDataType::IndexPriceUpdate => typed!(IndexPriceUpdate),
1566 NautilusDataType::FundingRateUpdate => typed!(FundingRateUpdate),
1567 NautilusDataType::OptionGreeks => typed!(OptionGreeks),
1568 NautilusDataType::InstrumentStatus => typed!(InstrumentStatus),
1569 NautilusDataType::InstrumentClose => typed!(InstrumentClose),
1570 #[cfg(feature = "defi")]
1571 NautilusDataType::Defi => {
1572 return Err(to_pytype_err("Defi data is not supported by query"));
1573 }
1574 #[cfg(not(feature = "defi"))]
1575 #[allow(
1576 unreachable_patterns,
1577 reason = "DeFi variants can exist without this crate's defi feature"
1578 )]
1579 _ => return Err(to_pytype_err("Defi data is not supported by query")),
1580 };
1581
1582 let mut python_objects = Vec::new();
1583 for item in data {
1584 python_objects.push(data_to_pyobject(py, item)?);
1585 }
1586
1587 Ok(python_objects)
1588 }
1589
1590 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1605 pub fn query_order_book_deltas(
1606 &mut self,
1607 py: Python<'_>,
1608 identifiers: Option<Vec<String>>,
1609 start: Option<u64>,
1610 end: Option<u64>,
1611 where_clause: Option<&str>,
1612 ) -> PyResult<Vec<OrderBookDelta>> {
1613 query_parquet_data::<OrderBookDelta>(
1614 py,
1615 &mut self.inner,
1616 identifiers,
1617 start,
1618 end,
1619 where_clause,
1620 None,
1621 true,
1622 "Failed to query data",
1623 )
1624 }
1625
1626 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1639 pub fn query_order_book_depths(
1640 &mut self,
1641 py: Python<'_>,
1642 identifiers: Option<Vec<String>>,
1643 start: Option<u64>,
1644 end: Option<u64>,
1645 where_clause: Option<&str>,
1646 ) -> PyResult<Vec<OrderBookDepth>> {
1647 query_parquet_data::<OrderBookDepth>(
1648 py,
1649 &mut self.inner,
1650 identifiers,
1651 start,
1652 end,
1653 where_clause,
1654 None,
1655 true,
1656 "Failed to query data",
1657 )
1658 }
1659
1660 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1675 pub fn query_quote_ticks(
1676 &mut self,
1677 py: Python<'_>,
1678 identifiers: Option<Vec<String>>,
1679 start: Option<u64>,
1680 end: Option<u64>,
1681 where_clause: Option<&str>,
1682 ) -> PyResult<Vec<QuoteTick>> {
1683 query_parquet_data::<QuoteTick>(
1684 py,
1685 &mut self.inner,
1686 identifiers,
1687 start,
1688 end,
1689 where_clause,
1690 None,
1691 true,
1692 "Failed to query data",
1693 )
1694 }
1695
1696 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1711 pub fn query_trade_ticks(
1712 &mut self,
1713 py: Python<'_>,
1714 identifiers: Option<Vec<String>>,
1715 start: Option<u64>,
1716 end: Option<u64>,
1717 where_clause: Option<&str>,
1718 ) -> PyResult<Vec<TradeTick>> {
1719 query_parquet_data::<TradeTick>(
1720 py,
1721 &mut self.inner,
1722 identifiers,
1723 start,
1724 end,
1725 where_clause,
1726 None,
1727 true,
1728 "Failed to query data",
1729 )
1730 }
1731
1732 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1747 pub fn query_bars(
1748 &mut self,
1749 py: Python<'_>,
1750 identifiers: Option<Vec<String>>,
1751 start: Option<u64>,
1752 end: Option<u64>,
1753 where_clause: Option<&str>,
1754 ) -> PyResult<Vec<Bar>> {
1755 query_parquet_data::<Bar>(
1756 py,
1757 &mut self.inner,
1758 identifiers,
1759 start,
1760 end,
1761 where_clause,
1762 None,
1763 true,
1764 "Failed to query data",
1765 )
1766 }
1767
1768 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1781 pub fn query_mark_price_updates(
1782 &mut self,
1783 py: Python<'_>,
1784 identifiers: Option<Vec<String>>,
1785 start: Option<u64>,
1786 end: Option<u64>,
1787 where_clause: Option<&str>,
1788 ) -> PyResult<Vec<MarkPriceUpdate>> {
1789 query_parquet_data::<MarkPriceUpdate>(
1790 py,
1791 &mut self.inner,
1792 identifiers,
1793 start,
1794 end,
1795 where_clause,
1796 None,
1797 true,
1798 "Failed to query data",
1799 )
1800 }
1801
1802 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1815 pub fn query_index_price_updates(
1816 &mut self,
1817 py: Python<'_>,
1818 identifiers: Option<Vec<String>>,
1819 start: Option<u64>,
1820 end: Option<u64>,
1821 where_clause: Option<&str>,
1822 ) -> PyResult<Vec<IndexPriceUpdate>> {
1823 query_parquet_data::<IndexPriceUpdate>(
1824 py,
1825 &mut self.inner,
1826 identifiers,
1827 start,
1828 end,
1829 where_clause,
1830 None,
1831 true,
1832 "Failed to query data",
1833 )
1834 }
1835
1836 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1849 pub fn query_funding_rate_updates(
1850 &mut self,
1851 py: Python<'_>,
1852 identifiers: Option<Vec<String>>,
1853 start: Option<u64>,
1854 end: Option<u64>,
1855 where_clause: Option<&str>,
1856 ) -> PyResult<Vec<FundingRateUpdate>> {
1857 query_parquet_data::<FundingRateUpdate>(
1858 py,
1859 &mut self.inner,
1860 identifiers,
1861 start,
1862 end,
1863 where_clause,
1864 None,
1865 true,
1866 "Failed to query data",
1867 )
1868 }
1869
1870 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1872 pub fn query_option_greeks(
1873 &mut self,
1874 py: Python<'_>,
1875 identifiers: Option<Vec<String>>,
1876 start: Option<u64>,
1877 end: Option<u64>,
1878 where_clause: Option<&str>,
1879 ) -> PyResult<Vec<OptionGreeks>> {
1880 query_parquet_data::<OptionGreeks>(
1881 py,
1882 &mut self.inner,
1883 identifiers,
1884 start,
1885 end,
1886 where_clause,
1887 None,
1888 true,
1889 "Failed to query data",
1890 )
1891 }
1892
1893 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1895 pub fn query_instrument_statuses(
1896 &mut self,
1897 py: Python<'_>,
1898 identifiers: Option<Vec<String>>,
1899 start: Option<u64>,
1900 end: Option<u64>,
1901 where_clause: Option<&str>,
1902 ) -> PyResult<Vec<InstrumentStatus>> {
1903 query_parquet_data::<InstrumentStatus>(
1904 py,
1905 &mut self.inner,
1906 identifiers,
1907 start,
1908 end,
1909 where_clause,
1910 None,
1911 true,
1912 "Failed to query data",
1913 )
1914 }
1915
1916 #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1918 pub fn query_instrument_closes(
1919 &mut self,
1920 py: Python<'_>,
1921 identifiers: Option<Vec<String>>,
1922 start: Option<u64>,
1923 end: Option<u64>,
1924 where_clause: Option<&str>,
1925 ) -> PyResult<Vec<InstrumentClose>> {
1926 query_parquet_data::<InstrumentClose>(
1927 py,
1928 &mut self.inner,
1929 identifiers,
1930 start,
1931 end,
1932 where_clause,
1933 None,
1934 true,
1935 "Failed to query data",
1936 )
1937 }
1938
1939 pub fn list_data_types(&self, py: Python<'_>) -> PyResult<Vec<PyCatalogDataType>> {
1945 py.detach(|| self.inner.list_data_types())
1946 .map(|data_types| data_types.into_iter().map(PyCatalogDataType::new).collect())
1947 .map_err(|e| PyIOError::new_err(format!("Failed to list data types: {e}")))
1948 }
1949
1950 pub fn list_live_runs(&self, py: Python<'_>) -> PyResult<Vec<String>> {
1956 py.detach(|| self.inner.list_live_runs())
1957 .map_err(|e| PyIOError::new_err(format!("Failed to list live runs: {e}")))
1958 }
1959
1960 pub fn list_backtest_runs(&self, py: Python<'_>) -> PyResult<Vec<String>> {
1966 py.detach(|| self.inner.list_backtest_runs())
1967 .map_err(|e| PyIOError::new_err(format!("Failed to list backtest runs: {e}")))
1968 }
1969
1970 pub fn list_backtests(&self, py: Python<'_>) -> PyResult<Vec<String>> {
1972 py.detach(|| self.inner.list_backtest_runs())
1973 .map_err(|e| PyIOError::new_err(format!("Failed to list backtests: {e}")))
1974 }
1975
1976 #[pyo3(signature = (instance_id))]
1986 pub fn read_live_run(&self, py: Python<'_>, instance_id: &str) -> PyResult<Vec<Py<PyAny>>> {
1987 let data = self
1988 .inner
1989 .read_live_run(instance_id)
1990 .map_err(|e| PyIOError::new_err(format!("Failed to read live run: {e}")))?;
1991
1992 let mut python_objects = Vec::new();
1993 for item in data {
1994 python_objects.push(data_to_pyobject(py, item)?);
1995 }
1996
1997 Ok(python_objects)
1998 }
1999
2000 #[pyo3(signature = (instance_id))]
2010 pub fn read_backtest(&self, py: Python<'_>, instance_id: &str) -> PyResult<Vec<Py<PyAny>>> {
2011 let data = self
2012 .inner
2013 .read_backtest(instance_id)
2014 .map_err(|e| PyIOError::new_err(format!("Failed to read backtest: {e}")))?;
2015
2016 let mut python_objects = Vec::new();
2017 for item in data {
2018 python_objects.push(data_to_pyobject(py, item)?);
2019 }
2020
2021 Ok(python_objects)
2022 }
2023
2024 #[pyo3(signature = (instance_id, data_type, environment=Environment::Backtest, identifiers=None, use_ts_event_for_ts_init=false))]
2061 #[expect(clippy::needless_pass_by_value)]
2062 pub fn convert_stream_to_data(
2063 &mut self,
2064 py: Python<'_>,
2065 instance_id: &str,
2066 data_type: PyCatalogDataType,
2067 environment: Environment,
2068 identifiers: Option<Vec<String>>,
2069 use_ts_event_for_ts_init: bool,
2070 ) -> PyResult<()> {
2071 let data_type = data_type.into_inner();
2072
2073 py.detach(|| {
2074 self.inner.convert_stream_to_data(
2075 instance_id,
2076 &data_type,
2077 environment,
2078 identifiers.as_deref(),
2079 use_ts_event_for_ts_init,
2080 )
2081 })
2082 .map_err(|e| PyIOError::new_err(format!("Failed to convert stream to data: {e}")))
2083 }
2084
2085 #[pyo3(signature = (data_type, identifiers=None, start=None, end=None, where_clause=None))]
2087 #[expect(clippy::needless_pass_by_value)]
2088 pub fn query_custom_data(
2089 &mut self,
2090 py: Python<'_>,
2091 #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
2092 '_,
2093 PyAny,
2094 >,
2095 identifiers: Option<Vec<String>>,
2096 start: Option<u64>,
2097 end: Option<u64>,
2098 where_clause: Option<&str>,
2099 ) -> PyResult<Vec<Py<PyAny>>> {
2100 let NautilusDataType::Custom { type_name } = nautilus_data_type_from_py(data_type)? else {
2101 return Err(to_pytype_err("data_type must be a custom NautilusDataType"));
2102 };
2103
2104 let start_nanos = start.map(UnixNanos::from);
2105 let end_nanos = end.map(UnixNanos::from);
2106
2107 let data = py
2108 .detach(|| {
2109 self.inner.query_custom_data_dynamic(
2110 &type_name,
2111 identifiers.as_deref(),
2112 start_nanos,
2113 end_nanos,
2114 where_clause,
2115 None,
2116 true,
2117 )
2118 })
2119 .map_err(|e| PyIOError::new_err(format!("Failed to query custom data: {e}")))?;
2120
2121 let mut python_objects = Vec::new();
2122
2123 for item in data {
2124 let py_obj: Py<PyAny> = match item {
2125 Data::Custom(custom) => Py::new(py, custom)?.into_any(),
2126 _ => return Err(PyIOError::new_err("Expected custom data")),
2127 };
2128
2129 python_objects.push(py_obj);
2130 }
2131
2132 Ok(python_objects)
2133 }
2134}
2135
2136impl PyParquetDataCatalog {
2137 fn mutate<T: Send>(
2138 &self,
2139 py: Python<'_>,
2140 operation: impl FnOnce() -> anyhow::Result<T> + Send,
2141 ) -> anyhow::Result<T> {
2142 py.detach(|| {
2143 let _guard = self
2144 .mutation_lock
2145 .lock()
2146 .map_err(|_| anyhow::anyhow!("catalog mutation lock is poisoned"))?;
2147 operation()
2148 })
2149 }
2150}
2151
2152fn catalog_compression_from_code(code: u8) -> PyResult<CatalogCompression> {
2155 match code {
2156 0 => Ok(CatalogCompression::Uncompressed),
2157 1 => Ok(CatalogCompression::Snappy),
2158 2 => Ok(CatalogCompression::Gzip),
2159 4 => Ok(CatalogCompression::Brotli),
2160 5 => Ok(CatalogCompression::Lz4),
2161 6 => Ok(CatalogCompression::Zstd),
2162 _ => Err(to_pyvalue_err(format!(
2163 "Invalid compression code {code}, expected one of 0 (UNCOMPRESSED), 1 (SNAPPY), \
2164 2 (GZIP), 4 (BROTLI), 5 (LZ4_RAW), or 6 (ZSTD)"
2165 ))),
2166 }
2167}
2168
2169#[cfg(test)]
2170mod tests {
2171 use std::{
2172 borrow::Borrow,
2173 cell::Cell,
2174 sync::{Arc, Barrier, Once, TryLockError, mpsc},
2175 thread,
2176 time::Duration,
2177 };
2178
2179 use arrow::{datatypes::Schema, error::ArrowError, record_batch::RecordBatch};
2180 use nautilus_model::{
2181 custom_data,
2182 data::{
2183 BookOrder, CustomData, CustomDataTrait, DataType, ensure_custom_data_json_registered,
2184 register_arrow,
2185 },
2186 enums::{BookAction, OrderSide},
2187 identifiers::InstrumentId,
2188 python::data::PyNautilusDataType,
2189 types::{Price, Quantity},
2190 };
2191 use nautilus_serialization::{
2192 arrow::{ArrowSchemaProvider, DecodeDataFromRecordBatch, EncodingError},
2193 arrow_custom_data,
2194 };
2195 use rstest::rstest;
2196 use tempfile::TempDir;
2197
2198 use super::*;
2199
2200 #[rstest]
2201 #[case::uncompressed(0, CatalogCompression::Uncompressed)]
2202 #[case::snappy(1, CatalogCompression::Snappy)]
2203 #[case::gzip(2, CatalogCompression::Gzip)]
2204 #[case::brotli(4, CatalogCompression::Brotli)]
2205 #[case::lz4(5, CatalogCompression::Lz4)]
2206 #[case::zstd(6, CatalogCompression::Zstd)]
2207 fn catalog_compression_from_code_maps_parquet_codes(
2208 #[case] code: u8,
2209 #[case] expected: CatalogCompression,
2210 ) {
2211 assert_eq!(catalog_compression_from_code(code).unwrap(), expected);
2212 }
2213
2214 #[rstest]
2215 fn test_write_parquet_data_releases_gil() {
2216 Python::initialize();
2217 let directory = TempDir::new().unwrap();
2218 let mut catalog = PyParquetDataCatalog::py_new(
2219 directory.path().to_str().unwrap(),
2220 None,
2221 None,
2222 None,
2223 None,
2224 )
2225 .unwrap();
2226 let delta = OrderBookDelta::clear(
2227 InstrumentId::from("ETH/USDT.BINANCE"),
2228 17,
2229 23.into(),
2230 29.into(),
2231 );
2232
2233 let path = Python::attach(|py| {
2234 write_parquet_data(
2235 py,
2236 &catalog,
2237 vec![GilCheckedDelta(delta, Some(&catalog.mutation_lock))],
2238 None,
2239 None,
2240 false,
2241 "deltas",
2242 )
2243 .unwrap()
2244 });
2245
2246 let loaded = catalog
2247 .inner
2248 .query::<OrderBookDelta>(None, None, None, None, None, true)
2249 .unwrap();
2250
2251 assert!(catalog.inner.file_exists(&path).unwrap());
2252 assert_eq!(loaded, vec![delta]);
2253 }
2254
2255 #[rstest]
2256 fn test_query_parquet_data_releases_gil() {
2257 Python::initialize();
2258 let directory = TempDir::new().unwrap();
2259 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2260 let delta = OrderBookDelta::clear(
2261 InstrumentId::from("ETH/USDT.BINANCE"),
2262 17,
2263 23.into(),
2264 29.into(),
2265 );
2266 catalog
2267 .write_to_parquet(&[delta], None, None, None)
2268 .unwrap();
2269
2270 let loaded = Python::attach(|py| {
2271 query_parquet_data::<GilCheckedDelta>(
2272 py,
2273 &mut catalog,
2274 None,
2275 None,
2276 None,
2277 None,
2278 None,
2279 true,
2280 "Query failed",
2281 )
2282 .unwrap()
2283 });
2284
2285 assert_eq!(loaded.len(), 1);
2286 assert_eq!(loaded[0].0, delta);
2287 assert!(loaded[0].1.is_none());
2288 assert_eq!(loaded[0].0.action, BookAction::Clear);
2289 }
2290
2291 #[rstest]
2292 fn test_catalog_python_delta_round_trip_and_maintenance() {
2293 Python::initialize();
2294 let directory = TempDir::new().unwrap();
2295 let instrument_id = InstrumentId::from("ETH/USDT.BINANCE");
2296 let deltas = vec![
2297 OrderBookDelta::new(
2298 instrument_id,
2299 BookAction::Add,
2300 BookOrder::new(
2301 OrderSide::Buy,
2302 Price::from("100.23"),
2303 Quantity::from("1.234"),
2304 41,
2305 ),
2306 7,
2307 17,
2308 23.into(),
2309 29.into(),
2310 ),
2311 OrderBookDelta::new(
2312 instrument_id,
2313 BookAction::Update,
2314 BookOrder::new(
2315 OrderSide::Sell,
2316 Price::from("101.45"),
2317 Quantity::from("2.345"),
2318 47,
2319 ),
2320 8,
2321 31,
2322 37.into(),
2323 43.into(),
2324 ),
2325 ];
2326
2327 Python::attach(|py| {
2328 let catalog = PyParquetDataCatalog::py_new(
2329 directory.path().to_str().unwrap(),
2330 None,
2331 None,
2332 None,
2333 None,
2334 )
2335 .unwrap();
2336 let catalog = Bound::new(py, catalog).unwrap();
2337 let data_type = PyNautilusDataType::new(NautilusDataType::OrderBookDelta);
2338 let data_type = Bound::new(py, data_type).unwrap();
2339 let path: String = catalog
2340 .call_method1("write_order_book_deltas", (deltas.clone(), 11, 53))
2341 .unwrap()
2342 .extract()
2343 .unwrap();
2344 let loaded: Vec<OrderBookDelta> = catalog
2345 .call_method0("query_order_book_deltas")
2346 .unwrap()
2347 .extract()
2348 .unwrap();
2349 let queried: Vec<OrderBookDelta> = catalog
2350 .call_method1("query", (&data_type,))
2351 .unwrap()
2352 .extract()
2353 .unwrap();
2354 let intervals: Vec<(u64, u64)> = catalog
2355 .call_method1("get_intervals", (&data_type, instrument_id.to_string()))
2356 .unwrap()
2357 .extract()
2358 .unwrap();
2359
2360 assert!(catalog.borrow().inner.file_exists(&path).unwrap());
2361 assert_eq!(loaded, deltas);
2362 assert_eq!(queried, deltas);
2363 assert_eq!(intervals, vec![(11, 53)]);
2364
2365 catalog
2366 .call_method1("consolidate_data", (&data_type, instrument_id.to_string()))
2367 .unwrap();
2368 catalog
2369 .call_method1(
2370 "delete_data_range",
2371 (&data_type, instrument_id.to_string(), 29, 29),
2372 )
2373 .unwrap();
2374 let remaining: Vec<OrderBookDelta> = catalog
2375 .call_method0("query_order_book_deltas")
2376 .unwrap()
2377 .extract()
2378 .unwrap();
2379
2380 assert_eq!(remaining, vec![deltas[1]]);
2381 });
2382 }
2383
2384 #[rstest]
2385 fn test_catalog_python_delta_write_errors_and_empty_input() {
2386 Python::initialize();
2387 let directory = TempDir::new().unwrap();
2388
2389 Python::attach(|py| {
2390 let catalog = PyParquetDataCatalog::py_new(
2391 directory.path().to_str().unwrap(),
2392 None,
2393 None,
2394 None,
2395 None,
2396 )
2397 .unwrap();
2398 let catalog = Bound::new(py, catalog).unwrap();
2399 let path: String = catalog
2400 .call_method1("write_order_book_deltas", (Vec::<OrderBookDelta>::new(),))
2401 .unwrap()
2402 .extract()
2403 .unwrap();
2404 let delta = OrderBookDelta::clear(
2405 InstrumentId::from("ETH/USDT.BINANCE"),
2406 17,
2407 23.into(),
2408 29.into(),
2409 );
2410
2411 let earlier = OrderBookDelta {
2412 ts_init: 19.into(),
2413 ..delta
2414 };
2415
2416 let error = catalog
2417 .call_method1("write_order_book_deltas", (vec![delta, earlier],))
2418 .unwrap_err();
2419 let loaded: Vec<OrderBookDelta> = catalog
2420 .call_method0("query_order_book_deltas")
2421 .unwrap()
2422 .extract()
2423 .unwrap();
2424
2425 assert_eq!(path, "");
2426 assert!(error.is_instance_of::<PyIOError>(py));
2427 assert_eq!(
2428 error.value(py).str().unwrap().to_str().unwrap(),
2429 "Failed to write order book deltas: nautilus_model_data_delta_order_book_delta timestamps must be in ascending order"
2430 );
2431 assert_eq!(loaded, Vec::<OrderBookDelta>::new());
2432 });
2433 }
2434
2435 #[rstest]
2436 fn test_concurrent_delta_writes_reject_overlapping_intervals() {
2437 Python::initialize();
2438 let directory = TempDir::new().unwrap();
2439 let mut catalog = PyParquetDataCatalog::py_new(
2440 directory.path().to_str().unwrap(),
2441 None,
2442 None,
2443 None,
2444 None,
2445 )
2446 .unwrap();
2447 let instrument_id = InstrumentId::from("ETH/USDT.BINANCE");
2448 let first = vec![
2449 OrderBookDelta::clear(instrument_id, 11, 17.into(), 23.into()),
2450 OrderBookDelta::clear(instrument_id, 29, 31.into(), 37.into()),
2451 ];
2452 let second = vec![
2453 OrderBookDelta::clear(instrument_id, 41, 43.into(), 37.into()),
2454 OrderBookDelta::clear(instrument_id, 47, 53.into(), 59.into()),
2455 ];
2456 let start = Barrier::new(2);
2457
2458 let (first_result, second_result) = thread::scope(|scope| {
2459 let first_thread = scope.spawn(|| {
2460 start.wait();
2461 Python::attach(|py| {
2462 catalog.write_order_book_deltas(py, first.clone(), None, None, false)
2463 })
2464 });
2465
2466 let second_thread = scope.spawn(|| {
2467 start.wait();
2468 Python::attach(|py| {
2469 catalog.write_order_book_deltas(py, second.clone(), None, None, false)
2470 })
2471 });
2472
2473 (first_thread.join().unwrap(), second_thread.join().unwrap())
2474 });
2475
2476 let (expected, error) = match (first_result, second_result) {
2477 (Ok(_), Err(e)) => (first, e),
2478 (Err(e), Ok(_)) => (second, e),
2479 results => {
2480 panic!("Expected one successful write and one overlap error, received {results:?}")
2481 }
2482 };
2483
2484 let loaded = catalog
2485 .inner
2486 .query::<OrderBookDelta>(None, None, None, None, None, true)
2487 .unwrap();
2488
2489 Python::attach(|py| {
2490 assert!(error.is_instance_of::<PyIOError>(py));
2491 assert!(
2492 error
2493 .value(py)
2494 .str()
2495 .unwrap()
2496 .to_str()
2497 .unwrap()
2498 .contains("would create non-disjoint intervals")
2499 );
2500 });
2501
2502 assert_eq!(loaded, expected);
2503 }
2504
2505 #[rstest]
2506 fn test_poisoned_mutation_lock_returns_python_io_error() {
2507 Python::initialize();
2508 let directory = TempDir::new().unwrap();
2509 let catalog = PyParquetDataCatalog::py_new(
2510 directory.path().to_str().unwrap(),
2511 None,
2512 None,
2513 None,
2514 None,
2515 )
2516 .unwrap();
2517
2518 let poisoned = thread::scope(|scope| {
2519 scope
2520 .spawn(|| {
2521 let _guard = catalog.mutation_lock.lock().unwrap();
2522 panic!("native mutation failed");
2523 })
2524 .join()
2525 .is_err()
2526 });
2527
2528 let error = Python::attach(|py| {
2529 catalog
2530 .write_order_book_deltas(py, Vec::new(), None, None, false)
2531 .unwrap_err()
2532 });
2533
2534 assert!(poisoned);
2535 Python::attach(|py| {
2536 assert!(error.is_instance_of::<PyIOError>(py));
2537 assert_eq!(
2538 error.value(py).str().unwrap().to_str().unwrap(),
2539 "Failed to write order book deltas: catalog mutation lock is poisoned"
2540 );
2541 });
2542 }
2543
2544 #[rstest]
2545 fn test_mutation_wait_releases_gil() {
2546 Python::initialize();
2547 let directory = TempDir::new().unwrap();
2548 let catalog = PyParquetDataCatalog::py_new(
2549 directory.path().to_str().unwrap(),
2550 None,
2551 None,
2552 None,
2553 None,
2554 )
2555 .unwrap();
2556 let delta = OrderBookDelta::clear(
2557 InstrumentId::from("ETH/USDT.BINANCE"),
2558 17,
2559 23.into(),
2560 29.into(),
2561 );
2562 let (started_tx, started_rx) = mpsc::channel();
2563 let (available_tx, available_rx) = mpsc::channel();
2564 let guard = catalog.mutation_lock.lock().unwrap();
2565
2566 let (gil_available, path) = thread::scope(|scope| {
2567 let writer = scope.spawn(|| {
2568 Python::attach(|py| {
2569 started_tx.send(()).unwrap();
2570 catalog.write_order_book_deltas(py, vec![delta], None, None, false)
2571 })
2572 });
2573
2574 started_rx.recv().unwrap();
2575
2576 let probe = scope.spawn(|| Python::attach(|_| available_tx.send(()).unwrap()));
2577 let gil_available = available_rx.recv_timeout(Duration::from_secs(5)).is_ok();
2578 drop(guard);
2579 let path = writer.join().unwrap().unwrap();
2580 probe.join().unwrap();
2581 (gil_available, path)
2582 });
2583
2584 assert!(
2585 gil_available,
2586 "GIL remained held while waiting for the mutation lock"
2587 );
2588 assert!(catalog.inner.file_exists(&path).unwrap());
2589 }
2590
2591 #[rstest]
2592 #[case::delete_data("delete_data_range")]
2593 #[case::delete_catalog("delete_catalog_range")]
2594 #[case::consolidate_data("consolidate_data_by_period")]
2595 #[case::consolidate_catalog("consolidate_catalog_by_period")]
2596 fn test_custom_maintenance_keeps_callbacks_attached(#[case] method: &str) {
2597 Python::initialize();
2598 register_gil_checked_custom_data();
2599 let directory = TempDir::new().unwrap();
2600
2601 let expected = [11u64, 29, 47]
2602 .into_iter()
2603 .map(|ts| {
2604 let value = GilCheckedCustomData {
2605 value: ts + 100,
2606 ts_event: (ts - 3).into(),
2607 ts_init: ts.into(),
2608 };
2609
2610 let data_type = DataType::new(GilCheckedCustomData::type_name_static(), None, None);
2611 CustomData::new(Arc::new(value), data_type)
2612 })
2613 .collect::<Vec<_>>();
2614
2615 Python::attach(|py| {
2616 let catalog = PyParquetDataCatalog::py_new(
2617 directory.path().to_str().unwrap(),
2618 None,
2619 None,
2620 None,
2621 None,
2622 )
2623 .unwrap();
2624 catalog
2625 .inner
2626 .write_custom_data_batch(&expected, None, None, None)
2627 .unwrap();
2628 let catalog = Bound::new(py, catalog).unwrap();
2629
2630 let data_type = PyNautilusDataType::new(NautilusDataType::Custom {
2631 type_name: GilCheckedCustomData::type_name_static().to_string(),
2632 });
2633
2634 let data_type = Bound::new(py, data_type).unwrap();
2635 CALLBACKS.with(|calls| calls.set((false, false)));
2636
2637 match method {
2638 "delete_data_range" => {
2639 catalog.call_method1(method, (&data_type, None::<String>, 29, 29))
2640 }
2641 "delete_catalog_range" => catalog.call_method1(method, (29, 29)),
2642 "consolidate_data_by_period" => {
2643 catalog.call_method1(method, (&data_type, None::<String>, 20))
2644 }
2645 "consolidate_catalog_by_period" => catalog.call_method1(method, (20,)),
2646 _ => unreachable!(),
2647 }
2648 .unwrap();
2649
2650 let callbacks = CALLBACKS.with(Cell::get);
2651 let loaded = catalog
2652 .borrow_mut()
2653 .inner
2654 .query_custom_data_dynamic(
2655 GilCheckedCustomData::type_name_static(),
2656 None,
2657 None,
2658 None,
2659 None,
2660 None,
2661 false,
2662 )
2663 .unwrap();
2664 let expected = expected
2665 .iter()
2666 .filter(|value| !method.starts_with("delete") || value.ts_init().as_u64() != 29)
2667 .cloned()
2668 .map(Data::Custom)
2669 .collect::<Vec<_>>();
2670
2671 assert_eq!(callbacks, (true, true));
2672 assert_eq!(loaded, expected);
2673 });
2674 }
2675
2676 #[rstest]
2677 #[case::empty(true)]
2678 #[case::nonempty(false)]
2679 fn test_custom_write_prepares_attached_before_native_mutation(#[case] empty: bool) {
2680 Python::initialize();
2681 register_gil_checked_custom_data();
2682 let directory = TempDir::new().unwrap();
2683
2684 Python::attach(|py| {
2685 let catalog = PyParquetDataCatalog::py_new(
2686 directory.path().to_str().unwrap(),
2687 None,
2688 None,
2689 None,
2690 None,
2691 )
2692 .unwrap();
2693 let catalog = Bound::new(py, catalog).unwrap();
2694
2695 let rows = if empty {
2696 Vec::new()
2697 } else {
2698 let value = GilCheckedCustomData {
2699 value: 113,
2700 ts_event: 17.into(),
2701 ts_init: 23.into(),
2702 };
2703
2704 let data_type = DataType::new(GilCheckedCustomData::type_name_static(), None, None);
2705 vec![CustomData::new(Arc::new(value), data_type)]
2706 };
2707
2708 let objects = rows
2709 .iter()
2710 .cloned()
2711 .map(|row| Py::new(py, row).unwrap())
2712 .collect::<Vec<_>>();
2713 CALLBACKS.with(|calls| calls.set((false, false)));
2714 let path: String = catalog
2715 .call_method1("write_custom_data", (objects,))
2716 .unwrap()
2717 .extract()
2718 .unwrap();
2719 let callbacks = CALLBACKS.with(Cell::get);
2720 let loaded = catalog
2721 .borrow_mut()
2722 .inner
2723 .query_custom_data_dynamic(
2724 GilCheckedCustomData::type_name_static(),
2725 None,
2726 None,
2727 None,
2728 None,
2729 None,
2730 false,
2731 )
2732 .unwrap();
2733
2734 assert_eq!(callbacks, (!empty, false));
2735 assert_eq!(
2736 loaded,
2737 rows.into_iter().map(Data::Custom).collect::<Vec<_>>()
2738 );
2739
2740 if empty {
2741 assert_eq!(path, "");
2742 } else {
2743 assert!(catalog.borrow().inner.file_exists(&path).unwrap());
2744 }
2745 });
2746 }
2747
2748 #[arrow_custom_data]
2749 #[custom_data]
2750 struct GilCheckedCustomData {
2751 value: u64,
2752 ts_event: UnixNanos,
2753 ts_init: UnixNanos,
2754 }
2755
2756 thread_local! {
2757 static CALLBACKS: Cell<(bool, bool)> = const { Cell::new((false, false)) };
2758 }
2759
2760 fn register_gil_checked_custom_data() {
2761 static REGISTER: Once = Once::new();
2762 REGISTER.call_once(|| {
2763 ensure_custom_data_json_registered::<GilCheckedCustomData>().unwrap();
2764 register_arrow(
2765 GilCheckedCustomData::type_name_static(),
2766 Arc::new(GilCheckedCustomData::get_schema(None)),
2767 Box::new(|items| {
2768 assert_gil_held();
2769 CALLBACKS.with(|calls| calls.set((true, calls.get().1)));
2770 Python::attach(|_| {
2771 let rows = items
2772 .iter()
2773 .map(|item| {
2774 item.as_any()
2775 .downcast_ref::<GilCheckedCustomData>()
2776 .unwrap()
2777 .clone()
2778 })
2779 .collect::<Vec<_>>();
2780
2781 let metadata = rows[0].metadata();
2782 GilCheckedCustomData::encode_batch(&metadata, &rows)
2783 .map_err(anyhow::Error::from)
2784 })
2785 }),
2786 Box::new(|metadata, batch| {
2787 assert_gil_held();
2788 CALLBACKS.with(|calls| calls.set((calls.get().0, true)));
2789 Python::attach(|_| GilCheckedCustomData::decode_data_batch(metadata, batch))
2790 .map_err(anyhow::Error::from)
2791 }),
2792 )
2793 .unwrap();
2794 });
2795 }
2796
2797 #[derive(Clone, Copy, Debug)]
2798 struct GilCheckedDelta<'a>(OrderBookDelta, Option<&'a Mutex<()>>);
2799
2800 impl HasTsInit for GilCheckedDelta<'_> {
2801 fn ts_init(&self) -> UnixNanos {
2802 self.0.ts_init
2803 }
2804 }
2805
2806 impl HasCatalogDataType for GilCheckedDelta<'_> {
2807 fn catalog_data_type() -> NautilusDataType {
2808 NautilusDataType::OrderBookDelta
2809 }
2810 }
2811
2812 impl ArrowSchemaProvider for GilCheckedDelta<'_> {
2813 fn get_schema(metadata: Option<HashMap<String, String>>) -> Schema {
2814 OrderBookDelta::get_schema(metadata)
2815 }
2816 }
2817
2818 impl EncodeToRecordBatch for GilCheckedDelta<'_> {
2819 fn encode_batch<T>(
2820 metadata: &HashMap<String, String>,
2821 data: &[T],
2822 ) -> Result<RecordBatch, ArrowError>
2823 where
2824 T: Borrow<Self>,
2825 {
2826 assert_gil_released();
2827 let deltas = data.iter().map(|item| item.borrow().0).collect::<Vec<_>>();
2828 OrderBookDelta::encode_batch(metadata, &deltas)
2829 }
2830
2831 fn metadata(&self) -> HashMap<String, String> {
2832 if let Some(lock) = self.1 {
2833 assert!(matches!(lock.try_lock(), Err(TryLockError::WouldBlock)));
2834 }
2835
2836 self.0.metadata()
2837 }
2838 }
2839
2840 impl DecodeTypedFromRecordBatch for GilCheckedDelta<'_> {
2841 fn decode_typed_batch(
2842 metadata: &HashMap<String, String>,
2843 record_batch: RecordBatch,
2844 ) -> Result<Vec<Self>, EncodingError> {
2845 assert_gil_released();
2846 OrderBookDelta::decode_typed_batch(metadata, record_batch)
2847 .map(|deltas| deltas.into_iter().map(|delta| Self(delta, None)).collect())
2848 }
2849 }
2850
2851 fn assert_gil_released() {
2852 assert_eq!(unsafe { pyo3::ffi::PyGILState_Check() }, 0);
2854 }
2855
2856 fn assert_gil_held() {
2857 assert_eq!(unsafe { pyo3::ffi::PyGILState_Check() }, 1);
2859 }
2860}