Skip to main content

nautilus_persistence/python/backend/parquet/
catalog.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16#![expect(
17    clippy::too_many_arguments,
18    reason = "PyO3 catalog wrapper mirrors Python API dispatch surface"
19)]
20
21use std::{collections::HashMap, sync::Mutex};
22
23use nautilus_common::{
24    config::ConfigError, enums::Environment, python::config_error_to_pyvalue_err,
25};
26use nautilus_core::{
27    UnixNanos,
28    python::{to_pytype_err, to_pyvalue_err},
29};
30use nautilus_model::{
31    data::{
32        Bar, Data, FundingRateUpdate, HasTsInit, IndexPriceUpdate, InstrumentStatus,
33        MarkPriceUpdate, NautilusDataType, OptionGreeks, OrderBookDelta, OrderBookDepth, QuoteTick,
34        TradeTick, close::InstrumentClose,
35    },
36    python::{
37        data::data_to_pyobject,
38        instruments::{
39            PyNautilusInstrumentType, instrument_any_to_pyobject, pyobject_to_instrument_any,
40        },
41    },
42};
43use nautilus_serialization::{
44    arrow::{
45        DecodeTypedFromRecordBatch, EncodeToRecordBatch, display::instrument::encode_instruments,
46    },
47    python::arrow::{arrow_record_batches_to_pyarrow_stream, arrow_record_batches_to_pybytes},
48};
49use pyo3::{
50    exceptions::PyIOError,
51    prelude::*,
52    types::{PyBytes, PyDict, PyList},
53};
54
55use crate::{
56    backend::{migration::build_catalog_migration_plan, parquet::catalog::ParquetDataCatalog},
57    catalog::{
58        traits::{CatalogQuery, CatalogReader, CatalogRecordQuery, CatalogWriter},
59        types::{CatalogDataType, HasCatalogDataType, parquet_catalog_data_type_path_prefixes},
60    },
61    config::CatalogCompression,
62    python::backend::{
63        PyCatalogDataType, arrow_ipc_batches, arrow_ipc_data_schema, arrow_ipc_record_schema,
64        arrow_record_batches_from_pybytes, catalog_metadata_to_pydict, catalog_record_type_from_py,
65        nautilus_data_type_from_py, to_pyio_err, write_record_params_from_py,
66    },
67};
68
69fn write_parquet_data<T>(
70    py: Python<'_>,
71    catalog: &PyParquetDataCatalog,
72    data: Vec<T>,
73    start: Option<u64>,
74    end: Option<u64>,
75    skip_disjoint_check: bool,
76    label: &str,
77) -> PyResult<String>
78where
79    T: HasTsInit + EncodeToRecordBatch + HasCatalogDataType + Send,
80{
81    catalog
82        .mutate(py, move || {
83            catalog.inner.write_to_parquet(
84                &data,
85                start.map(UnixNanos::from),
86                end.map(UnixNanos::from),
87                Some(skip_disjoint_check),
88            )
89        })
90        .map(|path| path.to_string_lossy().to_string())
91        .map_err(|e| PyIOError::new_err(format!("Failed to write {label}: {e}")))
92}
93
94fn query_parquet_data<T>(
95    py: Python<'_>,
96    catalog: &mut ParquetDataCatalog,
97    identifiers: Option<Vec<String>>,
98    start: Option<u64>,
99    end: Option<u64>,
100    where_clause: Option<&str>,
101    files: Option<Vec<String>>,
102    optimize_file_loading: bool,
103    error_context: &str,
104) -> PyResult<Vec<T>>
105where
106    T: DecodeTypedFromRecordBatch + HasCatalogDataType + HasTsInit + Send,
107{
108    py.detach(move || {
109        catalog.query::<T>(
110            identifiers,
111            start.map(UnixNanos::from),
112            end.map(UnixNanos::from),
113            where_clause,
114            files,
115            optimize_file_loading,
116        )
117    })
118    .map_err(|e| PyIOError::new_err(format!("{error_context}: {e}")))
119}
120
121fn reject_parquet_as_of(as_of: Option<&Bound<'_, PyAny>>) -> PyResult<()> {
122    if as_of.is_some() {
123        return Err(to_pyvalue_err("ParquetDataCatalog does not support as_of"));
124    }
125
126    Ok(())
127}
128
129/// A catalog for writing data to Parquet files.
130#[pyclass(name = "ParquetDataCatalog", module = "nautilus_trader.persistence")]
131#[pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.persistence")]
132pub struct PyParquetDataCatalog {
133    inner: ParquetDataCatalog,
134    mutation_lock: Mutex<()>,
135}
136
137#[pymethods]
138#[pyo3_stub_gen::derive::gen_stub_pymethods]
139impl PyParquetDataCatalog {
140    /// Create a new `ParquetCatalog` with the given base path and optional parameters.
141    ///
142    /// # Parameters
143    ///
144    /// - `base_path`: The base path for the catalog
145    /// - `storage_options`: Optional storage configuration for cloud backends
146    /// - `batch_size`: Optional batch size for processing (default: 10,000)
147    /// - `compression`: Optional compression code (0=UNCOMPRESSED, 1=SNAPPY, 2=GZIP, 4=BROTLI, 5=`LZ4_RAW`, 6=ZSTD)
148    /// - `max_row_group_size`: Optional maximum row group size (default: 131,072)
149    ///
150    /// # Errors
151    ///
152    /// Returns an error if `compression` is not one of the listed codes, `batch_size` or
153    /// `max_row_group_size` is zero, or the underlying [`ParquetDataCatalog`] cannot be created.
154    #[new]
155    #[pyo3(signature = (base_path, storage_options=None, batch_size=None, compression=None, max_row_group_size=None))]
156    pub fn py_new(
157        base_path: &str,
158        storage_options: Option<HashMap<String, String>>,
159        batch_size: Option<usize>,
160        compression: Option<u8>,
161        max_row_group_size: Option<usize>,
162    ) -> PyResult<Self> {
163        let compression = compression
164            .map(catalog_compression_from_code)
165            .transpose()?
166            .map(parquet::basic::Compression::from);
167
168        // Convert HashMap to AHashMap for internal use
169        let storage_options = storage_options.map(|m| m.into_iter().collect());
170
171        let inner = ParquetDataCatalog::from_uri(
172            base_path,
173            storage_options,
174            batch_size,
175            compression,
176            max_row_group_size,
177        )
178        .map_err(|e| match e.downcast::<ConfigError>() {
179            Ok(e) => config_error_to_pyvalue_err(e),
180            Err(e) => PyIOError::new_err(format!("Failed to create ParquetDataCatalog: {e}")),
181        })?;
182
183        Ok(Self {
184            inner,
185            mutation_lock: Mutex::new(()),
186        })
187    }
188
189    /// Rewrites a legacy Parquet catalog into this current Parquet catalog.
190    #[pyo3(signature = (parquet_path, storage_options=None, dry_run=false))]
191    pub fn migrate_from_legacy_parquet_path(
192        mut slf: PyRefMut<'_, Self>,
193        parquet_path: &str,
194        storage_options: Option<HashMap<String, String>>,
195        dry_run: bool,
196    ) -> PyResult<usize> {
197        let storage_options = storage_options.map(|m| m.into_iter().collect());
198        let source = ParquetDataCatalog::from_uri(parquet_path, storage_options, None, None, None)
199            .map_err(to_pyio_err)?;
200        let py = slf.py();
201        if dry_run {
202            return py
203                .detach(|| {
204                    let plan = build_catalog_migration_plan(&source)?;
205                    plan.ensure_ready()?;
206                    Ok::<usize, anyhow::Error>(0)
207                })
208                .map_err(to_pyio_err);
209        }
210
211        let inner = &mut slf.inner;
212        py.detach(|| {
213            inner
214                .migrate_from_legacy_parquet_catalog(&source)
215                .map(|report| report.migrated_rows)
216        })
217        .map_err(to_pyio_err)
218    }
219
220    /// Write order book delta data to Parquet files.
221    ///
222    /// # Parameters
223    ///
224    /// - `data`: Vector of order book deltas to write
225    /// - `start`: Optional start timestamp override (nanoseconds since Unix epoch)
226    /// - `end`: Optional end timestamp override (nanoseconds since Unix epoch)
227    ///
228    /// # Returns
229    ///
230    /// Returns the path of the created file as a string.
231    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
232    pub fn write_order_book_deltas(
233        &self,
234        py: Python<'_>,
235        data: Vec<OrderBookDelta>,
236        start: Option<u64>,
237        end: Option<u64>,
238        skip_disjoint_check: bool,
239    ) -> PyResult<String> {
240        write_parquet_data(
241            py,
242            self,
243            data,
244            start,
245            end,
246            skip_disjoint_check,
247            "order book deltas",
248        )
249    }
250
251    /// Write order book depth data to Parquet files.
252    ///
253    /// # Parameters
254    ///
255    /// - `data`: Vector of order book depths to write
256    /// - `start`: Optional start timestamp override (nanoseconds since Unix epoch)
257    /// - `end`: Optional end timestamp override (nanoseconds since Unix epoch)
258    ///
259    /// # Returns
260    ///
261    /// Returns the path of the created file as a string.
262    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
263    pub fn write_order_book_depths(
264        &self,
265        py: Python<'_>,
266        data: Vec<OrderBookDepth>,
267        start: Option<u64>,
268        end: Option<u64>,
269        skip_disjoint_check: bool,
270    ) -> PyResult<String> {
271        write_parquet_data(
272            py,
273            self,
274            data,
275            start,
276            end,
277            skip_disjoint_check,
278            "order book depths",
279        )
280    }
281
282    /// Write quote tick data to Parquet files.
283    ///
284    /// # Parameters
285    ///
286    /// - `data`: Vector of quote ticks to write
287    /// - `start`: Optional start timestamp override (nanoseconds since Unix epoch)
288    /// - `end`: Optional end timestamp override (nanoseconds since Unix epoch)
289    ///
290    /// # Returns
291    ///
292    /// Returns the path of the created file as a string.
293    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
294    pub fn write_quote_ticks(
295        &self,
296        py: Python<'_>,
297        data: Vec<QuoteTick>,
298        start: Option<u64>,
299        end: Option<u64>,
300        skip_disjoint_check: bool,
301    ) -> PyResult<String> {
302        write_parquet_data(
303            py,
304            self,
305            data,
306            start,
307            end,
308            skip_disjoint_check,
309            "quote ticks",
310        )
311    }
312
313    /// Write trade tick data to Parquet files.
314    ///
315    /// # Parameters
316    ///
317    /// - `data`: Vector of trade ticks to write
318    /// - `start`: Optional start timestamp override (nanoseconds since Unix epoch)
319    /// - `end`: Optional end timestamp override (nanoseconds since Unix epoch)
320    ///
321    /// # Returns
322    ///
323    /// Returns the path of the created file as a string.
324    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
325    pub fn write_trade_ticks(
326        &self,
327        py: Python<'_>,
328        data: Vec<TradeTick>,
329        start: Option<u64>,
330        end: Option<u64>,
331        skip_disjoint_check: bool,
332    ) -> PyResult<String> {
333        write_parquet_data(
334            py,
335            self,
336            data,
337            start,
338            end,
339            skip_disjoint_check,
340            "trade ticks",
341        )
342    }
343
344    /// Write bar data to Parquet files.
345    ///
346    /// # Parameters
347    ///
348    /// - `data`: Vector of bars to write
349    /// - `start`: Optional start timestamp override (nanoseconds since Unix epoch)
350    /// - `end`: Optional end timestamp override (nanoseconds since Unix epoch)
351    ///
352    /// # Returns
353    ///
354    /// Returns the path of the created file as a string.
355    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
356    pub fn write_bars(
357        &self,
358        py: Python<'_>,
359        data: Vec<Bar>,
360        start: Option<u64>,
361        end: Option<u64>,
362        skip_disjoint_check: bool,
363    ) -> PyResult<String> {
364        write_parquet_data(py, self, data, start, end, skip_disjoint_check, "bars")
365    }
366
367    /// Write mark price update data to Parquet files.
368    ///
369    /// # Parameters
370    ///
371    /// - `data`: Vector of mark price updates to write
372    /// - `start`: Optional start timestamp override (nanoseconds since Unix epoch)
373    /// - `end`: Optional end timestamp override (nanoseconds since Unix epoch)
374    ///
375    /// # Returns
376    ///
377    /// Returns the path of the created file as a string.
378    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
379    pub fn write_mark_price_updates(
380        &self,
381        py: Python<'_>,
382        data: Vec<MarkPriceUpdate>,
383        start: Option<u64>,
384        end: Option<u64>,
385        skip_disjoint_check: bool,
386    ) -> PyResult<String> {
387        write_parquet_data(
388            py,
389            self,
390            data,
391            start,
392            end,
393            skip_disjoint_check,
394            "mark price updates",
395        )
396    }
397
398    /// Write index price update data to Parquet files.
399    ///
400    /// # Parameters
401    ///
402    /// - `data`: Vector of index price updates to write
403    /// - `start`: Optional start timestamp override (nanoseconds since Unix epoch)
404    /// - `end`: Optional end timestamp override (nanoseconds since Unix epoch)
405    ///
406    /// # Returns
407    ///
408    /// Returns the path of the created file as a string.
409    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
410    pub fn write_index_price_updates(
411        &self,
412        py: Python<'_>,
413        data: Vec<IndexPriceUpdate>,
414        start: Option<u64>,
415        end: Option<u64>,
416        skip_disjoint_check: bool,
417    ) -> PyResult<String> {
418        write_parquet_data(
419            py,
420            self,
421            data,
422            start,
423            end,
424            skip_disjoint_check,
425            "index price updates",
426        )
427    }
428
429    /// Write funding rate update data to Parquet files.
430    ///
431    /// # Parameters
432    ///
433    /// - `data`: Vector of funding rate updates to write
434    /// - `start`: Optional start timestamp override (nanoseconds since Unix epoch)
435    /// - `end`: Optional end timestamp override (nanoseconds since Unix epoch)
436    ///
437    /// # Returns
438    ///
439    /// Returns the path of the created file as a string.
440    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
441    pub fn write_funding_rate_updates(
442        &self,
443        py: Python<'_>,
444        data: Vec<FundingRateUpdate>,
445        start: Option<u64>,
446        end: Option<u64>,
447        skip_disjoint_check: bool,
448    ) -> PyResult<String> {
449        write_parquet_data(
450            py,
451            self,
452            data,
453            start,
454            end,
455            skip_disjoint_check,
456            "funding rate updates",
457        )
458    }
459
460    /// Write option greeks data to Parquet files.
461    ///
462    /// # Parameters
463    ///
464    /// - `data`: Vector of option greeks to write
465    /// - `start`: Optional start timestamp override (nanoseconds since Unix epoch)
466    /// - `end`: Optional end timestamp override (nanoseconds since Unix epoch)
467    ///
468    /// # Returns
469    ///
470    /// Returns the path of the created file as a string.
471    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
472    pub fn write_option_greeks(
473        &self,
474        py: Python<'_>,
475        data: Vec<OptionGreeks>,
476        start: Option<u64>,
477        end: Option<u64>,
478        skip_disjoint_check: bool,
479    ) -> PyResult<String> {
480        write_parquet_data(
481            py,
482            self,
483            data,
484            start,
485            end,
486            skip_disjoint_check,
487            "option greeks",
488        )
489    }
490
491    /// Write instrument status data to Parquet files.
492    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
493    pub fn write_instrument_statuses(
494        &self,
495        py: Python<'_>,
496        data: Vec<InstrumentStatus>,
497        start: Option<u64>,
498        end: Option<u64>,
499        skip_disjoint_check: bool,
500    ) -> PyResult<String> {
501        write_parquet_data(
502            py,
503            self,
504            data,
505            start,
506            end,
507            skip_disjoint_check,
508            "instrument statuses",
509        )
510    }
511
512    /// Write instrument close data to Parquet files.
513    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
514    pub fn write_instrument_closes(
515        &self,
516        py: Python<'_>,
517        data: Vec<InstrumentClose>,
518        start: Option<u64>,
519        end: Option<u64>,
520        skip_disjoint_check: bool,
521    ) -> PyResult<String> {
522        write_parquet_data(
523            py,
524            self,
525            data,
526            start,
527            end,
528            skip_disjoint_check,
529            "instrument closes",
530        )
531    }
532
533    /// Write instruments to Parquet files in the catalog.
534    ///
535    /// Instruments are stored under `data/instruments/{instrument_id}/` using timestamp-ranged
536    /// parquet file names, allowing multiple historical versions of the same instrument to be
537    /// written across separate calls.
538    ///
539    /// # Parameters
540    ///
541    /// - `instruments`: A Python list of instrument objects (e.g. `CurrencyPair`, Equity).
542    ///
543    /// # Returns
544    ///
545    /// Returns a list of written file paths.
546    pub fn write_instruments(&self, instruments: &Bound<'_, PyAny>) -> PyResult<Vec<String>> {
547        let data = instruments;
548        let py = data.py();
549        let list = data.cast::<PyList>()?;
550        let mut instruments = Vec::with_capacity(list.len());
551        for item in list.iter() {
552            let py_item: Py<PyAny> = item.unbind();
553            let instrument = pyobject_to_instrument_any(py, py_item)?;
554            instruments.push(instrument);
555        }
556
557        self.mutate(py, || self.inner.write_instruments(instruments))
558            .map(|paths| {
559                paths
560                    .into_iter()
561                    .map(|p| p.to_string_lossy().to_string())
562                    .collect()
563            })
564            .map_err(|e| PyIOError::new_err(format!("Failed to write instruments: {e}")))
565    }
566
567    /// Query instruments from the catalog.
568    ///
569    /// # Parameters
570    ///
571    /// - `instrument_ids`: Optional list of instrument IDs to filter by. If `None`, returns all instruments.
572    /// - `start`: Optional inclusive lower bound for `ts_init` filtering.
573    /// - `end`: Optional inclusive upper bound for `ts_init` filtering.
574    /// - `where_clause`: Optional SQL WHERE clause for additional filtering.
575    ///
576    /// # Returns
577    ///
578    /// Returns a list of instrument objects (e.g. `CurrencyPair`, Equity).
579    #[pyo3(signature = (
580        instrument_ids=None,
581        start=None,
582        end=None,
583        where_clause=None,
584        instrument_type=None,
585    ))]
586    #[expect(clippy::needless_pass_by_value)]
587    pub fn instruments(
588        &mut self,
589        py: Python<'_>,
590        instrument_ids: Option<Vec<String>>,
591        start: Option<u64>,
592        end: Option<u64>,
593        where_clause: Option<&str>,
594        instrument_type: Option<PyRef<'_, PyNautilusInstrumentType>>,
595    ) -> PyResult<Vec<Py<PyAny>>> {
596        let instrument_type = instrument_type.map(|instrument_type| instrument_type.inner());
597
598        let rust_instruments = py
599            .detach(|| {
600                self.inner.query_instruments_filtered_with_where_and_type(
601                    instrument_ids.as_deref(),
602                    start.map(UnixNanos::from),
603                    end.map(UnixNanos::from),
604                    where_clause,
605                    instrument_type.as_ref(),
606                )
607            })
608            .map_err(|e| PyIOError::new_err(format!("Failed to query instruments: {e}")))?;
609
610        rust_instruments
611            .into_iter()
612            .map(|inst| instrument_any_to_pyobject(py, inst))
613            .collect()
614    }
615
616    /// Query instruments as display-friendly Arrow IPC stream bytes.
617    #[pyo3(signature = (
618        instrument_ids=None,
619        start=None,
620        end=None,
621        where_clause=None,
622        instrument_type=None,
623    ))]
624    #[expect(clippy::needless_pass_by_value)]
625    pub fn query_instrument_arrow_bytes(
626        &mut self,
627        py: Python<'_>,
628        instrument_ids: Option<Vec<String>>,
629        start: Option<u64>,
630        end: Option<u64>,
631        where_clause: Option<&str>,
632        instrument_type: Option<PyRef<'_, PyNautilusInstrumentType>>,
633    ) -> PyResult<Py<PyBytes>> {
634        let instrument_type = instrument_type.map(|instrument_type| instrument_type.inner());
635
636        let instruments = py
637            .detach(|| {
638                self.inner.query_instruments_filtered_with_where_and_type(
639                    instrument_ids.as_deref(),
640                    start.map(UnixNanos::from),
641                    end.map(UnixNanos::from),
642                    where_clause,
643                    instrument_type.as_ref(),
644                )
645            })
646            .map_err(|e| PyIOError::new_err(format!("Failed query instruments: {e}")))?;
647
648        let batch = encode_instruments(&instruments)
649            .map_err(|e| PyIOError::new_err(format!("Failed encode instruments: {e}")))?;
650        let schema = batch.schema().as_ref().clone();
651        arrow_record_batches_to_pybytes(py, &schema, &[batch])
652    }
653
654    /// Query instruments as an Arrow C stream `PyCapsule`.
655    #[pyo3(signature = (
656        instrument_ids=None,
657        start=None,
658        end=None,
659        where_clause=None,
660        instrument_type=None,
661    ))]
662    #[expect(clippy::needless_pass_by_value)]
663    pub fn query_instrument_arrow_stream(
664        &mut self,
665        py: Python<'_>,
666        instrument_ids: Option<Vec<String>>,
667        start: Option<u64>,
668        end: Option<u64>,
669        where_clause: Option<&str>,
670        instrument_type: Option<PyRef<'_, PyNautilusInstrumentType>>,
671    ) -> PyResult<Py<PyAny>> {
672        let instrument_type = instrument_type.map(|instrument_type| instrument_type.inner());
673
674        let instruments = py
675            .detach(|| {
676                self.inner.query_instruments_filtered_with_where_and_type(
677                    instrument_ids.as_deref(),
678                    start.map(UnixNanos::from),
679                    end.map(UnixNanos::from),
680                    where_clause,
681                    instrument_type.as_ref(),
682                )
683            })
684            .map_err(|e| PyIOError::new_err(format!("Failed query instruments: {e}")))?;
685
686        let batch = encode_instruments(&instruments)
687            .map_err(|e| PyIOError::new_err(format!("Failed encode instruments: {e}")))?;
688        let schema = batch.schema().as_ref().clone();
689        arrow_record_batches_to_pyarrow_stream(py, &schema, vec![batch])
690    }
691
692    /// Extend file names in the catalog with additional timestamp information.
693    ///
694    /// # Parameters
695    ///
696    /// - `data_type`: The stored family to target (data type, record type, or instrument type).
697    /// - `identifier`: Optional instrument ID or bar type filter
698    /// - `start`: Start timestamp (nanoseconds since Unix epoch)
699    /// - `end`: End timestamp (nanoseconds since Unix epoch)
700    #[pyo3(signature = (data_type, identifier=None, *, start, end))]
701    #[expect(clippy::needless_pass_by_value)]
702    pub fn extend_file_name(
703        &self,
704        py: Python<'_>,
705        data_type: PyCatalogDataType,
706        identifier: Option<String>,
707        start: u64,
708        end: u64,
709    ) -> PyResult<()> {
710        let data_type = data_type.into_inner();
711        let start_nanos = UnixNanos::from(start);
712        let end_nanos = UnixNanos::from(end);
713
714        self.mutate(py, || {
715            self.inner
716                .extend_file_name(&data_type, identifier.as_deref(), start_nanos, end_nanos)
717        })
718        .map_err(|e| PyIOError::new_err(format!("Failed to extend file name: {e}")))
719    }
720
721    /// Consolidate all data files in the catalog within the specified time range.
722    ///
723    /// # Parameters
724    ///
725    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
726    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
727    /// - `ensure_contiguous_files`: Optional flag to ensure files are contiguous
728    /// - `deduplicate`: Optional flag to deduplicate rows when combining files
729    #[pyo3(signature = (start=None, end=None, ensure_contiguous_files=None, deduplicate=None))]
730    pub fn consolidate_catalog(
731        &self,
732        py: Python<'_>,
733        start: Option<u64>,
734        end: Option<u64>,
735        ensure_contiguous_files: Option<bool>,
736        deduplicate: Option<bool>,
737    ) -> PyResult<()> {
738        let start_nanos = start.map(UnixNanos::from);
739        let end_nanos = end.map(UnixNanos::from);
740
741        self.mutate(py, || {
742            self.inner.consolidate_catalog(
743                start_nanos,
744                end_nanos,
745                ensure_contiguous_files,
746                deduplicate,
747            )
748        })
749        .map_err(|e| PyIOError::new_err(format!("Failed to consolidate catalog: {e}")))
750    }
751
752    /// Consolidate data files for a specific data type within the specified time range.
753    ///
754    /// # Parameters
755    ///
756    /// - `data_type`: The stored family to target (data type, record type, or instrument type).
757    /// - `identifier`: Optional instrument ID or bar type filter
758    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
759    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
760    /// - `ensure_contiguous_files`: Optional flag to ensure files are contiguous
761    /// - `deduplicate`: Optional flag to deduplicate rows when combining files
762    #[pyo3(signature = (data_type, identifier=None, start=None, end=None, ensure_contiguous_files=None, deduplicate=None))]
763    #[expect(clippy::needless_pass_by_value)]
764    pub fn consolidate_data(
765        &mut self,
766        py: Python<'_>,
767        data_type: PyCatalogDataType,
768        identifier: Option<String>,
769        start: Option<u64>,
770        end: Option<u64>,
771        ensure_contiguous_files: Option<bool>,
772        deduplicate: Option<bool>,
773    ) -> PyResult<()> {
774        let data_type = data_type.into_inner();
775        let start_nanos = start.map(UnixNanos::from);
776        let end_nanos = end.map(UnixNanos::from);
777
778        py.detach(|| {
779            self.inner.consolidate_data(
780                &data_type,
781                identifier.as_deref(),
782                start_nanos,
783                end_nanos,
784                ensure_contiguous_files,
785                deduplicate,
786            )
787        })
788        .map_err(|e| PyIOError::new_err(format!("Failed to consolidate data: {e}")))
789    }
790
791    /// Consolidate all data files in the catalog by splitting them into fixed time periods.
792    ///
793    /// This method identifies all leaf directories in the catalog that contain parquet files
794    /// and consolidates them by period. A leaf directory is one that contains files but no subdirectories.
795    /// This is a convenience method that effectively calls `consolidate_data_by_period` for all data types
796    /// and instrument IDs in the catalog.
797    ///
798    /// # Parameters
799    ///
800    /// - `period_nanos`: Optional period duration for consolidation in nanoseconds. Default is 1 day (86400000000000).
801    ///   Examples: 3600000000000 (1 hour), 604800000000000 (7 days), 1800000000000 (30 minutes)
802    /// - `start`: Optional start timestamp for the consolidation range (nanoseconds since Unix epoch)
803    /// - `end`: Optional end timestamp for the consolidation range (nanoseconds since Unix epoch)
804    /// - `ensure_contiguous_files`: Optional flag to control file naming strategy
805    #[pyo3(signature = (period_nanos=None, start=None, end=None, ensure_contiguous_files=None))]
806    pub fn consolidate_catalog_by_period(
807        &mut self,
808        period_nanos: Option<u64>,
809        start: Option<u64>,
810        end: Option<u64>,
811        ensure_contiguous_files: Option<bool>,
812    ) -> PyResult<()> {
813        let start_nanos = start.map(UnixNanos::from);
814        let end_nanos = end.map(UnixNanos::from);
815
816        self.inner
817            .consolidate_catalog_by_period(
818                period_nanos,
819                start_nanos,
820                end_nanos,
821                ensure_contiguous_files,
822            )
823            .map_err(|e| {
824                PyIOError::new_err(format!("Failed to consolidate catalog by period: {e}"))
825            })
826    }
827
828    /// Consolidate data files by splitting them into fixed time periods.
829    ///
830    /// This method queries data by period and writes consolidated files immediately,
831    /// using efficient period-based consolidation logic. When start/end boundaries intersect existing files,
832    /// the function automatically splits those files to preserve all data.
833    ///
834    /// # Parameters
835    ///
836    /// - `data_type`: The stored family to target (data type, record type, or instrument type).
837    /// - `identifier`: Optional instrument ID to consolidate. If None, consolidates all instruments
838    /// - `period_nanos`: Optional period duration for consolidation in nanoseconds. Default is 1 day (86400000000000).
839    ///   Examples: 3600000000000 (1 hour), 604800000000000 (7 days), 1800000000000 (30 minutes)
840    /// - `start`: Optional start timestamp for consolidation range (nanoseconds since Unix epoch)
841    /// - `end`: Optional end timestamp for consolidation range (nanoseconds since Unix epoch)
842    /// - `ensure_contiguous_files`: Optional flag to control file naming strategy
843    #[pyo3(signature = (data_type, identifier=None, period_nanos=None, start=None, end=None, ensure_contiguous_files=None))]
844    #[expect(clippy::needless_pass_by_value)]
845    pub fn consolidate_data_by_period(
846        &mut self,
847        py: Python<'_>,
848        data_type: PyCatalogDataType,
849        identifier: Option<String>,
850        period_nanos: Option<u64>,
851        start: Option<u64>,
852        end: Option<u64>,
853        ensure_contiguous_files: Option<bool>,
854    ) -> PyResult<()> {
855        let data_type = data_type.into_inner();
856        let start_nanos = start.map(UnixNanos::from);
857        let end_nanos = end.map(UnixNanos::from);
858
859        let mut consolidate = || {
860            self.inner.consolidate_data_by_period(
861                &data_type,
862                identifier.as_deref(),
863                period_nanos,
864                start_nanos,
865                end_nanos,
866                ensure_contiguous_files,
867            )
868        };
869
870        let result = if matches!(
871            &data_type,
872            CatalogDataType::Data(NautilusDataType::Custom { .. })
873        ) {
874            consolidate()
875        } else {
876            py.detach(consolidate)
877        };
878
879        result.map_err(|e| PyIOError::new_err(format!("Failed to consolidate data by period: {e}")))
880    }
881
882    /// Reset all catalog file names to their canonical form.
883    pub fn reset_all_file_names(&self, py: Python<'_>) -> PyResult<()> {
884        self.mutate(py, || self.inner.reset_all_file_names())
885            .map_err(|e| PyIOError::new_err(format!("Failed to reset catalog file names: {e}")))
886    }
887
888    /// Reset data file names for a specific data class to their canonical form.
889    ///
890    /// # Parameters
891    ///
892    /// - `data_type`: The stored family to target (data type, record type, or instrument type).
893    /// - `identifier`: Optional instrument ID or bar type filter
894    #[pyo3(signature = (data_type, identifier=None))]
895    #[expect(clippy::needless_pass_by_value)]
896    pub fn reset_data_file_names(
897        &self,
898        py: Python<'_>,
899        data_type: PyCatalogDataType,
900        identifier: Option<String>,
901    ) -> PyResult<()> {
902        let data_type = data_type.into_inner();
903        self.mutate(py, || {
904            self.inner
905                .reset_data_file_names(&data_type, identifier.as_deref())
906        })
907        .map_err(|e| PyIOError::new_err(format!("Failed to reset data file names: {e}")))
908    }
909
910    /// Delete data within a specified time range across the entire catalog.
911    ///
912    /// This method identifies all leaf directories in the catalog that contain parquet files
913    /// and deletes data within the specified time range from each directory. A leaf directory
914    /// is one that contains files but no subdirectories. This is a convenience method that
915    /// effectively calls `delete_data_range` for all data types and instrument IDs in the catalog.
916    ///
917    /// # Parameters
918    ///
919    /// - `start`: Optional start timestamp for the deletion range (nanoseconds since Unix epoch)
920    /// - `end`: Optional end timestamp for the deletion range (nanoseconds since Unix epoch)
921    ///
922    /// # Notes
923    ///
924    /// - This operation permanently removes data and cannot be undone
925    /// - The deletion process handles file intersections intelligently by splitting files
926    ///   when they partially overlap with the deletion range
927    /// - Files completely within the deletion range are removed entirely
928    /// - Files partially overlapping the deletion range are split to preserve data outside the range
929    /// - This method is useful for bulk data cleanup operations across the entire catalog
930    /// - Empty directories are not automatically removed after deletion
931    #[pyo3(signature = (start=None, end=None))]
932    pub fn delete_catalog_range(&mut self, start: Option<u64>, end: Option<u64>) -> PyResult<()> {
933        let start_nanos = start.map(UnixNanos::from);
934        let end_nanos = end.map(UnixNanos::from);
935
936        self.inner
937            .delete_catalog_range(start_nanos, end_nanos)
938            .map_err(|e| PyIOError::new_err(format!("Failed to delete catalog range: {e}")))
939    }
940
941    /// Delete data within a specified time range for a specific data type and instrument.
942    ///
943    /// This method identifies all parquet files that intersect with the specified time range
944    /// and handles them appropriately:
945    /// - Files completely within the range are deleted
946    /// - Files partially overlapping the range are split to preserve data outside the range
947    /// - The original intersecting files are removed after processing
948    ///
949    /// # Parameters
950    ///
951    /// - `data_type`: The data type to delete from.
952    /// - `identifier`: Optional identifier to delete data for. If None, deletes data across all identifiers
953    /// - `start`: Optional start timestamp for the deletion range (nanoseconds since Unix epoch)
954    /// - `end`: Optional end timestamp for the deletion range (nanoseconds since Unix epoch)
955    ///
956    /// # Notes
957    ///
958    /// - This operation permanently removes data and cannot be undone
959    /// - Files that partially overlap the deletion range are split to preserve data outside the range
960    /// - The method ensures data integrity by using atomic operations where possible
961    /// - Empty directories are not automatically removed after deletion
962    #[pyo3(signature = (data_type, identifier=None, start=None, end=None))]
963    #[expect(clippy::needless_pass_by_value)]
964    pub fn delete_data_range(
965        &mut self,
966        py: Python<'_>,
967        #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
968            '_,
969            PyAny,
970        >,
971        identifier: Option<String>,
972        start: Option<u64>,
973        end: Option<u64>,
974    ) -> PyResult<()> {
975        let data_type = nautilus_data_type_from_py(data_type)?;
976        let start_nanos = start.map(UnixNanos::from);
977        let end_nanos = end.map(UnixNanos::from);
978
979        let mut delete = || {
980            self.inner
981                .delete_data_range(&data_type, identifier.as_deref(), start_nanos, end_nanos)
982        };
983
984        let result = if matches!(&data_type, NautilusDataType::Custom { .. }) {
985            delete()
986        } else {
987            py.detach(delete)
988        };
989
990        result.map_err(|e| PyIOError::new_err(format!("Failed to delete data range: {e}")))
991    }
992
993    /// Writes custom data to Parquet files.
994    ///
995    /// Requires `CustomData` wrappers. Callers must wrap raw custom objects in
996    /// `CustomData(data_type=DataType(cls, metadata=...), data=...)` before writing.
997    ///
998    /// The registered Arrow schema must contain `ts_init`. Any `ts_event` or `ts_init` fields
999    /// must use `timestamp("ns", tz="UTC")`; incompatible schemas fail before writing.
1000    #[pyo3(signature = (data, start=None, end=None, skip_disjoint_check=false))]
1001    pub fn write_custom_data(
1002        &self,
1003        py: Python<'_>,
1004        data: Vec<Bound<'_, PyAny>>,
1005        start: Option<u64>,
1006        end: Option<u64>,
1007        skip_disjoint_check: bool,
1008    ) -> PyResult<String> {
1009        use nautilus_model::data::CustomData;
1010
1011        let mut custom_items: Vec<CustomData> = Vec::with_capacity(data.len());
1012        for obj in data {
1013            let custom = obj.extract::<CustomData>().map_err(|_| {
1014                to_pytype_err(
1015                    "write_custom_data requires CustomData wrappers; wrap with CustomData(data_type=DataType(cls, metadata=...), data=...)",
1016                )
1017            })?;
1018
1019            custom_items.push(custom);
1020        }
1021
1022        if custom_items.is_empty() {
1023            return Ok(String::new());
1024        }
1025
1026        let start_nanos = start.map(UnixNanos::from);
1027        let end_nanos = end.map(UnixNanos::from);
1028        let write_error =
1029            |e: anyhow::Error| PyIOError::new_err(format!("Failed to write custom data: {e}"));
1030        let custom_refs = custom_items.iter().collect::<Vec<_>>();
1031        let write = self
1032            .inner
1033            .prepare_custom_data_write(&custom_refs, start_nanos, end_nanos)
1034            .map_err(write_error)?;
1035
1036        self.mutate(py, || {
1037            self.inner
1038                .write_custom_data_prepared(&write, Some(skip_disjoint_check))
1039        })
1040        .map(|path| path.to_string_lossy().to_string())
1041        .map_err(write_error)
1042    }
1043
1044    /// List all instrument IDs available in the catalog for a given catalog type.
1045    pub fn list_instruments(
1046        &self,
1047        py: Python<'_>,
1048        data_type: PyCatalogDataType,
1049    ) -> PyResult<Vec<String>> {
1050        let data_type = data_type.into_inner();
1051        py.detach(|| self.inner.list_instruments(&data_type))
1052            .map_err(|e| PyIOError::new_err(format!("Failed to list instruments: {e}")))
1053    }
1054
1055    /// List all Parquet files in the catalog for a given data type and identifier.
1056    pub fn list_parquet_files(
1057        &self,
1058        py: Python<'_>,
1059        data_type: PyCatalogDataType,
1060        identifier: &str,
1061    ) -> PyResult<Vec<String>> {
1062        let data_type = data_type.into_inner();
1063        let list_files = |prefix: &str| -> PyResult<Vec<String>> {
1064            let directory = self
1065                .inner
1066                .make_path(prefix, Some(identifier))
1067                .map_err(|e| PyIOError::new_err(format!("Failed to list parquet files: {e}")))?;
1068            py.detach(|| self.inner.list_parquet_files(&directory))
1069                .map_err(|e| PyIOError::new_err(format!("Failed to list parquet files: {e}")))
1070        };
1071        let mut files = Vec::new();
1072
1073        for prefix in parquet_catalog_data_type_path_prefixes(&data_type) {
1074            files.extend(list_files(prefix.as_ref())?);
1075        }
1076
1077        Ok(files)
1078    }
1079
1080    /// Query files in the catalog matching the specified criteria.
1081    ///
1082    /// # Parameters
1083    ///
1084    /// - `data_type`: The stored family to target (data type, record type, or instrument type).
1085    /// - `identifiers`: Optional list of identifiers to filter by. Can be `instrument_id` strings
1086    ///   (e.g., "EUR/USD.SIM") or `bar_type` strings (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
1087    ///   For bars, partial matching is supported.
1088    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
1089    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
1090    ///
1091    /// # Returns
1092    ///
1093    /// Returns a list of file paths matching the criteria.
1094    #[pyo3(signature = (data_type, identifiers=None, start=None, end=None))]
1095    pub fn query_files(
1096        &self,
1097        py: Python<'_>,
1098        data_type: PyCatalogDataType,
1099        identifiers: Option<Vec<String>>,
1100        start: Option<u64>,
1101        end: Option<u64>,
1102    ) -> PyResult<Vec<String>> {
1103        let data_type = data_type.into_inner();
1104        let start_nanos = start.map(UnixNanos::from);
1105        let end_nanos = end.map(UnixNanos::from);
1106
1107        py.detach(|| {
1108            self.inner
1109                .query_files(&data_type, identifiers, start_nanos, end_nanos)
1110        })
1111        .map_err(|e| PyIOError::new_err(format!("Failed to query files list: {e}")))
1112    }
1113
1114    /// Query catalog metadata keyed by the first timestamp where each metadata is used.
1115    #[pyo3(signature = (data_type, identifiers=None, start=None, end=None, where_clause=None))]
1116    pub fn query_metadata(
1117        &mut self,
1118        py: Python<'_>,
1119        #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
1120            '_,
1121            PyAny,
1122        >,
1123        identifiers: Option<Vec<String>>,
1124        start: Option<u64>,
1125        end: Option<u64>,
1126        where_clause: Option<&str>,
1127    ) -> PyResult<Py<PyDict>> {
1128        let data_type = nautilus_data_type_from_py(data_type)?;
1129
1130        let metadata = py
1131            .detach(|| {
1132                CatalogReader::query_metadata(
1133                    &mut self.inner,
1134                    &CatalogQuery::new(data_type)
1135                        .with_identifiers(identifiers)
1136                        .with_range(start.map(UnixNanos::from), end.map(UnixNanos::from))
1137                        .with_where_clause(where_clause.map(str::to_string)),
1138                )
1139            })
1140            .map_err(|e| PyIOError::new_err(format!("Metadata query failed: {e}")))?;
1141
1142        catalog_metadata_to_pydict(py, metadata)
1143    }
1144
1145    /// Query catalog data as display-friendly Arrow IPC stream bytes.
1146    #[pyo3(signature = (
1147        data_type,
1148        identifiers=None,
1149        start=None,
1150        end=None,
1151        where_clause=None,
1152        display=true,
1153        as_of=None,
1154    ))]
1155    pub fn query_data_arrow_bytes(
1156        &mut self,
1157        py: Python<'_>,
1158        #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
1159            '_,
1160            PyAny,
1161        >,
1162        identifiers: Option<Vec<String>>,
1163        start: Option<u64>,
1164        end: Option<u64>,
1165        where_clause: Option<&str>,
1166        display: bool,
1167        as_of: Option<&Bound<'_, PyAny>>,
1168    ) -> PyResult<Py<PyBytes>> {
1169        let data_type = nautilus_data_type_from_py(data_type)?;
1170        reject_parquet_as_of(as_of)?;
1171        let query = CatalogQuery::new(data_type.clone())
1172            .with_identifiers(identifiers)
1173            .with_range(start.map(UnixNanos::from), end.map(UnixNanos::from))
1174            .with_where_clause(where_clause.map(str::to_string));
1175
1176        let batches = py
1177            .detach(|| {
1178                if display {
1179                    CatalogReader::query_display_record_batches(&mut self.inner, &query)
1180                } else {
1181                    let data = CatalogReader::query_batch(&mut self.inner, &query)?
1182                        .to_data_vec_for_compat();
1183                    crate::common::arrow::data_to_arrow_batches(&data_type, data)
1184                }
1185            })
1186            .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?;
1187
1188        let schema = arrow_ipc_data_schema(&data_type, &batches, display)?;
1189        let batches = arrow_ipc_batches(&schema, batches)?;
1190        arrow_record_batches_to_pybytes(py, &schema, &batches)
1191    }
1192
1193    /// Query catalog data as an Arrow C stream `PyCapsule`.
1194    #[pyo3(signature = (
1195        data_type,
1196        identifiers=None,
1197        start=None,
1198        end=None,
1199        where_clause=None,
1200        display=true,
1201        as_of=None,
1202    ))]
1203    pub fn query_data_arrow_stream(
1204        &mut self,
1205        py: Python<'_>,
1206        #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
1207            '_,
1208            PyAny,
1209        >,
1210        identifiers: Option<Vec<String>>,
1211        start: Option<u64>,
1212        end: Option<u64>,
1213        where_clause: Option<&str>,
1214        display: bool,
1215        as_of: Option<&Bound<'_, PyAny>>,
1216    ) -> PyResult<Py<PyAny>> {
1217        let data_type = nautilus_data_type_from_py(data_type)?;
1218        reject_parquet_as_of(as_of)?;
1219        let query = CatalogQuery::new(data_type.clone())
1220            .with_identifiers(identifiers)
1221            .with_range(start.map(UnixNanos::from), end.map(UnixNanos::from))
1222            .with_where_clause(where_clause.map(str::to_string));
1223
1224        let batches = py
1225            .detach(|| {
1226                if display {
1227                    CatalogReader::query_display_record_batches(&mut self.inner, &query)
1228                } else {
1229                    let data = CatalogReader::query_batch(&mut self.inner, &query)?
1230                        .to_data_vec_for_compat();
1231                    crate::common::arrow::data_to_arrow_batches(&data_type, data)
1232                }
1233            })
1234            .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?;
1235
1236        let schema = arrow_ipc_data_schema(&data_type, &batches, display)?;
1237        let batches = arrow_ipc_batches(&schema, batches)?;
1238        arrow_record_batches_to_pyarrow_stream(py, &schema, batches)
1239    }
1240
1241    /// Write catalog records from Arrow IPC stream bytes.
1242    #[pyo3(signature = (record_type, data, identifier=None, params=None))]
1243    pub fn write_record_arrow_bytes(
1244        &mut self,
1245        py: Python<'_>,
1246        #[gen_stub(override_type(type_repr = "model.NautilusRecordType"))] record_type: &Bound<
1247            '_,
1248            PyAny,
1249        >,
1250        data: Vec<u8>,
1251        identifier: Option<String>,
1252        params: Option<Py<PyDict>>,
1253    ) -> PyResult<()> {
1254        let record_type = catalog_record_type_from_py(record_type)?;
1255        let batches = arrow_record_batches_from_pybytes(data)?;
1256        let params = write_record_params_from_py(py, identifier, params)?;
1257
1258        py.detach(|| CatalogWriter::write_records(&mut self.inner, record_type, &batches, params))
1259            .map_err(|e| PyIOError::new_err(format!("Failed write records: {e}")))
1260    }
1261
1262    /// Query catalog records and return Arrow IPC stream bytes.
1263    #[pyo3(signature = (
1264        record_type,
1265        identifier=None,
1266        start=None,
1267        end=None,
1268        where_clause=None,
1269        display=true,
1270        as_of=None,
1271    ))]
1272    pub fn query_record_arrow_bytes(
1273        &mut self,
1274        py: Python<'_>,
1275        #[gen_stub(override_type(type_repr = "model.NautilusRecordType"))] record_type: &Bound<
1276            '_,
1277            PyAny,
1278        >,
1279        identifier: Option<String>,
1280        start: Option<u64>,
1281        end: Option<u64>,
1282        where_clause: Option<&str>,
1283        display: bool,
1284        as_of: Option<&Bound<'_, PyAny>>,
1285    ) -> PyResult<Py<PyBytes>> {
1286        let record_type = catalog_record_type_from_py(record_type)?;
1287        reject_parquet_as_of(as_of)?;
1288        let start_nanos = start.map(UnixNanos::from);
1289        let end_nanos = end.map(UnixNanos::from);
1290
1291        let batches = py
1292            .detach(|| {
1293                if display {
1294                    CatalogReader::query_record_display_batches(
1295                        &mut self.inner,
1296                        &CatalogRecordQuery::new(record_type)
1297                            .with_identifier(identifier)
1298                            .with_range(start_nanos, end_nanos)
1299                            .with_where_clause(where_clause.map(str::to_string)),
1300                    )
1301                } else {
1302                    CatalogReader::query_record_batches(
1303                        &mut self.inner,
1304                        &CatalogRecordQuery::new(record_type)
1305                            .with_identifier(identifier)
1306                            .with_range(start_nanos, end_nanos)
1307                            .with_where_clause(where_clause.map(str::to_string)),
1308                    )
1309                }
1310            })
1311            .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?;
1312
1313        let schema = arrow_ipc_record_schema(record_type, &batches)?;
1314        let batches = arrow_ipc_batches(&schema, batches)?;
1315
1316        arrow_record_batches_to_pybytes(py, &schema, &batches)
1317    }
1318
1319    /// Query catalog records as an Arrow C stream `PyCapsule`.
1320    #[pyo3(signature = (
1321        record_type,
1322        identifier=None,
1323        start=None,
1324        end=None,
1325        where_clause=None,
1326        display=true,
1327        as_of=None,
1328    ))]
1329    pub fn query_record_arrow_stream(
1330        &mut self,
1331        py: Python<'_>,
1332        #[gen_stub(override_type(type_repr = "model.NautilusRecordType"))] record_type: &Bound<
1333            '_,
1334            PyAny,
1335        >,
1336        identifier: Option<String>,
1337        start: Option<u64>,
1338        end: Option<u64>,
1339        where_clause: Option<&str>,
1340        display: bool,
1341        as_of: Option<&Bound<'_, PyAny>>,
1342    ) -> PyResult<Py<PyAny>> {
1343        let record_type = catalog_record_type_from_py(record_type)?;
1344        reject_parquet_as_of(as_of)?;
1345        let start_nanos = start.map(UnixNanos::from);
1346        let end_nanos = end.map(UnixNanos::from);
1347
1348        let batches = py
1349            .detach(|| {
1350                if display {
1351                    CatalogReader::query_record_display_batches(
1352                        &mut self.inner,
1353                        &CatalogRecordQuery::new(record_type)
1354                            .with_identifier(identifier)
1355                            .with_range(start_nanos, end_nanos)
1356                            .with_where_clause(where_clause.map(str::to_string)),
1357                    )
1358                } else {
1359                    CatalogReader::query_record_batches(
1360                        &mut self.inner,
1361                        &CatalogRecordQuery::new(record_type)
1362                            .with_identifier(identifier)
1363                            .with_range(start_nanos, end_nanos)
1364                            .with_where_clause(where_clause.map(str::to_string)),
1365                    )
1366                }
1367            })
1368            .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?;
1369
1370        let schema = arrow_ipc_record_schema(record_type, &batches)?;
1371        let batches = arrow_ipc_batches(&schema, batches)?;
1372
1373        arrow_record_batches_to_pyarrow_stream(py, &schema, batches)
1374    }
1375
1376    /// Get missing time intervals for a data request.
1377    ///
1378    /// # Parameters
1379    ///
1380    /// - `start`: Start timestamp (nanoseconds since Unix epoch)
1381    /// - `end`: End timestamp (nanoseconds since Unix epoch)
1382    /// - `data_type`: The stored family to target (data type, record type, or instrument type).
1383    /// - `identifier`: Optional instrument ID or bar type filter
1384    ///
1385    /// # Returns
1386    ///
1387    /// Returns a list of (start, end) timestamp tuples representing missing intervals.
1388    #[pyo3(signature = (start, end, data_type, identifier=None))]
1389    #[expect(clippy::needless_pass_by_value)]
1390    pub fn get_missing_intervals_for_request(
1391        &self,
1392        py: Python<'_>,
1393        start: u64,
1394        end: u64,
1395        data_type: PyCatalogDataType,
1396        identifier: Option<String>,
1397    ) -> PyResult<Vec<(u64, u64)>> {
1398        let data_type = data_type.into_inner();
1399        py.detach(|| {
1400            self.inner.get_missing_intervals_for_request(
1401                start,
1402                end,
1403                &data_type,
1404                identifier.as_deref(),
1405            )
1406        })
1407        .map_err(|e| PyIOError::new_err(format!("Failed to get missing intervals: {e}")))
1408    }
1409
1410    /// Query the first timestamp for a specific data class and instrument.
1411    ///
1412    /// # Parameters
1413    ///
1414    /// - `data_type`: The stored family to target (data type, record type, or instrument type).
1415    /// - `identifier`: Optional instrument ID or bar type filter
1416    ///
1417    /// # Returns
1418    ///
1419    /// Returns the first timestamp as nanoseconds since Unix epoch, or None if no data exists.
1420    #[pyo3(signature = (data_type, identifier=None))]
1421    #[expect(clippy::needless_pass_by_value)]
1422    pub fn query_first_timestamp(
1423        &self,
1424        py: Python<'_>,
1425        data_type: PyCatalogDataType,
1426        identifier: Option<String>,
1427    ) -> PyResult<Option<u64>> {
1428        let data_type = data_type.into_inner();
1429        py.detach(|| {
1430            self.inner
1431                .query_first_timestamp(&data_type, identifier.as_deref())
1432        })
1433        .map_err(|e| PyIOError::new_err(format!("Failed to query first timestamp: {e}")))
1434    }
1435
1436    /// Query the last timestamp for a specific data class and instrument.
1437    ///
1438    /// # Parameters
1439    ///
1440    /// - `data_type`: The stored family to target (data type, record type, or instrument type).
1441    /// - `identifier`: Optional instrument ID or bar type filter
1442    ///
1443    /// # Returns
1444    ///
1445    /// Returns the last timestamp as nanoseconds since Unix epoch, or None if no data exists.
1446    #[pyo3(signature = (data_type, identifier=None))]
1447    #[expect(clippy::needless_pass_by_value)]
1448    pub fn query_last_timestamp(
1449        &self,
1450        py: Python<'_>,
1451        data_type: PyCatalogDataType,
1452        identifier: Option<String>,
1453    ) -> PyResult<Option<u64>> {
1454        let data_type = data_type.into_inner();
1455        py.detach(|| {
1456            self.inner
1457                .query_last_timestamp(&data_type, identifier.as_deref())
1458        })
1459        .map_err(|e| PyIOError::new_err(format!("Failed to query last timestamp: {e}")))
1460    }
1461
1462    /// Get time intervals covered by data for a specific data class and instrument.
1463    ///
1464    /// # Parameters
1465    ///
1466    /// - `data_type`: The stored family to target (data type, record type, or instrument type).
1467    /// - `identifier`: Optional instrument ID or bar type filter
1468    ///
1469    /// # Returns
1470    ///
1471    /// Returns a list of (start, end) timestamp tuples representing covered intervals.
1472    #[pyo3(signature = (data_type, identifier=None))]
1473    #[expect(clippy::needless_pass_by_value)]
1474    pub fn get_intervals(
1475        &self,
1476        py: Python<'_>,
1477        data_type: PyCatalogDataType,
1478        identifier: Option<String>,
1479    ) -> PyResult<Vec<(u64, u64)>> {
1480        let data_type = data_type.into_inner();
1481        py.detach(|| self.inner.get_intervals(&data_type, identifier.as_deref()))
1482            .map_err(|e| PyIOError::new_err(format!("Failed to get intervals: {e}")))
1483    }
1484
1485    /// Queries one data family and returns the decoded Python objects.
1486    #[pyo3(signature = (data_type, identifiers=None, start=None, end=None, where_clause=None, files=None, optimize_file_loading=true))]
1487    #[expect(
1488        clippy::too_many_arguments,
1489        reason = "PyO3 signature mirrors the catalog query filters"
1490    )]
1491    pub fn query(
1492        &mut self,
1493        py: Python<'_>,
1494        #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
1495            '_,
1496            PyAny,
1497        >,
1498        identifiers: Option<Vec<String>>,
1499        start: Option<u64>,
1500        end: Option<u64>,
1501        where_clause: Option<&str>,
1502        files: Option<Vec<String>>,
1503        optimize_file_loading: bool,
1504    ) -> PyResult<Vec<Py<PyAny>>> {
1505        let data_type = nautilus_data_type_from_py(data_type)?;
1506        let start_nanos = start.map(UnixNanos::from);
1507        let end_nanos = end.map(UnixNanos::from);
1508
1509        macro_rules! typed {
1510            ($type:ty) => {
1511                query_parquet_data::<$type>(
1512                    py,
1513                    &mut self.inner,
1514                    identifiers,
1515                    start,
1516                    end,
1517                    where_clause,
1518                    files,
1519                    optimize_file_loading,
1520                    "Query failed",
1521                )?
1522                .into_iter()
1523                .map(Data::from)
1524                .collect::<Vec<Data>>()
1525            };
1526        }
1527
1528        let data = match data_type {
1529            NautilusDataType::Custom { type_name } => py
1530                .detach(|| {
1531                    self.inner.query_custom_data_dynamic(
1532                        &type_name,
1533                        identifiers.as_deref(),
1534                        start_nanos,
1535                        end_nanos,
1536                        where_clause,
1537                        files.clone(),
1538                        optimize_file_loading,
1539                    )
1540                })
1541                .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?,
1542            NautilusDataType::Instrument => {
1543                let instruments = py
1544                    .detach(|| {
1545                        self.inner.query_instruments_filtered_with_where(
1546                            identifiers.as_deref(),
1547                            start_nanos,
1548                            end_nanos,
1549                            where_clause,
1550                        )
1551                    })
1552                    .map_err(|e| PyIOError::new_err(format!("Query failed: {e}")))?;
1553
1554                return instruments
1555                    .into_iter()
1556                    .map(|instrument| instrument_any_to_pyobject(py, instrument))
1557                    .collect();
1558            }
1559            NautilusDataType::OrderBookDelta => typed!(OrderBookDelta),
1560            NautilusDataType::OrderBookDepth => typed!(OrderBookDepth),
1561            NautilusDataType::QuoteTick => typed!(QuoteTick),
1562            NautilusDataType::TradeTick => typed!(TradeTick),
1563            NautilusDataType::Bar => typed!(Bar),
1564            NautilusDataType::MarkPriceUpdate => typed!(MarkPriceUpdate),
1565            NautilusDataType::IndexPriceUpdate => typed!(IndexPriceUpdate),
1566            NautilusDataType::FundingRateUpdate => typed!(FundingRateUpdate),
1567            NautilusDataType::OptionGreeks => typed!(OptionGreeks),
1568            NautilusDataType::InstrumentStatus => typed!(InstrumentStatus),
1569            NautilusDataType::InstrumentClose => typed!(InstrumentClose),
1570            #[cfg(feature = "defi")]
1571            NautilusDataType::Defi => {
1572                return Err(to_pytype_err("Defi data is not supported by query"));
1573            }
1574            #[cfg(not(feature = "defi"))]
1575            #[allow(
1576                unreachable_patterns,
1577                reason = "DeFi variants can exist without this crate's defi feature"
1578            )]
1579            _ => return Err(to_pytype_err("Defi data is not supported by query")),
1580        };
1581
1582        let mut python_objects = Vec::new();
1583        for item in data {
1584            python_objects.push(data_to_pyobject(py, item)?);
1585        }
1586
1587        Ok(python_objects)
1588    }
1589
1590    /// Query order book delta data from Parquet files.
1591    ///
1592    /// # Parameters
1593    ///
1594    /// - `identifiers`: Optional list of identifiers to filter by. Can be `instrument_id` strings
1595    ///   (e.g., "EUR/USD.SIM") or `bar_type` strings (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
1596    ///   For bars, partial matching is supported.
1597    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
1598    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
1599    /// - `where_clause`: Optional SQL WHERE clause for additional filtering
1600    ///
1601    /// # Returns
1602    ///
1603    /// Returns a vector of `OrderBookDelta` objects matching the query criteria.
1604    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1605    pub fn query_order_book_deltas(
1606        &mut self,
1607        py: Python<'_>,
1608        identifiers: Option<Vec<String>>,
1609        start: Option<u64>,
1610        end: Option<u64>,
1611        where_clause: Option<&str>,
1612    ) -> PyResult<Vec<OrderBookDelta>> {
1613        query_parquet_data::<OrderBookDelta>(
1614            py,
1615            &mut self.inner,
1616            identifiers,
1617            start,
1618            end,
1619            where_clause,
1620            None,
1621            true,
1622            "Failed to query data",
1623        )
1624    }
1625
1626    /// Query order book depth data from Parquet files.
1627    ///
1628    /// # Parameters
1629    ///
1630    /// - `identifiers`: Optional list of identifiers to filter by.
1631    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
1632    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
1633    /// - `where_clause`: Optional SQL WHERE clause for additional filtering
1634    ///
1635    /// # Returns
1636    ///
1637    /// Returns a vector of `OrderBookDepth` objects matching the query criteria.
1638    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1639    pub fn query_order_book_depths(
1640        &mut self,
1641        py: Python<'_>,
1642        identifiers: Option<Vec<String>>,
1643        start: Option<u64>,
1644        end: Option<u64>,
1645        where_clause: Option<&str>,
1646    ) -> PyResult<Vec<OrderBookDepth>> {
1647        query_parquet_data::<OrderBookDepth>(
1648            py,
1649            &mut self.inner,
1650            identifiers,
1651            start,
1652            end,
1653            where_clause,
1654            None,
1655            true,
1656            "Failed to query data",
1657        )
1658    }
1659
1660    /// Query quote tick data from Parquet files.
1661    ///
1662    /// # Parameters
1663    ///
1664    /// - `identifiers`: Optional list of identifiers to filter by. Can be `instrument_id` strings
1665    ///   (e.g., "EUR/USD.SIM") or `bar_type` strings (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
1666    ///   For bars, partial matching is supported.
1667    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
1668    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
1669    /// - `where_clause`: Optional SQL WHERE clause for additional filtering
1670    ///
1671    /// # Returns
1672    ///
1673    /// Returns a vector of `QuoteTick` objects matching the query criteria.
1674    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1675    pub fn query_quote_ticks(
1676        &mut self,
1677        py: Python<'_>,
1678        identifiers: Option<Vec<String>>,
1679        start: Option<u64>,
1680        end: Option<u64>,
1681        where_clause: Option<&str>,
1682    ) -> PyResult<Vec<QuoteTick>> {
1683        query_parquet_data::<QuoteTick>(
1684            py,
1685            &mut self.inner,
1686            identifiers,
1687            start,
1688            end,
1689            where_clause,
1690            None,
1691            true,
1692            "Failed to query data",
1693        )
1694    }
1695
1696    /// Query trade tick data from Parquet files.
1697    ///
1698    /// # Parameters
1699    ///
1700    /// - `identifiers`: Optional list of identifiers to filter by. Can be `instrument_id` strings
1701    ///   (e.g., "EUR/USD.SIM") or `bar_type` strings (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
1702    ///   For bars, partial matching is supported.
1703    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
1704    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
1705    /// - `where_clause`: Optional SQL WHERE clause for additional filtering
1706    ///
1707    /// # Returns
1708    ///
1709    /// Returns a vector of `TradeTick` objects matching the query criteria.
1710    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1711    pub fn query_trade_ticks(
1712        &mut self,
1713        py: Python<'_>,
1714        identifiers: Option<Vec<String>>,
1715        start: Option<u64>,
1716        end: Option<u64>,
1717        where_clause: Option<&str>,
1718    ) -> PyResult<Vec<TradeTick>> {
1719        query_parquet_data::<TradeTick>(
1720            py,
1721            &mut self.inner,
1722            identifiers,
1723            start,
1724            end,
1725            where_clause,
1726            None,
1727            true,
1728            "Failed to query data",
1729        )
1730    }
1731
1732    /// Query bar data from Parquet files.
1733    ///
1734    /// # Parameters
1735    ///
1736    /// - `identifiers`: Optional list of identifiers to filter by. Can be `instrument_id` strings
1737    ///   (e.g., "EUR/USD.SIM") or `bar_type` strings (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
1738    ///   For bars, partial matching is supported (e.g., "EUR/USD.SIM" will match all bar types for that instrument).
1739    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
1740    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
1741    /// - `where_clause`: Optional SQL WHERE clause for additional filtering
1742    ///
1743    /// # Returns
1744    ///
1745    /// Returns a vector of Bar objects matching the query criteria.
1746    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1747    pub fn query_bars(
1748        &mut self,
1749        py: Python<'_>,
1750        identifiers: Option<Vec<String>>,
1751        start: Option<u64>,
1752        end: Option<u64>,
1753        where_clause: Option<&str>,
1754    ) -> PyResult<Vec<Bar>> {
1755        query_parquet_data::<Bar>(
1756            py,
1757            &mut self.inner,
1758            identifiers,
1759            start,
1760            end,
1761            where_clause,
1762            None,
1763            true,
1764            "Failed to query data",
1765        )
1766    }
1767
1768    /// Query mark price update data from Parquet files.
1769    ///
1770    /// # Parameters
1771    ///
1772    /// - `identifiers`: Optional list of identifiers to filter by.
1773    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
1774    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
1775    /// - `where_clause`: Optional SQL WHERE clause for additional filtering
1776    ///
1777    /// # Returns
1778    ///
1779    /// Returns a vector of `MarkPriceUpdate` objects matching the query criteria.
1780    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1781    pub fn query_mark_price_updates(
1782        &mut self,
1783        py: Python<'_>,
1784        identifiers: Option<Vec<String>>,
1785        start: Option<u64>,
1786        end: Option<u64>,
1787        where_clause: Option<&str>,
1788    ) -> PyResult<Vec<MarkPriceUpdate>> {
1789        query_parquet_data::<MarkPriceUpdate>(
1790            py,
1791            &mut self.inner,
1792            identifiers,
1793            start,
1794            end,
1795            where_clause,
1796            None,
1797            true,
1798            "Failed to query data",
1799        )
1800    }
1801
1802    /// Query index price update data from Parquet files.
1803    ///
1804    /// # Parameters
1805    ///
1806    /// - `identifiers`: Optional list of identifiers to filter by.
1807    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
1808    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
1809    /// - `where_clause`: Optional SQL WHERE clause for additional filtering
1810    ///
1811    /// # Returns
1812    ///
1813    /// Returns a vector of `IndexPriceUpdate` objects matching the query criteria.
1814    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1815    pub fn query_index_price_updates(
1816        &mut self,
1817        py: Python<'_>,
1818        identifiers: Option<Vec<String>>,
1819        start: Option<u64>,
1820        end: Option<u64>,
1821        where_clause: Option<&str>,
1822    ) -> PyResult<Vec<IndexPriceUpdate>> {
1823        query_parquet_data::<IndexPriceUpdate>(
1824            py,
1825            &mut self.inner,
1826            identifiers,
1827            start,
1828            end,
1829            where_clause,
1830            None,
1831            true,
1832            "Failed to query data",
1833        )
1834    }
1835
1836    /// Query funding rate update data from Parquet files.
1837    ///
1838    /// # Parameters
1839    ///
1840    /// - `identifiers`: Optional list of identifiers to filter by.
1841    /// - `start`: Optional start timestamp (nanoseconds since Unix epoch)
1842    /// - `end`: Optional end timestamp (nanoseconds since Unix epoch)
1843    /// - `where_clause`: Optional SQL WHERE clause for additional filtering
1844    ///
1845    /// # Returns
1846    ///
1847    /// Returns a vector of `FundingRateUpdate` objects matching the query criteria.
1848    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1849    pub fn query_funding_rate_updates(
1850        &mut self,
1851        py: Python<'_>,
1852        identifiers: Option<Vec<String>>,
1853        start: Option<u64>,
1854        end: Option<u64>,
1855        where_clause: Option<&str>,
1856    ) -> PyResult<Vec<FundingRateUpdate>> {
1857        query_parquet_data::<FundingRateUpdate>(
1858            py,
1859            &mut self.inner,
1860            identifiers,
1861            start,
1862            end,
1863            where_clause,
1864            None,
1865            true,
1866            "Failed to query data",
1867        )
1868    }
1869
1870    /// Query option greeks data from Parquet files.
1871    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1872    pub fn query_option_greeks(
1873        &mut self,
1874        py: Python<'_>,
1875        identifiers: Option<Vec<String>>,
1876        start: Option<u64>,
1877        end: Option<u64>,
1878        where_clause: Option<&str>,
1879    ) -> PyResult<Vec<OptionGreeks>> {
1880        query_parquet_data::<OptionGreeks>(
1881            py,
1882            &mut self.inner,
1883            identifiers,
1884            start,
1885            end,
1886            where_clause,
1887            None,
1888            true,
1889            "Failed to query data",
1890        )
1891    }
1892
1893    /// Queries instrument status data from Parquet files.
1894    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1895    pub fn query_instrument_statuses(
1896        &mut self,
1897        py: Python<'_>,
1898        identifiers: Option<Vec<String>>,
1899        start: Option<u64>,
1900        end: Option<u64>,
1901        where_clause: Option<&str>,
1902    ) -> PyResult<Vec<InstrumentStatus>> {
1903        query_parquet_data::<InstrumentStatus>(
1904            py,
1905            &mut self.inner,
1906            identifiers,
1907            start,
1908            end,
1909            where_clause,
1910            None,
1911            true,
1912            "Failed to query data",
1913        )
1914    }
1915
1916    /// Queries instrument close data from Parquet files.
1917    #[pyo3(signature = (identifiers=None, start=None, end=None, where_clause=None))]
1918    pub fn query_instrument_closes(
1919        &mut self,
1920        py: Python<'_>,
1921        identifiers: Option<Vec<String>>,
1922        start: Option<u64>,
1923        end: Option<u64>,
1924        where_clause: Option<&str>,
1925    ) -> PyResult<Vec<InstrumentClose>> {
1926        query_parquet_data::<InstrumentClose>(
1927            py,
1928            &mut self.inner,
1929            identifiers,
1930            start,
1931            end,
1932            where_clause,
1933            None,
1934            true,
1935            "Failed to query data",
1936        )
1937    }
1938
1939    /// List all data types available in the catalog.
1940    ///
1941    /// # Returns
1942    ///
1943    /// Returns the data, record, and instrument types stored in the catalog.
1944    pub fn list_data_types(&self, py: Python<'_>) -> PyResult<Vec<PyCatalogDataType>> {
1945        py.detach(|| self.inner.list_data_types())
1946            .map(|data_types| data_types.into_iter().map(PyCatalogDataType::new).collect())
1947            .map_err(|e| PyIOError::new_err(format!("Failed to list data types: {e}")))
1948    }
1949
1950    /// List all live run IDs available in the catalog.
1951    ///
1952    /// # Returns
1953    ///
1954    /// Returns a list of live run IDs (as directory stems) in the catalog.
1955    pub fn list_live_runs(&self, py: Python<'_>) -> PyResult<Vec<String>> {
1956        py.detach(|| self.inner.list_live_runs())
1957            .map_err(|e| PyIOError::new_err(format!("Failed to list live runs: {e}")))
1958    }
1959
1960    /// List all backtest run IDs available in the catalog.
1961    ///
1962    /// # Returns
1963    ///
1964    /// Returns a list of backtest run IDs (as directory stems) in the catalog.
1965    pub fn list_backtest_runs(&self, py: Python<'_>) -> PyResult<Vec<String>> {
1966        py.detach(|| self.inner.list_backtest_runs())
1967            .map_err(|e| PyIOError::new_err(format!("Failed to list backtest runs: {e}")))
1968    }
1969
1970    /// List all backtest run instances available in the catalog.
1971    pub fn list_backtests(&self, py: Python<'_>) -> PyResult<Vec<String>> {
1972        py.detach(|| self.inner.list_backtest_runs())
1973            .map_err(|e| PyIOError::new_err(format!("Failed to list backtests: {e}")))
1974    }
1975
1976    /// Read data from a live run instance.
1977    ///
1978    /// # Parameters
1979    ///
1980    /// - `instance_id`: The ID of the live run instance
1981    ///
1982    /// # Returns
1983    ///
1984    /// Returns a list of data objects from the live run, sorted by timestamp.
1985    #[pyo3(signature = (instance_id))]
1986    pub fn read_live_run(&self, py: Python<'_>, instance_id: &str) -> PyResult<Vec<Py<PyAny>>> {
1987        let data = self
1988            .inner
1989            .read_live_run(instance_id)
1990            .map_err(|e| PyIOError::new_err(format!("Failed to read live run: {e}")))?;
1991
1992        let mut python_objects = Vec::new();
1993        for item in data {
1994            python_objects.push(data_to_pyobject(py, item)?);
1995        }
1996
1997        Ok(python_objects)
1998    }
1999
2000    /// Read data from a backtest run instance.
2001    ///
2002    /// # Parameters
2003    ///
2004    /// - `instance_id`: The ID of the backtest run instance
2005    ///
2006    /// # Returns
2007    ///
2008    /// Returns a list of data objects from the backtest run, sorted by timestamp.
2009    #[pyo3(signature = (instance_id))]
2010    pub fn read_backtest(&self, py: Python<'_>, instance_id: &str) -> PyResult<Vec<Py<PyAny>>> {
2011        let data = self
2012            .inner
2013            .read_backtest(instance_id)
2014            .map_err(|e| PyIOError::new_err(format!("Failed to read backtest: {e}")))?;
2015
2016        let mut python_objects = Vec::new();
2017        for item in data {
2018            python_objects.push(data_to_pyobject(py, item)?);
2019        }
2020
2021        Ok(python_objects)
2022    }
2023
2024    /// Convert stream data from feather files to parquet files.
2025    ///
2026    /// This method reads data from feather files generated during a backtest or live run
2027    /// and writes it to the catalog in parquet format. It's useful for converting temporary
2028    /// stream data into a more permanent and queryable format.
2029    ///
2030    /// # Parameters
2031    ///
2032    /// - `instance_id`: The ID of the backtest or live run instance
2033    /// - `data_type`: The stored family to convert (data type or record type).
2034    /// - `environment`: The environment of the run, which names the folder holding its feather files (default: `Environment.BACKTEST`)
2035    /// - `identifiers`: Optional list of identifiers to filter by (instrument IDs or bar types)
2036    /// - `use_ts_event_for_ts_init`: If true, replaces the `ts_init` column with `ts_event` column values before deserializing
2037    ///
2038    /// # Returns
2039    ///
2040    /// Returns nothing on success.
2041    ///
2042    /// # Examples
2043    ///
2044    /// ```python
2045    /// # Convert backtest stream data to parquet
2046    /// catalog.convert_stream_to_data(
2047    ///     "instance-123",
2048    ///     NautilusDataType.QuoteTick,
2049    ///     environment=Environment.BACKTEST
2050    /// )
2051    ///
2052    /// # Convert live run data with identifier filtering
2053    /// catalog.convert_stream_to_data(
2054    ///     "instance-456",
2055    ///     NautilusDataType.TradeTick,
2056    ///     environment=Environment.LIVE,
2057    ///     identifiers=["EUR/USD.SIM"]
2058    /// )
2059    /// ```
2060    #[pyo3(signature = (instance_id, data_type, environment=Environment::Backtest, identifiers=None, use_ts_event_for_ts_init=false))]
2061    #[expect(clippy::needless_pass_by_value)]
2062    pub fn convert_stream_to_data(
2063        &mut self,
2064        py: Python<'_>,
2065        instance_id: &str,
2066        data_type: PyCatalogDataType,
2067        environment: Environment,
2068        identifiers: Option<Vec<String>>,
2069        use_ts_event_for_ts_init: bool,
2070    ) -> PyResult<()> {
2071        let data_type = data_type.into_inner();
2072
2073        py.detach(|| {
2074            self.inner.convert_stream_to_data(
2075                instance_id,
2076                &data_type,
2077                environment,
2078                identifiers.as_deref(),
2079                use_ts_event_for_ts_init,
2080            )
2081        })
2082        .map_err(|e| PyIOError::new_err(format!("Failed to convert stream to data: {e}")))
2083    }
2084
2085    /// Query custom data from Parquet files.
2086    #[pyo3(signature = (data_type, identifiers=None, start=None, end=None, where_clause=None))]
2087    #[expect(clippy::needless_pass_by_value)]
2088    pub fn query_custom_data(
2089        &mut self,
2090        py: Python<'_>,
2091        #[gen_stub(override_type(type_repr = "model.NautilusDataType"))] data_type: &Bound<
2092            '_,
2093            PyAny,
2094        >,
2095        identifiers: Option<Vec<String>>,
2096        start: Option<u64>,
2097        end: Option<u64>,
2098        where_clause: Option<&str>,
2099    ) -> PyResult<Vec<Py<PyAny>>> {
2100        let NautilusDataType::Custom { type_name } = nautilus_data_type_from_py(data_type)? else {
2101            return Err(to_pytype_err("data_type must be a custom NautilusDataType"));
2102        };
2103
2104        let start_nanos = start.map(UnixNanos::from);
2105        let end_nanos = end.map(UnixNanos::from);
2106
2107        let data = py
2108            .detach(|| {
2109                self.inner.query_custom_data_dynamic(
2110                    &type_name,
2111                    identifiers.as_deref(),
2112                    start_nanos,
2113                    end_nanos,
2114                    where_clause,
2115                    None,
2116                    true,
2117                )
2118            })
2119            .map_err(|e| PyIOError::new_err(format!("Failed to query custom data: {e}")))?;
2120
2121        let mut python_objects = Vec::new();
2122
2123        for item in data {
2124            let py_obj: Py<PyAny> = match item {
2125                Data::Custom(custom) => Py::new(py, custom)?.into_any(),
2126                _ => return Err(PyIOError::new_err("Expected custom data")),
2127            };
2128
2129            python_objects.push(py_obj);
2130        }
2131
2132        Ok(python_objects)
2133    }
2134}
2135
2136impl PyParquetDataCatalog {
2137    fn mutate<T: Send>(
2138        &self,
2139        py: Python<'_>,
2140        operation: impl FnOnce() -> anyhow::Result<T> + Send,
2141    ) -> anyhow::Result<T> {
2142        py.detach(|| {
2143            let _guard = self
2144                .mutation_lock
2145                .lock()
2146                .map_err(|_| anyhow::anyhow!("catalog mutation lock is poisoned"))?;
2147            operation()
2148        })
2149    }
2150}
2151
2152// Codes follow the Parquet format codec numbering, except that 5 writes LZ4_RAW rather than the
2153// deprecated Hadoop-framed LZ4; LZO (3) is excluded because the writer cannot produce it
2154fn catalog_compression_from_code(code: u8) -> PyResult<CatalogCompression> {
2155    match code {
2156        0 => Ok(CatalogCompression::Uncompressed),
2157        1 => Ok(CatalogCompression::Snappy),
2158        2 => Ok(CatalogCompression::Gzip),
2159        4 => Ok(CatalogCompression::Brotli),
2160        5 => Ok(CatalogCompression::Lz4),
2161        6 => Ok(CatalogCompression::Zstd),
2162        _ => Err(to_pyvalue_err(format!(
2163            "Invalid compression code {code}, expected one of 0 (UNCOMPRESSED), 1 (SNAPPY), \
2164             2 (GZIP), 4 (BROTLI), 5 (LZ4_RAW), or 6 (ZSTD)"
2165        ))),
2166    }
2167}
2168
2169#[cfg(test)]
2170mod tests {
2171    use std::{
2172        borrow::Borrow,
2173        cell::Cell,
2174        sync::{Arc, Barrier, Once, TryLockError, mpsc},
2175        thread,
2176        time::Duration,
2177    };
2178
2179    use arrow::{datatypes::Schema, error::ArrowError, record_batch::RecordBatch};
2180    use nautilus_model::{
2181        custom_data,
2182        data::{
2183            BookOrder, CustomData, CustomDataTrait, DataType, ensure_custom_data_json_registered,
2184            register_arrow,
2185        },
2186        enums::{BookAction, OrderSide},
2187        identifiers::InstrumentId,
2188        python::data::PyNautilusDataType,
2189        types::{Price, Quantity},
2190    };
2191    use nautilus_serialization::{
2192        arrow::{ArrowSchemaProvider, DecodeDataFromRecordBatch, EncodingError},
2193        arrow_custom_data,
2194    };
2195    use rstest::rstest;
2196    use tempfile::TempDir;
2197
2198    use super::*;
2199
2200    #[rstest]
2201    #[case::uncompressed(0, CatalogCompression::Uncompressed)]
2202    #[case::snappy(1, CatalogCompression::Snappy)]
2203    #[case::gzip(2, CatalogCompression::Gzip)]
2204    #[case::brotli(4, CatalogCompression::Brotli)]
2205    #[case::lz4(5, CatalogCompression::Lz4)]
2206    #[case::zstd(6, CatalogCompression::Zstd)]
2207    fn catalog_compression_from_code_maps_parquet_codes(
2208        #[case] code: u8,
2209        #[case] expected: CatalogCompression,
2210    ) {
2211        assert_eq!(catalog_compression_from_code(code).unwrap(), expected);
2212    }
2213
2214    #[rstest]
2215    fn test_write_parquet_data_releases_gil() {
2216        Python::initialize();
2217        let directory = TempDir::new().unwrap();
2218        let mut catalog = PyParquetDataCatalog::py_new(
2219            directory.path().to_str().unwrap(),
2220            None,
2221            None,
2222            None,
2223            None,
2224        )
2225        .unwrap();
2226        let delta = OrderBookDelta::clear(
2227            InstrumentId::from("ETH/USDT.BINANCE"),
2228            17,
2229            23.into(),
2230            29.into(),
2231        );
2232
2233        let path = Python::attach(|py| {
2234            write_parquet_data(
2235                py,
2236                &catalog,
2237                vec![GilCheckedDelta(delta, Some(&catalog.mutation_lock))],
2238                None,
2239                None,
2240                false,
2241                "deltas",
2242            )
2243            .unwrap()
2244        });
2245
2246        let loaded = catalog
2247            .inner
2248            .query::<OrderBookDelta>(None, None, None, None, None, true)
2249            .unwrap();
2250
2251        assert!(catalog.inner.file_exists(&path).unwrap());
2252        assert_eq!(loaded, vec![delta]);
2253    }
2254
2255    #[rstest]
2256    fn test_query_parquet_data_releases_gil() {
2257        Python::initialize();
2258        let directory = TempDir::new().unwrap();
2259        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2260        let delta = OrderBookDelta::clear(
2261            InstrumentId::from("ETH/USDT.BINANCE"),
2262            17,
2263            23.into(),
2264            29.into(),
2265        );
2266        catalog
2267            .write_to_parquet(&[delta], None, None, None)
2268            .unwrap();
2269
2270        let loaded = Python::attach(|py| {
2271            query_parquet_data::<GilCheckedDelta>(
2272                py,
2273                &mut catalog,
2274                None,
2275                None,
2276                None,
2277                None,
2278                None,
2279                true,
2280                "Query failed",
2281            )
2282            .unwrap()
2283        });
2284
2285        assert_eq!(loaded.len(), 1);
2286        assert_eq!(loaded[0].0, delta);
2287        assert!(loaded[0].1.is_none());
2288        assert_eq!(loaded[0].0.action, BookAction::Clear);
2289    }
2290
2291    #[rstest]
2292    fn test_catalog_python_delta_round_trip_and_maintenance() {
2293        Python::initialize();
2294        let directory = TempDir::new().unwrap();
2295        let instrument_id = InstrumentId::from("ETH/USDT.BINANCE");
2296        let deltas = vec![
2297            OrderBookDelta::new(
2298                instrument_id,
2299                BookAction::Add,
2300                BookOrder::new(
2301                    OrderSide::Buy,
2302                    Price::from("100.23"),
2303                    Quantity::from("1.234"),
2304                    41,
2305                ),
2306                7,
2307                17,
2308                23.into(),
2309                29.into(),
2310            ),
2311            OrderBookDelta::new(
2312                instrument_id,
2313                BookAction::Update,
2314                BookOrder::new(
2315                    OrderSide::Sell,
2316                    Price::from("101.45"),
2317                    Quantity::from("2.345"),
2318                    47,
2319                ),
2320                8,
2321                31,
2322                37.into(),
2323                43.into(),
2324            ),
2325        ];
2326
2327        Python::attach(|py| {
2328            let catalog = PyParquetDataCatalog::py_new(
2329                directory.path().to_str().unwrap(),
2330                None,
2331                None,
2332                None,
2333                None,
2334            )
2335            .unwrap();
2336            let catalog = Bound::new(py, catalog).unwrap();
2337            let data_type = PyNautilusDataType::new(NautilusDataType::OrderBookDelta);
2338            let data_type = Bound::new(py, data_type).unwrap();
2339            let path: String = catalog
2340                .call_method1("write_order_book_deltas", (deltas.clone(), 11, 53))
2341                .unwrap()
2342                .extract()
2343                .unwrap();
2344            let loaded: Vec<OrderBookDelta> = catalog
2345                .call_method0("query_order_book_deltas")
2346                .unwrap()
2347                .extract()
2348                .unwrap();
2349            let queried: Vec<OrderBookDelta> = catalog
2350                .call_method1("query", (&data_type,))
2351                .unwrap()
2352                .extract()
2353                .unwrap();
2354            let intervals: Vec<(u64, u64)> = catalog
2355                .call_method1("get_intervals", (&data_type, instrument_id.to_string()))
2356                .unwrap()
2357                .extract()
2358                .unwrap();
2359
2360            assert!(catalog.borrow().inner.file_exists(&path).unwrap());
2361            assert_eq!(loaded, deltas);
2362            assert_eq!(queried, deltas);
2363            assert_eq!(intervals, vec![(11, 53)]);
2364
2365            catalog
2366                .call_method1("consolidate_data", (&data_type, instrument_id.to_string()))
2367                .unwrap();
2368            catalog
2369                .call_method1(
2370                    "delete_data_range",
2371                    (&data_type, instrument_id.to_string(), 29, 29),
2372                )
2373                .unwrap();
2374            let remaining: Vec<OrderBookDelta> = catalog
2375                .call_method0("query_order_book_deltas")
2376                .unwrap()
2377                .extract()
2378                .unwrap();
2379
2380            assert_eq!(remaining, vec![deltas[1]]);
2381        });
2382    }
2383
2384    #[rstest]
2385    fn test_catalog_python_delta_write_errors_and_empty_input() {
2386        Python::initialize();
2387        let directory = TempDir::new().unwrap();
2388
2389        Python::attach(|py| {
2390            let catalog = PyParquetDataCatalog::py_new(
2391                directory.path().to_str().unwrap(),
2392                None,
2393                None,
2394                None,
2395                None,
2396            )
2397            .unwrap();
2398            let catalog = Bound::new(py, catalog).unwrap();
2399            let path: String = catalog
2400                .call_method1("write_order_book_deltas", (Vec::<OrderBookDelta>::new(),))
2401                .unwrap()
2402                .extract()
2403                .unwrap();
2404            let delta = OrderBookDelta::clear(
2405                InstrumentId::from("ETH/USDT.BINANCE"),
2406                17,
2407                23.into(),
2408                29.into(),
2409            );
2410
2411            let earlier = OrderBookDelta {
2412                ts_init: 19.into(),
2413                ..delta
2414            };
2415
2416            let error = catalog
2417                .call_method1("write_order_book_deltas", (vec![delta, earlier],))
2418                .unwrap_err();
2419            let loaded: Vec<OrderBookDelta> = catalog
2420                .call_method0("query_order_book_deltas")
2421                .unwrap()
2422                .extract()
2423                .unwrap();
2424
2425            assert_eq!(path, "");
2426            assert!(error.is_instance_of::<PyIOError>(py));
2427            assert_eq!(
2428                error.value(py).str().unwrap().to_str().unwrap(),
2429                "Failed to write order book deltas: nautilus_model_data_delta_order_book_delta timestamps must be in ascending order"
2430            );
2431            assert_eq!(loaded, Vec::<OrderBookDelta>::new());
2432        });
2433    }
2434
2435    #[rstest]
2436    fn test_concurrent_delta_writes_reject_overlapping_intervals() {
2437        Python::initialize();
2438        let directory = TempDir::new().unwrap();
2439        let mut catalog = PyParquetDataCatalog::py_new(
2440            directory.path().to_str().unwrap(),
2441            None,
2442            None,
2443            None,
2444            None,
2445        )
2446        .unwrap();
2447        let instrument_id = InstrumentId::from("ETH/USDT.BINANCE");
2448        let first = vec![
2449            OrderBookDelta::clear(instrument_id, 11, 17.into(), 23.into()),
2450            OrderBookDelta::clear(instrument_id, 29, 31.into(), 37.into()),
2451        ];
2452        let second = vec![
2453            OrderBookDelta::clear(instrument_id, 41, 43.into(), 37.into()),
2454            OrderBookDelta::clear(instrument_id, 47, 53.into(), 59.into()),
2455        ];
2456        let start = Barrier::new(2);
2457
2458        let (first_result, second_result) = thread::scope(|scope| {
2459            let first_thread = scope.spawn(|| {
2460                start.wait();
2461                Python::attach(|py| {
2462                    catalog.write_order_book_deltas(py, first.clone(), None, None, false)
2463                })
2464            });
2465
2466            let second_thread = scope.spawn(|| {
2467                start.wait();
2468                Python::attach(|py| {
2469                    catalog.write_order_book_deltas(py, second.clone(), None, None, false)
2470                })
2471            });
2472
2473            (first_thread.join().unwrap(), second_thread.join().unwrap())
2474        });
2475
2476        let (expected, error) = match (first_result, second_result) {
2477            (Ok(_), Err(e)) => (first, e),
2478            (Err(e), Ok(_)) => (second, e),
2479            results => {
2480                panic!("Expected one successful write and one overlap error, received {results:?}")
2481            }
2482        };
2483
2484        let loaded = catalog
2485            .inner
2486            .query::<OrderBookDelta>(None, None, None, None, None, true)
2487            .unwrap();
2488
2489        Python::attach(|py| {
2490            assert!(error.is_instance_of::<PyIOError>(py));
2491            assert!(
2492                error
2493                    .value(py)
2494                    .str()
2495                    .unwrap()
2496                    .to_str()
2497                    .unwrap()
2498                    .contains("would create non-disjoint intervals")
2499            );
2500        });
2501
2502        assert_eq!(loaded, expected);
2503    }
2504
2505    #[rstest]
2506    fn test_poisoned_mutation_lock_returns_python_io_error() {
2507        Python::initialize();
2508        let directory = TempDir::new().unwrap();
2509        let catalog = PyParquetDataCatalog::py_new(
2510            directory.path().to_str().unwrap(),
2511            None,
2512            None,
2513            None,
2514            None,
2515        )
2516        .unwrap();
2517
2518        let poisoned = thread::scope(|scope| {
2519            scope
2520                .spawn(|| {
2521                    let _guard = catalog.mutation_lock.lock().unwrap();
2522                    panic!("native mutation failed");
2523                })
2524                .join()
2525                .is_err()
2526        });
2527
2528        let error = Python::attach(|py| {
2529            catalog
2530                .write_order_book_deltas(py, Vec::new(), None, None, false)
2531                .unwrap_err()
2532        });
2533
2534        assert!(poisoned);
2535        Python::attach(|py| {
2536            assert!(error.is_instance_of::<PyIOError>(py));
2537            assert_eq!(
2538                error.value(py).str().unwrap().to_str().unwrap(),
2539                "Failed to write order book deltas: catalog mutation lock is poisoned"
2540            );
2541        });
2542    }
2543
2544    #[rstest]
2545    fn test_mutation_wait_releases_gil() {
2546        Python::initialize();
2547        let directory = TempDir::new().unwrap();
2548        let catalog = PyParquetDataCatalog::py_new(
2549            directory.path().to_str().unwrap(),
2550            None,
2551            None,
2552            None,
2553            None,
2554        )
2555        .unwrap();
2556        let delta = OrderBookDelta::clear(
2557            InstrumentId::from("ETH/USDT.BINANCE"),
2558            17,
2559            23.into(),
2560            29.into(),
2561        );
2562        let (started_tx, started_rx) = mpsc::channel();
2563        let (available_tx, available_rx) = mpsc::channel();
2564        let guard = catalog.mutation_lock.lock().unwrap();
2565
2566        let (gil_available, path) = thread::scope(|scope| {
2567            let writer = scope.spawn(|| {
2568                Python::attach(|py| {
2569                    started_tx.send(()).unwrap();
2570                    catalog.write_order_book_deltas(py, vec![delta], None, None, false)
2571                })
2572            });
2573
2574            started_rx.recv().unwrap();
2575
2576            let probe = scope.spawn(|| Python::attach(|_| available_tx.send(()).unwrap()));
2577            let gil_available = available_rx.recv_timeout(Duration::from_secs(5)).is_ok();
2578            drop(guard);
2579            let path = writer.join().unwrap().unwrap();
2580            probe.join().unwrap();
2581            (gil_available, path)
2582        });
2583
2584        assert!(
2585            gil_available,
2586            "GIL remained held while waiting for the mutation lock"
2587        );
2588        assert!(catalog.inner.file_exists(&path).unwrap());
2589    }
2590
2591    #[rstest]
2592    #[case::delete_data("delete_data_range")]
2593    #[case::delete_catalog("delete_catalog_range")]
2594    #[case::consolidate_data("consolidate_data_by_period")]
2595    #[case::consolidate_catalog("consolidate_catalog_by_period")]
2596    fn test_custom_maintenance_keeps_callbacks_attached(#[case] method: &str) {
2597        Python::initialize();
2598        register_gil_checked_custom_data();
2599        let directory = TempDir::new().unwrap();
2600
2601        let expected = [11u64, 29, 47]
2602            .into_iter()
2603            .map(|ts| {
2604                let value = GilCheckedCustomData {
2605                    value: ts + 100,
2606                    ts_event: (ts - 3).into(),
2607                    ts_init: ts.into(),
2608                };
2609
2610                let data_type = DataType::new(GilCheckedCustomData::type_name_static(), None, None);
2611                CustomData::new(Arc::new(value), data_type)
2612            })
2613            .collect::<Vec<_>>();
2614
2615        Python::attach(|py| {
2616            let catalog = PyParquetDataCatalog::py_new(
2617                directory.path().to_str().unwrap(),
2618                None,
2619                None,
2620                None,
2621                None,
2622            )
2623            .unwrap();
2624            catalog
2625                .inner
2626                .write_custom_data_batch(&expected, None, None, None)
2627                .unwrap();
2628            let catalog = Bound::new(py, catalog).unwrap();
2629
2630            let data_type = PyNautilusDataType::new(NautilusDataType::Custom {
2631                type_name: GilCheckedCustomData::type_name_static().to_string(),
2632            });
2633
2634            let data_type = Bound::new(py, data_type).unwrap();
2635            CALLBACKS.with(|calls| calls.set((false, false)));
2636
2637            match method {
2638                "delete_data_range" => {
2639                    catalog.call_method1(method, (&data_type, None::<String>, 29, 29))
2640                }
2641                "delete_catalog_range" => catalog.call_method1(method, (29, 29)),
2642                "consolidate_data_by_period" => {
2643                    catalog.call_method1(method, (&data_type, None::<String>, 20))
2644                }
2645                "consolidate_catalog_by_period" => catalog.call_method1(method, (20,)),
2646                _ => unreachable!(),
2647            }
2648            .unwrap();
2649
2650            let callbacks = CALLBACKS.with(Cell::get);
2651            let loaded = catalog
2652                .borrow_mut()
2653                .inner
2654                .query_custom_data_dynamic(
2655                    GilCheckedCustomData::type_name_static(),
2656                    None,
2657                    None,
2658                    None,
2659                    None,
2660                    None,
2661                    false,
2662                )
2663                .unwrap();
2664            let expected = expected
2665                .iter()
2666                .filter(|value| !method.starts_with("delete") || value.ts_init().as_u64() != 29)
2667                .cloned()
2668                .map(Data::Custom)
2669                .collect::<Vec<_>>();
2670
2671            assert_eq!(callbacks, (true, true));
2672            assert_eq!(loaded, expected);
2673        });
2674    }
2675
2676    #[rstest]
2677    #[case::empty(true)]
2678    #[case::nonempty(false)]
2679    fn test_custom_write_prepares_attached_before_native_mutation(#[case] empty: bool) {
2680        Python::initialize();
2681        register_gil_checked_custom_data();
2682        let directory = TempDir::new().unwrap();
2683
2684        Python::attach(|py| {
2685            let catalog = PyParquetDataCatalog::py_new(
2686                directory.path().to_str().unwrap(),
2687                None,
2688                None,
2689                None,
2690                None,
2691            )
2692            .unwrap();
2693            let catalog = Bound::new(py, catalog).unwrap();
2694
2695            let rows = if empty {
2696                Vec::new()
2697            } else {
2698                let value = GilCheckedCustomData {
2699                    value: 113,
2700                    ts_event: 17.into(),
2701                    ts_init: 23.into(),
2702                };
2703
2704                let data_type = DataType::new(GilCheckedCustomData::type_name_static(), None, None);
2705                vec![CustomData::new(Arc::new(value), data_type)]
2706            };
2707
2708            let objects = rows
2709                .iter()
2710                .cloned()
2711                .map(|row| Py::new(py, row).unwrap())
2712                .collect::<Vec<_>>();
2713            CALLBACKS.with(|calls| calls.set((false, false)));
2714            let path: String = catalog
2715                .call_method1("write_custom_data", (objects,))
2716                .unwrap()
2717                .extract()
2718                .unwrap();
2719            let callbacks = CALLBACKS.with(Cell::get);
2720            let loaded = catalog
2721                .borrow_mut()
2722                .inner
2723                .query_custom_data_dynamic(
2724                    GilCheckedCustomData::type_name_static(),
2725                    None,
2726                    None,
2727                    None,
2728                    None,
2729                    None,
2730                    false,
2731                )
2732                .unwrap();
2733
2734            assert_eq!(callbacks, (!empty, false));
2735            assert_eq!(
2736                loaded,
2737                rows.into_iter().map(Data::Custom).collect::<Vec<_>>()
2738            );
2739
2740            if empty {
2741                assert_eq!(path, "");
2742            } else {
2743                assert!(catalog.borrow().inner.file_exists(&path).unwrap());
2744            }
2745        });
2746    }
2747
2748    #[arrow_custom_data]
2749    #[custom_data]
2750    struct GilCheckedCustomData {
2751        value: u64,
2752        ts_event: UnixNanos,
2753        ts_init: UnixNanos,
2754    }
2755
2756    thread_local! {
2757        static CALLBACKS: Cell<(bool, bool)> = const { Cell::new((false, false)) };
2758    }
2759
2760    fn register_gil_checked_custom_data() {
2761        static REGISTER: Once = Once::new();
2762        REGISTER.call_once(|| {
2763            ensure_custom_data_json_registered::<GilCheckedCustomData>().unwrap();
2764            register_arrow(
2765                GilCheckedCustomData::type_name_static(),
2766                Arc::new(GilCheckedCustomData::get_schema(None)),
2767                Box::new(|items| {
2768                    assert_gil_held();
2769                    CALLBACKS.with(|calls| calls.set((true, calls.get().1)));
2770                    Python::attach(|_| {
2771                        let rows = items
2772                            .iter()
2773                            .map(|item| {
2774                                item.as_any()
2775                                    .downcast_ref::<GilCheckedCustomData>()
2776                                    .unwrap()
2777                                    .clone()
2778                            })
2779                            .collect::<Vec<_>>();
2780
2781                        let metadata = rows[0].metadata();
2782                        GilCheckedCustomData::encode_batch(&metadata, &rows)
2783                            .map_err(anyhow::Error::from)
2784                    })
2785                }),
2786                Box::new(|metadata, batch| {
2787                    assert_gil_held();
2788                    CALLBACKS.with(|calls| calls.set((calls.get().0, true)));
2789                    Python::attach(|_| GilCheckedCustomData::decode_data_batch(metadata, batch))
2790                        .map_err(anyhow::Error::from)
2791                }),
2792            )
2793            .unwrap();
2794        });
2795    }
2796
2797    #[derive(Clone, Copy, Debug)]
2798    struct GilCheckedDelta<'a>(OrderBookDelta, Option<&'a Mutex<()>>);
2799
2800    impl HasTsInit for GilCheckedDelta<'_> {
2801        fn ts_init(&self) -> UnixNanos {
2802            self.0.ts_init
2803        }
2804    }
2805
2806    impl HasCatalogDataType for GilCheckedDelta<'_> {
2807        fn catalog_data_type() -> NautilusDataType {
2808            NautilusDataType::OrderBookDelta
2809        }
2810    }
2811
2812    impl ArrowSchemaProvider for GilCheckedDelta<'_> {
2813        fn get_schema(metadata: Option<HashMap<String, String>>) -> Schema {
2814            OrderBookDelta::get_schema(metadata)
2815        }
2816    }
2817
2818    impl EncodeToRecordBatch for GilCheckedDelta<'_> {
2819        fn encode_batch<T>(
2820            metadata: &HashMap<String, String>,
2821            data: &[T],
2822        ) -> Result<RecordBatch, ArrowError>
2823        where
2824            T: Borrow<Self>,
2825        {
2826            assert_gil_released();
2827            let deltas = data.iter().map(|item| item.borrow().0).collect::<Vec<_>>();
2828            OrderBookDelta::encode_batch(metadata, &deltas)
2829        }
2830
2831        fn metadata(&self) -> HashMap<String, String> {
2832            if let Some(lock) = self.1 {
2833                assert!(matches!(lock.try_lock(), Err(TryLockError::WouldBlock)));
2834            }
2835
2836            self.0.metadata()
2837        }
2838    }
2839
2840    impl DecodeTypedFromRecordBatch for GilCheckedDelta<'_> {
2841        fn decode_typed_batch(
2842            metadata: &HashMap<String, String>,
2843            record_batch: RecordBatch,
2844        ) -> Result<Vec<Self>, EncodingError> {
2845            assert_gil_released();
2846            OrderBookDelta::decode_typed_batch(metadata, record_batch)
2847                .map(|deltas| deltas.into_iter().map(|delta| Self(delta, None)).collect())
2848        }
2849    }
2850
2851    fn assert_gil_released() {
2852        // SAFETY: Python is initialized before this check, which does not require the GIL.
2853        assert_eq!(unsafe { pyo3::ffi::PyGILState_Check() }, 0);
2854    }
2855
2856    fn assert_gil_held() {
2857        // SAFETY: Python is initialized before this check, which does not require the GIL.
2858        assert_eq!(unsafe { pyo3::ffi::PyGILState_Check() }, 1);
2859    }
2860}