Skip to main content

nautilus_persistence/backend/parquet/catalog/
query.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Parquet catalog query paths and typed query wrappers.
17
18#![expect(
19    clippy::missing_errors_doc,
20    reason = "query methods forward DataFusion errors"
21)]
22
23use nautilus_model::instruments::NautilusInstrumentType;
24use nautilus_serialization::arrow::{
25    catalog_identifier_from_metadata, instrument::decode_instrument_any_batch,
26    record_batch_with_identifier_column,
27};
28
29use super::{
30    ArrowSchemaProvider, Bar, CustomDataDecoder, Data, DecodeDataFromRecordBatch,
31    DecodeTypedFromRecordBatch, FundingRateUpdate, HasCatalogDataType, HasTsInit, HashMap,
32    INSTRUMENT_PATH_PREFIXES, InstrumentAny, InstrumentClose, NautilusDataType, OptionGreeks,
33    OrderBookDelta, OrderBookDepth, ParquetDataCatalog, Path, QuoteTick, RecordBatch, TradeTick,
34    UnixNanos, build_query, catalog_record_batch_to_display, datafusion,
35    decode_object_store_segment, extract_bar_type_instrument_id, extract_identifier_from_path,
36    extract_sql_safe_filename, filter_instruments_for_request_range, instrument_path_prefix,
37    is_monotonically_increasing_by_init, make_object_store_path, make_sql_safe_identifier,
38    parquet_data_path_prefix, parse_filename_timestamps, query_intersects_filename,
39    read_parquet_from_object_store, read_parquet_schema_from_object_store,
40    session::{MergedPages, TypedPages, decode_typed_pages},
41    urisafe_instrument_id,
42};
43use crate::{
44    catalog::types::{
45        CatalogDataType, parquet_catalog_data_type_path_prefixes,
46        parquet_catalog_data_type_table_stem,
47    },
48    common::arrow::{empty_display_batch_with_identifier, validate_catalog_schema},
49};
50
51impl ParquetDataCatalog {
52    /// Queries instruments from the catalog.
53    ///
54    /// Instruments are stored under v1-compatible concrete instrument type folders:
55    /// `data/{instrument_type}/{instrument_id}/`.
56    ///
57    /// # Parameters
58    ///
59    /// - `instrument_ids`: Optional list of instrument IDs to filter by. If `None`, queries all instruments.
60    ///
61    /// # Returns
62    ///
63    /// Returns a vector of `InstrumentAny` instances, or an error if the operation fails.
64    ///
65    /// # Errors
66    ///
67    /// Returns an error if:
68    /// - File discovery fails.
69    /// - File reading fails.
70    /// - Data deserialization fails.
71    ///
72    /// # Examples
73    ///
74    /// ```rust,no_run
75    /// use nautilus_model::instruments::InstrumentAny;
76    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
77    ///
78    /// let mut catalog = ParquetDataCatalog::new(
79    ///     std::path::Path::new("/tmp/nautilus_data"),
80    ///     None,
81    ///     None,
82    ///     None,
83    ///     None,
84    /// );
85    ///
86    /// // Query all instruments
87    /// let instruments = catalog.query_instruments(None)?;
88    ///
89    /// // Query specific instruments
90    /// let instrument_ids = vec!["EUR/USD.SIM".to_string()];
91    /// let instruments = catalog.query_instruments(Some(&instrument_ids))?;
92    /// # Ok::<(), anyhow::Error>(())
93    /// ```
94    pub fn query_instruments(
95        &self,
96        instrument_ids: Option<&[String]>,
97    ) -> anyhow::Result<Vec<InstrumentAny>> {
98        self.query_instruments_filtered(instrument_ids, None, None)
99    }
100
101    /// Queries instruments from the catalog with optional timestamp filtering.
102    ///
103    /// This reads all matching parquet files under
104    /// `data/{instrument_type}/{instrument_id}/`, decodes the records back to
105    /// `InstrumentAny`, and filters them by `ts_init` when a range is provided.
106    pub fn query_instruments_filtered(
107        &self,
108        instrument_ids: Option<&[String]>,
109        start: Option<UnixNanos>,
110        end: Option<UnixNanos>,
111    ) -> anyhow::Result<Vec<InstrumentAny>> {
112        let instrument_files = self.discover_instrument_files(instrument_ids, end, None)?;
113        self.decode_instrument_files(instrument_files, start, end)
114    }
115
116    /// Queries instruments from the catalog with optional timestamp and SQL filtering.
117    ///
118    /// When `where_clause` is provided, the predicate is applied through DataFusion
119    /// before instrument records are decoded.
120    pub fn query_instruments_filtered_with_where(
121        &mut self,
122        instrument_ids: Option<&[String]>,
123        start: Option<UnixNanos>,
124        end: Option<UnixNanos>,
125        where_clause: Option<&str>,
126    ) -> anyhow::Result<Vec<InstrumentAny>> {
127        self.query_instruments_filtered_with_where_and_type(
128            instrument_ids,
129            start,
130            end,
131            where_clause,
132            None,
133        )
134    }
135
136    /// Queries instruments from the catalog with optional timestamp, SQL, and instrument type
137    /// filtering.
138    ///
139    /// When `instrument_type` is provided, only that instrument class directory is read.
140    pub fn query_instruments_filtered_with_where_and_type(
141        &mut self,
142        instrument_ids: Option<&[String]>,
143        start: Option<UnixNanos>,
144        end: Option<UnixNanos>,
145        where_clause: Option<&str>,
146        instrument_type: Option<&NautilusInstrumentType>,
147    ) -> anyhow::Result<Vec<InstrumentAny>> {
148        let Some(where_clause) = where_clause else {
149            let instrument_files =
150                self.discover_instrument_files(instrument_ids, end, instrument_type)?;
151            return self.decode_instrument_files(instrument_files, start, end);
152        };
153
154        self.clear_session_tables();
155        self.register_remote_object_store()?;
156
157        let mut all_instruments = Vec::new();
158        let instrument_files =
159            self.discover_instrument_files(instrument_ids, end, instrument_type)?;
160
161        for (index, file_path) in instrument_files.into_iter().enumerate() {
162            let object_path = self.to_object_path_parsed(&file_path)?;
163
164            let (_, builder_schema) = self.execute_async(|| async {
165                read_parquet_from_object_store(self.object_store.clone(), &object_path).await
166            })?;
167
168            validate_catalog_schema(&builder_schema)?;
169            let metadata = builder_schema.metadata().clone();
170            let target_schema = InstrumentAny::get_schema(Some(metadata.clone()));
171
172            let table_name = format!(
173                "instruments_{}_{}",
174                index,
175                extract_sql_safe_filename(&file_path)
176            );
177
178            // The range filter below applies `start` to keep the latest pre-start definition
179            let query = build_query(&table_name, None, end, Some(where_clause));
180            let resolved_path = self.resolve_path_for_datafusion(&file_path);
181            let batches = self.session.collect_parquet_files_batches(
182                &table_name,
183                vec![resolved_path],
184                Some(&query),
185            )?;
186
187            for batch in batches {
188                let batch = datafusion::cast_record_batch_to_schema(&batch, &target_schema)?;
189                all_instruments.extend(decode_instrument_any_batch(&metadata, &batch)?);
190            }
191        }
192
193        Ok(filter_instruments_for_request_range(
194            all_instruments,
195            start,
196            end,
197        ))
198    }
199
200    /// Discovers instrument parquet files under `data/{instrument_type}/{instrument_id}/`,
201    /// filtered by instrument IDs and an optional `end` timestamp, sorted by path.
202    fn discover_instrument_files(
203        &self,
204        instrument_ids: Option<&[String]>,
205        end: Option<UnixNanos>,
206        instrument_type: Option<&NautilusInstrumentType>,
207    ) -> anyhow::Result<Vec<String>> {
208        let base_dir = make_object_store_path(&self.base_path, ["data"]);
209        let end_u64 = end.map(|ts| ts.as_u64());
210        let list_result = self.list_objects(&base_dir)?;
211
212        let mut instrument_files = Vec::new();
213
214        for object in list_result {
215            let path_str = object.location.to_string();
216            if !path_str.ends_with(".parquet") {
217                continue;
218            }
219
220            let directory = path_str
221                .rsplit_once('/')
222                .map_or(path_str.as_str(), |(directory, _)| directory);
223
224            let (Some(type_dir), Some(identifier)) =
225                self.extract_data_cls_and_identifier_from_path(directory)?
226            else {
227                continue;
228            };
229
230            let type_dir = decode_object_store_segment(&type_dir);
231            if !is_parquet_instrument_type_prefix(&type_dir)
232                || instrument_type.is_some_and(|value| instrument_path_prefix(value) != type_dir)
233            {
234                continue;
235            }
236
237            let instrument_id_dir = decode_object_store_segment(&identifier);
238
239            if let Some(ids) = instrument_ids
240                && !ids
241                    .iter()
242                    .map(|id| urisafe_instrument_id(id))
243                    .any(|x| x.as_str() == urisafe_instrument_id(&instrument_id_dir))
244            {
245                continue;
246            }
247
248            let include_file = if path_str.ends_with("/instrument.parquet") {
249                true
250            } else if let Some((file_start, _)) = parse_filename_timestamps(&path_str) {
251                end_u64.is_none_or(|end| file_start <= end)
252            } else {
253                // Include files with nonstandard names rather than silently dropping
254                // instruments written by external or older tooling.
255                log::warn!(
256                    "Including instrument file with unparsable interval filename: {path_str}"
257                );
258                true
259            };
260
261            if include_file {
262                instrument_files.push(path_str);
263            }
264        }
265
266        instrument_files.sort();
267        Ok(instrument_files)
268    }
269
270    fn decode_instrument_files(
271        &self,
272        instrument_files: Vec<String>,
273        start: Option<UnixNanos>,
274        end: Option<UnixNanos>,
275    ) -> anyhow::Result<Vec<InstrumentAny>> {
276        let mut instruments = Vec::new();
277
278        for file_path in instrument_files {
279            let object_path = self.to_object_path_parsed(&file_path)?;
280
281            let (batches, builder_schema) = self.execute_async(|| async {
282                read_parquet_from_object_store(self.object_store.clone(), &object_path).await
283            })?;
284
285            validate_catalog_schema(&builder_schema)?;
286            let metadata = builder_schema.metadata().clone();
287            let target_schema = InstrumentAny::get_schema(Some(metadata.clone()));
288
289            for batch in batches {
290                let batch = datafusion::cast_record_batch_to_schema(&batch, &target_schema)?;
291                instruments.extend(decode_instrument_any_batch(&metadata, &batch)?);
292            }
293        }
294
295        Ok(filter_instruments_for_request_range(
296            instruments,
297            start,
298            end,
299        ))
300    }
301
302    /// Queries data of type `T` from the catalog.
303    ///
304    /// # Type Parameters
305    ///
306    /// - `T`: The data type to query and return, which selects the catalog directory and decoder.
307    ///
308    /// # Parameters
309    ///
310    /// - `identifiers`: Optional list of identifiers to filter by. Can be `instrument_id` strings (e.g., "EUR/USD.SIM")
311    ///   or `bar_type` strings (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL"). If `None`, queries all identifiers.
312    ///   For bars, partial matching is supported (e.g., "EUR/USD.SIM" will match "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
313    /// - `start`: Optional start timestamp for filtering (inclusive). If `None`, queries from the beginning.
314    /// - `end`: Optional end timestamp for filtering (inclusive). If `None`, queries to the end.
315    /// - `where_clause`: Optional SQL WHERE clause for additional filtering. Use standard SQL syntax
316    ///   with column names matching the Parquet schema (e.g., "`bid_price` > 1.2000", "volume > 1000").
317    /// - `files`: Optional list of catalog files to read in place of the files matching the filters.
318    /// - `optimize_file_loading`: Whether to register each parent directory as one table rather
319    ///   than each file as its own table. When `true`, every file in those directories is read,
320    ///   including files not listed in `files`. A directory whose files cannot merge into one
321    ///   schema, such as one instrument written at two precisions, instead registers only the
322    ///   files selected by `files` or the filters, each as its own table.
323    ///
324    /// # Returns
325    ///
326    /// Returns a vector of the specific data type `T`, sorted by timestamp. The vector will be
327    /// empty if no data matches the query criteria.
328    ///
329    /// # Errors
330    ///
331    /// Returns an error if:
332    /// - The underlying query execution fails.
333    /// - Record batch decoding fails.
334    /// - A file uses a legacy catalog schema.
335    /// - Object store access fails.
336    /// - Invalid WHERE clause syntax is provided.
337    ///
338    /// # Performance Considerations
339    ///
340    /// - Use specific instrument IDs and time ranges to minimize data scanning.
341    /// - WHERE clauses are pushed down to Parquet readers when possible.
342    /// - Results are automatically sorted by timestamp during collection.
343    /// - Memory usage scales with the amount of data returned.
344    ///
345    /// # Examples
346    ///
347    /// ```rust,no_run
348    /// use nautilus_core::UnixNanos;
349    /// use nautilus_model::data::{Bar, QuoteTick, TradeTick};
350    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
351    ///
352    /// let mut catalog = ParquetDataCatalog::new(
353    ///     std::path::Path::new("/tmp/nautilus_data"),
354    ///     None,
355    ///     None,
356    ///     None,
357    ///     None,
358    /// );
359    ///
360    /// // Query all quotes for a specific instrument
361    /// let quotes: Vec<QuoteTick> = catalog.query(
362    ///     Some(vec!["EUR/USD.SIM".to_string()]),
363    ///     None,
364    ///     None,
365    ///     None,
366    ///     None,
367    ///     true,
368    /// )?;
369    ///
370    /// // Query trades within a specific time range
371    /// let trades: Vec<TradeTick> = catalog.query(
372    ///     Some(vec!["BTC/USD.SIM".to_string()]),
373    ///     Some(UnixNanos::from(1609459200000000000)),
374    ///     Some(UnixNanos::from(1609545600000000000)),
375    ///     None,
376    ///     None,
377    ///     true,
378    /// )?;
379    ///
380    /// // Query bars with volume filter (using instrument_id - partial match for bar_type)
381    /// let bars: Vec<Bar> = catalog.query(
382    ///     Some(vec!["AAPL.NASDAQ".to_string()]),
383    ///     None,
384    ///     None,
385    ///     Some("volume > 1000000"),
386    ///     None,
387    ///     true,
388    /// )?;
389    ///
390    /// // Query bars with specific bar_type
391    /// let bars: Vec<Bar> = catalog.query(
392    ///     Some(vec!["AAPL.NASDAQ-1-MINUTE-LAST-EXTERNAL".to_string()]),
393    ///     None,
394    ///     None,
395    ///     None,
396    ///     None,
397    ///     true,
398    /// )?;
399    ///
400    /// // Query multiple instruments with price filter
401    /// let quotes: Vec<QuoteTick> = catalog.query(
402    ///     Some(vec!["EUR/USD.SIM".to_string(), "GBP/USD.SIM".to_string()]),
403    ///     None,
404    ///     None,
405    ///     Some("bid_price > 1.2000 AND ask_price < 1.3000"),
406    ///     None,
407    ///     true,
408    /// )?;
409    /// # Ok::<(), anyhow::Error>(())
410    /// ```
411    pub fn query<T>(
412        &mut self,
413        identifiers: Option<Vec<String>>,
414        start: Option<UnixNanos>,
415        end: Option<UnixNanos>,
416        where_clause: Option<&str>,
417        files: Option<Vec<String>>,
418        optimize_file_loading: bool,
419    ) -> anyhow::Result<Vec<T>>
420    where
421        T: DecodeTypedFromRecordBatch + HasCatalogDataType + HasTsInit,
422    {
423        self.clear_session_tables();
424
425        self.register_remote_object_store()?;
426
427        let data_type = T::catalog_data_type();
428        let path_prefix = parquet_data_path_prefix(&data_type);
429
430        let files_list = if let Some(files) = files {
431            files
432        } else {
433            self.query_files(
434                &CatalogDataType::Data(data_type.clone()),
435                identifiers,
436                start,
437                end,
438            )?
439        };
440
441        let table_prefix = make_sql_safe_identifier(path_prefix.as_ref());
442        let tables =
443            self.resolve_tables_for_datafusion(&table_prefix, &files_list, optimize_file_loading)?;
444        let mut all_records = Vec::new();
445
446        for table in tables {
447            let query = build_query(&table.name, start, end, where_clause);
448            let batches = self.session.collect_parquet_files_batches(
449                &table.name,
450                vec![table.path],
451                Some(&query),
452            )?;
453
454            all_records.extend(self.convert_record_batches_to_typed::<T>(batches)?);
455        }
456
457        if !is_monotonically_increasing_by_init(&all_records) {
458            all_records.sort_by_key(HasTsInit::ts_init);
459        }
460
461        Ok(all_records)
462    }
463
464    pub(super) fn query_typed_pages<T>(
465        &mut self,
466        identifiers: Option<Vec<String>>,
467        start: Option<UnixNanos>,
468        end: Option<UnixNanos>,
469        where_clause: Option<&str>,
470        files: Option<Vec<String>>,
471        optimize_file_loading: bool,
472    ) -> anyhow::Result<TypedPages<T>>
473    where
474        T: DecodeTypedFromRecordBatch + HasCatalogDataType + HasTsInit + Send + 'static,
475    {
476        self.clear_session_tables();
477        self.register_remote_object_store()?;
478        let data_type = T::catalog_data_type();
479
480        let files = match files {
481            Some(files) => files,
482            None => self.query_files(&CatalogDataType::Data(data_type), identifiers, start, end)?,
483        };
484
485        let tables =
486            self.resolve_tables_for_datafusion("parquet", &files, optimize_file_loading)?;
487        let mut sources = Vec::with_capacity(tables.len());
488        for table in tables {
489            let sql = build_query(&table.name, start, end, where_clause);
490            let stream = self.session.parquet_files_batch_stream(
491                &table.name,
492                vec![table.path],
493                Some(&sql),
494            )?;
495            let pages = decode_typed_pages::<T>(stream);
496            sources.push(
497                Box::new(datafusion::BlockingBatchStream::from_stream_with_runtime(
498                    pages,
499                    &self.session.runtime,
500                )) as TypedPages<T>,
501            );
502        }
503
504        Ok(Box::new(MergedPages::new(sources, self.batch_size)))
505    }
506
507    /// Queries raw catalog Arrow record batches for any supported record table.
508    ///
509    /// # Errors
510    ///
511    /// Returns an error if file discovery or DataFusion query execution fails.
512    pub fn query_record_batches(
513        &mut self,
514        data_type: &CatalogDataType,
515        identifier: Option<String>,
516        start: Option<UnixNanos>,
517        end: Option<UnixNanos>,
518        where_clause: Option<&str>,
519        optimize_file_loading: bool,
520    ) -> anyhow::Result<Vec<RecordBatch>> {
521        self.clear_session_tables();
522        self.register_remote_object_store()?;
523
524        let identifiers = identifier.map(|value| vec![value]);
525        let files_list = self.query_files(data_type, identifiers, start, end)?;
526        let mut record_batches = Vec::new();
527        let table_prefix =
528            make_sql_safe_identifier(&parquet_catalog_data_type_table_stem(data_type));
529
530        let tables =
531            self.resolve_tables_for_datafusion(&table_prefix, &files_list, optimize_file_loading)?;
532
533        for table in tables {
534            let query = build_query(&table.name, start, end, where_clause);
535            record_batches.extend(self.session.collect_parquet_files_batches(
536                &table.name,
537                vec![table.path],
538                Some(&query),
539            )?);
540        }
541
542        Ok(record_batches)
543    }
544
545    /// Queries raw catalog batches and converts them to display-friendly Arrow batches.
546    ///
547    /// # Errors
548    ///
549    /// Returns an error if file discovery, DataFusion query execution, or catalog display
550    /// conversion fails.
551    pub fn query_display_record_batches(
552        &mut self,
553        data_type: &NautilusDataType,
554        identifiers: Option<Vec<String>>,
555        start: Option<UnixNanos>,
556        end: Option<UnixNanos>,
557        where_clause: Option<&str>,
558        optimize_file_loading: bool,
559    ) -> anyhow::Result<Vec<RecordBatch>> {
560        self.clear_session_tables();
561        self.register_remote_object_store()?;
562
563        let data_path_prefix = parquet_data_path_prefix(data_type);
564        let files_list = self.query_files(
565            &CatalogDataType::Data(data_type.clone()),
566            identifiers,
567            start,
568            end,
569        )?;
570        let mut display_batches = Vec::new();
571        let table_prefix = make_sql_safe_identifier(data_path_prefix.as_ref());
572        let tables =
573            self.resolve_tables_for_datafusion(&table_prefix, &files_list, optimize_file_loading)?;
574
575        for table in tables {
576            let path_identifier = display_identifier(data_type, &table.directory);
577            let query = build_query(&table.name, start, end, where_clause);
578            let batches = self.session.collect_parquet_files_batches(
579                &table.name,
580                vec![table.path],
581                Some(&query),
582            )?;
583
584            for batch in batches {
585                let identifier =
586                    display_batch_identifier(data_type, &batch, path_identifier.as_deref());
587                let batch = record_batch_with_identifier_column(batch, identifier.as_deref())?;
588                let metadata = batch.schema().metadata().clone();
589                display_batches.push(catalog_record_batch_to_display(
590                    data_type, &metadata, &batch,
591                )?);
592            }
593        }
594
595        if display_batches.is_empty() {
596            display_batches.push(empty_display_batch_with_identifier(data_type)?);
597        }
598
599        Ok(display_batches)
600    }
601
602    /// Queries concrete catalog identifiers for matching data rows.
603    pub fn query_identifiers(
604        &mut self,
605        data_type: &CatalogDataType,
606        identifiers: Option<Vec<String>>,
607        start: Option<UnixNanos>,
608        end: Option<UnixNanos>,
609        where_clause: Option<&str>,
610        _optimize_file_loading: bool,
611    ) -> anyhow::Result<Vec<String>> {
612        self.clear_session_tables();
613        self.register_remote_object_store()?;
614
615        let files_list = self.query_files(data_type, identifiers, start, end)?;
616        let table_prefix =
617            make_sql_safe_identifier(&parquet_catalog_data_type_table_stem(data_type));
618        let tables = self.resolve_tables_for_datafusion(&table_prefix, &files_list, true)?;
619        let mut identifiers = Vec::new();
620
621        for table in tables {
622            let identifier = dir_identifier(&table.directory);
623            let query = format!(
624                "{} LIMIT 1",
625                build_query(&table.name, start, end, where_clause)
626            );
627            let batches = self.session.collect_parquet_files_batches(
628                &table.name,
629                vec![table.path],
630                Some(&query),
631            )?;
632
633            if batches.iter().any(|batch| batch.num_rows() != 0) {
634                identifiers.push(decode_object_store_segment(&identifier));
635            }
636        }
637
638        identifiers.sort();
639        identifiers.dedup();
640        Ok(identifiers)
641    }
642
643    // Registers each directory table up front, so a directory whose file schemas cannot merge
644    // (such as one instrument written at two precisions) falls back to one table per listed file.
645    // File tables register when queried.
646    fn resolve_tables_for_datafusion(
647        &mut self,
648        table_prefix: &str,
649        files: &[String],
650        optimize_file_loading: bool,
651    ) -> anyhow::Result<Vec<DataFusionTable>> {
652        let table_name = |index: usize| format!("{table_prefix}_{index}");
653        let mut tables = Vec::new();
654
655        if !optimize_file_loading {
656            for file in files {
657                tables.push(self.file_table(table_name(tables.len()), file)?);
658            }
659            return Ok(tables);
660        }
661
662        // Deterministic registration order so equal-ts_init tie order is reproducible.
663        for directory in parent_directories(files) {
664            let name = table_name(tables.len());
665            let path = self.resolve_directory_for_datafusion(&directory);
666
667            if !self.has_pending_consolidation(&directory)?
668                && self
669                    .session
670                    .try_register_parquet_files_table(&name, vec![path.clone()])?
671            {
672                tables.push(DataFusionTable {
673                    name,
674                    directory,
675                    path,
676                });
677                continue;
678            }
679
680            for file in files
681                .iter()
682                .filter(|file| parent_directory(file).as_ref() == Some(&directory))
683            {
684                tables.push(self.file_table(table_name(tables.len()), file)?);
685            }
686        }
687
688        Ok(tables)
689    }
690
691    fn file_table(&self, name: String, file: &str) -> anyhow::Result<DataFusionTable> {
692        let directory = parent_directory(file)
693            .ok_or_else(|| anyhow::anyhow!("Cannot extract directory from '{file}'"))?;
694        let path = self.resolve_path_for_datafusion(file);
695
696        Ok(DataFusionTable {
697            name,
698            directory,
699            path,
700        })
701    }
702
703    /// Queries custom data dynamically by type name.
704    ///
705    /// This method allows querying custom data types without compile-time knowledge of the type.
706    /// It uses dynamic schema decoding based on the type name stored in metadata.
707    ///
708    /// # Parameters
709    ///
710    /// - `type_name`: The name of the custom data type to query.
711    /// - `identifiers`: Optional list of instrument identifiers to filter by.
712    /// - `start`: Optional start timestamp for filtering.
713    /// - `end`: Optional end timestamp for filtering.
714    /// - `where_clause`: Optional SQL WHERE clause for additional filtering.
715    /// - `files`: Optional list of specific files to query.
716    /// - `_optimize_file_loading`: Whether to optimize file loading (currently unused).
717    ///
718    /// # Returns
719    ///
720    /// Returns a vector of `Data` enum variants containing the custom data.
721    ///
722    /// # Errors
723    ///
724    /// Returns an error if:
725    /// - File discovery fails.
726    /// - Data decoding fails.
727    /// - Query execution fails.
728    #[expect(clippy::too_many_arguments)]
729    pub fn query_custom_data_dynamic(
730        &mut self,
731        type_name: &str,
732        identifiers: Option<&[String]>,
733        start: Option<UnixNanos>,
734        end: Option<UnixNanos>,
735        where_clause: Option<&str>,
736        files: Option<Vec<String>>,
737        _optimize_file_loading: bool,
738    ) -> anyhow::Result<Vec<Data>> {
739        self.clear_session_tables();
740
741        self.register_remote_object_store()?;
742
743        let files = if let Some(f) = files {
744            f.into_iter()
745                .map(|p| self.to_object_path_parsed(&p).map(|op| op.to_string()))
746                .collect::<anyhow::Result<Vec<_>>>()?
747        } else {
748            self.list_parquet_files_with_criteria(
749                &CatalogDataType::Data(NautilusDataType::Custom {
750                    type_name: type_name.to_string(),
751                }),
752                identifiers,
753                start,
754                end,
755            )?
756        };
757
758        if files.is_empty() {
759            return Ok(Vec::new());
760        }
761
762        // Use CustomDataDecoder for all custom data. Pass type_name so decode can look up
763        // the type when Parquet/DataFusion does not preserve schema metadata. Callers must
764        // ensure Rust custom types are registered via ensure_custom_data_registered::<T>().
765        let mut lookup_metadata = HashMap::new();
766        lookup_metadata.insert("type_name".to_string(), type_name.to_string());
767        let registered_schema = CustomDataDecoder::get_schema(Some(lookup_metadata.clone()));
768        registered_schema.field_with_name("ts_init").map_err(|_| {
769            anyhow::anyhow!(
770                "custom data type '{type_name}' is not registered with an Arrow schema containing ts_init; \
771                 call ensure_custom_data_registered::<T>() before querying"
772            )
773        })?;
774
775        let mut all_data = Vec::new();
776
777        for (index, file) in files.into_iter().enumerate() {
778            let object_path = self.to_object_path_parsed(&file)?;
779
780            let mut decode_metadata = self.execute_async(|| async {
781                let schema =
782                    read_parquet_schema_from_object_store(self.object_store.clone(), &object_path)
783                        .await?;
784                validate_catalog_schema(&schema)?;
785                Ok::<HashMap<String, String>, anyhow::Error>(schema.metadata().clone())
786            })?;
787
788            decode_metadata.extend(lookup_metadata.clone());
789            let identifier = extract_identifier_from_path(&file)
790                .ok_or_else(|| anyhow::anyhow!("Cannot extract identifier from path '{file}'"))?;
791            let safe_type_name = make_sql_safe_identifier(type_name);
792            let safe_sql_identifier = make_sql_safe_identifier(identifier);
793            let safe_filename = extract_sql_safe_filename(&file);
794            let table_name =
795                format!("custom_{safe_type_name}_{safe_sql_identifier}_{safe_filename}_{index}");
796            let resolved_path = self.resolve_path_for_datafusion(&file);
797            let sql_query = build_query(&table_name, start, end, where_clause);
798
799            // Use schemaless registration so DataFusion preserves the parquet file's
800            // schema metadata (e.g. `bar_type`) on output batches, since the
801            // explicit-schema variant strips per-batch metadata that decoders rely on.
802            let batches = self.session.collect_parquet_files_batches(
803                &table_name,
804                vec![resolved_path],
805                Some(&sql_query),
806            )?;
807
808            for batch in batches {
809                all_data.extend(CustomDataDecoder::decode_data_batch(
810                    &decode_metadata,
811                    batch,
812                )?);
813            }
814        }
815
816        all_data.sort_by_key(HasTsInit::ts_init);
817        Ok(all_data)
818    }
819
820    /// Queries all Parquet files for a specific data type and optional instrument IDs.
821    ///
822    /// This method finds all Parquet files that match the specified criteria and returns
823    /// their full URIs. The files are filtered by data type, instrument IDs (if provided),
824    /// and timestamp range (if provided).
825    ///
826    /// # Parameters
827    ///
828    /// - `data_type`: The stored family to read.
829    /// - `identifiers`: Optional list of identifiers to filter by. Can be `instrument_id` strings
830    ///   (e.g., "EUR/USD.SIM") or `bar_type` strings (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
831    ///   For bars, partial matching is supported.
832    /// - `start`: Optional start timestamp to filter files by their time range.
833    /// - `end`: Optional end timestamp to filter files by their time range.
834    ///
835    /// # Returns
836    ///
837    /// Returns a vector of file URI strings that match the query criteria,
838    /// or an error if the query fails.
839    ///
840    /// # Errors
841    ///
842    /// Returns an error if:
843    /// - The directory path cannot be constructed.
844    /// - Object store listing operations fail.
845    /// - URI reconstruction fails.
846    ///
847    /// # Examples
848    ///
849    /// ```rust,no_run
850    /// use nautilus_core::UnixNanos;
851    /// use nautilus_model::data::NautilusDataType;
852    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
853    ///
854    /// let mut catalog = ParquetDataCatalog::new(
855    ///     std::path::Path::new("/tmp/nautilus_data"),
856    ///     None,
857    ///     None,
858    ///     None,
859    ///     None,
860    /// );
861    ///
862    /// // Query all quote files
863    /// let files = catalog.query_files(&NautilusDataType::QuoteTick.into(), None, None, None)?;
864    ///
865    /// // Query trade files for specific instruments within a time range
866    /// let files = catalog.query_files(
867    ///     &NautilusDataType::TradeTick.into(),
868    ///     Some(vec!["BTC/USD.SIM".to_string(), "ETH/USD.SIM".to_string()]),
869    ///     Some(UnixNanos::from(1609459200000000000)),
870    ///     Some(UnixNanos::from(1609545600000000000)),
871    /// )?;
872    /// # Ok::<(), anyhow::Error>(())
873    /// ```
874    pub fn query_files(
875        &self,
876        data_type: &CatalogDataType,
877        identifiers: Option<Vec<String>>,
878        start: Option<UnixNanos>,
879        end: Option<UnixNanos>,
880    ) -> anyhow::Result<Vec<String>> {
881        // Take the identifiers once so every prefix shares them without cloning per directory.
882        let identifiers = identifiers.map(Vec::into_boxed_slice);
883
884        let mut files = Vec::new();
885
886        for prefix in parquet_catalog_data_type_path_prefixes(data_type) {
887            files.extend(self.query_prefix_files(
888                prefix.as_ref(),
889                identifiers.as_deref(),
890                start,
891                end,
892            )?);
893        }
894
895        files.sort();
896        files.dedup();
897
898        Ok(files)
899    }
900
901    fn query_prefix_files(
902        &self,
903        data_cls: &str,
904        identifiers: Option<&[String]>,
905        start: Option<UnixNanos>,
906        end: Option<UnixNanos>,
907    ) -> anyhow::Result<Vec<String>> {
908        let mut files = Vec::new();
909
910        let start_u64 = start.map(|s| s.as_u64());
911        let end_u64 = end.map(|e| e.as_u64());
912
913        let base_dir = self.make_path(data_cls, None)?;
914
915        // Use recursive listing to match Python's glob behavior
916        let list_result = self.list_objects(&base_dir)?;
917
918        let mut file_paths: Vec<String> = list_result
919            .into_iter()
920            .filter_map(|object| {
921                let path_str = object.location.to_string();
922                if path_str.ends_with(".parquet") {
923                    Some(path_str)
924                } else {
925                    None
926                }
927            })
928            .collect();
929
930        if let Some(identifiers) = identifiers {
931            file_paths =
932                filter_identifier_files(file_paths, identifiers, is_parquet_bar_prefix(data_cls));
933        }
934
935        // Apply timestamp filtering
936        file_paths.retain(|file_path| query_intersects_filename(file_path, start_u64, end_u64));
937
938        for file_path in file_paths {
939            files.push(self.path_for_query_list(&file_path));
940        }
941
942        Ok(files)
943    }
944
945    /// Queries quote tick data for the specified instrument(s) and time range.
946    pub fn quote_ticks(
947        &mut self,
948        instrument_ids: Option<Vec<String>>,
949        start: Option<UnixNanos>,
950        end: Option<UnixNanos>,
951    ) -> anyhow::Result<Vec<QuoteTick>> {
952        self.query::<QuoteTick>(instrument_ids, start, end, None, None, true)
953    }
954
955    /// Queries trade tick data for the specified instrument(s) and time range.
956    pub fn trade_ticks(
957        &mut self,
958        instrument_ids: Option<Vec<String>>,
959        start: Option<UnixNanos>,
960        end: Option<UnixNanos>,
961    ) -> anyhow::Result<Vec<TradeTick>> {
962        self.query::<TradeTick>(instrument_ids, start, end, None, None, true)
963    }
964
965    /// Queries bar data for the specified instrument(s) and time range.
966    pub fn bars(
967        &mut self,
968        instrument_ids: Option<Vec<String>>,
969        start: Option<UnixNanos>,
970        end: Option<UnixNanos>,
971    ) -> anyhow::Result<Vec<Bar>> {
972        self.query::<Bar>(instrument_ids, start, end, None, None, true)
973    }
974
975    /// Queries order book delta data for the specified instrument(s) and time range.
976    pub fn order_book_deltas(
977        &mut self,
978        instrument_ids: Option<Vec<String>>,
979        start: Option<UnixNanos>,
980        end: Option<UnixNanos>,
981    ) -> anyhow::Result<Vec<OrderBookDelta>> {
982        self.query::<OrderBookDelta>(instrument_ids, start, end, None, None, true)
983    }
984
985    /// Queries order book depth data for the specified instrument(s) and time range.
986    pub fn order_book_depths(
987        &mut self,
988        instrument_ids: Option<Vec<String>>,
989        start: Option<UnixNanos>,
990        end: Option<UnixNanos>,
991    ) -> anyhow::Result<Vec<OrderBookDepth>> {
992        self.query::<OrderBookDepth>(instrument_ids, start, end, None, None, true)
993    }
994
995    /// Queries funding rate updates for the specified instrument(s) and time range.
996    pub fn funding_rates(
997        &mut self,
998        instrument_ids: Option<Vec<String>>,
999        start: Option<UnixNanos>,
1000        end: Option<UnixNanos>,
1001    ) -> anyhow::Result<Vec<FundingRateUpdate>> {
1002        self.query::<FundingRateUpdate>(instrument_ids, start, end, None, None, true)
1003    }
1004
1005    /// Queries instrument close data for the specified instrument(s) and time range.
1006    pub fn instrument_closes(
1007        &mut self,
1008        instrument_ids: Option<Vec<String>>,
1009        start: Option<UnixNanos>,
1010        end: Option<UnixNanos>,
1011    ) -> anyhow::Result<Vec<InstrumentClose>> {
1012        self.query::<InstrumentClose>(instrument_ids, start, end, None, None, true)
1013    }
1014
1015    /// Queries option greeks data for the specified instrument(s) and time range.
1016    pub fn option_greeks(
1017        &mut self,
1018        instrument_ids: Option<Vec<String>>,
1019        start: Option<UnixNanos>,
1020        end: Option<UnixNanos>,
1021    ) -> anyhow::Result<Vec<OptionGreeks>> {
1022        self.query::<OptionGreeks>(instrument_ids, start, end, None, None, true)
1023    }
1024
1025    /// Queries any instrument data for the specified instrument(s) and time range.
1026    pub fn instruments(
1027        &self,
1028        instrument_ids: Option<&[String]>,
1029        start: Option<UnixNanos>,
1030        end: Option<UnixNanos>,
1031    ) -> anyhow::Result<Vec<InstrumentAny>> {
1032        self.query_instruments_filtered(instrument_ids, start, end)
1033    }
1034
1035    /// Retrieves a list of file paths for a given data type.
1036    ///
1037    /// This method constructs a path pattern to find all parquet files
1038    /// associated with the specified data type in the catalog's directory structure.
1039    ///
1040    /// # Parameters
1041    ///
1042    /// - `data_type`: The stored family to read.
1043    ///
1044    /// # Returns
1045    ///
1046    /// Returns a vector of file paths matching the data type, or an error if the operation fails.
1047    ///
1048    /// # Errors
1049    ///
1050    /// Returns an error if:
1051    /// - Object store listing operations fail.
1052    /// - Directory access is denied.
1053    ///
1054    /// # Examples
1055    ///
1056    /// ```rust,no_run
1057    /// use nautilus_model::data::NautilusDataType;
1058    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
1059    ///
1060    /// let mut catalog = ParquetDataCatalog::new(
1061    ///     std::path::Path::new("/tmp/nautilus_data"),
1062    ///     None,
1063    ///     None,
1064    ///     None,
1065    ///     None,
1066    /// );
1067    /// let files = catalog.get_file_list_from_data_cls(&NautilusDataType::QuoteTick.into())?;
1068    ///
1069    /// for file in files {
1070    ///     println!("Found file: {}", file);
1071    /// }
1072    /// # Ok::<(), anyhow::Error>(())
1073    /// ```
1074    pub fn get_file_list_from_data_cls(
1075        &self,
1076        data_type: &CatalogDataType,
1077    ) -> anyhow::Result<Vec<String>> {
1078        let mut file_paths = Vec::new();
1079
1080        for data_cls in parquet_catalog_data_type_path_prefixes(data_type) {
1081            let directory = self.make_path(data_cls.as_ref(), None)?;
1082            file_paths.extend(self.list_parquet_files(&directory)?);
1083        }
1084
1085        Ok(file_paths)
1086    }
1087
1088    /// Filters a list of file paths based on identifiers and time range.
1089    ///
1090    /// This method filters the provided file paths by:
1091    /// 1. Matching identifiers (exact match for instruments, prefix match for bars)
1092    /// 2. Intersecting with the specified time range
1093    ///
1094    /// # Parameters
1095    ///
1096    /// - `data_type`: The stored family to read.
1097    /// - `file_paths`: List of file paths to filter.
1098    /// - `identifiers`: Optional list of identifiers to match against file paths.
1099    /// - `start`: Optional start timestamp for filtering.
1100    /// - `end`: Optional end timestamp for filtering.
1101    ///
1102    /// # Returns
1103    ///
1104    /// Returns a filtered vector of file paths that match the criteria.
1105    ///
1106    /// # Notes
1107    ///
1108    /// For Bar data types, if exact identifier matching fails, the function attempts
1109    /// partial matching by checking if the file's identifier starts with the provided identifier
1110    /// followed by a dash (to match bar type patterns).
1111    ///
1112    /// # Examples
1113    ///
1114    /// ```rust,no_run
1115    /// use nautilus_core::UnixNanos;
1116    /// use nautilus_model::data::NautilusDataType;
1117    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
1118    ///
1119    /// let mut catalog = ParquetDataCatalog::new(
1120    ///     std::path::Path::new("/tmp/nautilus_data"),
1121    ///     None,
1122    ///     None,
1123    ///     None,
1124    ///     None,
1125    /// );
1126    /// let all_files = catalog.get_file_list_from_data_cls(&NautilusDataType::QuoteTick.into())?;
1127    ///
1128    /// let filtered = catalog.filter_files(
1129    ///     &NautilusDataType::QuoteTick.into(),
1130    ///     all_files,
1131    ///     Some(vec!["EUR/USD.SIM".to_string()]),
1132    ///     Some(UnixNanos::from(1609459200000000000)),
1133    ///     Some(UnixNanos::from(1609545600000000000)),
1134    /// )?;
1135    /// # Ok::<(), anyhow::Error>(())
1136    /// ```
1137    pub fn filter_files(
1138        &self,
1139        data_type: &CatalogDataType,
1140        file_paths: Vec<String>,
1141        identifiers: Option<Vec<String>>,
1142        start: Option<UnixNanos>,
1143        end: Option<UnixNanos>,
1144    ) -> anyhow::Result<Vec<String>> {
1145        let has_bar_prefix = parquet_catalog_data_type_path_prefixes(data_type)
1146            .iter()
1147            .any(|data_cls| is_parquet_bar_prefix(data_cls.as_ref()));
1148        let mut filtered_paths = file_paths;
1149
1150        if let Some(identifiers) = identifiers {
1151            filtered_paths = filter_identifier_files(filtered_paths, &identifiers, has_bar_prefix);
1152        }
1153
1154        // Apply timestamp filtering
1155        let start_u64 = start.map(|s| s.as_u64());
1156        let end_u64 = end.map(|e| e.as_u64());
1157        filtered_paths.retain(|file_path| query_intersects_filename(file_path, start_u64, end_u64));
1158
1159        Ok(filtered_paths)
1160    }
1161}
1162
1163fn is_parquet_instrument_type_prefix(prefix: &str) -> bool {
1164    INSTRUMENT_PATH_PREFIXES.contains(&prefix)
1165}
1166
1167pub(super) fn is_parquet_bar_prefix(data_cls: &str) -> bool {
1168    data_cls == parquet_data_path_prefix(&NautilusDataType::Bar).as_ref()
1169}
1170
1171// Each identifier matches its URI-safe directory exactly. For bars, an identifier with no exact
1172// match falls back to the bar-type directories of that instrument ID, decided per identifier so
1173// one exact match does not disable the fallback for the others.
1174pub(super) fn filter_identifier_files(
1175    file_paths: Vec<String>,
1176    identifiers: &[String],
1177    bars: bool,
1178) -> Vec<String> {
1179    let directories = file_paths
1180        .iter()
1181        .map(|file_path| identifier_directory(file_path))
1182        .collect::<Vec<_>>();
1183    let mut keep = vec![false; file_paths.len()];
1184
1185    for safe_id in identifiers.iter().map(|id| urisafe_instrument_id(id)) {
1186        let has_exact = directories
1187            .iter()
1188            .flatten()
1189            .any(|directory| *directory == safe_id);
1190        let partial = bars && !has_exact;
1191
1192        for (keep, directory) in keep.iter_mut().zip(&directories) {
1193            let Some(directory) = directory else {
1194                continue;
1195            };
1196
1197            *keep |= if partial {
1198                extract_bar_type_instrument_id(directory) == Some(safe_id.as_str())
1199            } else {
1200                *directory == safe_id
1201            };
1202        }
1203    }
1204
1205    file_paths
1206        .into_iter()
1207        .zip(keep)
1208        .filter_map(|(file_path, keep)| keep.then_some(file_path))
1209        .collect()
1210}
1211
1212fn identifier_directory(file_path: &str) -> Option<String> {
1213    let mut segments = file_path.rsplit('/');
1214    segments.next()?;
1215    segments.next().map(decode_object_store_segment)
1216}
1217
1218// A table over a parent directory or one of its files, with `path` resolved for DataFusion and
1219// `directory` naming the parent directory in either case.
1220struct DataFusionTable {
1221    name: String,
1222    directory: String,
1223    path: String,
1224}
1225
1226/// Returns the sorted, deduplicated parent directories (everything except the filename)
1227/// of the given file URIs.
1228fn parent_directories(files: &[String]) -> Vec<String> {
1229    let mut directories: Vec<String> = files
1230        .iter()
1231        .filter_map(|file_uri| parent_directory(file_uri))
1232        .collect();
1233
1234    directories.sort();
1235    directories.dedup();
1236    directories
1237}
1238
1239fn parent_directory(file_uri: &str) -> Option<String> {
1240    Path::new(file_uri)
1241        .parent()
1242        .map(|path| path.to_string_lossy().to_string())
1243}
1244
1245/// Extracts the identifier from a directory path (last component).
1246fn dir_identifier(directory: &str) -> String {
1247    directory
1248        .rsplit_once('/')
1249        .map_or(directory, |(_, name)| name)
1250        .to_string()
1251}
1252
1253fn display_identifier(data_type: &NautilusDataType, directory: &str) -> Option<String> {
1254    let identifier = dir_identifier(directory);
1255    let is_unpartitioned_custom = matches!(data_type, NautilusDataType::Custom { .. })
1256        && Path::new(directory)
1257            .ends_with(Path::new("data").join(parquet_data_path_prefix(data_type).as_ref()));
1258
1259    (!is_unpartitioned_custom).then(|| decode_object_store_segment(&identifier))
1260}
1261
1262fn display_batch_identifier(
1263    data_type: &NautilusDataType,
1264    batch: &RecordBatch,
1265    path_identifier: Option<&str>,
1266) -> Option<String> {
1267    if matches!(data_type, NautilusDataType::Custom { .. }) {
1268        path_identifier.map(str::to_string)
1269    } else {
1270        catalog_identifier_from_metadata(batch.schema().metadata())
1271            .or_else(|| path_identifier.map(str::to_string))
1272    }
1273}