Skip to main content

nautilus_persistence/backend/parquet/catalog/
store.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Object-store paths, listing, and file administration for the Parquet catalog.
17
18#![expect(
19    clippy::missing_errors_doc,
20    reason = "catalog store functions forward object-store errors"
21)]
22
23use anyhow::Context;
24use datafusion::datasource::listing::ListingTableUrl;
25use nautilus_common::live::block_on_nautilus_with;
26use nautilus_model::data::NautilusDataType;
27use object_store::ObjectMeta;
28
29use super::{
30    HashSet, ObjectPath, ObjectStore, ObjectStoreExt, ParquetDataCatalog, PathBuf, StreamExt,
31    UnixNanos, append_path_to_file_uri, are_intervals_disjoint, decode_object_store_segment,
32    is_remote_uri_scheme, make_object_store_path, parse_filename_timestamps,
33    query::{filter_identifier_files, is_parquet_bar_prefix},
34    query_intersects_filename, remote_full_uri, remote_store_root_url, timestamps_to_filename,
35    urisafe_instrument_id,
36};
37use crate::{
38    backend::parquet::consolidation_state,
39    catalog::types::{
40        CatalogDataType, catalog_data_type_from_path, parquet_catalog_data_type_path_prefixes,
41    },
42    common::{paths::normalize_path_separators, storage::StorageBackend},
43};
44
45impl ParquetDataCatalog {
46    /// Extends the timestamp range of an existing Parquet file by renaming it.
47    ///
48    /// This method finds an existing file that is adjacent to the specified time range
49    /// and renames it to include the new range. This is useful when appending data
50    /// that extends the time coverage of existing files.
51    /// The proposed extension is validated against the other files before renaming,
52    /// so a rejected extension leaves existing files unchanged.
53    ///
54    /// If no file is adjacent to the specified range, this method does nothing and
55    /// returns `Ok(())` after confirming the existing intervals are disjoint.
56    ///
57    /// # Parameters
58    ///
59    /// - `data_type`: The stored family to target.
60    /// - `identifier`: Optional identifier to target a specific instrument's data. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
61    /// - `start`: Start timestamp of the new range to extend to.
62    /// - `end`: End timestamp of the new range to extend to.
63    ///
64    /// # Returns
65    ///
66    /// Returns `Ok(())` on success, or an error if the operation fails.
67    ///
68    /// # Errors
69    ///
70    /// Returns an error if:
71    /// - The range is reversed (`start` is after `end`).
72    /// - The directory path cannot be constructed.
73    /// - The proposed extension would overlap another file.
74    /// - The existing intervals are already overlapping.
75    /// - File rename operations fail.
76    ///
77    /// # Examples
78    ///
79    /// ```rust,no_run
80    /// use nautilus_core::UnixNanos;
81    /// use nautilus_model::data::NautilusDataType;
82    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
83    ///
84    /// let mut catalog = ParquetDataCatalog::new(
85    ///     std::path::Path::new("/tmp/nautilus_data"),
86    ///     None,
87    ///     None,
88    ///     None,
89    ///     None,
90    /// );
91    ///
92    /// // Extend a file's range backwards or forwards
93    /// catalog.extend_file_name(
94    ///     &NautilusDataType::QuoteTick.into(),
95    ///     Some("BTC/USD.SIM"),
96    ///     UnixNanos::from(1609459200000000000),
97    ///     UnixNanos::from(1609545600000000000),
98    /// )?;
99    /// # Ok::<(), anyhow::Error>(())
100    /// ```
101    pub fn extend_file_name(
102        &self,
103        data_type: &CatalogDataType,
104        identifier: Option<&str>,
105        start: UnixNanos,
106        end: UnixNanos,
107    ) -> anyhow::Result<()> {
108        let prefixes = parquet_catalog_data_type_path_prefixes(data_type);
109
110        if let [data_cls] = prefixes.as_slice() {
111            let directory = self.make_path(data_cls.as_ref(), identifier)?;
112            return self.extend_file_name_in_directory(&directory, start, end);
113        }
114
115        // The aggregate instrument family spans every class directory, and one identifier can be
116        // stored under several classes, so extend each directory that already holds it rather
117        // than inventing a class for it.
118        let mut extended = false;
119
120        for data_cls in &prefixes {
121            let directory = self.make_path(data_cls.as_ref(), identifier)?;
122            if !self.get_directory_intervals(&directory)?.is_empty() {
123                self.extend_file_name_in_directory(&directory, start, end)?;
124                extended = true;
125            }
126        }
127
128        anyhow::ensure!(
129            extended,
130            "Cannot extend file name for {data_type}: no instrument class holds {}; \
131             name the class with a NautilusInstrumentType",
132            identifier.unwrap_or("any identifier"),
133        );
134
135        Ok(())
136    }
137
138    pub(super) fn extend_file_name_in_directory(
139        &self,
140        directory: &str,
141        start: UnixNanos,
142        end: UnixNanos,
143    ) -> anyhow::Result<()> {
144        let start = start.as_u64();
145        let end = end.as_u64();
146
147        anyhow::ensure!(
148            start <= end,
149            "Cannot extend file in directory '{directory}': reversed range ({start}, {end})",
150        );
151
152        let intervals = self.get_directory_intervals(directory)?;
153
154        anyhow::ensure!(
155            are_intervals_disjoint(&intervals),
156            "Intervals are not disjoint in directory '{directory}': {intervals:?}",
157        );
158
159        let adjacent = intervals.iter().enumerate().find_map(|(index, interval)| {
160            if end.checked_add(1) == Some(interval.0) {
161                // Extend backwards: new file covers [start, interval.1]
162                Some((index, *interval, (start, interval.1)))
163            } else if start.checked_sub(1) == Some(interval.1) {
164                // Extend forwards: new file covers [interval.0, end]
165                Some((index, *interval, (interval.0, end)))
166            } else {
167                None
168            }
169        });
170
171        let Some((index, original, proposed)) = adjacent else {
172            return Ok(());
173        };
174
175        let mut extended = intervals.clone();
176        extended[index] = proposed;
177
178        anyhow::ensure!(
179            are_intervals_disjoint(&extended),
180            "Extending file interval {original:?} to {proposed:?} in directory '{directory}' \
181            with range ({start}, {end}) would create non-disjoint intervals. \
182            Existing intervals: {intervals:?}",
183        );
184
185        self.rename_parquet_file(directory, original.0, original.1, proposed.0, proposed.1)
186    }
187
188    /// Helper method to rename a parquet file by moving it via object store operations
189    fn rename_parquet_file(
190        &self,
191        directory: &str,
192        old_start: u64,
193        old_end: u64,
194        new_start: u64,
195        new_end: u64,
196    ) -> anyhow::Result<()> {
197        let new_filename =
198            timestamps_to_filename(UnixNanos::from(new_start), UnixNanos::from(new_end));
199        let new_path = format!("{directory}/{new_filename}");
200        let matches = self
201            .list_parquet_files(directory)?
202            .into_iter()
203            .filter(|file| parse_filename_timestamps(file) == Some((old_start, old_end)))
204            .collect::<Vec<_>>();
205
206        anyhow::ensure!(
207            matches.len() == 1,
208            "expected one parquet file for interval ({old_start}, {old_end}) in {directory}, \
209             found {}",
210            matches.len()
211        );
212
213        let old_path = &matches[0];
214        if old_path.ends_with(&new_filename) {
215            return Ok(());
216        }
217
218        let old_object_path = self.to_object_path_parsed(old_path)?;
219        let new_object_path = self.to_object_path(&new_path)?;
220        self.move_file(&old_object_path, &new_object_path)
221    }
222
223    /// Lists all Parquet files under a directory, including its subdirectories.
224    ///
225    /// This method scans a directory and returns the full paths of all files with the `.parquet`
226    /// extension. It works with both local filesystems and remote object stores, making it
227    /// suitable for various storage backends.
228    ///
229    /// # Parameters
230    ///
231    /// - `directory`: The directory path to scan for Parquet files, unencoded as
232    ///   [`Self::make_path`] returns it.
233    ///
234    /// # Returns
235    ///
236    /// Returns a vector of full file paths (as strings) for all Parquet files under the directory.
237    /// The paths are percent-encoded object-store keys relative to the store root; convert one with
238    /// [`Self::to_object_path_parsed`].
239    /// Returns an empty vector if the directory doesn't exist or contains no Parquet files.
240    ///
241    /// # Errors
242    ///
243    /// Returns an error if:
244    /// - Object store listing operations fail.
245    /// - Directory access is denied.
246    /// - Network issues occur (for remote object stores).
247    ///
248    /// # Notes
249    ///
250    /// - Only files ending with `.parquet` are included.
251    /// - Subdirectories are scanned recursively, so a type directory such as `data/quotes`
252    ///   returns the files of every identifier under it.
253    /// - File paths are returned in the order provided by the object store.
254    /// - Works with all supported object store backends (local, S3, GCS, Azure, etc.).
255    ///
256    /// # Examples
257    ///
258    /// ```rust,no_run
259    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
260    ///
261    /// let mut catalog = ParquetDataCatalog::new(
262    ///     std::path::Path::new("/tmp/nautilus_data"),
263    ///     None,
264    ///     None,
265    ///     None,
266    ///     None,
267    /// );
268    /// let files = catalog.list_parquet_files("data/quotes/EURUSD")?;
269    ///
270    /// for file in files {
271    ///     println!("Found Parquet file: {}", file);
272    /// }
273    /// # Ok::<(), anyhow::Error>(())
274    /// ```
275    pub fn list_parquet_files(&self, directory: &str) -> anyhow::Result<Vec<String>> {
276        let files = self
277            .list_objects(directory)?
278            .into_iter()
279            .filter(|object| object.location.as_ref().ends_with(".parquet"))
280            .map(|object| object.location.to_string())
281            .collect();
282
283        Ok(files)
284    }
285
286    /// Lists all instrument identifiers for a specific data type.
287    ///
288    /// This method scans the data directory for a given data type and extracts
289    /// all unique instrument identifiers from the directory structure.
290    ///
291    /// # Parameters
292    ///
293    /// - `data_type`: The stored family to target.
294    ///
295    /// # Returns
296    ///
297    /// Returns a vector of instrument identifier strings.
298    ///
299    /// # Errors
300    ///
301    /// Returns an error if directory listing fails.
302    pub fn list_instruments(&self, data_type: &CatalogDataType) -> anyhow::Result<Vec<String>> {
303        let mut instruments = Vec::new();
304
305        for prefix in parquet_catalog_data_type_path_prefixes(data_type) {
306            instruments.extend(self.list_prefix_instruments(prefix.as_ref())?);
307        }
308
309        // The same identifier can live under more than one instrument class.
310        instruments.sort();
311        instruments.dedup();
312
313        Ok(instruments)
314    }
315
316    fn list_prefix_instruments(&self, data_type: &str) -> anyhow::Result<Vec<String>> {
317        let directory = self.make_path(data_type, None)?;
318        let prefix = ObjectPath::from(directory.as_str());
319        let mut instruments = HashSet::new();
320
321        for object in self.list_objects(&directory)? {
322            // Relative to the prefix a datum is `{identifier}/{filename}.parquet`
323            let Some(relative) = object.location.prefix_match(&prefix) else {
324                continue;
325            };
326            let segments: Vec<_> = relative.collect();
327
328            if let [identifier, filename] = segments.as_slice()
329                && filename.as_ref().ends_with(".parquet")
330            {
331                instruments.insert(decode_object_store_segment(identifier.as_ref()));
332            }
333        }
334
335        Ok(instruments.into_iter().collect())
336    }
337
338    /// Lists Parquet files matching specific criteria (data type, identifiers, time range).
339    ///
340    /// This method finds all Parquet files that match the specified criteria by filtering
341    /// files based on their directory structure and filename timestamps.
342    ///
343    /// # Parameters
344    ///
345    /// - `data_type`: The stored family to target.
346    /// - `identifiers`: Optional list of identifiers to filter by.
347    /// - `start`: Optional start timestamp to filter files by their time range.
348    /// - `end`: Optional end timestamp to filter files by their time range.
349    ///
350    /// # Returns
351    ///
352    /// Returns a vector of file paths that match the criteria.
353    ///
354    /// # Errors
355    ///
356    /// Returns an error if directory listing or file filtering fails.
357    pub fn list_parquet_files_with_criteria(
358        &self,
359        data_type: &CatalogDataType,
360        identifiers: Option<&[String]>,
361        start: Option<UnixNanos>,
362        end: Option<UnixNanos>,
363    ) -> anyhow::Result<Vec<String>> {
364        let mut all_files = Vec::new();
365
366        for data_cls in parquet_catalog_data_type_path_prefixes(data_type) {
367            all_files.extend(self.list_prefix_files_with_criteria(
368                data_cls.as_ref(),
369                identifiers,
370                start,
371                end,
372            )?);
373        }
374
375        Ok(all_files)
376    }
377
378    fn list_prefix_files_with_criteria(
379        &self,
380        data_cls: &str,
381        identifiers: Option<&[String]>,
382        start: Option<UnixNanos>,
383        end: Option<UnixNanos>,
384    ) -> anyhow::Result<Vec<String>> {
385        let start_u64 = start.map(|s| s.as_u64());
386        let end_u64 = end.map(|e| e.as_u64());
387
388        let base_dir = self.make_path(data_cls, None)?;
389
390        // Use recursive listing to match Python's glob behavior
391        let mut all_files = self.list_parquet_files(&base_dir)?;
392
393        if let Some(identifiers) = identifiers {
394            all_files =
395                filter_identifier_files(all_files, identifiers, is_parquet_bar_prefix(data_cls));
396        }
397
398        all_files.retain(|path| query_intersects_filename(path, start_u64, end_u64));
399        Ok(all_files)
400    }
401
402    /// Recursively lists all objects under `{dir}/` in the object store.
403    pub(crate) fn list_objects(&self, dir: &str) -> anyhow::Result<Vec<ObjectMeta>> {
404        self.execute_async(|| async {
405            let prefix = ObjectPath::from(format!("{dir}/"));
406            let mut stream = self.object_store.list(Some(&prefix));
407            let mut objects = Vec::new();
408            while let Some(object) = stream.next().await {
409                objects.push(object?);
410            }
411
412            consolidation_state::visible_objects(&self.object_store, objects).await
413        })
414    }
415
416    pub(crate) fn recover_consolidation(&self, directory: &ObjectPath) -> anyhow::Result<()> {
417        self.execute_async(|| async {
418            consolidation_state::recover(&self.object_store, directory).await
419        })
420    }
421
422    pub(super) fn has_pending_consolidation(&self, directory: &str) -> anyhow::Result<bool> {
423        let directory = if self.is_remote_uri() {
424            self.to_object_path_parsed(directory)?
425        } else {
426            let uri = self.resolve_directory_for_datafusion(directory);
427            let path = ListingTableUrl::parse(&uri)?
428                .get_url()
429                .to_file_path()
430                .map_err(|()| anyhow::anyhow!("Invalid local file URI: {directory}"))?;
431            self.to_object_path_parsed(&path.to_string_lossy())?
432        };
433
434        self.execute_async(|| async {
435            consolidation_state::has_pending(&self.object_store, &directory).await
436        })
437    }
438
439    /// Helper method to reconstruct full URI for remote object store paths
440    #[must_use]
441    pub fn reconstruct_full_uri(&self, path_str: &str) -> String {
442        if path_str.contains("://") {
443            return path_str.to_string();
444        }
445
446        // Check if this is a remote URI scheme that needs reconstruction
447        if self.is_remote_uri() {
448            let path = self.path_under_base(path_str);
449            if let Ok(uri) = remote_full_uri(&self.original_uri, &path) {
450                return uri;
451            }
452        }
453
454        // For local paths, extract the directory from the original URI
455        if self.original_uri.starts_with("file://") {
456            // Extract the path from the file:// URI
457            if let Ok(url) = url::Url::parse(&self.original_uri)
458                && let Ok(base_path) = url.to_file_path()
459            {
460                // Use platform-appropriate path separator for display
461                // but object store paths always use forward slashes
462                let base_str = base_path.to_string_lossy();
463                return make_object_store_path(&base_str, [path_str]);
464            }
465        }
466
467        // For local paths without file:// prefix, use the original URI as base
468        if self.base_path.is_empty() {
469            // If base_path is empty and not a file URI, try using original_uri as base
470            if self.original_uri.contains("://") {
471                // Fallback: return the path as-is
472                path_str.to_string()
473            } else {
474                make_object_store_path(self.original_uri.trim_end_matches('/'), [path_str])
475            }
476        } else {
477            let base = self.base_path.trim_end_matches('/');
478            make_object_store_path(base, [path_str])
479        }
480    }
481
482    /// Resolves a path for use with DataFusion (avoiding Windows path doubling for file://).
483    /// A remote path resolves to a URL under the store root whose path decodes to the object
484    /// key. A local path is returned as-is if it is already a full URI or absolute; otherwise it
485    /// joins the file:// base or goes through `reconstruct_full_uri`.
486    #[must_use]
487    pub(crate) fn resolve_path_for_datafusion(&self, path: &str) -> String {
488        // DataFusion decodes the URL path once, so encode the key again
489        if self.is_remote_uri()
490            && let Ok(key) = self.object_store_path(path)
491            && let Ok(root) = remote_store_root_url(&self.original_uri)
492        {
493            return append_path_to_file_uri(root.as_str(), &key);
494        }
495
496        if path.contains("://") {
497            return path.to_string();
498        }
499
500        if path.starts_with('/') {
501            return path.to_string();
502        }
503
504        if self.original_uri.starts_with("file://") {
505            return append_path_to_file_uri(&self.original_uri, path);
506        }
507
508        self.reconstruct_full_uri(path)
509    }
510
511    /// Like `resolve_path_for_datafusion` but ensures the result ends with a trailing slash.
512    #[must_use]
513    pub(super) fn resolve_directory_for_datafusion(&self, directory: &str) -> String {
514        let mut resolved = self.resolve_path_for_datafusion(directory);
515        if !resolved.ends_with('/') {
516            resolved.push('/');
517        }
518
519        resolved
520    }
521
522    /// Returns the path string to push in `query_files` result list: relative for file://,
523    /// full URI for remote (so callers can pass to `resolve_path_for_datafusion` later).
524    #[must_use]
525    pub(super) fn path_for_query_list(&self, path: &str) -> String {
526        if self.original_uri.starts_with("file://") {
527            path.to_string()
528        } else {
529            self.reconstruct_full_uri(path)
530        }
531    }
532
533    /// Returns the native path string for the catalog root (for `std::fs`). Only valid when
534    /// !`is_remote_uri()`; uses parquet's `file_uri_to_native_path` for file:// URIs.
535    #[must_use]
536    pub(crate) fn native_base_path_string(&self) -> String {
537        if self.original_uri.starts_with("file://") {
538            crate::backend::parquet::io::file_uri_to_native_path(&self.original_uri)
539        } else {
540            self.original_uri.clone()
541        }
542    }
543
544    pub(crate) fn register_remote_object_store(&mut self) -> anyhow::Result<()> {
545        if self.is_remote_uri() {
546            let base_url = remote_store_root_url(&self.original_uri)?;
547            self.session
548                .register_object_store(&base_url, self.object_store.clone());
549        }
550
551        Ok(())
552    }
553
554    /// Helper method to check if the original URI uses a remote object store scheme
555    #[must_use]
556    pub fn is_remote_uri(&self) -> bool {
557        self.original_uri
558            .split_once("://")
559            .is_some_and(|(scheme, _)| is_remote_uri_scheme(scheme))
560    }
561
562    /// Constructs a directory path for storing data of a specific type and instrument.
563    ///
564    /// This method builds the hierarchical directory structure used by the catalog to organize
565    /// data by type and instrument. The path follows the pattern: `{base_path}/data/{type_name}/{instrument_id}`.
566    /// Instrument IDs are automatically converted to URI-safe format by removing forward slashes.
567    ///
568    /// # Parameters
569    ///
570    /// - `type_name`: The data type directory name (e.g., "quotes", "trades", "bars").
571    /// - `identifier`: Optional identifier. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL"). If provided, creates a subdirectory for the identifier. If `None`, returns the path to the data type directory.
572    ///
573    /// # Returns
574    ///
575    /// Returns the constructed directory path as a string, or an error if path construction fails.
576    ///
577    /// # Errors
578    ///
579    /// Returns an error if:
580    /// - The instrument ID contains invalid characters that cannot be made URI-safe.
581    /// - Path construction fails due to system limitations.
582    ///
583    /// # Path Structure
584    ///
585    /// - Without identifier: `{base_path}/data/{type_name}`.
586    /// - With identifier: `{base_path}/data/{type_name}/{safe_identifier}`.
587    /// - If `base_path` is empty: `data/{type_name}[/{safe_identifier}]`.
588    ///
589    /// # Examples
590    ///
591    /// ```rust,no_run
592    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
593    ///
594    /// let mut catalog = ParquetDataCatalog::new(
595    ///     std::path::Path::new("/tmp/nautilus_data"),
596    ///     None,
597    ///     None,
598    ///     None,
599    ///     None,
600    /// );
601    ///
602    /// // Path for all quote data
603    /// let quotes_path = catalog.make_path("quotes", None)?;
604    /// // Returns: "/base/path/data/quotes"
605    ///
606    /// // Path for specific instrument quotes
607    /// let eurusd_quotes = catalog.make_path("quotes", Some("EUR/USD"))?;
608    /// // Returns: "/base/path/data/quotes/EURUSD" (slash removed)
609    ///
610    /// // Path for bar data with complex instrument ID
611    /// let bars_path = catalog.make_path("bars", Some("BTC/USD-1H"))?;
612    /// // Returns: "/base/path/data/bars/BTCUSD-1H"
613    /// # Ok::<(), anyhow::Error>(())
614    /// ```
615    pub fn make_path(&self, type_name: &str, identifier: Option<&str>) -> anyhow::Result<String> {
616        let mut components = vec!["data".to_string(), type_name.to_string()];
617
618        if let Some(id) = identifier {
619            let safe_id = urisafe_instrument_id(id);
620            components.push(safe_id);
621        }
622
623        let path = make_object_store_path(&self.base_path, components);
624        Ok(path)
625    }
626
627    /// Builds the directory path for custom data: `data/custom/{type_name}[/{identifier}]`.
628    pub fn make_path_custom_data(
629        &self,
630        type_name: &str,
631        identifier: Option<&str>,
632    ) -> anyhow::Result<String> {
633        let mut components = vec![
634            "data".to_string(),
635            "custom".to_string(),
636            type_name.to_string(),
637        ];
638
639        if let Some(id) = identifier {
640            let safe_id = urisafe_instrument_id(id);
641
642            if !safe_id.is_empty() {
643                components.push(safe_id);
644            }
645        }
646
647        let path = make_object_store_path(&self.base_path, components);
648        Ok(path)
649    }
650
651    /// Converts a catalog path string to an [`ObjectPath`] for object store operations.
652    ///
653    /// This method handles the conversion between catalog-relative paths and object store paths,
654    /// taking into account the catalog's base path configuration. It automatically preserves the
655    /// base path prefix for remote catalogs and strips it for local catalog paths.
656    ///
657    /// # Parameters
658    ///
659    /// - `path`: The catalog path string to convert. Can be absolute or relative.
660    ///
661    /// # Returns
662    ///
663    /// Returns an [`ObjectPath`] suitable for use with object store operations.
664    ///
665    /// # Path Handling
666    ///
667    /// - If `base_path` is empty, the path is used as-is.
668    /// - If `base_path` is set for a remote catalog, it's preserved or prepended.
669    /// - If `base_path` is set for a local catalog, it's stripped from the path if present.
670    /// - Trailing slashes and backslashes are automatically handled.
671    /// - The resulting path is relative to the object store root.
672    /// - All paths are normalized to use forward slashes (object store convention).
673    ///
674    /// # Errors
675    ///
676    /// Returns an error if the catalog is remote and `path` is a full URI whose scheme/host does
677    /// not match the catalog's own root (cross-bucket misuse). Without this guard the caller could
678    /// silently write to or read from the wrong bucket.
679    ///
680    /// # Examples
681    ///
682    /// Local catalog paths (absolute or relative) strip the catalog's base directory:
683    ///
684    /// ```rust,no_run
685    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
686    /// # let catalog: ParquetDataCatalog = unimplemented!();
687    /// let object_path = catalog.to_object_path("/base/data/quotes/file.parquet")?;
688    /// // ObjectPath("data/quotes/file.parquet")
689    /// # Ok::<(), anyhow::Error>(())
690    /// ```
691    ///
692    /// Remote catalog paths (relative or full URI) preserve or prepend the base prefix:
693    ///
694    /// ```rust,no_run
695    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
696    /// # let catalog: ParquetDataCatalog = unimplemented!();
697    /// let object_path = catalog.to_object_path("data/trades/file.parquet")?;
698    /// // ObjectPath("base/data/trades/file.parquet")
699    /// # Ok::<(), anyhow::Error>(())
700    /// ```
701    pub fn to_object_path(&self, path: &str) -> anyhow::Result<ObjectPath> {
702        Ok(ObjectPath::from(self.object_store_path(path)?))
703    }
704
705    /// Converts a path string to [`ObjectPath`] using parse (no percent-encoding).
706    ///
707    /// Use this for paths that were returned by the object store (e.g. from `list()`),
708    /// which may already be percent-encoded. Using [`Self::to_object_path`] (which uses
709    /// `Path::from`) on such paths would double-encode (e.g. `%5E` -> `%255E`).
710    ///
711    /// # Errors
712    ///
713    /// Returns an error if the same cross-bucket condition as [`Self::to_object_path`] occurs or
714    /// the resulting string fails [`ObjectPath::parse`].
715    pub fn to_object_path_parsed(&self, path: &str) -> anyhow::Result<ObjectPath> {
716        let to_parse = self.object_store_path(path)?;
717        ObjectPath::parse(&to_parse).map_err(anyhow::Error::from)
718    }
719
720    pub(super) fn object_store_path(&self, path: &str) -> anyhow::Result<String> {
721        let normalized_path = normalize_path_separators(path);
722
723        if self.is_remote_uri() {
724            if normalized_path.contains("://") {
725                let path_under_root = self.remote_uri_object_path(&normalized_path)?;
726                return Ok(self.path_under_base(&path_under_root));
727            }
728
729            return Ok(self.path_under_base(&normalized_path));
730        }
731
732        Ok(self.path_without_local_base(&normalized_path))
733    }
734
735    fn remote_uri_object_path(&self, path: &str) -> anyhow::Result<String> {
736        let path_url = url::Url::parse(path)
737            .map_err(|e| anyhow::anyhow!("Failed to parse object store URI {path}: {e}"))?;
738        if !is_remote_uri_scheme(path_url.scheme()) {
739            anyhow::bail!(
740                "URI {path} uses non-remote scheme {} for remote catalog at {}",
741                path_url.scheme(),
742                self.original_uri,
743            );
744        }
745
746        let catalog_root = remote_store_root_url(&self.original_uri)?;
747        let path_root = remote_store_root_url(path)?;
748        if catalog_root.as_str().trim_end_matches('/') != path_root.as_str().trim_end_matches('/') {
749            anyhow::bail!(
750                "Cross-store URI {path} (root {}) does not belong to catalog rooted at {} ({})",
751                path_root.as_str().trim_end_matches('/'),
752                self.original_uri,
753                catalog_root.as_str().trim_end_matches('/'),
754            );
755        }
756
757        // The URL crate keeps the path component percent-encoded (e.g. `%5E`),
758        // so preserve that encoding for `ObjectPath::parse` round-trips through
759        // `object_store::list`/`get`.
760        Ok(path_url.path().trim_start_matches('/').to_string())
761    }
762
763    pub(crate) fn path_without_local_base(&self, path: &str) -> String {
764        let base_path = if self.base_path.is_empty() {
765            self.native_base_path_string()
766        } else {
767            self.base_path.clone()
768        };
769
770        let normalized_base = normalize_path_separators(&base_path);
771        let base = normalized_base.trim_end_matches('/');
772
773        if base.is_empty() {
774            path.to_string()
775        } else if path == base {
776            String::new()
777        } else if let Some(without_base) = path.strip_prefix(&format!("{base}/")) {
778            without_base.to_string()
779        } else {
780            path.to_string()
781        }
782    }
783
784    fn path_under_base(&self, path: &str) -> String {
785        let normalized_path = normalize_path_separators(path);
786        let path = normalized_path
787            .trim_start_matches('/')
788            .trim_end_matches('/');
789
790        if self.base_path.is_empty() {
791            return path.to_string();
792        }
793
794        let normalized_base = normalize_path_separators(&self.base_path);
795        let base = normalized_base
796            .trim_start_matches('/')
797            .trim_end_matches('/');
798
799        if base.is_empty() || path == base || path.starts_with(&format!("{base}/")) {
800            path.to_string()
801        } else if path.is_empty() {
802            base.to_string()
803        } else {
804            make_object_store_path(base, [path])
805        }
806    }
807
808    /// Helper method to move a file using object store rename operation
809    pub fn move_file(&self, old_path: &ObjectPath, new_path: &ObjectPath) -> anyhow::Result<()> {
810        if old_path == new_path {
811            return Ok(());
812        }
813
814        self.recover_consolidation(&old_path.parent().unwrap_or_default())?;
815        self.recover_consolidation(&new_path.parent().unwrap_or_default())?;
816
817        self.execute_async(|| async {
818            self.object_store
819                .rename(old_path, new_path)
820                .await
821                .map_err(anyhow::Error::from)
822        })
823    }
824
825    /// Helper method to execute async operations with a runtime
826    pub fn execute_async<C, F, R>(&self, create_future: C) -> anyhow::Result<R>
827    where
828        C: FnOnce() -> F + Send,
829        F: std::future::Future<Output = anyhow::Result<R>>,
830        R: Send,
831    {
832        block_on_nautilus_with(create_future)
833    }
834
835    /// Lists directory stems (directory names without path) in a subdirectory.
836    ///
837    /// This method scans a subdirectory and returns the names of all immediate
838    /// subdirectories. It's used to list data types, backtest runs, and live runs.
839    ///
840    /// # Parameters
841    ///
842    /// - `subdirectory`: The subdirectory path to scan (e.g., "data", "backtest", "live").
843    ///
844    /// # Returns
845    ///
846    /// Returns the sorted directory names (stems) found in the subdirectory, decoded from their
847    /// percent-encoded object-store form, or an error if the operation fails.
848    ///
849    /// # Errors
850    ///
851    /// Returns an error if:
852    /// - Object store listing operations fail.
853    /// - Directory access is denied.
854    ///
855    /// # Examples
856    ///
857    /// ```rust,no_run
858    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
859    ///
860    /// let mut catalog = ParquetDataCatalog::new(
861    ///     std::path::Path::new("/tmp/nautilus_data"),
862    ///     None,
863    ///     None,
864    ///     None,
865    ///     None,
866    /// );
867    ///
868    /// // List all data types
869    /// let data_types = catalog.list_directory_stems("data")?;
870    /// for data_type in data_types {
871    ///     println!("Found data type: {}", data_type);
872    /// }
873    /// # Ok::<(), anyhow::Error>(())
874    /// ```
875    pub fn list_directory_stems(&self, subdirectory: &str) -> anyhow::Result<Vec<String>> {
876        // For local filesystem paths, use filesystem operations to detect empty directories
877        // For remote object stores, we can only list directories that contain files
878        if !self.is_remote_uri() {
879            let directory = PathBuf::from(self.native_base_path_string()).join(subdirectory);
880
881            let entries = match std::fs::read_dir(&directory) {
882                Ok(entries) => entries,
883                Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
884                Err(e) => {
885                    return Err(e)
886                        .with_context(|| format!("failed to list {}", directory.display()));
887                }
888            };
889
890            let mut directories = Vec::new();
891
892            for entry in entries {
893                let entry =
894                    entry.with_context(|| format!("failed to list {}", directory.display()))?;
895                let file_type = entry
896                    .file_type()
897                    .with_context(|| format!("failed to read {}", entry.path().display()))?;
898
899                // Use file_name() to get the directory name (not file_stem which removes extension)
900                if file_type.is_dir()
901                    && let Some(name) = entry.path().file_name()
902                {
903                    directories.push(decode_object_store_segment(&name.to_string_lossy()));
904                }
905            }
906
907            directories.sort();
908            return Ok(directories);
909        }
910
911        let storage = StorageBackend {
912            object_store: self.object_store.clone(),
913            base_path: self.base_path.clone(),
914            original_uri: self.original_uri.clone(),
915        };
916
917        self.execute_async(|| async { storage.list_directory_stems(subdirectory).await })
918    }
919
920    /// Lists the data, record, and instrument types stored in the catalog.
921    ///
922    /// Each `data/<type>` directory maps to its catalog type, and each `data/custom/<type_name>`
923    /// directory to a custom data type. Unrecognized directories are skipped with a warning.
924    ///
925    /// # Returns
926    ///
927    /// Returns the catalog types, or an error if the operation fails.
928    ///
929    /// # Errors
930    ///
931    /// Returns an error if:
932    /// - Object store listing operations fail.
933    /// - Directory access is denied.
934    ///
935    /// # Examples
936    ///
937    /// ```rust,no_run
938    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
939    ///
940    /// let mut catalog = ParquetDataCatalog::new(
941    ///     std::path::Path::new("/tmp/nautilus_data"),
942    ///     None,
943    ///     None,
944    ///     None,
945    ///     None,
946    /// );
947    ///
948    /// // List all data types
949    /// let data_types = catalog.list_data_types()?;
950    /// for data_type in data_types {
951    ///     println!("Available data type: {data_type}");
952    /// }
953    /// # Ok::<(), anyhow::Error>(())
954    /// ```
955    pub fn list_data_types(&self) -> anyhow::Result<Vec<CatalogDataType>> {
956        let mut data_types = Vec::new();
957
958        for stem in self.list_directory_stems("data")? {
959            if stem == "custom" {
960                for type_name in self.list_directory_stems("data/custom")? {
961                    data_types.push(CatalogDataType::from(NautilusDataType::Custom {
962                        type_name,
963                    }));
964                }
965
966                continue;
967            }
968
969            match catalog_data_type_from_path(&stem) {
970                Ok(data_type) => data_types.push(data_type),
971                Err(e) => log::warn!("Skipping catalog directory data/{stem}: {e}"),
972            }
973        }
974
975        Ok(data_types)
976    }
977
978    /// Lists all backtest run IDs available in the catalog.
979    ///
980    /// This method returns the names of all backtest run directories in the catalog.
981    /// Each backtest run corresponds to a specific backtest execution instance.
982    ///
983    /// # Returns
984    ///
985    /// Returns a vector of backtest run IDs, or an error if the operation fails.
986    ///
987    /// # Errors
988    ///
989    /// Returns an error if:
990    /// - Object store listing operations fail.
991    /// - Directory access is denied.
992    ///
993    /// # Examples
994    ///
995    /// ```rust,no_run
996    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
997    ///
998    /// let mut catalog = ParquetDataCatalog::new(
999    ///     std::path::Path::new("/tmp/nautilus_data"),
1000    ///     None,
1001    ///     None,
1002    ///     None,
1003    ///     None,
1004    /// );
1005    ///
1006    /// // List all backtest runs
1007    /// let runs = catalog.list_backtest_runs()?;
1008    /// for run_id in runs {
1009    ///     println!("Backtest run: {}", run_id);
1010    /// }
1011    /// # Ok::<(), anyhow::Error>(())
1012    /// ```
1013    pub fn list_backtest_runs(&self) -> anyhow::Result<Vec<String>> {
1014        self.list_directory_stems("backtest")
1015    }
1016
1017    /// Lists all live run IDs available in the catalog.
1018    ///
1019    /// This method returns the names of all live run directories in the catalog.
1020    /// Each live run corresponds to a specific live trading execution instance.
1021    ///
1022    /// # Returns
1023    ///
1024    /// Returns a vector of live run IDs, or an error if the operation fails.
1025    ///
1026    /// # Errors
1027    ///
1028    /// Returns an error if:
1029    /// - Object store listing operations fail.
1030    /// - Directory access is denied.
1031    ///
1032    /// # Examples
1033    ///
1034    /// ```rust,no_run
1035    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
1036    ///
1037    /// let mut catalog = ParquetDataCatalog::new(
1038    ///     std::path::Path::new("/tmp/nautilus_data"),
1039    ///     None,
1040    ///     None,
1041    ///     None,
1042    ///     None,
1043    /// );
1044    ///
1045    /// // List all live runs
1046    /// let runs = catalog.list_live_runs()?;
1047    /// for run_id in runs {
1048    ///     println!("Live run: {}", run_id);
1049    /// }
1050    /// # Ok::<(), anyhow::Error>(())
1051    /// ```
1052    pub fn list_live_runs(&self) -> anyhow::Result<Vec<String>> {
1053        self.list_directory_stems("live")
1054    }
1055}
1056
1057#[cfg(test)]
1058mod tests {
1059    use std::sync::Arc;
1060
1061    use nautilus_model::data::NautilusDataType;
1062    use object_store::{ObjectStoreExt, PutPayload, memory::InMemory, path::Path as ObjectPath};
1063    use rstest::rstest;
1064
1065    use super::ParquetDataCatalog;
1066    use crate::{catalog::types::CatalogDataType, common::datafusion::DataBackendSession};
1067
1068    /// Builds a catalog over an in-memory store. Only remote catalogs carry a non-empty
1069    /// `base_path`, so seeding one here is the sole way to reproduce a bucket sub-prefix.
1070    fn memory_catalog(base_path: &str) -> ParquetDataCatalog {
1071        ParquetDataCatalog {
1072            base_path: base_path.to_string(),
1073            original_uri: "memory://".to_string(),
1074            object_store: Arc::new(InMemory::new()),
1075            session: DataBackendSession::new(5_000),
1076            batch_size: 5_000,
1077            compression: parquet::basic::Compression::SNAPPY,
1078            max_row_group_size: 5_000,
1079        }
1080    }
1081
1082    fn seed(catalog: &ParquetDataCatalog, keys: &[&str]) {
1083        catalog
1084            .execute_async(|| async {
1085                for key in keys {
1086                    catalog
1087                        .object_store
1088                        .put(&ObjectPath::from(*key), PutPayload::from_static(b"x"))
1089                        .await?;
1090                }
1091                Ok(())
1092            })
1093            .unwrap();
1094    }
1095
1096    #[rstest]
1097    #[case("nautilus-data", "nautilus-data/")]
1098    #[case("", "")]
1099    fn list_instruments_returns_ids_under_base_path(
1100        #[case] base_path: &str,
1101        #[case] key_prefix: &str,
1102    ) {
1103        let catalog = memory_catalog(base_path);
1104        let keys = [
1105            "data/quotes/EURUSD.SIM/0-1.parquet",
1106            "data/quotes/EURUSD.SIM/2-3.parquet",
1107            "data/quotes/GBPUSD.SIM/0-1.parquet",
1108            "data/trades/AUDUSD.SIM/0-1.parquet",
1109        ]
1110        .map(|key| format!("{key_prefix}{key}"));
1111        seed(&catalog, &keys.each_ref().map(String::as_str));
1112
1113        assert_eq!(
1114            catalog
1115                .list_instruments(&NautilusDataType::QuoteTick.into())
1116                .unwrap(),
1117            ["EURUSD.SIM", "GBPUSD.SIM"]
1118        );
1119    }
1120
1121    #[rstest]
1122    #[case("nautilus-data", "nautilus-data/")]
1123    #[case("", "")]
1124    fn list_parquet_files_lists_type_directory_recursively(
1125        #[case] base_path: &str,
1126        #[case] key_prefix: &str,
1127    ) {
1128        let catalog = memory_catalog(base_path);
1129        let keys = [
1130            "data/quotes/EURUSD.SIM/0-1.parquet",
1131            "data/quotes/EURUSD.SIM/0-1.json",
1132            "data/quotes/BTC€.SIM/2-3.parquet",
1133            "data/trades/AUDUSD.SIM/0-1.parquet",
1134        ]
1135        .map(|key| format!("{key_prefix}{key}"));
1136        seed(&catalog, &keys.each_ref().map(String::as_str));
1137
1138        let directory = catalog.make_path("quotes", None).unwrap();
1139        let mut files = catalog.list_parquet_files(&directory).unwrap();
1140        files.sort();
1141
1142        assert_eq!(
1143            files,
1144            [
1145                format!("{key_prefix}data/quotes/BTC%E2%82%AC.SIM/2-3.parquet"),
1146                format!("{key_prefix}data/quotes/EURUSD.SIM/0-1.parquet"),
1147            ]
1148        );
1149    }
1150
1151    #[rstest]
1152    fn list_instruments_ignores_unpartitioned_files() {
1153        // An empty `base_path` keeps this distinct from the remote prefix defect, so an empty
1154        // result can only come from the layout check.
1155        let catalog = memory_catalog("");
1156        seed(
1157            &catalog,
1158            &[
1159                "data/custom/MyType/1-2.parquet",
1160                "data/custom/MyType/1-2.json",
1161            ],
1162        );
1163
1164        let custom = CatalogDataType::Data(NautilusDataType::Custom {
1165            type_name: "MyType".to_string(),
1166        });
1167
1168        assert!(catalog.list_instruments(&custom).unwrap().is_empty());
1169    }
1170
1171    #[rstest]
1172    fn list_instruments_decodes_percent_encoded_ids() {
1173        let catalog = memory_catalog("nautilus-data");
1174        seed(
1175            &catalog,
1176            &["nautilus-data/data/quotes/BTC€.SIM/0-1.parquet"],
1177        );
1178
1179        assert_eq!(
1180            catalog
1181                .list_instruments(&NautilusDataType::QuoteTick.into())
1182                .unwrap(),
1183            ["BTC€.SIM"]
1184        );
1185    }
1186}