Skip to main content

nautilus_persistence/backend/parquet/catalog/
store.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Object-store paths, listing, and file administration for the Parquet catalog.
17
18#![expect(
19    clippy::missing_errors_doc,
20    reason = "catalog store functions forward object-store errors"
21)]
22
23use anyhow::Context;
24use datafusion::datasource::listing::ListingTableUrl;
25use nautilus_common::live::block_on_nautilus_with;
26use nautilus_model::data::NautilusDataType;
27use object_store::ObjectMeta;
28
29use super::{
30    HashSet, ObjectPath, ObjectStore, ObjectStoreExt, ParquetDataCatalog, PathBuf, StreamExt,
31    UnixNanos, append_path_to_file_uri, are_intervals_disjoint, decode_object_store_segment,
32    is_remote_uri_scheme, make_object_store_path, parse_filename_timestamps,
33    query::{filter_identifier_files, is_parquet_bar_prefix},
34    query_intersects_filename, remote_full_uri, remote_store_root_url, timestamps_to_filename,
35    urisafe_instrument_id,
36};
37use crate::{
38    backend::parquet::consolidation_state,
39    catalog::types::{
40        CatalogDataType, catalog_data_type_from_path, parquet_catalog_data_type_path_prefixes,
41    },
42    common::{
43        paths::{escape_local_file_uri, normalize_path_separators},
44        storage::StorageBackend,
45    },
46};
47
48impl ParquetDataCatalog {
49    /// Extends the timestamp range of an existing Parquet file by renaming it.
50    ///
51    /// This method finds an existing file that is adjacent to the specified time range
52    /// and renames it to include the new range. This is useful when appending data
53    /// that extends the time coverage of existing files.
54    /// The proposed extension is validated against the other files before renaming,
55    /// so a rejected extension leaves existing files unchanged.
56    ///
57    /// If no file is adjacent to the specified range, this method does nothing and
58    /// returns `Ok(())` after confirming the existing intervals are disjoint.
59    ///
60    /// # Parameters
61    ///
62    /// - `data_type`: The stored family to target.
63    /// - `identifier`: Optional identifier to target a specific instrument's data. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
64    /// - `start`: Start timestamp of the new range to extend to.
65    /// - `end`: End timestamp of the new range to extend to.
66    ///
67    /// # Returns
68    ///
69    /// Returns `Ok(())` on success, or an error if the operation fails.
70    ///
71    /// # Errors
72    ///
73    /// Returns an error if:
74    /// - The range is reversed (`start` is after `end`).
75    /// - The directory path cannot be constructed.
76    /// - The proposed extension would overlap another file.
77    /// - The existing intervals are already overlapping.
78    /// - File rename operations fail.
79    ///
80    /// # Examples
81    ///
82    /// ```rust,no_run
83    /// use nautilus_core::UnixNanos;
84    /// use nautilus_model::data::NautilusDataType;
85    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
86    ///
87    /// let mut catalog = ParquetDataCatalog::new(
88    ///     std::path::Path::new("/tmp/nautilus_data"),
89    ///     None,
90    ///     None,
91    ///     None,
92    ///     None,
93    /// );
94    ///
95    /// // Extend a file's range backwards or forwards
96    /// catalog.extend_file_name(
97    ///     &NautilusDataType::QuoteTick.into(),
98    ///     Some("BTC/USD.SIM"),
99    ///     UnixNanos::from(1609459200000000000),
100    ///     UnixNanos::from(1609545600000000000),
101    /// )?;
102    /// # Ok::<(), anyhow::Error>(())
103    /// ```
104    pub fn extend_file_name(
105        &self,
106        data_type: &CatalogDataType,
107        identifier: Option<&str>,
108        start: UnixNanos,
109        end: UnixNanos,
110    ) -> anyhow::Result<()> {
111        let prefixes = parquet_catalog_data_type_path_prefixes(data_type);
112
113        if let [data_cls] = prefixes.as_slice() {
114            let directory = self.make_path(data_cls.as_ref(), identifier)?;
115            return self.extend_file_name_in_directory(&directory, start, end);
116        }
117
118        // The aggregate instrument family spans every class directory, and one identifier can be
119        // stored under several classes, so extend each directory that already holds it rather
120        // than inventing a class for it.
121        let mut extended = false;
122
123        for data_cls in &prefixes {
124            let directory = self.make_path(data_cls.as_ref(), identifier)?;
125            if !self.get_directory_intervals(&directory)?.is_empty() {
126                self.extend_file_name_in_directory(&directory, start, end)?;
127                extended = true;
128            }
129        }
130
131        anyhow::ensure!(
132            extended,
133            "Cannot extend file name for {data_type}: no instrument class holds {}; \
134             name the class with a NautilusInstrumentType",
135            identifier.unwrap_or("any identifier"),
136        );
137
138        Ok(())
139    }
140
141    pub(super) fn extend_file_name_in_directory(
142        &self,
143        directory: &str,
144        start: UnixNanos,
145        end: UnixNanos,
146    ) -> anyhow::Result<()> {
147        let start = start.as_u64();
148        let end = end.as_u64();
149
150        anyhow::ensure!(
151            start <= end,
152            "Cannot extend file in directory '{directory}': reversed range ({start}, {end})",
153        );
154
155        let intervals = self.get_directory_intervals(directory)?;
156
157        anyhow::ensure!(
158            are_intervals_disjoint(&intervals),
159            "Intervals are not disjoint in directory '{directory}': {intervals:?}",
160        );
161
162        let adjacent = intervals.iter().enumerate().find_map(|(index, interval)| {
163            if end.checked_add(1) == Some(interval.0) {
164                // Extend backwards: new file covers [start, interval.1]
165                Some((index, *interval, (start, interval.1)))
166            } else if start.checked_sub(1) == Some(interval.1) {
167                // Extend forwards: new file covers [interval.0, end]
168                Some((index, *interval, (interval.0, end)))
169            } else {
170                None
171            }
172        });
173
174        let Some((index, original, proposed)) = adjacent else {
175            return Ok(());
176        };
177
178        let mut extended = intervals.clone();
179        extended[index] = proposed;
180
181        anyhow::ensure!(
182            are_intervals_disjoint(&extended),
183            "Extending file interval {original:?} to {proposed:?} in directory '{directory}' \
184            with range ({start}, {end}) would create non-disjoint intervals. \
185            Existing intervals: {intervals:?}",
186        );
187
188        self.rename_parquet_file(directory, original.0, original.1, proposed.0, proposed.1)
189    }
190
191    /// Helper method to rename a parquet file by moving it via object store operations
192    fn rename_parquet_file(
193        &self,
194        directory: &str,
195        old_start: u64,
196        old_end: u64,
197        new_start: u64,
198        new_end: u64,
199    ) -> anyhow::Result<()> {
200        let new_filename =
201            timestamps_to_filename(UnixNanos::from(new_start), UnixNanos::from(new_end));
202        let new_path = format!("{directory}/{new_filename}");
203        let matches = self
204            .list_parquet_files(directory)?
205            .into_iter()
206            .filter(|file| parse_filename_timestamps(file) == Some((old_start, old_end)))
207            .collect::<Vec<_>>();
208
209        anyhow::ensure!(
210            matches.len() == 1,
211            "expected one parquet file for interval ({old_start}, {old_end}) in {directory}, \
212             found {}",
213            matches.len()
214        );
215
216        let old_path = &matches[0];
217        if old_path.ends_with(&new_filename) {
218            return Ok(());
219        }
220
221        let old_object_path = self.to_object_path_parsed(old_path)?;
222        let new_object_path = self.to_object_path(&new_path)?;
223        self.move_file(&old_object_path, &new_object_path)
224    }
225
226    /// Lists all Parquet files under a directory, including its subdirectories.
227    ///
228    /// This method scans a directory and returns the full paths of all files with the `.parquet`
229    /// extension. It works with both local filesystems and remote object stores, making it
230    /// suitable for various storage backends.
231    ///
232    /// # Parameters
233    ///
234    /// - `directory`: The directory path to scan for Parquet files, unencoded as
235    ///   [`Self::make_path`] returns it.
236    ///
237    /// # Returns
238    ///
239    /// Returns a vector of full file paths (as strings) for all Parquet files under the directory.
240    /// The paths are percent-encoded object-store keys relative to the store root; convert one with
241    /// [`Self::to_object_path_parsed`].
242    /// Returns an empty vector if the directory doesn't exist or contains no Parquet files.
243    ///
244    /// # Errors
245    ///
246    /// Returns an error if:
247    /// - Object store listing operations fail.
248    /// - Directory access is denied.
249    /// - Network issues occur (for remote object stores).
250    ///
251    /// # Notes
252    ///
253    /// - Only files ending with `.parquet` are included.
254    /// - Subdirectories are scanned recursively, so a type directory such as `data/quotes`
255    ///   returns the files of every identifier under it.
256    /// - File paths are returned in the order provided by the object store.
257    /// - Works with all supported object store backends (local, S3, GCS, Azure, etc.).
258    ///
259    /// # Examples
260    ///
261    /// ```rust,no_run
262    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
263    ///
264    /// let mut catalog = ParquetDataCatalog::new(
265    ///     std::path::Path::new("/tmp/nautilus_data"),
266    ///     None,
267    ///     None,
268    ///     None,
269    ///     None,
270    /// );
271    /// let files = catalog.list_parquet_files("data/quotes/EURUSD")?;
272    ///
273    /// for file in files {
274    ///     println!("Found Parquet file: {}", file);
275    /// }
276    /// # Ok::<(), anyhow::Error>(())
277    /// ```
278    pub fn list_parquet_files(&self, directory: &str) -> anyhow::Result<Vec<String>> {
279        let files = self
280            .list_objects(directory)?
281            .into_iter()
282            .filter(|object| object.location.as_ref().ends_with(".parquet"))
283            .map(|object| object.location.to_string())
284            .collect();
285
286        Ok(files)
287    }
288
289    /// Lists all instrument identifiers for a specific data type.
290    ///
291    /// This method scans the data directory for a given data type and extracts
292    /// all unique instrument identifiers from the directory structure.
293    ///
294    /// # Parameters
295    ///
296    /// - `data_type`: The stored family to target.
297    ///
298    /// # Returns
299    ///
300    /// Returns a vector of instrument identifier strings.
301    ///
302    /// # Errors
303    ///
304    /// Returns an error if directory listing fails.
305    pub fn list_instruments(&self, data_type: &CatalogDataType) -> anyhow::Result<Vec<String>> {
306        let mut instruments = Vec::new();
307
308        for prefix in parquet_catalog_data_type_path_prefixes(data_type) {
309            instruments.extend(self.list_prefix_instruments(prefix.as_ref())?);
310        }
311
312        // The same identifier can live under more than one instrument class.
313        instruments.sort();
314        instruments.dedup();
315
316        Ok(instruments)
317    }
318
319    fn list_prefix_instruments(&self, data_type: &str) -> anyhow::Result<Vec<String>> {
320        let directory = self.make_path(data_type, None)?;
321        let prefix = ObjectPath::from(directory.as_str());
322        let mut instruments = HashSet::new();
323
324        for object in self.list_objects(&directory)? {
325            // Relative to the prefix a datum is `{identifier}/{filename}.parquet`
326            let Some(relative) = object.location.prefix_match(&prefix) else {
327                continue;
328            };
329            let segments: Vec<_> = relative.collect();
330
331            if let [identifier, filename] = segments.as_slice()
332                && filename.as_ref().ends_with(".parquet")
333            {
334                instruments.insert(decode_object_store_segment(identifier.as_ref()));
335            }
336        }
337
338        Ok(instruments.into_iter().collect())
339    }
340
341    /// Lists Parquet files matching specific criteria (data type, identifiers, time range).
342    ///
343    /// This method finds all Parquet files that match the specified criteria by filtering
344    /// files based on their directory structure and filename timestamps.
345    ///
346    /// # Parameters
347    ///
348    /// - `data_type`: The stored family to target.
349    /// - `identifiers`: Optional list of identifiers to filter by.
350    /// - `start`: Optional start timestamp to filter files by their time range.
351    /// - `end`: Optional end timestamp to filter files by their time range.
352    ///
353    /// # Returns
354    ///
355    /// Returns a vector of file paths that match the criteria.
356    ///
357    /// # Errors
358    ///
359    /// Returns an error if directory listing or file filtering fails.
360    pub fn list_parquet_files_with_criteria(
361        &self,
362        data_type: &CatalogDataType,
363        identifiers: Option<&[String]>,
364        start: Option<UnixNanos>,
365        end: Option<UnixNanos>,
366    ) -> anyhow::Result<Vec<String>> {
367        let mut all_files = Vec::new();
368
369        for data_cls in parquet_catalog_data_type_path_prefixes(data_type) {
370            all_files.extend(self.list_prefix_files_with_criteria(
371                data_cls.as_ref(),
372                identifiers,
373                start,
374                end,
375            )?);
376        }
377
378        Ok(all_files)
379    }
380
381    fn list_prefix_files_with_criteria(
382        &self,
383        data_cls: &str,
384        identifiers: Option<&[String]>,
385        start: Option<UnixNanos>,
386        end: Option<UnixNanos>,
387    ) -> anyhow::Result<Vec<String>> {
388        let start_u64 = start.map(|s| s.as_u64());
389        let end_u64 = end.map(|e| e.as_u64());
390
391        let base_dir = self.make_path(data_cls, None)?;
392
393        // Use recursive listing to match Python's glob behavior
394        let mut all_files = self.list_parquet_files(&base_dir)?;
395
396        if let Some(identifiers) = identifiers {
397            all_files =
398                filter_identifier_files(all_files, identifiers, is_parquet_bar_prefix(data_cls));
399        }
400
401        all_files.retain(|path| query_intersects_filename(path, start_u64, end_u64));
402        Ok(all_files)
403    }
404
405    fn is_windows_local(&self) -> bool {
406        cfg!(target_os = "windows") && !self.is_remote_uri()
407    }
408
409    /// Encodes colons as `%3A` in an [`ObjectPath`] for Windows local filesystem compatibility.
410    pub(crate) fn encode_windows_colon_path(path: &ObjectPath) -> ObjectPath {
411        if path.as_ref().contains(':')
412            && let Ok(encoded) = ObjectPath::parse(path.as_ref().replace(':', "%3A"))
413        {
414            encoded
415        } else {
416            path.clone()
417        }
418    }
419
420    /// Converts a directory path to an [`ObjectPath`] listing prefix.
421    ///
422    /// On Windows local filesystem catalogs, colons in instrument identifiers are encoded
423    /// as `%3A` to match the on-disk directory names stored by `object_store::local::LocalFileSystem`.
424    pub(super) fn directory_listing_prefix(&self, dir: &str) -> ObjectPath {
425        let prefix = ObjectPath::from(format!("{dir}/"));
426
427        if self.is_windows_local() {
428            Self::encode_windows_colon_path(&prefix)
429        } else {
430            prefix
431        }
432    }
433
434    /// Recursively lists all objects under `{dir}/` in the object store.
435    pub(crate) fn list_objects(&self, dir: &str) -> anyhow::Result<Vec<ObjectMeta>> {
436        let prefix = self.directory_listing_prefix(dir);
437        self.execute_async(|| async {
438            let mut stream = self.object_store.list(Some(&prefix));
439            let mut objects = Vec::new();
440            while let Some(object) = stream.next().await {
441                objects.push(object?);
442            }
443
444            consolidation_state::visible_objects(&self.object_store, objects).await
445        })
446    }
447
448    pub(crate) fn recover_consolidation(&self, directory: &ObjectPath) -> anyhow::Result<()> {
449        self.execute_async(|| async {
450            consolidation_state::recover(&self.object_store, directory).await
451        })
452    }
453
454    pub(super) fn has_pending_consolidation(&self, directory: &str) -> anyhow::Result<bool> {
455        let directory = if self.is_remote_uri() {
456            self.to_object_path_parsed(directory)?
457        } else {
458            let uri = self.resolve_directory_for_datafusion(directory);
459            let path = ListingTableUrl::parse(&uri)?
460                .get_url()
461                .to_file_path()
462                .map_err(|()| anyhow::anyhow!("Invalid local file URI: {directory}"))?;
463            self.to_object_path_parsed(&path.to_string_lossy())?
464        };
465
466        self.execute_async(|| async {
467            consolidation_state::has_pending(&self.object_store, &directory).await
468        })
469    }
470
471    /// Reconstructs a local filesystem path or a remote object-store URI.
472    ///
473    /// Local catalog roots retain literal `%`, `#`, and `?` in directory names.
474    #[must_use]
475    pub fn reconstruct_full_uri(&self, path_str: &str) -> String {
476        if path_str.contains("://") {
477            return path_str.to_string();
478        }
479
480        // Check if this is a remote URI scheme that needs reconstruction
481        if self.is_remote_uri() {
482            let path = self.path_under_base(path_str);
483            if let Ok(uri) = remote_full_uri(&self.original_uri, &path) {
484                return uri;
485            }
486        }
487
488        // For local paths, extract the directory from the original URI
489        if self.original_uri.starts_with("file://") {
490            // Extract the path from the file:// URI
491            if let Ok(url) = url::Url::parse(&escape_local_file_uri(&self.original_uri))
492                && let Ok(base_path) = url.to_file_path()
493            {
494                // Use platform-appropriate path separator for display
495                // but object store paths always use forward slashes
496                let base_str = base_path.to_string_lossy();
497                return make_object_store_path(&base_str, [path_str]);
498            }
499        }
500
501        // For local paths without file:// prefix, use the original URI as base
502        if self.base_path.is_empty() {
503            // If base_path is empty and not a file URI, try using original_uri as base
504            if self.original_uri.contains("://") {
505                // Fallback: return the path as-is
506                path_str.to_string()
507            } else {
508                make_object_store_path(self.original_uri.trim_end_matches('/'), [path_str])
509            }
510        } else {
511            let base = self.base_path.trim_end_matches('/');
512            make_object_store_path(base, [path_str])
513        }
514    }
515
516    /// Resolves a path for use with DataFusion (avoiding Windows path doubling for file://).
517    /// A remote path resolves to a URL under the store root whose path decodes to the object
518    /// key. A local path is returned as-is if it is already a full URI or absolute; otherwise it
519    /// joins the file:// base or goes through `reconstruct_full_uri`.
520    #[must_use]
521    pub(crate) fn resolve_path_for_datafusion(&self, path: &str) -> String {
522        // DataFusion decodes the URL path once, so encode the key again
523        if self.is_remote_uri()
524            && let Ok(key) = self.object_store_path(path)
525            && let Ok(root) = remote_store_root_url(&self.original_uri)
526        {
527            return append_path_to_file_uri(root.as_str(), &key);
528        }
529
530        if path.contains("://") {
531            return path.to_string();
532        }
533
534        if path.starts_with('/') {
535            return path.to_string();
536        }
537
538        if self.original_uri.starts_with("file://") {
539            return append_path_to_file_uri(&escape_local_file_uri(&self.original_uri), path);
540        }
541
542        self.reconstruct_full_uri(path)
543    }
544
545    /// Like `resolve_path_for_datafusion` but ensures the result ends with a trailing slash.
546    #[must_use]
547    pub(super) fn resolve_directory_for_datafusion(&self, directory: &str) -> String {
548        let mut resolved = self.resolve_path_for_datafusion(directory);
549        if !resolved.ends_with('/') {
550            resolved.push('/');
551        }
552
553        resolved
554    }
555
556    /// Returns the path string to push in `query_files` result list: relative for file://,
557    /// full URI for remote (so callers can pass to `resolve_path_for_datafusion` later).
558    #[must_use]
559    pub(super) fn path_for_query_list(&self, path: &str) -> String {
560        if self.original_uri.starts_with("file://") {
561            path.to_string()
562        } else {
563            self.reconstruct_full_uri(path)
564        }
565    }
566
567    /// Returns the native path string for the catalog root (for `std::fs`). Only valid when
568    /// !`is_remote_uri()`; uses parquet's `file_uri_to_native_path` for file:// URIs.
569    #[must_use]
570    pub(crate) fn native_base_path_string(&self) -> String {
571        if self.original_uri.starts_with("file://") {
572            crate::backend::parquet::io::file_uri_to_native_path(&self.original_uri)
573        } else {
574            self.original_uri.clone()
575        }
576    }
577
578    pub(crate) fn register_remote_object_store(&mut self) -> anyhow::Result<()> {
579        if self.is_remote_uri() {
580            let base_url = remote_store_root_url(&self.original_uri)?;
581            self.session
582                .register_object_store(&base_url, self.object_store.clone());
583        }
584
585        Ok(())
586    }
587
588    /// Helper method to check if the original URI uses a remote object store scheme
589    #[must_use]
590    pub fn is_remote_uri(&self) -> bool {
591        self.original_uri
592            .split_once("://")
593            .is_some_and(|(scheme, _)| is_remote_uri_scheme(scheme))
594    }
595
596    /// Constructs a directory path for storing data of a specific type and instrument.
597    ///
598    /// This method builds the hierarchical directory structure used by the catalog to organize
599    /// data by type and instrument. The path follows the pattern: `{base_path}/data/{type_name}/{instrument_id}`.
600    /// Instrument IDs are automatically converted to URI-safe format by removing forward slashes.
601    ///
602    /// # Parameters
603    ///
604    /// - `type_name`: The data type directory name (e.g., "quotes", "trades", "bars").
605    /// - `identifier`: Optional identifier. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL"). If provided, creates a subdirectory for the identifier. If `None`, returns the path to the data type directory.
606    ///
607    /// # Returns
608    ///
609    /// Returns the constructed directory path as a string, or an error if path construction fails.
610    ///
611    /// # Errors
612    ///
613    /// Returns an error if:
614    /// - The instrument ID contains invalid characters that cannot be made URI-safe.
615    /// - Path construction fails due to system limitations.
616    ///
617    /// # Path Structure
618    ///
619    /// - Without identifier: `{base_path}/data/{type_name}`.
620    /// - With identifier: `{base_path}/data/{type_name}/{safe_identifier}`.
621    /// - If `base_path` is empty: `data/{type_name}[/{safe_identifier}]`.
622    ///
623    /// # Examples
624    ///
625    /// ```rust,no_run
626    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
627    ///
628    /// let mut catalog = ParquetDataCatalog::new(
629    ///     std::path::Path::new("/tmp/nautilus_data"),
630    ///     None,
631    ///     None,
632    ///     None,
633    ///     None,
634    /// );
635    ///
636    /// // Path for all quote data
637    /// let quotes_path = catalog.make_path("quotes", None)?;
638    /// // Returns: "/base/path/data/quotes"
639    ///
640    /// // Path for specific instrument quotes
641    /// let eurusd_quotes = catalog.make_path("quotes", Some("EUR/USD"))?;
642    /// // Returns: "/base/path/data/quotes/EURUSD" (slash removed)
643    ///
644    /// // Path for bar data with complex instrument ID
645    /// let bars_path = catalog.make_path("bars", Some("BTC/USD-1H"))?;
646    /// // Returns: "/base/path/data/bars/BTCUSD-1H"
647    /// # Ok::<(), anyhow::Error>(())
648    /// ```
649    pub fn make_path(&self, type_name: &str, identifier: Option<&str>) -> anyhow::Result<String> {
650        let mut components = vec!["data".to_string(), type_name.to_string()];
651
652        if let Some(id) = identifier {
653            let safe_id = urisafe_instrument_id(id);
654            components.push(safe_id);
655        }
656
657        let path = make_object_store_path(&self.base_path, components);
658        Ok(path)
659    }
660
661    /// Builds the directory path for custom data: `data/custom/{type_name}[/{identifier}]`.
662    pub fn make_path_custom_data(
663        &self,
664        type_name: &str,
665        identifier: Option<&str>,
666    ) -> anyhow::Result<String> {
667        let mut components = vec![
668            "data".to_string(),
669            "custom".to_string(),
670            type_name.to_string(),
671        ];
672
673        if let Some(id) = identifier {
674            let safe_id = urisafe_instrument_id(id);
675
676            if !safe_id.is_empty() {
677                components.push(safe_id);
678            }
679        }
680
681        let path = make_object_store_path(&self.base_path, components);
682        Ok(path)
683    }
684
685    /// Converts a catalog path string to an [`ObjectPath`] for object store operations.
686    ///
687    /// This method handles the conversion between catalog-relative paths and object store paths,
688    /// taking into account the catalog's base path configuration. It automatically preserves the
689    /// base path prefix for remote catalogs and strips it for local catalog paths.
690    ///
691    /// # Parameters
692    ///
693    /// - `path`: The catalog path string to convert. Can be absolute or relative.
694    ///
695    /// # Returns
696    ///
697    /// Returns an [`ObjectPath`] suitable for use with object store operations.
698    ///
699    /// # Path Handling
700    ///
701    /// - If `base_path` is empty, the path is used as-is.
702    /// - If `base_path` is set for a remote catalog, it's preserved or prepended.
703    /// - If `base_path` is set for a local catalog, it's stripped from the path if present.
704    /// - Trailing slashes and backslashes are automatically handled.
705    /// - The resulting path is relative to the object store root.
706    /// - All paths are normalized to use forward slashes (object store convention).
707    ///
708    /// # Errors
709    ///
710    /// Returns an error if the catalog is remote and `path` is a full URI whose scheme/host does
711    /// not match the catalog's own root (cross-bucket misuse). Without this guard the caller could
712    /// silently write to or read from the wrong bucket.
713    ///
714    /// # Examples
715    ///
716    /// Local catalog paths (absolute or relative) strip the catalog's base directory:
717    ///
718    /// ```rust,no_run
719    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
720    /// # let catalog: ParquetDataCatalog = unimplemented!();
721    /// let object_path = catalog.to_object_path("/base/data/quotes/file.parquet")?;
722    /// // ObjectPath("data/quotes/file.parquet")
723    /// # Ok::<(), anyhow::Error>(())
724    /// ```
725    ///
726    /// Remote catalog paths (relative or full URI) preserve or prepend the base prefix:
727    ///
728    /// ```rust,no_run
729    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
730    /// # let catalog: ParquetDataCatalog = unimplemented!();
731    /// let object_path = catalog.to_object_path("data/trades/file.parquet")?;
732    /// // ObjectPath("base/data/trades/file.parquet")
733    /// # Ok::<(), anyhow::Error>(())
734    /// ```
735    pub fn to_object_path(&self, path: &str) -> anyhow::Result<ObjectPath> {
736        let store_path = self.object_store_path(path)?;
737        let object_path = ObjectPath::from(store_path);
738
739        if self.is_windows_local() {
740            Ok(Self::encode_windows_colon_path(&object_path))
741        } else {
742            Ok(object_path)
743        }
744    }
745
746    /// Converts a path string to [`ObjectPath`] using parse (no percent-encoding).
747    ///
748    /// Use this for paths that were returned by the object store (e.g. from `list()`),
749    /// which may already be percent-encoded. Using [`Self::to_object_path`] (which uses
750    /// `Path::from`) on such paths would double-encode (e.g. `%5E` -> `%255E`).
751    ///
752    /// # Errors
753    ///
754    /// Returns an error if the same cross-bucket condition as [`Self::to_object_path`] occurs or
755    /// the resulting string fails [`ObjectPath::parse`].
756    pub fn to_object_path_parsed(&self, path: &str) -> anyhow::Result<ObjectPath> {
757        let to_parse = self.object_store_path(path)?;
758        ObjectPath::parse(&to_parse).map_err(anyhow::Error::from)
759    }
760
761    pub(super) fn object_store_path(&self, path: &str) -> anyhow::Result<String> {
762        let normalized_path = normalize_path_separators(path);
763
764        if self.is_remote_uri() {
765            if normalized_path.contains("://") {
766                let path_under_root = self.remote_uri_object_path(&normalized_path)?;
767                return Ok(self.path_under_base(&path_under_root));
768            }
769
770            return Ok(self.path_under_base(&normalized_path));
771        }
772
773        Ok(self.path_without_local_base(&normalized_path))
774    }
775
776    fn remote_uri_object_path(&self, path: &str) -> anyhow::Result<String> {
777        let path_url = url::Url::parse(path)
778            .map_err(|e| anyhow::anyhow!("Failed to parse object store URI {path}: {e}"))?;
779        if !is_remote_uri_scheme(path_url.scheme()) {
780            anyhow::bail!(
781                "URI {path} uses non-remote scheme {} for remote catalog at {}",
782                path_url.scheme(),
783                self.original_uri,
784            );
785        }
786
787        let catalog_root = remote_store_root_url(&self.original_uri)?;
788        let path_root = remote_store_root_url(path)?;
789        if catalog_root.as_str().trim_end_matches('/') != path_root.as_str().trim_end_matches('/') {
790            anyhow::bail!(
791                "Cross-store URI {path} (root {}) does not belong to catalog rooted at {} ({})",
792                path_root.as_str().trim_end_matches('/'),
793                self.original_uri,
794                catalog_root.as_str().trim_end_matches('/'),
795            );
796        }
797
798        // The URL crate keeps the path component percent-encoded (e.g. `%5E`),
799        // so preserve that encoding for `ObjectPath::parse` round-trips through
800        // `object_store::list`/`get`.
801        Ok(path_url.path().trim_start_matches('/').to_string())
802    }
803
804    pub(crate) fn path_without_local_base(&self, path: &str) -> String {
805        let base_path = if self.base_path.is_empty() {
806            self.native_base_path_string()
807        } else {
808            self.base_path.clone()
809        };
810
811        let normalized_base = normalize_path_separators(&base_path);
812        let base = normalized_base.trim_end_matches('/');
813
814        if base.is_empty() {
815            path.to_string()
816        } else if path == base {
817            String::new()
818        } else if let Some(without_base) = path.strip_prefix(&format!("{base}/")) {
819            without_base.to_string()
820        } else {
821            path.to_string()
822        }
823    }
824
825    fn path_under_base(&self, path: &str) -> String {
826        let normalized_path = normalize_path_separators(path);
827        let path = normalized_path
828            .trim_start_matches('/')
829            .trim_end_matches('/');
830
831        if self.base_path.is_empty() {
832            return path.to_string();
833        }
834
835        let normalized_base = normalize_path_separators(&self.base_path);
836        let base = normalized_base
837            .trim_start_matches('/')
838            .trim_end_matches('/');
839
840        if base.is_empty() || path == base || path.starts_with(&format!("{base}/")) {
841            path.to_string()
842        } else if path.is_empty() {
843            base.to_string()
844        } else {
845            make_object_store_path(base, [path])
846        }
847    }
848
849    /// Helper method to move a file using object store rename operation
850    pub fn move_file(&self, old_path: &ObjectPath, new_path: &ObjectPath) -> anyhow::Result<()> {
851        if old_path == new_path {
852            return Ok(());
853        }
854
855        self.recover_consolidation(&old_path.parent().unwrap_or_default())?;
856        self.recover_consolidation(&new_path.parent().unwrap_or_default())?;
857
858        self.execute_async(|| async {
859            self.object_store
860                .rename(old_path, new_path)
861                .await
862                .map_err(anyhow::Error::from)
863        })
864    }
865
866    /// Helper method to execute async operations with a runtime
867    pub fn execute_async<C, F, R>(&self, create_future: C) -> anyhow::Result<R>
868    where
869        C: FnOnce() -> F + Send,
870        F: std::future::Future<Output = anyhow::Result<R>>,
871        R: Send,
872    {
873        block_on_nautilus_with(create_future)
874    }
875
876    /// Lists directory stems (directory names without path) in a subdirectory.
877    ///
878    /// This method scans a subdirectory and returns the names of all immediate
879    /// subdirectories. It's used to list data types, backtest runs, and live runs.
880    ///
881    /// # Parameters
882    ///
883    /// - `subdirectory`: The subdirectory path to scan (e.g., "data", "backtest", "live").
884    ///
885    /// # Returns
886    ///
887    /// Returns the sorted directory names (stems) found in the subdirectory, decoded from their
888    /// percent-encoded object-store form, or an error if the operation fails.
889    ///
890    /// # Errors
891    ///
892    /// Returns an error if:
893    /// - Object store listing operations fail.
894    /// - Directory access is denied.
895    ///
896    /// # Examples
897    ///
898    /// ```rust,no_run
899    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
900    ///
901    /// let mut catalog = ParquetDataCatalog::new(
902    ///     std::path::Path::new("/tmp/nautilus_data"),
903    ///     None,
904    ///     None,
905    ///     None,
906    ///     None,
907    /// );
908    ///
909    /// // List all data types
910    /// let data_types = catalog.list_directory_stems("data")?;
911    /// for data_type in data_types {
912    ///     println!("Found data type: {}", data_type);
913    /// }
914    /// # Ok::<(), anyhow::Error>(())
915    /// ```
916    pub fn list_directory_stems(&self, subdirectory: &str) -> anyhow::Result<Vec<String>> {
917        // For local filesystem paths, use filesystem operations to detect empty directories
918        // For remote object stores, we can only list directories that contain files
919        if !self.is_remote_uri() {
920            let directory = PathBuf::from(self.native_base_path_string()).join(subdirectory);
921
922            let entries = match std::fs::read_dir(&directory) {
923                Ok(entries) => entries,
924                Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
925                Err(e) => {
926                    return Err(e)
927                        .with_context(|| format!("failed to list {}", directory.display()));
928                }
929            };
930
931            let mut directories = Vec::new();
932
933            for entry in entries {
934                let entry =
935                    entry.with_context(|| format!("failed to list {}", directory.display()))?;
936                let file_type = entry
937                    .file_type()
938                    .with_context(|| format!("failed to read {}", entry.path().display()))?;
939
940                // Use file_name() to get the directory name (not file_stem which removes extension)
941                if file_type.is_dir()
942                    && let Some(name) = entry.path().file_name()
943                {
944                    directories.push(decode_object_store_segment(&name.to_string_lossy()));
945                }
946            }
947
948            directories.sort();
949            return Ok(directories);
950        }
951
952        let storage = StorageBackend {
953            object_store: self.object_store.clone(),
954            base_path: self.base_path.clone(),
955            original_uri: self.original_uri.clone(),
956        };
957
958        self.execute_async(|| async { storage.list_directory_stems(subdirectory).await })
959    }
960
961    /// Lists the data, record, and instrument types stored in the catalog.
962    ///
963    /// Each `data/<type>` directory maps to its catalog type, and each `data/custom/<type_name>`
964    /// directory to a custom data type. Unrecognized directories are skipped with a warning.
965    ///
966    /// # Returns
967    ///
968    /// Returns the catalog types, or an error if the operation fails.
969    ///
970    /// # Errors
971    ///
972    /// Returns an error if:
973    /// - Object store listing operations fail.
974    /// - Directory access is denied.
975    ///
976    /// # Examples
977    ///
978    /// ```rust,no_run
979    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
980    ///
981    /// let mut catalog = ParquetDataCatalog::new(
982    ///     std::path::Path::new("/tmp/nautilus_data"),
983    ///     None,
984    ///     None,
985    ///     None,
986    ///     None,
987    /// );
988    ///
989    /// // List all data types
990    /// let data_types = catalog.list_data_types()?;
991    /// for data_type in data_types {
992    ///     println!("Available data type: {data_type}");
993    /// }
994    /// # Ok::<(), anyhow::Error>(())
995    /// ```
996    pub fn list_data_types(&self) -> anyhow::Result<Vec<CatalogDataType>> {
997        let mut data_types = Vec::new();
998
999        for stem in self.list_directory_stems("data")? {
1000            if stem == "custom" {
1001                for type_name in self.list_directory_stems("data/custom")? {
1002                    data_types.push(CatalogDataType::from(NautilusDataType::Custom {
1003                        type_name,
1004                    }));
1005                }
1006
1007                continue;
1008            }
1009
1010            match catalog_data_type_from_path(&stem) {
1011                Ok(data_type) => data_types.push(data_type),
1012                Err(e) => log::warn!("Skipping catalog directory data/{stem}: {e}"),
1013            }
1014        }
1015
1016        Ok(data_types)
1017    }
1018
1019    /// Lists all backtest run IDs available in the catalog.
1020    ///
1021    /// This method returns the names of all backtest run directories in the catalog.
1022    /// Each backtest run corresponds to a specific backtest execution instance.
1023    ///
1024    /// # Returns
1025    ///
1026    /// Returns a vector of backtest run IDs, or an error if the operation fails.
1027    ///
1028    /// # Errors
1029    ///
1030    /// Returns an error if:
1031    /// - Object store listing operations fail.
1032    /// - Directory access is denied.
1033    ///
1034    /// # Examples
1035    ///
1036    /// ```rust,no_run
1037    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
1038    ///
1039    /// let mut catalog = ParquetDataCatalog::new(
1040    ///     std::path::Path::new("/tmp/nautilus_data"),
1041    ///     None,
1042    ///     None,
1043    ///     None,
1044    ///     None,
1045    /// );
1046    ///
1047    /// // List all backtest runs
1048    /// let runs = catalog.list_backtest_runs()?;
1049    /// for run_id in runs {
1050    ///     println!("Backtest run: {}", run_id);
1051    /// }
1052    /// # Ok::<(), anyhow::Error>(())
1053    /// ```
1054    pub fn list_backtest_runs(&self) -> anyhow::Result<Vec<String>> {
1055        self.list_directory_stems("backtest")
1056    }
1057
1058    /// Lists all live run IDs available in the catalog.
1059    ///
1060    /// This method returns the names of all live run directories in the catalog.
1061    /// Each live run corresponds to a specific live trading execution instance.
1062    ///
1063    /// # Returns
1064    ///
1065    /// Returns a vector of live run IDs, or an error if the operation fails.
1066    ///
1067    /// # Errors
1068    ///
1069    /// Returns an error if:
1070    /// - Object store listing operations fail.
1071    /// - Directory access is denied.
1072    ///
1073    /// # Examples
1074    ///
1075    /// ```rust,no_run
1076    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
1077    ///
1078    /// let mut catalog = ParquetDataCatalog::new(
1079    ///     std::path::Path::new("/tmp/nautilus_data"),
1080    ///     None,
1081    ///     None,
1082    ///     None,
1083    ///     None,
1084    /// );
1085    ///
1086    /// // List all live runs
1087    /// let runs = catalog.list_live_runs()?;
1088    /// for run_id in runs {
1089    ///     println!("Live run: {}", run_id);
1090    /// }
1091    /// # Ok::<(), anyhow::Error>(())
1092    /// ```
1093    pub fn list_live_runs(&self) -> anyhow::Result<Vec<String>> {
1094        self.list_directory_stems("live")
1095    }
1096}
1097
1098#[cfg(test)]
1099mod tests {
1100    use std::sync::Arc;
1101
1102    use nautilus_model::data::NautilusDataType;
1103    use object_store::{ObjectStoreExt, PutPayload, memory::InMemory, path::Path as ObjectPath};
1104    use rstest::rstest;
1105
1106    use super::ParquetDataCatalog;
1107    use crate::{catalog::types::CatalogDataType, common::datafusion::DataBackendSession};
1108
1109    /// Builds a catalog over an in-memory store. Only remote catalogs carry a non-empty
1110    /// `base_path`, so seeding one here is the sole way to reproduce a bucket sub-prefix.
1111    fn memory_catalog(base_path: &str) -> ParquetDataCatalog {
1112        ParquetDataCatalog {
1113            base_path: base_path.to_string(),
1114            original_uri: "memory://".to_string(),
1115            object_store: Arc::new(InMemory::new()),
1116            session: DataBackendSession::new(5_000),
1117            batch_size: 5_000,
1118            compression: parquet::basic::Compression::SNAPPY,
1119            max_row_group_size: 5_000,
1120        }
1121    }
1122
1123    fn seed(catalog: &ParquetDataCatalog, keys: &[&str]) {
1124        catalog
1125            .execute_async(|| async {
1126                for key in keys {
1127                    catalog
1128                        .object_store
1129                        .put(&ObjectPath::from(*key), PutPayload::from_static(b"x"))
1130                        .await?;
1131                }
1132                Ok(())
1133            })
1134            .unwrap();
1135    }
1136
1137    #[rstest]
1138    #[case("nautilus-data", "nautilus-data/")]
1139    #[case("", "")]
1140    fn list_instruments_returns_ids_under_base_path(
1141        #[case] base_path: &str,
1142        #[case] key_prefix: &str,
1143    ) {
1144        let catalog = memory_catalog(base_path);
1145        let keys = [
1146            "data/quotes/EURUSD.SIM/0-1.parquet",
1147            "data/quotes/EURUSD.SIM/2-3.parquet",
1148            "data/quotes/GBPUSD.SIM/0-1.parquet",
1149            "data/trades/AUDUSD.SIM/0-1.parquet",
1150        ]
1151        .map(|key| format!("{key_prefix}{key}"));
1152        seed(&catalog, &keys.each_ref().map(String::as_str));
1153
1154        assert_eq!(
1155            catalog
1156                .list_instruments(&NautilusDataType::QuoteTick.into())
1157                .unwrap(),
1158            ["EURUSD.SIM", "GBPUSD.SIM"]
1159        );
1160    }
1161
1162    #[rstest]
1163    #[case("nautilus-data", "nautilus-data/")]
1164    #[case("", "")]
1165    fn list_parquet_files_lists_type_directory_recursively(
1166        #[case] base_path: &str,
1167        #[case] key_prefix: &str,
1168    ) {
1169        let catalog = memory_catalog(base_path);
1170        let keys = [
1171            "data/quotes/EURUSD.SIM/0-1.parquet",
1172            "data/quotes/EURUSD.SIM/0-1.json",
1173            "data/quotes/BTC€.SIM/2-3.parquet",
1174            "data/trades/AUDUSD.SIM/0-1.parquet",
1175        ]
1176        .map(|key| format!("{key_prefix}{key}"));
1177        seed(&catalog, &keys.each_ref().map(String::as_str));
1178
1179        let directory = catalog.make_path("quotes", None).unwrap();
1180        let mut files = catalog.list_parquet_files(&directory).unwrap();
1181        files.sort();
1182
1183        assert_eq!(
1184            files,
1185            [
1186                format!("{key_prefix}data/quotes/BTC%E2%82%AC.SIM/2-3.parquet"),
1187                format!("{key_prefix}data/quotes/EURUSD.SIM/0-1.parquet"),
1188            ]
1189        );
1190    }
1191
1192    #[rstest]
1193    fn list_instruments_ignores_unpartitioned_files() {
1194        // An empty `base_path` keeps this distinct from the remote prefix defect, so an empty
1195        // result can only come from the layout check.
1196        let catalog = memory_catalog("");
1197        seed(
1198            &catalog,
1199            &[
1200                "data/custom/MyType/1-2.parquet",
1201                "data/custom/MyType/1-2.json",
1202            ],
1203        );
1204
1205        let custom = CatalogDataType::Data(NautilusDataType::Custom {
1206            type_name: "MyType".to_string(),
1207        });
1208
1209        assert!(catalog.list_instruments(&custom).unwrap().is_empty());
1210    }
1211
1212    #[rstest]
1213    fn list_instruments_decodes_percent_encoded_ids() {
1214        let catalog = memory_catalog("nautilus-data");
1215        seed(
1216            &catalog,
1217            &["nautilus-data/data/quotes/BTC€.SIM/0-1.parquet"],
1218        );
1219
1220        assert_eq!(
1221            catalog
1222                .list_instruments(&NautilusDataType::QuoteTick.into())
1223                .unwrap(),
1224            ["BTC€.SIM"]
1225        );
1226    }
1227
1228    #[rstest]
1229    #[case::local(false)]
1230    #[case::remote(true)]
1231    fn test_directory_listing_prefix_matches_object_path(#[case] remote: bool) {
1232        let mut catalog = memory_catalog("");
1233        if remote {
1234            catalog.original_uri = "s3://test-bucket".to_string();
1235        }
1236
1237        let directory = "data/trades/io:ANTH-USD-PERP.HYPERLIQUID";
1238        let prefix = catalog.directory_listing_prefix(directory);
1239        let target = catalog.to_object_path(directory).unwrap();
1240
1241        let expected = if cfg!(target_os = "windows") && !remote {
1242            "data/trades/io%3AANTH-USD-PERP.HYPERLIQUID"
1243        } else {
1244            directory
1245        };
1246
1247        assert_eq!(prefix.as_ref(), expected);
1248        assert_eq!(target, prefix);
1249    }
1250
1251    #[rstest]
1252    #[case(
1253        "data/trades/io:ANTH-USD-PERP.HYPERLIQUID",
1254        "data/trades/io%3AANTH-USD-PERP.HYPERLIQUID"
1255    )]
1256    #[case("data/quotes/BTC€.SIM", "data/quotes/BTC%E2%82%AC.SIM")]
1257    #[case("data/quotes/EURUSD", "data/quotes/EURUSD")]
1258    fn test_encode_windows_colon_path(#[case] input: &str, #[case] expected: &str) {
1259        let path = ObjectPath::from(input);
1260        let encoded = ParquetDataCatalog::encode_windows_colon_path(&path);
1261        assert_eq!(encoded.as_ref(), expected);
1262    }
1263}