nautilus_persistence/backend/parquet/catalog/store.rs
1// -------------------------------------------------------------------------------------------------
2// Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3// https://nautechsystems.io
4//
5// Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6// You may not use this file except in compliance with the License.
7// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Object-store paths, listing, and file administration for the Parquet catalog.
17
18#![expect(
19 clippy::missing_errors_doc,
20 reason = "catalog store functions forward object-store errors"
21)]
22
23use anyhow::Context;
24use datafusion::datasource::listing::ListingTableUrl;
25use nautilus_common::live::block_on_nautilus_with;
26use nautilus_model::data::NautilusDataType;
27use object_store::ObjectMeta;
28
29use super::{
30 HashSet, ObjectPath, ObjectStore, ObjectStoreExt, ParquetDataCatalog, PathBuf, StreamExt,
31 UnixNanos, append_path_to_file_uri, are_intervals_disjoint, decode_object_store_segment,
32 is_remote_uri_scheme, make_object_store_path, parse_filename_timestamps,
33 query::{filter_identifier_files, is_parquet_bar_prefix},
34 query_intersects_filename, remote_full_uri, remote_store_root_url, timestamps_to_filename,
35 urisafe_instrument_id,
36};
37use crate::{
38 backend::parquet::consolidation_state,
39 catalog::types::{
40 CatalogDataType, catalog_data_type_from_path, parquet_catalog_data_type_path_prefixes,
41 },
42 common::{
43 paths::{escape_local_file_uri, normalize_path_separators},
44 storage::StorageBackend,
45 },
46};
47
48impl ParquetDataCatalog {
49 /// Extends the timestamp range of an existing Parquet file by renaming it.
50 ///
51 /// This method finds an existing file that is adjacent to the specified time range
52 /// and renames it to include the new range. This is useful when appending data
53 /// that extends the time coverage of existing files.
54 /// The proposed extension is validated against the other files before renaming,
55 /// so a rejected extension leaves existing files unchanged.
56 ///
57 /// If no file is adjacent to the specified range, this method does nothing and
58 /// returns `Ok(())` after confirming the existing intervals are disjoint.
59 ///
60 /// # Parameters
61 ///
62 /// - `data_type`: The stored family to target.
63 /// - `identifier`: Optional identifier to target a specific instrument's data. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
64 /// - `start`: Start timestamp of the new range to extend to.
65 /// - `end`: End timestamp of the new range to extend to.
66 ///
67 /// # Returns
68 ///
69 /// Returns `Ok(())` on success, or an error if the operation fails.
70 ///
71 /// # Errors
72 ///
73 /// Returns an error if:
74 /// - The range is reversed (`start` is after `end`).
75 /// - The directory path cannot be constructed.
76 /// - The proposed extension would overlap another file.
77 /// - The existing intervals are already overlapping.
78 /// - File rename operations fail.
79 ///
80 /// # Examples
81 ///
82 /// ```rust,no_run
83 /// use nautilus_core::UnixNanos;
84 /// use nautilus_model::data::NautilusDataType;
85 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
86 ///
87 /// let mut catalog = ParquetDataCatalog::new(
88 /// std::path::Path::new("/tmp/nautilus_data"),
89 /// None,
90 /// None,
91 /// None,
92 /// None,
93 /// );
94 ///
95 /// // Extend a file's range backwards or forwards
96 /// catalog.extend_file_name(
97 /// &NautilusDataType::QuoteTick.into(),
98 /// Some("BTC/USD.SIM"),
99 /// UnixNanos::from(1609459200000000000),
100 /// UnixNanos::from(1609545600000000000),
101 /// )?;
102 /// # Ok::<(), anyhow::Error>(())
103 /// ```
104 pub fn extend_file_name(
105 &self,
106 data_type: &CatalogDataType,
107 identifier: Option<&str>,
108 start: UnixNanos,
109 end: UnixNanos,
110 ) -> anyhow::Result<()> {
111 let prefixes = parquet_catalog_data_type_path_prefixes(data_type);
112
113 if let [data_cls] = prefixes.as_slice() {
114 let directory = self.make_path(data_cls.as_ref(), identifier)?;
115 return self.extend_file_name_in_directory(&directory, start, end);
116 }
117
118 // The aggregate instrument family spans every class directory, and one identifier can be
119 // stored under several classes, so extend each directory that already holds it rather
120 // than inventing a class for it.
121 let mut extended = false;
122
123 for data_cls in &prefixes {
124 let directory = self.make_path(data_cls.as_ref(), identifier)?;
125 if !self.get_directory_intervals(&directory)?.is_empty() {
126 self.extend_file_name_in_directory(&directory, start, end)?;
127 extended = true;
128 }
129 }
130
131 anyhow::ensure!(
132 extended,
133 "Cannot extend file name for {data_type}: no instrument class holds {}; \
134 name the class with a NautilusInstrumentType",
135 identifier.unwrap_or("any identifier"),
136 );
137
138 Ok(())
139 }
140
141 pub(super) fn extend_file_name_in_directory(
142 &self,
143 directory: &str,
144 start: UnixNanos,
145 end: UnixNanos,
146 ) -> anyhow::Result<()> {
147 let start = start.as_u64();
148 let end = end.as_u64();
149
150 anyhow::ensure!(
151 start <= end,
152 "Cannot extend file in directory '{directory}': reversed range ({start}, {end})",
153 );
154
155 let intervals = self.get_directory_intervals(directory)?;
156
157 anyhow::ensure!(
158 are_intervals_disjoint(&intervals),
159 "Intervals are not disjoint in directory '{directory}': {intervals:?}",
160 );
161
162 let adjacent = intervals.iter().enumerate().find_map(|(index, interval)| {
163 if end.checked_add(1) == Some(interval.0) {
164 // Extend backwards: new file covers [start, interval.1]
165 Some((index, *interval, (start, interval.1)))
166 } else if start.checked_sub(1) == Some(interval.1) {
167 // Extend forwards: new file covers [interval.0, end]
168 Some((index, *interval, (interval.0, end)))
169 } else {
170 None
171 }
172 });
173
174 let Some((index, original, proposed)) = adjacent else {
175 return Ok(());
176 };
177
178 let mut extended = intervals.clone();
179 extended[index] = proposed;
180
181 anyhow::ensure!(
182 are_intervals_disjoint(&extended),
183 "Extending file interval {original:?} to {proposed:?} in directory '{directory}' \
184 with range ({start}, {end}) would create non-disjoint intervals. \
185 Existing intervals: {intervals:?}",
186 );
187
188 self.rename_parquet_file(directory, original.0, original.1, proposed.0, proposed.1)
189 }
190
191 /// Helper method to rename a parquet file by moving it via object store operations
192 fn rename_parquet_file(
193 &self,
194 directory: &str,
195 old_start: u64,
196 old_end: u64,
197 new_start: u64,
198 new_end: u64,
199 ) -> anyhow::Result<()> {
200 let new_filename =
201 timestamps_to_filename(UnixNanos::from(new_start), UnixNanos::from(new_end));
202 let new_path = format!("{directory}/{new_filename}");
203 let matches = self
204 .list_parquet_files(directory)?
205 .into_iter()
206 .filter(|file| parse_filename_timestamps(file) == Some((old_start, old_end)))
207 .collect::<Vec<_>>();
208
209 anyhow::ensure!(
210 matches.len() == 1,
211 "expected one parquet file for interval ({old_start}, {old_end}) in {directory}, \
212 found {}",
213 matches.len()
214 );
215
216 let old_path = &matches[0];
217 if old_path.ends_with(&new_filename) {
218 return Ok(());
219 }
220
221 let old_object_path = self.to_object_path_parsed(old_path)?;
222 let new_object_path = self.to_object_path(&new_path)?;
223 self.move_file(&old_object_path, &new_object_path)
224 }
225
226 /// Lists all Parquet files under a directory, including its subdirectories.
227 ///
228 /// This method scans a directory and returns the full paths of all files with the `.parquet`
229 /// extension. It works with both local filesystems and remote object stores, making it
230 /// suitable for various storage backends.
231 ///
232 /// # Parameters
233 ///
234 /// - `directory`: The directory path to scan for Parquet files, unencoded as
235 /// [`Self::make_path`] returns it.
236 ///
237 /// # Returns
238 ///
239 /// Returns a vector of full file paths (as strings) for all Parquet files under the directory.
240 /// The paths are percent-encoded object-store keys relative to the store root; convert one with
241 /// [`Self::to_object_path_parsed`].
242 /// Returns an empty vector if the directory doesn't exist or contains no Parquet files.
243 ///
244 /// # Errors
245 ///
246 /// Returns an error if:
247 /// - Object store listing operations fail.
248 /// - Directory access is denied.
249 /// - Network issues occur (for remote object stores).
250 ///
251 /// # Notes
252 ///
253 /// - Only files ending with `.parquet` are included.
254 /// - Subdirectories are scanned recursively, so a type directory such as `data/quotes`
255 /// returns the files of every identifier under it.
256 /// - File paths are returned in the order provided by the object store.
257 /// - Works with all supported object store backends (local, S3, GCS, Azure, etc.).
258 ///
259 /// # Examples
260 ///
261 /// ```rust,no_run
262 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
263 ///
264 /// let mut catalog = ParquetDataCatalog::new(
265 /// std::path::Path::new("/tmp/nautilus_data"),
266 /// None,
267 /// None,
268 /// None,
269 /// None,
270 /// );
271 /// let files = catalog.list_parquet_files("data/quotes/EURUSD")?;
272 ///
273 /// for file in files {
274 /// println!("Found Parquet file: {}", file);
275 /// }
276 /// # Ok::<(), anyhow::Error>(())
277 /// ```
278 pub fn list_parquet_files(&self, directory: &str) -> anyhow::Result<Vec<String>> {
279 let files = self
280 .list_objects(directory)?
281 .into_iter()
282 .filter(|object| object.location.as_ref().ends_with(".parquet"))
283 .map(|object| object.location.to_string())
284 .collect();
285
286 Ok(files)
287 }
288
289 /// Lists all instrument identifiers for a specific data type.
290 ///
291 /// This method scans the data directory for a given data type and extracts
292 /// all unique instrument identifiers from the directory structure.
293 ///
294 /// # Parameters
295 ///
296 /// - `data_type`: The stored family to target.
297 ///
298 /// # Returns
299 ///
300 /// Returns a vector of instrument identifier strings.
301 ///
302 /// # Errors
303 ///
304 /// Returns an error if directory listing fails.
305 pub fn list_instruments(&self, data_type: &CatalogDataType) -> anyhow::Result<Vec<String>> {
306 let mut instruments = Vec::new();
307
308 for prefix in parquet_catalog_data_type_path_prefixes(data_type) {
309 instruments.extend(self.list_prefix_instruments(prefix.as_ref())?);
310 }
311
312 // The same identifier can live under more than one instrument class.
313 instruments.sort();
314 instruments.dedup();
315
316 Ok(instruments)
317 }
318
319 fn list_prefix_instruments(&self, data_type: &str) -> anyhow::Result<Vec<String>> {
320 let directory = self.make_path(data_type, None)?;
321 let prefix = ObjectPath::from(directory.as_str());
322 let mut instruments = HashSet::new();
323
324 for object in self.list_objects(&directory)? {
325 // Relative to the prefix a datum is `{identifier}/{filename}.parquet`
326 let Some(relative) = object.location.prefix_match(&prefix) else {
327 continue;
328 };
329 let segments: Vec<_> = relative.collect();
330
331 if let [identifier, filename] = segments.as_slice()
332 && filename.as_ref().ends_with(".parquet")
333 {
334 instruments.insert(decode_object_store_segment(identifier.as_ref()));
335 }
336 }
337
338 Ok(instruments.into_iter().collect())
339 }
340
341 /// Lists Parquet files matching specific criteria (data type, identifiers, time range).
342 ///
343 /// This method finds all Parquet files that match the specified criteria by filtering
344 /// files based on their directory structure and filename timestamps.
345 ///
346 /// # Parameters
347 ///
348 /// - `data_type`: The stored family to target.
349 /// - `identifiers`: Optional list of identifiers to filter by.
350 /// - `start`: Optional start timestamp to filter files by their time range.
351 /// - `end`: Optional end timestamp to filter files by their time range.
352 ///
353 /// # Returns
354 ///
355 /// Returns a vector of file paths that match the criteria.
356 ///
357 /// # Errors
358 ///
359 /// Returns an error if directory listing or file filtering fails.
360 pub fn list_parquet_files_with_criteria(
361 &self,
362 data_type: &CatalogDataType,
363 identifiers: Option<&[String]>,
364 start: Option<UnixNanos>,
365 end: Option<UnixNanos>,
366 ) -> anyhow::Result<Vec<String>> {
367 let mut all_files = Vec::new();
368
369 for data_cls in parquet_catalog_data_type_path_prefixes(data_type) {
370 all_files.extend(self.list_prefix_files_with_criteria(
371 data_cls.as_ref(),
372 identifiers,
373 start,
374 end,
375 )?);
376 }
377
378 Ok(all_files)
379 }
380
381 fn list_prefix_files_with_criteria(
382 &self,
383 data_cls: &str,
384 identifiers: Option<&[String]>,
385 start: Option<UnixNanos>,
386 end: Option<UnixNanos>,
387 ) -> anyhow::Result<Vec<String>> {
388 let start_u64 = start.map(|s| s.as_u64());
389 let end_u64 = end.map(|e| e.as_u64());
390
391 let base_dir = self.make_path(data_cls, None)?;
392
393 // Use recursive listing to match Python's glob behavior
394 let mut all_files = self.list_parquet_files(&base_dir)?;
395
396 if let Some(identifiers) = identifiers {
397 all_files =
398 filter_identifier_files(all_files, identifiers, is_parquet_bar_prefix(data_cls));
399 }
400
401 all_files.retain(|path| query_intersects_filename(path, start_u64, end_u64));
402 Ok(all_files)
403 }
404
405 fn is_windows_local(&self) -> bool {
406 cfg!(target_os = "windows") && !self.is_remote_uri()
407 }
408
409 /// Encodes colons as `%3A` in an [`ObjectPath`] for Windows local filesystem compatibility.
410 pub(crate) fn encode_windows_colon_path(path: &ObjectPath) -> ObjectPath {
411 if path.as_ref().contains(':')
412 && let Ok(encoded) = ObjectPath::parse(path.as_ref().replace(':', "%3A"))
413 {
414 encoded
415 } else {
416 path.clone()
417 }
418 }
419
420 /// Converts a directory path to an [`ObjectPath`] listing prefix.
421 ///
422 /// On Windows local filesystem catalogs, colons in instrument identifiers are encoded
423 /// as `%3A` to match the on-disk directory names stored by `object_store::local::LocalFileSystem`.
424 pub(super) fn directory_listing_prefix(&self, dir: &str) -> ObjectPath {
425 let prefix = ObjectPath::from(format!("{dir}/"));
426
427 if self.is_windows_local() {
428 Self::encode_windows_colon_path(&prefix)
429 } else {
430 prefix
431 }
432 }
433
434 /// Recursively lists all objects under `{dir}/` in the object store.
435 pub(crate) fn list_objects(&self, dir: &str) -> anyhow::Result<Vec<ObjectMeta>> {
436 let prefix = self.directory_listing_prefix(dir);
437 self.execute_async(|| async {
438 let mut stream = self.object_store.list(Some(&prefix));
439 let mut objects = Vec::new();
440 while let Some(object) = stream.next().await {
441 objects.push(object?);
442 }
443
444 consolidation_state::visible_objects(&self.object_store, objects).await
445 })
446 }
447
448 pub(crate) fn recover_consolidation(&self, directory: &ObjectPath) -> anyhow::Result<()> {
449 self.execute_async(|| async {
450 consolidation_state::recover(&self.object_store, directory).await
451 })
452 }
453
454 pub(super) fn has_pending_consolidation(&self, directory: &str) -> anyhow::Result<bool> {
455 let directory = if self.is_remote_uri() {
456 self.to_object_path_parsed(directory)?
457 } else {
458 let uri = self.resolve_directory_for_datafusion(directory);
459 let path = ListingTableUrl::parse(&uri)?
460 .get_url()
461 .to_file_path()
462 .map_err(|()| anyhow::anyhow!("Invalid local file URI: {directory}"))?;
463 self.to_object_path_parsed(&path.to_string_lossy())?
464 };
465
466 self.execute_async(|| async {
467 consolidation_state::has_pending(&self.object_store, &directory).await
468 })
469 }
470
471 /// Reconstructs a local filesystem path or a remote object-store URI.
472 ///
473 /// Local catalog roots retain literal `%`, `#`, and `?` in directory names.
474 #[must_use]
475 pub fn reconstruct_full_uri(&self, path_str: &str) -> String {
476 if path_str.contains("://") {
477 return path_str.to_string();
478 }
479
480 // Check if this is a remote URI scheme that needs reconstruction
481 if self.is_remote_uri() {
482 let path = self.path_under_base(path_str);
483 if let Ok(uri) = remote_full_uri(&self.original_uri, &path) {
484 return uri;
485 }
486 }
487
488 // For local paths, extract the directory from the original URI
489 if self.original_uri.starts_with("file://") {
490 // Extract the path from the file:// URI
491 if let Ok(url) = url::Url::parse(&escape_local_file_uri(&self.original_uri))
492 && let Ok(base_path) = url.to_file_path()
493 {
494 // Use platform-appropriate path separator for display
495 // but object store paths always use forward slashes
496 let base_str = base_path.to_string_lossy();
497 return make_object_store_path(&base_str, [path_str]);
498 }
499 }
500
501 // For local paths without file:// prefix, use the original URI as base
502 if self.base_path.is_empty() {
503 // If base_path is empty and not a file URI, try using original_uri as base
504 if self.original_uri.contains("://") {
505 // Fallback: return the path as-is
506 path_str.to_string()
507 } else {
508 make_object_store_path(self.original_uri.trim_end_matches('/'), [path_str])
509 }
510 } else {
511 let base = self.base_path.trim_end_matches('/');
512 make_object_store_path(base, [path_str])
513 }
514 }
515
516 /// Resolves a path for use with DataFusion (avoiding Windows path doubling for file://).
517 /// A remote path resolves to a URL under the store root whose path decodes to the object
518 /// key. A local path is returned as-is if it is already a full URI or absolute; otherwise it
519 /// joins the file:// base or goes through `reconstruct_full_uri`.
520 #[must_use]
521 pub(crate) fn resolve_path_for_datafusion(&self, path: &str) -> String {
522 // DataFusion decodes the URL path once, so encode the key again
523 if self.is_remote_uri()
524 && let Ok(key) = self.object_store_path(path)
525 && let Ok(root) = remote_store_root_url(&self.original_uri)
526 {
527 return append_path_to_file_uri(root.as_str(), &key);
528 }
529
530 if path.contains("://") {
531 return path.to_string();
532 }
533
534 if path.starts_with('/') {
535 return path.to_string();
536 }
537
538 if self.original_uri.starts_with("file://") {
539 return append_path_to_file_uri(&escape_local_file_uri(&self.original_uri), path);
540 }
541
542 self.reconstruct_full_uri(path)
543 }
544
545 /// Like `resolve_path_for_datafusion` but ensures the result ends with a trailing slash.
546 #[must_use]
547 pub(super) fn resolve_directory_for_datafusion(&self, directory: &str) -> String {
548 let mut resolved = self.resolve_path_for_datafusion(directory);
549 if !resolved.ends_with('/') {
550 resolved.push('/');
551 }
552
553 resolved
554 }
555
556 /// Returns the path string to push in `query_files` result list: relative for file://,
557 /// full URI for remote (so callers can pass to `resolve_path_for_datafusion` later).
558 #[must_use]
559 pub(super) fn path_for_query_list(&self, path: &str) -> String {
560 if self.original_uri.starts_with("file://") {
561 path.to_string()
562 } else {
563 self.reconstruct_full_uri(path)
564 }
565 }
566
567 /// Returns the native path string for the catalog root (for `std::fs`). Only valid when
568 /// !`is_remote_uri()`; uses parquet's `file_uri_to_native_path` for file:// URIs.
569 #[must_use]
570 pub(crate) fn native_base_path_string(&self) -> String {
571 if self.original_uri.starts_with("file://") {
572 crate::backend::parquet::io::file_uri_to_native_path(&self.original_uri)
573 } else {
574 self.original_uri.clone()
575 }
576 }
577
578 pub(crate) fn register_remote_object_store(&mut self) -> anyhow::Result<()> {
579 if self.is_remote_uri() {
580 let base_url = remote_store_root_url(&self.original_uri)?;
581 self.session
582 .register_object_store(&base_url, self.object_store.clone());
583 }
584
585 Ok(())
586 }
587
588 /// Helper method to check if the original URI uses a remote object store scheme
589 #[must_use]
590 pub fn is_remote_uri(&self) -> bool {
591 self.original_uri
592 .split_once("://")
593 .is_some_and(|(scheme, _)| is_remote_uri_scheme(scheme))
594 }
595
596 /// Constructs a directory path for storing data of a specific type and instrument.
597 ///
598 /// This method builds the hierarchical directory structure used by the catalog to organize
599 /// data by type and instrument. The path follows the pattern: `{base_path}/data/{type_name}/{instrument_id}`.
600 /// Instrument IDs are automatically converted to URI-safe format by removing forward slashes.
601 ///
602 /// # Parameters
603 ///
604 /// - `type_name`: The data type directory name (e.g., "quotes", "trades", "bars").
605 /// - `identifier`: Optional identifier. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL"). If provided, creates a subdirectory for the identifier. If `None`, returns the path to the data type directory.
606 ///
607 /// # Returns
608 ///
609 /// Returns the constructed directory path as a string, or an error if path construction fails.
610 ///
611 /// # Errors
612 ///
613 /// Returns an error if:
614 /// - The instrument ID contains invalid characters that cannot be made URI-safe.
615 /// - Path construction fails due to system limitations.
616 ///
617 /// # Path Structure
618 ///
619 /// - Without identifier: `{base_path}/data/{type_name}`.
620 /// - With identifier: `{base_path}/data/{type_name}/{safe_identifier}`.
621 /// - If `base_path` is empty: `data/{type_name}[/{safe_identifier}]`.
622 ///
623 /// # Examples
624 ///
625 /// ```rust,no_run
626 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
627 ///
628 /// let mut catalog = ParquetDataCatalog::new(
629 /// std::path::Path::new("/tmp/nautilus_data"),
630 /// None,
631 /// None,
632 /// None,
633 /// None,
634 /// );
635 ///
636 /// // Path for all quote data
637 /// let quotes_path = catalog.make_path("quotes", None)?;
638 /// // Returns: "/base/path/data/quotes"
639 ///
640 /// // Path for specific instrument quotes
641 /// let eurusd_quotes = catalog.make_path("quotes", Some("EUR/USD"))?;
642 /// // Returns: "/base/path/data/quotes/EURUSD" (slash removed)
643 ///
644 /// // Path for bar data with complex instrument ID
645 /// let bars_path = catalog.make_path("bars", Some("BTC/USD-1H"))?;
646 /// // Returns: "/base/path/data/bars/BTCUSD-1H"
647 /// # Ok::<(), anyhow::Error>(())
648 /// ```
649 pub fn make_path(&self, type_name: &str, identifier: Option<&str>) -> anyhow::Result<String> {
650 let mut components = vec!["data".to_string(), type_name.to_string()];
651
652 if let Some(id) = identifier {
653 let safe_id = urisafe_instrument_id(id);
654 components.push(safe_id);
655 }
656
657 let path = make_object_store_path(&self.base_path, components);
658 Ok(path)
659 }
660
661 /// Builds the directory path for custom data: `data/custom/{type_name}[/{identifier}]`.
662 pub fn make_path_custom_data(
663 &self,
664 type_name: &str,
665 identifier: Option<&str>,
666 ) -> anyhow::Result<String> {
667 let mut components = vec![
668 "data".to_string(),
669 "custom".to_string(),
670 type_name.to_string(),
671 ];
672
673 if let Some(id) = identifier {
674 let safe_id = urisafe_instrument_id(id);
675
676 if !safe_id.is_empty() {
677 components.push(safe_id);
678 }
679 }
680
681 let path = make_object_store_path(&self.base_path, components);
682 Ok(path)
683 }
684
685 /// Converts a catalog path string to an [`ObjectPath`] for object store operations.
686 ///
687 /// This method handles the conversion between catalog-relative paths and object store paths,
688 /// taking into account the catalog's base path configuration. It automatically preserves the
689 /// base path prefix for remote catalogs and strips it for local catalog paths.
690 ///
691 /// # Parameters
692 ///
693 /// - `path`: The catalog path string to convert. Can be absolute or relative.
694 ///
695 /// # Returns
696 ///
697 /// Returns an [`ObjectPath`] suitable for use with object store operations.
698 ///
699 /// # Path Handling
700 ///
701 /// - If `base_path` is empty, the path is used as-is.
702 /// - If `base_path` is set for a remote catalog, it's preserved or prepended.
703 /// - If `base_path` is set for a local catalog, it's stripped from the path if present.
704 /// - Trailing slashes and backslashes are automatically handled.
705 /// - The resulting path is relative to the object store root.
706 /// - All paths are normalized to use forward slashes (object store convention).
707 ///
708 /// # Errors
709 ///
710 /// Returns an error if the catalog is remote and `path` is a full URI whose scheme/host does
711 /// not match the catalog's own root (cross-bucket misuse). Without this guard the caller could
712 /// silently write to or read from the wrong bucket.
713 ///
714 /// # Examples
715 ///
716 /// Local catalog paths (absolute or relative) strip the catalog's base directory:
717 ///
718 /// ```rust,no_run
719 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
720 /// # let catalog: ParquetDataCatalog = unimplemented!();
721 /// let object_path = catalog.to_object_path("/base/data/quotes/file.parquet")?;
722 /// // ObjectPath("data/quotes/file.parquet")
723 /// # Ok::<(), anyhow::Error>(())
724 /// ```
725 ///
726 /// Remote catalog paths (relative or full URI) preserve or prepend the base prefix:
727 ///
728 /// ```rust,no_run
729 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
730 /// # let catalog: ParquetDataCatalog = unimplemented!();
731 /// let object_path = catalog.to_object_path("data/trades/file.parquet")?;
732 /// // ObjectPath("base/data/trades/file.parquet")
733 /// # Ok::<(), anyhow::Error>(())
734 /// ```
735 pub fn to_object_path(&self, path: &str) -> anyhow::Result<ObjectPath> {
736 let store_path = self.object_store_path(path)?;
737 let object_path = ObjectPath::from(store_path);
738
739 if self.is_windows_local() {
740 Ok(Self::encode_windows_colon_path(&object_path))
741 } else {
742 Ok(object_path)
743 }
744 }
745
746 /// Converts a path string to [`ObjectPath`] using parse (no percent-encoding).
747 ///
748 /// Use this for paths that were returned by the object store (e.g. from `list()`),
749 /// which may already be percent-encoded. Using [`Self::to_object_path`] (which uses
750 /// `Path::from`) on such paths would double-encode (e.g. `%5E` -> `%255E`).
751 ///
752 /// # Errors
753 ///
754 /// Returns an error if the same cross-bucket condition as [`Self::to_object_path`] occurs or
755 /// the resulting string fails [`ObjectPath::parse`].
756 pub fn to_object_path_parsed(&self, path: &str) -> anyhow::Result<ObjectPath> {
757 let to_parse = self.object_store_path(path)?;
758 ObjectPath::parse(&to_parse).map_err(anyhow::Error::from)
759 }
760
761 pub(super) fn object_store_path(&self, path: &str) -> anyhow::Result<String> {
762 let normalized_path = normalize_path_separators(path);
763
764 if self.is_remote_uri() {
765 if normalized_path.contains("://") {
766 let path_under_root = self.remote_uri_object_path(&normalized_path)?;
767 return Ok(self.path_under_base(&path_under_root));
768 }
769
770 return Ok(self.path_under_base(&normalized_path));
771 }
772
773 Ok(self.path_without_local_base(&normalized_path))
774 }
775
776 fn remote_uri_object_path(&self, path: &str) -> anyhow::Result<String> {
777 let path_url = url::Url::parse(path)
778 .map_err(|e| anyhow::anyhow!("Failed to parse object store URI {path}: {e}"))?;
779 if !is_remote_uri_scheme(path_url.scheme()) {
780 anyhow::bail!(
781 "URI {path} uses non-remote scheme {} for remote catalog at {}",
782 path_url.scheme(),
783 self.original_uri,
784 );
785 }
786
787 let catalog_root = remote_store_root_url(&self.original_uri)?;
788 let path_root = remote_store_root_url(path)?;
789 if catalog_root.as_str().trim_end_matches('/') != path_root.as_str().trim_end_matches('/') {
790 anyhow::bail!(
791 "Cross-store URI {path} (root {}) does not belong to catalog rooted at {} ({})",
792 path_root.as_str().trim_end_matches('/'),
793 self.original_uri,
794 catalog_root.as_str().trim_end_matches('/'),
795 );
796 }
797
798 // The URL crate keeps the path component percent-encoded (e.g. `%5E`),
799 // so preserve that encoding for `ObjectPath::parse` round-trips through
800 // `object_store::list`/`get`.
801 Ok(path_url.path().trim_start_matches('/').to_string())
802 }
803
804 pub(crate) fn path_without_local_base(&self, path: &str) -> String {
805 let base_path = if self.base_path.is_empty() {
806 self.native_base_path_string()
807 } else {
808 self.base_path.clone()
809 };
810
811 let normalized_base = normalize_path_separators(&base_path);
812 let base = normalized_base.trim_end_matches('/');
813
814 if base.is_empty() {
815 path.to_string()
816 } else if path == base {
817 String::new()
818 } else if let Some(without_base) = path.strip_prefix(&format!("{base}/")) {
819 without_base.to_string()
820 } else {
821 path.to_string()
822 }
823 }
824
825 fn path_under_base(&self, path: &str) -> String {
826 let normalized_path = normalize_path_separators(path);
827 let path = normalized_path
828 .trim_start_matches('/')
829 .trim_end_matches('/');
830
831 if self.base_path.is_empty() {
832 return path.to_string();
833 }
834
835 let normalized_base = normalize_path_separators(&self.base_path);
836 let base = normalized_base
837 .trim_start_matches('/')
838 .trim_end_matches('/');
839
840 if base.is_empty() || path == base || path.starts_with(&format!("{base}/")) {
841 path.to_string()
842 } else if path.is_empty() {
843 base.to_string()
844 } else {
845 make_object_store_path(base, [path])
846 }
847 }
848
849 /// Helper method to move a file using object store rename operation
850 pub fn move_file(&self, old_path: &ObjectPath, new_path: &ObjectPath) -> anyhow::Result<()> {
851 if old_path == new_path {
852 return Ok(());
853 }
854
855 self.recover_consolidation(&old_path.parent().unwrap_or_default())?;
856 self.recover_consolidation(&new_path.parent().unwrap_or_default())?;
857
858 self.execute_async(|| async {
859 self.object_store
860 .rename(old_path, new_path)
861 .await
862 .map_err(anyhow::Error::from)
863 })
864 }
865
866 /// Helper method to execute async operations with a runtime
867 pub fn execute_async<C, F, R>(&self, create_future: C) -> anyhow::Result<R>
868 where
869 C: FnOnce() -> F + Send,
870 F: std::future::Future<Output = anyhow::Result<R>>,
871 R: Send,
872 {
873 block_on_nautilus_with(create_future)
874 }
875
876 /// Lists directory stems (directory names without path) in a subdirectory.
877 ///
878 /// This method scans a subdirectory and returns the names of all immediate
879 /// subdirectories. It's used to list data types, backtest runs, and live runs.
880 ///
881 /// # Parameters
882 ///
883 /// - `subdirectory`: The subdirectory path to scan (e.g., "data", "backtest", "live").
884 ///
885 /// # Returns
886 ///
887 /// Returns the sorted directory names (stems) found in the subdirectory, decoded from their
888 /// percent-encoded object-store form, or an error if the operation fails.
889 ///
890 /// # Errors
891 ///
892 /// Returns an error if:
893 /// - Object store listing operations fail.
894 /// - Directory access is denied.
895 ///
896 /// # Examples
897 ///
898 /// ```rust,no_run
899 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
900 ///
901 /// let mut catalog = ParquetDataCatalog::new(
902 /// std::path::Path::new("/tmp/nautilus_data"),
903 /// None,
904 /// None,
905 /// None,
906 /// None,
907 /// );
908 ///
909 /// // List all data types
910 /// let data_types = catalog.list_directory_stems("data")?;
911 /// for data_type in data_types {
912 /// println!("Found data type: {}", data_type);
913 /// }
914 /// # Ok::<(), anyhow::Error>(())
915 /// ```
916 pub fn list_directory_stems(&self, subdirectory: &str) -> anyhow::Result<Vec<String>> {
917 // For local filesystem paths, use filesystem operations to detect empty directories
918 // For remote object stores, we can only list directories that contain files
919 if !self.is_remote_uri() {
920 let directory = PathBuf::from(self.native_base_path_string()).join(subdirectory);
921
922 let entries = match std::fs::read_dir(&directory) {
923 Ok(entries) => entries,
924 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
925 Err(e) => {
926 return Err(e)
927 .with_context(|| format!("failed to list {}", directory.display()));
928 }
929 };
930
931 let mut directories = Vec::new();
932
933 for entry in entries {
934 let entry =
935 entry.with_context(|| format!("failed to list {}", directory.display()))?;
936 let file_type = entry
937 .file_type()
938 .with_context(|| format!("failed to read {}", entry.path().display()))?;
939
940 // Use file_name() to get the directory name (not file_stem which removes extension)
941 if file_type.is_dir()
942 && let Some(name) = entry.path().file_name()
943 {
944 directories.push(decode_object_store_segment(&name.to_string_lossy()));
945 }
946 }
947
948 directories.sort();
949 return Ok(directories);
950 }
951
952 let storage = StorageBackend {
953 object_store: self.object_store.clone(),
954 base_path: self.base_path.clone(),
955 original_uri: self.original_uri.clone(),
956 };
957
958 self.execute_async(|| async { storage.list_directory_stems(subdirectory).await })
959 }
960
961 /// Lists the data, record, and instrument types stored in the catalog.
962 ///
963 /// Each `data/<type>` directory maps to its catalog type, and each `data/custom/<type_name>`
964 /// directory to a custom data type. Unrecognized directories are skipped with a warning.
965 ///
966 /// # Returns
967 ///
968 /// Returns the catalog types, or an error if the operation fails.
969 ///
970 /// # Errors
971 ///
972 /// Returns an error if:
973 /// - Object store listing operations fail.
974 /// - Directory access is denied.
975 ///
976 /// # Examples
977 ///
978 /// ```rust,no_run
979 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
980 ///
981 /// let mut catalog = ParquetDataCatalog::new(
982 /// std::path::Path::new("/tmp/nautilus_data"),
983 /// None,
984 /// None,
985 /// None,
986 /// None,
987 /// );
988 ///
989 /// // List all data types
990 /// let data_types = catalog.list_data_types()?;
991 /// for data_type in data_types {
992 /// println!("Available data type: {data_type}");
993 /// }
994 /// # Ok::<(), anyhow::Error>(())
995 /// ```
996 pub fn list_data_types(&self) -> anyhow::Result<Vec<CatalogDataType>> {
997 let mut data_types = Vec::new();
998
999 for stem in self.list_directory_stems("data")? {
1000 if stem == "custom" {
1001 for type_name in self.list_directory_stems("data/custom")? {
1002 data_types.push(CatalogDataType::from(NautilusDataType::Custom {
1003 type_name,
1004 }));
1005 }
1006
1007 continue;
1008 }
1009
1010 match catalog_data_type_from_path(&stem) {
1011 Ok(data_type) => data_types.push(data_type),
1012 Err(e) => log::warn!("Skipping catalog directory data/{stem}: {e}"),
1013 }
1014 }
1015
1016 Ok(data_types)
1017 }
1018
1019 /// Lists all backtest run IDs available in the catalog.
1020 ///
1021 /// This method returns the names of all backtest run directories in the catalog.
1022 /// Each backtest run corresponds to a specific backtest execution instance.
1023 ///
1024 /// # Returns
1025 ///
1026 /// Returns a vector of backtest run IDs, or an error if the operation fails.
1027 ///
1028 /// # Errors
1029 ///
1030 /// Returns an error if:
1031 /// - Object store listing operations fail.
1032 /// - Directory access is denied.
1033 ///
1034 /// # Examples
1035 ///
1036 /// ```rust,no_run
1037 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
1038 ///
1039 /// let mut catalog = ParquetDataCatalog::new(
1040 /// std::path::Path::new("/tmp/nautilus_data"),
1041 /// None,
1042 /// None,
1043 /// None,
1044 /// None,
1045 /// );
1046 ///
1047 /// // List all backtest runs
1048 /// let runs = catalog.list_backtest_runs()?;
1049 /// for run_id in runs {
1050 /// println!("Backtest run: {}", run_id);
1051 /// }
1052 /// # Ok::<(), anyhow::Error>(())
1053 /// ```
1054 pub fn list_backtest_runs(&self) -> anyhow::Result<Vec<String>> {
1055 self.list_directory_stems("backtest")
1056 }
1057
1058 /// Lists all live run IDs available in the catalog.
1059 ///
1060 /// This method returns the names of all live run directories in the catalog.
1061 /// Each live run corresponds to a specific live trading execution instance.
1062 ///
1063 /// # Returns
1064 ///
1065 /// Returns a vector of live run IDs, or an error if the operation fails.
1066 ///
1067 /// # Errors
1068 ///
1069 /// Returns an error if:
1070 /// - Object store listing operations fail.
1071 /// - Directory access is denied.
1072 ///
1073 /// # Examples
1074 ///
1075 /// ```rust,no_run
1076 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
1077 ///
1078 /// let mut catalog = ParquetDataCatalog::new(
1079 /// std::path::Path::new("/tmp/nautilus_data"),
1080 /// None,
1081 /// None,
1082 /// None,
1083 /// None,
1084 /// );
1085 ///
1086 /// // List all live runs
1087 /// let runs = catalog.list_live_runs()?;
1088 /// for run_id in runs {
1089 /// println!("Live run: {}", run_id);
1090 /// }
1091 /// # Ok::<(), anyhow::Error>(())
1092 /// ```
1093 pub fn list_live_runs(&self) -> anyhow::Result<Vec<String>> {
1094 self.list_directory_stems("live")
1095 }
1096}
1097
1098#[cfg(test)]
1099mod tests {
1100 use std::sync::Arc;
1101
1102 use nautilus_model::data::NautilusDataType;
1103 use object_store::{ObjectStoreExt, PutPayload, memory::InMemory, path::Path as ObjectPath};
1104 use rstest::rstest;
1105
1106 use super::ParquetDataCatalog;
1107 use crate::{catalog::types::CatalogDataType, common::datafusion::DataBackendSession};
1108
1109 /// Builds a catalog over an in-memory store. Only remote catalogs carry a non-empty
1110 /// `base_path`, so seeding one here is the sole way to reproduce a bucket sub-prefix.
1111 fn memory_catalog(base_path: &str) -> ParquetDataCatalog {
1112 ParquetDataCatalog {
1113 base_path: base_path.to_string(),
1114 original_uri: "memory://".to_string(),
1115 object_store: Arc::new(InMemory::new()),
1116 session: DataBackendSession::new(5_000),
1117 batch_size: 5_000,
1118 compression: parquet::basic::Compression::SNAPPY,
1119 max_row_group_size: 5_000,
1120 }
1121 }
1122
1123 fn seed(catalog: &ParquetDataCatalog, keys: &[&str]) {
1124 catalog
1125 .execute_async(|| async {
1126 for key in keys {
1127 catalog
1128 .object_store
1129 .put(&ObjectPath::from(*key), PutPayload::from_static(b"x"))
1130 .await?;
1131 }
1132 Ok(())
1133 })
1134 .unwrap();
1135 }
1136
1137 #[rstest]
1138 #[case("nautilus-data", "nautilus-data/")]
1139 #[case("", "")]
1140 fn list_instruments_returns_ids_under_base_path(
1141 #[case] base_path: &str,
1142 #[case] key_prefix: &str,
1143 ) {
1144 let catalog = memory_catalog(base_path);
1145 let keys = [
1146 "data/quotes/EURUSD.SIM/0-1.parquet",
1147 "data/quotes/EURUSD.SIM/2-3.parquet",
1148 "data/quotes/GBPUSD.SIM/0-1.parquet",
1149 "data/trades/AUDUSD.SIM/0-1.parquet",
1150 ]
1151 .map(|key| format!("{key_prefix}{key}"));
1152 seed(&catalog, &keys.each_ref().map(String::as_str));
1153
1154 assert_eq!(
1155 catalog
1156 .list_instruments(&NautilusDataType::QuoteTick.into())
1157 .unwrap(),
1158 ["EURUSD.SIM", "GBPUSD.SIM"]
1159 );
1160 }
1161
1162 #[rstest]
1163 #[case("nautilus-data", "nautilus-data/")]
1164 #[case("", "")]
1165 fn list_parquet_files_lists_type_directory_recursively(
1166 #[case] base_path: &str,
1167 #[case] key_prefix: &str,
1168 ) {
1169 let catalog = memory_catalog(base_path);
1170 let keys = [
1171 "data/quotes/EURUSD.SIM/0-1.parquet",
1172 "data/quotes/EURUSD.SIM/0-1.json",
1173 "data/quotes/BTC€.SIM/2-3.parquet",
1174 "data/trades/AUDUSD.SIM/0-1.parquet",
1175 ]
1176 .map(|key| format!("{key_prefix}{key}"));
1177 seed(&catalog, &keys.each_ref().map(String::as_str));
1178
1179 let directory = catalog.make_path("quotes", None).unwrap();
1180 let mut files = catalog.list_parquet_files(&directory).unwrap();
1181 files.sort();
1182
1183 assert_eq!(
1184 files,
1185 [
1186 format!("{key_prefix}data/quotes/BTC%E2%82%AC.SIM/2-3.parquet"),
1187 format!("{key_prefix}data/quotes/EURUSD.SIM/0-1.parquet"),
1188 ]
1189 );
1190 }
1191
1192 #[rstest]
1193 fn list_instruments_ignores_unpartitioned_files() {
1194 // An empty `base_path` keeps this distinct from the remote prefix defect, so an empty
1195 // result can only come from the layout check.
1196 let catalog = memory_catalog("");
1197 seed(
1198 &catalog,
1199 &[
1200 "data/custom/MyType/1-2.parquet",
1201 "data/custom/MyType/1-2.json",
1202 ],
1203 );
1204
1205 let custom = CatalogDataType::Data(NautilusDataType::Custom {
1206 type_name: "MyType".to_string(),
1207 });
1208
1209 assert!(catalog.list_instruments(&custom).unwrap().is_empty());
1210 }
1211
1212 #[rstest]
1213 fn list_instruments_decodes_percent_encoded_ids() {
1214 let catalog = memory_catalog("nautilus-data");
1215 seed(
1216 &catalog,
1217 &["nautilus-data/data/quotes/BTC€.SIM/0-1.parquet"],
1218 );
1219
1220 assert_eq!(
1221 catalog
1222 .list_instruments(&NautilusDataType::QuoteTick.into())
1223 .unwrap(),
1224 ["BTC€.SIM"]
1225 );
1226 }
1227
1228 #[rstest]
1229 #[case::local(false)]
1230 #[case::remote(true)]
1231 fn test_directory_listing_prefix_matches_object_path(#[case] remote: bool) {
1232 let mut catalog = memory_catalog("");
1233 if remote {
1234 catalog.original_uri = "s3://test-bucket".to_string();
1235 }
1236
1237 let directory = "data/trades/io:ANTH-USD-PERP.HYPERLIQUID";
1238 let prefix = catalog.directory_listing_prefix(directory);
1239 let target = catalog.to_object_path(directory).unwrap();
1240
1241 let expected = if cfg!(target_os = "windows") && !remote {
1242 "data/trades/io%3AANTH-USD-PERP.HYPERLIQUID"
1243 } else {
1244 directory
1245 };
1246
1247 assert_eq!(prefix.as_ref(), expected);
1248 assert_eq!(target, prefix);
1249 }
1250
1251 #[rstest]
1252 #[case(
1253 "data/trades/io:ANTH-USD-PERP.HYPERLIQUID",
1254 "data/trades/io%3AANTH-USD-PERP.HYPERLIQUID"
1255 )]
1256 #[case("data/quotes/BTC€.SIM", "data/quotes/BTC%E2%82%AC.SIM")]
1257 #[case("data/quotes/EURUSD", "data/quotes/EURUSD")]
1258 fn test_encode_windows_colon_path(#[case] input: &str, #[case] expected: &str) {
1259 let path = ObjectPath::from(input);
1260 let encoded = ParquetDataCatalog::encode_windows_colon_path(&path);
1261 assert_eq!(encoded.as_ref(), expected);
1262 }
1263}