nautilus_persistence/backend/parquet/catalog/store.rs
1// -------------------------------------------------------------------------------------------------
2// Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3// https://nautechsystems.io
4//
5// Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6// You may not use this file except in compliance with the License.
7// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Object-store paths, listing, and file administration for the Parquet catalog.
17
18#![expect(
19 clippy::missing_errors_doc,
20 reason = "catalog store functions forward object-store errors"
21)]
22
23use anyhow::Context;
24use datafusion::datasource::listing::ListingTableUrl;
25use nautilus_common::live::block_on_nautilus_with;
26use nautilus_model::data::NautilusDataType;
27use object_store::ObjectMeta;
28
29use super::{
30 HashSet, ObjectPath, ObjectStore, ObjectStoreExt, ParquetDataCatalog, PathBuf, StreamExt,
31 UnixNanos, append_path_to_file_uri, are_intervals_disjoint, decode_object_store_segment,
32 is_remote_uri_scheme, make_object_store_path, parse_filename_timestamps,
33 query::{filter_identifier_files, is_parquet_bar_prefix},
34 query_intersects_filename, remote_full_uri, remote_store_root_url, timestamps_to_filename,
35 urisafe_instrument_id,
36};
37use crate::{
38 backend::parquet::consolidation_state,
39 catalog::types::{
40 CatalogDataType, catalog_data_type_from_path, parquet_catalog_data_type_path_prefixes,
41 },
42 common::{paths::normalize_path_separators, storage::StorageBackend},
43};
44
45impl ParquetDataCatalog {
46 /// Extends the timestamp range of an existing Parquet file by renaming it.
47 ///
48 /// This method finds an existing file that is adjacent to the specified time range
49 /// and renames it to include the new range. This is useful when appending data
50 /// that extends the time coverage of existing files.
51 /// The proposed extension is validated against the other files before renaming,
52 /// so a rejected extension leaves existing files unchanged.
53 ///
54 /// If no file is adjacent to the specified range, this method does nothing and
55 /// returns `Ok(())` after confirming the existing intervals are disjoint.
56 ///
57 /// # Parameters
58 ///
59 /// - `data_type`: The stored family to target.
60 /// - `identifier`: Optional identifier to target a specific instrument's data. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
61 /// - `start`: Start timestamp of the new range to extend to.
62 /// - `end`: End timestamp of the new range to extend to.
63 ///
64 /// # Returns
65 ///
66 /// Returns `Ok(())` on success, or an error if the operation fails.
67 ///
68 /// # Errors
69 ///
70 /// Returns an error if:
71 /// - The range is reversed (`start` is after `end`).
72 /// - The directory path cannot be constructed.
73 /// - The proposed extension would overlap another file.
74 /// - The existing intervals are already overlapping.
75 /// - File rename operations fail.
76 ///
77 /// # Examples
78 ///
79 /// ```rust,no_run
80 /// use nautilus_core::UnixNanos;
81 /// use nautilus_model::data::NautilusDataType;
82 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
83 ///
84 /// let mut catalog = ParquetDataCatalog::new(
85 /// std::path::Path::new("/tmp/nautilus_data"),
86 /// None,
87 /// None,
88 /// None,
89 /// None,
90 /// );
91 ///
92 /// // Extend a file's range backwards or forwards
93 /// catalog.extend_file_name(
94 /// &NautilusDataType::QuoteTick.into(),
95 /// Some("BTC/USD.SIM"),
96 /// UnixNanos::from(1609459200000000000),
97 /// UnixNanos::from(1609545600000000000),
98 /// )?;
99 /// # Ok::<(), anyhow::Error>(())
100 /// ```
101 pub fn extend_file_name(
102 &self,
103 data_type: &CatalogDataType,
104 identifier: Option<&str>,
105 start: UnixNanos,
106 end: UnixNanos,
107 ) -> anyhow::Result<()> {
108 let prefixes = parquet_catalog_data_type_path_prefixes(data_type);
109
110 if let [data_cls] = prefixes.as_slice() {
111 let directory = self.make_path(data_cls.as_ref(), identifier)?;
112 return self.extend_file_name_in_directory(&directory, start, end);
113 }
114
115 // The aggregate instrument family spans every class directory, and one identifier can be
116 // stored under several classes, so extend each directory that already holds it rather
117 // than inventing a class for it.
118 let mut extended = false;
119
120 for data_cls in &prefixes {
121 let directory = self.make_path(data_cls.as_ref(), identifier)?;
122 if !self.get_directory_intervals(&directory)?.is_empty() {
123 self.extend_file_name_in_directory(&directory, start, end)?;
124 extended = true;
125 }
126 }
127
128 anyhow::ensure!(
129 extended,
130 "Cannot extend file name for {data_type}: no instrument class holds {}; \
131 name the class with a NautilusInstrumentType",
132 identifier.unwrap_or("any identifier"),
133 );
134
135 Ok(())
136 }
137
138 pub(super) fn extend_file_name_in_directory(
139 &self,
140 directory: &str,
141 start: UnixNanos,
142 end: UnixNanos,
143 ) -> anyhow::Result<()> {
144 let start = start.as_u64();
145 let end = end.as_u64();
146
147 anyhow::ensure!(
148 start <= end,
149 "Cannot extend file in directory '{directory}': reversed range ({start}, {end})",
150 );
151
152 let intervals = self.get_directory_intervals(directory)?;
153
154 anyhow::ensure!(
155 are_intervals_disjoint(&intervals),
156 "Intervals are not disjoint in directory '{directory}': {intervals:?}",
157 );
158
159 let adjacent = intervals.iter().enumerate().find_map(|(index, interval)| {
160 if end.checked_add(1) == Some(interval.0) {
161 // Extend backwards: new file covers [start, interval.1]
162 Some((index, *interval, (start, interval.1)))
163 } else if start.checked_sub(1) == Some(interval.1) {
164 // Extend forwards: new file covers [interval.0, end]
165 Some((index, *interval, (interval.0, end)))
166 } else {
167 None
168 }
169 });
170
171 let Some((index, original, proposed)) = adjacent else {
172 return Ok(());
173 };
174
175 let mut extended = intervals.clone();
176 extended[index] = proposed;
177
178 anyhow::ensure!(
179 are_intervals_disjoint(&extended),
180 "Extending file interval {original:?} to {proposed:?} in directory '{directory}' \
181 with range ({start}, {end}) would create non-disjoint intervals. \
182 Existing intervals: {intervals:?}",
183 );
184
185 self.rename_parquet_file(directory, original.0, original.1, proposed.0, proposed.1)
186 }
187
188 /// Helper method to rename a parquet file by moving it via object store operations
189 fn rename_parquet_file(
190 &self,
191 directory: &str,
192 old_start: u64,
193 old_end: u64,
194 new_start: u64,
195 new_end: u64,
196 ) -> anyhow::Result<()> {
197 let new_filename =
198 timestamps_to_filename(UnixNanos::from(new_start), UnixNanos::from(new_end));
199 let new_path = format!("{directory}/{new_filename}");
200 let matches = self
201 .list_parquet_files(directory)?
202 .into_iter()
203 .filter(|file| parse_filename_timestamps(file) == Some((old_start, old_end)))
204 .collect::<Vec<_>>();
205
206 anyhow::ensure!(
207 matches.len() == 1,
208 "expected one parquet file for interval ({old_start}, {old_end}) in {directory}, \
209 found {}",
210 matches.len()
211 );
212
213 let old_path = &matches[0];
214 if old_path.ends_with(&new_filename) {
215 return Ok(());
216 }
217
218 let old_object_path = self.to_object_path_parsed(old_path)?;
219 let new_object_path = self.to_object_path(&new_path)?;
220 self.move_file(&old_object_path, &new_object_path)
221 }
222
223 /// Lists all Parquet files under a directory, including its subdirectories.
224 ///
225 /// This method scans a directory and returns the full paths of all files with the `.parquet`
226 /// extension. It works with both local filesystems and remote object stores, making it
227 /// suitable for various storage backends.
228 ///
229 /// # Parameters
230 ///
231 /// - `directory`: The directory path to scan for Parquet files, unencoded as
232 /// [`Self::make_path`] returns it.
233 ///
234 /// # Returns
235 ///
236 /// Returns a vector of full file paths (as strings) for all Parquet files under the directory.
237 /// The paths are percent-encoded object-store keys relative to the store root; convert one with
238 /// [`Self::to_object_path_parsed`].
239 /// Returns an empty vector if the directory doesn't exist or contains no Parquet files.
240 ///
241 /// # Errors
242 ///
243 /// Returns an error if:
244 /// - Object store listing operations fail.
245 /// - Directory access is denied.
246 /// - Network issues occur (for remote object stores).
247 ///
248 /// # Notes
249 ///
250 /// - Only files ending with `.parquet` are included.
251 /// - Subdirectories are scanned recursively, so a type directory such as `data/quotes`
252 /// returns the files of every identifier under it.
253 /// - File paths are returned in the order provided by the object store.
254 /// - Works with all supported object store backends (local, S3, GCS, Azure, etc.).
255 ///
256 /// # Examples
257 ///
258 /// ```rust,no_run
259 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
260 ///
261 /// let mut catalog = ParquetDataCatalog::new(
262 /// std::path::Path::new("/tmp/nautilus_data"),
263 /// None,
264 /// None,
265 /// None,
266 /// None,
267 /// );
268 /// let files = catalog.list_parquet_files("data/quotes/EURUSD")?;
269 ///
270 /// for file in files {
271 /// println!("Found Parquet file: {}", file);
272 /// }
273 /// # Ok::<(), anyhow::Error>(())
274 /// ```
275 pub fn list_parquet_files(&self, directory: &str) -> anyhow::Result<Vec<String>> {
276 let files = self
277 .list_objects(directory)?
278 .into_iter()
279 .filter(|object| object.location.as_ref().ends_with(".parquet"))
280 .map(|object| object.location.to_string())
281 .collect();
282
283 Ok(files)
284 }
285
286 /// Lists all instrument identifiers for a specific data type.
287 ///
288 /// This method scans the data directory for a given data type and extracts
289 /// all unique instrument identifiers from the directory structure.
290 ///
291 /// # Parameters
292 ///
293 /// - `data_type`: The stored family to target.
294 ///
295 /// # Returns
296 ///
297 /// Returns a vector of instrument identifier strings.
298 ///
299 /// # Errors
300 ///
301 /// Returns an error if directory listing fails.
302 pub fn list_instruments(&self, data_type: &CatalogDataType) -> anyhow::Result<Vec<String>> {
303 let mut instruments = Vec::new();
304
305 for prefix in parquet_catalog_data_type_path_prefixes(data_type) {
306 instruments.extend(self.list_prefix_instruments(prefix.as_ref())?);
307 }
308
309 // The same identifier can live under more than one instrument class.
310 instruments.sort();
311 instruments.dedup();
312
313 Ok(instruments)
314 }
315
316 fn list_prefix_instruments(&self, data_type: &str) -> anyhow::Result<Vec<String>> {
317 let directory = self.make_path(data_type, None)?;
318 let prefix = ObjectPath::from(directory.as_str());
319 let mut instruments = HashSet::new();
320
321 for object in self.list_objects(&directory)? {
322 // Relative to the prefix a datum is `{identifier}/{filename}.parquet`
323 let Some(relative) = object.location.prefix_match(&prefix) else {
324 continue;
325 };
326 let segments: Vec<_> = relative.collect();
327
328 if let [identifier, filename] = segments.as_slice()
329 && filename.as_ref().ends_with(".parquet")
330 {
331 instruments.insert(decode_object_store_segment(identifier.as_ref()));
332 }
333 }
334
335 Ok(instruments.into_iter().collect())
336 }
337
338 /// Lists Parquet files matching specific criteria (data type, identifiers, time range).
339 ///
340 /// This method finds all Parquet files that match the specified criteria by filtering
341 /// files based on their directory structure and filename timestamps.
342 ///
343 /// # Parameters
344 ///
345 /// - `data_type`: The stored family to target.
346 /// - `identifiers`: Optional list of identifiers to filter by.
347 /// - `start`: Optional start timestamp to filter files by their time range.
348 /// - `end`: Optional end timestamp to filter files by their time range.
349 ///
350 /// # Returns
351 ///
352 /// Returns a vector of file paths that match the criteria.
353 ///
354 /// # Errors
355 ///
356 /// Returns an error if directory listing or file filtering fails.
357 pub fn list_parquet_files_with_criteria(
358 &self,
359 data_type: &CatalogDataType,
360 identifiers: Option<&[String]>,
361 start: Option<UnixNanos>,
362 end: Option<UnixNanos>,
363 ) -> anyhow::Result<Vec<String>> {
364 let mut all_files = Vec::new();
365
366 for data_cls in parquet_catalog_data_type_path_prefixes(data_type) {
367 all_files.extend(self.list_prefix_files_with_criteria(
368 data_cls.as_ref(),
369 identifiers,
370 start,
371 end,
372 )?);
373 }
374
375 Ok(all_files)
376 }
377
378 fn list_prefix_files_with_criteria(
379 &self,
380 data_cls: &str,
381 identifiers: Option<&[String]>,
382 start: Option<UnixNanos>,
383 end: Option<UnixNanos>,
384 ) -> anyhow::Result<Vec<String>> {
385 let start_u64 = start.map(|s| s.as_u64());
386 let end_u64 = end.map(|e| e.as_u64());
387
388 let base_dir = self.make_path(data_cls, None)?;
389
390 // Use recursive listing to match Python's glob behavior
391 let mut all_files = self.list_parquet_files(&base_dir)?;
392
393 if let Some(identifiers) = identifiers {
394 all_files =
395 filter_identifier_files(all_files, identifiers, is_parquet_bar_prefix(data_cls));
396 }
397
398 all_files.retain(|path| query_intersects_filename(path, start_u64, end_u64));
399 Ok(all_files)
400 }
401
402 /// Recursively lists all objects under `{dir}/` in the object store.
403 pub(crate) fn list_objects(&self, dir: &str) -> anyhow::Result<Vec<ObjectMeta>> {
404 self.execute_async(|| async {
405 let prefix = ObjectPath::from(format!("{dir}/"));
406 let mut stream = self.object_store.list(Some(&prefix));
407 let mut objects = Vec::new();
408 while let Some(object) = stream.next().await {
409 objects.push(object?);
410 }
411
412 consolidation_state::visible_objects(&self.object_store, objects).await
413 })
414 }
415
416 pub(crate) fn recover_consolidation(&self, directory: &ObjectPath) -> anyhow::Result<()> {
417 self.execute_async(|| async {
418 consolidation_state::recover(&self.object_store, directory).await
419 })
420 }
421
422 pub(super) fn has_pending_consolidation(&self, directory: &str) -> anyhow::Result<bool> {
423 let directory = if self.is_remote_uri() {
424 self.to_object_path_parsed(directory)?
425 } else {
426 let uri = self.resolve_directory_for_datafusion(directory);
427 let path = ListingTableUrl::parse(&uri)?
428 .get_url()
429 .to_file_path()
430 .map_err(|()| anyhow::anyhow!("Invalid local file URI: {directory}"))?;
431 self.to_object_path_parsed(&path.to_string_lossy())?
432 };
433
434 self.execute_async(|| async {
435 consolidation_state::has_pending(&self.object_store, &directory).await
436 })
437 }
438
439 /// Helper method to reconstruct full URI for remote object store paths
440 #[must_use]
441 pub fn reconstruct_full_uri(&self, path_str: &str) -> String {
442 if path_str.contains("://") {
443 return path_str.to_string();
444 }
445
446 // Check if this is a remote URI scheme that needs reconstruction
447 if self.is_remote_uri() {
448 let path = self.path_under_base(path_str);
449 if let Ok(uri) = remote_full_uri(&self.original_uri, &path) {
450 return uri;
451 }
452 }
453
454 // For local paths, extract the directory from the original URI
455 if self.original_uri.starts_with("file://") {
456 // Extract the path from the file:// URI
457 if let Ok(url) = url::Url::parse(&self.original_uri)
458 && let Ok(base_path) = url.to_file_path()
459 {
460 // Use platform-appropriate path separator for display
461 // but object store paths always use forward slashes
462 let base_str = base_path.to_string_lossy();
463 return make_object_store_path(&base_str, [path_str]);
464 }
465 }
466
467 // For local paths without file:// prefix, use the original URI as base
468 if self.base_path.is_empty() {
469 // If base_path is empty and not a file URI, try using original_uri as base
470 if self.original_uri.contains("://") {
471 // Fallback: return the path as-is
472 path_str.to_string()
473 } else {
474 make_object_store_path(self.original_uri.trim_end_matches('/'), [path_str])
475 }
476 } else {
477 let base = self.base_path.trim_end_matches('/');
478 make_object_store_path(base, [path_str])
479 }
480 }
481
482 /// Resolves a path for use with DataFusion (avoiding Windows path doubling for file://).
483 /// A remote path resolves to a URL under the store root whose path decodes to the object
484 /// key. A local path is returned as-is if it is already a full URI or absolute; otherwise it
485 /// joins the file:// base or goes through `reconstruct_full_uri`.
486 #[must_use]
487 pub(crate) fn resolve_path_for_datafusion(&self, path: &str) -> String {
488 // DataFusion decodes the URL path once, so encode the key again
489 if self.is_remote_uri()
490 && let Ok(key) = self.object_store_path(path)
491 && let Ok(root) = remote_store_root_url(&self.original_uri)
492 {
493 return append_path_to_file_uri(root.as_str(), &key);
494 }
495
496 if path.contains("://") {
497 return path.to_string();
498 }
499
500 if path.starts_with('/') {
501 return path.to_string();
502 }
503
504 if self.original_uri.starts_with("file://") {
505 return append_path_to_file_uri(&self.original_uri, path);
506 }
507
508 self.reconstruct_full_uri(path)
509 }
510
511 /// Like `resolve_path_for_datafusion` but ensures the result ends with a trailing slash.
512 #[must_use]
513 pub(super) fn resolve_directory_for_datafusion(&self, directory: &str) -> String {
514 let mut resolved = self.resolve_path_for_datafusion(directory);
515 if !resolved.ends_with('/') {
516 resolved.push('/');
517 }
518
519 resolved
520 }
521
522 /// Returns the path string to push in `query_files` result list: relative for file://,
523 /// full URI for remote (so callers can pass to `resolve_path_for_datafusion` later).
524 #[must_use]
525 pub(super) fn path_for_query_list(&self, path: &str) -> String {
526 if self.original_uri.starts_with("file://") {
527 path.to_string()
528 } else {
529 self.reconstruct_full_uri(path)
530 }
531 }
532
533 /// Returns the native path string for the catalog root (for `std::fs`). Only valid when
534 /// !`is_remote_uri()`; uses parquet's `file_uri_to_native_path` for file:// URIs.
535 #[must_use]
536 pub(crate) fn native_base_path_string(&self) -> String {
537 if self.original_uri.starts_with("file://") {
538 crate::backend::parquet::io::file_uri_to_native_path(&self.original_uri)
539 } else {
540 self.original_uri.clone()
541 }
542 }
543
544 pub(crate) fn register_remote_object_store(&mut self) -> anyhow::Result<()> {
545 if self.is_remote_uri() {
546 let base_url = remote_store_root_url(&self.original_uri)?;
547 self.session
548 .register_object_store(&base_url, self.object_store.clone());
549 }
550
551 Ok(())
552 }
553
554 /// Helper method to check if the original URI uses a remote object store scheme
555 #[must_use]
556 pub fn is_remote_uri(&self) -> bool {
557 self.original_uri
558 .split_once("://")
559 .is_some_and(|(scheme, _)| is_remote_uri_scheme(scheme))
560 }
561
562 /// Constructs a directory path for storing data of a specific type and instrument.
563 ///
564 /// This method builds the hierarchical directory structure used by the catalog to organize
565 /// data by type and instrument. The path follows the pattern: `{base_path}/data/{type_name}/{instrument_id}`.
566 /// Instrument IDs are automatically converted to URI-safe format by removing forward slashes.
567 ///
568 /// # Parameters
569 ///
570 /// - `type_name`: The data type directory name (e.g., "quotes", "trades", "bars").
571 /// - `identifier`: Optional identifier. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL"). If provided, creates a subdirectory for the identifier. If `None`, returns the path to the data type directory.
572 ///
573 /// # Returns
574 ///
575 /// Returns the constructed directory path as a string, or an error if path construction fails.
576 ///
577 /// # Errors
578 ///
579 /// Returns an error if:
580 /// - The instrument ID contains invalid characters that cannot be made URI-safe.
581 /// - Path construction fails due to system limitations.
582 ///
583 /// # Path Structure
584 ///
585 /// - Without identifier: `{base_path}/data/{type_name}`.
586 /// - With identifier: `{base_path}/data/{type_name}/{safe_identifier}`.
587 /// - If `base_path` is empty: `data/{type_name}[/{safe_identifier}]`.
588 ///
589 /// # Examples
590 ///
591 /// ```rust,no_run
592 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
593 ///
594 /// let mut catalog = ParquetDataCatalog::new(
595 /// std::path::Path::new("/tmp/nautilus_data"),
596 /// None,
597 /// None,
598 /// None,
599 /// None,
600 /// );
601 ///
602 /// // Path for all quote data
603 /// let quotes_path = catalog.make_path("quotes", None)?;
604 /// // Returns: "/base/path/data/quotes"
605 ///
606 /// // Path for specific instrument quotes
607 /// let eurusd_quotes = catalog.make_path("quotes", Some("EUR/USD"))?;
608 /// // Returns: "/base/path/data/quotes/EURUSD" (slash removed)
609 ///
610 /// // Path for bar data with complex instrument ID
611 /// let bars_path = catalog.make_path("bars", Some("BTC/USD-1H"))?;
612 /// // Returns: "/base/path/data/bars/BTCUSD-1H"
613 /// # Ok::<(), anyhow::Error>(())
614 /// ```
615 pub fn make_path(&self, type_name: &str, identifier: Option<&str>) -> anyhow::Result<String> {
616 let mut components = vec!["data".to_string(), type_name.to_string()];
617
618 if let Some(id) = identifier {
619 let safe_id = urisafe_instrument_id(id);
620 components.push(safe_id);
621 }
622
623 let path = make_object_store_path(&self.base_path, components);
624 Ok(path)
625 }
626
627 /// Builds the directory path for custom data: `data/custom/{type_name}[/{identifier}]`.
628 pub fn make_path_custom_data(
629 &self,
630 type_name: &str,
631 identifier: Option<&str>,
632 ) -> anyhow::Result<String> {
633 let mut components = vec![
634 "data".to_string(),
635 "custom".to_string(),
636 type_name.to_string(),
637 ];
638
639 if let Some(id) = identifier {
640 let safe_id = urisafe_instrument_id(id);
641
642 if !safe_id.is_empty() {
643 components.push(safe_id);
644 }
645 }
646
647 let path = make_object_store_path(&self.base_path, components);
648 Ok(path)
649 }
650
651 /// Converts a catalog path string to an [`ObjectPath`] for object store operations.
652 ///
653 /// This method handles the conversion between catalog-relative paths and object store paths,
654 /// taking into account the catalog's base path configuration. It automatically preserves the
655 /// base path prefix for remote catalogs and strips it for local catalog paths.
656 ///
657 /// # Parameters
658 ///
659 /// - `path`: The catalog path string to convert. Can be absolute or relative.
660 ///
661 /// # Returns
662 ///
663 /// Returns an [`ObjectPath`] suitable for use with object store operations.
664 ///
665 /// # Path Handling
666 ///
667 /// - If `base_path` is empty, the path is used as-is.
668 /// - If `base_path` is set for a remote catalog, it's preserved or prepended.
669 /// - If `base_path` is set for a local catalog, it's stripped from the path if present.
670 /// - Trailing slashes and backslashes are automatically handled.
671 /// - The resulting path is relative to the object store root.
672 /// - All paths are normalized to use forward slashes (object store convention).
673 ///
674 /// # Errors
675 ///
676 /// Returns an error if the catalog is remote and `path` is a full URI whose scheme/host does
677 /// not match the catalog's own root (cross-bucket misuse). Without this guard the caller could
678 /// silently write to or read from the wrong bucket.
679 ///
680 /// # Examples
681 ///
682 /// Local catalog paths (absolute or relative) strip the catalog's base directory:
683 ///
684 /// ```rust,no_run
685 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
686 /// # let catalog: ParquetDataCatalog = unimplemented!();
687 /// let object_path = catalog.to_object_path("/base/data/quotes/file.parquet")?;
688 /// // ObjectPath("data/quotes/file.parquet")
689 /// # Ok::<(), anyhow::Error>(())
690 /// ```
691 ///
692 /// Remote catalog paths (relative or full URI) preserve or prepend the base prefix:
693 ///
694 /// ```rust,no_run
695 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
696 /// # let catalog: ParquetDataCatalog = unimplemented!();
697 /// let object_path = catalog.to_object_path("data/trades/file.parquet")?;
698 /// // ObjectPath("base/data/trades/file.parquet")
699 /// # Ok::<(), anyhow::Error>(())
700 /// ```
701 pub fn to_object_path(&self, path: &str) -> anyhow::Result<ObjectPath> {
702 Ok(ObjectPath::from(self.object_store_path(path)?))
703 }
704
705 /// Converts a path string to [`ObjectPath`] using parse (no percent-encoding).
706 ///
707 /// Use this for paths that were returned by the object store (e.g. from `list()`),
708 /// which may already be percent-encoded. Using [`Self::to_object_path`] (which uses
709 /// `Path::from`) on such paths would double-encode (e.g. `%5E` -> `%255E`).
710 ///
711 /// # Errors
712 ///
713 /// Returns an error if the same cross-bucket condition as [`Self::to_object_path`] occurs or
714 /// the resulting string fails [`ObjectPath::parse`].
715 pub fn to_object_path_parsed(&self, path: &str) -> anyhow::Result<ObjectPath> {
716 let to_parse = self.object_store_path(path)?;
717 ObjectPath::parse(&to_parse).map_err(anyhow::Error::from)
718 }
719
720 pub(super) fn object_store_path(&self, path: &str) -> anyhow::Result<String> {
721 let normalized_path = normalize_path_separators(path);
722
723 if self.is_remote_uri() {
724 if normalized_path.contains("://") {
725 let path_under_root = self.remote_uri_object_path(&normalized_path)?;
726 return Ok(self.path_under_base(&path_under_root));
727 }
728
729 return Ok(self.path_under_base(&normalized_path));
730 }
731
732 Ok(self.path_without_local_base(&normalized_path))
733 }
734
735 fn remote_uri_object_path(&self, path: &str) -> anyhow::Result<String> {
736 let path_url = url::Url::parse(path)
737 .map_err(|e| anyhow::anyhow!("Failed to parse object store URI {path}: {e}"))?;
738 if !is_remote_uri_scheme(path_url.scheme()) {
739 anyhow::bail!(
740 "URI {path} uses non-remote scheme {} for remote catalog at {}",
741 path_url.scheme(),
742 self.original_uri,
743 );
744 }
745
746 let catalog_root = remote_store_root_url(&self.original_uri)?;
747 let path_root = remote_store_root_url(path)?;
748 if catalog_root.as_str().trim_end_matches('/') != path_root.as_str().trim_end_matches('/') {
749 anyhow::bail!(
750 "Cross-store URI {path} (root {}) does not belong to catalog rooted at {} ({})",
751 path_root.as_str().trim_end_matches('/'),
752 self.original_uri,
753 catalog_root.as_str().trim_end_matches('/'),
754 );
755 }
756
757 // The URL crate keeps the path component percent-encoded (e.g. `%5E`),
758 // so preserve that encoding for `ObjectPath::parse` round-trips through
759 // `object_store::list`/`get`.
760 Ok(path_url.path().trim_start_matches('/').to_string())
761 }
762
763 pub(crate) fn path_without_local_base(&self, path: &str) -> String {
764 let base_path = if self.base_path.is_empty() {
765 self.native_base_path_string()
766 } else {
767 self.base_path.clone()
768 };
769
770 let normalized_base = normalize_path_separators(&base_path);
771 let base = normalized_base.trim_end_matches('/');
772
773 if base.is_empty() {
774 path.to_string()
775 } else if path == base {
776 String::new()
777 } else if let Some(without_base) = path.strip_prefix(&format!("{base}/")) {
778 without_base.to_string()
779 } else {
780 path.to_string()
781 }
782 }
783
784 fn path_under_base(&self, path: &str) -> String {
785 let normalized_path = normalize_path_separators(path);
786 let path = normalized_path
787 .trim_start_matches('/')
788 .trim_end_matches('/');
789
790 if self.base_path.is_empty() {
791 return path.to_string();
792 }
793
794 let normalized_base = normalize_path_separators(&self.base_path);
795 let base = normalized_base
796 .trim_start_matches('/')
797 .trim_end_matches('/');
798
799 if base.is_empty() || path == base || path.starts_with(&format!("{base}/")) {
800 path.to_string()
801 } else if path.is_empty() {
802 base.to_string()
803 } else {
804 make_object_store_path(base, [path])
805 }
806 }
807
808 /// Helper method to move a file using object store rename operation
809 pub fn move_file(&self, old_path: &ObjectPath, new_path: &ObjectPath) -> anyhow::Result<()> {
810 if old_path == new_path {
811 return Ok(());
812 }
813
814 self.recover_consolidation(&old_path.parent().unwrap_or_default())?;
815 self.recover_consolidation(&new_path.parent().unwrap_or_default())?;
816
817 self.execute_async(|| async {
818 self.object_store
819 .rename(old_path, new_path)
820 .await
821 .map_err(anyhow::Error::from)
822 })
823 }
824
825 /// Helper method to execute async operations with a runtime
826 pub fn execute_async<C, F, R>(&self, create_future: C) -> anyhow::Result<R>
827 where
828 C: FnOnce() -> F + Send,
829 F: std::future::Future<Output = anyhow::Result<R>>,
830 R: Send,
831 {
832 block_on_nautilus_with(create_future)
833 }
834
835 /// Lists directory stems (directory names without path) in a subdirectory.
836 ///
837 /// This method scans a subdirectory and returns the names of all immediate
838 /// subdirectories. It's used to list data types, backtest runs, and live runs.
839 ///
840 /// # Parameters
841 ///
842 /// - `subdirectory`: The subdirectory path to scan (e.g., "data", "backtest", "live").
843 ///
844 /// # Returns
845 ///
846 /// Returns the sorted directory names (stems) found in the subdirectory, decoded from their
847 /// percent-encoded object-store form, or an error if the operation fails.
848 ///
849 /// # Errors
850 ///
851 /// Returns an error if:
852 /// - Object store listing operations fail.
853 /// - Directory access is denied.
854 ///
855 /// # Examples
856 ///
857 /// ```rust,no_run
858 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
859 ///
860 /// let mut catalog = ParquetDataCatalog::new(
861 /// std::path::Path::new("/tmp/nautilus_data"),
862 /// None,
863 /// None,
864 /// None,
865 /// None,
866 /// );
867 ///
868 /// // List all data types
869 /// let data_types = catalog.list_directory_stems("data")?;
870 /// for data_type in data_types {
871 /// println!("Found data type: {}", data_type);
872 /// }
873 /// # Ok::<(), anyhow::Error>(())
874 /// ```
875 pub fn list_directory_stems(&self, subdirectory: &str) -> anyhow::Result<Vec<String>> {
876 // For local filesystem paths, use filesystem operations to detect empty directories
877 // For remote object stores, we can only list directories that contain files
878 if !self.is_remote_uri() {
879 let directory = PathBuf::from(self.native_base_path_string()).join(subdirectory);
880
881 let entries = match std::fs::read_dir(&directory) {
882 Ok(entries) => entries,
883 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
884 Err(e) => {
885 return Err(e)
886 .with_context(|| format!("failed to list {}", directory.display()));
887 }
888 };
889
890 let mut directories = Vec::new();
891
892 for entry in entries {
893 let entry =
894 entry.with_context(|| format!("failed to list {}", directory.display()))?;
895 let file_type = entry
896 .file_type()
897 .with_context(|| format!("failed to read {}", entry.path().display()))?;
898
899 // Use file_name() to get the directory name (not file_stem which removes extension)
900 if file_type.is_dir()
901 && let Some(name) = entry.path().file_name()
902 {
903 directories.push(decode_object_store_segment(&name.to_string_lossy()));
904 }
905 }
906
907 directories.sort();
908 return Ok(directories);
909 }
910
911 let storage = StorageBackend {
912 object_store: self.object_store.clone(),
913 base_path: self.base_path.clone(),
914 original_uri: self.original_uri.clone(),
915 };
916
917 self.execute_async(|| async { storage.list_directory_stems(subdirectory).await })
918 }
919
920 /// Lists the data, record, and instrument types stored in the catalog.
921 ///
922 /// Each `data/<type>` directory maps to its catalog type, and each `data/custom/<type_name>`
923 /// directory to a custom data type. Unrecognized directories are skipped with a warning.
924 ///
925 /// # Returns
926 ///
927 /// Returns the catalog types, or an error if the operation fails.
928 ///
929 /// # Errors
930 ///
931 /// Returns an error if:
932 /// - Object store listing operations fail.
933 /// - Directory access is denied.
934 ///
935 /// # Examples
936 ///
937 /// ```rust,no_run
938 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
939 ///
940 /// let mut catalog = ParquetDataCatalog::new(
941 /// std::path::Path::new("/tmp/nautilus_data"),
942 /// None,
943 /// None,
944 /// None,
945 /// None,
946 /// );
947 ///
948 /// // List all data types
949 /// let data_types = catalog.list_data_types()?;
950 /// for data_type in data_types {
951 /// println!("Available data type: {data_type}");
952 /// }
953 /// # Ok::<(), anyhow::Error>(())
954 /// ```
955 pub fn list_data_types(&self) -> anyhow::Result<Vec<CatalogDataType>> {
956 let mut data_types = Vec::new();
957
958 for stem in self.list_directory_stems("data")? {
959 if stem == "custom" {
960 for type_name in self.list_directory_stems("data/custom")? {
961 data_types.push(CatalogDataType::from(NautilusDataType::Custom {
962 type_name,
963 }));
964 }
965
966 continue;
967 }
968
969 match catalog_data_type_from_path(&stem) {
970 Ok(data_type) => data_types.push(data_type),
971 Err(e) => log::warn!("Skipping catalog directory data/{stem}: {e}"),
972 }
973 }
974
975 Ok(data_types)
976 }
977
978 /// Lists all backtest run IDs available in the catalog.
979 ///
980 /// This method returns the names of all backtest run directories in the catalog.
981 /// Each backtest run corresponds to a specific backtest execution instance.
982 ///
983 /// # Returns
984 ///
985 /// Returns a vector of backtest run IDs, or an error if the operation fails.
986 ///
987 /// # Errors
988 ///
989 /// Returns an error if:
990 /// - Object store listing operations fail.
991 /// - Directory access is denied.
992 ///
993 /// # Examples
994 ///
995 /// ```rust,no_run
996 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
997 ///
998 /// let mut catalog = ParquetDataCatalog::new(
999 /// std::path::Path::new("/tmp/nautilus_data"),
1000 /// None,
1001 /// None,
1002 /// None,
1003 /// None,
1004 /// );
1005 ///
1006 /// // List all backtest runs
1007 /// let runs = catalog.list_backtest_runs()?;
1008 /// for run_id in runs {
1009 /// println!("Backtest run: {}", run_id);
1010 /// }
1011 /// # Ok::<(), anyhow::Error>(())
1012 /// ```
1013 pub fn list_backtest_runs(&self) -> anyhow::Result<Vec<String>> {
1014 self.list_directory_stems("backtest")
1015 }
1016
1017 /// Lists all live run IDs available in the catalog.
1018 ///
1019 /// This method returns the names of all live run directories in the catalog.
1020 /// Each live run corresponds to a specific live trading execution instance.
1021 ///
1022 /// # Returns
1023 ///
1024 /// Returns a vector of live run IDs, or an error if the operation fails.
1025 ///
1026 /// # Errors
1027 ///
1028 /// Returns an error if:
1029 /// - Object store listing operations fail.
1030 /// - Directory access is denied.
1031 ///
1032 /// # Examples
1033 ///
1034 /// ```rust,no_run
1035 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
1036 ///
1037 /// let mut catalog = ParquetDataCatalog::new(
1038 /// std::path::Path::new("/tmp/nautilus_data"),
1039 /// None,
1040 /// None,
1041 /// None,
1042 /// None,
1043 /// );
1044 ///
1045 /// // List all live runs
1046 /// let runs = catalog.list_live_runs()?;
1047 /// for run_id in runs {
1048 /// println!("Live run: {}", run_id);
1049 /// }
1050 /// # Ok::<(), anyhow::Error>(())
1051 /// ```
1052 pub fn list_live_runs(&self) -> anyhow::Result<Vec<String>> {
1053 self.list_directory_stems("live")
1054 }
1055}
1056
1057#[cfg(test)]
1058mod tests {
1059 use std::sync::Arc;
1060
1061 use nautilus_model::data::NautilusDataType;
1062 use object_store::{ObjectStoreExt, PutPayload, memory::InMemory, path::Path as ObjectPath};
1063 use rstest::rstest;
1064
1065 use super::ParquetDataCatalog;
1066 use crate::{catalog::types::CatalogDataType, common::datafusion::DataBackendSession};
1067
1068 /// Builds a catalog over an in-memory store. Only remote catalogs carry a non-empty
1069 /// `base_path`, so seeding one here is the sole way to reproduce a bucket sub-prefix.
1070 fn memory_catalog(base_path: &str) -> ParquetDataCatalog {
1071 ParquetDataCatalog {
1072 base_path: base_path.to_string(),
1073 original_uri: "memory://".to_string(),
1074 object_store: Arc::new(InMemory::new()),
1075 session: DataBackendSession::new(5_000),
1076 batch_size: 5_000,
1077 compression: parquet::basic::Compression::SNAPPY,
1078 max_row_group_size: 5_000,
1079 }
1080 }
1081
1082 fn seed(catalog: &ParquetDataCatalog, keys: &[&str]) {
1083 catalog
1084 .execute_async(|| async {
1085 for key in keys {
1086 catalog
1087 .object_store
1088 .put(&ObjectPath::from(*key), PutPayload::from_static(b"x"))
1089 .await?;
1090 }
1091 Ok(())
1092 })
1093 .unwrap();
1094 }
1095
1096 #[rstest]
1097 #[case("nautilus-data", "nautilus-data/")]
1098 #[case("", "")]
1099 fn list_instruments_returns_ids_under_base_path(
1100 #[case] base_path: &str,
1101 #[case] key_prefix: &str,
1102 ) {
1103 let catalog = memory_catalog(base_path);
1104 let keys = [
1105 "data/quotes/EURUSD.SIM/0-1.parquet",
1106 "data/quotes/EURUSD.SIM/2-3.parquet",
1107 "data/quotes/GBPUSD.SIM/0-1.parquet",
1108 "data/trades/AUDUSD.SIM/0-1.parquet",
1109 ]
1110 .map(|key| format!("{key_prefix}{key}"));
1111 seed(&catalog, &keys.each_ref().map(String::as_str));
1112
1113 assert_eq!(
1114 catalog
1115 .list_instruments(&NautilusDataType::QuoteTick.into())
1116 .unwrap(),
1117 ["EURUSD.SIM", "GBPUSD.SIM"]
1118 );
1119 }
1120
1121 #[rstest]
1122 #[case("nautilus-data", "nautilus-data/")]
1123 #[case("", "")]
1124 fn list_parquet_files_lists_type_directory_recursively(
1125 #[case] base_path: &str,
1126 #[case] key_prefix: &str,
1127 ) {
1128 let catalog = memory_catalog(base_path);
1129 let keys = [
1130 "data/quotes/EURUSD.SIM/0-1.parquet",
1131 "data/quotes/EURUSD.SIM/0-1.json",
1132 "data/quotes/BTC€.SIM/2-3.parquet",
1133 "data/trades/AUDUSD.SIM/0-1.parquet",
1134 ]
1135 .map(|key| format!("{key_prefix}{key}"));
1136 seed(&catalog, &keys.each_ref().map(String::as_str));
1137
1138 let directory = catalog.make_path("quotes", None).unwrap();
1139 let mut files = catalog.list_parquet_files(&directory).unwrap();
1140 files.sort();
1141
1142 assert_eq!(
1143 files,
1144 [
1145 format!("{key_prefix}data/quotes/BTC%E2%82%AC.SIM/2-3.parquet"),
1146 format!("{key_prefix}data/quotes/EURUSD.SIM/0-1.parquet"),
1147 ]
1148 );
1149 }
1150
1151 #[rstest]
1152 fn list_instruments_ignores_unpartitioned_files() {
1153 // An empty `base_path` keeps this distinct from the remote prefix defect, so an empty
1154 // result can only come from the layout check.
1155 let catalog = memory_catalog("");
1156 seed(
1157 &catalog,
1158 &[
1159 "data/custom/MyType/1-2.parquet",
1160 "data/custom/MyType/1-2.json",
1161 ],
1162 );
1163
1164 let custom = CatalogDataType::Data(NautilusDataType::Custom {
1165 type_name: "MyType".to_string(),
1166 });
1167
1168 assert!(catalog.list_instruments(&custom).unwrap().is_empty());
1169 }
1170
1171 #[rstest]
1172 fn list_instruments_decodes_percent_encoded_ids() {
1173 let catalog = memory_catalog("nautilus-data");
1174 seed(
1175 &catalog,
1176 &["nautilus-data/data/quotes/BTC€.SIM/0-1.parquet"],
1177 );
1178
1179 assert_eq!(
1180 catalog
1181 .list_instruments(&NautilusDataType::QuoteTick.into())
1182 .unwrap(),
1183 ["BTC€.SIM"]
1184 );
1185 }
1186}