Skip to main content

nautilus_persistence/backend/parquet/
file_admin.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! File-level admin operations: existence checks, deletion, name resets, leaf-directory walk.
17
18use std::collections::BTreeMap;
19
20use ahash::AHashSet;
21use nautilus_core::UnixNanos;
22use object_store::{ObjectStoreExt, path::Path as ObjectPath};
23
24use crate::{
25    backend::parquet::{
26        catalog::ParquetDataCatalog,
27        intervals::are_intervals_disjoint,
28        io::{decode_object_store_segment, min_max_from_parquet_metadata_object_store},
29        paths::{make_object_store_path, timestamps_to_filename},
30    },
31    catalog::types::{CatalogDataType, parquet_catalog_data_type_path_prefixes},
32};
33
34impl ParquetDataCatalog {
35    /// Checks if a file exists in the object store.
36    ///
37    /// This method performs a HEAD operation on the object store to determine if a file
38    /// exists without downloading its content. It works with both local and remote object stores.
39    ///
40    /// # Parameters
41    ///
42    /// - `path`: The file path to check, relative to the catalog structure.
43    ///
44    /// # Returns
45    ///
46    /// Returns `true` if the file exists, `false` if it doesn't exist.
47    ///
48    /// # Errors
49    ///
50    /// Returns an error if the object store operation fails due to network issues,
51    /// authentication problems, or other I/O errors.
52    pub(crate) fn file_exists(&self, path: &str) -> anyhow::Result<bool> {
53        let object_path = self.to_object_path(path)?;
54
55        self.execute_async(|| async {
56            match self.object_store.head(&object_path).await {
57                Ok(_) => Ok(true),
58                Err(object_store::Error::NotFound { .. }) => Ok(false),
59                Err(e) => Err(e.into()),
60            }
61        })
62    }
63
64    /// Deletes a file from the object store.
65    ///
66    /// This method removes a file from the object store. The operation is permanent
67    /// and cannot be undone. It works with both local filesystems and remote object stores.
68    ///
69    /// # Parameters
70    ///
71    /// - `path`: The file path to delete, as an object-store listing returns it.
72    ///
73    /// # Returns
74    ///
75    /// Returns `Ok(())` on successful deletion.
76    ///
77    /// # Errors
78    ///
79    /// Returns an error if:
80    /// - The file doesn't exist.
81    /// - Permission is denied.
82    /// - Network issues occur (for remote stores).
83    /// - The object store operation fails.
84    ///
85    /// # Safety
86    ///
87    /// This operation is irreversible. Ensure the file is no longer needed before deletion.
88    pub(crate) fn delete_file(&self, path: &str) -> anyhow::Result<()> {
89        let object_path = self.to_object_path_parsed(path)?;
90        self.recover_consolidation(&object_path.parent().unwrap_or_default())?;
91        self.execute_async(|| async {
92            self.object_store
93                .delete(&object_path)
94                .await
95                .map_err(anyhow::Error::from)
96        })
97    }
98
99    /// Resets the filenames of all Parquet files in the catalog to match their actual content timestamps.
100    ///
101    /// This method scans all leaf data directories in the catalog and renames files based on
102    /// the actual timestamp range of their content. This is useful when files have been
103    /// modified or when filename conventions have changed.
104    ///
105    /// # Returns
106    ///
107    /// Returns `Ok(())` on success, or an error if the operation fails.
108    ///
109    /// # Errors
110    ///
111    /// Returns an error if:
112    /// - Directory listing fails.
113    /// - File metadata reading fails.
114    /// - File rename operations fail.
115    /// - The content intervals in a directory are not disjoint, or a new name matches another
116    ///   file's current name. Both are checked before any file in that directory is renamed.
117    ///
118    /// # Examples
119    ///
120    /// ```rust,no_run
121    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
122    ///
123    /// let mut catalog = ParquetDataCatalog::new(
124    ///     std::path::Path::new("/tmp/nautilus_data"),
125    ///     None,
126    ///     None,
127    ///     None,
128    ///     None,
129    /// );
130    ///
131    /// // Reset all filenames in the catalog
132    /// catalog.reset_all_file_names()?;
133    /// # Ok::<(), anyhow::Error>(())
134    /// ```
135    pub fn reset_all_file_names(&self) -> anyhow::Result<()> {
136        let leaf_directories = self.find_leaf_data_directories()?;
137
138        for directory in leaf_directories {
139            self.reset_file_names(&directory)?;
140        }
141
142        Ok(())
143    }
144
145    /// Resets the filenames of Parquet files for a specific data type and identifier.
146    ///
147    /// This method renames files in a specific directory based on the actual timestamp
148    /// range of their content. This is useful for correcting filenames after data
149    /// modifications or when filename conventions have changed.
150    ///
151    /// # Parameters
152    ///
153    /// - `data_type`: The stored family to target.
154    /// - `identifier`: Optional identifier to target a specific instrument's data. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
155    ///
156    /// # Returns
157    ///
158    /// Returns `Ok(())` on success, or an error if the operation fails.
159    ///
160    /// # Errors
161    ///
162    /// Returns an error if:
163    /// - The directory path cannot be constructed.
164    /// - File metadata reading fails.
165    /// - File rename operations fail.
166    /// - The content intervals in a directory are not disjoint, or a new name matches another
167    ///   file's current name. Both are checked before any file in that directory is renamed.
168    ///
169    /// # Examples
170    ///
171    /// ```rust,no_run
172    /// use nautilus_model::data::NautilusDataType;
173    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
174    ///
175    /// let mut catalog = ParquetDataCatalog::new(
176    ///     std::path::Path::new("/tmp/nautilus_data"),
177    ///     None,
178    ///     None,
179    ///     None,
180    ///     None,
181    /// );
182    ///
183    /// // Reset filenames for all quote files
184    /// catalog.reset_data_file_names(&NautilusDataType::QuoteTick.into(), None)?;
185    ///
186    /// // Reset filenames for a specific instrument's trade files
187    /// catalog.reset_data_file_names(&NautilusDataType::TradeTick.into(), Some("BTCUSD"))?;
188    /// # Ok::<(), anyhow::Error>(())
189    /// ```
190    pub fn reset_data_file_names(
191        &self,
192        data_type: &CatalogDataType,
193        identifier: Option<&str>,
194    ) -> anyhow::Result<()> {
195        for type_name in parquet_catalog_data_type_path_prefixes(data_type) {
196            let directory = self.make_path(type_name.as_ref(), identifier)?;
197            self.reset_file_names(&directory)?;
198        }
199
200        Ok(())
201    }
202
203    /// Resets the filenames of Parquet files in a directory to match their actual content timestamps.
204    ///
205    /// This internal method scans all Parquet files in a directory, reads their metadata to
206    /// determine the actual timestamp range of their content, and renames the files accordingly.
207    /// This ensures that filenames accurately reflect the data they contain. The listing includes
208    /// subdirectories, so each file keeps its own directory and is validated against the files
209    /// that share it.
210    ///
211    /// # Parameters
212    ///
213    /// - `directory`: The directory path containing Parquet files to rename.
214    ///
215    /// # Returns
216    ///
217    /// Returns `Ok(())` on success, or an error if the operation fails.
218    ///
219    /// # Process
220    ///
221    /// 1. Lists all Parquet files in the directory
222    /// 2. For each file, reads metadata to extract min/max timestamps
223    /// 3. Generates a new filename based on actual timestamp range
224    /// 4. Validates the new names before moving any file
225    /// 5. Moves each file to its new name using object store operations
226    ///
227    /// # Errors
228    ///
229    /// Returns an error if:
230    /// - Directory listing fails.
231    /// - Metadata reading fails for any file.
232    /// - The content intervals within a file's directory are not disjoint.
233    /// - A new name matches the current name of another file.
234    /// - File move operations fail.
235    /// - Object store operations fail.
236    ///
237    /// # Notes
238    ///
239    /// - This operation can be time-consuming for directories with many files.
240    /// - A validation error leaves every file name unchanged.
241    /// - Files are renamed one at a time. Object stores without a native rename copy then
242    ///   delete, so a failure can leave a file under both names.
243    fn reset_file_names(&self, directory: &str) -> anyhow::Result<()> {
244        self.recover_consolidation(&self.to_object_path(directory)?)?;
245        let parquet_files = self.list_parquet_files(directory)?;
246        let mut intervals_by_directory = BTreeMap::<ObjectPath, Vec<(u64, u64)>>::new();
247        let mut moves = Vec::with_capacity(parquet_files.len());
248
249        for file in parquet_files {
250            let object_path = self.to_object_path_parsed(&file)?;
251
252            let (first_ts, last_ts) = self.execute_async(|| async {
253                min_max_from_parquet_metadata_object_store(
254                    self.object_store.clone(),
255                    &object_path,
256                    "ts_init",
257                )
258                .await
259            })?;
260
261            let new_filename =
262                timestamps_to_filename(UnixNanos::from(first_ts), UnixNanos::from(last_ts));
263            let file_directory = object_path.parent().unwrap_or_default();
264            let new_object_path = file_directory.clone().join(new_filename.as_str());
265
266            intervals_by_directory
267                .entry(file_directory)
268                .or_default()
269                .push((first_ts, last_ts));
270            moves.push((object_path, new_object_path));
271        }
272
273        for (file_directory, intervals) in &intervals_by_directory {
274            anyhow::ensure!(
275                are_intervals_disjoint(intervals),
276                "Cannot reset file names in directory '{file_directory}': content intervals are \
277                 not disjoint: {intervals:?}",
278            );
279        }
280
281        // Moves overwrite, so a new name held by another file would replace that file
282        let current_paths = moves
283            .iter()
284            .map(|(old_path, _)| old_path)
285            .collect::<AHashSet<_>>();
286
287        if let Some((old_path, new_path)) = moves
288            .iter()
289            .find(|(old_path, new_path)| old_path != new_path && current_paths.contains(new_path))
290        {
291            anyhow::bail!(
292                "Cannot reset file names in directory '{directory}': new name {new_path} for \
293                 {old_path} is held by another file; rename that file to an unused interval \
294                 name and retry"
295            );
296        }
297
298        for (old_path, new_path) in &moves {
299            self.move_file(old_path, new_path)?;
300        }
301
302        Ok(())
303    }
304
305    /// Finds all leaf data directories in the catalog.
306    ///
307    /// A leaf directory is one that contains data files but no subdirectories.
308    /// This method is used to identify directories that can be processed for
309    /// consolidation or other operations.
310    ///
311    /// # Returns
312    ///
313    /// Returns a vector of directory path strings representing leaf directories,
314    /// or an error if directory traversal fails.
315    ///
316    /// # Errors
317    ///
318    /// Returns an error if:
319    /// - Object store listing operations fail.
320    /// - Directory structure cannot be analyzed.
321    ///
322    /// # Examples
323    ///
324    /// ```rust,no_run
325    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
326    ///
327    /// let mut catalog = ParquetDataCatalog::new(
328    ///     std::path::Path::new("/tmp/nautilus_data"),
329    ///     None,
330    ///     None,
331    ///     None,
332    ///     None,
333    /// );
334    ///
335    /// let leaf_dirs = catalog.find_leaf_data_directories()?;
336    /// for dir in leaf_dirs {
337    ///     println!("Found leaf directory: {}", dir);
338    /// }
339    /// # Ok::<(), anyhow::Error>(())
340    /// ```
341    pub fn find_leaf_data_directories(&self) -> anyhow::Result<Vec<String>> {
342        let data_dir = make_object_store_path(&self.base_path, ["data"]);
343        let objects = self.list_objects(&data_dir)?;
344
345        let mut directories = AHashSet::new();
346
347        // List all objects under the data directory
348        for object in objects {
349            let parts = object.location.parts().collect::<Vec<_>>();
350
351            // Decode to match `make_path` output, which callers encode again
352            if let Some((_, parent)) = parts.split_last() {
353                let directory = parent
354                    .iter()
355                    .map(|part| decode_object_store_segment(part.as_ref()))
356                    .collect::<Vec<_>>()
357                    .join("/");
358                directories.insert(directory);
359            }
360        }
361
362        // Find leaf directories (every listed directory contains at least one file,
363        // so a leaf is one without subdirectories)
364        let mut leaf_dirs = Vec::new();
365
366        for dir in &directories {
367            let prefix = format!("{dir}/");
368            let has_subdirs = directories.iter().any(|d| d.starts_with(&prefix));
369
370            if !has_subdirs {
371                leaf_dirs.push(dir.clone());
372            }
373        }
374
375        leaf_dirs.sort();
376
377        Ok(leaf_dirs)
378    }
379}