Skip to main content

nautilus_persistence/backend/parquet/
consolidation.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Period-based and bulk consolidation of parquet files in a catalog directory.
17
18#![expect(
19    clippy::missing_errors_doc,
20    reason = "consolidation operations forward catalog/storage errors and operate on validated batches"
21)]
22
23use std::collections::BTreeMap;
24
25use ahash::AHashSet;
26use arrow::record_batch::RecordBatch;
27use nautilus_core::{ClosedInterval, UnixNanos, string::conversions::to_snake_case};
28use nautilus_model::data::{
29    Bar, CustomData, Data, HasTsInit, IndexPriceUpdate, MarkPriceUpdate, NautilusDataType,
30    OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick, close::InstrumentClose,
31};
32use nautilus_serialization::arrow::{
33    DecodeTypedFromRecordBatch, EncodeToRecordBatch, catalog_identifier_from_metadata,
34};
35use object_store::path::Path as ObjectPath;
36
37use crate::{
38    backend::parquet::{
39        catalog::ParquetDataCatalog,
40        intervals::{are_intervals_contiguous, are_intervals_disjoint},
41        io::consolidate_parquet_files_from_object_store,
42        paths::{
43            extract_path_components, make_object_store_path, normalize_path_separators,
44            parse_filename_timestamps, timestamps_to_filename,
45        },
46    },
47    catalog::types::{
48        CatalogDataType, HasCatalogDataType, parquet_catalog_data_type_path_prefixes,
49        parquet_data_path_prefix,
50    },
51    common::coverage::merge_closed_intervals,
52};
53
54/// Information about a consolidation query to be executed.
55#[derive(Debug, Clone)]
56pub struct ConsolidationQuery {
57    /// Start timestamp for the query range (inclusive, in nanoseconds).
58    pub query_start: u64,
59    /// End timestamp for the query range (inclusive, in nanoseconds).
60    pub query_end: u64,
61    /// Whether to use period boundaries for file naming (true) or actual data timestamps (false).
62    pub use_period_boundaries: bool,
63}
64
65impl ParquetDataCatalog {
66    /// Consolidates all data files in the catalog.
67    ///
68    /// This method identifies all leaf directories in the catalog that contain parquet files
69    /// and consolidates them. A leaf directory is one that contains files but no subdirectories.
70    /// This is a convenience method that effectively calls `consolidate_data` for all data types
71    /// and instrument IDs in the catalog.
72    ///
73    /// # Parameters
74    ///
75    /// - `start`: Optional start timestamp for the consolidation range. Only files with timestamps
76    ///   greater than or equal to this value will be consolidated. If None, all files
77    ///   from the beginning of time will be considered.
78    /// - `end`: Optional end timestamp for the consolidation range. Only files with timestamps
79    ///   less than or equal to this value will be consolidated. If None, all files
80    ///   up to the end of time will be considered.
81    /// - `ensure_contiguous_files`: Whether to validate that consolidated intervals are contiguous (default: true).
82    ///
83    /// # Returns
84    ///
85    /// Returns `Ok(())` on success, or an error if consolidation fails for any directory.
86    ///
87    /// # Errors
88    ///
89    /// Returns an error if:
90    /// - Directory listing fails.
91    /// - File consolidation operations fail.
92    /// - Interval validation fails (when `ensure_contiguous_files` is true).
93    ///
94    /// # Examples
95    ///
96    /// ```rust,no_run
97    /// use nautilus_core::UnixNanos;
98    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
99    ///
100    /// let mut catalog = ParquetDataCatalog::new(
101    ///     std::path::Path::new("/tmp/nautilus_data"),
102    ///     None,
103    ///     None,
104    ///     None,
105    ///     None,
106    /// );
107    ///
108    /// // Consolidate all files in the catalog
109    /// catalog.consolidate_catalog(None, None, None, None)?;
110    ///
111    /// // Consolidate only files within a specific time range
112    /// catalog.consolidate_catalog(
113    ///     Some(UnixNanos::from(1609459200000000000)),
114    ///     Some(UnixNanos::from(1609545600000000000)),
115    ///     Some(true),
116    ///     None,
117    /// )?;
118    /// # Ok::<(), anyhow::Error>(())
119    /// ```
120    pub fn consolidate_catalog(
121        &self,
122        start: Option<UnixNanos>,
123        end: Option<UnixNanos>,
124        ensure_contiguous_files: Option<bool>,
125        deduplicate: Option<bool>,
126    ) -> anyhow::Result<()> {
127        let leaf_directories = self.find_leaf_data_directories()?;
128
129        for directory in leaf_directories {
130            self.consolidate_directory(
131                &directory,
132                start,
133                end,
134                ensure_contiguous_files,
135                deduplicate,
136            )?;
137        }
138
139        Ok(())
140    }
141
142    /// Consolidates data files for a specific data type and identifier.
143    ///
144    /// This method selects directories by data type and optional identifier, then merges files
145    /// within each directory. It preserves identifier partitions while reducing file count.
146    ///
147    /// # Parameters
148    ///
149    /// - `data_type`: The stored family to consolidate.
150    /// - `identifier`: Optional identifier to target a specific instrument's data. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
151    /// - `start`: Optional start timestamp to limit consolidation to files within this range.
152    /// - `end`: Optional end timestamp to limit consolidation to files within this range.
153    /// - `ensure_contiguous_files`: Whether to validate that consolidated intervals are contiguous (default: true).
154    ///
155    /// # Returns
156    ///
157    /// Returns `Ok(())` on success, or an error if consolidation fails.
158    ///
159    /// # Errors
160    ///
161    /// Returns an error if:
162    /// - The directory path cannot be constructed.
163    /// - File consolidation operations fail.
164    /// - Interval validation fails (when `ensure_contiguous_files` is true).
165    ///
166    /// # Examples
167    ///
168    /// ```rust,no_run
169    /// use nautilus_core::UnixNanos;
170    /// use nautilus_model::data::NautilusDataType;
171    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
172    ///
173    /// let mut catalog = ParquetDataCatalog::new(
174    ///     std::path::Path::new("/tmp/nautilus_data"),
175    ///     None,
176    ///     None,
177    ///     None,
178    ///     None,
179    /// );
180    ///
181    /// // Consolidate all quote files for a specific instrument
182    /// catalog.consolidate_data(
183    ///     &NautilusDataType::QuoteTick.into(),
184    ///     Some("BTCUSD"),
185    ///     None,
186    ///     None,
187    ///     None,
188    ///     None,
189    /// )?;
190    ///
191    /// // Consolidate trade files within a time range
192    /// catalog.consolidate_data(
193    ///     &NautilusDataType::TradeTick.into(),
194    ///     None,
195    ///     Some(UnixNanos::from(1609459200000000000)),
196    ///     Some(UnixNanos::from(1609545600000000000)),
197    ///     Some(true),
198    ///     None,
199    /// )?;
200    /// # Ok::<(), anyhow::Error>(())
201    /// ```
202    pub fn consolidate_data(
203        &mut self,
204        data_type: &CatalogDataType,
205        identifier: Option<&str>,
206        start: Option<UnixNanos>,
207        end: Option<UnixNanos>,
208        ensure_contiguous_files: Option<bool>,
209        deduplicate: Option<bool>,
210    ) -> anyhow::Result<()> {
211        for type_name in parquet_catalog_data_type_path_prefixes(data_type) {
212            self.consolidate_prefix_data(
213                type_name.as_ref(),
214                identifier,
215                start,
216                end,
217                ensure_contiguous_files,
218                deduplicate,
219            )?;
220        }
221
222        Ok(())
223    }
224
225    fn consolidate_prefix_data(
226        &mut self,
227        type_name: &str,
228        identifier: Option<&str>,
229        start: Option<UnixNanos>,
230        end: Option<UnixNanos>,
231        ensure_contiguous_files: Option<bool>,
232        deduplicate: Option<bool>,
233    ) -> anyhow::Result<()> {
234        let directory = self.make_path(type_name, identifier)?;
235        let raw_result = self.consolidate_directory(
236            &directory,
237            start,
238            end,
239            ensure_contiguous_files,
240            deduplicate,
241        );
242
243        match raw_result {
244            Ok(()) => Ok(()),
245            Err(raw_error)
246                if can_rewrite_consolidation_by_period(type_name)
247                    && is_schema_incompatibility(&raw_error) =>
248            {
249                if deduplicate.unwrap_or(false) {
250                    anyhow::bail!(
251                        "Raw parquet consolidation failed due to incompatible file schemas, \
252                         but typed period consolidation cannot preserve deduplicate=true. \
253                         Raw consolidation error: {raw_error}"
254                    );
255                }
256
257                log::warn!(
258                    "Raw parquet consolidation failed due to incompatible file schemas for \
259                     {type_name}; retrying with typed period consolidation. Raw error: {raw_error}"
260                );
261
262                self.consolidate_prefix_data_by_period(
263                    type_name,
264                    identifier,
265                    None,
266                    start,
267                    end,
268                    ensure_contiguous_files,
269                )
270                .map_err(|typed_error| {
271                    anyhow::anyhow!(
272                        "Raw parquet consolidation failed due to incompatible file schemas, \
273                         and typed period consolidation also failed. Raw error: {raw_error}; \
274                         typed period error: {typed_error}"
275                    )
276                })
277            }
278            Err(e) => Err(e),
279        }
280    }
281
282    /// Consolidates Parquet files within a specific directory by merging them into a single file.
283    ///
284    /// This internal method performs the actual consolidation work for a single directory.
285    /// It identifies files within the specified time range, validates their intervals,
286    /// and combines them into a single Parquet file with optimized storage.
287    ///
288    /// # Parameters
289    ///
290    /// - `directory`: The directory path containing Parquet files to consolidate.
291    /// - `start`: Optional start timestamp to limit consolidation to files within this range.
292    /// - `end`: Optional end timestamp to limit consolidation to files within this range.
293    /// - `ensure_contiguous_files`: Whether to validate that consolidated intervals are contiguous.
294    ///
295    /// # Returns
296    ///
297    /// Returns `Ok(())` on success, or an error if consolidation fails.
298    ///
299    /// # Behavior
300    ///
301    /// - Skips directories containing 1 or fewer files.
302    /// - Filters files by timestamp range if start/end are specified.
303    /// - Sorts files by interval bounds and path before consolidation.
304    /// - Creates a new file spanning the entire time range of input files.
305    /// - Validates interval disjointness and contiguity before merging (if enabled).
306    ///
307    /// # Errors
308    ///
309    /// Returns an error if:
310    /// - Directory listing fails.
311    /// - File combination operations fail.
312    /// - Interval validation fails (when `ensure_contiguous_files` is true).
313    /// - Object store operations fail.
314    fn consolidate_directory(
315        &self,
316        directory: &str,
317        start: Option<UnixNanos>,
318        end: Option<UnixNanos>,
319        ensure_contiguous_files: Option<bool>,
320        deduplicate: Option<bool>,
321    ) -> anyhow::Result<()> {
322        self.recover_consolidation(&self.to_object_path(directory)?)?;
323        let parquet_files = self.list_parquet_files(directory)?;
324        let mut directories = BTreeMap::<ObjectPath, Vec<String>>::new();
325
326        for file in parquet_files {
327            let path = self.to_object_path_parsed(&file)?;
328            let directory = path.parent().unwrap_or_default();
329            directories.entry(directory).or_default().push(file);
330        }
331
332        for (directory, files) in directories {
333            self.consolidate_files(
334                &directory,
335                files,
336                start,
337                end,
338                ensure_contiguous_files,
339                deduplicate,
340            )?;
341        }
342
343        Ok(())
344    }
345
346    fn consolidate_files(
347        &self,
348        directory: &ObjectPath,
349        parquet_files: Vec<String>,
350        start: Option<UnixNanos>,
351        end: Option<UnixNanos>,
352        ensure_contiguous_files: Option<bool>,
353        deduplicate: Option<bool>,
354    ) -> anyhow::Result<()> {
355        if parquet_files.len() <= 1 {
356            return Ok(());
357        }
358
359        let mut files_to_consolidate = Vec::new();
360        let start = start.map(|t| t.as_u64());
361        let end = end.map(|t| t.as_u64());
362
363        for file in parquet_files {
364            let Some(interval) = parse_filename_timestamps(&file) else {
365                continue;
366            };
367
368            if start.is_none_or(|s| interval.0 >= s) && end.is_none_or(|e| interval.1 <= e) {
369                files_to_consolidate.push((file, interval));
370            }
371        }
372
373        files_to_consolidate.sort_by(|(path_a, interval_a), (path_b, interval_b)| {
374            interval_a.cmp(interval_b).then_with(|| path_a.cmp(path_b))
375        });
376
377        let intervals: Vec<(u64, u64)> = files_to_consolidate
378            .iter()
379            .map(|&(_, interval)| interval)
380            .collect();
381
382        if ensure_contiguous_files.unwrap_or(true) {
383            anyhow::ensure!(
384                are_intervals_disjoint(&intervals),
385                "Intervals are not disjoint before consolidating a directory"
386            );
387            anyhow::ensure!(
388                are_intervals_contiguous(&intervals),
389                "Intervals are not contiguous before consolidating a directory"
390            );
391        }
392
393        let Some(last_end) = intervals.iter().map(|interval| interval.1).max() else {
394            return Ok(());
395        };
396
397        let file_name =
398            timestamps_to_filename(UnixNanos::from(intervals[0].0), UnixNanos::from(last_end));
399        let path = directory.clone().join(file_name);
400
401        let object_paths = files_to_consolidate
402            .iter()
403            .map(|(path, _)| self.to_object_path_parsed(path))
404            .collect::<anyhow::Result<Vec<_>>>()?;
405
406        self.execute_async(|| async {
407            consolidate_parquet_files_from_object_store(
408                self.object_store.clone(),
409                object_paths,
410                &path,
411                Some(self.compression),
412                Some(self.max_row_group_size),
413                deduplicate,
414            )
415            .await
416        })
417    }
418
419    /// Consolidates all data files in the catalog by splitting them into fixed time periods.
420    ///
421    /// This method identifies all leaf directories in the catalog that contain parquet files
422    /// and consolidates them by period. A leaf directory is one that contains files but no subdirectories.
423    /// This is a convenience method that effectively calls `consolidate_data_by_period` for all data types
424    /// and instrument IDs in the catalog.
425    ///
426    /// # Parameters
427    ///
428    /// - `period_nanos`: The period duration for consolidation in nanoseconds. Default is 1 day (86400000000000).
429    ///   Examples: 3600000000000 (1 hour), 604800000000000 (7 days), 1800000000000 (30 minutes)
430    /// - `start`: Optional start timestamp for the consolidation range. Only files with timestamps
431    ///   greater than or equal to this value will be consolidated. If None, all files
432    ///   from the beginning of time will be considered.
433    /// - `end`: Optional end timestamp for the consolidation range. Only files with timestamps
434    ///   less than or equal to this value will be consolidated. If None, all files
435    ///   up to the end of time will be considered.
436    /// - `ensure_contiguous_files`: If true, uses period boundaries clipped to `start` and `end` for file naming.
437    ///   If false, uses actual data timestamps for file naming.
438    ///
439    /// # Returns
440    ///
441    /// Returns `Ok(())` on success, or an error if consolidation fails for any directory.
442    ///
443    /// # Errors
444    ///
445    /// Returns an error if:
446    /// - `period_nanos` is zero and a directory holds data to consolidate.
447    /// - Directory listing fails.
448    /// - Data type extraction from path fails.
449    /// - Period-based consolidation operations fail.
450    ///
451    /// # Notes
452    ///
453    /// - This operation can be resource-intensive for large catalogs with many data types.
454    ///   and instruments.
455    /// - The consolidation process splits data into fixed time periods rather than combining.
456    ///   all files into a single file per directory.
457    /// - Uses the same period-based consolidation logic as `consolidate_data_by_period`.
458    /// - Original files are removed and replaced with period-based consolidated files.
459    /// - This method is useful for periodic maintenance of the catalog to standardize.
460    ///   file organization by time periods.
461    ///
462    /// # Examples
463    ///
464    /// ```rust,no_run
465    /// use nautilus_core::UnixNanos;
466    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
467    ///
468    /// let mut catalog = ParquetDataCatalog::new(
469    ///     std::path::Path::new("/tmp/nautilus_data"),
470    ///     None,
471    ///     None,
472    ///     None,
473    ///     None,
474    /// );
475    ///
476    /// // Consolidate all files in the catalog by 1-day periods
477    /// catalog.consolidate_catalog_by_period(
478    ///     Some(86400000000000), // 1 day in nanoseconds
479    ///     None,
480    ///     None,
481    ///     Some(true),
482    /// )?;
483    ///
484    /// // Consolidate only files within a specific time range by 1-hour periods
485    /// catalog.consolidate_catalog_by_period(
486    ///     Some(3600000000000), // 1 hour in nanoseconds
487    ///     Some(UnixNanos::from(1609459200000000000)),
488    ///     Some(UnixNanos::from(1609545600000000000)),
489    ///     Some(false),
490    /// )?;
491    /// # Ok::<(), anyhow::Error>(())
492    /// ```
493    pub fn consolidate_catalog_by_period(
494        &mut self,
495        period_nanos: Option<u64>,
496        start: Option<UnixNanos>,
497        end: Option<UnixNanos>,
498        ensure_contiguous_files: Option<bool>,
499    ) -> anyhow::Result<()> {
500        let leaf_directories = self.find_leaf_data_directories()?;
501
502        for directory in leaf_directories {
503            let (Some(data_cls_name), identifier) =
504                self.extract_data_cls_and_identifier_from_path(&directory)?
505            else {
506                continue;
507            };
508
509            if !self.dispatch_consolidate_data_by_period(
510                &data_cls_name,
511                identifier.as_deref(),
512                period_nanos,
513                start,
514                end,
515                ensure_contiguous_files,
516            )? {
517                // Skip unknown data types
518                log::warn!("Unknown data type for consolidation: {data_cls_name}");
519            }
520        }
521
522        Ok(())
523    }
524
525    /// Extracts data class and identifier from a directory path.
526    ///
527    /// This method parses a directory path to extract the data type and optional
528    /// instrument identifier. It's used to determine what type of data consolidation
529    /// to perform for each directory.
530    ///
531    /// # Parameters
532    ///
533    /// - `path`: The directory path to parse.
534    ///
535    /// # Returns
536    ///
537    /// Returns a tuple of (`data_class`, identifier) where both are optional strings.
538    pub fn extract_data_cls_and_identifier_from_path(
539        &self,
540        path: &str,
541    ) -> anyhow::Result<(Option<String>, Option<String>)> {
542        // Parse from the catalog root so a base path containing a `data` segment is skipped
543        let path = self.path_without_local_base(&normalize_path_separators(path));
544        let path_components = extract_path_components(&path);
545
546        // Find the "data" directory in the path
547        if let Some(data_index) = path_components.iter().position(|part| part == "data")
548            && data_index + 1 < path_components.len()
549        {
550            let second = &path_components[data_index + 1];
551
552            if second == "custom" {
553                let Some(type_name) = path_components.get(data_index + 2) else {
554                    return Ok((None, None));
555                };
556
557                let identifier = (path_components.len() > data_index + 3)
558                    .then(|| path_components[data_index + 3..].join("/"));
559                return Ok((Some(format!("custom/{type_name}")), identifier));
560            }
561
562            let data_cls = second.clone();
563            let identifier = path_components.get(data_index + 2).cloned();
564
565            return Ok((Some(data_cls), identifier));
566        }
567
568        // If we can't parse the path, return None for both
569        Ok((None, None))
570    }
571
572    /// Consolidates data files by splitting them into fixed time periods.
573    ///
574    /// This method queries data by period and writes consolidated files immediately,
575    /// using efficient period-based consolidation logic. When start/end boundaries intersect existing files,
576    /// the function automatically splits those files to preserve all data.
577    ///
578    /// # Parameters
579    ///
580    /// - `data_type`: The stored family to consolidate.
581    /// - `identifier`: Optional instrument ID to consolidate. If None, consolidates all instruments.
582    /// - `period_nanos`: The period duration for consolidation in nanoseconds. Default is 1 day (86400000000000).
583    ///   Examples: 3600000000000 (1 hour), 604800000000000 (7 days), 1800000000000 (30 minutes)
584    /// - `start`: Optional start timestamp for consolidation range. If None, uses earliest available data.
585    ///   If specified and intersects existing files, those files will be split to preserve
586    ///   data outside the consolidation range.
587    /// - `end`: Optional end timestamp for consolidation range. If None, uses latest available data.
588    ///   If specified and intersects existing files, those files will be split to preserve
589    ///   data outside the consolidation range.
590    /// - `ensure_contiguous_files`: If true, uses period boundaries clipped to `start` and `end` for file naming.
591    ///   If false, uses actual data timestamps for file naming.
592    ///
593    /// # Returns
594    ///
595    /// Returns `Ok(())` on success, or an error if consolidation fails.
596    ///
597    /// # Errors
598    ///
599    /// Returns an error if:
600    /// - `period_nanos` is zero and the directory holds data to consolidate.
601    /// - `start` exceeds `end` and the directory holds data to consolidate.
602    /// - `data_type` is a record family or an instrument selector, which have no
603    ///   period-typed rewrite; use [`Self::consolidate_data`] for those.
604    /// - The directory path cannot be constructed.
605    /// - File operations fail.
606    /// - Data querying or writing fails.
607    ///
608    /// # Notes
609    ///
610    /// - Uses two-phase approach: first determines all queries, then executes them.
611    /// - Groups intervals into contiguous groups to preserve holes between groups.
612    /// - Allows consolidation across multiple files within each contiguous group.
613    /// - Validates existing targets before keeping them and removing their source fragments.
614    /// - Removes source files only after processing their entire time range.
615    /// - When `ensure_contiguous_files=false`, file timestamps match actual data range.
616    /// - When `ensure_contiguous_files=true`, file timestamps use period boundaries clipped to `start` and `end`.
617    /// - Existing-target checks compare row values without counting identical copies.
618    /// - Uses modulo arithmetic for efficient period boundary calculation.
619    /// - Preserves holes in data by preventing queries from spanning across gaps.
620    /// - Automatically splits files at start/end boundaries to preserve all data.
621    /// - Split operations are executed before consolidation to ensure data preservation.
622    ///
623    /// # Examples
624    ///
625    /// ```rust,no_run
626    /// use nautilus_core::UnixNanos;
627    /// use nautilus_model::data::NautilusDataType;
628    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
629    ///
630    /// let mut catalog = ParquetDataCatalog::new(
631    ///     std::path::Path::new("/tmp/nautilus_data"),
632    ///     None,
633    ///     None,
634    ///     None,
635    ///     None,
636    /// );
637    ///
638    /// // Consolidate all quote files by 1-day periods
639    /// catalog.consolidate_data_by_period(
640    ///     &NautilusDataType::QuoteTick.into(),
641    ///     None,
642    ///     Some(86400000000000), // 1 day in nanoseconds
643    ///     None,
644    ///     None,
645    ///     Some(true),
646    /// )?;
647    ///
648    /// // Consolidate specific instrument by 1-hour periods
649    /// catalog.consolidate_data_by_period(
650    ///     &NautilusDataType::TradeTick.into(),
651    ///     Some("BTCUSD"),
652    ///     Some(3600000000000), // 1 hour in nanoseconds
653    ///     Some(UnixNanos::from(1609459200000000000)),
654    ///     Some(UnixNanos::from(1609545600000000000)),
655    ///     Some(false),
656    /// )?;
657    /// # Ok::<(), anyhow::Error>(())
658    /// ```
659    pub fn consolidate_data_by_period(
660        &mut self,
661        data_type: &CatalogDataType,
662        identifier: Option<&str>,
663        period_nanos: Option<u64>,
664        start: Option<UnixNanos>,
665        end: Option<UnixNanos>,
666        ensure_contiguous_files: Option<bool>,
667    ) -> anyhow::Result<()> {
668        anyhow::ensure!(
669            matches!(
670                data_type,
671                CatalogDataType::Data(data_type) if *data_type != NautilusDataType::Instrument
672            ),
673            "Period consolidation applies to data families only, not {data_type}; \
674             use consolidate_data",
675        );
676
677        for type_name in parquet_catalog_data_type_path_prefixes(data_type) {
678            self.consolidate_prefix_data_by_period(
679                type_name.as_ref(),
680                identifier,
681                period_nanos,
682                start,
683                end,
684                ensure_contiguous_files,
685            )?;
686        }
687
688        Ok(())
689    }
690
691    fn consolidate_prefix_data_by_period(
692        &mut self,
693        type_name: &str,
694        identifier: Option<&str>,
695        period_nanos: Option<u64>,
696        start: Option<UnixNanos>,
697        end: Option<UnixNanos>,
698        ensure_contiguous_files: Option<bool>,
699    ) -> anyhow::Result<()> {
700        if !self.dispatch_consolidate_data_by_period(
701            type_name,
702            identifier,
703            period_nanos,
704            start,
705            end,
706            ensure_contiguous_files,
707        )? {
708            anyhow::bail!("Unknown data type for consolidation: {type_name}");
709        }
710
711        Ok(())
712    }
713
714    /// Dispatches period-based consolidation for `type_name`, returning `Ok(false)` for
715    /// unknown data types so callers choose whether to warn or fail.
716    fn dispatch_consolidate_data_by_period(
717        &mut self,
718        type_name: &str,
719        identifier: Option<&str>,
720        period_nanos: Option<u64>,
721        start: Option<UnixNanos>,
722        end: Option<UnixNanos>,
723        ensure_contiguous_files: Option<bool>,
724    ) -> anyhow::Result<bool> {
725        match type_name {
726            "quotes" => {
727                self.consolidate_data_by_period_generic::<QuoteTick>(
728                    identifier,
729                    period_nanos,
730                    start,
731                    end,
732                    ensure_contiguous_files,
733                )?;
734            }
735            "trades" => {
736                self.consolidate_data_by_period_generic::<TradeTick>(
737                    identifier,
738                    period_nanos,
739                    start,
740                    end,
741                    ensure_contiguous_files,
742                )?;
743            }
744            "order_book_deltas" => {
745                self.consolidate_data_by_period_generic::<OrderBookDelta>(
746                    identifier,
747                    period_nanos,
748                    start,
749                    end,
750                    ensure_contiguous_files,
751                )?;
752            }
753            "order_book_depths" => {
754                self.consolidate_data_by_period_generic::<OrderBookDepth>(
755                    identifier,
756                    period_nanos,
757                    start,
758                    end,
759                    ensure_contiguous_files,
760                )?;
761            }
762            "bars" => {
763                self.consolidate_data_by_period_generic::<Bar>(
764                    identifier,
765                    period_nanos,
766                    start,
767                    end,
768                    ensure_contiguous_files,
769                )?;
770            }
771            "index_prices" => {
772                self.consolidate_data_by_period_generic::<IndexPriceUpdate>(
773                    identifier,
774                    period_nanos,
775                    start,
776                    end,
777                    ensure_contiguous_files,
778                )?;
779            }
780            "mark_prices" => {
781                self.consolidate_data_by_period_generic::<MarkPriceUpdate>(
782                    identifier,
783                    period_nanos,
784                    start,
785                    end,
786                    ensure_contiguous_files,
787                )?;
788            }
789            "instrument_closes" => {
790                self.consolidate_data_by_period_generic::<InstrumentClose>(
791                    identifier,
792                    period_nanos,
793                    start,
794                    end,
795                    ensure_contiguous_files,
796                )?;
797            }
798            _ => {
799                if let Some(custom_type_name) = type_name.strip_prefix("custom/") {
800                    self.consolidate_custom_data_by_period(
801                        custom_type_name,
802                        identifier,
803                        period_nanos,
804                        start,
805                        end,
806                        ensure_contiguous_files,
807                    )?;
808                } else {
809                    return Ok(false);
810                }
811            }
812        }
813
814        Ok(true)
815    }
816
817    /// Generic consolidate data files by splitting them into fixed time periods.
818    ///
819    /// This is a type-safe version of `consolidate_data_by_period` that uses generic types
820    /// to ensure compile-time correctness and enable reuse across different data types.
821    ///
822    /// # Type Parameters
823    ///
824    /// - `T`: The data type to consolidate, must implement required traits for serialization.
825    ///
826    /// # Parameters
827    ///
828    /// - `identifier`: Optional instrument ID to target a specific instrument's data.
829    /// - `period_nanos`: Optional period size in nanoseconds (default: 1 day).
830    /// - `start`: Optional start timestamp for consolidation range.
831    /// - `end`: Optional end timestamp for consolidation range.
832    /// - `ensure_contiguous_files`: Optional flag to control file naming strategy.
833    ///
834    /// # Returns
835    ///
836    /// Returns `Ok(())` on success, or an error if consolidation fails.
837    pub fn consolidate_data_by_period_generic<T>(
838        &mut self,
839        identifier: Option<&str>,
840        period_nanos: Option<u64>,
841        start: Option<UnixNanos>,
842        end: Option<UnixNanos>,
843        ensure_contiguous_files: Option<bool>,
844    ) -> anyhow::Result<()>
845    where
846        T: DecodeTypedFromRecordBatch
847            + HasCatalogDataType
848            + EncodeToRecordBatch
849            + HasTsInit
850            + TryFrom<Data>
851            + Clone,
852    {
853        let period_nanos = period_nanos.unwrap_or(86_400_000_000_000); // Default: 1 day
854        let ensure_contiguous_files = ensure_contiguous_files.unwrap_or(true);
855
856        // Use get_intervals for cleaner implementation
857        let data_type = T::catalog_data_type();
858        let path_prefix = parquet_data_path_prefix(&data_type);
859        let intervals =
860            self.get_intervals(&CatalogDataType::Data(data_type.clone()), identifier)?;
861
862        if intervals.is_empty() {
863            return Ok(()); // No files to consolidate
864        }
865
866        // Use auxiliary function to prepare all queries for execution
867        let queries_to_execute = self.prepare_consolidation_queries(
868            path_prefix.as_ref(),
869            identifier,
870            &intervals,
871            period_nanos,
872            start,
873            end,
874            ensure_contiguous_files,
875        )?;
876
877        if queries_to_execute.is_empty() {
878            return Ok(()); // No queries to execute
879        }
880
881        // Get directory for file operations
882        let directory = self.make_path(path_prefix.as_ref(), identifier)?;
883        let mut existing_files = self.list_parquet_files(&directory)?;
884        existing_files.sort();
885
886        // Simulate source removals and target creation before mutating storage, so validation
887        // observes the same files as execution while holding only one period's rows.
888        let type_name = to_snake_case(std::any::type_name::<T>());
889
890        for write_files in [false, true] {
891            let mut files = ConsolidationFiles::new(&existing_files, start, end);
892            let mut file_start_ns: Option<u64> = None;
893
894            for query_info in &queries_to_execute {
895                let period_data = self.query_period::<T>(identifier, query_info, &files.sources)?;
896
897                let (Some(first), Some(last)) = (period_data.first(), period_data.last()) else {
898                    if query_info.use_period_boundaries {
899                        file_start_ns.get_or_insert(query_info.query_start);
900                    }
901
902                    files.complete(self, query_info, write_files)?;
903                    continue;
904                };
905
906                let (start_ts, end_ts) =
907                    query_info.file_range(&mut file_start_ns, first.ts_init(), last.ts_init());
908
909                Self::check_ascending_timestamps(&period_data, &type_name)?;
910                Self::check_chunk_identity(&period_data, &type_name)?;
911                let batches = self.data_to_record_batches(&period_data)?;
912                let metadata = T::chunk_metadata(&period_data);
913                let target_identifier = catalog_identifier_from_metadata(&metadata);
914                let target_directory =
915                    self.make_path(path_prefix.as_ref(), target_identifier.as_deref())?;
916                let target_filename = make_object_store_path(
917                    &target_directory,
918                    [timestamps_to_filename(start_ts, end_ts)],
919                );
920                let target = self.to_object_path(&target_filename)?.to_string();
921
922                if files.prepare_target(self, &target, &batches)? && write_files {
923                    self.write_parquet_file_checked(
924                        &target_directory,
925                        start_ts,
926                        end_ts,
927                        &batches,
928                        true,
929                        "File",
930                        Some(&format!("{type_name} data")),
931                        None,
932                    )?;
933                }
934
935                files.complete(self, query_info, write_files)?;
936
937                // Reset so next period starts a new contiguous segment
938                file_start_ns = None;
939            }
940        }
941
942        Ok(())
943    }
944
945    /// Consolidates custom data files by splitting them into fixed time periods.
946    ///
947    /// This method provides consolidation for custom data types that don't have compile-time
948    /// type information. It uses dynamic querying and writing methods.
949    ///
950    /// # Parameters
951    ///
952    /// - `type_name`: The custom data type name (without "custom/" prefix).
953    /// - `identifier`: Optional instrument ID to consolidate.
954    /// - `period_nanos`: Optional period size in nanoseconds (default: 1 day).
955    /// - `start`: Optional start timestamp for consolidation range.
956    /// - `end`: Optional end timestamp for consolidation range.
957    /// - `ensure_contiguous_files`: Optional flag to control file naming strategy.
958    ///
959    /// # Returns
960    ///
961    /// Returns `Ok(())` on success, or an error if consolidation fails.
962    fn consolidate_custom_data_by_period(
963        &mut self,
964        type_name: &str,
965        identifier: Option<&str>,
966        period_nanos: Option<u64>,
967        start: Option<UnixNanos>,
968        end: Option<UnixNanos>,
969        ensure_contiguous_files: Option<bool>,
970    ) -> anyhow::Result<()> {
971        let period_nanos = period_nanos.unwrap_or(86_400_000_000_000); // Default: 1 day
972        let ensure_contiguous_files = ensure_contiguous_files.unwrap_or(true);
973
974        // Get intervals for the custom data type
975        let data_type = NautilusDataType::Custom {
976            type_name: type_name.to_string(),
977        };
978
979        let path_prefix = parquet_data_path_prefix(&data_type);
980        let intervals = self.get_intervals(&CatalogDataType::Data(data_type), identifier)?;
981
982        if intervals.is_empty() {
983            return Ok(()); // No files to consolidate
984        }
985
986        // Use auxiliary function to prepare all queries for execution
987        let queries_to_execute = self.prepare_consolidation_queries(
988            path_prefix.as_ref(),
989            identifier,
990            &intervals,
991            period_nanos,
992            start,
993            end,
994            ensure_contiguous_files,
995        )?;
996
997        if queries_to_execute.is_empty() {
998            return Ok(()); // No queries to execute
999        }
1000
1001        // Get directory for file operations
1002        let directory = self.make_path(path_prefix.as_ref(), identifier)?;
1003        let mut existing_files = self.list_parquet_files(&directory)?;
1004        existing_files.sort();
1005
1006        // Simulate source removals and target creation before mutating storage, so validation
1007        // observes the same files as execution while holding only one period's rows.
1008        for write_files in [false, true] {
1009            let mut files = ConsolidationFiles::new(&existing_files, start, end);
1010            let mut file_start_ns: Option<u64> = None;
1011
1012            for query_info in &queries_to_execute {
1013                let custom_items =
1014                    self.query_custom_period(type_name, identifier, query_info, &files.sources)?;
1015
1016                let (Some(first), Some(last)) = (custom_items.first(), custom_items.last()) else {
1017                    if query_info.use_period_boundaries {
1018                        file_start_ns.get_or_insert(query_info.query_start);
1019                    }
1020
1021                    files.complete(self, query_info, write_files)?;
1022                    continue;
1023                };
1024
1025                let (start_ts, end_ts) = query_info.file_range(
1026                    &mut file_start_ns,
1027                    first.data.ts_init(),
1028                    last.data.ts_init(),
1029                );
1030
1031                self.consolidate_custom_period(
1032                    &mut files,
1033                    &custom_items,
1034                    start_ts,
1035                    end_ts,
1036                    write_files,
1037                )?;
1038                files.complete(self, query_info, write_files)?;
1039
1040                // Reset so next period starts a new contiguous segment
1041                file_start_ns = None;
1042            }
1043        }
1044
1045        Ok(())
1046    }
1047
1048    fn consolidate_custom_period(
1049        &self,
1050        files: &mut ConsolidationFiles,
1051        rows: &[CustomData],
1052        start: UnixNanos,
1053        end: UnixNanos,
1054        write_files: bool,
1055    ) -> anyhow::Result<()> {
1056        let writes = self.prepare_custom_data_groups(rows, Some(start), Some(end), Some(true))?;
1057
1058        for write in &writes {
1059            let target = self
1060                .to_object_path(&write.path().to_string_lossy())?
1061                .to_string();
1062
1063            if files.prepare_target(self, &target, &write.batches)? && write_files {
1064                self.write_custom_data_prepared(write, Some(true))?;
1065            }
1066        }
1067
1068        Ok(())
1069    }
1070
1071    // Queries the data of one consolidation period from `files` only.
1072    fn query_period<T>(
1073        &mut self,
1074        identifier: Option<&str>,
1075        query_info: &ConsolidationQuery,
1076        files: &[String],
1077    ) -> anyhow::Result<Vec<T>>
1078    where
1079        T: DecodeTypedFromRecordBatch + HasCatalogDataType + HasTsInit,
1080    {
1081        let instrument_ids = identifier.map(|id| vec![id.to_string()]);
1082
1083        // Use optimize_file_loading=false to match Python behavior:
1084        // During consolidation, we want to read only the specific files being consolidated,
1085        // not the entire directory. This ensures precise file control during consolidation.
1086        self.query::<T>(
1087            instrument_ids,
1088            Some(UnixNanos::from(query_info.query_start)),
1089            Some(UnixNanos::from(query_info.query_end)),
1090            None,
1091            Some(files.to_vec()),
1092            false, // optimize_file_loading=false for precise file control during consolidation
1093        )
1094    }
1095
1096    // Queries the custom data of one consolidation period from `files` only.
1097    fn query_custom_period(
1098        &mut self,
1099        type_name: &str,
1100        identifier: Option<&str>,
1101        query_info: &ConsolidationQuery,
1102        files: &[String],
1103    ) -> anyhow::Result<Vec<CustomData>> {
1104        let instrument_ids = identifier.map(|id| vec![id.to_string()]);
1105        let period_data = self.query_custom_data_dynamic(
1106            type_name,
1107            instrument_ids.as_deref(),
1108            Some(UnixNanos::from(query_info.query_start)),
1109            Some(UnixNanos::from(query_info.query_end)),
1110            None,
1111            Some(files.to_vec()),
1112            false, // optimize_file_loading=false for precise file control during consolidation
1113        )?;
1114
1115        Ok(period_data
1116            .into_iter()
1117            .filter_map(|data| match data {
1118                Data::Custom(c) => Some(c),
1119                _ => None,
1120            })
1121            .collect())
1122    }
1123
1124    /// Prepares all queries for consolidation by filtering, grouping, and handling splits.
1125    ///
1126    /// Includes every period and the segments outside the requested bounds. Existing targets
1127    /// remain in the plan so execution can verify their rows before removing source files.
1128    ///
1129    /// # Errors
1130    ///
1131    /// Returns an error if:
1132    /// - `period_nanos` is zero.
1133    /// - `start` exceeds `end`.
1134    /// - The intervals are not contiguous when `ensure_contiguous_files` is true.
1135    #[expect(
1136        clippy::too_many_arguments,
1137        reason = "Consolidation keeps its window and policy arguments explicit"
1138    )]
1139    pub fn prepare_consolidation_queries(
1140        &self,
1141        _type_name: &str,
1142        _identifier: Option<&str>,
1143        intervals: &[(u64, u64)],
1144        period_nanos: u64,
1145        start: Option<UnixNanos>,
1146        end: Option<UnixNanos>,
1147        ensure_contiguous_files: bool,
1148    ) -> anyhow::Result<Vec<ConsolidationQuery>> {
1149        anyhow::ensure!(period_nanos > 0, "period_nanos must be positive");
1150
1151        if let (Some(start), Some(end)) = (start, end) {
1152            anyhow::ensure!(start <= end, "consolidation start must not exceed end");
1153        }
1154
1155        // Filter intervals by time range if specified
1156        let used_start = start.map(|s| s.as_u64());
1157        let used_end = end.map(|e| e.as_u64());
1158
1159        let mut filtered_intervals = Vec::new();
1160
1161        for &(interval_start, interval_end) in intervals {
1162            // Check if interval overlaps with the specified range
1163            if interval_intersects_range((interval_start, interval_end), start, end) {
1164                filtered_intervals.push((interval_start, interval_end));
1165            }
1166        }
1167
1168        if filtered_intervals.is_empty() {
1169            return Ok(Vec::new()); // No intervals in the specified range
1170        }
1171
1172        // Check contiguity of filtered intervals if required
1173        if ensure_contiguous_files && !are_intervals_contiguous(&filtered_intervals) {
1174            anyhow::bail!(
1175                "Intervals are not contiguous. When ensure_contiguous_files=true, \
1176                 all files in the consolidation range must have contiguous timestamps."
1177            );
1178        }
1179
1180        // Group intervals by the target period: split only when the gap between files
1181        // exceeds one period, since sub-period gaps land in the same consolidated file.
1182        let contiguous_groups = self.group_contiguous_intervals(&filtered_intervals, period_nanos);
1183
1184        let mut queries_to_execute = Vec::new();
1185
1186        // Handle interval splitting by creating split operations for data preservation
1187        if !filtered_intervals.is_empty() {
1188            if let Some(start_ts) = used_start {
1189                let first_interval = filtered_intervals[0];
1190                if first_interval.0 < start_ts && start_ts <= first_interval.1 {
1191                    // Split before start: preserve data from interval_start to start-1
1192                    queries_to_execute.push(ConsolidationQuery {
1193                        query_start: first_interval.0,
1194                        query_end: start_ts - 1,
1195                        use_period_boundaries: false,
1196                    });
1197                }
1198            }
1199
1200            if let Some(end_ts) = used_end {
1201                let last_end = filtered_intervals
1202                    .iter()
1203                    .map(|interval| interval.1)
1204                    .max()
1205                    .unwrap_or(end_ts);
1206                if end_ts < last_end {
1207                    // Split after end: preserve data from end+1 to interval_end
1208                    queries_to_execute.push(ConsolidationQuery {
1209                        query_start: end_ts + 1,
1210                        query_end: last_end,
1211                        use_period_boundaries: false,
1212                    });
1213                }
1214            }
1215        }
1216
1217        // Generate period-based consolidation queries for each contiguous group
1218        for group in contiguous_groups {
1219            let group_start = group[0].0;
1220            let group_end = group
1221                .iter()
1222                .fold(group[0].1, |end, interval| end.max(interval.1));
1223
1224            // Apply start/end filtering to the group
1225            let effective_start = used_start.map_or(group_start, |s| s.max(group_start));
1226            let effective_end = used_end.map_or(group_end, |e| e.min(group_end));
1227
1228            if effective_start > effective_end {
1229                continue; // Skip if no overlap
1230            }
1231
1232            // Generate period-based queries within this contiguous group
1233            let mut current_start_ns = (effective_start / period_nanos) * period_nanos;
1234
1235            loop {
1236                let current_end_ns = current_start_ns
1237                    .saturating_add(period_nanos - 1)
1238                    .min(effective_end);
1239
1240                queries_to_execute.push(ConsolidationQuery {
1241                    query_start: current_start_ns.max(used_start.unwrap_or(current_start_ns)),
1242                    query_end: current_end_ns,
1243                    use_period_boundaries: ensure_contiguous_files,
1244                });
1245
1246                match current_start_ns.checked_add(period_nanos) {
1247                    Some(next_start_ns) if next_start_ns <= effective_end => {
1248                        current_start_ns = next_start_ns;
1249                    }
1250                    _ => break,
1251                }
1252            }
1253        }
1254
1255        queries_to_execute.sort_by_key(|q| q.query_start);
1256
1257        Ok(queries_to_execute)
1258    }
1259
1260    /// Groups intervals for period-based consolidation.
1261    ///
1262    /// Groups adjacent intervals into the same bucket unless the gap between them exceeds
1263    /// `period_nanos`. Sub-period gaps land in the same consolidated file anyway, so they
1264    /// do not warrant a split. Gaps larger than one period represent genuine data holes.
1265    ///
1266    /// # Parameters
1267    ///
1268    /// - `intervals`: A slice of timestamp intervals as (start, end) tuples, sorted by start.
1269    /// - `period_nanos`: The target consolidation period; gaps larger than this split groups.
1270    ///
1271    /// # Returns
1272    ///
1273    /// Returns a vector of groups. Returns an empty vector if the input is empty.
1274    ///
1275    /// # Examples
1276    ///
1277    /// ```text
1278    /// Legacy chunked files with period=86_400_000_000_000 (1 day):
1279    ///   [(1,5), (6,10), (11,15)] -> [[(1,5), (6,10), (11,15)]]
1280    ///
1281    /// Small period=1 with mixed gaps:
1282    ///   [(1,5), (8,10), (12,15)] -> [[(1,5)], [(8,10)], [(12,15)]]
1283    /// ```
1284    #[must_use]
1285    pub fn group_contiguous_intervals(
1286        &self,
1287        intervals: &[(u64, u64)],
1288        period_nanos: u64,
1289    ) -> Vec<Vec<(u64, u64)>> {
1290        if intervals.is_empty() {
1291            return Vec::new();
1292        }
1293
1294        // Split groups only when the gap between files exceeds one period,
1295        // since sub-period gaps land in the same consolidated file anyway.
1296        // This works for both legacy chunked files (gap ~1ns) and fragment-per-flush
1297        // catalogs (gap ~bar interval) without inferring spacing from the data.
1298        let mut contiguous_groups = Vec::new();
1299        let mut current_group = vec![intervals[0]];
1300        let mut current_end = intervals[0].1;
1301        for &(curr_start, curr_end) in intervals.iter().skip(1) {
1302            if curr_start.saturating_sub(current_end) > period_nanos {
1303                contiguous_groups.push(current_group);
1304                current_group = vec![(curr_start, curr_end)];
1305                current_end = curr_end;
1306            } else {
1307                current_group.push((curr_start, curr_end));
1308                current_end = current_end.max(curr_end);
1309            }
1310        }
1311
1312        contiguous_groups.push(current_group);
1313
1314        contiguous_groups
1315    }
1316}
1317
1318impl ConsolidationQuery {
1319    fn file_range(
1320        &self,
1321        start_ns: &mut Option<u64>,
1322        first: UnixNanos,
1323        last: UnixNanos,
1324    ) -> (UnixNanos, UnixNanos) {
1325        if self.use_period_boundaries {
1326            let start = *start_ns.get_or_insert(self.query_start);
1327            (UnixNanos::from(start), UnixNanos::from(self.query_end))
1328        } else {
1329            (first, last)
1330        }
1331    }
1332}
1333
1334struct ConsolidationFiles {
1335    sources: Vec<String>,
1336    paths: AHashSet<String>,
1337    completed: Vec<ClosedInterval>,
1338}
1339
1340impl ConsolidationFiles {
1341    fn new(files: &[String], start: Option<UnixNanos>, end: Option<UnixNanos>) -> Self {
1342        Self {
1343            sources: files
1344                .iter()
1345                .filter(|file| {
1346                    parse_filename_timestamps(file)
1347                        .is_some_and(|interval| interval_intersects_range(interval, start, end))
1348                })
1349                .cloned()
1350                .collect(),
1351            paths: files.iter().cloned().collect(),
1352            completed: Vec::new(),
1353        }
1354    }
1355
1356    fn prepare_target(
1357        &mut self,
1358        catalog: &ParquetDataCatalog,
1359        target: &str,
1360        batches: &[RecordBatch],
1361    ) -> anyhow::Result<bool> {
1362        if !self.paths.contains(target) {
1363            self.paths.insert(target.to_string());
1364            return Ok(true);
1365        }
1366
1367        anyhow::ensure!(
1368            catalog.target_holds_rows(target, batches)?,
1369            "Cannot consolidate into existing target {target}: it does not hold every source row",
1370        );
1371        self.sources.retain(|file| file != target);
1372        Ok(false)
1373    }
1374
1375    fn complete(
1376        &mut self,
1377        catalog: &ParquetDataCatalog,
1378        query: &ConsolidationQuery,
1379        write_files: bool,
1380    ) -> anyhow::Result<()> {
1381        self.completed.push(ClosedInterval {
1382            start: query.query_start,
1383            end: query.query_end,
1384        });
1385
1386        self.completed = merge_closed_intervals(&self.completed);
1387
1388        for file in self.sources.clone() {
1389            if let Some((start, end)) = parse_filename_timestamps(&file)
1390                && self
1391                    .completed
1392                    .iter()
1393                    .any(|covered| covered.start <= start && end <= covered.end)
1394            {
1395                self.sources.retain(|source| source != &file);
1396                self.paths.remove(&file);
1397                if write_files {
1398                    catalog.delete_file(&file)?;
1399                }
1400            }
1401        }
1402
1403        Ok(())
1404    }
1405}
1406
1407fn interval_intersects_range(
1408    (interval_start, interval_end): (u64, u64),
1409    start: Option<UnixNanos>,
1410    end: Option<UnixNanos>,
1411) -> bool {
1412    start.is_none_or(|start| start.as_u64() <= interval_end)
1413        && end.is_none_or(|end| interval_start <= end.as_u64())
1414}
1415
1416fn can_rewrite_consolidation_by_period(type_name: &str) -> bool {
1417    matches!(
1418        type_name,
1419        "quotes"
1420            | "trades"
1421            | "order_book_deltas"
1422            | "order_book_depths"
1423            | "bars"
1424            | "index_prices"
1425            | "mark_prices"
1426    )
1427}
1428
1429fn is_schema_incompatibility(error: &anyhow::Error) -> bool {
1430    let message = error.to_string().to_lowercase();
1431    [
1432        "schema",
1433        "field",
1434        "column",
1435        "data type",
1436        "datatype",
1437        "not compatible",
1438        "mismatch",
1439    ]
1440    .iter()
1441    .any(|needle| message.contains(needle))
1442}
1443
1444#[cfg(test)]
1445mod tests {
1446    use rstest::rstest;
1447
1448    use super::*;
1449
1450    #[rstest]
1451    #[case("quotes", true)]
1452    #[case("bars", true)]
1453    #[case("instrument_closes", false)]
1454    #[case("custom/signal", false)]
1455    fn can_rewrite_consolidation_by_period_only_for_supported_types(
1456        #[case] type_name: &str,
1457        #[case] expected: bool,
1458    ) {
1459        assert_eq!(can_rewrite_consolidation_by_period(type_name), expected);
1460    }
1461
1462    #[rstest]
1463    #[case("schema mismatch while writing record batch", true)]
1464    #[case("object store request timed out", false)]
1465    fn schema_incompatibility_detection_matches_schema_errors(
1466        #[case] message: &str,
1467        #[case] expected: bool,
1468    ) {
1469        assert_eq!(
1470            is_schema_incompatibility(&anyhow::anyhow!(message.to_string())),
1471            expected
1472        );
1473    }
1474}