nautilus_persistence/backend/parquet/consolidation.rs
1// -------------------------------------------------------------------------------------------------
2// Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3// https://nautechsystems.io
4//
5// Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6// You may not use this file except in compliance with the License.
7// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Period-based and bulk consolidation of parquet files in a catalog directory.
17
18#![expect(
19 clippy::missing_errors_doc,
20 reason = "consolidation operations forward catalog/storage errors and operate on validated batches"
21)]
22
23use std::collections::BTreeMap;
24
25use ahash::AHashSet;
26use arrow::record_batch::RecordBatch;
27use nautilus_core::{ClosedInterval, UnixNanos, string::conversions::to_snake_case};
28use nautilus_model::data::{
29 Bar, CustomData, Data, HasTsInit, IndexPriceUpdate, MarkPriceUpdate, NautilusDataType,
30 OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick, close::InstrumentClose,
31};
32use nautilus_serialization::arrow::{
33 DecodeTypedFromRecordBatch, EncodeToRecordBatch, catalog_identifier_from_metadata,
34};
35use object_store::path::Path as ObjectPath;
36
37use crate::{
38 backend::parquet::{
39 catalog::ParquetDataCatalog,
40 intervals::{are_intervals_contiguous, are_intervals_disjoint},
41 io::consolidate_parquet_files_from_object_store,
42 paths::{
43 extract_path_components, make_object_store_path, normalize_path_separators,
44 parse_filename_timestamps, timestamps_to_filename,
45 },
46 },
47 catalog::types::{
48 CatalogDataType, HasCatalogDataType, parquet_catalog_data_type_path_prefixes,
49 parquet_data_path_prefix,
50 },
51 common::coverage::merge_closed_intervals,
52};
53
54/// Information about a consolidation query to be executed.
55#[derive(Debug, Clone)]
56pub struct ConsolidationQuery {
57 /// Start timestamp for the query range (inclusive, in nanoseconds).
58 pub query_start: u64,
59 /// End timestamp for the query range (inclusive, in nanoseconds).
60 pub query_end: u64,
61 /// Whether to use period boundaries for file naming (true) or actual data timestamps (false).
62 pub use_period_boundaries: bool,
63}
64
65impl ParquetDataCatalog {
66 /// Consolidates all data files in the catalog.
67 ///
68 /// This method identifies all leaf directories in the catalog that contain parquet files
69 /// and consolidates them. A leaf directory is one that contains files but no subdirectories.
70 /// This is a convenience method that effectively calls `consolidate_data` for all data types
71 /// and instrument IDs in the catalog.
72 ///
73 /// # Parameters
74 ///
75 /// - `start`: Optional start timestamp for the consolidation range. Only files with timestamps
76 /// greater than or equal to this value will be consolidated. If None, all files
77 /// from the beginning of time will be considered.
78 /// - `end`: Optional end timestamp for the consolidation range. Only files with timestamps
79 /// less than or equal to this value will be consolidated. If None, all files
80 /// up to the end of time will be considered.
81 /// - `ensure_contiguous_files`: Whether to validate that consolidated intervals are contiguous (default: true).
82 ///
83 /// # Returns
84 ///
85 /// Returns `Ok(())` on success, or an error if consolidation fails for any directory.
86 ///
87 /// # Errors
88 ///
89 /// Returns an error if:
90 /// - Directory listing fails.
91 /// - File consolidation operations fail.
92 /// - Interval validation fails (when `ensure_contiguous_files` is true).
93 ///
94 /// # Examples
95 ///
96 /// ```rust,no_run
97 /// use nautilus_core::UnixNanos;
98 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
99 ///
100 /// let mut catalog = ParquetDataCatalog::new(
101 /// std::path::Path::new("/tmp/nautilus_data"),
102 /// None,
103 /// None,
104 /// None,
105 /// None,
106 /// );
107 ///
108 /// // Consolidate all files in the catalog
109 /// catalog.consolidate_catalog(None, None, None, None)?;
110 ///
111 /// // Consolidate only files within a specific time range
112 /// catalog.consolidate_catalog(
113 /// Some(UnixNanos::from(1609459200000000000)),
114 /// Some(UnixNanos::from(1609545600000000000)),
115 /// Some(true),
116 /// None,
117 /// )?;
118 /// # Ok::<(), anyhow::Error>(())
119 /// ```
120 pub fn consolidate_catalog(
121 &self,
122 start: Option<UnixNanos>,
123 end: Option<UnixNanos>,
124 ensure_contiguous_files: Option<bool>,
125 deduplicate: Option<bool>,
126 ) -> anyhow::Result<()> {
127 let leaf_directories = self.find_leaf_data_directories()?;
128
129 for directory in leaf_directories {
130 self.consolidate_directory(
131 &directory,
132 start,
133 end,
134 ensure_contiguous_files,
135 deduplicate,
136 )?;
137 }
138
139 Ok(())
140 }
141
142 /// Consolidates data files for a specific data type and identifier.
143 ///
144 /// This method selects directories by data type and optional identifier, then merges files
145 /// within each directory. It preserves identifier partitions while reducing file count.
146 ///
147 /// # Parameters
148 ///
149 /// - `data_type`: The stored family to consolidate.
150 /// - `identifier`: Optional identifier to target a specific instrument's data. Can be an `instrument_id` (e.g., "EUR/USD.SIM") or a `bar_type` (e.g., "EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL").
151 /// - `start`: Optional start timestamp to limit consolidation to files within this range.
152 /// - `end`: Optional end timestamp to limit consolidation to files within this range.
153 /// - `ensure_contiguous_files`: Whether to validate that consolidated intervals are contiguous (default: true).
154 ///
155 /// # Returns
156 ///
157 /// Returns `Ok(())` on success, or an error if consolidation fails.
158 ///
159 /// # Errors
160 ///
161 /// Returns an error if:
162 /// - The directory path cannot be constructed.
163 /// - File consolidation operations fail.
164 /// - Interval validation fails (when `ensure_contiguous_files` is true).
165 ///
166 /// # Examples
167 ///
168 /// ```rust,no_run
169 /// use nautilus_core::UnixNanos;
170 /// use nautilus_model::data::NautilusDataType;
171 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
172 ///
173 /// let mut catalog = ParquetDataCatalog::new(
174 /// std::path::Path::new("/tmp/nautilus_data"),
175 /// None,
176 /// None,
177 /// None,
178 /// None,
179 /// );
180 ///
181 /// // Consolidate all quote files for a specific instrument
182 /// catalog.consolidate_data(
183 /// &NautilusDataType::QuoteTick.into(),
184 /// Some("BTCUSD"),
185 /// None,
186 /// None,
187 /// None,
188 /// None,
189 /// )?;
190 ///
191 /// // Consolidate trade files within a time range
192 /// catalog.consolidate_data(
193 /// &NautilusDataType::TradeTick.into(),
194 /// None,
195 /// Some(UnixNanos::from(1609459200000000000)),
196 /// Some(UnixNanos::from(1609545600000000000)),
197 /// Some(true),
198 /// None,
199 /// )?;
200 /// # Ok::<(), anyhow::Error>(())
201 /// ```
202 pub fn consolidate_data(
203 &mut self,
204 data_type: &CatalogDataType,
205 identifier: Option<&str>,
206 start: Option<UnixNanos>,
207 end: Option<UnixNanos>,
208 ensure_contiguous_files: Option<bool>,
209 deduplicate: Option<bool>,
210 ) -> anyhow::Result<()> {
211 for type_name in parquet_catalog_data_type_path_prefixes(data_type) {
212 self.consolidate_prefix_data(
213 type_name.as_ref(),
214 identifier,
215 start,
216 end,
217 ensure_contiguous_files,
218 deduplicate,
219 )?;
220 }
221
222 Ok(())
223 }
224
225 fn consolidate_prefix_data(
226 &mut self,
227 type_name: &str,
228 identifier: Option<&str>,
229 start: Option<UnixNanos>,
230 end: Option<UnixNanos>,
231 ensure_contiguous_files: Option<bool>,
232 deduplicate: Option<bool>,
233 ) -> anyhow::Result<()> {
234 let directory = self.make_path(type_name, identifier)?;
235 let raw_result = self.consolidate_directory(
236 &directory,
237 start,
238 end,
239 ensure_contiguous_files,
240 deduplicate,
241 );
242
243 match raw_result {
244 Ok(()) => Ok(()),
245 Err(raw_error)
246 if can_rewrite_consolidation_by_period(type_name)
247 && is_schema_incompatibility(&raw_error) =>
248 {
249 if deduplicate.unwrap_or(false) {
250 anyhow::bail!(
251 "Raw parquet consolidation failed due to incompatible file schemas, \
252 but typed period consolidation cannot preserve deduplicate=true. \
253 Raw consolidation error: {raw_error}"
254 );
255 }
256
257 log::warn!(
258 "Raw parquet consolidation failed due to incompatible file schemas for \
259 {type_name}; retrying with typed period consolidation. Raw error: {raw_error}"
260 );
261
262 self.consolidate_prefix_data_by_period(
263 type_name,
264 identifier,
265 None,
266 start,
267 end,
268 ensure_contiguous_files,
269 )
270 .map_err(|typed_error| {
271 anyhow::anyhow!(
272 "Raw parquet consolidation failed due to incompatible file schemas, \
273 and typed period consolidation also failed. Raw error: {raw_error}; \
274 typed period error: {typed_error}"
275 )
276 })
277 }
278 Err(e) => Err(e),
279 }
280 }
281
282 /// Consolidates Parquet files within a specific directory by merging them into a single file.
283 ///
284 /// This internal method performs the actual consolidation work for a single directory.
285 /// It identifies files within the specified time range, validates their intervals,
286 /// and combines them into a single Parquet file with optimized storage.
287 ///
288 /// # Parameters
289 ///
290 /// - `directory`: The directory path containing Parquet files to consolidate.
291 /// - `start`: Optional start timestamp to limit consolidation to files within this range.
292 /// - `end`: Optional end timestamp to limit consolidation to files within this range.
293 /// - `ensure_contiguous_files`: Whether to validate that consolidated intervals are contiguous.
294 ///
295 /// # Returns
296 ///
297 /// Returns `Ok(())` on success, or an error if consolidation fails.
298 ///
299 /// # Behavior
300 ///
301 /// - Skips directories containing 1 or fewer files.
302 /// - Filters files by timestamp range if start/end are specified.
303 /// - Sorts files by interval bounds and path before consolidation.
304 /// - Creates a new file spanning the entire time range of input files.
305 /// - Validates interval disjointness and contiguity before merging (if enabled).
306 ///
307 /// # Errors
308 ///
309 /// Returns an error if:
310 /// - Directory listing fails.
311 /// - File combination operations fail.
312 /// - Interval validation fails (when `ensure_contiguous_files` is true).
313 /// - Object store operations fail.
314 fn consolidate_directory(
315 &self,
316 directory: &str,
317 start: Option<UnixNanos>,
318 end: Option<UnixNanos>,
319 ensure_contiguous_files: Option<bool>,
320 deduplicate: Option<bool>,
321 ) -> anyhow::Result<()> {
322 self.recover_consolidation(&self.to_object_path(directory)?)?;
323 let parquet_files = self.list_parquet_files(directory)?;
324 let mut directories = BTreeMap::<ObjectPath, Vec<String>>::new();
325
326 for file in parquet_files {
327 let path = self.to_object_path_parsed(&file)?;
328 let directory = path.parent().unwrap_or_default();
329 directories.entry(directory).or_default().push(file);
330 }
331
332 for (directory, files) in directories {
333 self.consolidate_files(
334 &directory,
335 files,
336 start,
337 end,
338 ensure_contiguous_files,
339 deduplicate,
340 )?;
341 }
342
343 Ok(())
344 }
345
346 fn consolidate_files(
347 &self,
348 directory: &ObjectPath,
349 parquet_files: Vec<String>,
350 start: Option<UnixNanos>,
351 end: Option<UnixNanos>,
352 ensure_contiguous_files: Option<bool>,
353 deduplicate: Option<bool>,
354 ) -> anyhow::Result<()> {
355 if parquet_files.len() <= 1 {
356 return Ok(());
357 }
358
359 let mut files_to_consolidate = Vec::new();
360 let start = start.map(|t| t.as_u64());
361 let end = end.map(|t| t.as_u64());
362
363 for file in parquet_files {
364 let Some(interval) = parse_filename_timestamps(&file) else {
365 continue;
366 };
367
368 if start.is_none_or(|s| interval.0 >= s) && end.is_none_or(|e| interval.1 <= e) {
369 files_to_consolidate.push((file, interval));
370 }
371 }
372
373 files_to_consolidate.sort_by(|(path_a, interval_a), (path_b, interval_b)| {
374 interval_a.cmp(interval_b).then_with(|| path_a.cmp(path_b))
375 });
376
377 let intervals: Vec<(u64, u64)> = files_to_consolidate
378 .iter()
379 .map(|&(_, interval)| interval)
380 .collect();
381
382 if ensure_contiguous_files.unwrap_or(true) {
383 anyhow::ensure!(
384 are_intervals_disjoint(&intervals),
385 "Intervals are not disjoint before consolidating a directory"
386 );
387 anyhow::ensure!(
388 are_intervals_contiguous(&intervals),
389 "Intervals are not contiguous before consolidating a directory"
390 );
391 }
392
393 let Some(last_end) = intervals.iter().map(|interval| interval.1).max() else {
394 return Ok(());
395 };
396
397 let file_name =
398 timestamps_to_filename(UnixNanos::from(intervals[0].0), UnixNanos::from(last_end));
399 let path = directory.clone().join(file_name);
400
401 let object_paths = files_to_consolidate
402 .iter()
403 .map(|(path, _)| self.to_object_path_parsed(path))
404 .collect::<anyhow::Result<Vec<_>>>()?;
405
406 self.execute_async(|| async {
407 consolidate_parquet_files_from_object_store(
408 self.object_store.clone(),
409 object_paths,
410 &path,
411 Some(self.compression),
412 Some(self.max_row_group_size),
413 deduplicate,
414 )
415 .await
416 })
417 }
418
419 /// Consolidates all data files in the catalog by splitting them into fixed time periods.
420 ///
421 /// This method identifies all leaf directories in the catalog that contain parquet files
422 /// and consolidates them by period. A leaf directory is one that contains files but no subdirectories.
423 /// This is a convenience method that effectively calls `consolidate_data_by_period` for all data types
424 /// and instrument IDs in the catalog.
425 ///
426 /// # Parameters
427 ///
428 /// - `period_nanos`: The period duration for consolidation in nanoseconds. Default is 1 day (86400000000000).
429 /// Examples: 3600000000000 (1 hour), 604800000000000 (7 days), 1800000000000 (30 minutes)
430 /// - `start`: Optional start timestamp for the consolidation range. Only files with timestamps
431 /// greater than or equal to this value will be consolidated. If None, all files
432 /// from the beginning of time will be considered.
433 /// - `end`: Optional end timestamp for the consolidation range. Only files with timestamps
434 /// less than or equal to this value will be consolidated. If None, all files
435 /// up to the end of time will be considered.
436 /// - `ensure_contiguous_files`: If true, uses period boundaries clipped to `start` and `end` for file naming.
437 /// If false, uses actual data timestamps for file naming.
438 ///
439 /// # Returns
440 ///
441 /// Returns `Ok(())` on success, or an error if consolidation fails for any directory.
442 ///
443 /// # Errors
444 ///
445 /// Returns an error if:
446 /// - `period_nanos` is zero and a directory holds data to consolidate.
447 /// - Directory listing fails.
448 /// - Data type extraction from path fails.
449 /// - Period-based consolidation operations fail.
450 ///
451 /// # Notes
452 ///
453 /// - This operation can be resource-intensive for large catalogs with many data types.
454 /// and instruments.
455 /// - The consolidation process splits data into fixed time periods rather than combining.
456 /// all files into a single file per directory.
457 /// - Uses the same period-based consolidation logic as `consolidate_data_by_period`.
458 /// - Original files are removed and replaced with period-based consolidated files.
459 /// - This method is useful for periodic maintenance of the catalog to standardize.
460 /// file organization by time periods.
461 ///
462 /// # Examples
463 ///
464 /// ```rust,no_run
465 /// use nautilus_core::UnixNanos;
466 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
467 ///
468 /// let mut catalog = ParquetDataCatalog::new(
469 /// std::path::Path::new("/tmp/nautilus_data"),
470 /// None,
471 /// None,
472 /// None,
473 /// None,
474 /// );
475 ///
476 /// // Consolidate all files in the catalog by 1-day periods
477 /// catalog.consolidate_catalog_by_period(
478 /// Some(86400000000000), // 1 day in nanoseconds
479 /// None,
480 /// None,
481 /// Some(true),
482 /// )?;
483 ///
484 /// // Consolidate only files within a specific time range by 1-hour periods
485 /// catalog.consolidate_catalog_by_period(
486 /// Some(3600000000000), // 1 hour in nanoseconds
487 /// Some(UnixNanos::from(1609459200000000000)),
488 /// Some(UnixNanos::from(1609545600000000000)),
489 /// Some(false),
490 /// )?;
491 /// # Ok::<(), anyhow::Error>(())
492 /// ```
493 pub fn consolidate_catalog_by_period(
494 &mut self,
495 period_nanos: Option<u64>,
496 start: Option<UnixNanos>,
497 end: Option<UnixNanos>,
498 ensure_contiguous_files: Option<bool>,
499 ) -> anyhow::Result<()> {
500 let leaf_directories = self.find_leaf_data_directories()?;
501
502 for directory in leaf_directories {
503 let (Some(data_cls_name), identifier) =
504 self.extract_data_cls_and_identifier_from_path(&directory)?
505 else {
506 continue;
507 };
508
509 if !self.dispatch_consolidate_data_by_period(
510 &data_cls_name,
511 identifier.as_deref(),
512 period_nanos,
513 start,
514 end,
515 ensure_contiguous_files,
516 )? {
517 // Skip unknown data types
518 log::warn!("Unknown data type for consolidation: {data_cls_name}");
519 }
520 }
521
522 Ok(())
523 }
524
525 /// Extracts data class and identifier from a directory path.
526 ///
527 /// This method parses a directory path to extract the data type and optional
528 /// instrument identifier. It's used to determine what type of data consolidation
529 /// to perform for each directory.
530 ///
531 /// # Parameters
532 ///
533 /// - `path`: The directory path to parse.
534 ///
535 /// # Returns
536 ///
537 /// Returns a tuple of (`data_class`, identifier) where both are optional strings.
538 pub fn extract_data_cls_and_identifier_from_path(
539 &self,
540 path: &str,
541 ) -> anyhow::Result<(Option<String>, Option<String>)> {
542 // Parse from the catalog root so a base path containing a `data` segment is skipped
543 let path = self.path_without_local_base(&normalize_path_separators(path));
544 let path_components = extract_path_components(&path);
545
546 // Find the "data" directory in the path
547 if let Some(data_index) = path_components.iter().position(|part| part == "data")
548 && data_index + 1 < path_components.len()
549 {
550 let second = &path_components[data_index + 1];
551
552 if second == "custom" {
553 let Some(type_name) = path_components.get(data_index + 2) else {
554 return Ok((None, None));
555 };
556
557 let identifier = (path_components.len() > data_index + 3)
558 .then(|| path_components[data_index + 3..].join("/"));
559 return Ok((Some(format!("custom/{type_name}")), identifier));
560 }
561
562 let data_cls = second.clone();
563 let identifier = path_components.get(data_index + 2).cloned();
564
565 return Ok((Some(data_cls), identifier));
566 }
567
568 // If we can't parse the path, return None for both
569 Ok((None, None))
570 }
571
572 /// Consolidates data files by splitting them into fixed time periods.
573 ///
574 /// This method queries data by period and writes consolidated files immediately,
575 /// using efficient period-based consolidation logic. When start/end boundaries intersect existing files,
576 /// the function automatically splits those files to preserve all data.
577 ///
578 /// # Parameters
579 ///
580 /// - `data_type`: The stored family to consolidate.
581 /// - `identifier`: Optional instrument ID to consolidate. If None, consolidates all instruments.
582 /// - `period_nanos`: The period duration for consolidation in nanoseconds. Default is 1 day (86400000000000).
583 /// Examples: 3600000000000 (1 hour), 604800000000000 (7 days), 1800000000000 (30 minutes)
584 /// - `start`: Optional start timestamp for consolidation range. If None, uses earliest available data.
585 /// If specified and intersects existing files, those files will be split to preserve
586 /// data outside the consolidation range.
587 /// - `end`: Optional end timestamp for consolidation range. If None, uses latest available data.
588 /// If specified and intersects existing files, those files will be split to preserve
589 /// data outside the consolidation range.
590 /// - `ensure_contiguous_files`: If true, uses period boundaries clipped to `start` and `end` for file naming.
591 /// If false, uses actual data timestamps for file naming.
592 ///
593 /// # Returns
594 ///
595 /// Returns `Ok(())` on success, or an error if consolidation fails.
596 ///
597 /// # Errors
598 ///
599 /// Returns an error if:
600 /// - `period_nanos` is zero and the directory holds data to consolidate.
601 /// - `start` exceeds `end` and the directory holds data to consolidate.
602 /// - `data_type` is a record family or an instrument selector, which have no
603 /// period-typed rewrite; use [`Self::consolidate_data`] for those.
604 /// - The directory path cannot be constructed.
605 /// - File operations fail.
606 /// - Data querying or writing fails.
607 ///
608 /// # Notes
609 ///
610 /// - Uses two-phase approach: first determines all queries, then executes them.
611 /// - Groups intervals into contiguous groups to preserve holes between groups.
612 /// - Allows consolidation across multiple files within each contiguous group.
613 /// - Validates existing targets before keeping them and removing their source fragments.
614 /// - Removes source files only after processing their entire time range.
615 /// - When `ensure_contiguous_files=false`, file timestamps match actual data range.
616 /// - When `ensure_contiguous_files=true`, file timestamps use period boundaries clipped to `start` and `end`.
617 /// - Existing-target checks compare row values without counting identical copies.
618 /// - Uses modulo arithmetic for efficient period boundary calculation.
619 /// - Preserves holes in data by preventing queries from spanning across gaps.
620 /// - Automatically splits files at start/end boundaries to preserve all data.
621 /// - Split operations are executed before consolidation to ensure data preservation.
622 ///
623 /// # Examples
624 ///
625 /// ```rust,no_run
626 /// use nautilus_core::UnixNanos;
627 /// use nautilus_model::data::NautilusDataType;
628 /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
629 ///
630 /// let mut catalog = ParquetDataCatalog::new(
631 /// std::path::Path::new("/tmp/nautilus_data"),
632 /// None,
633 /// None,
634 /// None,
635 /// None,
636 /// );
637 ///
638 /// // Consolidate all quote files by 1-day periods
639 /// catalog.consolidate_data_by_period(
640 /// &NautilusDataType::QuoteTick.into(),
641 /// None,
642 /// Some(86400000000000), // 1 day in nanoseconds
643 /// None,
644 /// None,
645 /// Some(true),
646 /// )?;
647 ///
648 /// // Consolidate specific instrument by 1-hour periods
649 /// catalog.consolidate_data_by_period(
650 /// &NautilusDataType::TradeTick.into(),
651 /// Some("BTCUSD"),
652 /// Some(3600000000000), // 1 hour in nanoseconds
653 /// Some(UnixNanos::from(1609459200000000000)),
654 /// Some(UnixNanos::from(1609545600000000000)),
655 /// Some(false),
656 /// )?;
657 /// # Ok::<(), anyhow::Error>(())
658 /// ```
659 pub fn consolidate_data_by_period(
660 &mut self,
661 data_type: &CatalogDataType,
662 identifier: Option<&str>,
663 period_nanos: Option<u64>,
664 start: Option<UnixNanos>,
665 end: Option<UnixNanos>,
666 ensure_contiguous_files: Option<bool>,
667 ) -> anyhow::Result<()> {
668 anyhow::ensure!(
669 matches!(
670 data_type,
671 CatalogDataType::Data(data_type) if *data_type != NautilusDataType::Instrument
672 ),
673 "Period consolidation applies to data families only, not {data_type}; \
674 use consolidate_data",
675 );
676
677 for type_name in parquet_catalog_data_type_path_prefixes(data_type) {
678 self.consolidate_prefix_data_by_period(
679 type_name.as_ref(),
680 identifier,
681 period_nanos,
682 start,
683 end,
684 ensure_contiguous_files,
685 )?;
686 }
687
688 Ok(())
689 }
690
691 fn consolidate_prefix_data_by_period(
692 &mut self,
693 type_name: &str,
694 identifier: Option<&str>,
695 period_nanos: Option<u64>,
696 start: Option<UnixNanos>,
697 end: Option<UnixNanos>,
698 ensure_contiguous_files: Option<bool>,
699 ) -> anyhow::Result<()> {
700 if !self.dispatch_consolidate_data_by_period(
701 type_name,
702 identifier,
703 period_nanos,
704 start,
705 end,
706 ensure_contiguous_files,
707 )? {
708 anyhow::bail!("Unknown data type for consolidation: {type_name}");
709 }
710
711 Ok(())
712 }
713
714 /// Dispatches period-based consolidation for `type_name`, returning `Ok(false)` for
715 /// unknown data types so callers choose whether to warn or fail.
716 fn dispatch_consolidate_data_by_period(
717 &mut self,
718 type_name: &str,
719 identifier: Option<&str>,
720 period_nanos: Option<u64>,
721 start: Option<UnixNanos>,
722 end: Option<UnixNanos>,
723 ensure_contiguous_files: Option<bool>,
724 ) -> anyhow::Result<bool> {
725 match type_name {
726 "quotes" => {
727 self.consolidate_data_by_period_generic::<QuoteTick>(
728 identifier,
729 period_nanos,
730 start,
731 end,
732 ensure_contiguous_files,
733 )?;
734 }
735 "trades" => {
736 self.consolidate_data_by_period_generic::<TradeTick>(
737 identifier,
738 period_nanos,
739 start,
740 end,
741 ensure_contiguous_files,
742 )?;
743 }
744 "order_book_deltas" => {
745 self.consolidate_data_by_period_generic::<OrderBookDelta>(
746 identifier,
747 period_nanos,
748 start,
749 end,
750 ensure_contiguous_files,
751 )?;
752 }
753 "order_book_depths" => {
754 self.consolidate_data_by_period_generic::<OrderBookDepth>(
755 identifier,
756 period_nanos,
757 start,
758 end,
759 ensure_contiguous_files,
760 )?;
761 }
762 "bars" => {
763 self.consolidate_data_by_period_generic::<Bar>(
764 identifier,
765 period_nanos,
766 start,
767 end,
768 ensure_contiguous_files,
769 )?;
770 }
771 "index_prices" => {
772 self.consolidate_data_by_period_generic::<IndexPriceUpdate>(
773 identifier,
774 period_nanos,
775 start,
776 end,
777 ensure_contiguous_files,
778 )?;
779 }
780 "mark_prices" => {
781 self.consolidate_data_by_period_generic::<MarkPriceUpdate>(
782 identifier,
783 period_nanos,
784 start,
785 end,
786 ensure_contiguous_files,
787 )?;
788 }
789 "instrument_closes" => {
790 self.consolidate_data_by_period_generic::<InstrumentClose>(
791 identifier,
792 period_nanos,
793 start,
794 end,
795 ensure_contiguous_files,
796 )?;
797 }
798 _ => {
799 if let Some(custom_type_name) = type_name.strip_prefix("custom/") {
800 self.consolidate_custom_data_by_period(
801 custom_type_name,
802 identifier,
803 period_nanos,
804 start,
805 end,
806 ensure_contiguous_files,
807 )?;
808 } else {
809 return Ok(false);
810 }
811 }
812 }
813
814 Ok(true)
815 }
816
817 /// Generic consolidate data files by splitting them into fixed time periods.
818 ///
819 /// This is a type-safe version of `consolidate_data_by_period` that uses generic types
820 /// to ensure compile-time correctness and enable reuse across different data types.
821 ///
822 /// # Type Parameters
823 ///
824 /// - `T`: The data type to consolidate, must implement required traits for serialization.
825 ///
826 /// # Parameters
827 ///
828 /// - `identifier`: Optional instrument ID to target a specific instrument's data.
829 /// - `period_nanos`: Optional period size in nanoseconds (default: 1 day).
830 /// - `start`: Optional start timestamp for consolidation range.
831 /// - `end`: Optional end timestamp for consolidation range.
832 /// - `ensure_contiguous_files`: Optional flag to control file naming strategy.
833 ///
834 /// # Returns
835 ///
836 /// Returns `Ok(())` on success, or an error if consolidation fails.
837 pub fn consolidate_data_by_period_generic<T>(
838 &mut self,
839 identifier: Option<&str>,
840 period_nanos: Option<u64>,
841 start: Option<UnixNanos>,
842 end: Option<UnixNanos>,
843 ensure_contiguous_files: Option<bool>,
844 ) -> anyhow::Result<()>
845 where
846 T: DecodeTypedFromRecordBatch
847 + HasCatalogDataType
848 + EncodeToRecordBatch
849 + HasTsInit
850 + TryFrom<Data>
851 + Clone,
852 {
853 let period_nanos = period_nanos.unwrap_or(86_400_000_000_000); // Default: 1 day
854 let ensure_contiguous_files = ensure_contiguous_files.unwrap_or(true);
855
856 // Use get_intervals for cleaner implementation
857 let data_type = T::catalog_data_type();
858 let path_prefix = parquet_data_path_prefix(&data_type);
859 let intervals =
860 self.get_intervals(&CatalogDataType::Data(data_type.clone()), identifier)?;
861
862 if intervals.is_empty() {
863 return Ok(()); // No files to consolidate
864 }
865
866 // Use auxiliary function to prepare all queries for execution
867 let queries_to_execute = self.prepare_consolidation_queries(
868 path_prefix.as_ref(),
869 identifier,
870 &intervals,
871 period_nanos,
872 start,
873 end,
874 ensure_contiguous_files,
875 )?;
876
877 if queries_to_execute.is_empty() {
878 return Ok(()); // No queries to execute
879 }
880
881 // Get directory for file operations
882 let directory = self.make_path(path_prefix.as_ref(), identifier)?;
883 let mut existing_files = self.list_parquet_files(&directory)?;
884 existing_files.sort();
885
886 // Simulate source removals and target creation before mutating storage, so validation
887 // observes the same files as execution while holding only one period's rows.
888 let type_name = to_snake_case(std::any::type_name::<T>());
889
890 for write_files in [false, true] {
891 let mut files = ConsolidationFiles::new(&existing_files, start, end);
892 let mut file_start_ns: Option<u64> = None;
893
894 for query_info in &queries_to_execute {
895 let period_data = self.query_period::<T>(identifier, query_info, &files.sources)?;
896
897 let (Some(first), Some(last)) = (period_data.first(), period_data.last()) else {
898 if query_info.use_period_boundaries {
899 file_start_ns.get_or_insert(query_info.query_start);
900 }
901
902 files.complete(self, query_info, write_files)?;
903 continue;
904 };
905
906 let (start_ts, end_ts) =
907 query_info.file_range(&mut file_start_ns, first.ts_init(), last.ts_init());
908
909 Self::check_ascending_timestamps(&period_data, &type_name)?;
910 Self::check_chunk_identity(&period_data, &type_name)?;
911 let batches = self.data_to_record_batches(&period_data)?;
912 let metadata = T::chunk_metadata(&period_data);
913 let target_identifier = catalog_identifier_from_metadata(&metadata);
914 let target_directory =
915 self.make_path(path_prefix.as_ref(), target_identifier.as_deref())?;
916 let target_filename = make_object_store_path(
917 &target_directory,
918 [timestamps_to_filename(start_ts, end_ts)],
919 );
920 let target = self.to_object_path(&target_filename)?.to_string();
921
922 if files.prepare_target(self, &target, &batches)? && write_files {
923 self.write_parquet_file_checked(
924 &target_directory,
925 start_ts,
926 end_ts,
927 &batches,
928 true,
929 "File",
930 Some(&format!("{type_name} data")),
931 None,
932 )?;
933 }
934
935 files.complete(self, query_info, write_files)?;
936
937 // Reset so next period starts a new contiguous segment
938 file_start_ns = None;
939 }
940 }
941
942 Ok(())
943 }
944
945 /// Consolidates custom data files by splitting them into fixed time periods.
946 ///
947 /// This method provides consolidation for custom data types that don't have compile-time
948 /// type information. It uses dynamic querying and writing methods.
949 ///
950 /// # Parameters
951 ///
952 /// - `type_name`: The custom data type name (without "custom/" prefix).
953 /// - `identifier`: Optional instrument ID to consolidate.
954 /// - `period_nanos`: Optional period size in nanoseconds (default: 1 day).
955 /// - `start`: Optional start timestamp for consolidation range.
956 /// - `end`: Optional end timestamp for consolidation range.
957 /// - `ensure_contiguous_files`: Optional flag to control file naming strategy.
958 ///
959 /// # Returns
960 ///
961 /// Returns `Ok(())` on success, or an error if consolidation fails.
962 fn consolidate_custom_data_by_period(
963 &mut self,
964 type_name: &str,
965 identifier: Option<&str>,
966 period_nanos: Option<u64>,
967 start: Option<UnixNanos>,
968 end: Option<UnixNanos>,
969 ensure_contiguous_files: Option<bool>,
970 ) -> anyhow::Result<()> {
971 let period_nanos = period_nanos.unwrap_or(86_400_000_000_000); // Default: 1 day
972 let ensure_contiguous_files = ensure_contiguous_files.unwrap_or(true);
973
974 // Get intervals for the custom data type
975 let data_type = NautilusDataType::Custom {
976 type_name: type_name.to_string(),
977 };
978
979 let path_prefix = parquet_data_path_prefix(&data_type);
980 let intervals = self.get_intervals(&CatalogDataType::Data(data_type), identifier)?;
981
982 if intervals.is_empty() {
983 return Ok(()); // No files to consolidate
984 }
985
986 // Use auxiliary function to prepare all queries for execution
987 let queries_to_execute = self.prepare_consolidation_queries(
988 path_prefix.as_ref(),
989 identifier,
990 &intervals,
991 period_nanos,
992 start,
993 end,
994 ensure_contiguous_files,
995 )?;
996
997 if queries_to_execute.is_empty() {
998 return Ok(()); // No queries to execute
999 }
1000
1001 // Get directory for file operations
1002 let directory = self.make_path(path_prefix.as_ref(), identifier)?;
1003 let mut existing_files = self.list_parquet_files(&directory)?;
1004 existing_files.sort();
1005
1006 // Simulate source removals and target creation before mutating storage, so validation
1007 // observes the same files as execution while holding only one period's rows.
1008 for write_files in [false, true] {
1009 let mut files = ConsolidationFiles::new(&existing_files, start, end);
1010 let mut file_start_ns: Option<u64> = None;
1011
1012 for query_info in &queries_to_execute {
1013 let custom_items =
1014 self.query_custom_period(type_name, identifier, query_info, &files.sources)?;
1015
1016 let (Some(first), Some(last)) = (custom_items.first(), custom_items.last()) else {
1017 if query_info.use_period_boundaries {
1018 file_start_ns.get_or_insert(query_info.query_start);
1019 }
1020
1021 files.complete(self, query_info, write_files)?;
1022 continue;
1023 };
1024
1025 let (start_ts, end_ts) = query_info.file_range(
1026 &mut file_start_ns,
1027 first.data.ts_init(),
1028 last.data.ts_init(),
1029 );
1030
1031 self.consolidate_custom_period(
1032 &mut files,
1033 &custom_items,
1034 start_ts,
1035 end_ts,
1036 write_files,
1037 )?;
1038 files.complete(self, query_info, write_files)?;
1039
1040 // Reset so next period starts a new contiguous segment
1041 file_start_ns = None;
1042 }
1043 }
1044
1045 Ok(())
1046 }
1047
1048 fn consolidate_custom_period(
1049 &self,
1050 files: &mut ConsolidationFiles,
1051 rows: &[CustomData],
1052 start: UnixNanos,
1053 end: UnixNanos,
1054 write_files: bool,
1055 ) -> anyhow::Result<()> {
1056 let writes = self.prepare_custom_data_groups(rows, Some(start), Some(end), Some(true))?;
1057
1058 for write in &writes {
1059 let target = self
1060 .to_object_path(&write.path().to_string_lossy())?
1061 .to_string();
1062
1063 if files.prepare_target(self, &target, &write.batches)? && write_files {
1064 self.write_custom_data_prepared(write, Some(true))?;
1065 }
1066 }
1067
1068 Ok(())
1069 }
1070
1071 // Queries the data of one consolidation period from `files` only.
1072 fn query_period<T>(
1073 &mut self,
1074 identifier: Option<&str>,
1075 query_info: &ConsolidationQuery,
1076 files: &[String],
1077 ) -> anyhow::Result<Vec<T>>
1078 where
1079 T: DecodeTypedFromRecordBatch + HasCatalogDataType + HasTsInit,
1080 {
1081 let instrument_ids = identifier.map(|id| vec![id.to_string()]);
1082
1083 // Use optimize_file_loading=false to match Python behavior:
1084 // During consolidation, we want to read only the specific files being consolidated,
1085 // not the entire directory. This ensures precise file control during consolidation.
1086 self.query::<T>(
1087 instrument_ids,
1088 Some(UnixNanos::from(query_info.query_start)),
1089 Some(UnixNanos::from(query_info.query_end)),
1090 None,
1091 Some(files.to_vec()),
1092 false, // optimize_file_loading=false for precise file control during consolidation
1093 )
1094 }
1095
1096 // Queries the custom data of one consolidation period from `files` only.
1097 fn query_custom_period(
1098 &mut self,
1099 type_name: &str,
1100 identifier: Option<&str>,
1101 query_info: &ConsolidationQuery,
1102 files: &[String],
1103 ) -> anyhow::Result<Vec<CustomData>> {
1104 let instrument_ids = identifier.map(|id| vec![id.to_string()]);
1105 let period_data = self.query_custom_data_dynamic(
1106 type_name,
1107 instrument_ids.as_deref(),
1108 Some(UnixNanos::from(query_info.query_start)),
1109 Some(UnixNanos::from(query_info.query_end)),
1110 None,
1111 Some(files.to_vec()),
1112 false, // optimize_file_loading=false for precise file control during consolidation
1113 )?;
1114
1115 Ok(period_data
1116 .into_iter()
1117 .filter_map(|data| match data {
1118 Data::Custom(c) => Some(c),
1119 _ => None,
1120 })
1121 .collect())
1122 }
1123
1124 /// Prepares all queries for consolidation by filtering, grouping, and handling splits.
1125 ///
1126 /// Includes every period and the segments outside the requested bounds. Existing targets
1127 /// remain in the plan so execution can verify their rows before removing source files.
1128 ///
1129 /// # Errors
1130 ///
1131 /// Returns an error if:
1132 /// - `period_nanos` is zero.
1133 /// - `start` exceeds `end`.
1134 /// - The intervals are not contiguous when `ensure_contiguous_files` is true.
1135 #[expect(
1136 clippy::too_many_arguments,
1137 reason = "Consolidation keeps its window and policy arguments explicit"
1138 )]
1139 pub fn prepare_consolidation_queries(
1140 &self,
1141 _type_name: &str,
1142 _identifier: Option<&str>,
1143 intervals: &[(u64, u64)],
1144 period_nanos: u64,
1145 start: Option<UnixNanos>,
1146 end: Option<UnixNanos>,
1147 ensure_contiguous_files: bool,
1148 ) -> anyhow::Result<Vec<ConsolidationQuery>> {
1149 anyhow::ensure!(period_nanos > 0, "period_nanos must be positive");
1150
1151 if let (Some(start), Some(end)) = (start, end) {
1152 anyhow::ensure!(start <= end, "consolidation start must not exceed end");
1153 }
1154
1155 // Filter intervals by time range if specified
1156 let used_start = start.map(|s| s.as_u64());
1157 let used_end = end.map(|e| e.as_u64());
1158
1159 let mut filtered_intervals = Vec::new();
1160
1161 for &(interval_start, interval_end) in intervals {
1162 // Check if interval overlaps with the specified range
1163 if interval_intersects_range((interval_start, interval_end), start, end) {
1164 filtered_intervals.push((interval_start, interval_end));
1165 }
1166 }
1167
1168 if filtered_intervals.is_empty() {
1169 return Ok(Vec::new()); // No intervals in the specified range
1170 }
1171
1172 // Check contiguity of filtered intervals if required
1173 if ensure_contiguous_files && !are_intervals_contiguous(&filtered_intervals) {
1174 anyhow::bail!(
1175 "Intervals are not contiguous. When ensure_contiguous_files=true, \
1176 all files in the consolidation range must have contiguous timestamps."
1177 );
1178 }
1179
1180 // Group intervals by the target period: split only when the gap between files
1181 // exceeds one period, since sub-period gaps land in the same consolidated file.
1182 let contiguous_groups = self.group_contiguous_intervals(&filtered_intervals, period_nanos);
1183
1184 let mut queries_to_execute = Vec::new();
1185
1186 // Handle interval splitting by creating split operations for data preservation
1187 if !filtered_intervals.is_empty() {
1188 if let Some(start_ts) = used_start {
1189 let first_interval = filtered_intervals[0];
1190 if first_interval.0 < start_ts && start_ts <= first_interval.1 {
1191 // Split before start: preserve data from interval_start to start-1
1192 queries_to_execute.push(ConsolidationQuery {
1193 query_start: first_interval.0,
1194 query_end: start_ts - 1,
1195 use_period_boundaries: false,
1196 });
1197 }
1198 }
1199
1200 if let Some(end_ts) = used_end {
1201 let last_end = filtered_intervals
1202 .iter()
1203 .map(|interval| interval.1)
1204 .max()
1205 .unwrap_or(end_ts);
1206 if end_ts < last_end {
1207 // Split after end: preserve data from end+1 to interval_end
1208 queries_to_execute.push(ConsolidationQuery {
1209 query_start: end_ts + 1,
1210 query_end: last_end,
1211 use_period_boundaries: false,
1212 });
1213 }
1214 }
1215 }
1216
1217 // Generate period-based consolidation queries for each contiguous group
1218 for group in contiguous_groups {
1219 let group_start = group[0].0;
1220 let group_end = group
1221 .iter()
1222 .fold(group[0].1, |end, interval| end.max(interval.1));
1223
1224 // Apply start/end filtering to the group
1225 let effective_start = used_start.map_or(group_start, |s| s.max(group_start));
1226 let effective_end = used_end.map_or(group_end, |e| e.min(group_end));
1227
1228 if effective_start > effective_end {
1229 continue; // Skip if no overlap
1230 }
1231
1232 // Generate period-based queries within this contiguous group
1233 let mut current_start_ns = (effective_start / period_nanos) * period_nanos;
1234
1235 loop {
1236 let current_end_ns = current_start_ns
1237 .saturating_add(period_nanos - 1)
1238 .min(effective_end);
1239
1240 queries_to_execute.push(ConsolidationQuery {
1241 query_start: current_start_ns.max(used_start.unwrap_or(current_start_ns)),
1242 query_end: current_end_ns,
1243 use_period_boundaries: ensure_contiguous_files,
1244 });
1245
1246 match current_start_ns.checked_add(period_nanos) {
1247 Some(next_start_ns) if next_start_ns <= effective_end => {
1248 current_start_ns = next_start_ns;
1249 }
1250 _ => break,
1251 }
1252 }
1253 }
1254
1255 queries_to_execute.sort_by_key(|q| q.query_start);
1256
1257 Ok(queries_to_execute)
1258 }
1259
1260 /// Groups intervals for period-based consolidation.
1261 ///
1262 /// Groups adjacent intervals into the same bucket unless the gap between them exceeds
1263 /// `period_nanos`. Sub-period gaps land in the same consolidated file anyway, so they
1264 /// do not warrant a split. Gaps larger than one period represent genuine data holes.
1265 ///
1266 /// # Parameters
1267 ///
1268 /// - `intervals`: A slice of timestamp intervals as (start, end) tuples, sorted by start.
1269 /// - `period_nanos`: The target consolidation period; gaps larger than this split groups.
1270 ///
1271 /// # Returns
1272 ///
1273 /// Returns a vector of groups. Returns an empty vector if the input is empty.
1274 ///
1275 /// # Examples
1276 ///
1277 /// ```text
1278 /// Legacy chunked files with period=86_400_000_000_000 (1 day):
1279 /// [(1,5), (6,10), (11,15)] -> [[(1,5), (6,10), (11,15)]]
1280 ///
1281 /// Small period=1 with mixed gaps:
1282 /// [(1,5), (8,10), (12,15)] -> [[(1,5)], [(8,10)], [(12,15)]]
1283 /// ```
1284 #[must_use]
1285 pub fn group_contiguous_intervals(
1286 &self,
1287 intervals: &[(u64, u64)],
1288 period_nanos: u64,
1289 ) -> Vec<Vec<(u64, u64)>> {
1290 if intervals.is_empty() {
1291 return Vec::new();
1292 }
1293
1294 // Split groups only when the gap between files exceeds one period,
1295 // since sub-period gaps land in the same consolidated file anyway.
1296 // This works for both legacy chunked files (gap ~1ns) and fragment-per-flush
1297 // catalogs (gap ~bar interval) without inferring spacing from the data.
1298 let mut contiguous_groups = Vec::new();
1299 let mut current_group = vec![intervals[0]];
1300 let mut current_end = intervals[0].1;
1301 for &(curr_start, curr_end) in intervals.iter().skip(1) {
1302 if curr_start.saturating_sub(current_end) > period_nanos {
1303 contiguous_groups.push(current_group);
1304 current_group = vec![(curr_start, curr_end)];
1305 current_end = curr_end;
1306 } else {
1307 current_group.push((curr_start, curr_end));
1308 current_end = current_end.max(curr_end);
1309 }
1310 }
1311
1312 contiguous_groups.push(current_group);
1313
1314 contiguous_groups
1315 }
1316}
1317
1318impl ConsolidationQuery {
1319 fn file_range(
1320 &self,
1321 start_ns: &mut Option<u64>,
1322 first: UnixNanos,
1323 last: UnixNanos,
1324 ) -> (UnixNanos, UnixNanos) {
1325 if self.use_period_boundaries {
1326 let start = *start_ns.get_or_insert(self.query_start);
1327 (UnixNanos::from(start), UnixNanos::from(self.query_end))
1328 } else {
1329 (first, last)
1330 }
1331 }
1332}
1333
1334struct ConsolidationFiles {
1335 sources: Vec<String>,
1336 paths: AHashSet<String>,
1337 completed: Vec<ClosedInterval>,
1338}
1339
1340impl ConsolidationFiles {
1341 fn new(files: &[String], start: Option<UnixNanos>, end: Option<UnixNanos>) -> Self {
1342 Self {
1343 sources: files
1344 .iter()
1345 .filter(|file| {
1346 parse_filename_timestamps(file)
1347 .is_some_and(|interval| interval_intersects_range(interval, start, end))
1348 })
1349 .cloned()
1350 .collect(),
1351 paths: files.iter().cloned().collect(),
1352 completed: Vec::new(),
1353 }
1354 }
1355
1356 fn prepare_target(
1357 &mut self,
1358 catalog: &ParquetDataCatalog,
1359 target: &str,
1360 batches: &[RecordBatch],
1361 ) -> anyhow::Result<bool> {
1362 if !self.paths.contains(target) {
1363 self.paths.insert(target.to_string());
1364 return Ok(true);
1365 }
1366
1367 anyhow::ensure!(
1368 catalog.target_holds_rows(target, batches)?,
1369 "Cannot consolidate into existing target {target}: it does not hold every source row",
1370 );
1371 self.sources.retain(|file| file != target);
1372 Ok(false)
1373 }
1374
1375 fn complete(
1376 &mut self,
1377 catalog: &ParquetDataCatalog,
1378 query: &ConsolidationQuery,
1379 write_files: bool,
1380 ) -> anyhow::Result<()> {
1381 self.completed.push(ClosedInterval {
1382 start: query.query_start,
1383 end: query.query_end,
1384 });
1385
1386 self.completed = merge_closed_intervals(&self.completed);
1387
1388 for file in self.sources.clone() {
1389 if let Some((start, end)) = parse_filename_timestamps(&file)
1390 && self
1391 .completed
1392 .iter()
1393 .any(|covered| covered.start <= start && end <= covered.end)
1394 {
1395 self.sources.retain(|source| source != &file);
1396 self.paths.remove(&file);
1397 if write_files {
1398 catalog.delete_file(&file)?;
1399 }
1400 }
1401 }
1402
1403 Ok(())
1404 }
1405}
1406
1407fn interval_intersects_range(
1408 (interval_start, interval_end): (u64, u64),
1409 start: Option<UnixNanos>,
1410 end: Option<UnixNanos>,
1411) -> bool {
1412 start.is_none_or(|start| start.as_u64() <= interval_end)
1413 && end.is_none_or(|end| interval_start <= end.as_u64())
1414}
1415
1416fn can_rewrite_consolidation_by_period(type_name: &str) -> bool {
1417 matches!(
1418 type_name,
1419 "quotes"
1420 | "trades"
1421 | "order_book_deltas"
1422 | "order_book_depths"
1423 | "bars"
1424 | "index_prices"
1425 | "mark_prices"
1426 )
1427}
1428
1429fn is_schema_incompatibility(error: &anyhow::Error) -> bool {
1430 let message = error.to_string().to_lowercase();
1431 [
1432 "schema",
1433 "field",
1434 "column",
1435 "data type",
1436 "datatype",
1437 "not compatible",
1438 "mismatch",
1439 ]
1440 .iter()
1441 .any(|needle| message.contains(needle))
1442}
1443
1444#[cfg(test)]
1445mod tests {
1446 use nautilus_model::{
1447 enums::AggressorSide,
1448 identifiers::{InstrumentId, TradeId},
1449 types::{Price, Quantity},
1450 };
1451 use nautilus_serialization::arrow::DecodeFromRecordBatch;
1452 use rstest::rstest;
1453
1454 use super::*;
1455 use crate::backend::parquet::io::{
1456 read_parquet_from_object_store, write_batches_to_object_store,
1457 };
1458
1459 #[rstest]
1460 fn repeated_consolidation_preserves_windows_encoded_target() {
1461 let root = tempfile::tempdir().unwrap();
1462 let catalog = ParquetDataCatalog::new(root.path(), None, None, None, None);
1463 let id = "io:ANTH-USD-PERP.HYPERLIQUID";
1464
1465 let trades = [10_000u64, 20_000].map(|ts| {
1466 TradeTick::new(
1467 InstrumentId::from(id),
1468 Price::from("12.34"),
1469 Quantity::from("5.67"),
1470 AggressorSide::Buy,
1471 TradeId::new(ts.to_string()),
1472 (ts - 1).into(),
1473 ts.into(),
1474 )
1475 });
1476
1477 let batches = catalog.data_to_record_batches(&trades).unwrap();
1478 let filename = timestamps_to_filename(10_000.into(), 20_000.into());
1479 let stored = ObjectPath::parse(format!(
1480 "data/trades/io%3AANTH-USD-PERP.HYPERLIQUID/{filename}"
1481 ))
1482 .unwrap();
1483 let directory = catalog.make_path("trades", Some(id)).unwrap();
1484 let raw_target = ObjectPath::from(format!("{directory}/{filename}"));
1485 let target = ParquetDataCatalog::encode_windows_colon_path(&raw_target);
1486 catalog
1487 .execute_async(|| async {
1488 write_batches_to_object_store(
1489 &batches,
1490 catalog.object_store.clone(),
1491 &stored,
1492 None,
1493 None,
1494 None,
1495 )
1496 .await
1497 })
1498 .unwrap();
1499
1500 let query = ConsolidationQuery {
1501 query_start: 10_000,
1502 query_end: 20_000,
1503 use_period_boundaries: false,
1504 };
1505
1506 for _ in 0..2 {
1507 let existing = catalog.list_parquet_files("data/trades").unwrap();
1508 let mut files = ConsolidationFiles::new(&existing, None, None);
1509 let write_target = files
1510 .prepare_target(&catalog, target.as_ref(), &batches)
1511 .unwrap();
1512 files.complete(&catalog, &query, true).unwrap();
1513 let remaining = catalog.list_parquet_files("data/trades").unwrap();
1514
1515 let (stored_batches, schema) = catalog
1516 .execute_async(|| async {
1517 read_parquet_from_object_store(catalog.object_store.clone(), &stored).await
1518 })
1519 .unwrap();
1520
1521 let actual = stored_batches
1522 .into_iter()
1523 .flat_map(|batch| TradeTick::decode_batch(schema.metadata(), batch).unwrap())
1524 .collect::<Vec<_>>();
1525
1526 assert_eq!(target, stored);
1527 assert!(!write_target);
1528 assert_eq!(remaining, vec![stored.to_string()]);
1529 assert_eq!(actual, trades);
1530 }
1531 }
1532
1533 #[rstest]
1534 #[case("quotes", true)]
1535 #[case("bars", true)]
1536 #[case("instrument_closes", false)]
1537 #[case("custom/signal", false)]
1538 fn can_rewrite_consolidation_by_period_only_for_supported_types(
1539 #[case] type_name: &str,
1540 #[case] expected: bool,
1541 ) {
1542 assert_eq!(can_rewrite_consolidation_by_period(type_name), expected);
1543 }
1544
1545 #[rstest]
1546 #[case("schema mismatch while writing record batch", true)]
1547 #[case("object store request timed out", false)]
1548 fn schema_incompatibility_detection_matches_schema_errors(
1549 #[case] message: &str,
1550 #[case] expected: bool,
1551 ) {
1552 assert_eq!(
1553 is_schema_incompatibility(&anyhow::anyhow!(message.to_string())),
1554 expected
1555 );
1556 }
1557}