Skip to main content

nautilus_persistence/backend/parquet/catalog/
write.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Parquet catalog write paths.
17
18#![expect(
19    clippy::missing_errors_doc,
20    reason = "catalog write functions validate catalog-controlled batches and forward storage errors"
21)]
22
23use nautilus_serialization::arrow::catalog_identifier_from_metadata;
24
25use super::{
26    BTreeMap, CustomData, Data, DataBatch, EncodeToRecordBatch, HasCatalogDataType, HasTsInit,
27    Instrument, InstrumentAny, NautilusDataType, NautilusRecordType, ObjectPath, ObjectStoreExt,
28    Params, ParquetDataCatalog, PathBuf, RecordBatch, Serialize, UnixNanos,
29    WRITE_SKIP_DISJOINT_CHECK, are_intervals_disjoint, group_custom_data_by_type,
30    instrument_any_type, instrument_path_prefix, local_to_object_store_path,
31    parquet_data_path_prefix, prepare_custom_data_batch, record_batch_without_identifier_column,
32    record_path_prefix, timestamps_to_filename, to_snake_case, write_batches_to_object_store,
33    write_catalog_batch,
34};
35use crate::{
36    backend::parquet::{io::write_batches_to_object_store_create, paths::catalog_filename},
37    common::metadata::record_batch_ts_init_range,
38};
39
40impl ParquetDataCatalog {
41    /// Writes mixed data types to the catalog by separating them into type-specific collections.
42    ///
43    /// This method takes a heterogeneous collection of market data and separates it by type,
44    /// then writes each type to its appropriate location in the catalog. This is useful when
45    /// processing mixed data streams or bulk data imports.
46    ///
47    /// # Parameters
48    ///
49    /// - `data`: A vector of mixed [`Data`] enum variants.
50    /// - `start`: Optional start timestamp to override the data's natural range.
51    /// - `end`: Optional end timestamp to override the data's natural range.
52    ///
53    /// # Notes
54    ///
55    /// - Data is automatically sorted by type before writing.
56    /// - Each data type is written to its own directory structure.
57    /// - Instrument data handling is not yet implemented (TODO).
58    ///
59    /// # Examples
60    ///
61    /// ```rust,no_run
62    /// use nautilus_model::data::Data;
63    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
64    ///
65    /// let mut catalog = ParquetDataCatalog::new(
66    ///     std::path::Path::new("/tmp/nautilus_data"),
67    ///     None,
68    ///     None,
69    ///     None,
70    ///     None,
71    /// );
72    /// let mixed_data: Vec<Data> = vec![/* mixed data types */];
73    ///
74    /// catalog.write_data_enum(&mixed_data, None, None, None)?;
75    /// # Ok::<(), anyhow::Error>(())
76    /// ```
77    pub fn write_data_enum(
78        &self,
79        data: &[Data],
80        start: Option<UnixNanos>,
81        end: Option<UnixNanos>,
82        skip_disjoint_check: Option<bool>,
83    ) -> anyhow::Result<()> {
84        let mut batches = Vec::new();
85        let mut custom = Vec::new();
86
87        for batch in DataBatch::from_data_vec_grouped(data)? {
88            match batch {
89                DataBatch::Custom(data) => custom.extend(data.iter().cloned()),
90                batch => batches.push(batch),
91            }
92        }
93
94        // Check every custom group of every type name before any file is written
95        let custom_writes =
96            self.prepare_custom_data_groups(&custom, start, end, skip_disjoint_check)?;
97
98        for batch in &batches {
99            write_catalog_batch(self, batch, start, end, skip_disjoint_check)?;
100        }
101
102        for write in &custom_writes {
103            self.write_custom_data_prepared(write, skip_disjoint_check)?;
104        }
105
106        Ok(())
107    }
108
109    pub(super) fn write_grouped_to_parquet<T>(
110        &self,
111        data: &[T],
112        start: Option<UnixNanos>,
113        end: Option<UnixNanos>,
114        skip_disjoint_check: Option<bool>,
115    ) -> anyhow::Result<()>
116    where
117        T: Clone + HasTsInit + EncodeToRecordBatch + HasCatalogDataType,
118    {
119        let mut groups: BTreeMap<Option<String>, Vec<T>> = BTreeMap::new();
120
121        for item in data {
122            let identifier = catalog_identifier_from_metadata(&item.metadata());
123            groups.entry(identifier).or_default().push(item.clone());
124        }
125
126        for items in groups.into_values() {
127            self.write_to_parquet(&items, start, end, skip_disjoint_check)?;
128        }
129
130        Ok(())
131    }
132
133    /// Writes Arrow batches into catalog under a record type optional identifier.
134    ///
135    /// # Errors
136    ///
137    /// Returns an error if batches do not contain `ts_init` or cannot be persisted.
138    pub fn write_record_batches(
139        &mut self,
140        record_type: &NautilusRecordType,
141        identifier: Option<&str>,
142        batches: &[RecordBatch],
143        params: &Params,
144    ) -> anyhow::Result<()> {
145        if batches.is_empty() || batches.iter().all(|batch| batch.num_rows() == 0) {
146            return Ok(());
147        }
148
149        let (start_ts, end_ts) = record_batch_ts_init_range(batches)?;
150        let record_prefix = record_path_prefix(record_type);
151        let directory = self.make_path(record_prefix.as_ref(), identifier)?;
152        self.recover_consolidation(&self.to_object_path(&directory)?)?;
153        let filename = timestamps_to_filename(UnixNanos::from(start_ts), UnixNanos::from(end_ts));
154        let path = PathBuf::from(directory.clone()).join(&filename);
155        let object_path = self.to_object_path(&path.to_string_lossy())?;
156        let skip_disjoint_check = params.get_bool(WRITE_SKIP_DISJOINT_CHECK).unwrap_or(false);
157
158        if !skip_disjoint_check {
159            let current_intervals = self.get_directory_intervals(&directory)?;
160            let mut intervals = current_intervals.clone();
161            intervals.push((start_ts, end_ts));
162            anyhow::ensure!(
163                are_intervals_disjoint(&intervals),
164                "Writing file {filename} interval ({start_ts}, {end_ts}) would create non-disjoint intervals. Existing intervals: {current_intervals:?}",
165            );
166        }
167
168        self.execute_async(|| async {
169            write_batches_to_object_store(
170                batches,
171                self.object_store.clone(),
172                &object_path,
173                Some(self.compression),
174                Some(self.max_row_group_size),
175                None,
176            )
177            .await
178        })
179    }
180
181    /// Writes typed data to a Parquet file in the catalog.
182    ///
183    /// This is the core method for persisting market data to the catalog. It handles data
184    /// validation, batching, compression, and ensures proper file organization with
185    /// timestamp-based naming.
186    ///
187    /// # Type Parameters
188    ///
189    /// - `T`: The data type to write, must implement required traits for serialization and cataloging.
190    ///
191    /// # Parameters
192    ///
193    /// - `data`: Vector of data records to write (must be in ascending timestamp order).
194    /// - `start`: Optional start timestamp to override the natural data range.
195    /// - `end`: Optional end timestamp to override the natural data range.
196    ///
197    /// # Returns
198    ///
199    /// Returns the [`PathBuf`] of the created file, or an empty path if no data was provided.
200    /// If the target file already exists, returns the path without writing (skips write).
201    ///
202    /// # Errors
203    ///
204    /// Returns an error if:
205    /// - Data serialization to Arrow record batches fails.
206    /// - Object store write operations fail.
207    /// - File path construction fails.
208    /// - Writing would create non-disjoint timestamp intervals.
209    ///
210    /// # Panics
211    ///
212    /// Panics if:
213    /// - Data timestamps are not in ascending order.
214    /// - Record batches are empty after conversion.
215    /// - Required metadata is missing from the schema.
216    ///
217    /// # Examples
218    ///
219    /// ```rust,no_run
220    /// use nautilus_model::data::QuoteTick;
221    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
222    ///
223    /// let mut catalog = ParquetDataCatalog::new(
224    ///     std::path::Path::new("/tmp/nautilus_data"),
225    ///     None,
226    ///     None,
227    ///     None,
228    ///     None,
229    /// );
230    /// let quotes: Vec<QuoteTick> = vec![/* quote data */];
231    ///
232    /// let path = catalog.write_to_parquet(&quotes, None, None, None)?;
233    /// println!("Data written to: {:?}", path);
234    /// # Ok::<(), anyhow::Error>(())
235    /// ```
236    pub fn write_to_parquet<T>(
237        &self,
238        data: &[T],
239        start: Option<UnixNanos>,
240        end: Option<UnixNanos>,
241        skip_disjoint_check: Option<bool>,
242    ) -> anyhow::Result<PathBuf>
243    where
244        T: HasTsInit + EncodeToRecordBatch + HasCatalogDataType,
245    {
246        let (Some(first), Some(last)) = (data.first(), data.last()) else {
247            return Ok(PathBuf::new());
248        };
249
250        let type_name = to_snake_case(std::any::type_name::<T>());
251        Self::check_ascending_timestamps(data, &type_name)?;
252        Self::check_chunk_identity(data, &type_name)?;
253
254        let start_ts = start.unwrap_or(first.ts_init());
255        let end_ts = end.unwrap_or(last.ts_init());
256
257        let batches = self.data_to_record_batches(data)?;
258        let schema = batches.first().expect("Batches are empty.").schema();
259
260        let data_type = T::catalog_data_type();
261        let path_prefix = parquet_data_path_prefix(&data_type);
262
263        let identifier = if matches!(data_type, NautilusDataType::Bar) {
264            schema.metadata.get("bar_type").cloned()
265        } else {
266            schema.metadata.get("instrument_id").cloned()
267        };
268
269        let directory = self.make_path(path_prefix.as_ref(), identifier.as_deref())?;
270        self.write_parquet_file_checked(
271            &directory,
272            start_ts,
273            end_ts,
274            &batches,
275            skip_disjoint_check.unwrap_or(false),
276            "File",
277            Some(&format!("{type_name} data")),
278            None,
279        )
280    }
281
282    /// Writes custom data to a Parquet file in the catalog.
283    ///
284    /// This method handles writing custom data types that implement `CustomDataTrait`.
285    /// Custom data is organized by type name in a `custom/{type_name}/` directory structure.
286    ///
287    /// # Parameters
288    ///
289    /// - `data`: Vector of custom data items to write (must be in ascending timestamp order).
290    /// - `start`: Optional start timestamp to override the natural data range.
291    /// - `end`: Optional end timestamp to override the natural data range.
292    /// - `skip_disjoint_check`: Whether to skip interval disjointness validation.
293    ///
294    /// # Returns
295    ///
296    /// Returns the [`PathBuf`] of the created file, or an empty path if no data was provided.
297    ///
298    /// # Errors
299    ///
300    /// Returns an error if:
301    /// - The registered Arrow schema omits `ts_init` or uses incompatible timestamp types.
302    /// - Data serialization to Arrow record batches fails.
303    /// - Object store write operations fail.
304    /// - File path construction fails.
305    /// - Writing would create non-disjoint timestamp intervals (unless skipped).
306    pub fn write_custom_data_batch<D>(
307        &self,
308        data: D,
309        start: Option<UnixNanos>,
310        end: Option<UnixNanos>,
311        skip_disjoint_check: Option<bool>,
312    ) -> anyhow::Result<PathBuf>
313    where
314        D: AsRef<[CustomData]>,
315    {
316        let data = data.as_ref().iter().collect::<Vec<_>>();
317        self.write_custom_data_refs_batch(&data, start, end, skip_disjoint_check)
318    }
319
320    fn write_custom_data_refs_batch(
321        &self,
322        data: &[&CustomData],
323        start: Option<UnixNanos>,
324        end: Option<UnixNanos>,
325        skip_disjoint_check: Option<bool>,
326    ) -> anyhow::Result<PathBuf> {
327        if data.is_empty() {
328            return Ok(PathBuf::new());
329        }
330
331        let write = self.prepare_custom_data_write(data, start, end)?;
332        self.write_custom_data_prepared(&write, skip_disjoint_check)
333    }
334
335    /// Writes custom data as one file per [`DataType`](nautilus_model::data::DataType) group.
336    ///
337    /// Every group is prepared and checked before any file is written, so a call whose groups
338    /// would share a file, or overlap within a directory while the disjoint check is active,
339    /// fails without writing anything.
340    pub(crate) fn write_custom_data_groups(
341        &self,
342        data: &[CustomData],
343        start: Option<UnixNanos>,
344        end: Option<UnixNanos>,
345        skip_disjoint_check: Option<bool>,
346    ) -> anyhow::Result<()> {
347        for write in &self.prepare_custom_data_groups(data, start, end, skip_disjoint_check)? {
348            self.write_custom_data_prepared(write, skip_disjoint_check)?;
349        }
350
351        Ok(())
352    }
353
354    // Prepares one write per custom data group and rejects groups that would share a file or,
355    // while the disjoint check is active, overlap within a directory. A file holds one data type,
356    // so two such groups cannot be stored together, and writing them in turn would skip the
357    // second as an existing file.
358    pub(crate) fn prepare_custom_data_groups(
359        &self,
360        data: &[CustomData],
361        start: Option<UnixNanos>,
362        end: Option<UnixNanos>,
363        skip_disjoint_check: Option<bool>,
364    ) -> anyhow::Result<Vec<CustomDataWrite>> {
365        let check_disjoint = !skip_disjoint_check.unwrap_or(false);
366        let mut writes = Vec::new();
367        let mut directory_intervals: BTreeMap<String, Vec<(u64, u64)>> = BTreeMap::new();
368
369        for group in group_custom_data_by_type(data) {
370            let write = self.prepare_custom_data_write(&group, start, end)?;
371            let interval = (write.start_ts.as_u64(), write.end_ts.as_u64());
372            let intervals = directory_intervals
373                .entry(write.directory.clone())
374                .or_default();
375
376            anyhow::ensure!(
377                !intervals.contains(&interval),
378                "Cannot write custom data for {} with different metadata to the same file {}: \
379                 a file holds one data type",
380                write.type_name,
381                PathBuf::from(&write.directory)
382                    .join(catalog_filename(write.start_ts, write.end_ts, None))
383                    .display(),
384            );
385            intervals.push(interval);
386            anyhow::ensure!(
387                !check_disjoint || are_intervals_disjoint(intervals),
388                "Cannot write custom data for {} with different metadata over overlapping \
389                 intervals {intervals:?} in {}",
390                write.type_name,
391                write.directory,
392            );
393
394            writes.push(write);
395        }
396
397        Ok(writes)
398    }
399
400    pub(crate) fn prepare_custom_data_write(
401        &self,
402        data: &[&CustomData],
403        start: Option<UnixNanos>,
404        end: Option<UnixNanos>,
405    ) -> anyhow::Result<CustomDataWrite> {
406        let (batch, type_name, identifier, start_ts, end_ts) = prepare_custom_data_batch(data)?;
407        let directory = self.make_path_custom_data(&type_name, identifier.as_deref())?;
408
409        Ok(CustomDataWrite {
410            type_name,
411            directory,
412            start_ts: start.unwrap_or(start_ts),
413            end_ts: end.unwrap_or(end_ts),
414            batches: vec![record_batch_without_identifier_column(batch)?],
415        })
416    }
417
418    pub(crate) fn write_custom_data_prepared(
419        &self,
420        write: &CustomDataWrite,
421        skip_disjoint_check: Option<bool>,
422    ) -> anyhow::Result<PathBuf> {
423        self.write_parquet_file_checked(
424            &write.directory,
425            write.start_ts,
426            write.end_ts,
427            &write.batches,
428            skip_disjoint_check.unwrap_or(false),
429            "File",
430            None,
431            None,
432        )
433    }
434
435    /// Writes instruments to Parquet files in the catalog.
436    ///
437    /// Instruments are stored under their instrument ID directory using timestamp-ranged
438    /// file names, allowing multiple historical versions of the same instrument to be
439    /// appended over time:
440    /// `data/instruments/{instrument_id}/{start_ts}-{end_ts}.parquet`
441    ///
442    /// # Parameters
443    ///
444    /// - `instruments`: Vector of instruments to write.
445    ///
446    /// # Returns
447    ///
448    /// Returns a vector of paths to the created files.
449    ///
450    /// # Errors
451    ///
452    /// Returns an error if:
453    /// - Data serialization fails.
454    /// - Object store write operations fail.
455    /// - File path construction fails.
456    ///
457    /// # Examples
458    ///
459    /// ```rust,no_run
460    /// use nautilus_model::instruments::InstrumentAny;
461    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
462    ///
463    /// let mut catalog = ParquetDataCatalog::new(
464    ///     std::path::Path::new("/tmp/nautilus_data"),
465    ///     None,
466    ///     None,
467    ///     None,
468    ///     None,
469    /// );
470    /// let instruments: Vec<InstrumentAny> = vec![/* instruments */];
471    ///
472    /// let paths = catalog.write_instruments(instruments)?;
473    /// # Ok::<(), anyhow::Error>(())
474    /// ```
475    pub fn write_instruments(
476        &self,
477        instruments: Vec<InstrumentAny>,
478    ) -> anyhow::Result<Vec<PathBuf>> {
479        if instruments.is_empty() {
480            return Ok(Vec::new());
481        }
482
483        // Group instruments by concrete type and instrument_id so mixed InstrumentAny
484        // inputs are written as separate parquet batches with stable ordering.
485        let mut by_type_and_id: BTreeMap<(String, String), Vec<InstrumentAny>> = BTreeMap::new();
486
487        for instrument in instruments {
488            let instrument_type = instrument_any_type(&instrument);
489            let instrument_prefix = instrument_path_prefix(&instrument_type).to_string();
490            let instrument_id = Instrument::id(&instrument).to_string();
491            by_type_and_id
492                .entry((instrument_prefix, instrument_id))
493                .or_default()
494                .push(instrument);
495        }
496
497        let mut paths = Vec::new();
498
499        for ((instrument_prefix, instrument_id), instrument_group) in by_type_and_id {
500            Self::check_ascending_timestamps(&instrument_group, "instrument")?;
501
502            let (Some(first), Some(last)) = (instrument_group.first(), instrument_group.last())
503            else {
504                continue;
505            };
506
507            let start_ts = HasTsInit::ts_init(first);
508            let end_ts = HasTsInit::ts_init(last);
509            let batches = self.data_to_record_batches(&instrument_group)?;
510
511            let directory = self.make_path(&instrument_prefix, Some(instrument_id.as_str()))?;
512
513            // ArrowWriter stores the full schema (including `type_name` metadata) in ARROW:schema.
514            // When reading, use the builder's schema for metadata (see query_instruments).
515            let path = self.write_parquet_file_checked(
516                &directory,
517                start_ts,
518                end_ts,
519                &batches,
520                false,
521                "Instrument file",
522                Some(&format!("instrument data for {instrument_id}")),
523                None,
524            )?;
525
526            paths.push(path);
527        }
528
529        Ok(paths)
530    }
531
532    /// Writes `batches` to a timestamp-named parquet file in `directory`, skipping when the
533    /// target file already exists and validating interval disjointness unless `skip_disjoint_check`.
534    ///
535    /// `file_label` and `data_description` parameterize the log messages so callers keep their
536    /// site-specific wording (`data_description = None` suppresses the pre-write log).
537    #[expect(clippy::too_many_arguments)]
538    pub(crate) fn write_parquet_file_checked(
539        &self,
540        directory: &str,
541        start_ts: UnixNanos,
542        end_ts: UnixNanos,
543        batches: &[RecordBatch],
544        skip_disjoint_check: bool,
545        file_label: &str,
546        data_description: Option<&str>,
547        replay_identity: Option<&str>,
548    ) -> anyhow::Result<PathBuf> {
549        self.recover_consolidation(&self.to_object_path(directory)?)?;
550        let filename = catalog_filename(start_ts, end_ts, replay_identity);
551
552        let path = PathBuf::from(directory).join(&filename);
553        let object_path = self.to_object_path(&path.to_string_lossy())?;
554
555        if self.file_exists(&path.to_string_lossy())? {
556            log::info!(
557                "{file_label} {} already exists, skipping write",
558                path.display()
559            );
560            return Ok(path);
561        }
562
563        if !skip_disjoint_check {
564            let current_intervals = self.get_directory_intervals(directory)?;
565            let new_interval = (start_ts.as_u64(), end_ts.as_u64());
566            let mut new_intervals = current_intervals.clone();
567            new_intervals.push(new_interval);
568
569            anyhow::ensure!(
570                are_intervals_disjoint(&new_intervals),
571                "Writing file {filename} with interval ({start_ts}, {end_ts}) would create \
572                non-disjoint intervals. Existing intervals: {current_intervals:?}"
573            );
574        }
575
576        if let Some(data_description) = data_description {
577            log::info!(
578                "Writing {} batches of {data_description} to {}",
579                batches.len(),
580                path.display(),
581            );
582        }
583
584        self.execute_async(|| async {
585            if replay_identity.is_none() {
586                return write_batches_to_object_store(
587                    batches,
588                    self.object_store.clone(),
589                    &object_path,
590                    Some(self.compression),
591                    Some(self.max_row_group_size),
592                    None,
593                )
594                .await;
595            }
596
597            let result = write_batches_to_object_store_create(
598                batches,
599                self.object_store.clone(),
600                &object_path,
601                Some(self.compression),
602                Some(self.max_row_group_size),
603                None,
604            )
605            .await;
606
607            match result {
608                Err(e)
609                    if matches!(
610                        e.downcast_ref::<object_store::Error>(),
611                        Some(object_store::Error::AlreadyExists { .. })
612                    ) =>
613                {
614                    Ok(())
615                }
616                result => result,
617            }
618        })?;
619
620        Ok(path)
621    }
622
623    /// Writes typed data to a JSON file in the catalog.
624    ///
625    /// This method provides an alternative to Parquet format for data export and debugging.
626    /// JSON files are human-readable but less efficient for large datasets.
627    ///
628    /// # Type Parameters
629    ///
630    /// - `T`: The data type to write, must implement serialization and cataloging traits.
631    ///
632    /// # Parameters
633    ///
634    /// - `data`: Vector of data records to write (must be in ascending timestamp order).
635    /// - `path`: Optional custom directory path (defaults to catalog's standard structure).
636    /// - `write_metadata`: Whether to write a separate metadata file alongside the data.
637    ///
638    /// # Returns
639    ///
640    /// Returns the [`PathBuf`] of the created JSON file.
641    ///
642    /// # Errors
643    ///
644    /// Returns an error if:
645    /// - JSON serialization fails.
646    /// - Object store write operations fail.
647    /// - File path construction fails.
648    ///
649    /// # Examples
650    ///
651    /// ```rust,no_run
652    /// use std::path::PathBuf;
653    /// use nautilus_model::data::TradeTick;
654    /// use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
655    ///
656    /// let mut catalog = ParquetDataCatalog::new(
657    ///     std::path::Path::new("/tmp/nautilus_data"),
658    ///     None,
659    ///     None,
660    ///     None,
661    ///     None,
662    /// );
663    /// let trades: Vec<TradeTick> = vec![/* trade data */];
664    ///
665    /// let path = catalog.write_to_json(
666    ///     trades,
667    ///     Some(PathBuf::from("/custom/path")),
668    ///     true  // write metadata
669    /// )?;
670    /// # Ok::<(), anyhow::Error>(())
671    /// ```
672    pub fn write_to_json<T>(
673        &self,
674        data: Vec<T>,
675        path: Option<PathBuf>,
676        write_metadata: bool,
677    ) -> anyhow::Result<PathBuf>
678    where
679        T: HasTsInit + Serialize + HasCatalogDataType + EncodeToRecordBatch,
680    {
681        let (Some(first), Some(last)) = (data.first(), data.last()) else {
682            return Ok(PathBuf::new());
683        };
684
685        let type_name = to_snake_case(std::any::type_name::<T>());
686        Self::check_ascending_timestamps(&data, &type_name)?;
687
688        let start_ts = first.ts_init();
689        let end_ts = last.ts_init();
690
691        let data_type = T::catalog_data_type();
692        let path_prefix = parquet_data_path_prefix(&data_type);
693
694        let directory = match path {
695            Some(path) => path,
696            None => PathBuf::from(self.make_path(path_prefix.as_ref(), None)?),
697        };
698
699        let filename = timestamps_to_filename(start_ts, end_ts).replace(".parquet", ".json");
700        let json_path = directory.join(&filename);
701
702        log::info!(
703            "Writing {} records of {type_name} data to {}",
704            data.len(),
705            json_path.display(),
706        );
707
708        if write_metadata {
709            let metadata = T::chunk_metadata(&data);
710            let metadata_path = json_path.with_extension("metadata.json");
711            log::info!("Writing metadata to {}", metadata_path.display());
712
713            // Use object store for metadata file
714            let metadata_object_path = ObjectPath::from(local_to_object_store_path(&metadata_path));
715            let metadata_json = serde_json::to_vec_pretty(&metadata)?;
716            self.execute_async(|| async {
717                let _: object_store::PutResult = self
718                    .object_store
719                    .put(&metadata_object_path, metadata_json.into())
720                    .await?;
721                Ok(())
722            })?;
723        }
724
725        // Use object store for main JSON file
726        let json_object_path = ObjectPath::from(local_to_object_store_path(&json_path));
727        let json_data = serde_json::to_vec_pretty(&serde_json::to_value(data)?)?;
728        self.execute_async(|| async {
729            let _: object_store::PutResult = self
730                .object_store
731                .put(&json_object_path, json_data.into())
732                .await?;
733            Ok(())
734        })?;
735
736        Ok(json_path)
737    }
738
739    /// Validates that data timestamps are in ascending order.
740    ///
741    /// # Parameters
742    ///
743    /// - `data`: Slice of data records to validate.
744    /// - `type_name`: Name of the data type for error messages.
745    pub fn check_ascending_timestamps<T: HasTsInit>(
746        data: &[T],
747        type_name: &str,
748    ) -> anyhow::Result<()> {
749        anyhow::ensure!(
750            data.array_windows()
751                .all(|[a, b]| a.ts_init() <= b.ts_init()),
752            "{type_name} timestamps must be in ascending order"
753        );
754        Ok(())
755    }
756
757    // Rejects data whose elements carry different chunk metadata, such as two instruments or
758    // precisions, since one file stores one identity.
759    pub(crate) fn check_chunk_identity<T: EncodeToRecordBatch>(
760        data: &[T],
761        type_name: &str,
762    ) -> anyhow::Result<()> {
763        if data.is_empty() {
764            return Ok(());
765        }
766        let chunk_metadata = T::chunk_metadata(data);
767
768        if let Some(position) = data
769            .iter()
770            .position(|item| !item.matches_chunk_metadata(&chunk_metadata))
771        {
772            anyhow::bail!(
773                "Cannot write {type_name} data with mixed identities: element {position} has \
774                 metadata {:?} but the chunk has {chunk_metadata:?}; write each \
775                 instrument or bar type separately",
776                data[position].metadata(),
777            );
778        }
779
780        Ok(())
781    }
782
783    /// Converts data into Arrow record batches for Parquet serialization.
784    ///
785    /// This method chunks the data according to the configured batch size and converts
786    /// each chunk into an Arrow record batch with appropriate metadata.
787    ///
788    /// # Type Parameters
789    ///
790    /// - `T`: The data type to convert, must implement required encoding traits.
791    ///
792    /// # Parameters
793    ///
794    /// - `data`: Vector of data records to convert.
795    ///
796    /// # Returns
797    ///
798    /// Returns a vector of Arrow [`RecordBatch`] instances ready for Parquet serialization.
799    ///
800    /// # Errors
801    ///
802    /// Returns an error if record batch encoding fails for any chunk.
803    pub fn data_to_record_batches<T>(&self, data: &[T]) -> anyhow::Result<Vec<RecordBatch>>
804    where
805        T: HasTsInit + EncodeToRecordBatch,
806    {
807        if data.is_empty() {
808            return Ok(Vec::new());
809        }
810
811        let mut batches = Vec::new();
812        let metadata = EncodeToRecordBatch::chunk_metadata(data);
813
814        for chunk in data.chunks(self.batch_size) {
815            let record_batch = T::encode_batch(&metadata, chunk)?;
816            let record_batch = record_batch_without_identifier_column(record_batch)?;
817            batches.push(record_batch);
818        }
819
820        Ok(batches)
821    }
822}
823
824// A custom data group prepared for writing, with its target directory and file range.
825pub(crate) struct CustomDataWrite {
826    type_name: String,
827    directory: String,
828    start_ts: UnixNanos,
829    end_ts: UnixNanos,
830    pub(crate) batches: Vec<RecordBatch>,
831}
832
833impl CustomDataWrite {
834    pub(crate) fn path(&self) -> PathBuf {
835        let filename = catalog_filename(self.start_ts, self.end_ts, None);
836        PathBuf::from(&self.directory).join(filename)
837    }
838}
839
840#[cfg(test)]
841mod tests {
842    use std::{
843        fmt::Display,
844        fs::File,
845        sync::{
846            Arc,
847            atomic::{AtomicBool, AtomicUsize, Ordering},
848        },
849    };
850
851    use arrow::{
852        array::Int64Array,
853        datatypes::{DataType, Field, Schema},
854        record_batch::RecordBatch,
855    };
856    use futures::{StreamExt, TryStreamExt, stream::BoxStream};
857    use nautilus_core::UnixNanos;
858    use nautilus_model::{
859        data::{
860            NautilusDataType, OrderBookDelta, OrderBookDepth, QuoteTick,
861            stubs::{stub_delta, stub_depth10},
862        },
863        identifiers::InstrumentId,
864        types::{Price, Quantity},
865    };
866    use nautilus_serialization::arrow::{
867        DecodeFromRecordBatch, KEY_PRICE_PRECISION, KEY_SIZE_PRECISION,
868    };
869    use object_store::{
870        CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
871        ObjectStoreExt, PutMode, PutMultipartOptions, PutOptions, PutPayload, PutResult,
872        Result as ObjectStoreResult, memory::InMemory, path::Path as ObjectPath,
873    };
874    use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
875    use rstest::rstest;
876    use tempfile::TempDir;
877
878    use super::ParquetDataCatalog;
879    use crate::common::datafusion::DataBackendSession;
880
881    #[rstest]
882    fn depth_write_shares_file_metadata_across_chunks(stub_depth10: OrderBookDepth) {
883        let mut empty = stub_depth10.clone();
884        empty.bids.clear();
885        empty.asks.clear();
886        empty.bid_counts.clear();
887        empty.ask_counts.clear();
888        let directory = TempDir::new().unwrap();
889        let catalog = ParquetDataCatalog::from_uri(
890            directory.path().to_str().unwrap(),
891            None,
892            Some(2),
893            None,
894            Some(2),
895        )
896        .unwrap();
897        let data = vec![empty.clone(), empty, stub_depth10];
898
899        let path = catalog.write_to_parquet(&data, None, None, None).unwrap();
900        let builder = ParquetRecordBatchReaderBuilder::try_new(
901            File::open(directory.path().join(path)).unwrap(),
902        )
903        .unwrap();
904        let metadata = builder.schema().metadata().clone();
905        let batches = builder
906            .build()
907            .unwrap()
908            .collect::<Result<Vec<_>, _>>()
909            .unwrap();
910        let decoded = batches
911            .iter()
912            .cloned()
913            .flat_map(|batch| OrderBookDepth::decode_batch(&metadata, batch).unwrap())
914            .collect::<Vec<_>>();
915
916        assert_eq!(metadata[KEY_PRICE_PRECISION], "2");
917        assert_eq!(metadata[KEY_SIZE_PRECISION], "0");
918        assert_eq!(decoded.len(), 3);
919        assert_eq!(decoded, data);
920        assert_eq!(decoded[2].bids[0].price.precision, 2);
921    }
922
923    #[rstest]
924    fn leading_clear_delta_writes_with_following_precision(stub_delta: OrderBookDelta) {
925        let directory = TempDir::new().unwrap();
926        let catalog = ParquetDataCatalog::from_uri(
927            directory.path().to_str().unwrap(),
928            None,
929            Some(2),
930            None,
931            None,
932        )
933        .unwrap();
934        let clear = OrderBookDelta::clear(
935            stub_delta.instrument_id,
936            0,
937            UnixNanos::from(1),
938            UnixNanos::from(1),
939        );
940        let second_clear = OrderBookDelta::clear(
941            stub_delta.instrument_id,
942            0,
943            UnixNanos::from(2),
944            UnixNanos::from(2),
945        );
946
947        let path = catalog
948            .write_to_parquet(&[clear, second_clear, stub_delta], None, None, None)
949            .unwrap();
950        let builder = ParquetRecordBatchReaderBuilder::try_new(
951            File::open(directory.path().join(&path)).unwrap(),
952        )
953        .unwrap();
954        let metadata = builder.schema().metadata().clone();
955        let decoded = builder
956            .build()
957            .unwrap()
958            .map(|batch| OrderBookDelta::decode_batch(&metadata, batch.unwrap()).unwrap())
959            .collect::<Vec<_>>()
960            .concat();
961
962        assert!(directory.path().join(path).exists());
963        assert_eq!(metadata[KEY_PRICE_PRECISION], "2");
964        assert_eq!(decoded[2].order.price.precision, 2);
965    }
966
967    #[derive(Debug)]
968    struct CreateRaceStore {
969        inner: InMemory,
970        create_calls: AtomicUsize,
971        already_exists: bool,
972        fail_head: AtomicBool,
973    }
974
975    impl Display for CreateRaceStore {
976        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
977            f.write_str("create-race")
978        }
979    }
980
981    #[async_trait::async_trait]
982    impl ObjectStore for CreateRaceStore {
983        async fn put_opts(
984            &self,
985            location: &ObjectPath,
986            payload: PutPayload,
987            opts: PutOptions,
988        ) -> ObjectStoreResult<PutResult> {
989            if opts.mode == PutMode::Create {
990                self.create_calls.fetch_add(1, Ordering::Relaxed);
991
992                if !self.already_exists {
993                    return Err(object_store::Error::Generic {
994                        store: "create-race",
995                        source: "injected create failure".into(),
996                    });
997                }
998
999                return Err(object_store::Error::AlreadyExists {
1000                    path: location.to_string(),
1001                    source: Box::new(std::io::Error::new(
1002                        std::io::ErrorKind::AlreadyExists,
1003                        "injected create race",
1004                    )),
1005                });
1006            }
1007
1008            self.inner.put_opts(location, payload, opts).await
1009        }
1010
1011        async fn put_multipart_opts(
1012            &self,
1013            location: &ObjectPath,
1014            opts: PutMultipartOptions,
1015        ) -> ObjectStoreResult<Box<dyn MultipartUpload>> {
1016            self.inner.put_multipart_opts(location, opts).await
1017        }
1018
1019        async fn get_opts(
1020            &self,
1021            location: &ObjectPath,
1022            options: GetOptions,
1023        ) -> ObjectStoreResult<GetResult> {
1024            if options.head && self.fail_head.load(Ordering::Relaxed) {
1025                return Err(object_store::Error::Generic {
1026                    store: "create-race",
1027                    source: "injected head failure".into(),
1028                });
1029            }
1030
1031            self.inner.get_opts(location, options).await
1032        }
1033
1034        fn list(
1035            &self,
1036            prefix: Option<&ObjectPath>,
1037        ) -> BoxStream<'static, ObjectStoreResult<ObjectMeta>> {
1038            self.inner.list(prefix)
1039        }
1040
1041        async fn list_with_delimiter(
1042            &self,
1043            prefix: Option<&ObjectPath>,
1044        ) -> ObjectStoreResult<ListResult> {
1045            self.inner.list_with_delimiter(prefix).await
1046        }
1047
1048        fn delete_stream(
1049            &self,
1050            locations: BoxStream<'static, ObjectStoreResult<ObjectPath>>,
1051        ) -> BoxStream<'static, ObjectStoreResult<ObjectPath>> {
1052            self.inner.delete_stream(locations)
1053        }
1054
1055        async fn copy_opts(
1056            &self,
1057            from: &ObjectPath,
1058            to: &ObjectPath,
1059            opts: CopyOptions,
1060        ) -> ObjectStoreResult<()> {
1061            self.inner.copy_opts(from, to, opts).await
1062        }
1063    }
1064
1065    #[rstest]
1066    fn promotion_writes_to_memory_store_without_copy_support() {
1067        let catalog = ParquetDataCatalog {
1068            base_path: "catalog".to_string(),
1069            original_uri: "memory://".to_string(),
1070            object_store: Arc::new(InMemory::new()),
1071            session: DataBackendSession::new(5_000),
1072            batch_size: 5_000,
1073            compression: parquet::basic::Compression::SNAPPY,
1074            max_row_group_size: 5_000,
1075        };
1076
1077        let batch = RecordBatch::try_new(
1078            Arc::new(Schema::new(vec![Field::new(
1079                "ts_init",
1080                DataType::Int64,
1081                false,
1082            )])),
1083            vec![Arc::new(Int64Array::from(vec![1]))],
1084        )
1085        .unwrap();
1086
1087        let path = catalog
1088            .write_parquet_file_checked(
1089                "quotes/TEST",
1090                UnixNanos::from(1),
1091                UnixNanos::from(1),
1092                &[batch],
1093                false,
1094                "Promoted file",
1095                None,
1096                Some("memory-replay"),
1097            )
1098            .unwrap();
1099        let object_path = catalog.to_object_path(&path.to_string_lossy()).unwrap();
1100        catalog
1101            .execute_async(|| async {
1102                catalog.object_store.head(&object_path).await?;
1103                Ok(())
1104            })
1105            .unwrap();
1106    }
1107
1108    #[rstest]
1109    fn promotion_accepts_already_exists_after_head_miss() {
1110        let object_store = Arc::new(CreateRaceStore {
1111            inner: InMemory::new(),
1112            create_calls: AtomicUsize::new(0),
1113            already_exists: true,
1114            fail_head: AtomicBool::new(false),
1115        });
1116
1117        let catalog = ParquetDataCatalog {
1118            base_path: "catalog".to_string(),
1119            original_uri: "memory://".to_string(),
1120            object_store: object_store.clone(),
1121            session: DataBackendSession::new(5_000),
1122            batch_size: 5_000,
1123            compression: parquet::basic::Compression::SNAPPY,
1124            max_row_group_size: 5_000,
1125        };
1126
1127        let batch = RecordBatch::try_new(
1128            Arc::new(Schema::new(vec![Field::new(
1129                "ts_init",
1130                DataType::Int64,
1131                false,
1132            )])),
1133            vec![Arc::new(Int64Array::from(vec![1]))],
1134        )
1135        .unwrap();
1136
1137        catalog
1138            .write_parquet_file_checked(
1139                "quotes/TEST",
1140                UnixNanos::from(1),
1141                UnixNanos::from(1),
1142                &[batch],
1143                true,
1144                "Promoted file",
1145                None,
1146                Some("racing-replay"),
1147            )
1148            .unwrap();
1149
1150        assert_eq!(object_store.create_calls.load(Ordering::Relaxed), 1);
1151    }
1152
1153    #[rstest]
1154    fn promotion_propagates_create_errors_other_than_already_exists() {
1155        let object_store = Arc::new(CreateRaceStore {
1156            inner: InMemory::new(),
1157            create_calls: AtomicUsize::new(0),
1158            already_exists: false,
1159            fail_head: AtomicBool::new(false),
1160        });
1161
1162        let catalog = ParquetDataCatalog {
1163            base_path: "catalog".to_string(),
1164            original_uri: "memory://".to_string(),
1165            object_store: object_store.clone(),
1166            session: DataBackendSession::new(5_000),
1167            batch_size: 5_000,
1168            compression: parquet::basic::Compression::SNAPPY,
1169            max_row_group_size: 5_000,
1170        };
1171
1172        let batch = RecordBatch::try_new(
1173            Arc::new(Schema::new(vec![Field::new(
1174                "ts_init",
1175                DataType::Int64,
1176                false,
1177            )])),
1178            vec![Arc::new(Int64Array::from(vec![1]))],
1179        )
1180        .unwrap();
1181
1182        let error = catalog
1183            .write_parquet_file_checked(
1184                "quotes/TEST",
1185                UnixNanos::from(1),
1186                UnixNanos::from(1),
1187                &[batch],
1188                true,
1189                "Promoted file",
1190                None,
1191                Some("failing-replay"),
1192            )
1193            .unwrap_err();
1194
1195        assert!(
1196            format!("{error:#}").contains("injected create failure"),
1197            "unexpected error: {error:#}"
1198        );
1199        assert_eq!(object_store.create_calls.load(Ordering::Relaxed), 1);
1200    }
1201
1202    #[rstest]
1203    #[case::head_failure(true, None)]
1204    #[case::missing_file(false, Some(false))]
1205    fn file_exists_propagates_storage_errors_other_than_not_found(
1206        #[case] fail_head: bool,
1207        #[case] expected: Option<bool>,
1208    ) {
1209        let catalog = ParquetDataCatalog {
1210            base_path: "catalog".to_string(),
1211            original_uri: "memory://".to_string(),
1212            object_store: Arc::new(CreateRaceStore {
1213                inner: InMemory::new(),
1214                create_calls: AtomicUsize::new(0),
1215                already_exists: true,
1216                fail_head: AtomicBool::new(fail_head),
1217            }),
1218            session: DataBackendSession::new(5_000),
1219            batch_size: 5_000,
1220            compression: parquet::basic::Compression::SNAPPY,
1221            max_row_group_size: 5_000,
1222        };
1223
1224        let result = catalog.file_exists("quotes/TEST/missing.parquet");
1225
1226        match expected {
1227            Some(exists) => assert_eq!(result.unwrap(), exists),
1228            None => assert!(
1229                format!("{:#}", result.unwrap_err()).contains("injected head failure"),
1230                "head failure must propagate"
1231            ),
1232        }
1233    }
1234
1235    #[rstest]
1236    fn write_propagates_head_failure_instead_of_writing() {
1237        let object_store = Arc::new(CreateRaceStore {
1238            inner: InMemory::new(),
1239            create_calls: AtomicUsize::new(0),
1240            already_exists: true,
1241            fail_head: AtomicBool::new(true),
1242        });
1243
1244        let catalog = ParquetDataCatalog {
1245            base_path: "catalog".to_string(),
1246            original_uri: "memory://".to_string(),
1247            object_store: object_store.clone(),
1248            session: DataBackendSession::new(5_000),
1249            batch_size: 5_000,
1250            compression: parquet::basic::Compression::SNAPPY,
1251            max_row_group_size: 5_000,
1252        };
1253
1254        let batch = RecordBatch::try_new(
1255            Arc::new(Schema::new(vec![Field::new(
1256                "ts_init",
1257                DataType::Int64,
1258                false,
1259            )])),
1260            vec![Arc::new(Int64Array::from(vec![1]))],
1261        )
1262        .unwrap();
1263
1264        let error = catalog
1265            .write_parquet_file_checked(
1266                "quotes/TEST",
1267                UnixNanos::from(1),
1268                UnixNanos::from(1),
1269                &[batch],
1270                true,
1271                "Data file",
1272                None,
1273                None,
1274            )
1275            .unwrap_err();
1276
1277        let stored = catalog
1278            .execute_async(|| async {
1279                Ok(object_store
1280                    .inner
1281                    .list(None)
1282                    .collect::<Vec<_>>()
1283                    .await
1284                    .len())
1285            })
1286            .unwrap();
1287
1288        assert!(
1289            format!("{error:#}").contains("injected head failure"),
1290            "unexpected error: {error:#}"
1291        );
1292        assert_eq!(stored, 0);
1293    }
1294
1295    #[rstest]
1296    fn period_consolidation_stops_on_head_failure() {
1297        let object_store = Arc::new(CreateRaceStore {
1298            inner: InMemory::new(),
1299            create_calls: AtomicUsize::new(0),
1300            already_exists: true,
1301            fail_head: AtomicBool::new(false),
1302        });
1303
1304        let mut catalog = ParquetDataCatalog {
1305            base_path: String::new(),
1306            original_uri: "s3://test-bucket".to_string(),
1307            object_store: object_store.clone(),
1308            session: DataBackendSession::new(5_000),
1309            batch_size: 5_000,
1310            compression: parquet::basic::Compression::SNAPPY,
1311            max_row_group_size: 5_000,
1312        };
1313
1314        let quote = |ts: u64| {
1315            QuoteTick::new(
1316                InstrumentId::from("ETH/USDT.BINANCE"),
1317                Price::from("1.0001"),
1318                Price::from("1.0002"),
1319                Quantity::from("100"),
1320                Quantity::from("100"),
1321                UnixNanos::from(ts),
1322                UnixNanos::from(ts),
1323            )
1324        };
1325
1326        catalog
1327            .write_to_parquet(&[quote(1)], None, None, None)
1328            .unwrap();
1329        catalog
1330            .write_to_parquet(&[quote(2)], None, None, None)
1331            .unwrap();
1332        object_store.fail_head.store(true, Ordering::Relaxed);
1333
1334        let error = catalog
1335            .consolidate_data_by_period(
1336                &NautilusDataType::QuoteTick.into(),
1337                Some("ETH/USDT.BINANCE"),
1338                Some(86_400_000_000_000),
1339                None,
1340                None,
1341                Some(true),
1342            )
1343            .unwrap_err();
1344
1345        let files = catalog
1346            .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
1347            .unwrap();
1348        assert!(
1349            format!("{error:#}").contains("injected head failure"),
1350            "unexpected error: {error:#}"
1351        );
1352        assert_eq!(files.len(), 2);
1353    }
1354
1355    #[rstest]
1356    fn write_to_json_normalizes_windows_separators() {
1357        let catalog = ParquetDataCatalog {
1358            base_path: String::new(),
1359            original_uri: "memory://".to_string(),
1360            object_store: Arc::new(InMemory::new()),
1361            session: DataBackendSession::new(5_000),
1362            batch_size: 5_000,
1363            compression: parquet::basic::Compression::SNAPPY,
1364            max_row_group_size: 5_000,
1365        };
1366
1367        let quote = QuoteTick::new(
1368            InstrumentId::from("ETH/USDT.BINANCE"),
1369            Price::from("1.0001"),
1370            Price::from("1.0002"),
1371            Quantity::from("100"),
1372            Quantity::from("100"),
1373            UnixNanos::from(1),
1374            UnixNanos::from(1),
1375        );
1376
1377        catalog
1378            .write_to_json(vec![quote], Some(r"data\quotes".into()), true)
1379            .unwrap();
1380
1381        let mut keys = catalog
1382            .execute_async(|| async {
1383                Ok(catalog
1384                    .object_store
1385                    .list(None)
1386                    .map_ok(|object| object.location.to_string())
1387                    .try_collect::<Vec<_>>()
1388                    .await?)
1389            })
1390            .unwrap();
1391
1392        keys.sort();
1393
1394        assert_eq!(
1395            keys,
1396            vec![
1397                "data/quotes/1970-01-01T00-00-00-000000001Z_1970-01-01T00-00-00-000000001Z.json",
1398                "data/quotes/1970-01-01T00-00-00-000000001Z_1970-01-01T00-00-00-000000001Z.metadata.json",
1399            ]
1400        );
1401    }
1402}