Skip to main content

nautilus_persistence/backend/parquet/
migration.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Legacy Parquet catalog rewrite into the current Parquet schema and layout.
17
18use arrow::record_batch::RecordBatch;
19use futures::StreamExt;
20use nautilus_core::UnixNanos;
21use nautilus_model::instruments::NautilusInstrumentType;
22use nautilus_serialization::arrow::record_batch_without_identifier_column;
23use object_store::{PutMode, PutOptions, path::Path as ObjectPath};
24
25use super::{
26    catalog::ParquetDataCatalog,
27    io::write_batches_to_object_store_create,
28    paths::{parse_filename_timestamps, timestamps_to_filename},
29};
30use crate::{
31    backend::migration::{
32        CatalogMigrationPlan, CatalogMigrationReport, IdentifierSource, ParquetCatalogSource,
33        PlannedMigrationFile, build_catalog_migration_plan, ensure_distinct_migration_locations,
34        ensure_planned_file_unchanged, prepare_migration_parts, read_planned_migration_file,
35    },
36    catalog::types::instrument_path_prefix,
37    common::{
38        metadata::record_batch_ts_init_range, paths::escape_local_file_uri,
39        storage::normalize_storage_location,
40    },
41};
42
43/// Settings for converting a legacy Parquet catalog into a separate native catalog.
44#[derive(Debug)]
45pub struct ParquetMigrationConfig {
46    pub source_uri: String,
47    pub target_uri: String,
48    pub source_options: Vec<(String, String)>,
49    pub target_options: Vec<(String, String)>,
50    pub dry_run: bool,
51}
52
53/// Converts the current or legacy Arrow representation of a Parquet catalog.
54///
55/// The source remains unchanged. The destination must be empty, and preflight rejects
56/// unsupported schemas before any destination files are written.
57///
58/// # Errors
59///
60/// Returns an error if locations overlap, schemas are invalid, the destination is nonempty, or an
61/// object-store read or write fails.
62pub fn migrate_parquet_catalog(
63    config: ParquetMigrationConfig,
64) -> anyhow::Result<CatalogMigrationReport> {
65    let source_uri = normalize_storage_location(&config.source_uri)?;
66    let target_uri = normalize_storage_location(&config.target_uri)?;
67    ensure_distinct_migration_locations(&source_uri, &target_uri)?;
68    let source = ParquetDataCatalog::from_uri(
69        &source_uri,
70        Some(config.source_options.into_iter().collect()),
71        None,
72        None,
73        None,
74    )?;
75    let plan = build_catalog_migration_plan(&source)?;
76    plan.ensure_ready()?;
77    if config.dry_run {
78        return Ok(CatalogMigrationReport::from_plan(&plan, true));
79    }
80
81    if target_uri.starts_with("file://") {
82        let path = url::Url::parse(&escape_local_file_uri(&target_uri))?
83            .to_file_path()
84            .map_err(|()| anyhow::anyhow!("Invalid destination file URI"))?;
85        std::fs::create_dir_all(path)?;
86    }
87
88    let target = ParquetDataCatalog::from_uri(
89        &target_uri,
90        Some(config.target_options.into_iter().collect()),
91        None,
92        None,
93        None,
94    )?;
95    target.migrate_from_legacy_parquet_catalog_plan(&source, &plan)
96}
97
98impl ParquetCatalogSource for ParquetDataCatalog {
99    fn object_store(&self) -> std::sync::Arc<dyn object_store::ObjectStore> {
100        self.object_store.clone()
101    }
102    fn base_path(&self) -> &str {
103        &self.base_path
104    }
105    fn original_uri(&self) -> &str {
106        &self.original_uri
107    }
108
109    fn to_object_path_parsed(&self, path: &str) -> anyhow::Result<ObjectPath> {
110        Self::to_object_path_parsed(self, path)
111    }
112}
113
114impl ParquetDataCatalog {
115    /// Rewrites a legacy Parquet catalog into this current Parquet catalog.
116    ///
117    /// # Errors
118    ///
119    /// Returns an error if source preflight fails, this catalog contains any leaf object, or a
120    /// source file cannot be read, converted, or written.
121    pub fn migrate_from_legacy_parquet_catalog(
122        &self,
123        source: &Self,
124    ) -> anyhow::Result<CatalogMigrationReport> {
125        ensure_distinct_migration_locations(&source.original_uri, &self.original_uri)?;
126        let plan = build_catalog_migration_plan(source)?;
127        plan.ensure_ready()?;
128        self.migrate_from_legacy_parquet_catalog_plan(source, &plan)
129    }
130
131    fn migrate_from_legacy_parquet_catalog_plan(
132        &self,
133        source: &Self,
134        plan: &CatalogMigrationPlan,
135    ) -> anyhow::Result<CatalogMigrationReport> {
136        self.ensure_migration_target_empty()?;
137        let mut report = CatalogMigrationReport::from_plan(plan, false);
138
139        for file in &plan.files {
140            if file.size == 0 {
141                let source_path = source.to_object_path_parsed(&file.path)?;
142                ensure_planned_file_unchanged(source, file, &source_path)?;
143
144                let target_prefix = match file.target_type_name.as_str() {
145                    "instruments" => file.source_type_name.clone(),
146                    "custom" => file
147                        .source_type_name
148                        .strip_prefix("custom_")
149                        .map_or_else(|| "custom".to_string(), |name| format!("custom/{name}")),
150                    _ => file.target_type_name.clone(),
151                };
152
153                let relative_path = file.relative_path.replacen(
154                    &format!("data/{}/", file.source_type_name),
155                    &format!("data/{target_prefix}/"),
156                    1,
157                );
158                let target_path = self.to_object_path_parsed(&relative_path)?;
159                self.execute_async(|| async {
160                    self.object_store
161                        .put_opts(
162                            &target_path,
163                            Vec::new().into(),
164                            PutOptions {
165                                mode: PutMode::Create,
166                                ..Default::default()
167                            },
168                        )
169                        .await?;
170
171                    Ok(())
172                })?;
173
174                report.record_migrated_file(file, 0, 0);
175                continue;
176            }
177
178            let batches = read_planned_migration_file(source, file)?;
179            let (migrated_rows, path_identifier_rows) =
180                self.write_migration_batches(file, batches)?;
181            if migrated_rows == 0 {
182                report.record_skipped_file(file);
183            } else {
184                report.record_migrated_file(file, migrated_rows, path_identifier_rows);
185            }
186        }
187
188        Ok(report)
189    }
190
191    fn write_migration_batches(
192        &self,
193        file: &PlannedMigrationFile,
194        batches: Vec<RecordBatch>,
195    ) -> anyhow::Result<(usize, usize)> {
196        let mut migrated_rows = 0;
197        let mut path_identifier_rows = 0;
198
199        let parts = prepare_migration_parts(file, batches)?;
200        let coverage = parse_filename_timestamps(&file.relative_path);
201
202        for part in parts {
203            if part.row_count == 0 {
204                continue;
205            }
206
207            let directory = if let Some(custom_type_name) =
208                file.target_type_name.strip_prefix("custom/")
209            {
210                self.make_path_custom_data(custom_type_name, part.identifier.as_deref())?
211            } else {
212                let prefix = if let Some(class) = file.target_table.strip_prefix("instruments/") {
213                    instrument_path_prefix(&class.parse::<NautilusInstrumentType>()?)
214                } else {
215                    &file.target_type_name
216                };
217
218                self.make_path(prefix, part.identifier.as_deref())?
219            };
220
221            let (row_start, row_end) = record_batch_ts_init_range(&part.batches)?;
222            let (start_ts, end_ts) = coverage.unwrap_or((row_start, row_end));
223            anyhow::ensure!(
224                start_ts <= row_start && row_end <= end_ts,
225                "Parquet migration source coverage does not contain its rows: {}",
226                file.relative_path
227            );
228            let filename =
229                timestamps_to_filename(UnixNanos::from(start_ts), UnixNanos::from(end_ts));
230            let object_path = self.to_object_path(&format!("{directory}/{filename}"))?;
231            let batches = part
232                .batches
233                .into_iter()
234                .map(record_batch_without_identifier_column)
235                .collect::<Result<Vec<_>, _>>()?;
236
237            self.execute_async(|| async {
238                write_batches_to_object_store_create(
239                    &batches,
240                    self.object_store.clone(),
241                    &object_path,
242                    Some(self.compression),
243                    Some(self.max_row_group_size),
244                    None,
245                )
246                .await
247            })
248            .map_err(|e| {
249                anyhow::anyhow!(
250                    "Parquet migration target object already exists or cannot be created: \
251                     {object_path}: {e}"
252                )
253            })?;
254
255            migrated_rows += part.row_count;
256            if part.identifier_source == IdentifierSource::Path {
257                path_identifier_rows += part.row_count;
258            }
259        }
260
261        Ok((migrated_rows, path_identifier_rows))
262    }
263
264    fn ensure_migration_target_empty(&self) -> anyhow::Result<()> {
265        let prefix = (!self.base_path.is_empty()).then(|| ObjectPath::from(self.base_path.clone()));
266        self.execute_async(|| async {
267            let mut objects = self.object_store.list(prefix.as_ref());
268            anyhow::ensure!(
269                objects.next().await.transpose()?.is_none(),
270                "Parquet migration target must be new or empty",
271            );
272            Ok(())
273        })
274    }
275}