nautilus_persistence/backend/parquet/
migration.rs1use arrow::record_batch::RecordBatch;
19use futures::StreamExt;
20use nautilus_core::UnixNanos;
21use nautilus_model::instruments::NautilusInstrumentType;
22use nautilus_serialization::arrow::record_batch_without_identifier_column;
23use object_store::{PutMode, PutOptions, path::Path as ObjectPath};
24
25use super::{
26 catalog::ParquetDataCatalog,
27 io::write_batches_to_object_store_create,
28 paths::{parse_filename_timestamps, timestamps_to_filename},
29};
30use crate::{
31 backend::migration::{
32 CatalogMigrationPlan, CatalogMigrationReport, IdentifierSource, ParquetCatalogSource,
33 PlannedMigrationFile, build_catalog_migration_plan, ensure_distinct_migration_locations,
34 ensure_planned_file_unchanged, prepare_migration_parts, read_planned_migration_file,
35 },
36 catalog::types::instrument_path_prefix,
37 common::{
38 metadata::record_batch_ts_init_range, paths::escape_local_file_uri,
39 storage::normalize_storage_location,
40 },
41};
42
43#[derive(Debug)]
45pub struct ParquetMigrationConfig {
46 pub source_uri: String,
47 pub target_uri: String,
48 pub source_options: Vec<(String, String)>,
49 pub target_options: Vec<(String, String)>,
50 pub dry_run: bool,
51}
52
53pub fn migrate_parquet_catalog(
63 config: ParquetMigrationConfig,
64) -> anyhow::Result<CatalogMigrationReport> {
65 let source_uri = normalize_storage_location(&config.source_uri)?;
66 let target_uri = normalize_storage_location(&config.target_uri)?;
67 ensure_distinct_migration_locations(&source_uri, &target_uri)?;
68 let source = ParquetDataCatalog::from_uri(
69 &source_uri,
70 Some(config.source_options.into_iter().collect()),
71 None,
72 None,
73 None,
74 )?;
75 let plan = build_catalog_migration_plan(&source)?;
76 plan.ensure_ready()?;
77 if config.dry_run {
78 return Ok(CatalogMigrationReport::from_plan(&plan, true));
79 }
80
81 if target_uri.starts_with("file://") {
82 let path = url::Url::parse(&escape_local_file_uri(&target_uri))?
83 .to_file_path()
84 .map_err(|()| anyhow::anyhow!("Invalid destination file URI"))?;
85 std::fs::create_dir_all(path)?;
86 }
87
88 let target = ParquetDataCatalog::from_uri(
89 &target_uri,
90 Some(config.target_options.into_iter().collect()),
91 None,
92 None,
93 None,
94 )?;
95 target.migrate_from_legacy_parquet_catalog_plan(&source, &plan)
96}
97
98impl ParquetCatalogSource for ParquetDataCatalog {
99 fn object_store(&self) -> std::sync::Arc<dyn object_store::ObjectStore> {
100 self.object_store.clone()
101 }
102 fn base_path(&self) -> &str {
103 &self.base_path
104 }
105 fn original_uri(&self) -> &str {
106 &self.original_uri
107 }
108
109 fn to_object_path_parsed(&self, path: &str) -> anyhow::Result<ObjectPath> {
110 Self::to_object_path_parsed(self, path)
111 }
112}
113
114impl ParquetDataCatalog {
115 pub fn migrate_from_legacy_parquet_catalog(
122 &self,
123 source: &Self,
124 ) -> anyhow::Result<CatalogMigrationReport> {
125 ensure_distinct_migration_locations(&source.original_uri, &self.original_uri)?;
126 let plan = build_catalog_migration_plan(source)?;
127 plan.ensure_ready()?;
128 self.migrate_from_legacy_parquet_catalog_plan(source, &plan)
129 }
130
131 fn migrate_from_legacy_parquet_catalog_plan(
132 &self,
133 source: &Self,
134 plan: &CatalogMigrationPlan,
135 ) -> anyhow::Result<CatalogMigrationReport> {
136 self.ensure_migration_target_empty()?;
137 let mut report = CatalogMigrationReport::from_plan(plan, false);
138
139 for file in &plan.files {
140 if file.size == 0 {
141 let source_path = source.to_object_path_parsed(&file.path)?;
142 ensure_planned_file_unchanged(source, file, &source_path)?;
143
144 let target_prefix = match file.target_type_name.as_str() {
145 "instruments" => file.source_type_name.clone(),
146 "custom" => file
147 .source_type_name
148 .strip_prefix("custom_")
149 .map_or_else(|| "custom".to_string(), |name| format!("custom/{name}")),
150 _ => file.target_type_name.clone(),
151 };
152
153 let relative_path = file.relative_path.replacen(
154 &format!("data/{}/", file.source_type_name),
155 &format!("data/{target_prefix}/"),
156 1,
157 );
158 let target_path = self.to_object_path_parsed(&relative_path)?;
159 self.execute_async(|| async {
160 self.object_store
161 .put_opts(
162 &target_path,
163 Vec::new().into(),
164 PutOptions {
165 mode: PutMode::Create,
166 ..Default::default()
167 },
168 )
169 .await?;
170
171 Ok(())
172 })?;
173
174 report.record_migrated_file(file, 0, 0);
175 continue;
176 }
177
178 let batches = read_planned_migration_file(source, file)?;
179 let (migrated_rows, path_identifier_rows) =
180 self.write_migration_batches(file, batches)?;
181 if migrated_rows == 0 {
182 report.record_skipped_file(file);
183 } else {
184 report.record_migrated_file(file, migrated_rows, path_identifier_rows);
185 }
186 }
187
188 Ok(report)
189 }
190
191 fn write_migration_batches(
192 &self,
193 file: &PlannedMigrationFile,
194 batches: Vec<RecordBatch>,
195 ) -> anyhow::Result<(usize, usize)> {
196 let mut migrated_rows = 0;
197 let mut path_identifier_rows = 0;
198
199 let parts = prepare_migration_parts(file, batches)?;
200 let coverage = parse_filename_timestamps(&file.relative_path);
201
202 for part in parts {
203 if part.row_count == 0 {
204 continue;
205 }
206
207 let directory = if let Some(custom_type_name) =
208 file.target_type_name.strip_prefix("custom/")
209 {
210 self.make_path_custom_data(custom_type_name, part.identifier.as_deref())?
211 } else {
212 let prefix = if let Some(class) = file.target_table.strip_prefix("instruments/") {
213 instrument_path_prefix(&class.parse::<NautilusInstrumentType>()?)
214 } else {
215 &file.target_type_name
216 };
217
218 self.make_path(prefix, part.identifier.as_deref())?
219 };
220
221 let (row_start, row_end) = record_batch_ts_init_range(&part.batches)?;
222 let (start_ts, end_ts) = coverage.unwrap_or((row_start, row_end));
223 anyhow::ensure!(
224 start_ts <= row_start && row_end <= end_ts,
225 "Parquet migration source coverage does not contain its rows: {}",
226 file.relative_path
227 );
228 let filename =
229 timestamps_to_filename(UnixNanos::from(start_ts), UnixNanos::from(end_ts));
230 let object_path = self.to_object_path(&format!("{directory}/{filename}"))?;
231 let batches = part
232 .batches
233 .into_iter()
234 .map(record_batch_without_identifier_column)
235 .collect::<Result<Vec<_>, _>>()?;
236
237 self.execute_async(|| async {
238 write_batches_to_object_store_create(
239 &batches,
240 self.object_store.clone(),
241 &object_path,
242 Some(self.compression),
243 Some(self.max_row_group_size),
244 None,
245 )
246 .await
247 })
248 .map_err(|e| {
249 anyhow::anyhow!(
250 "Parquet migration target object already exists or cannot be created: \
251 {object_path}: {e}"
252 )
253 })?;
254
255 migrated_rows += part.row_count;
256 if part.identifier_source == IdentifierSource::Path {
257 path_identifier_rows += part.row_count;
258 }
259 }
260
261 Ok((migrated_rows, path_identifier_rows))
262 }
263
264 fn ensure_migration_target_empty(&self) -> anyhow::Result<()> {
265 let prefix = (!self.base_path.is_empty()).then(|| ObjectPath::from(self.base_path.clone()));
266 self.execute_async(|| async {
267 let mut objects = self.object_store.list(prefix.as_ref());
268 anyhow::ensure!(
269 objects.next().await.transpose()?.is_none(),
270 "Parquet migration target must be new or empty",
271 );
272 Ok(())
273 })
274 }
275}