Skip to main content

nautilus_persistence/backend/
migration.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Target-neutral planning and reading for legacy Parquet catalog migration.
17
18use std::{
19    collections::{BTreeMap, HashMap},
20    fmt::{Debug, Display},
21    sync::Arc,
22};
23
24use ahash::AHashSet;
25use arrow::{
26    array::{
27        Array, ArrayRef, BinaryArray, Decimal128Array, FixedSizeListArray, ListArray,
28        StringBuilder, StructArray, UInt32Array, UInt64Array,
29    },
30    buffer::{OffsetBuffer, ScalarBuffer},
31    compute::{cast, take_record_batch},
32    datatypes::{DataType as ArrowDataType, Field, Fields, Schema, TimeUnit},
33    record_batch::RecordBatch,
34};
35use futures::{StreamExt, TryStreamExt};
36use nautilus_model::{
37    data::{
38        Bar, FundingRateUpdate, IndexPriceUpdate, InstrumentClose, InstrumentStatus,
39        MarkPriceUpdate, NautilusDataType, NautilusRecordType, OptionGreeks, OrderBookDelta,
40        OrderBookDepth, QuoteTick, TradeTick, depth::DEPTH10_LEN,
41    },
42    instruments::{InstrumentAny, NautilusInstrumentType},
43};
44use nautilus_serialization::arrow::{
45    ArrowSchemaProvider, EncodeToRecordBatch, KEY_BAR_TYPE, KEY_IDENTIFIER, KEY_INSTRUMENT_ID,
46    KEY_TYPE_NAME, StringColumnRef,
47    instrument::decode_instrument_any_batch,
48    legacy::{
49        LegacyArrowError, LegacySchemaResolution, LegacyTranscodeKind, LegacyTranscodeState,
50        SchemaFingerprint, is_nautilus_legacy_schema, is_nautilus_timestamp_schema,
51        normalize_legacy_fixed_columns,
52        normalized_legacy_data_type as normalize_legacy_arrow_data_type, normalized_timestamp_type,
53        resolve_legacy_schema, schema_fingerprint, transcode_legacy_record_batch_with_state,
54    },
55    record_batch_with_identifier_column, schema_without_identifier_column,
56};
57use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt, path::Path as ObjectPath};
58use parquet::{read_parquet_from_object_store, read_parquet_schema_from_object_store};
59use serde::Serialize;
60use strum::IntoEnumIterator;
61
62use crate::{
63    backend::parquet::io as parquet,
64    catalog::types::{
65        INSTRUMENT_PATH_PREFIXES, data_path_prefix, data_type_from_data_path_prefix,
66        record_path_prefix,
67    },
68    common::{
69        arrow::{catalog_record_schema, round_trip_batches, round_trip_catalog_record_batches},
70        paths::normalize_path_separators,
71        storage::normalize_storage_location,
72    },
73};
74
75const SCHEMA_READ_CONCURRENCY: usize = 16;
76const LEGACY_KEY_CLASS: &str = "class";
77
78/// Default maximum number of source rows written in one open-catalog migration commit.
79pub const DEFAULT_MIGRATION_COMMIT_ROWS: usize = 500_000;
80
81/// Object-store source used to plan and read a legacy Parquet migration.
82pub trait ParquetCatalogSource: Sync {
83    /// Returns the object store containing the source catalog.
84    fn object_store(&self) -> Arc<dyn ObjectStore>;
85    /// Returns the catalog path within the object store.
86    fn base_path(&self) -> &str;
87    /// Returns the URI identifying the source catalog.
88    fn original_uri(&self) -> &str;
89
90    /// Resolves a plan-relative path within the source catalog.
91    ///
92    /// # Errors
93    ///
94    /// Returns an error if the combined object-store path is invalid.
95    fn to_object_path_parsed(&self, path: &str) -> anyhow::Result<ObjectPath> {
96        let normalized = normalize_path_separators(path);
97        let base = self.base_path().trim_matches('/');
98
99        let full = if base.is_empty() {
100            normalized
101        } else {
102            format!("{base}/{}", normalized.trim_start_matches('/'))
103        };
104
105        ObjectPath::parse(full.trim_start_matches('/')).map_err(anyhow::Error::from)
106    }
107}
108
109/// Migration counters for one current target type.
110#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)]
111pub struct CatalogMigrationTypeReport {
112    pub planned_files: usize,
113    pub migrated_files: usize,
114    pub skipped_files: usize,
115    pub migrated_rows: usize,
116    pub transcoded_rows: usize,
117    pub path_identifier_rows: usize,
118}
119
120/// Structured result shared by Parquet and Delta migration entry points.
121#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)]
122pub struct CatalogMigrationReport {
123    pub dry_run: bool,
124    pub total_leaf_files: usize,
125    pub migrated_files: usize,
126    pub skipped_files: usize,
127    pub migrated_rows: usize,
128    pub transcoded_rows: usize,
129    pub path_identifier_rows: usize,
130    pub unmigrated: Vec<UnmigratedFile>,
131    pub types: BTreeMap<String, CatalogMigrationTypeReport>,
132}
133
134impl CatalogMigrationReport {
135    /// Builds a report with preflight file accounting and no writes.
136    #[must_use]
137    pub fn from_plan(plan: &CatalogMigrationPlan, dry_run: bool) -> Self {
138        let mut types = BTreeMap::new();
139        for file in &plan.files {
140            types
141                .entry(file.target_type_name.clone())
142                .or_insert_with(CatalogMigrationTypeReport::default)
143                .planned_files += 1;
144        }
145
146        Self {
147            dry_run,
148            total_leaf_files: plan.total_leaf_files,
149            unmigrated: plan.unmigrated.clone(),
150            types,
151            ..Self::default()
152        }
153    }
154
155    pub(crate) fn record_migrated_file(
156        &mut self,
157        file: &PlannedMigrationFile,
158        rows: usize,
159        path_identifier_rows: usize,
160    ) {
161        let transcoded_rows = if file.transcode_kind == LegacyTranscodeKind::PassThrough {
162            0
163        } else {
164            rows
165        };
166
167        self.migrated_files += 1;
168        self.migrated_rows += rows;
169        self.transcoded_rows += transcoded_rows;
170        self.path_identifier_rows += path_identifier_rows;
171        let type_report = self.types.entry(file.target_type_name.clone()).or_default();
172        type_report.migrated_files += 1;
173        type_report.migrated_rows += rows;
174        type_report.transcoded_rows += transcoded_rows;
175        type_report.path_identifier_rows += path_identifier_rows;
176    }
177
178    pub(crate) fn record_skipped_file(&mut self, file: &PlannedMigrationFile) {
179        self.skipped_files += 1;
180        self.types
181            .entry(file.target_type_name.clone())
182            .or_default()
183            .skipped_files += 1;
184    }
185}
186
187impl Display for CatalogMigrationReport {
188    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
189        writeln!(
190            f,
191            "{}: {} planned leaf files, {} migrated files, {} migrated rows, {} skipped files, {} \
192             unmigrated files",
193            if self.dry_run {
194                "Migration dry-run report"
195            } else {
196                "Migration report"
197            },
198            self.total_leaf_files,
199            self.migrated_files,
200            self.migrated_rows,
201            self.skipped_files,
202            self.unmigrated.len(),
203        )?;
204
205        for (type_name, report) in &self.types {
206            writeln!(
207                f,
208                "{type_name}: {} planned files, {} migrated files, {} rows, {} transcoded rows, {} \
209                 path-derived identifier rows, {} skipped files",
210                report.planned_files,
211                report.migrated_files,
212                report.migrated_rows,
213                report.transcoded_rows,
214                report.path_identifier_rows,
215                report.skipped_files,
216            )?;
217        }
218
219        let mut unmigrated_directories = BTreeMap::new();
220
221        for file in &self.unmigrated {
222            let directory = file
223                .path
224                .rsplit_once('/')
225                .map_or(".", |(directory, _)| directory);
226            *unmigrated_directories.entry(directory).or_insert(0_usize) += 1;
227        }
228
229        for (directory, file_count) in unmigrated_directories {
230            writeln!(f, "Unmigrated directory {directory}: {file_count} files")?;
231        }
232
233        for file in &self.unmigrated {
234            writeln!(f, "Unmigrated {}: {}", file.path, file.reason)?;
235        }
236
237        Ok(())
238    }
239}
240
241/// One source file accepted by migration preflight.
242#[derive(Clone, Debug)]
243pub struct PlannedMigrationFile {
244    pub path: String,
245    pub relative_path: String,
246    pub source_type_name: String,
247    pub target_type_name: String,
248    pub target_table: String,
249    pub size: u64,
250    pub e_tag: Option<String>,
251    pub version: Option<String>,
252    pub last_modified: String,
253    pub source_fingerprint: SchemaFingerprint,
254    pub target_fingerprint: SchemaFingerprint,
255    pub transcode_kind: LegacyTranscodeKind,
256}
257
258/// One source path excluded from migration.
259#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
260pub struct UnmigratedFile {
261    pub path: String,
262    pub reason: String,
263}
264
265/// One schema and example source path in a target-table conflict.
266#[derive(Clone, Debug, Eq, PartialEq)]
267pub struct SchemaConflictExample {
268    pub fingerprint: SchemaFingerprint,
269    pub path: String,
270}
271
272/// Incompatible final schemas targeting one output table.
273#[derive(Clone, Debug, Eq, PartialEq)]
274pub struct SchemaConflict {
275    pub target_table: String,
276    pub examples: Vec<SchemaConflictExample>,
277}
278
279/// One source file whose schema needs an unregistered transcoder.
280#[derive(Clone, Debug, Eq, PartialEq)]
281pub struct UnresolvedSchema {
282    pub path: String,
283    pub message: String,
284}
285
286/// Complete read-only result of source discovery and schema preflight.
287#[derive(Clone, Debug, Default)]
288pub struct CatalogMigrationPlan {
289    pub files: Vec<PlannedMigrationFile>,
290    pub unmigrated: Vec<UnmigratedFile>,
291    pub conflicts: Vec<SchemaConflict>,
292    pub unresolved_schemas: Vec<UnresolvedSchema>,
293    pub total_leaf_files: usize,
294}
295
296impl CatalogMigrationPlan {
297    /// Returns whether preflight found a condition that prevents writing.
298    #[must_use]
299    pub const fn has_errors(&self) -> bool {
300        !self.conflicts.is_empty() || !self.unresolved_schemas.is_empty()
301    }
302
303    /// Rejects a plan that cannot be written safely.
304    ///
305    /// # Errors
306    ///
307    /// Returns an error if any schema conflicts or unresolved schemas remain. The error summarizes
308    /// every schema conflict and unresolved schema.
309    pub fn ensure_ready(&self) -> anyhow::Result<()> {
310        if !self.has_errors() {
311            return Ok(());
312        }
313
314        let mut messages = self
315            .conflicts
316            .iter()
317            .map(|conflict| {
318                let examples = conflict
319                    .examples
320                    .iter()
321                    .map(|example| format!("{} ({})", example.path, example.fingerprint))
322                    .collect::<Vec<_>>()
323                    .join(", ");
324                format!(
325                    "target table {} has conflicting schemas: {examples}",
326                    conflict.target_table
327                )
328            })
329            .collect::<Vec<_>>();
330
331        messages.extend(
332            self.unresolved_schemas
333                .iter()
334                .map(|schema| schema.message.clone()),
335        );
336        anyhow::bail!(
337            "Catalog migration preflight failed:\n{}",
338            messages.join("\n")
339        );
340    }
341}
342
343/// How a migrated identifier was resolved.
344#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
345pub enum IdentifierSource {
346    Metadata,
347    Row,
348    Path,
349    Absent,
350}
351
352/// Batches from one source file that share one resolved identifier.
353#[derive(Debug)]
354pub struct PreparedMigrationPart {
355    pub identifier: Option<String>,
356    pub identifier_source: IdentifierSource,
357    pub batches: Vec<RecordBatch>,
358    pub row_count: usize,
359}
360
361/// Enumerates and resolves every source leaf file without writing a target.
362///
363/// # Errors
364///
365/// Returns an error if source listing or Parquet schema reading fails.
366pub fn build_catalog_migration_plan(
367    source: &dyn ParquetCatalogSource,
368) -> anyhow::Result<CatalogMigrationPlan> {
369    let objects = list_source_objects(source)?;
370    let total_leaf_files = objects.len();
371    let mut candidates = Vec::new();
372    let mut unmigrated = Vec::new();
373
374    for object in objects {
375        let relative_path = relative_object_path(source, &object.location);
376        match classify_source_path(&relative_path) {
377            SourceClassification::Migratable {
378                source_type_name,
379                target_type_name,
380            } if relative_path.ends_with(".parquet") => {
381                candidates.push(SchemaCandidate {
382                    object_path: object.location.clone(),
383                    object,
384                    relative_path,
385                    source_type_name,
386                    target_type_name,
387                });
388            }
389            SourceClassification::Migratable { .. } => {
390                unmigrated.push(UnmigratedFile {
391                    path: relative_path,
392                    reason: "recognized catalog directory contains a non-Parquet leaf".to_string(),
393                });
394            }
395            SourceClassification::Unmigrated(reason) => {
396                unmigrated.push(UnmigratedFile {
397                    path: relative_path,
398                    reason,
399                });
400            }
401        }
402    }
403
404    let (mut files, unresolved_schemas) = resolve_candidate_schemas(source, candidates)?;
405    files.sort_by(|left, right| left.path.cmp(&right.path));
406    unmigrated.sort_by(|left, right| left.path.cmp(&right.path));
407    let conflicts = schema_conflicts(&files);
408
409    Ok(CatalogMigrationPlan {
410        files,
411        unmigrated,
412        conflicts,
413        unresolved_schemas,
414        total_leaf_files,
415    })
416}
417
418/// Reads, normalizes, and transcodes one planned source file.
419///
420/// # Errors
421///
422/// Returns an error if the source file cannot be read, normalized, or transcoded.
423pub fn read_planned_migration_file(
424    source: &dyn ParquetCatalogSource,
425    file: &PlannedMigrationFile,
426) -> anyhow::Result<Vec<RecordBatch>> {
427    let object_path = source.to_object_path_parsed(&file.path)?;
428
429    let (batches, schema) = execute_async(|| async {
430        read_parquet_from_object_store(source.object_store(), &object_path).await
431    })?;
432
433    ensure_planned_file_unchanged(source, file, &object_path)?;
434    let mut state = LegacyTranscodeState::default();
435    let mut transcoded = Vec::new();
436
437    for batch in record_batches_with_schema(batches, &schema)? {
438        let batch = with_inferred_custom_type_name(file, batch)?;
439        let batch = normalize_legacy_parquet_columns(&batch)?;
440        let result = transcode_legacy_record_batch_with_state(
441            &file.target_type_name,
442            &file.path,
443            batch,
444            &mut state,
445        )?;
446        transcoded.extend(result.batches);
447    }
448
449    Ok(transcoded)
450}
451
452/// Resolves identifiers, re-encodes built-in batches with the current encoders, and groups
453/// batches from one source file.
454///
455/// # Errors
456///
457/// Returns an error if an identifier column is not string-like, a batch cannot be sliced or rebuilt
458/// with current identifier metadata, or a built-in batch fails its current decoder.
459pub fn prepare_migration_parts(
460    file: &PlannedMigrationFile,
461    batches: Vec<RecordBatch>,
462) -> anyhow::Result<Vec<PreparedMigrationPart>> {
463    let mut grouped: BTreeMap<(Option<String>, IdentifierSource), Vec<RecordBatch>> =
464        BTreeMap::new();
465
466    for batch in batches {
467        for (identifier, source, batch) in split_batch_by_identifier(file, batch)? {
468            if batch.num_rows() == 0 {
469                continue;
470            }
471
472            let batch =
473                batch_with_identifier(&file.target_type_name, identifier.as_deref(), batch)?;
474            grouped.entry((identifier, source)).or_default().push(batch);
475        }
476    }
477
478    grouped
479        .into_iter()
480        .map(|((identifier, identifier_source), batches)| {
481            let row_count = batches.iter().map(RecordBatch::num_rows).sum();
482            let batches = round_trip_migrated_batches(&file.target_type_name, batches)?;
483            Ok(PreparedMigrationPart {
484                identifier,
485                identifier_source,
486                batches,
487                row_count,
488            })
489        })
490        .collect()
491}
492
493/// Parses a storage option expressed as `key=value`.
494///
495/// # Errors
496///
497/// Returns an error if the separator or either side is absent.
498pub fn parse_storage_option(option: &str) -> Result<(String, String), String> {
499    let (key, value) = option
500        .split_once('=')
501        .ok_or_else(|| format!("Storage option must use key=value: {option}"))?;
502
503    if key.is_empty() || value.is_empty() {
504        return Err(format!(
505            "Storage option must use non-empty key=value: {option}"
506        ));
507    }
508
509    Ok((key.to_string(), value.to_string()))
510}
511
512pub(crate) fn ensure_distinct_migration_locations(
513    source_uri: &str,
514    target_uri: &str,
515) -> anyhow::Result<()> {
516    let source_uri = normalize_storage_location(source_uri)?;
517    let target_uri = normalize_storage_location(target_uri)?;
518    let source_uri = source_uri.trim_end_matches('/');
519    let target_uri = target_uri.trim_end_matches('/');
520    anyhow::ensure!(
521        source_uri != target_uri
522            && !target_uri.starts_with(&format!("{source_uri}/"))
523            && !source_uri.starts_with(&format!("{target_uri}/")),
524        "Migration source and target must be distinct, non-overlapping locations",
525    );
526    Ok(())
527}
528
529fn list_source_objects(source: &dyn ParquetCatalogSource) -> anyhow::Result<Vec<ObjectMeta>> {
530    let prefix =
531        (!source.base_path().is_empty()).then(|| ObjectPath::from(source.base_path().to_string()));
532
533    let mut objects = execute_async(|| async {
534        Ok(source
535            .object_store()
536            .list(prefix.as_ref())
537            .try_collect::<Vec<_>>()
538            .await?)
539    })?;
540
541    // The OpenDAL filesystem adapter lists directory entries alongside leaves; an entry
542    // that is the parent of another listed entry is a directory, not a migratable leaf.
543    objects.sort_by(|left, right| left.location.as_ref().cmp(right.location.as_ref()));
544
545    let parents = objects
546        .windows(2)
547        .filter(|pair| {
548            pair[1]
549                .location
550                .as_ref()
551                .starts_with(&format!("{}/", pair[0].location.as_ref()))
552        })
553        .map(|pair| pair[0].location.clone())
554        .collect::<AHashSet<_>>();
555
556    objects.retain(|object| {
557        !parents.contains(&object.location)
558            && !relative_object_path(source, &object.location).is_empty()
559    });
560
561    Ok(objects)
562}
563
564fn execute_async<C, F, R>(create_future: C) -> anyhow::Result<R>
565where
566    C: FnOnce() -> F + Send,
567    F: std::future::Future<Output = anyhow::Result<R>>,
568    R: Send,
569{
570    nautilus_common::live::block_on_nautilus_with(create_future)
571}
572
573fn relative_object_path(source: &dyn ParquetCatalogSource, path: &ObjectPath) -> String {
574    let path = path.as_ref();
575    let base = source.base_path().trim_matches('/');
576    if base.is_empty() {
577        return path.to_string();
578    }
579
580    path.strip_prefix(&format!("{base}/"))
581        .unwrap_or(path)
582        .to_string()
583}
584
585#[derive(Clone, Debug)]
586enum SourceClassification {
587    Migratable {
588        source_type_name: String,
589        target_type_name: String,
590    },
591    Unmigrated(String),
592}
593
594fn classify_source_path(path: &str) -> SourceClassification {
595    let parts = path.split('/').collect::<Vec<_>>();
596
597    let Some(root) = parts.first().copied() else {
598        return SourceClassification::Unmigrated("empty source path".to_string());
599    };
600
601    if matches!(root, "backtest" | "live") {
602        return SourceClassification::Unmigrated(format!(
603            "{root} Feather trees are outside catalog migration scope"
604        ));
605    }
606
607    if root != "data" || parts.len() < 2 {
608        return SourceClassification::Unmigrated(
609            "path is outside the recognized data catalog tree".to_string(),
610        );
611    }
612
613    let source_type_name = parts[1];
614    if INSTRUMENT_PATH_PREFIXES.contains(&source_type_name) {
615        return SourceClassification::Migratable {
616            source_type_name: source_type_name.to_string(),
617            target_type_name: "instruments".to_string(),
618        };
619    }
620
621    if source_type_name == "custom" || source_type_name.starts_with("custom_") {
622        return SourceClassification::Migratable {
623            source_type_name: source_type_name.to_string(),
624            target_type_name: "custom".to_string(),
625        };
626    }
627
628    // v1 catalogs named the depth directory after the removed `OrderBookDepth10` type
629    let data_type = match source_type_name {
630        "order_book_depth10" => Ok(NautilusDataType::OrderBookDepth),
631        _ => data_type_from_data_path_prefix(source_type_name),
632    };
633
634    if let Ok(data_type) = data_type {
635        return SourceClassification::Migratable {
636            source_type_name: source_type_name.to_string(),
637            target_type_name: data_path_prefix(&data_type).into_owned(),
638        };
639    }
640
641    if NautilusRecordType::iter()
642        .any(|record_type| record_path_prefix(&record_type).as_ref() == source_type_name)
643    {
644        return SourceClassification::Migratable {
645            source_type_name: source_type_name.to_string(),
646            target_type_name: source_type_name.to_string(),
647        };
648    }
649
650    let reason = if source_type_name == "portfolio_snapshot" {
651        "no NautilusRecordType maps to data/portfolio_snapshot"
652    } else {
653        "unrecognized catalog data directory"
654    };
655
656    SourceClassification::Unmigrated(reason.to_string())
657}
658
659#[derive(Clone, Debug)]
660struct SchemaCandidate {
661    object: ObjectMeta,
662    relative_path: String,
663    source_type_name: String,
664    target_type_name: String,
665    object_path: ObjectPath,
666}
667
668#[expect(
669    clippy::too_many_lines,
670    reason = "Preflight resolves every candidate before assembling the migration plan"
671)]
672fn resolve_candidate_schemas(
673    source: &dyn ParquetCatalogSource,
674    candidates: Vec<SchemaCandidate>,
675) -> anyhow::Result<(Vec<PlannedMigrationFile>, Vec<UnresolvedSchema>)> {
676    let object_store = source.object_store();
677
678    let resolved = execute_async(|| async move {
679        futures::stream::iter(candidates)
680            .map(|candidate| {
681                let object_store = object_store.clone();
682                async move {
683                    let schema =
684                        read_parquet_schema_from_object_store(object_store, &candidate.object_path)
685                            .await?;
686                    Ok::<_, anyhow::Error>((candidate, schema))
687                }
688            })
689            .buffer_unordered(SCHEMA_READ_CONCURRENCY)
690            .try_collect::<Vec<_>>()
691            .await
692    })?;
693
694    let mut files = Vec::new();
695    let mut unresolved = Vec::new();
696
697    for (mut candidate, schema) in resolved {
698        if candidate.object.size == 0 {
699            // Mirror the data-file inference so markers land beside their data files
700            if candidate.target_type_name == "custom"
701                && let Some(inferred) = legacy_custom_type_name(&candidate.source_type_name)
702            {
703                candidate.target_type_name = format!("custom/{inferred}");
704            }
705
706            let fingerprint = schema_fingerprint(&Schema::empty());
707            files.push(ResolvedCandidate {
708                target_table: candidate.target_type_name.clone(),
709                candidate,
710                resolution: LegacySchemaResolution {
711                    kind: LegacyTranscodeKind::PassThrough,
712                    source_fingerprint: fingerprint.clone(),
713                    target_fingerprint: fingerprint,
714                },
715            });
716
717            continue;
718        }
719
720        // Resolve the custom target before normalization, so the preflight fingerprint
721        // reflects the same type_name the execution path injects.
722        if candidate.target_type_name == "custom" {
723            if let Some(type_name) = schema.metadata().get("type_name") {
724                candidate.target_type_name = format!("custom/{type_name}");
725            } else if let Some(inferred) = legacy_custom_type_name(&candidate.source_type_name) {
726                candidate.target_type_name = format!("custom/{inferred}");
727            } else {
728                unresolved.push(UnresolvedSchema {
729                    path: candidate.relative_path.clone(),
730                    message: format!(
731                        "Parquet custom data file {} is missing type_name metadata",
732                        candidate.relative_path
733                    ),
734                });
735
736                continue;
737            }
738        }
739
740        // Custom targets skip fingerprint validation, so reject unconvertible
741        // timestamps here instead of migrating to an unreadable destination.
742        if candidate.target_type_name.starts_with("custom/")
743            && !custom_timestamps_convertible(&schema)
744        {
745            let detail = if schema.metadata().contains_key("type_name") {
746                "has non-UInt64 timestamps with no transcoder"
747            } else {
748                "is missing type_name metadata and has non-UInt64 timestamps with no transcoder"
749            };
750
751            unresolved.push(UnresolvedSchema {
752                path: candidate.relative_path.clone(),
753                message: format!(
754                    "Parquet custom data file {} {detail}",
755                    candidate.relative_path
756                ),
757            });
758
759            continue;
760        }
761
762        let schema = with_target_custom_type_name(&candidate, &schema);
763        let schema = normalize_legacy_parquet_schema(&schema);
764
765        let target_table = if candidate.target_type_name == "instruments" {
766            let Some(instrument_type) = schema.metadata().get(KEY_TYPE_NAME) else {
767                unresolved.push(UnresolvedSchema {
768                    path: candidate.relative_path.clone(),
769                    message: format!(
770                        "Parquet instrument file {} is missing type_name metadata",
771                        candidate.relative_path
772                    ),
773                });
774
775                continue;
776            };
777
778            format!("instruments/{instrument_type}")
779        } else {
780            candidate.target_type_name.clone()
781        };
782
783        if let Ok(record_type) = candidate.target_type_name.parse::<NautilusRecordType>()
784            && let Ok(current) = catalog_record_schema(record_type)
785        {
786            let expected = schema_without_identifier_column(&current);
787            let actual = schema_without_identifier_column(&schema);
788
789            if schema_fingerprint(&actual) != schema_fingerprint(&expected) {
790                unresolved.push(UnresolvedSchema {
791                    path: candidate.relative_path.clone(),
792                    message: format!(
793                        "Record file {} does not match the registered Arrow schema",
794                        candidate.relative_path
795                    ),
796                });
797
798                continue;
799            }
800        }
801
802        if schema
803            .fields()
804            .iter()
805            .any(|field| contains_legacy_fixed_binary(field.data_type()))
806        {
807            unresolved.push(UnresolvedSchema {
808                path: candidate.relative_path.clone(),
809                message: format!(
810                    "No final-format transcoder is registered for fixed binary columns in {}",
811                    candidate.relative_path
812                ),
813            });
814
815            continue;
816        }
817
818        match resolve_legacy_schema(
819            &candidate.target_type_name,
820            &candidate.relative_path,
821            &schema,
822        ) {
823            Ok(resolution) => files.push(ResolvedCandidate {
824                candidate,
825                target_table,
826                resolution,
827            }),
828            Err(error @ LegacyArrowError::UnknownSchema { .. }) => {
829                unresolved.push(UnresolvedSchema {
830                    path: candidate.relative_path,
831                    message: error.to_string(),
832                });
833            }
834            Err(e) => return Err(e.into()),
835        }
836    }
837
838    let files = files
839        .into_iter()
840        .map(|resolved| PlannedMigrationFile {
841            path: resolved.candidate.object.location.to_string(),
842            relative_path: resolved.candidate.relative_path,
843            source_type_name: resolved.candidate.source_type_name,
844            target_type_name: resolved.candidate.target_type_name,
845            target_table: resolved.target_table,
846            size: resolved.candidate.object.size,
847            e_tag: resolved.candidate.object.e_tag,
848            version: resolved.candidate.object.version,
849            last_modified: resolved.candidate.object.last_modified.to_rfc3339(),
850            source_fingerprint: resolved.resolution.source_fingerprint,
851            target_fingerprint: resolved.resolution.target_fingerprint,
852            transcode_kind: resolved.resolution.kind,
853        })
854        .collect();
855
856    Ok((files, unresolved))
857}
858
859#[derive(Debug)]
860struct ResolvedCandidate {
861    candidate: SchemaCandidate,
862    target_table: String,
863    resolution: LegacySchemaResolution,
864}
865
866/// Infers a custom type name from a legacy `custom_<snake_case>` directory.
867///
868/// Old Python-written catalogs stored custom data under `data/custom_<snake_case>` without
869/// `type_name` schema metadata. Best-effort reversal to PascalCase; acronyms do not survive
870/// the round trip, but the known legacy layouts (e.g. `custom_binance_bar` -> `BinanceBar`)
871/// map exactly. Returns `None` for the canonical `custom` directory, which carries no name.
872fn legacy_custom_type_name(source_type_name: &str) -> Option<String> {
873    let legacy = source_type_name.strip_prefix("custom_")?;
874    let mut pascal = String::with_capacity(legacy.len());
875    for part in legacy.split('_') {
876        let mut chars = part.chars();
877        if let Some(first) = chars.next() {
878            pascal.extend(first.to_uppercase());
879            pascal.extend(chars);
880        }
881    }
882
883    (!pascal.is_empty()).then_some(pascal)
884}
885
886/// Returns true when a custom schema can migrate: timestamp normalization converts
887/// `UInt64` `ts_event`/`ts_init` and passes timestamps through, so any other
888/// physical type (notably legacy `Int64`) has no transcoder.
889fn custom_timestamps_convertible(schema: &Schema) -> bool {
890    ["ts_event", "ts_init"].iter().all(|name| {
891        schema.field_with_name(name).is_ok_and(|field| {
892            matches!(
893                field.data_type(),
894                ArrowDataType::UInt64 | ArrowDataType::Timestamp(TimeUnit::Nanosecond, _)
895            )
896        })
897    })
898}
899
900/// Attaches the planned custom `type_name` to a preflight schema that lacks it,
901/// mirroring the execution-time injection so fingerprints match written output.
902fn with_target_custom_type_name(candidate: &SchemaCandidate, schema: &Schema) -> Schema {
903    let Some(type_name) = candidate.target_type_name.strip_prefix("custom/") else {
904        return schema.clone();
905    };
906
907    inject_type_name_metadata(schema, type_name)
908}
909
910/// Attaches a custom `type_name` to a schema that lacks it, leaving other
911/// schemas untouched.
912fn inject_type_name_metadata(schema: &Schema, type_name: &str) -> Schema {
913    let mut schema = schema.clone();
914    schema
915        .metadata
916        .entry("type_name".to_string())
917        .or_insert_with(|| type_name.to_string());
918    schema
919}
920
921fn contains_legacy_fixed_binary(data_type: &ArrowDataType) -> bool {
922    match data_type {
923        ArrowDataType::FixedSizeBinary(_) => true,
924        ArrowDataType::List(field)
925        | ArrowDataType::LargeList(field)
926        | ArrowDataType::FixedSizeList(field, _)
927        | ArrowDataType::Map(field, _) => contains_legacy_fixed_binary(field.data_type()),
928        ArrowDataType::Struct(fields) => fields
929            .iter()
930            .any(|field| contains_legacy_fixed_binary(field.data_type())),
931        ArrowDataType::Dictionary(_, value) => contains_legacy_fixed_binary(value),
932        _ => false,
933    }
934}
935
936pub(crate) fn ensure_planned_file_unchanged(
937    source: &dyn ParquetCatalogSource,
938    file: &PlannedMigrationFile,
939    object_path: &ObjectPath,
940) -> anyhow::Result<()> {
941    let object = execute_async(|| async { Ok(source.object_store().head(object_path).await?) })?;
942    anyhow::ensure!(
943        object.size == file.size
944            && object.e_tag == file.e_tag
945            && object.version == file.version
946            && object.last_modified.to_rfc3339() == file.last_modified,
947        "Source file changed after migration preflight: {}",
948        file.relative_path,
949    );
950    Ok(())
951}
952
953fn schema_conflicts(files: &[PlannedMigrationFile]) -> Vec<SchemaConflict> {
954    let mut by_table: BTreeMap<String, BTreeMap<String, SchemaConflictExample>> = BTreeMap::new();
955    for file in files.iter().filter(|file| file.size != 0) {
956        by_table
957            .entry(file.target_table.clone())
958            .or_default()
959            .entry(file.target_fingerprint.to_string())
960            .or_insert_with(|| SchemaConflictExample {
961                fingerprint: file.target_fingerprint.clone(),
962                path: file.relative_path.clone(),
963            });
964    }
965
966    by_table
967        .into_iter()
968        .filter_map(|(target_table, examples)| {
969            (examples.len() > 1).then(|| SchemaConflict {
970                target_table,
971                examples: examples.into_values().collect(),
972            })
973        })
974        .collect()
975}
976
977fn split_batch_by_identifier(
978    file: &PlannedMigrationFile,
979    batch: RecordBatch,
980) -> anyhow::Result<Vec<(Option<String>, IdentifierSource, RecordBatch)>> {
981    if let Some(identifier) = metadata_identifier(&file.target_type_name, &batch) {
982        return Ok(vec![(Some(identifier), IdentifierSource::Metadata, batch)]);
983    }
984
985    if let Some(column) = identifier_column(&file.target_type_name, &batch)? {
986        let groups = row_identifier_groups(&column, batch.num_rows())?;
987        return groups
988            .into_iter()
989            .map(|(identifier, indices)| {
990                let batch = take_record_batch(&batch, &UInt32Array::from(indices))?;
991                Ok((identifier, IdentifierSource::Row, batch))
992            })
993            .collect();
994    }
995
996    if let Some(identifier) =
997        record_identifier_from_path(&file.relative_path, &file.source_type_name)
998    {
999        return Ok(vec![(Some(identifier), IdentifierSource::Path, batch)]);
1000    }
1001
1002    Ok(vec![(None, IdentifierSource::Absent, batch)])
1003}
1004
1005fn metadata_identifier(type_name: &str, batch: &RecordBatch) -> Option<String> {
1006    batch
1007        .schema()
1008        .metadata()
1009        .get(identifier_metadata_key(type_name))
1010        .cloned()
1011}
1012
1013fn identifier_metadata_key(type_name: &str) -> &'static str {
1014    if type_name == "bars" {
1015        KEY_BAR_TYPE
1016    } else {
1017        KEY_INSTRUMENT_ID
1018    }
1019}
1020
1021fn identifier_column<'a>(
1022    type_name: &str,
1023    batch: &'a RecordBatch,
1024) -> anyhow::Result<Option<StringColumnRef<'a>>> {
1025    let candidates = if type_name == "bars" {
1026        [KEY_IDENTIFIER, KEY_BAR_TYPE, KEY_INSTRUMENT_ID, "id"]
1027    } else {
1028        [KEY_IDENTIFIER, KEY_INSTRUMENT_ID, KEY_BAR_TYPE, "id"]
1029    };
1030
1031    for name in candidates {
1032        if let Some(column) = batch.column_by_name(name) {
1033            return StringColumnRef::try_from_array(column.as_ref())
1034                .map(Some)
1035                .ok_or_else(|| anyhow::anyhow!("Identifier column {name} is not string-like"));
1036        }
1037    }
1038
1039    Ok(None)
1040}
1041
1042fn row_identifier_groups(
1043    column: &StringColumnRef<'_>,
1044    row_count: usize,
1045) -> anyhow::Result<BTreeMap<Option<String>, Vec<u32>>> {
1046    let mut groups: BTreeMap<Option<String>, Vec<u32>> = BTreeMap::new();
1047    for row in 0..row_count {
1048        groups
1049            .entry(column.value_opt(row).map(ToString::to_string))
1050            .or_default()
1051            .push(u32::try_from(row)?);
1052    }
1053
1054    Ok(groups)
1055}
1056
1057fn batch_with_identifier(
1058    type_name: &str,
1059    identifier: Option<&str>,
1060    batch: RecordBatch,
1061) -> anyhow::Result<RecordBatch> {
1062    let batch = record_batch_with_identifier_column(batch, identifier)?;
1063
1064    let Some(identifier) = identifier else {
1065        return Ok(batch);
1066    };
1067
1068    if type_name.starts_with("custom/") {
1069        return Ok(batch);
1070    }
1071
1072    let mut schema = batch.schema().as_ref().clone();
1073    schema.metadata.insert(
1074        identifier_metadata_key(type_name).to_string(),
1075        identifier.to_string(),
1076    );
1077    Ok(RecordBatch::try_new(
1078        Arc::new(schema),
1079        batch.columns().to_vec(),
1080    )?)
1081}
1082
1083// Built-in batches of one output file take the current encoder's exact schema and metadata, so
1084// migrated files consolidate with later catalog writes
1085fn round_trip_migrated_batches(
1086    type_name: &str,
1087    batches: Vec<RecordBatch>,
1088) -> anyhow::Result<Vec<RecordBatch>> {
1089    if let Ok(record_type) = type_name.parse::<NautilusRecordType>() {
1090        return round_trip_catalog_record_batches(record_type, batches);
1091    }
1092
1093    let data_type = data_type_from_data_path_prefix(type_name)?;
1094
1095    macro_rules! round_trip_data_type {
1096        (
1097            ($data_type:ident, $batches:ident);
1098            (Instrument, InstrumentAny, Instrument, Instrument, $instrument_prefix:literal),
1099            $(($variant:ident, $type:ident, $data:ident, $batch_variant:ident, $prefix:literal)),+ $(,)?
1100        ) => {
1101            match $data_type {
1102                NautilusDataType::Instrument => {
1103                    let mut instruments = Vec::new();
1104
1105                    for batch in &$batches {
1106                        let metadata = batch.schema().metadata().clone();
1107                        instruments.extend(decode_instrument_any_batch(&metadata, batch)?);
1108                    }
1109
1110                    let metadata = InstrumentAny::chunk_metadata(&instruments);
1111                    Ok(vec![InstrumentAny::encode_batch(&metadata, &instruments)?])
1112                }
1113                $(
1114                    NautilusDataType::$variant => {
1115                        let batches = $batches
1116                            .iter()
1117                            .map(project_current_columns::<$type>)
1118                            .collect::<anyhow::Result<Vec<_>>>()?;
1119                        round_trip_batches::<$type>(batches)
1120                    }
1121                )+
1122                NautilusDataType::Custom { .. } => Ok($batches),
1123                #[cfg(feature = "defi")]
1124                NautilusDataType::Defi => {
1125                    anyhow::bail!("Parquet migration does not support DeFi data")
1126                }
1127                #[cfg(not(feature = "defi"))]
1128                #[allow(
1129                    unreachable_patterns,
1130                    reason = "DeFi variants can exist without this crate's defi feature"
1131                )]
1132                _ => anyhow::bail!("Parquet migration does not support DeFi data"),
1133            }
1134        };
1135    }
1136
1137    nautilus_model::for_each_data_type!(round_trip_data_type, data_type, batches)
1138}
1139
1140// Positional decoders read the current column order, while sources may order columns differently
1141// or carry identity columns that current schemas keep in metadata. Current nullability applies so
1142// a null in a required column fails here instead of decoding as a default value.
1143fn project_current_columns<T: ArrowSchemaProvider>(
1144    batch: &RecordBatch,
1145) -> anyhow::Result<RecordBatch> {
1146    let schema = batch.schema();
1147    let mut fields = Vec::new();
1148    let mut columns = Vec::new();
1149
1150    for expected in T::get_schema(None).fields() {
1151        if expected.name() == KEY_IDENTIFIER {
1152            continue;
1153        }
1154
1155        let index = schema.index_of(expected.name())?;
1156        fields.push(
1157            schema
1158                .field(index)
1159                .clone()
1160                .with_nullable(expected.is_nullable()),
1161        );
1162        columns.push(batch.column(index).clone());
1163    }
1164
1165    Ok(RecordBatch::try_new(
1166        Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone())),
1167        columns,
1168    )?)
1169}
1170
1171fn record_identifier_from_path(file_path: &str, type_name: &str) -> Option<String> {
1172    let path_parts = file_path.split('/').collect::<Vec<_>>();
1173    let type_parts = type_name.split('/').collect::<Vec<_>>();
1174
1175    for start in 0..path_parts.len() {
1176        if path_parts.get(start) != Some(&"data") {
1177            continue;
1178        }
1179
1180        let type_start = start + 1;
1181        let type_end = type_start + type_parts.len();
1182        if path_parts.get(type_start..type_end) != Some(type_parts.as_slice()) {
1183            continue;
1184        }
1185
1186        let remaining = &path_parts[type_end..];
1187        if remaining.len() <= 1 {
1188            return None;
1189        }
1190
1191        return Some(remaining[0].to_string());
1192    }
1193
1194    None
1195}
1196
1197fn record_batches_with_schema(
1198    batches: Vec<RecordBatch>,
1199    schema: &Arc<Schema>,
1200) -> anyhow::Result<Vec<RecordBatch>> {
1201    batches
1202        .into_iter()
1203        .map(|batch| {
1204            RecordBatch::try_new(schema.clone(), batch.columns().to_vec()).map_err(Into::into)
1205        })
1206        .collect()
1207}
1208
1209/// Attaches the planned custom `type_name` to batches that lack it.
1210///
1211/// Legacy Python-written custom files predate `type_name` schema metadata. Timestamp
1212/// normalization keys off that metadata, so without it `uint64` timestamps would pass
1213/// through unconverted. Files that already carry `type_name` are returned unchanged.
1214fn with_inferred_custom_type_name(
1215    file: &PlannedMigrationFile,
1216    batch: RecordBatch,
1217) -> anyhow::Result<RecordBatch> {
1218    let Some(type_name) = file.target_type_name.strip_prefix("custom/") else {
1219        return Ok(batch);
1220    };
1221
1222    let schema = Arc::new(inject_type_name_metadata(&batch.schema(), type_name));
1223    Ok(RecordBatch::try_new(schema, batch.columns().to_vec())?)
1224}
1225
1226/// Normalizes supported legacy Parquet physical encodings for explicit migration.
1227///
1228/// Older Python/v1 catalog writers can emit low-cardinality strings as Arrow dictionary
1229/// arrays and instrument `info` values as the JSON bytes `null` rather than Arrow nulls.
1230fn normalize_legacy_parquet_columns(batch: &RecordBatch) -> anyhow::Result<RecordBatch> {
1231    if let Some(schema) = normalize_legacy_record_schema(batch.schema_ref()) {
1232        let batch = normalize_legacy_info_column(batch)?;
1233        let batch = normalize_legacy_fixed_columns(&batch)?;
1234        return Ok(nautilus_serialization::arrow::record_batch_with_timestamps(
1235            Arc::new(schema),
1236            batch.columns().to_vec(),
1237        )?);
1238    }
1239
1240    if is_legacy_instrument_schema(batch.schema_ref()) {
1241        let metadata = legacy_instrument_metadata(batch.schema().metadata());
1242        if batch.num_rows() == 0 {
1243            return Ok(RecordBatch::new_empty(Arc::new(InstrumentAny::get_schema(
1244                Some(metadata),
1245            ))));
1246        }
1247
1248        let batch = normalize_dictionary_string_columns(batch)?;
1249        let batch = normalize_legacy_info_column(&batch)?;
1250        let instruments = decode_instrument_any_batch(&metadata, &batch)?;
1251        return Ok(InstrumentAny::encode_batch(&metadata, &instruments)?);
1252    }
1253
1254    let normalize_legacy = uses_legacy_string_encoding(batch.schema_ref());
1255    let batch = if normalize_legacy {
1256        normalize_dictionary_string_columns(batch)?
1257    } else {
1258        batch.clone()
1259    };
1260
1261    let batch = normalize_legacy_fixed_columns(&batch)?;
1262    let batch = normalize_legacy_info_column(&batch)?;
1263    normalize_legacy_depth_columns(&batch)
1264}
1265
1266/// Normalizes the Arrow schema changes made by [`normalize_legacy_parquet_columns`].
1267#[must_use]
1268fn normalize_legacy_parquet_schema(schema: &Schema) -> Schema {
1269    if let Some(schema) = normalize_legacy_record_schema(schema) {
1270        return schema;
1271    }
1272
1273    if is_legacy_instrument_schema(schema) {
1274        return InstrumentAny::get_schema(Some(legacy_instrument_metadata(schema.metadata())));
1275    }
1276
1277    let normalize_fixed = is_nautilus_legacy_schema(schema);
1278    let normalize_strings = uses_legacy_string_encoding(schema);
1279    let normalize_timestamps = is_nautilus_timestamp_schema(schema);
1280
1281    let fields = schema
1282        .fields()
1283        .iter()
1284        .map(|field| {
1285            let data_type = if normalize_strings {
1286                normalized_dictionary_data_type(field.data_type())
1287            } else {
1288                field.data_type().clone()
1289            };
1290
1291            let data_type = normalized_legacy_data_type(field.name(), &data_type, normalize_fixed);
1292
1293            let data_type = if schema.metadata().contains_key("type_name")
1294                && matches!(field.name().as_str(), "ts_event" | "ts_init")
1295                && data_type == ArrowDataType::UInt64
1296            {
1297                nautilus_serialization::arrow::timestamp_data_type()
1298            } else if normalize_timestamps {
1299                normalized_timestamp_type(&data_type)
1300            } else {
1301                data_type
1302            };
1303
1304            let nullable = field.is_nullable()
1305                || (normalize_fixed
1306                    && matches!(field.data_type(), ArrowDataType::FixedSizeBinary(8 | 16)))
1307                || (field.name() == "info" && field.data_type() == &ArrowDataType::Binary);
1308
1309            Arc::new(
1310                field
1311                    .as_ref()
1312                    .clone()
1313                    .with_data_type(data_type)
1314                    .with_nullable(nullable),
1315            )
1316        })
1317        .collect::<Vec<_>>();
1318
1319    normalize_legacy_depth_schema(Schema::new_with_metadata(fields, schema.metadata().clone()))
1320}
1321
1322fn uses_legacy_string_encoding(schema: &Schema) -> bool {
1323    is_nautilus_legacy_schema(schema)
1324        || (is_nautilus_timestamp_schema(schema)
1325            && schema
1326                .field_with_name("ts_init")
1327                .is_ok_and(|field| field.data_type() == &ArrowDataType::UInt64))
1328}
1329
1330fn normalize_legacy_record_schema(schema: &Schema) -> Option<Schema> {
1331    let record_type = schema
1332        .metadata()
1333        .get("type")?
1334        .parse::<NautilusRecordType>()
1335        .ok()?;
1336    let current = catalog_record_schema(record_type).ok()?;
1337
1338    let fields = schema
1339        .fields()
1340        .iter()
1341        .map(|field| {
1342            let Ok(expected) = current.field_with_name(field.name()) else {
1343                return field.clone();
1344            };
1345
1346            if field.data_type() == expected.data_type()
1347                || (expected.data_type() == &nautilus_serialization::arrow::timestamp_data_type()
1348                    && (field.data_type() == &ArrowDataType::UInt64
1349                        || normalized_timestamp_type(field.data_type()) == *expected.data_type()))
1350                || (field.name() == "info" && field.data_type() == &ArrowDataType::Binary)
1351            {
1352                Arc::new(expected.clone())
1353            } else {
1354                field.clone()
1355            }
1356        })
1357        .collect::<Vec<_>>();
1358
1359    Some(Schema::new_with_metadata(fields, schema.metadata().clone()))
1360}
1361
1362fn is_legacy_instrument_schema(schema: &Schema) -> bool {
1363    schema
1364        .metadata()
1365        .get(LEGACY_KEY_CLASS)
1366        .is_some_and(|type_name| type_name.parse::<NautilusInstrumentType>().is_ok())
1367        && schema.field_with_name("ts_init").is_ok_and(|field| {
1368            field.data_type() == &ArrowDataType::UInt64
1369                || field.data_type() == &nautilus_serialization::arrow::timestamp_data_type()
1370        })
1371}
1372
1373// Legacy catalogs name the instrument type under `class`; current schemas use `type_name`
1374fn legacy_instrument_metadata(metadata: &HashMap<String, String>) -> HashMap<String, String> {
1375    let mut metadata = metadata.clone();
1376    if let Some(instrument_type) = metadata.remove(LEGACY_KEY_CLASS) {
1377        metadata.insert(KEY_TYPE_NAME.to_string(), instrument_type);
1378    }
1379
1380    metadata
1381}
1382
1383/// Casts dictionary-encoded string columns from legacy Parquet files to plain UTF-8 columns.
1384///
1385/// Older Python/v1 catalog writers can emit low-cardinality strings as Arrow dictionary
1386/// arrays. Rust decoders generally expect concrete `Utf8` columns, so Parquet reads normalize
1387/// this physical encoding before typed decoding.
1388fn normalize_dictionary_string_columns(batch: &RecordBatch) -> anyhow::Result<RecordBatch> {
1389    let schema = batch.schema();
1390    let mut changed = false;
1391    let mut fields = Vec::with_capacity(schema.fields().len());
1392    let mut columns: Vec<ArrayRef> = Vec::with_capacity(batch.num_columns());
1393
1394    for (field, column) in schema.fields().iter().zip(batch.columns()) {
1395        let data_type = normalized_dictionary_data_type(field.data_type());
1396
1397        changed |= &data_type != field.data_type();
1398        fields.push(Arc::new(
1399            field.as_ref().clone().with_data_type(data_type.clone()),
1400        ));
1401
1402        if column.data_type() == &data_type {
1403            columns.push(column.clone());
1404        } else {
1405            columns.push(cast(column.as_ref(), &data_type)?);
1406        }
1407    }
1408
1409    if !changed {
1410        return Ok(batch.clone());
1411    }
1412
1413    let schema = Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone()));
1414    Ok(RecordBatch::try_new(schema, columns)?)
1415}
1416
1417fn normalized_legacy_data_type(
1418    name: &str,
1419    data_type: &ArrowDataType,
1420    normalize_fixed: bool,
1421) -> ArrowDataType {
1422    if normalize_fixed
1423        && let ArrowDataType::FixedSizeList(item, length) = data_type
1424        && matches!(item.data_type(), ArrowDataType::FixedSizeBinary(8 | 16))
1425    {
1426        return ArrowDataType::FixedSizeList(
1427            Arc::new(
1428                item.as_ref()
1429                    .clone()
1430                    .with_data_type(normalize_legacy_arrow_data_type(name, item.data_type()))
1431                    .with_nullable(true),
1432            ),
1433            *length,
1434        );
1435    }
1436
1437    if normalize_fixed {
1438        let normalized = normalize_legacy_arrow_data_type(name, data_type);
1439        if &normalized != data_type {
1440            return normalized;
1441        }
1442    }
1443
1444    match data_type {
1445        ArrowDataType::Binary if name == "info" => ArrowDataType::Utf8,
1446        _ if normalize_fixed => normalized_dictionary_data_type(data_type),
1447        _ => data_type.clone(),
1448    }
1449}
1450
1451fn normalized_dictionary_data_type(data_type: &ArrowDataType) -> ArrowDataType {
1452    match data_type {
1453        ArrowDataType::Dictionary(_, value_type)
1454            if matches!(value_type.as_ref(), ArrowDataType::Utf8) =>
1455        {
1456            ArrowDataType::Utf8
1457        }
1458        _ => data_type.clone(),
1459    }
1460}
1461
1462fn normalize_legacy_info_column(batch: &RecordBatch) -> anyhow::Result<RecordBatch> {
1463    let Some(info_index) = batch.schema().index_of("info").ok() else {
1464        return Ok(batch.clone());
1465    };
1466
1467    let column = batch.column(info_index);
1468    if column.data_type() != &ArrowDataType::Binary {
1469        return Ok(batch.clone());
1470    }
1471
1472    let info = column
1473        .as_any()
1474        .downcast_ref::<BinaryArray>()
1475        .expect("Binary column should downcast to BinaryArray");
1476
1477    let mut builder = StringBuilder::new();
1478
1479    for row in 0..info.len() {
1480        if info.is_null(row) || info.value(row) == b"null" {
1481            builder.append_null();
1482        } else {
1483            builder.append_value(std::str::from_utf8(info.value(row))?);
1484        }
1485    }
1486
1487    let mut fields = batch.schema().fields().iter().cloned().collect::<Vec<_>>();
1488
1489    fields[info_index] = Arc::new(
1490        fields[info_index]
1491            .as_ref()
1492            .clone()
1493            .with_data_type(ArrowDataType::Utf8)
1494            .with_nullable(true),
1495    );
1496    let mut columns = batch.columns().to_vec();
1497    columns[info_index] = Arc::new(builder.finish());
1498
1499    let schema = Arc::new(Schema::new_with_metadata(
1500        fields,
1501        batch.schema().metadata().clone(),
1502    ));
1503    Ok(RecordBatch::try_new(schema, columns)?)
1504}
1505
1506fn normalize_legacy_depth_schema(schema: Schema) -> Schema {
1507    if !has_legacy_depth_columns(&schema) {
1508        return schema;
1509    }
1510
1511    let mut fields = vec![depth_side_field("bids"), depth_side_field("asks")];
1512    fields.extend(
1513        schema
1514            .fields()
1515            .iter()
1516            .filter(|field| !is_legacy_depth_column(field.name()))
1517            .cloned(),
1518    );
1519    Schema::new_with_metadata(fields, schema.metadata().clone())
1520}
1521
1522fn normalize_legacy_depth_columns(batch: &RecordBatch) -> anyhow::Result<RecordBatch> {
1523    if !has_legacy_depth_columns(batch.schema().as_ref()) {
1524        return Ok(batch.clone());
1525    }
1526
1527    let flat = batch.schema().index_of("bid_price_0").is_ok();
1528
1529    let side_values = |side: &str, value: &str| {
1530        let name = format!("{side}_{value}");
1531
1532        if flat {
1533            return match value {
1534                "price" | "size" => decimal_depth_list(batch, &name),
1535                "count" => u32_depth_list(batch, &name),
1536                "order_id" => u64_depth_list(batch, &name),
1537                _ => unreachable!("depth field inventory is fixed"),
1538            }
1539            .and_then(|list| depth_list_values(&list));
1540        }
1541
1542        if let Some(column) = batch.column_by_name(&name) {
1543            return depth_list_values(column);
1544        }
1545
1546        anyhow::ensure!(
1547            matches!(value, "count" | "order_id"),
1548            "Missing legacy depth column '{name}'"
1549        );
1550        let width = legacy_fixed_list_width(batch, side)?;
1551
1552        let len = batch
1553            .num_rows()
1554            .checked_mul(width)
1555            .ok_or_else(|| anyhow::anyhow!("Legacy depth column '{name}' length overflow"))?;
1556
1557        match value {
1558            "count" => Ok(Arc::new(UInt32Array::from(vec![0; len])) as ArrayRef),
1559            "order_id" => Ok(Arc::new(UInt64Array::from(vec![0; len])) as ArrayRef),
1560            _ => unreachable!("missing legacy depth defaults are fixed"),
1561        }
1562    };
1563
1564    let mut columns = vec![
1565        depth_side_array(
1566            &side_values("bid", "price")?,
1567            &side_values("bid", "size")?,
1568            &side_values("bid", "count")?,
1569            &side_values("bid", "order_id")?,
1570            batch.num_rows(),
1571        )?,
1572        depth_side_array(
1573            &side_values("ask", "price")?,
1574            &side_values("ask", "size")?,
1575            &side_values("ask", "count")?,
1576            &side_values("ask", "order_id")?,
1577            batch.num_rows(),
1578        )?,
1579    ];
1580    columns.extend(
1581        batch
1582            .schema()
1583            .fields()
1584            .iter()
1585            .zip(batch.columns())
1586            .filter(|(field, _)| !is_legacy_depth_column(field.name()))
1587            .map(|(_, column)| column.clone()),
1588    );
1589
1590    let schema = Arc::new(normalize_legacy_depth_schema(
1591        batch.schema().as_ref().clone(),
1592    ));
1593    Ok(RecordBatch::try_new(schema, columns)?)
1594}
1595
1596fn legacy_fixed_list_width(batch: &RecordBatch, side: &str) -> anyhow::Result<usize> {
1597    let schema = batch.schema();
1598    ["price", "size", "count", "order_id"]
1599        .iter()
1600        .filter_map(|value| schema.field_with_name(&format!("{side}_{value}")).ok())
1601        .find_map(|field| match field.data_type() {
1602            ArrowDataType::FixedSizeList(_, width) => usize::try_from(*width).ok(),
1603            _ => None,
1604        })
1605        .ok_or_else(|| anyhow::anyhow!("Missing legacy depth FixedSizeList width for '{side}'"))
1606}
1607
1608fn has_legacy_depth_columns(schema: &Schema) -> bool {
1609    if schema.index_of("bids").is_ok() {
1610        return false;
1611    }
1612
1613    let has_fixed_lists = ["bid_price", "ask_price", "bid_size", "ask_size"]
1614        .iter()
1615        .all(|name| {
1616            schema
1617                .field_with_name(name)
1618                .is_ok_and(|field| matches!(field.data_type(), ArrowDataType::FixedSizeList(_, _)))
1619        });
1620
1621    let has_flat_levels = ["bid_price_0", "ask_price_0", "bid_size_0", "ask_size_0"]
1622        .iter()
1623        .all(|name| schema.index_of(name).is_ok());
1624
1625    has_fixed_lists || has_flat_levels
1626}
1627
1628fn depth_level_fields() -> Fields {
1629    vec![
1630        Field::new("price", ArrowDataType::Decimal128(38, 16), false),
1631        Field::new("size", ArrowDataType::Decimal128(38, 16), false),
1632        Field::new("count", ArrowDataType::UInt32, false),
1633        Field::new("order_id", ArrowDataType::UInt64, false),
1634    ]
1635    .into()
1636}
1637
1638fn depth_side_field(name: &str) -> Arc<Field> {
1639    let fields = depth_level_fields();
1640
1641    Arc::new(Field::new(
1642        name,
1643        ArrowDataType::List(Arc::new(Field::new(
1644            "item",
1645            ArrowDataType::Struct(fields),
1646            false,
1647        ))),
1648        false,
1649    ))
1650}
1651
1652fn decimal_depth_list(batch: &RecordBatch, prefix: &str) -> anyhow::Result<ArrayRef> {
1653    let arrays = (0..DEPTH10_LEN)
1654        .map(|level| {
1655            let name = format!("{prefix}_{level}");
1656            batch
1657                .column_by_name(&name)
1658                .and_then(|column| column.as_any().downcast_ref::<Decimal128Array>())
1659                .ok_or_else(|| anyhow::anyhow!("Legacy depth column '{name}' must be Decimal128"))
1660        })
1661        .collect::<Result<Vec<_>, _>>()?;
1662
1663    let mut values = Vec::with_capacity(batch.num_rows() * DEPTH10_LEN);
1664    for row in 0..batch.num_rows() {
1665        for array in &arrays {
1666            values.push((!array.is_null(row)).then(|| array.value(row)));
1667        }
1668    }
1669
1670    let values = Decimal128Array::from(values).with_precision_and_scale(38, 16)?;
1671    Ok(depth_list_array(Arc::new(values), true))
1672}
1673
1674fn u64_depth_list(batch: &RecordBatch, prefix: &str) -> anyhow::Result<ArrayRef> {
1675    let arrays = (0..DEPTH10_LEN)
1676        .map(|level| {
1677            let name = format!("{prefix}_{level}");
1678            batch
1679                .column_by_name(&name)
1680                .map(|column| {
1681                    column
1682                        .as_any()
1683                        .downcast_ref::<UInt64Array>()
1684                        .ok_or_else(|| {
1685                            anyhow::anyhow!("Legacy depth column '{name}' must be UInt64")
1686                        })
1687                })
1688                .transpose()
1689        })
1690        .collect::<Result<Vec<_>, _>>()?;
1691
1692    let mut values = Vec::with_capacity(batch.num_rows() * DEPTH10_LEN);
1693    for row in 0..batch.num_rows() {
1694        for array in &arrays {
1695            values.push(array.map_or(0, |array| array.value(row)));
1696        }
1697    }
1698
1699    Ok(depth_list_array(Arc::new(UInt64Array::from(values)), false))
1700}
1701
1702fn u32_depth_list(batch: &RecordBatch, prefix: &str) -> anyhow::Result<ArrayRef> {
1703    let arrays = (0..DEPTH10_LEN)
1704        .map(|level| {
1705            let name = format!("{prefix}_{level}");
1706            batch
1707                .column_by_name(&name)
1708                .and_then(|column| column.as_any().downcast_ref::<UInt32Array>())
1709                .ok_or_else(|| anyhow::anyhow!("Legacy depth column '{name}' must be UInt32"))
1710        })
1711        .collect::<Result<Vec<_>, _>>()?;
1712
1713    let mut values = Vec::with_capacity(batch.num_rows() * DEPTH10_LEN);
1714    for row in 0..batch.num_rows() {
1715        for array in &arrays {
1716            values.push(array.value(row));
1717        }
1718    }
1719
1720    Ok(depth_list_array(Arc::new(UInt32Array::from(values)), false))
1721}
1722
1723fn depth_list_array(values: ArrayRef, values_nullable: bool) -> ArrayRef {
1724    Arc::new(FixedSizeListArray::new(
1725        Arc::new(Field::new(
1726            "item",
1727            values.data_type().clone(),
1728            values_nullable,
1729        )),
1730        i32::try_from(DEPTH10_LEN).expect("depth-10 length fits i32"),
1731        values,
1732        None,
1733    ))
1734}
1735
1736fn depth_list_values(list: &ArrayRef) -> anyhow::Result<ArrayRef> {
1737    list.as_any()
1738        .downcast_ref::<FixedSizeListArray>()
1739        .map(|list| list.values().clone())
1740        .ok_or_else(|| anyhow::anyhow!("Legacy depth column must be FixedSizeList"))
1741}
1742
1743fn depth_side_array(
1744    prices: &ArrayRef,
1745    sizes: &ArrayRef,
1746    counts: &ArrayRef,
1747    order_ids: &ArrayRef,
1748    rows: usize,
1749) -> anyhow::Result<ArrayRef> {
1750    let prices = prices
1751        .as_any()
1752        .downcast_ref::<Decimal128Array>()
1753        .ok_or_else(|| anyhow::anyhow!("Legacy depth prices must be Decimal128"))?;
1754    let sizes = sizes
1755        .as_any()
1756        .downcast_ref::<Decimal128Array>()
1757        .ok_or_else(|| anyhow::anyhow!("Legacy depth sizes must be Decimal128"))?;
1758    let counts = counts
1759        .as_any()
1760        .downcast_ref::<UInt32Array>()
1761        .ok_or_else(|| anyhow::anyhow!("Legacy depth counts must be UInt32"))?;
1762    let order_ids = order_ids
1763        .as_any()
1764        .downcast_ref::<UInt64Array>()
1765        .ok_or_else(|| anyhow::anyhow!("Legacy depth order IDs must be UInt64"))?;
1766    let expected_len = prices.len();
1767    anyhow::ensure!(
1768        sizes.len() == expected_len
1769            && counts.len() == expected_len
1770            && order_ids.len() == expected_len,
1771        "Legacy depth columns must contain the same number of values"
1772    );
1773
1774    let width = if rows == 0 {
1775        0
1776    } else {
1777        anyhow::ensure!(
1778            expected_len.is_multiple_of(rows),
1779            "Legacy depth column length {expected_len} is not divisible by row count {rows}"
1780        );
1781        expected_len / rows
1782    };
1783
1784    let mut open_prices = Vec::with_capacity(expected_len);
1785    let mut open_sizes = Vec::with_capacity(expected_len);
1786    let mut open_counts = Vec::with_capacity(expected_len);
1787    let mut open_order_ids = Vec::with_capacity(expected_len);
1788    let mut offsets = Vec::with_capacity(rows + 1);
1789    offsets.push(0);
1790
1791    for row in 0..rows {
1792        for level in 0..width {
1793            let index = row * width + level;
1794            if prices.is_null(index) || sizes.is_null(index) {
1795                continue;
1796            }
1797
1798            open_prices.push(prices.value(index));
1799            open_sizes.push(sizes.value(index));
1800            open_counts.push(counts.value(index));
1801            open_order_ids.push(order_ids.value(index));
1802        }
1803
1804        offsets.push(i32::try_from(open_prices.len())?);
1805    }
1806
1807    let fields = depth_level_fields();
1808    let values = StructArray::try_new(
1809        fields.clone(),
1810        vec![
1811            Arc::new(Decimal128Array::from(open_prices).with_precision_and_scale(38, 16)?),
1812            Arc::new(Decimal128Array::from(open_sizes).with_precision_and_scale(38, 16)?),
1813            Arc::new(UInt32Array::from(open_counts)),
1814            Arc::new(UInt64Array::from(open_order_ids)),
1815        ],
1816        None,
1817    )?;
1818    Ok(Arc::new(ListArray::try_new(
1819        Arc::new(Field::new("item", ArrowDataType::Struct(fields), false)),
1820        OffsetBuffer::new(ScalarBuffer::from(offsets)),
1821        Arc::new(values),
1822        None,
1823    )?))
1824}
1825
1826fn is_legacy_depth_column(name: &str) -> bool {
1827    const LIST_COLUMNS: &[&str] = &[
1828        "bid_price",
1829        "ask_price",
1830        "bid_size",
1831        "ask_size",
1832        "bid_order_id",
1833        "ask_order_id",
1834        "bid_count",
1835        "ask_count",
1836    ];
1837    LIST_COLUMNS.iter().any(|column| {
1838        name.strip_prefix(column).is_some_and(|suffix| {
1839            suffix.is_empty()
1840                || suffix
1841                    .strip_prefix('_')
1842                    .and_then(|level| level.parse::<usize>().ok())
1843                    .is_some_and(|level| level < DEPTH10_LEN)
1844        })
1845    })
1846}
1847
1848#[cfg(test)]
1849mod tests {
1850    use std::{collections::HashMap, fs, sync::Arc};
1851
1852    use ::parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
1853    use arrow::{
1854        array::{
1855            BooleanArray, FixedSizeBinaryArray, Float64Array, Int64Array, StringArray,
1856            StringDictionaryBuilder, TimestampNanosecondArray, UInt8Array, UInt64Array,
1857        },
1858        datatypes::{DataType, Field, Int8Type, Schema},
1859        record_batch::RecordBatch,
1860    };
1861    use nautilus_core::UnixNanos;
1862    use nautilus_model::{
1863        data::{greeks::OptionGreekValues, stubs::stub_depth10},
1864        enums::{GreeksConvention, MarketStatusAction},
1865        identifiers::InstrumentId,
1866        types::{Price, Quantity},
1867    };
1868    use nautilus_serialization::arrow::{
1869        DecodeFromRecordBatch, KEY_PRICE_PRECISION, KEY_TYPE_NAME,
1870        record_batch_without_identifier_column, timestamp_array, timestamp_data_type,
1871    };
1872    use rstest::rstest;
1873    use tempfile::TempDir;
1874
1875    use super::*;
1876
1877    #[rstest]
1878    fn storage_options_require_non_empty_key_and_value() {
1879        assert_eq!(
1880            parse_storage_option("region=us-east-1"),
1881            Ok(("region".to_string(), "us-east-1".to_string()))
1882        );
1883        assert_eq!(
1884            parse_storage_option("region").unwrap_err(),
1885            "Storage option must use key=value: region"
1886        );
1887        assert_eq!(
1888            parse_storage_option("=us-east-1").unwrap_err(),
1889            "Storage option must use non-empty key=value: =us-east-1"
1890        );
1891        assert_eq!(
1892            parse_storage_option("region=").unwrap_err(),
1893            "Storage option must use non-empty key=value: region="
1894        );
1895    }
1896
1897    #[rstest]
1898    fn dry_run_report_lists_unmigrated_files_and_reasons() {
1899        let plan = CatalogMigrationPlan {
1900            total_leaf_files: 1,
1901            unmigrated: vec![UnmigratedFile {
1902                path: "backtest/run/data.feather".to_string(),
1903                reason: "Feather trees are outside catalog migration scope".to_string(),
1904            }],
1905            ..CatalogMigrationPlan::default()
1906        };
1907
1908        let report = CatalogMigrationReport::from_plan(&plan, true).to_string();
1909
1910        assert!(report.contains("Migration dry-run report: 1 planned leaf files"));
1911        assert!(report.contains("Unmigrated directory backtest/run: 1 files"));
1912        assert!(report.contains(
1913            "Unmigrated backtest/run/data.feather: Feather trees are outside catalog migration scope"
1914        ));
1915    }
1916
1917    #[rstest]
1918    #[case("/tmp/source", "/tmp/source")]
1919    #[case("/tmp/source/", "/tmp/source")]
1920    #[case("/tmp/source", "/tmp/source/target")]
1921    #[case("/tmp/source/child", "/tmp/source")]
1922    fn migration_locations_must_not_overlap(#[case] source: &str, #[case] target: &str) {
1923        assert_eq!(
1924            ensure_distinct_migration_locations(source, target)
1925                .unwrap_err()
1926                .to_string(),
1927            "Migration source and target must be distinct, non-overlapping locations"
1928        );
1929    }
1930
1931    #[rstest]
1932    fn migration_locations_normalize_local_paths() {
1933        let source = std::env::current_dir().unwrap().join("catalog");
1934        let source_uri = normalize_storage_location(source.to_str().unwrap()).unwrap();
1935
1936        assert!(ensure_distinct_migration_locations("catalog", &source_uri).is_err());
1937        assert!(ensure_distinct_migration_locations("catalog-a", "catalog-b").is_ok());
1938    }
1939
1940    #[rstest]
1941    #[case("depths.parquet", "order_book_depths")]
1942    #[case("quotes.parquet", "quotes")]
1943    #[case("trades.parquet", "trades")]
1944    #[case("bars.parquet", "bars")]
1945    #[case("deltas.parquet", "order_book_deltas")]
1946    fn committed_legacy_fixture_plans_as_pass_through(
1947        #[case] file_name: &str,
1948        #[case] type_name: &str,
1949    ) {
1950        let precision_dir = if cfg!(feature = "high-precision") {
1951            "128-bit"
1952        } else {
1953            "64-bit"
1954        };
1955
1956        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
1957            .join("../../test_data/nautilus/legacy")
1958            .join(precision_dir)
1959            .join(file_name);
1960        let file = std::fs::File::open(path).unwrap();
1961        let builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
1962        let schema = normalize_legacy_parquet_schema(builder.schema().as_ref());
1963
1964        let resolution = resolve_legacy_schema(type_name, file_name, &schema).unwrap();
1965
1966        assert_eq!(resolution.kind, LegacyTranscodeKind::PassThrough);
1967    }
1968
1969    #[rstest]
1970    fn classify_source_path_maps_legacy_depth_directory() {
1971        let classification =
1972            classify_source_path("data/order_book_depth10/AAPL.XNAS/part-0.parquet");
1973
1974        assert!(matches!(
1975            classification,
1976            SourceClassification::Migratable { source_type_name, target_type_name }
1977                if source_type_name == "order_book_depth10"
1978                    && target_type_name == "order_book_depths"
1979        ));
1980    }
1981
1982    #[rstest]
1983    fn migration_plan_discovers_bar_file_under_bare_instrument_folder() {
1984        let temp = TempDir::new().unwrap();
1985        let source_path = temp.path().join("source");
1986        let bar_dir = source_path.join("data").join("bars").join("AUDUSD.SIM");
1987        fs::create_dir_all(&bar_dir).unwrap();
1988
1989        let precision_dir = if cfg!(feature = "high-precision") {
1990            "128-bit"
1991        } else {
1992            "64-bit"
1993        };
1994
1995        let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
1996            .join("../../test_data/nautilus/legacy")
1997            .join(precision_dir)
1998            .join("bars.parquet");
1999        fs::copy(fixture, bar_dir.join("bars.parquet")).unwrap();
2000        let source = crate::backend::parquet::catalog::ParquetDataCatalog::from_uri(
2001            source_path.to_str().unwrap(),
2002            None,
2003            None,
2004            None,
2005            None,
2006        )
2007        .unwrap();
2008
2009        let plan = build_catalog_migration_plan(&source).unwrap();
2010
2011        assert_eq!(plan.total_leaf_files, 1);
2012        assert_eq!(plan.files.len(), 1);
2013        assert_eq!(
2014            plan.files[0].relative_path,
2015            "data/bars/AUDUSD.SIM/bars.parquet",
2016        );
2017        assert_eq!(plan.files[0].source_type_name, "bars");
2018        assert_eq!(plan.files[0].target_type_name, "bars");
2019        assert!(plan.unmigrated.is_empty());
2020        assert!(plan.unresolved_schemas.is_empty());
2021        plan.ensure_ready().unwrap();
2022    }
2023
2024    #[rstest]
2025    fn migration_plan_rejects_unrecognized_schema_during_preflight() {
2026        let temp = TempDir::new().unwrap();
2027        let source_path = temp.path().join("source");
2028        let quote_dir = source_path.join("data").join("quotes").join("AUDUSD.SIM");
2029        fs::create_dir_all(&quote_dir).unwrap();
2030
2031        let schema = Arc::new(Schema::new(vec![Field::new(
2032            "unknown",
2033            DataType::Int64,
2034            false,
2035        )]));
2036        let batch = RecordBatch::try_new(
2037            schema.clone(),
2038            vec![Arc::new(Int64Array::from(vec![1_i64]))],
2039        )
2040        .unwrap();
2041        let file = fs::File::create(quote_dir.join("unknown.parquet")).unwrap();
2042        let mut writer = ::parquet::arrow::ArrowWriter::try_new(file, schema, None).unwrap();
2043        writer.write(&batch).unwrap();
2044        writer.close().unwrap();
2045        let source = crate::backend::parquet::catalog::ParquetDataCatalog::from_uri(
2046            source_path.to_str().unwrap(),
2047            None,
2048            None,
2049            None,
2050            None,
2051        )
2052        .unwrap();
2053
2054        let plan = build_catalog_migration_plan(&source).unwrap();
2055        let error = plan.ensure_ready().unwrap_err();
2056
2057        assert_eq!(plan.total_leaf_files, 1);
2058        assert!(plan.files.is_empty());
2059        assert_eq!(plan.unresolved_schemas.len(), 1);
2060        assert_eq!(
2061            plan.unresolved_schemas[0].path,
2062            "data/quotes/AUDUSD.SIM/unknown.parquet",
2063        );
2064        assert_eq!(
2065            error.to_string(),
2066            format!(
2067                "Catalog migration preflight failed:\n{}",
2068                plan.unresolved_schemas[0].message,
2069            ),
2070        );
2071    }
2072
2073    #[rstest]
2074    #[case::bars(
2075        "bars",
2076        Some("AUD/USD.SIM-1-MINUTE-BID-EXTERNAL"),
2077        Some("AUD/USD.SIM-1-MINUTE-BID-EXTERNAL"),
2078        None
2079    )]
2080    #[case::quotes("quotes", Some("AUD/USD.SIM"), None, Some("AUD/USD.SIM"))]
2081    #[case::custom("custom/RustTestCustomData", Some("AUD/USD.SIM"), None, None)]
2082    #[case::no_identifier("quotes", None, None, None)]
2083    fn batch_with_identifier_writes_the_type_specific_metadata_key(
2084        #[case] type_name: &str,
2085        #[case] identifier: Option<&str>,
2086        #[case] expected_bar_type: Option<&str>,
2087        #[case] expected_instrument_id: Option<&str>,
2088    ) {
2089        let batch = RecordBatch::try_new(
2090            Arc::new(Schema::new(vec![Field::new(
2091                "ts_init",
2092                DataType::Int64,
2093                false,
2094            )])),
2095            vec![Arc::new(Int64Array::from(vec![1]))],
2096        )
2097        .unwrap();
2098
2099        let batch = batch_with_identifier(type_name, identifier, batch).unwrap();
2100
2101        let schema = batch.schema();
2102        let metadata = schema.metadata();
2103        assert_eq!(
2104            metadata.get(KEY_BAR_TYPE).map(String::as_str),
2105            expected_bar_type
2106        );
2107        assert_eq!(
2108            metadata.get(KEY_INSTRUMENT_ID).map(String::as_str),
2109            expected_instrument_id
2110        );
2111        assert_eq!(
2112            metadata_identifier(type_name, &batch).as_deref(),
2113            expected_bar_type.or(expected_instrument_id)
2114        );
2115    }
2116
2117    #[rstest]
2118    #[case::absent(None, "Planned")]
2119    #[case::present(Some("Existing"), "Existing")]
2120    fn inject_type_name_metadata_keeps_an_existing_type_name(
2121        #[case] existing: Option<&str>,
2122        #[case] expected: &str,
2123    ) {
2124        let metadata = existing
2125            .map_or_default(|name| HashMap::from([("type_name".to_string(), name.to_string())]));
2126        let schema = Schema::new_with_metadata(
2127            vec![Field::new("ts_init", DataType::Int64, false)],
2128            metadata,
2129        );
2130
2131        let injected = inject_type_name_metadata(&schema, "Planned");
2132
2133        assert_eq!(
2134            injected.metadata().get("type_name").map(String::as_str),
2135            Some(expected)
2136        );
2137        assert_eq!(injected.fields(), schema.fields());
2138    }
2139
2140    #[rstest]
2141    fn prepare_migration_parts_splits_legacy_status_and_matches_fresh_writes() {
2142        let expected = [
2143            InstrumentStatus::new(
2144                InstrumentId::from("AAA.XNAS"),
2145                MarketStatusAction::Trading,
2146                UnixNanos::from(1),
2147                UnixNanos::from(2),
2148                Some("Normal trading".into()),
2149                None,
2150                Some(true),
2151                None,
2152                Some(false),
2153            ),
2154            InstrumentStatus::new(
2155                InstrumentId::from("BBB.XNAS"),
2156                MarketStatusAction::Halt,
2157                UnixNanos::from(3),
2158                UnixNanos::from(4),
2159                None,
2160                Some("MARKET_HALT".into()),
2161                Some(false),
2162                Some(false),
2163                None,
2164            ),
2165        ];
2166
2167        // Legacy status files carried the instrument per row, with no identity metadata
2168        let source = RecordBatch::try_new(
2169            Arc::new(Schema::new(vec![
2170                Field::new("instrument_id", DataType::Utf8, true),
2171                Field::new("action", DataType::Utf8, true),
2172                Field::new("reason", DataType::Utf8, true),
2173                Field::new("trading_event", DataType::Utf8, true),
2174                Field::new("is_trading", DataType::Boolean, true),
2175                Field::new("is_quoting", DataType::Boolean, true),
2176                Field::new("is_short_sell_restricted", DataType::Boolean, true),
2177                Field::new("ts_event", DataType::UInt64, true),
2178                Field::new("ts_init", DataType::UInt64, true),
2179            ])),
2180            vec![
2181                Arc::new(StringArray::from(vec!["AAA.XNAS", "BBB.XNAS"])),
2182                Arc::new(StringArray::from(vec!["TRADING", "HALT"])),
2183                Arc::new(StringArray::from(vec![Some("Normal trading"), None])),
2184                Arc::new(StringArray::from(vec![None, Some("MARKET_HALT")])),
2185                Arc::new(BooleanArray::from(vec![Some(true), Some(false)])),
2186                Arc::new(BooleanArray::from(vec![None, Some(false)])),
2187                Arc::new(BooleanArray::from(vec![Some(false), None])),
2188                Arc::new(UInt64Array::from(vec![1, 3])),
2189                Arc::new(UInt64Array::from(vec![2, 4])),
2190            ],
2191        )
2192        .unwrap();
2193
2194        let parts = migrate_source_batches("instrument_status", &[source]);
2195
2196        assert_eq!(
2197            parts
2198                .iter()
2199                .map(|part| (part.identifier.as_deref(), part.identifier_source))
2200                .collect::<Vec<_>>(),
2201            vec![
2202                (Some("AAA.XNAS"), IdentifierSource::Row),
2203                (Some("BBB.XNAS"), IdentifierSource::Row),
2204            ]
2205        );
2206
2207        for (part, expected) in parts.into_iter().zip(expected) {
2208            assert_part_matches_fresh_write(part, &[expected]);
2209        }
2210    }
2211
2212    #[rstest]
2213    fn prepare_migration_parts_reorders_legacy_greeks_and_matches_fresh_writes() {
2214        let instrument_id = InstrumentId::from("BTC-20260529-100000-C.OKX");
2215
2216        let expected = OptionGreeks {
2217            instrument_id,
2218            convention: GreeksConvention::PriceAdjusted,
2219            greeks: OptionGreekValues {
2220                delta: 0.55,
2221                gamma: 0.012,
2222                vega: 3.4,
2223                theta: -1.2,
2224                rho: 0.01,
2225            },
2226            mark_iv: Some(0.64),
2227            bid_iv: None,
2228            ask_iv: Some(0.66),
2229            underlying_price: Some(100_000.0),
2230            open_interest: None,
2231            ts_event: UnixNanos::from(5),
2232            ts_init: UnixNanos::from(6),
2233        };
2234
2235        // Legacy Greeks files led with an instrument column and ended with the convention
2236        let float_field = |name| Field::new(name, DataType::Float64, false);
2237        let optional_float_field = |name| Field::new(name, DataType::Float64, true);
2238        let source = RecordBatch::try_new(
2239            Arc::new(Schema::new_with_metadata(
2240                vec![
2241                    Field::new("instrument_id", DataType::Utf8, false),
2242                    float_field("delta"),
2243                    float_field("gamma"),
2244                    float_field("vega"),
2245                    float_field("theta"),
2246                    float_field("rho"),
2247                    optional_float_field("mark_iv"),
2248                    optional_float_field("bid_iv"),
2249                    optional_float_field("ask_iv"),
2250                    optional_float_field("underlying_price"),
2251                    optional_float_field("open_interest"),
2252                    Field::new("ts_event", DataType::UInt64, false),
2253                    Field::new("ts_init", DataType::UInt64, false),
2254                    Field::new("convention", DataType::Utf8, false),
2255                ],
2256                HashMap::from([("type".to_string(), "OptionGreeks".to_string())]),
2257            )),
2258            vec![
2259                Arc::new(StringArray::from(vec![instrument_id.to_string()])),
2260                Arc::new(Float64Array::from(vec![0.55])),
2261                Arc::new(Float64Array::from(vec![0.012])),
2262                Arc::new(Float64Array::from(vec![3.4])),
2263                Arc::new(Float64Array::from(vec![-1.2])),
2264                Arc::new(Float64Array::from(vec![0.01])),
2265                Arc::new(Float64Array::from(vec![Some(0.64)])),
2266                Arc::new(Float64Array::from(vec![None])),
2267                Arc::new(Float64Array::from(vec![Some(0.66)])),
2268                Arc::new(Float64Array::from(vec![Some(100_000.0)])),
2269                Arc::new(Float64Array::from(vec![None])),
2270                Arc::new(UInt64Array::from(vec![5])),
2271                Arc::new(UInt64Array::from(vec![6])),
2272                Arc::new(StringArray::from(vec!["PRICE_ADJUSTED"])),
2273            ],
2274        )
2275        .unwrap();
2276
2277        let parts = migrate_source_batches("option_greeks", &[source]);
2278
2279        assert_eq!(parts.len(), 1);
2280        assert_eq!(
2281            parts[0].identifier.as_deref(),
2282            Some("BTC-20260529-100000-C.OKX")
2283        );
2284        assert_eq!(
2285            parts[0].batches[0].schema().metadata().get(KEY_TYPE_NAME),
2286            Some(&"OptionGreeks".to_string())
2287        );
2288        assert_part_matches_fresh_write(parts.into_iter().next().unwrap(), &[expected]);
2289    }
2290
2291    #[rstest]
2292    fn prepare_migration_parts_rejects_null_in_required_column() {
2293        let source = RecordBatch::try_new(
2294            Arc::new(Schema::new_with_metadata(
2295                vec![
2296                    Field::new("instrument_id", DataType::Utf8, false),
2297                    Field::new("convention", DataType::Utf8, false),
2298                    Field::new("delta", DataType::Float64, true),
2299                    Field::new("gamma", DataType::Float64, false),
2300                    Field::new("vega", DataType::Float64, false),
2301                    Field::new("theta", DataType::Float64, false),
2302                    Field::new("rho", DataType::Float64, false),
2303                    Field::new("mark_iv", DataType::Float64, true),
2304                    Field::new("bid_iv", DataType::Float64, true),
2305                    Field::new("ask_iv", DataType::Float64, true),
2306                    Field::new("underlying_price", DataType::Float64, true),
2307                    Field::new("open_interest", DataType::Float64, true),
2308                    Field::new("ts_event", timestamp_data_type(), false),
2309                    Field::new("ts_init", timestamp_data_type(), false),
2310                ],
2311                HashMap::from([("type".to_string(), "OptionGreeks".to_string())]),
2312            )),
2313            vec![
2314                Arc::new(StringArray::from(vec!["BTC-20260529-100000-C.OKX"])),
2315                Arc::new(StringArray::from(vec!["BLACK_SCHOLES"])),
2316                Arc::new(Float64Array::from(vec![None])),
2317                Arc::new(Float64Array::from(vec![0.012])),
2318                Arc::new(Float64Array::from(vec![3.4])),
2319                Arc::new(Float64Array::from(vec![-1.2])),
2320                Arc::new(Float64Array::from(vec![0.01])),
2321                Arc::new(Float64Array::from(vec![None])),
2322                Arc::new(Float64Array::from(vec![None])),
2323                Arc::new(Float64Array::from(vec![None])),
2324                Arc::new(Float64Array::from(vec![None])),
2325                Arc::new(Float64Array::from(vec![None])),
2326                Arc::new(timestamp_array([5]).unwrap()),
2327                Arc::new(timestamp_array([6]).unwrap()),
2328            ],
2329        )
2330        .unwrap();
2331
2332        let error = try_migrate_source_batches("option_greeks", &[source]).unwrap_err();
2333
2334        assert_eq!(
2335            error.to_string(),
2336            "Invalid argument error: Column 'delta' is declared as non-nullable but contains null values"
2337        );
2338    }
2339
2340    #[rstest]
2341    fn prepare_migration_parts_selects_metadata_across_every_batch_of_a_file() {
2342        let populated = stub_depth10();
2343        let mut empty = populated.clone();
2344        empty.bids.clear();
2345        empty.asks.clear();
2346        empty.bid_counts.clear();
2347        empty.ask_counts.clear();
2348        let expected = [empty, populated];
2349
2350        // A reader splits one file into batches that share the file's metadata, and a batch of
2351        // empty snapshots alone would select zero precision
2352        let file_metadata = OrderBookDepth::chunk_metadata(&expected);
2353
2354        let sources = expected
2355            .iter()
2356            .map(|depth| {
2357                let batch =
2358                    OrderBookDepth::encode_batch(&file_metadata, std::slice::from_ref(depth))
2359                        .unwrap();
2360                record_batch_without_identifier_column(batch).unwrap()
2361            })
2362            .collect::<Vec<_>>();
2363
2364        let parts = migrate_source_batches("order_book_depths", &sources);
2365
2366        assert_eq!(parts.len(), 1);
2367        assert_eq!(
2368            parts[0]
2369                .batches
2370                .iter()
2371                .map(|batch| batch.schema().metadata()[KEY_PRICE_PRECISION].clone())
2372                .collect::<Vec<_>>(),
2373            vec!["2", "2"]
2374        );
2375        assert_part_matches_fresh_write(parts.into_iter().next().unwrap(), &expected);
2376    }
2377
2378    // Runs the per-batch steps of `read_planned_migration_file` without an object store
2379    fn migrate_source_batches(
2380        type_name: &str,
2381        sources: &[RecordBatch],
2382    ) -> Vec<PreparedMigrationPart> {
2383        try_migrate_source_batches(type_name, sources).unwrap()
2384    }
2385
2386    fn try_migrate_source_batches(
2387        type_name: &str,
2388        sources: &[RecordBatch],
2389    ) -> anyhow::Result<Vec<PreparedMigrationPart>> {
2390        let mut state = LegacyTranscodeState::default();
2391        let mut batches = Vec::new();
2392        let mut transcode_kind = LegacyTranscodeKind::PassThrough;
2393
2394        for source in sources {
2395            let normalized = normalize_legacy_parquet_columns(source).unwrap();
2396            let transcoded = transcode_legacy_record_batch_with_state(
2397                type_name,
2398                "legacy.parquet",
2399                normalized,
2400                &mut state,
2401            )
2402            .unwrap();
2403            transcode_kind = transcoded.kind;
2404            batches.extend(transcoded.batches);
2405        }
2406
2407        let fingerprint = schema_fingerprint(sources[0].schema_ref());
2408
2409        let file = PlannedMigrationFile {
2410            path: format!("data/{type_name}/legacy.parquet"),
2411            relative_path: format!("data/{type_name}/legacy.parquet"),
2412            source_type_name: type_name.to_string(),
2413            target_type_name: type_name.to_string(),
2414            target_table: type_name.to_string(),
2415            size: 1,
2416            e_tag: None,
2417            version: None,
2418            last_modified: String::new(),
2419            source_fingerprint: fingerprint.clone(),
2420            target_fingerprint: fingerprint,
2421            transcode_kind,
2422        };
2423
2424        prepare_migration_parts(&file, batches)
2425    }
2426
2427    fn assert_part_matches_fresh_write<T>(part: PreparedMigrationPart, expected: &[T])
2428    where
2429        T: Clone + DecodeFromRecordBatch + EncodeToRecordBatch + PartialEq + std::fmt::Debug,
2430    {
2431        let fresh = T::encode_batch(&T::chunk_metadata(expected), expected).unwrap();
2432
2433        let mut decoded = Vec::new();
2434
2435        for batch in part.batches {
2436            let schema = batch.schema();
2437            assert_eq!(schema, fresh.schema());
2438            decoded.extend(T::decode_batch(schema.metadata(), batch).unwrap());
2439        }
2440
2441        assert_eq!(part.row_count, expected.len());
2442        assert_eq!(decoded, expected);
2443    }
2444
2445    #[rstest]
2446    #[case::instrument_u64("CryptoPerpetual", DataType::UInt64, true)]
2447    #[case::instrument_utc("CryptoPerpetual", timestamp_data_type(), true)]
2448    #[case::instrument_no_timezone(
2449        "CryptoPerpetual",
2450        DataType::Timestamp(TimeUnit::Nanosecond, None),
2451        false
2452    )]
2453    #[case::instrument_other_timezone("CryptoPerpetual", DataType::Timestamp(TimeUnit::Nanosecond, Some("America/New_York".into())), false)]
2454    #[case::quote_u64("QuoteTick", DataType::UInt64, false)]
2455    #[case::quote_utc("QuoteTick", timestamp_data_type(), false)]
2456    fn legacy_instrument_detection_requires_known_class_and_supported_timestamps(
2457        #[case] class: &str,
2458        #[case] timestamp_type: DataType,
2459        #[case] expected: bool,
2460    ) {
2461        let schema = Schema::new_with_metadata(
2462            vec![Field::new("ts_init", timestamp_type, false)],
2463            HashMap::from([(LEGACY_KEY_CLASS.to_string(), class.to_string())]),
2464        );
2465
2466        assert_eq!(is_legacy_instrument_schema(&schema), expected);
2467    }
2468
2469    #[rstest]
2470    #[case::utc(Some("UTC"))]
2471    #[case::canonical(None)]
2472    fn v2_timestamp_normalization_matches_preflight_and_preserves_values(
2473        #[case] timezone: Option<&str>,
2474    ) {
2475        let first = QuoteTick {
2476            instrument_id: InstrumentId::from("AAPL.XNAS"),
2477            bid_price: Price::from("123.45"),
2478            ask_price: Price::from("123.67"),
2479            bid_size: Quantity::from(17),
2480            ask_size: Quantity::from(29),
2481            ts_event: 1_788_652_800_123_456_789_u64.into(),
2482            ts_init: 1_788_652_800_123_456_799_u64.into(),
2483        };
2484
2485        let values = vec![
2486            first,
2487            QuoteTick {
2488                ts_event: 1_788_652_800_123_456_801_u64.into(),
2489                ts_init: 1_788_652_800_123_456_899_u64.into(),
2490                ..first
2491            },
2492        ];
2493        let metadata = QuoteTick::get_metadata(&first.instrument_id, 2, 0);
2494        let expected = QuoteTick::encode_batch(&metadata, &values).unwrap();
2495        let source_type = DataType::Timestamp(TimeUnit::Nanosecond, timezone.map(Into::into));
2496
2497        let fields = expected
2498            .schema()
2499            .fields()
2500            .iter()
2501            .map(|field| {
2502                let field = field.as_ref().clone();
2503
2504                if matches!(field.data_type(), DataType::Timestamp(_, _)) {
2505                    field.with_data_type(source_type.clone())
2506                } else {
2507                    field
2508                }
2509            })
2510            .collect::<Vec<_>>();
2511
2512        let columns = expected
2513            .columns()
2514            .iter()
2515            .map(|column| {
2516                if let Some(timestamps) = column.as_any().downcast_ref::<TimestampNanosecondArray>()
2517                {
2518                    Arc::new(
2519                        timestamps
2520                            .clone()
2521                            .with_timezone_opt(timezone.map(Arc::<str>::from)),
2522                    ) as ArrayRef
2523                } else {
2524                    column.clone()
2525                }
2526            })
2527            .collect::<Vec<_>>();
2528
2529        let source = RecordBatch::try_new(
2530            Arc::new(Schema::new_with_metadata(
2531                fields,
2532                expected.schema().metadata().clone(),
2533            )),
2534            columns,
2535        )
2536        .unwrap();
2537        let preflight = normalize_legacy_parquet_schema(source.schema_ref());
2538        let normalized = normalize_legacy_parquet_columns(&source).unwrap();
2539        assert_eq!(&preflight, expected.schema_ref().as_ref());
2540        assert_eq!(normalized, expected);
2541        assert_eq!(
2542            QuoteTick::decode_batch(&metadata, normalized).unwrap(),
2543            values
2544        );
2545    }
2546
2547    #[rstest]
2548    fn timestamp_normalization_preserves_custom_nulls_and_unrelated_numeric_fields() {
2549        let schema = Arc::new(Schema::new_with_metadata(
2550            vec![
2551                Field::new(
2552                    "ts_event",
2553                    DataType::Timestamp(TimeUnit::Nanosecond, None),
2554                    true,
2555                ),
2556                Field::new("ts_count", DataType::UInt64, false),
2557            ],
2558            HashMap::from([("type_name".to_string(), "TimestampSample".to_string())]),
2559        ));
2560        let timestamps =
2561            TimestampNanosecondArray::from(vec![Some(1_788_652_800_123_456_789), None]);
2562        let source = RecordBatch::try_new(
2563            schema,
2564            vec![
2565                Arc::new(timestamps),
2566                Arc::new(UInt64Array::from(vec![17, 29])),
2567            ],
2568        )
2569        .unwrap();
2570        let normalized = normalize_legacy_parquet_columns(&source).unwrap();
2571        assert_eq!(
2572            &normalize_legacy_parquet_schema(source.schema_ref()),
2573            normalized.schema_ref().as_ref()
2574        );
2575        assert_eq!(
2576            normalized.schema().field(0).data_type(),
2577            &DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
2578        );
2579        assert_eq!(
2580            normalized
2581                .column(0)
2582                .as_any()
2583                .downcast_ref::<TimestampNanosecondArray>()
2584                .unwrap()
2585                .iter()
2586                .collect::<Vec<_>>(),
2587            vec![Some(1_788_652_800_123_456_789), None]
2588        );
2589        assert_eq!(normalized.column(1), source.column(1));
2590    }
2591
2592    #[rstest]
2593    fn normalize_legacy_info_schema_makes_binary_info_nullable() {
2594        let schema = Schema::new(vec![Field::new("info", DataType::Binary, false)]);
2595
2596        let normalized = normalize_legacy_parquet_schema(&schema);
2597
2598        let info = normalized.field_with_name("info").unwrap();
2599        assert_eq!(info.data_type(), &DataType::Utf8);
2600        assert!(info.is_nullable());
2601    }
2602
2603    #[rstest]
2604    fn normalize_dictionary_string_columns_casts_string_dictionaries_to_utf8() {
2605        let mut builder = StringDictionaryBuilder::<Int8Type>::new();
2606        builder.append("AUD/USD.SIM").unwrap();
2607        builder.append("EUR/USD.SIM").unwrap();
2608        let dictionary = Arc::new(builder.finish()) as ArrayRef;
2609
2610        let schema = Arc::new(Schema::new(vec![
2611            Field::new("instrument_id", dictionary.data_type().clone(), false),
2612            Field::new("ts_init", DataType::UInt64, false),
2613        ]));
2614        let batch = RecordBatch::try_new(
2615            schema,
2616            vec![
2617                dictionary,
2618                Arc::new(UInt64Array::from(vec![1_u64, 2])) as ArrayRef,
2619            ],
2620        )
2621        .unwrap();
2622
2623        let normalized = normalize_dictionary_string_columns(&batch).unwrap();
2624
2625        assert_eq!(
2626            normalized
2627                .schema()
2628                .field_with_name("instrument_id")
2629                .unwrap()
2630                .data_type(),
2631            &DataType::Utf8,
2632        );
2633        let values = normalized
2634            .column_by_name("instrument_id")
2635            .unwrap()
2636            .as_any()
2637            .downcast_ref::<StringArray>()
2638            .unwrap()
2639            .iter()
2640            .collect::<Vec<_>>();
2641        assert_eq!(values, vec![Some("AUD/USD.SIM"), Some("EUR/USD.SIM")]);
2642    }
2643
2644    #[rstest]
2645    fn normalize_legacy_parquet_columns_preserves_unrecognized_dictionary() {
2646        let mut builder = StringDictionaryBuilder::<Int8Type>::new();
2647        builder.append("alpha").unwrap();
2648        let dictionary = Arc::new(builder.finish()) as ArrayRef;
2649
2650        let schema = Arc::new(Schema::new(vec![Field::new(
2651            "label",
2652            dictionary.data_type().clone(),
2653            false,
2654        )]));
2655        let batch = RecordBatch::try_new(schema, vec![dictionary]).unwrap();
2656
2657        let normalized = normalize_legacy_parquet_columns(&batch).unwrap();
2658
2659        assert_eq!(normalized, batch);
2660    }
2661
2662    #[rstest]
2663    #[case::without_ts_init(false)]
2664    #[case::timestamp_ts_init(true)]
2665    fn normalize_open_custom_columns_preserves_dictionary_with_type_metadata(
2666        #[case] has_ts_init: bool,
2667    ) {
2668        let mut builder = StringDictionaryBuilder::<Int8Type>::new();
2669        builder.append("alpha").unwrap();
2670        let dictionary = Arc::new(builder.finish()) as ArrayRef;
2671        let decimal = Arc::new(
2672            Decimal128Array::from(vec![Some(123_i128)])
2673                .with_precision_and_scale(38, 16)
2674                .unwrap(),
2675        ) as ArrayRef;
2676
2677        let mut fields = vec![
2678            Field::new("label", dictionary.data_type().clone(), false),
2679            Field::new("price", decimal.data_type().clone(), false),
2680            Field::new("ts_recv", DataType::UInt64, false),
2681        ];
2682        let mut columns = vec![dictionary, decimal, Arc::new(UInt64Array::from(vec![7]))];
2683
2684        if has_ts_init {
2685            fields.push(Field::new("ts_init", timestamp_data_type(), false));
2686            columns.push(Arc::new(timestamp_array([13]).unwrap()));
2687        }
2688
2689        let schema = Arc::new(Schema::new_with_metadata(
2690            fields,
2691            HashMap::from([("type_name".to_string(), "CustomData".to_string())]),
2692        ));
2693        let batch = RecordBatch::try_new(schema, columns).unwrap();
2694
2695        let normalized = normalize_legacy_parquet_columns(&batch).unwrap();
2696        let normalized_schema = normalize_legacy_parquet_schema(batch.schema_ref());
2697
2698        assert_eq!(normalized, batch);
2699        assert_eq!(&normalized_schema, batch.schema_ref().as_ref());
2700    }
2701
2702    #[rstest]
2703    fn normalize_legacy_parquet_schema_preserves_unrecognized_fixed_binary() {
2704        let schema = Schema::new(vec![Field::new(
2705            "price",
2706            DataType::FixedSizeBinary(8),
2707            false,
2708        )]);
2709
2710        let normalized = normalize_legacy_parquet_schema(&schema);
2711
2712        assert_eq!(normalized, schema);
2713    }
2714
2715    #[rstest]
2716    fn normalize_legacy_parquet_columns_converts_binary_info_null_to_arrow_null() {
2717        let schema = Arc::new(Schema::new(vec![
2718            Field::new("info", DataType::Binary, true),
2719            Field::new("ts_init", DataType::UInt64, false),
2720        ]));
2721        let batch = RecordBatch::try_new(
2722            schema,
2723            vec![
2724                Arc::new(BinaryArray::from_vec(vec![b"null".as_slice()])) as ArrayRef,
2725                Arc::new(UInt64Array::from(vec![1_u64])) as ArrayRef,
2726            ],
2727        )
2728        .unwrap();
2729
2730        let normalized = normalize_legacy_parquet_columns(&batch).unwrap();
2731        let info = normalized
2732            .column_by_name("info")
2733            .unwrap()
2734            .as_any()
2735            .downcast_ref::<StringArray>()
2736            .unwrap();
2737
2738        assert_eq!(
2739            normalized
2740                .schema()
2741                .field_with_name("info")
2742                .unwrap()
2743                .data_type(),
2744            &DataType::Utf8,
2745        );
2746        assert!(info.is_null(0));
2747    }
2748
2749    #[rstest]
2750    fn normalize_legacy_parquet_columns_preserves_quote_price_columns() {
2751        let decimal = DataType::Decimal128(38, 16);
2752
2753        let schema = Arc::new(Schema::new(vec![
2754            Field::new("bid_price", decimal.clone(), false),
2755            Field::new("ask_price", decimal.clone(), false),
2756            Field::new("bid_size", decimal.clone(), false),
2757            Field::new("ask_size", decimal, false),
2758        ]));
2759
2760        let values = || {
2761            Arc::new(
2762                Decimal128Array::from(vec![1_i128])
2763                    .with_precision_and_scale(38, 16)
2764                    .unwrap(),
2765            ) as ArrayRef
2766        };
2767
2768        let batch =
2769            RecordBatch::try_new(schema.clone(), vec![values(), values(), values(), values()])
2770                .unwrap();
2771
2772        let normalized = normalize_legacy_parquet_columns(&batch).unwrap();
2773
2774        assert_eq!(normalized.schema(), schema);
2775        assert_eq!(normalized, batch);
2776    }
2777
2778    #[rstest]
2779    fn normalize_legacy_depth_flat_columns_builds_structured_sides() {
2780        let mut fields = Vec::new();
2781        let mut columns = Vec::new();
2782
2783        for side in ["bid", "ask"] {
2784            for level in 0..DEPTH10_LEN {
2785                for (name, value) in [("price", 11_i128), ("size", 22_i128)] {
2786                    fields.push(Field::new(
2787                        format!("{side}_{name}_{level}"),
2788                        DataType::Decimal128(38, 16),
2789                        true,
2790                    ));
2791                    let value = (level == 0).then_some(value);
2792                    columns.push(Arc::new(
2793                        Decimal128Array::from(vec![value])
2794                            .with_precision_and_scale(38, 16)
2795                            .unwrap(),
2796                    ) as ArrayRef);
2797                }
2798
2799                fields.push(Field::new(
2800                    format!("{side}_count_{level}"),
2801                    DataType::UInt32,
2802                    false,
2803                ));
2804                columns.push(Arc::new(UInt32Array::from(vec![33])) as ArrayRef);
2805                fields.push(Field::new(
2806                    format!("{side}_order_id_{level}"),
2807                    DataType::UInt64,
2808                    false,
2809                ));
2810                columns.push(Arc::new(UInt64Array::from(vec![44])) as ArrayRef);
2811            }
2812        }
2813
2814        let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap();
2815
2816        let normalized = normalize_legacy_parquet_columns(&batch).unwrap();
2817
2818        assert_eq!(normalized.num_columns(), 2);
2819        assert_normalized_depth(&normalized, 1, 11, 22, 33, 44);
2820    }
2821
2822    #[rstest]
2823    fn normalize_legacy_depth_fixed_lists_builds_structured_sides() {
2824        let decimal_values = |value| {
2825            Arc::new(
2826                Decimal128Array::from(
2827                    (0..DEPTH10_LEN)
2828                        .map(|level| (level == 0).then_some(value))
2829                        .collect::<Vec<_>>(),
2830                )
2831                .with_precision_and_scale(38, 16)
2832                .unwrap(),
2833            ) as ArrayRef
2834        };
2835
2836        let counts = Arc::new(UInt32Array::from(vec![33; DEPTH10_LEN])) as ArrayRef;
2837        let order_ids = Arc::new(UInt64Array::from(vec![44; DEPTH10_LEN])) as ArrayRef;
2838        let mut fields = Vec::new();
2839        let mut columns = Vec::new();
2840
2841        for side in ["bid", "ask"] {
2842            for (name, column) in [
2843                ("price", depth_list_array(decimal_values(11), true)),
2844                ("size", depth_list_array(decimal_values(22), true)),
2845                ("count", depth_list_array(counts.clone(), false)),
2846                ("order_id", depth_list_array(order_ids.clone(), false)),
2847            ] {
2848                fields.push(Field::new(
2849                    format!("{side}_{name}"),
2850                    column.data_type().clone(),
2851                    false,
2852                ));
2853                columns.push(column);
2854            }
2855        }
2856
2857        let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap();
2858
2859        let normalized = normalize_legacy_parquet_columns(&batch).unwrap();
2860
2861        assert_eq!(normalized.num_columns(), 2);
2862        assert_normalized_depth(&normalized, 1, 11, 22, 33, 44);
2863    }
2864
2865    #[rstest]
2866    fn normalize_legacy_depth_missing_counts_and_order_ids_uses_list_width() {
2867        const WIDTH: i32 = 3;
2868
2869        let decimal_values = |value| {
2870            Arc::new(
2871                Decimal128Array::from(vec![value; WIDTH as usize])
2872                    .with_precision_and_scale(38, 16)
2873                    .unwrap(),
2874            ) as ArrayRef
2875        };
2876
2877        let list = |values: ArrayRef| {
2878            Arc::new(FixedSizeListArray::new(
2879                Arc::new(Field::new("item", values.data_type().clone(), false)),
2880                WIDTH,
2881                values,
2882                None,
2883            )) as ArrayRef
2884        };
2885
2886        let mut fields = Vec::new();
2887        let mut columns = Vec::new();
2888
2889        for side in ["bid", "ask"] {
2890            for (name, column) in [
2891                ("price", list(decimal_values(11))),
2892                ("size", list(decimal_values(22))),
2893            ] {
2894                fields.push(Field::new(
2895                    format!("{side}_{name}"),
2896                    column.data_type().clone(),
2897                    false,
2898                ));
2899                columns.push(column);
2900            }
2901        }
2902
2903        let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap();
2904
2905        let normalized = normalize_legacy_parquet_columns(&batch).unwrap();
2906
2907        assert_eq!(normalized.num_columns(), 2);
2908        assert_normalized_depth(&normalized, WIDTH as usize, 11, 22, 0, 0);
2909    }
2910
2911    #[rstest]
2912    #[case::with_order_ids(true, 44)]
2913    #[case::without_order_ids(false, 0)]
2914    fn normalize_legacy_depth_fixed_binary_lists_matches_schema(
2915        #[case] include_order_ids: bool,
2916        #[case] expected_order_id: u64,
2917    ) {
2918        let fixed_values = |value: [u8; 8]| {
2919            let values = (0..DEPTH10_LEN)
2920                .map(|level| (level == 0).then_some(value))
2921                .collect::<Vec<_>>();
2922            Arc::new(
2923                FixedSizeBinaryArray::try_from_sparse_iter_with_size(
2924                    values
2925                        .iter()
2926                        .map(Option::as_ref)
2927                        .map(|value| value.map(<[u8; 8]>::as_slice)),
2928                    8,
2929                )
2930                .unwrap(),
2931            ) as ArrayRef
2932        };
2933
2934        let counts = Arc::new(UInt32Array::from(vec![33; DEPTH10_LEN])) as ArrayRef;
2935        let order_ids = Arc::new(UInt64Array::from(vec![44; DEPTH10_LEN])) as ArrayRef;
2936        let mut fields = Vec::new();
2937        let mut columns = Vec::new();
2938
2939        for side in ["bid", "ask"] {
2940            let mut side_columns = vec![
2941                (
2942                    "price",
2943                    depth_list_array(fixed_values(11_i64.to_le_bytes()), true),
2944                ),
2945                (
2946                    "size",
2947                    depth_list_array(fixed_values(22_u64.to_le_bytes()), true),
2948                ),
2949                ("count", depth_list_array(counts.clone(), false)),
2950            ];
2951
2952            if include_order_ids {
2953                side_columns.push(("order_id", depth_list_array(order_ids.clone(), false)));
2954            }
2955
2956            for (name, column) in side_columns {
2957                fields.push(Field::new(
2958                    format!("{side}_{name}"),
2959                    column.data_type().clone(),
2960                    false,
2961                ));
2962                columns.push(column);
2963            }
2964        }
2965
2966        let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap();
2967
2968        assert!(is_nautilus_legacy_schema(batch.schema_ref()));
2969        let normalized_schema = normalize_legacy_parquet_schema(batch.schema_ref());
2970        let normalized_batch = normalize_legacy_parquet_columns(&batch).unwrap();
2971
2972        assert_eq!(
2973            normalized_schema,
2974            normalized_batch.schema().as_ref().clone()
2975        );
2976        assert_normalized_depth(
2977            &normalized_batch,
2978            1,
2979            110_000_000,
2980            220_000_000,
2981            33,
2982            expected_order_id,
2983        );
2984    }
2985
2986    #[rstest]
2987    fn normalize_legacy_depth_flat_fixed_columns_preserves_order_ids() {
2988        let fixed_price = || {
2989            let bytes = 11_i64.to_le_bytes();
2990            Arc::new(
2991                FixedSizeBinaryArray::try_from_sparse_iter_with_size(
2992                    [Some(bytes.as_slice())].into_iter(),
2993                    8,
2994                )
2995                .unwrap(),
2996            ) as ArrayRef
2997        };
2998
2999        let fixed_size = || {
3000            let bytes = 22_u64.to_le_bytes();
3001            Arc::new(
3002                FixedSizeBinaryArray::try_from_sparse_iter_with_size(
3003                    [Some(bytes.as_slice())].into_iter(),
3004                    8,
3005                )
3006                .unwrap(),
3007            ) as ArrayRef
3008        };
3009
3010        let mut fields = Vec::new();
3011        let mut columns = Vec::new();
3012
3013        for side in ["bid", "ask"] {
3014            for level in 0..DEPTH10_LEN {
3015                fields.push(Field::new(
3016                    format!("{side}_price_{level}"),
3017                    DataType::FixedSizeBinary(8),
3018                    false,
3019                ));
3020                columns.push(fixed_price());
3021                fields.push(Field::new(
3022                    format!("{side}_size_{level}"),
3023                    DataType::FixedSizeBinary(8),
3024                    false,
3025                ));
3026                columns.push(fixed_size());
3027                fields.push(Field::new(
3028                    format!("{side}_count_{level}"),
3029                    DataType::UInt32,
3030                    false,
3031                ));
3032                columns.push(Arc::new(UInt32Array::from(vec![33])) as ArrayRef);
3033                fields.push(Field::new(
3034                    format!("{side}_order_id_{level}"),
3035                    DataType::UInt64,
3036                    false,
3037                ));
3038                columns.push(Arc::new(UInt64Array::from(vec![44])) as ArrayRef);
3039            }
3040        }
3041
3042        for (field, column) in [
3043            (
3044                Field::new("flags", DataType::UInt8, false),
3045                Arc::new(UInt8Array::from(vec![0])) as ArrayRef,
3046            ),
3047            (
3048                Field::new("sequence", DataType::UInt64, false),
3049                Arc::new(UInt64Array::from(vec![1])) as ArrayRef,
3050            ),
3051            (
3052                Field::new("ts_event", DataType::UInt64, false),
3053                Arc::new(UInt64Array::from(vec![2])) as ArrayRef,
3054            ),
3055            (
3056                Field::new("ts_init", DataType::UInt64, false),
3057                Arc::new(UInt64Array::from(vec![3])) as ArrayRef,
3058            ),
3059        ] {
3060            fields.push(field);
3061            columns.push(column);
3062        }
3063
3064        let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap();
3065        assert!(is_nautilus_legacy_schema(batch.schema_ref()));
3066
3067        let normalized = normalize_legacy_parquet_columns(&batch).unwrap();
3068
3069        for side in ["bids", "asks"] {
3070            let list = normalized
3071                .column_by_name(side)
3072                .unwrap()
3073                .as_any()
3074                .downcast_ref::<ListArray>()
3075                .unwrap();
3076            let levels = list.value(0);
3077            let levels = levels.as_any().downcast_ref::<StructArray>().unwrap();
3078            let order_ids = levels
3079                .column_by_name("order_id")
3080                .unwrap()
3081                .as_any()
3082                .downcast_ref::<UInt64Array>()
3083                .unwrap();
3084            assert_eq!(order_ids.values(), &[44; DEPTH10_LEN]);
3085        }
3086    }
3087
3088    #[rstest]
3089    fn normalize_legacy_depth_fixture_matches_open_shape() {
3090        let precision_dir = if cfg!(feature = "high-precision") {
3091            "128-bit"
3092        } else {
3093            "64-bit"
3094        };
3095
3096        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
3097            .join("../../test_data/nautilus/legacy")
3098            .join(precision_dir)
3099            .join("depths.parquet");
3100        let file = std::fs::File::open(path).unwrap();
3101        let builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
3102        let metadata = builder
3103            .metadata()
3104            .file_metadata()
3105            .key_value_metadata()
3106            .unwrap();
3107
3108        for (key, value) in [
3109            ("instrument_id", "AAPL.XNAS"),
3110            ("price_precision", "4"),
3111            ("size_precision", "1"),
3112        ] {
3113            assert_eq!(
3114                metadata
3115                    .iter()
3116                    .find(|entry| entry.key == key)
3117                    .and_then(|entry| entry.value.as_deref()),
3118                Some(value),
3119            );
3120        }
3121
3122        let batch = builder.build().unwrap().next().unwrap().unwrap();
3123
3124        let normalized = normalize_legacy_parquet_columns(&batch).unwrap();
3125
3126        assert_normalized_depth(
3127            &normalized,
3128            DEPTH10_LEN,
3129            12_345_000_000_000_000,
3130            25_000_000_000_000_000,
3131            3,
3132            0,
3133        );
3134        assert_eq!(normalized.num_columns(), 6);
3135        assert_eq!(
3136            normalized
3137                .column_by_name("flags")
3138                .unwrap()
3139                .as_any()
3140                .downcast_ref::<UInt8Array>()
3141                .unwrap()
3142                .value(0),
3143            32
3144        );
3145        assert_eq!(
3146            normalized
3147                .column_by_name("sequence")
3148                .unwrap()
3149                .as_any()
3150                .downcast_ref::<UInt64Array>()
3151                .unwrap()
3152                .value(0),
3153            7
3154        );
3155
3156        for name in ["ts_event", "ts_init"] {
3157            assert_eq!(
3158                normalized
3159                    .schema()
3160                    .field_with_name(name)
3161                    .unwrap()
3162                    .data_type(),
3163                &DataType::Timestamp(arrow::datatypes::TimeUnit::Nanosecond, Some("UTC".into())),
3164            );
3165        }
3166    }
3167
3168    #[rstest]
3169    #[case("quotes.parquet", "bid_price", None)]
3170    #[case("trades.parquet", "price", Some("aggressor_side"))]
3171    #[case("bars.parquet", "open", None)]
3172    #[case("deltas.parquet", "price", Some("action"))]
3173    fn legacy_market_fixture_matches_open_types(
3174        #[case] file_name: &str,
3175        #[case] fixed_field: &str,
3176        #[case] enum_field: Option<&str>,
3177    ) {
3178        let precision_dir = if cfg!(feature = "high-precision") {
3179            "128-bit"
3180        } else {
3181            "64-bit"
3182        };
3183
3184        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
3185            .join("../../test_data/nautilus/legacy")
3186            .join(precision_dir)
3187            .join(file_name);
3188        let file = std::fs::File::open(path).unwrap();
3189        let batch = ParquetRecordBatchReaderBuilder::try_new(file)
3190            .unwrap()
3191            .build()
3192            .unwrap()
3193            .next()
3194            .unwrap()
3195            .unwrap();
3196
3197        let normalized = normalize_legacy_parquet_columns(&batch).unwrap();
3198
3199        assert_eq!(
3200            normalized
3201                .schema()
3202                .field_with_name(fixed_field)
3203                .unwrap()
3204                .data_type(),
3205            &DataType::Decimal128(38, 16),
3206        );
3207        assert_eq!(
3208            normalized
3209                .schema()
3210                .field_with_name("ts_init")
3211                .unwrap()
3212                .data_type(),
3213            &DataType::Timestamp(arrow::datatypes::TimeUnit::Nanosecond, Some("UTC".into())),
3214        );
3215
3216        if let Some(enum_field) = enum_field {
3217            assert!(matches!(
3218                normalized
3219                    .schema()
3220                    .field_with_name(enum_field)
3221                    .unwrap()
3222                    .data_type(),
3223                DataType::Dictionary(_, value) if value.as_ref() == &DataType::Utf8
3224            ));
3225        }
3226    }
3227
3228    #[rstest]
3229    #[case("depths.parquet")]
3230    #[case("quotes.parquet")]
3231    #[case("trades.parquet")]
3232    #[case("bars.parquet")]
3233    #[case("deltas.parquet")]
3234    #[case("dictionary-trade")]
3235    fn legacy_fixture_schema_normalization_matches_batch(#[case] file_name: &str) {
3236        let (schema, batch) = if file_name == "dictionary-trade" {
3237            let dictionary = |value: &str| {
3238                let mut builder = StringDictionaryBuilder::<Int8Type>::new();
3239                builder.append(value).unwrap();
3240                Arc::new(builder.finish()) as ArrayRef
3241            };
3242
3243            let price = 11_i64.to_le_bytes();
3244            let size = 22_u64.to_le_bytes();
3245            let trade_ids = dictionary("trade-1");
3246            let identifiers = dictionary("AAPL.XNAS");
3247
3248            let schema = Arc::new(Schema::new_with_metadata(
3249                vec![
3250                    Field::new("price", DataType::FixedSizeBinary(8), false),
3251                    Field::new("size", DataType::FixedSizeBinary(8), false),
3252                    Field::new("aggressor_side", DataType::UInt8, false),
3253                    Field::new("trade_id", trade_ids.data_type().clone(), false),
3254                    Field::new("ts_event", DataType::UInt64, false),
3255                    Field::new("ts_init", DataType::UInt64, false),
3256                    Field::new(KEY_IDENTIFIER, identifiers.data_type().clone(), false),
3257                ],
3258                HashMap::from([("type".to_string(), "TradeTick".to_string())]),
3259            ));
3260            let batch = RecordBatch::try_new(
3261                Arc::clone(&schema),
3262                vec![
3263                    Arc::new(
3264                        FixedSizeBinaryArray::try_from_sparse_iter_with_size(
3265                            [Some(price.as_slice())].into_iter(),
3266                            8,
3267                        )
3268                        .unwrap(),
3269                    ),
3270                    Arc::new(
3271                        FixedSizeBinaryArray::try_from_sparse_iter_with_size(
3272                            [Some(size.as_slice())].into_iter(),
3273                            8,
3274                        )
3275                        .unwrap(),
3276                    ),
3277                    Arc::new(UInt8Array::from(vec![1])),
3278                    trade_ids,
3279                    Arc::new(UInt64Array::from(vec![1])),
3280                    Arc::new(UInt64Array::from(vec![2])),
3281                    identifiers,
3282                ],
3283            )
3284            .unwrap();
3285            (schema, batch)
3286        } else {
3287            let precision_dir = if cfg!(feature = "high-precision") {
3288                "128-bit"
3289            } else {
3290                "64-bit"
3291            };
3292
3293            let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
3294                .join("../../test_data/nautilus/legacy")
3295                .join(precision_dir)
3296                .join(file_name);
3297            let file = std::fs::File::open(path).unwrap();
3298            let builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
3299            let schema = builder.schema().clone();
3300            let batch = builder.build().unwrap().next().unwrap().unwrap();
3301            let batch =
3302                RecordBatch::try_new(Arc::clone(&schema), batch.columns().to_vec()).unwrap();
3303            (schema, batch)
3304        };
3305
3306        let normalized_schema = normalize_legacy_parquet_schema(schema.as_ref());
3307        let normalized_batch = normalize_legacy_parquet_columns(&batch).unwrap();
3308
3309        assert_eq!(
3310            normalized_schema,
3311            normalized_batch.schema().as_ref().clone()
3312        );
3313
3314        if file_name == "dictionary-trade" {
3315            assert_eq!(
3316                normalized_batch
3317                    .schema()
3318                    .field_with_name("trade_id")
3319                    .unwrap()
3320                    .data_type(),
3321                &DataType::Utf8,
3322            );
3323        }
3324    }
3325
3326    fn assert_normalized_depth(
3327        batch: &RecordBatch,
3328        level_count: usize,
3329        price: i128,
3330        size: i128,
3331        count: u32,
3332        order_id: u64,
3333    ) {
3334        let schema = batch.schema();
3335        assert_eq!(schema.field(0).name(), "bids");
3336        assert_eq!(schema.field(1).name(), "asks");
3337
3338        for side in ["bids", "asks"] {
3339            let list = batch
3340                .column_by_name(side)
3341                .unwrap()
3342                .as_any()
3343                .downcast_ref::<ListArray>()
3344                .unwrap();
3345            let levels = list.value(0);
3346            let levels = levels.as_any().downcast_ref::<StructArray>().unwrap();
3347            let prices = levels
3348                .column_by_name("price")
3349                .unwrap()
3350                .as_any()
3351                .downcast_ref::<Decimal128Array>()
3352                .unwrap();
3353            let sizes = levels
3354                .column_by_name("size")
3355                .unwrap()
3356                .as_any()
3357                .downcast_ref::<Decimal128Array>()
3358                .unwrap();
3359            let counts = levels
3360                .column_by_name("count")
3361                .unwrap()
3362                .as_any()
3363                .downcast_ref::<UInt32Array>()
3364                .unwrap();
3365            let order_ids = levels
3366                .column_by_name("order_id")
3367                .unwrap()
3368                .as_any()
3369                .downcast_ref::<UInt64Array>()
3370                .unwrap();
3371
3372            assert_eq!(levels.len(), level_count);
3373            assert_eq!(prices.value(0), price);
3374            assert_eq!(sizes.value(0), size);
3375            assert_eq!(counts.value(0), count);
3376            assert_eq!(order_ids.value(0), order_id);
3377        }
3378    }
3379}