1use std::{
19 collections::{BTreeMap, HashMap},
20 fmt::{Display, Write},
21 str::FromStr,
22 sync::Arc,
23};
24
25use arrow::{
26 array::{
27 Array, ArrayRef, BinaryArray, Decimal128Array, FixedSizeBinaryArray, FixedSizeListArray,
28 StringArray, StringBuilder, TimestampNanosecondArray, UInt8Array, UInt16Array, UInt64Array,
29 UInt64Builder, new_null_array,
30 },
31 compute::cast,
32 datatypes::{DataType, Field, Schema, TimeUnit},
33 error::ArrowError,
34 record_batch::RecordBatch,
35};
36use nautilus_model::{
37 data::{
38 Bar, DEPTH10_LEN, FundingRateUpdate, IndexPriceUpdate, InstrumentClose, InstrumentStatus,
39 MarkPriceUpdate, OptionGreeks, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick,
40 },
41 enums::{AggressorSide, BookAction, FromU8, InstrumentCloseType, OrderSide},
42 types::Price,
43};
44
45use super::{
46 ArrowSchemaProvider, FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE, KEY_IDENTIFIER,
47 KEY_INSTRUMENT_ID, KEY_PRICE_PRECISION, STANDARD_TO_DECIMAL_SCALE, enum_dictionary_array,
48 enum_dictionary_data_type, fixed_decimal_data_type, price_decimal_array, timestamp_column,
49 timestamp_data_type,
50};
51
52#[derive(Clone, Debug, Eq, Hash, PartialEq)]
54pub struct SchemaFingerprint(String);
55
56impl Display for SchemaFingerprint {
57 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
58 f.write_str(&self.0)
59 }
60}
61
62#[derive(Clone, Copy, Debug, Eq, PartialEq)]
64pub enum LegacyTranscodeKind {
65 PassThrough,
66 InstrumentStatusV1,
67 FundingRateUpdateV1,
68 InstrumentCloseV1,
69}
70
71#[derive(Debug)]
73pub struct LegacyTranscodeResult {
74 pub batches: Vec<RecordBatch>,
75 pub kind: LegacyTranscodeKind,
76}
77
78#[derive(Debug, Default)]
80pub struct LegacyTranscodeState {
81 instrument_close_precisions: BTreeMap<String, u8>,
82}
83
84#[derive(Clone, Debug, Eq, PartialEq)]
86pub struct LegacySchemaResolution {
87 pub kind: LegacyTranscodeKind,
88 pub source_fingerprint: SchemaFingerprint,
89 pub target_fingerprint: SchemaFingerprint,
90}
91
92#[derive(Debug, thiserror::Error)]
94pub enum LegacyArrowError {
95 #[error(
96 "No Arrow transcoder is registered for type {type_name}, file {file_path}, \
97 and schema fingerprint {fingerprint}"
98 )]
99 UnknownSchema {
100 type_name: String,
101 file_path: String,
102 fingerprint: SchemaFingerprint,
103 },
104 #[error("Failed to transcode type {type_name} from {file_path}: {message}")]
105 Transcode {
106 type_name: String,
107 file_path: String,
108 message: String,
109 },
110}
111
112#[must_use]
114pub fn schema_fingerprint(schema: &Schema) -> SchemaFingerprint {
115 let mut value = String::new();
116
117 for field in schema.fields() {
118 write!(
119 value,
120 "{}:{:?}:{};",
121 field.name(),
122 field.data_type(),
123 field.is_nullable()
124 )
125 .expect("writing a schema fingerprint to a string cannot fail");
126 }
127 SchemaFingerprint(value)
128}
129
130pub fn transcode_legacy_record_batch(
138 type_name: &str,
139 file_path: &str,
140 batch: RecordBatch,
141) -> Result<LegacyTranscodeResult, LegacyArrowError> {
142 transcode_legacy_record_batch_with_state(
143 type_name,
144 file_path,
145 batch,
146 &mut LegacyTranscodeState::default(),
147 )
148}
149
150pub fn transcode_legacy_record_batch_with_state(
158 type_name: &str,
159 file_path: &str,
160 batch: RecordBatch,
161 state: &mut LegacyTranscodeState,
162) -> Result<LegacyTranscodeResult, LegacyArrowError> {
163 let resolution = resolve_legacy_schema(type_name, file_path, batch.schema().as_ref())?;
164 if resolution.kind == LegacyTranscodeKind::PassThrough {
165 return Ok(pass_through(batch));
166 }
167 let batches = match resolution.kind {
168 LegacyTranscodeKind::InstrumentStatusV1 => {
169 vec![transcode_instrument_status(&batch, file_path)?]
170 }
171 LegacyTranscodeKind::FundingRateUpdateV1 => {
172 vec![transcode_funding_rate(&batch, file_path)?]
173 }
174 LegacyTranscodeKind::InstrumentCloseV1 => {
175 transcode_instrument_close(&batch, file_path, state)?
176 }
177 LegacyTranscodeKind::PassThrough => {
178 unreachable!("pass-through schemas are handled before legacy registry lookup")
179 }
180 };
181
182 Ok(LegacyTranscodeResult {
183 batches,
184 kind: resolution.kind,
185 })
186}
187
188pub fn resolve_legacy_schema(
194 type_name: &str,
195 file_path: &str,
196 schema: &Schema,
197) -> Result<LegacySchemaResolution, LegacyArrowError> {
198 let source_fingerprint = schema_fingerprint(schema);
199 let Some(current_schema) = current_schema(type_name, schema.metadata().clone()) else {
200 return Ok(LegacySchemaResolution {
201 kind: LegacyTranscodeKind::PassThrough,
202 target_fingerprint: source_fingerprint.clone(),
203 source_fingerprint,
204 });
205 };
206 let target_fingerprint = schema_fingerprint(¤t_schema);
207
208 let kind = if let Some(kind) = registered_legacy_kind(type_name, &source_fingerprint) {
210 kind
211 } else if matches_current_schema(schema, ¤t_schema) {
212 LegacyTranscodeKind::PassThrough
213 } else {
214 return Err(LegacyArrowError::UnknownSchema {
215 type_name: type_name.to_string(),
216 file_path: file_path.to_string(),
217 fingerprint: source_fingerprint,
218 });
219 };
220
221 Ok(LegacySchemaResolution {
222 kind,
223 source_fingerprint,
224 target_fingerprint,
225 })
226}
227
228fn matches_current_schema(source: &Schema, current: &Schema) -> bool {
231 let current_columns_present = current
232 .fields()
233 .iter()
234 .filter(|field| field.name() != KEY_IDENTIFIER)
235 .all(|expected| {
236 source.field_with_name(expected.name()).is_ok_and(|field| {
237 field.data_type() == expected.data_type()
238 || (matches!(expected.data_type(), DataType::Dictionary(_, value_type)
239 if value_type.as_ref() == &DataType::Utf8)
240 && field.data_type() == &DataType::Utf8)
241 })
242 });
243
244 current_columns_present
245 && source.fields().iter().all(|field| {
246 current.field_with_name(field.name()).is_ok()
247 || matches!(field.name().as_str(), KEY_IDENTIFIER | KEY_INSTRUMENT_ID)
248 })
249}
250
251fn pass_through(batch: RecordBatch) -> LegacyTranscodeResult {
252 LegacyTranscodeResult {
253 batches: vec![batch],
254 kind: LegacyTranscodeKind::PassThrough,
255 }
256}
257
258fn current_schema(type_name: &str, metadata: HashMap<String, String>) -> Option<Schema> {
259 let metadata = Some(metadata);
260 match type_name {
261 "quotes" => Some(QuoteTick::get_schema(metadata)),
262 "trades" => Some(TradeTick::get_schema(metadata)),
263 "bars" => Some(Bar::get_schema(metadata)),
264 "order_book_deltas" => Some(OrderBookDelta::get_schema(metadata)),
265 "order_book_depths" => Some(OrderBookDepth::get_schema(metadata)),
266 "mark_prices" => Some(MarkPriceUpdate::get_schema(metadata)),
267 "index_prices" => Some(IndexPriceUpdate::get_schema(metadata)),
268 "funding_rates" => Some(FundingRateUpdate::get_schema(metadata)),
269 "instrument_status" => Some(InstrumentStatus::get_schema(metadata)),
270 "option_greeks" => Some(OptionGreeks::get_schema(metadata)),
271 "instrument_closes" => Some(InstrumentClose::get_schema(metadata)),
272 _ => None,
273 }
274}
275
276fn registered_legacy_kind(
277 type_name: &str,
278 fingerprint: &SchemaFingerprint,
279) -> Option<LegacyTranscodeKind> {
280 let entries = [
281 (
282 "instrument_status",
283 legacy_instrument_status_schema(),
284 LegacyTranscodeKind::InstrumentStatusV1,
285 ),
286 (
287 "funding_rates",
288 legacy_funding_rate_schema(),
289 LegacyTranscodeKind::FundingRateUpdateV1,
290 ),
291 (
292 "instrument_closes",
293 legacy_instrument_close_schema(),
294 LegacyTranscodeKind::InstrumentCloseV1,
295 ),
296 ];
297
298 entries
299 .into_iter()
300 .find(|(entry_type, schema, _)| {
301 type_name == *entry_type
302 && (schema_fingerprint(schema) == *fingerprint
303 || schema_fingerprint(&schema_with_normalized_legacy_types(schema))
304 == *fingerprint)
305 })
306 .map(|(_, _, kind)| kind)
307}
308
309fn schema_with_normalized_legacy_types(schema: &Schema) -> Schema {
310 let fields = schema
311 .fields()
312 .iter()
313 .map(|field| {
314 Arc::new(
315 field
316 .as_ref()
317 .clone()
318 .with_data_type(normalized_legacy_data_type(field.name(), field.data_type())),
319 )
320 })
321 .collect::<Vec<_>>();
322 Schema::new_with_metadata(fields, schema.metadata().clone())
323}
324
325fn legacy_instrument_status_schema() -> Schema {
326 Schema::new(vec![
327 Field::new("instrument_id", DataType::Utf8, true),
328 Field::new("action", DataType::Utf8, true),
329 Field::new("reason", DataType::Utf8, true),
330 Field::new("trading_event", DataType::Utf8, true),
331 Field::new("is_trading", DataType::Boolean, true),
332 Field::new("is_quoting", DataType::Boolean, true),
333 Field::new("is_short_sell_restricted", DataType::Boolean, true),
334 Field::new("ts_event", DataType::UInt64, true),
335 Field::new("ts_init", DataType::UInt64, true),
336 ])
337}
338
339fn legacy_funding_rate_schema() -> Schema {
340 Schema::new(vec![
341 Field::new("rate", DataType::Binary, false),
342 Field::new("interval", DataType::UInt16, true),
343 Field::new("next_funding_ns", DataType::UInt64, true),
344 Field::new("ts_event", DataType::UInt64, false),
345 Field::new("ts_init", DataType::UInt64, false),
346 ])
347}
348
349fn legacy_instrument_close_schema() -> Schema {
350 Schema::new(vec![
351 Field::new("instrument_id", DataType::Utf8, true),
352 Field::new("close_type", DataType::Utf8, true),
353 Field::new("close_price", DataType::Utf8, true),
354 Field::new("ts_event", DataType::UInt64, true),
355 Field::new("ts_init", DataType::UInt64, true),
356 ])
357}
358
359fn transcode_instrument_status(
360 batch: &RecordBatch,
361 file_path: &str,
362) -> Result<RecordBatch, LegacyArrowError> {
363 let schema = InstrumentStatus::get_schema(Some(batch.schema().metadata().clone()));
364 let instrument_id = required_column(batch, "instrument_id", file_path)?;
365 let mut columns = Vec::with_capacity(schema.fields().len());
366
367 for field in schema.fields() {
368 let column = source_status_column(batch, field, &instrument_id, file_path)?;
369 let convertible_timestamp = field.data_type() == &super::timestamp_data_type()
370 && column.data_type() == &DataType::UInt64;
371 if column.data_type() != field.data_type() && !convertible_timestamp {
372 return Err(transcode_error(
373 "instrument_status",
374 file_path,
375 format!(
376 "field {} expected {:?}, found {:?}",
377 field.name(),
378 field.data_type(),
379 column.data_type()
380 ),
381 ));
382 }
383
384 if !field.is_nullable() && column.null_count() > 0 {
385 return Err(transcode_error(
386 "instrument_status",
387 file_path,
388 format!("required field {} contains nulls", field.name()),
389 ));
390 }
391 columns.push(column);
392 }
393
394 super::record_batch_with_timestamps(Arc::new(schema), columns)
395 .map_err(|e| transcode_error("instrument_status", file_path, e.to_string()))
396}
397
398fn source_status_column(
399 batch: &RecordBatch,
400 field: &Field,
401 instrument_id: &ArrayRef,
402 file_path: &str,
403) -> Result<ArrayRef, LegacyArrowError> {
404 if field.name() == KEY_IDENTIFIER {
405 return Ok(Arc::clone(instrument_id));
406 }
407
408 let Some(column) = batch.column_by_name(field.name()) else {
409 if field.is_nullable() {
410 return Ok(new_null_array(field.data_type(), batch.num_rows()));
411 }
412
413 return Err(transcode_error(
414 "instrument_status",
415 file_path,
416 format!("required field {} is absent", field.name()),
417 ));
418 };
419
420 if field.data_type() == &enum_dictionary_data_type() && column.data_type() == &DataType::Utf8 {
421 return cast(column, field.data_type())
422 .map_err(|e| transcode_error("instrument_status", file_path, e.to_string()));
423 }
424
425 Ok(Arc::clone(column))
426}
427
428fn transcode_funding_rate(
429 batch: &RecordBatch,
430 file_path: &str,
431) -> Result<RecordBatch, LegacyArrowError> {
432 let instrument_id = batch
433 .schema()
434 .metadata()
435 .get(KEY_INSTRUMENT_ID)
436 .cloned()
437 .ok_or_else(|| {
438 transcode_error(
439 "funding_rates",
440 file_path,
441 "instrument_id schema metadata is absent",
442 )
443 })?;
444 let rate = required_typed_column::<BinaryArray>(batch, "rate", file_path, "funding_rates")?;
445 let interval =
446 required_typed_column::<UInt16Array>(batch, "interval", file_path, "funding_rates")?;
447 let mut rate_builder = StringBuilder::with_capacity(batch.num_rows(), rate.value_data().len());
448
449 for row in 0..batch.num_rows() {
450 if rate.is_null(row) {
451 return Err(transcode_error(
452 "funding_rates",
453 file_path,
454 format!("required field rate is null at row {row}"),
455 ));
456 }
457 let value = serde_json::from_slice::<String>(rate.value(row)).map_err(|e| {
458 transcode_error(
459 "funding_rates",
460 file_path,
461 format!("rate at row {row} is not a msgspec JSON string: {e}"),
462 )
463 })?;
464 rate_builder.append_value(value);
465 }
466
467 let schema = FundingRateUpdate::get_schema(Some(batch.schema().metadata().clone()));
468 let identifiers = StringArray::from(vec![instrument_id; batch.num_rows()]);
469 let columns: Vec<ArrayRef> = vec![
470 Arc::new(rate_builder.finish()),
471 cast(interval, &DataType::UInt64)
472 .map_err(|e| transcode_error("funding_rates", file_path, e.to_string()))?,
473 required_column(batch, "next_funding_ns", file_path)?,
474 required_column(batch, "ts_event", file_path)?,
475 required_column(batch, "ts_init", file_path)?,
476 Arc::new(identifiers),
477 ];
478
479 super::record_batch_with_timestamps(Arc::new(schema), columns)
480 .map_err(|e| transcode_error("funding_rates", file_path, e.to_string()))
481}
482
483fn transcode_instrument_close(
484 batch: &RecordBatch,
485 file_path: &str,
486 state: &mut LegacyTranscodeState,
487) -> Result<Vec<RecordBatch>, LegacyArrowError> {
488 let normalized = super::record_batch_with_u64_timestamps(batch)
489 .map_err(|e| transcode_error("instrument_closes", file_path, e.to_string()))?;
490 let batch = &normalized;
491 let instrument_ids = required_typed_column::<StringArray>(
492 batch,
493 "instrument_id",
494 file_path,
495 "instrument_closes",
496 )?;
497 let close_types =
498 required_typed_column::<StringArray>(batch, "close_type", file_path, "instrument_closes")?;
499 let close_prices =
500 required_typed_column::<StringArray>(batch, "close_price", file_path, "instrument_closes")?;
501 let ts_events =
502 required_typed_column::<UInt64Array>(batch, "ts_event", file_path, "instrument_closes")?;
503 let ts_inits =
504 required_typed_column::<UInt64Array>(batch, "ts_init", file_path, "instrument_closes")?;
505 let mut rows_by_instrument: BTreeMap<String, Vec<usize>> = BTreeMap::new();
506
507 for row in 0..batch.num_rows() {
508 if instrument_ids.is_null(row) {
509 return Err(transcode_error(
510 "instrument_closes",
511 file_path,
512 format!("instrument_id is null at row {row}"),
513 ));
514 }
515 rows_by_instrument
516 .entry(instrument_ids.value(row).to_string())
517 .or_default()
518 .push(row);
519 }
520
521 let mut batches = Vec::with_capacity(rows_by_instrument.len());
522 for (instrument_id, rows) in rows_by_instrument {
523 let (batch, precision) = transcode_instrument_close_rows(
524 file_path,
525 &instrument_id,
526 &rows,
527 close_types,
528 close_prices,
529 ts_events,
530 ts_inits,
531 state
532 .instrument_close_precisions
533 .get(&instrument_id)
534 .copied(),
535 )?;
536 state
537 .instrument_close_precisions
538 .insert(instrument_id, precision);
539 batches.push(batch);
540 }
541 Ok(batches)
542}
543
544#[expect(
545 clippy::too_many_arguments,
546 reason = "the arguments are the validated source columns for one legacy close batch"
547)]
548fn transcode_instrument_close_rows(
549 file_path: &str,
550 instrument_id: &str,
551 rows: &[usize],
552 close_types: &StringArray,
553 close_prices: &StringArray,
554 ts_events: &UInt64Array,
555 ts_inits: &UInt64Array,
556 expected_precision: Option<u8>,
557) -> Result<(RecordBatch, u8), LegacyArrowError> {
558 let mut prices = Vec::with_capacity(rows.len());
559 let mut types = Vec::with_capacity(rows.len());
560 let mut event_builder = UInt64Builder::with_capacity(rows.len());
561 let mut init_builder = UInt64Builder::with_capacity(rows.len());
562 let mut precision = expected_precision;
563
564 for &row in rows {
565 if close_types.is_null(row)
566 || close_prices.is_null(row)
567 || ts_events.is_null(row)
568 || ts_inits.is_null(row)
569 {
570 return Err(transcode_error(
571 "instrument_closes",
572 file_path,
573 format!("required close field is null at row {row}"),
574 ));
575 }
576
577 let value = close_prices.value(row);
578 let row_precision = decimal_precision(value).map_err(|message| {
579 transcode_error(
580 "instrument_closes",
581 file_path,
582 format!("invalid close_price {value:?} at row {row}: {message}"),
583 )
584 })?;
585
586 if let Some(existing) = precision
587 && existing != row_precision
588 {
589 return Err(transcode_error(
590 "instrument_closes",
591 file_path,
592 format!(
593 "close_price precision conflict at row {row}: found {row_precision}, \
594 expected {existing}"
595 ),
596 ));
597 }
598 precision = Some(row_precision);
599
600 let price = Price::from_str(value).map_err(|e| {
601 transcode_error(
602 "instrument_closes",
603 file_path,
604 format!("invalid close_price {value:?} at row {row}: {e}"),
605 )
606 })?;
607 let close_type = InstrumentCloseType::from_str(close_types.value(row)).map_err(|e| {
608 transcode_error(
609 "instrument_closes",
610 file_path,
611 format!(
612 "unknown close_type {:?} at row {row}: {e}",
613 close_types.value(row)
614 ),
615 )
616 })?;
617
618 prices.push(price.raw());
619 types.push(close_type);
620 event_builder.append_value(ts_events.value(row));
621 init_builder.append_value(ts_inits.value(row));
622 }
623
624 let precision = precision.unwrap_or(0);
625 let metadata = HashMap::from([
626 (KEY_INSTRUMENT_ID.to_string(), instrument_id.to_string()),
627 (KEY_PRICE_PRECISION.to_string(), precision.to_string()),
628 ]);
629 let schema = InstrumentClose::get_schema(Some(metadata));
630 let identifiers = StringArray::from(vec![instrument_id; rows.len()]);
631
632 let batch = super::record_batch_with_timestamps(
633 Arc::new(schema),
634 vec![
635 Arc::new(
636 price_decimal_array(prices, "close_price")
637 .map_err(|e| transcode_error("instrument_closes", file_path, e.to_string()))?,
638 ),
639 Arc::new(
640 enum_dictionary_array(types)
641 .map_err(|e| transcode_error("instrument_closes", file_path, e.to_string()))?,
642 ),
643 Arc::new(event_builder.finish()),
644 Arc::new(init_builder.finish()),
645 Arc::new(identifiers),
646 ],
647 )
648 .map_err(|e| transcode_error("instrument_closes", file_path, e.to_string()))?;
649 Ok((batch, precision))
650}
651
652fn decimal_precision(value: &str) -> Result<u8, String> {
653 let precision = value
654 .split_once('.')
655 .map_or(0, |(_, fractional)| fractional.len());
656 u8::try_from(precision).map_err(|e| e.to_string())
657}
658
659fn required_column(
660 batch: &RecordBatch,
661 name: &str,
662 file_path: &str,
663) -> Result<ArrayRef, LegacyArrowError> {
664 batch.column_by_name(name).cloned().ok_or_else(|| {
665 transcode_error(
666 "legacy_arrow",
667 file_path,
668 format!("required field {name} is absent"),
669 )
670 })
671}
672
673fn required_typed_column<'a, T: Array + 'static>(
674 batch: &'a RecordBatch,
675 name: &str,
676 file_path: &str,
677 type_name: &str,
678) -> Result<&'a T, LegacyArrowError> {
679 let column = batch.column_by_name(name).ok_or_else(|| {
680 transcode_error(
681 type_name,
682 file_path,
683 format!("required field {name} is absent"),
684 )
685 })?;
686 column.as_any().downcast_ref::<T>().ok_or_else(|| {
687 transcode_error(
688 type_name,
689 file_path,
690 format!("field {name} has unexpected type {:?}", column.data_type()),
691 )
692 })
693}
694
695fn transcode_error(
696 type_name: &str,
697 file_path: &str,
698 message: impl Into<String>,
699) -> LegacyArrowError {
700 LegacyArrowError::Transcode {
701 type_name: type_name.to_string(),
702 file_path: file_path.to_string(),
703 message: message.into(),
704 }
705}
706
707#[must_use]
709pub fn normalized_legacy_data_type(name: &str, data_type: &DataType) -> DataType {
710 match data_type {
711 DataType::FixedSizeBinary(8 | 16) => fixed_decimal_data_type(),
712 DataType::UInt64 if is_timestamp_field(name) => timestamp_data_type(),
713 DataType::Timestamp(TimeUnit::Nanosecond, None) if is_timestamp_field(name) => {
714 timestamp_data_type()
715 }
716 DataType::UInt8 if is_legacy_enum_field(name) => enum_dictionary_data_type(),
717 _ => data_type.clone(),
718 }
719}
720
721pub fn normalize_legacy_fixed_columns(batch: &RecordBatch) -> Result<RecordBatch, ArrowError> {
727 let mut changed = false;
728 let mut fields = Vec::with_capacity(batch.num_columns());
729 let mut columns = Vec::with_capacity(batch.num_columns());
730 let normalize_named_fields = is_nautilus_legacy_schema(batch.schema_ref());
731 let normalize_timestamps = is_nautilus_timestamp_schema(batch.schema_ref());
732
733 for (field, column) in batch.schema().fields().iter().zip(batch.columns()) {
734 let custom_timestamp = batch.schema().metadata().contains_key("type_name")
735 && matches!(field.name().as_str(), "ts_event" | "ts_init");
736 if field.data_type() == &DataType::UInt64
737 && ((normalize_named_fields && is_timestamp_field(field.name())) || custom_timestamp)
738 {
739 let timestamps = timestamp_column(field, column.as_ref())?;
740 fields.push(Arc::new(field.as_ref().clone().with_data_type(
741 normalized_legacy_data_type(field.name(), field.data_type()),
742 )));
743 columns.push(timestamps);
744 changed = true;
745 continue;
746 }
747
748 if normalize_timestamps
749 && normalized_timestamp_type(field.data_type()) != *field.data_type()
750 {
751 fields.push(Arc::new(
752 field.as_ref().clone().with_data_type(timestamp_data_type()),
753 ));
754 let timestamps = column
755 .as_any()
756 .downcast_ref::<TimestampNanosecondArray>()
757 .ok_or_else(|| {
758 ArrowError::CastError(format!(
759 "Column '{}' is not a nanosecond timestamp",
760 field.name()
761 ))
762 })?;
763 columns.push(Arc::new(
764 timestamps.clone().with_data_type(timestamp_data_type()),
765 ));
766 changed = true;
767 continue;
768 }
769
770 if normalize_named_fields
771 && field.data_type() == &DataType::UInt8
772 && is_legacy_enum_field(field.name())
773 {
774 fields.push(Arc::new(field.as_ref().clone().with_data_type(
775 normalized_legacy_data_type(field.name(), field.data_type()),
776 )));
777 columns.push(legacy_enum_dictionary_column(field, column.as_ref())?);
778 changed = true;
779 continue;
780 }
781
782 if normalize_named_fields
783 && let DataType::FixedSizeList(item, length) = field.data_type()
784 && let DataType::FixedSizeBinary(width @ (8 | 16)) = item.data_type()
785 {
786 let list = column
787 .as_any()
788 .downcast_ref::<FixedSizeListArray>()
789 .ok_or_else(|| {
790 ArrowError::CastError(format!("Column '{}' is not FixedSizeList", field.name()))
791 })?;
792 let values = list
793 .values()
794 .as_any()
795 .downcast_ref::<FixedSizeBinaryArray>()
796 .ok_or_else(|| {
797 ArrowError::CastError(format!(
798 "Column '{}' values are not FixedSizeBinary",
799 field.name()
800 ))
801 })?;
802 let decimal = normalize_legacy_fixed_array(field.name(), values, *width)?;
803 let item = Arc::new(
804 item.as_ref()
805 .clone()
806 .with_data_type(fixed_decimal_data_type())
807 .with_nullable(true),
808 );
809 let list = FixedSizeListArray::try_new(
810 Arc::clone(&item),
811 *length,
812 Arc::new(decimal),
813 list.nulls().cloned(),
814 )?;
815 fields.push(Arc::new(
816 field
817 .as_ref()
818 .clone()
819 .with_data_type(DataType::FixedSizeList(item, *length)),
820 ));
821 columns.push(Arc::new(list) as ArrayRef);
822 changed = true;
823 continue;
824 }
825
826 let DataType::FixedSizeBinary(width @ (8 | 16)) = field.data_type() else {
827 fields.push(Arc::clone(field));
828 columns.push(Arc::clone(column));
829 continue;
830 };
831
832 if !normalize_named_fields {
833 fields.push(Arc::clone(field));
834 columns.push(Arc::clone(column));
835 continue;
836 }
837 let values = column
838 .as_any()
839 .downcast_ref::<FixedSizeBinaryArray>()
840 .ok_or_else(|| {
841 ArrowError::CastError(format!("Column '{}' is not FixedSizeBinary", field.name()))
842 })?;
843 let decimal = normalize_legacy_fixed_array(field.name(), values, *width)?;
844 fields.push(Arc::new(
845 field
846 .as_ref()
847 .clone()
848 .with_data_type(normalized_legacy_data_type(field.name(), field.data_type()))
849 .with_nullable(true),
850 ));
851 columns.push(Arc::new(decimal) as ArrayRef);
852 changed = true;
853 }
854
855 if !changed {
856 return Ok(batch.clone());
857 }
858
859 let schema = Arc::new(Schema::new_with_metadata(
860 fields,
861 batch.schema().metadata().clone(),
862 ));
863 RecordBatch::try_new(schema, columns)
864}
865
866pub(super) fn legacy_enum_dictionary_column(
867 field: &Field,
868 column: &dyn Array,
869) -> Result<ArrayRef, ArrowError> {
870 let values = column
871 .as_any()
872 .downcast_ref::<UInt8Array>()
873 .ok_or_else(|| ArrowError::CastError(format!("Column '{}' is not UInt8", field.name())))?;
874
875 let names = (0..values.len())
876 .map(|row| {
877 if values.is_null(row) {
878 return Err(ArrowError::CastError(format!(
879 "Legacy enum column '{}' is null at row {row}",
880 field.name()
881 )));
882 }
883
884 legacy_enum_name(field.name(), values.value(row))
885 })
886 .collect::<Result<Vec<_>, _>>()?;
887 Ok(Arc::new(enum_dictionary_array(names)?) as ArrayRef)
888}
889
890fn normalize_legacy_fixed_array(
891 name: &str,
892 values: &FixedSizeBinaryArray,
893 width: i32,
894) -> Result<Decimal128Array, ArrowError> {
895 let quantity = is_legacy_quantity_field(name);
896 let mut decimals = Vec::with_capacity(values.len());
897
898 for row in 0..values.len() {
899 if values.is_null(row) {
900 decimals.push(None);
901 continue;
902 }
903
904 let bytes = values.value(row);
905 let decimal = match (width, quantity) {
906 (8, false) => {
907 let raw = i64::from_le_bytes(bytes.try_into().map_err(|e| {
908 ArrowError::CastError(format!(
909 "Invalid legacy price column '{name}' at row {row}: {e}"
910 ))
911 })?);
912
913 if raw == i64::MIN {
914 return Err(ArrowError::CastError(format!(
915 "Legacy price column '{name}' contains PRICE_ERROR raw value {raw} at row {row}",
916 )));
917 }
918 (raw != i64::MAX).then_some(i128::from(raw) * STANDARD_TO_DECIMAL_SCALE)
919 }
920 (8, true) => {
921 let raw = u64::from_le_bytes(bytes.try_into().map_err(|e| {
922 ArrowError::CastError(format!(
923 "Invalid legacy quantity column '{name}' at row {row}: {e}"
924 ))
925 })?);
926 (raw != u64::MAX).then_some(i128::from(raw) * STANDARD_TO_DECIMAL_SCALE)
927 }
928 (16, false) => {
929 let raw = i128::from_le_bytes(bytes.try_into().map_err(|e| {
930 ArrowError::CastError(format!(
931 "Invalid legacy price column '{name}' at row {row}: {e}"
932 ))
933 })?);
934
935 if raw == i128::MIN {
936 return Err(ArrowError::CastError(format!(
937 "Legacy price column '{name}' contains PRICE_ERROR raw value {raw} at row {row}",
938 )));
939 }
940 (raw != i128::MAX).then_some(raw)
941 }
942 (16, true) => {
943 let raw = u128::from_le_bytes(bytes.try_into().map_err(|e| {
944 ArrowError::CastError(format!(
945 "Invalid legacy quantity column '{name}' at row {row}: {e}"
946 ))
947 })?);
948
949 if raw == u128::MAX {
950 None
951 } else {
952 Some(i128::try_from(raw).map_err(|_| {
953 ArrowError::CastError(format!(
954 "Legacy quantity column '{name}' exceeds Decimal128 at row {row}"
955 ))
956 })?)
957 }
958 }
959 _ => unreachable!("legacy fixed width checked above"),
960 };
961 decimals.push(decimal);
962 }
963
964 Decimal128Array::from(decimals)
965 .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
966}
967
968type LegacySchemaFields = &'static [(&'static str, LegacyFieldType, bool)];
969
970const LEGACY_SCHEMA_FAMILIES: &[(&[&str], LegacySchemaFields)] = &[
971 (
972 &["OrderBookDelta"],
973 &[
974 ("action", LegacyFieldType::UInt8, false),
975 ("side", LegacyFieldType::UInt8, false),
976 ("price", LegacyFieldType::Fixed, false),
977 ("size", LegacyFieldType::Fixed, false),
978 ("order_id", LegacyFieldType::UInt64, false),
979 ("flags", LegacyFieldType::UInt8, false),
980 ("sequence", LegacyFieldType::UInt64, false),
981 ("ts_event", LegacyFieldType::UInt64, false),
982 ("ts_init", LegacyFieldType::UInt64, false),
983 ],
984 ),
985 (
986 &["TradeTick"],
987 &[
988 ("price", LegacyFieldType::Fixed, false),
989 ("size", LegacyFieldType::Fixed, false),
990 ("aggressor_side", LegacyFieldType::UInt8, false),
991 ("trade_id", LegacyFieldType::Utf8, false),
992 ("ts_event", LegacyFieldType::UInt64, false),
993 ("ts_init", LegacyFieldType::UInt64, false),
994 ],
995 ),
996 (
997 &["InstrumentClose"],
998 &[
999 ("close_price", LegacyFieldType::Fixed, false),
1000 ("close_type", LegacyFieldType::UInt8, false),
1001 ("ts_event", LegacyFieldType::UInt64, false),
1002 ("ts_init", LegacyFieldType::UInt64, false),
1003 ],
1004 ),
1005 (
1006 &["InstrumentClose"],
1007 &[
1008 ("instrument_id", LegacyFieldType::Utf8, true),
1009 ("close_type", LegacyFieldType::Utf8, true),
1010 ("close_price", LegacyFieldType::Utf8, true),
1011 ("ts_event", LegacyFieldType::UInt64, true),
1012 ("ts_init", LegacyFieldType::UInt64, true),
1013 ],
1014 ),
1015 (
1016 &["QuoteTick"],
1017 &[
1018 ("bid_price", LegacyFieldType::Fixed, false),
1019 ("ask_price", LegacyFieldType::Fixed, false),
1020 ("bid_size", LegacyFieldType::Fixed, false),
1021 ("ask_size", LegacyFieldType::Fixed, false),
1022 ("ts_event", LegacyFieldType::UInt64, false),
1023 ("ts_init", LegacyFieldType::UInt64, false),
1024 ],
1025 ),
1026 (
1027 &["Bar"],
1028 &[
1029 ("open", LegacyFieldType::Fixed, false),
1030 ("high", LegacyFieldType::Fixed, false),
1031 ("low", LegacyFieldType::Fixed, false),
1032 ("close", LegacyFieldType::Fixed, false),
1033 ("volume", LegacyFieldType::Fixed, false),
1034 ("ts_event", LegacyFieldType::UInt64, false),
1035 ("ts_init", LegacyFieldType::UInt64, false),
1036 ],
1037 ),
1038 (
1039 &["MarkPriceUpdate", "IndexPriceUpdate"],
1040 &[
1041 ("value", LegacyFieldType::Fixed, false),
1042 ("ts_event", LegacyFieldType::UInt64, false),
1043 ("ts_init", LegacyFieldType::UInt64, false),
1044 ],
1045 ),
1046 (
1047 &["FundingRateUpdate"],
1048 &[
1049 ("rate", LegacyFieldType::Binary, false),
1050 ("interval", LegacyFieldType::UInt16, true),
1051 ("next_funding_ns", LegacyFieldType::UInt64, true),
1052 ("ts_event", LegacyFieldType::UInt64, false),
1053 ("ts_init", LegacyFieldType::UInt64, false),
1054 ],
1055 ),
1056 (
1057 &["InstrumentStatus"],
1058 &[
1059 ("instrument_id", LegacyFieldType::Utf8, true),
1060 ("action", LegacyFieldType::Utf8, true),
1061 ("reason", LegacyFieldType::Utf8, true),
1062 ("trading_event", LegacyFieldType::Utf8, true),
1063 ("is_trading", LegacyFieldType::Boolean, true),
1064 ("is_quoting", LegacyFieldType::Boolean, true),
1065 ("is_short_sell_restricted", LegacyFieldType::Boolean, true),
1066 ("ts_event", LegacyFieldType::UInt64, true),
1067 ("ts_init", LegacyFieldType::UInt64, true),
1068 ],
1069 ),
1070 (
1071 &["OptionGreeks"],
1072 &[
1073 ("instrument_id", LegacyFieldType::Utf8, false),
1074 ("delta", LegacyFieldType::Float64, false),
1075 ("gamma", LegacyFieldType::Float64, false),
1076 ("vega", LegacyFieldType::Float64, false),
1077 ("theta", LegacyFieldType::Float64, false),
1078 ("rho", LegacyFieldType::Float64, false),
1079 ("mark_iv", LegacyFieldType::Float64, true),
1080 ("bid_iv", LegacyFieldType::Float64, true),
1081 ("ask_iv", LegacyFieldType::Float64, true),
1082 ("underlying_price", LegacyFieldType::Float64, true),
1083 ("open_interest", LegacyFieldType::Float64, true),
1084 ("ts_event", LegacyFieldType::UInt64, false),
1085 ("ts_init", LegacyFieldType::UInt64, false),
1086 ("convention", LegacyFieldType::Utf8, false),
1087 ],
1088 ),
1089];
1090
1091#[must_use]
1093pub fn is_nautilus_legacy_schema(schema: &Schema) -> bool {
1094 if let Some(type_name) = schema
1095 .metadata()
1096 .get("type_name")
1097 .or_else(|| schema.metadata().get("type"))
1098 {
1099 return legacy_metadata_family_matches(schema, type_name);
1100 }
1101
1102 LEGACY_SCHEMA_FAMILIES
1103 .iter()
1104 .any(|(_, fields)| schema_fingerprint_matches(schema, fields))
1105 || legacy_flat_depth_fingerprint_matches(schema)
1106 || legacy_fixed_list_depth_fingerprint_matches(schema)
1107}
1108
1109#[must_use]
1111pub fn is_nautilus_timestamp_schema(schema: &Schema) -> bool {
1112 if schema.metadata().contains_key("type_name") || schema.metadata().contains_key("type") {
1113 return true;
1114 }
1115
1116 if is_nautilus_legacy_schema(schema) {
1117 return true;
1118 }
1119 let fields = schema
1120 .fields()
1121 .iter()
1122 .filter(|field| field.name() != KEY_IDENTIFIER)
1123 .collect::<Vec<_>>();
1124 LEGACY_SCHEMA_FAMILIES.iter().any(|(_, expected)| {
1125 fields.len() == expected.len()
1126 && fields
1127 .iter()
1128 .zip(*expected)
1129 .all(|(field, (name, expected, _))| {
1130 field.name() == *name
1131 && (legacy_field_type_matches(field.data_type(), *expected)
1132 || (is_timestamp_field(name)
1133 && matches!(
1134 field.data_type(),
1135 DataType::Timestamp(TimeUnit::Nanosecond, _)
1136 ))
1137 || (matches!(expected, LegacyFieldType::Fixed)
1138 && field.data_type() == &fixed_decimal_data_type())
1139 || (is_legacy_enum_field(name)
1140 && field.data_type() == &enum_dictionary_data_type()))
1141 })
1142 })
1143}
1144
1145#[must_use]
1147pub fn normalized_timestamp_type(data_type: &DataType) -> DataType {
1148 match data_type {
1149 DataType::Timestamp(TimeUnit::Nanosecond, None) => timestamp_data_type(),
1150 _ => data_type.clone(),
1151 }
1152}
1153
1154fn legacy_metadata_family_matches(schema: &Schema, type_name: &str) -> bool {
1155 match type_name {
1156 "OrderBookDepth10" | "OrderBookDepth" => {
1157 return legacy_flat_depth_fingerprint_matches(schema)
1158 || legacy_fixed_list_depth_fingerprint_matches(schema);
1159 }
1160 _ => {}
1161 }
1162
1163 LEGACY_SCHEMA_FAMILIES
1164 .iter()
1165 .filter(|(names, _)| names.contains(&type_name))
1166 .any(|(_, fields)| schema_fingerprint_matches(schema, fields))
1167}
1168
1169#[derive(Clone, Copy)]
1170enum LegacyFieldType {
1171 Binary,
1172 Float64,
1173 Fixed,
1174 UInt8,
1175 UInt16,
1176 UInt32,
1177 UInt64,
1178 Utf8,
1179 Boolean,
1180}
1181
1182fn schema_fingerprint_matches(schema: &Schema, expected: &[(&str, LegacyFieldType, bool)]) -> bool {
1183 let fields = schema
1184 .fields()
1185 .iter()
1186 .filter(|field| field.name() != KEY_IDENTIFIER)
1187 .collect::<Vec<_>>();
1188
1189 let fields_match = fields.len() == expected.len()
1190 && fields
1191 .iter()
1192 .zip(expected)
1193 .all(|(field, (name, data_type, _))| {
1194 field.name() == *name && legacy_field_type_matches(field.data_type(), *data_type)
1195 });
1196 let nullability_matches = fields
1197 .iter()
1198 .zip(expected)
1199 .all(|(field, (_, _, nullable))| field.is_nullable() == *nullable)
1200 || fields.iter().all(|field| field.is_nullable());
1201
1202 fields_match
1203 && nullability_matches
1204 && fixed_width_is_consistent(fields.iter().map(|field| field.data_type()))
1205}
1206
1207fn legacy_field_type_matches(data_type: &DataType, expected: LegacyFieldType) -> bool {
1208 match expected {
1209 LegacyFieldType::Binary => data_type == &DataType::Binary,
1210 LegacyFieldType::Float64 => data_type == &DataType::Float64,
1211 LegacyFieldType::Fixed => matches!(data_type, DataType::FixedSizeBinary(8 | 16)),
1212 LegacyFieldType::UInt8 => data_type == &DataType::UInt8,
1213 LegacyFieldType::UInt16 => data_type == &DataType::UInt16,
1214 LegacyFieldType::UInt32 => data_type == &DataType::UInt32,
1215 LegacyFieldType::UInt64 => data_type == &DataType::UInt64,
1216 LegacyFieldType::Utf8 => match data_type {
1217 DataType::Utf8 | DataType::Utf8View => true,
1218 DataType::Dictionary(_, value) => value.as_ref() == &DataType::Utf8,
1219 _ => false,
1220 },
1221 LegacyFieldType::Boolean => data_type == &DataType::Boolean,
1222 }
1223}
1224
1225fn fixed_width_is_consistent<'a>(data_types: impl Iterator<Item = &'a DataType>) -> bool {
1226 let mut width = None;
1227 data_types
1228 .filter_map(|data_type| match data_type {
1229 DataType::FixedSizeBinary(width) => Some(*width),
1230 _ => None,
1231 })
1232 .all(|current| *width.get_or_insert(current) == current)
1233}
1234
1235fn legacy_flat_depth_fingerprint_matches(schema: &Schema) -> bool {
1236 let fields = schema
1237 .fields()
1238 .iter()
1239 .filter(|field| field.name() != KEY_IDENTIFIER)
1240 .collect::<Vec<_>>();
1241 let market_fields = fields
1242 .iter()
1243 .filter(|field| {
1244 matches!(
1245 field.name().as_str(),
1246 "flags" | "sequence" | "ts_event" | "ts_init"
1247 )
1248 })
1249 .count();
1250
1251 let order_id_fields = fields
1252 .iter()
1253 .filter(|field| {
1254 field.name().starts_with("bid_order_id_") || field.name().starts_with("ask_order_id_")
1255 })
1256 .count();
1257 let all_nullable = fields.iter().all(|field| field.is_nullable());
1258
1259 fields.len() == 6 * DEPTH10_LEN + order_id_fields + market_fields
1260 && market_fields <= 4
1261 && matches!(order_id_fields, 0 | 20)
1262 && fixed_width_is_consistent(fields.iter().map(|field| field.data_type()))
1263 && ["bid", "ask"].iter().all(|side| {
1264 (0..DEPTH10_LEN).all(|level| {
1265 [
1266 ("price", LegacyFieldType::Fixed),
1267 ("size", LegacyFieldType::Fixed),
1268 ("count", LegacyFieldType::UInt32),
1269 ]
1270 .iter()
1271 .all(|(value, data_type)| {
1272 schema
1273 .field_with_name(&format!("{side}_{value}_{level}"))
1274 .is_ok_and(|field| {
1275 (all_nullable || !field.is_nullable())
1276 && legacy_field_type_matches(field.data_type(), *data_type)
1277 })
1278 })
1279 })
1280 })
1281 && (order_id_fields == 0
1282 || ["bid", "ask"].iter().all(|side| {
1283 (0..DEPTH10_LEN).all(|level| {
1284 schema
1285 .field_with_name(&format!("{side}_order_id_{level}"))
1286 .is_ok_and(|field| {
1287 (all_nullable || !field.is_nullable())
1288 && legacy_field_type_matches(
1289 field.data_type(),
1290 LegacyFieldType::UInt64,
1291 )
1292 })
1293 })
1294 }))
1295 && [
1296 ("flags", LegacyFieldType::UInt8),
1297 ("sequence", LegacyFieldType::UInt64),
1298 ("ts_event", LegacyFieldType::UInt64),
1299 ("ts_init", LegacyFieldType::UInt64),
1300 ]
1301 .iter()
1302 .all(|(name, data_type)| {
1303 let Ok(field) = schema.field_with_name(name) else {
1304 return true;
1305 };
1306 (all_nullable || !field.is_nullable())
1307 && legacy_field_type_matches(field.data_type(), *data_type)
1308 })
1309}
1310
1311fn legacy_fixed_list_depth_fingerprint_matches(schema: &Schema) -> bool {
1312 let fixed_list_matches = |name: &str, expected: &DataType| {
1313 schema.field_with_name(name).is_ok_and(|field| {
1314 matches!(
1315 field.data_type(),
1316 DataType::FixedSizeList(item, length)
1317 if *length == i32::try_from(DEPTH10_LEN).expect("depth length fits i32")
1318 && item.data_type() == expected
1319 )
1320 })
1321 };
1322 let widths = ["bid_price", "ask_price", "bid_size", "ask_size"]
1323 .iter()
1324 .filter_map(|name| {
1325 let field = schema.field_with_name(name).ok()?;
1326 let DataType::FixedSizeList(item, _) = field.data_type() else {
1327 return None;
1328 };
1329 let DataType::FixedSizeBinary(width @ (8 | 16)) = item.data_type() else {
1330 return None;
1331 };
1332 Some(*width)
1333 })
1334 .collect::<Vec<_>>();
1335 let order_id_fields = ["bid_order_id", "ask_order_id"]
1336 .iter()
1337 .filter(|name| schema.field_with_name(name).is_ok())
1338 .count();
1339
1340 widths.len() == 4
1341 && widths.iter().all(|width| *width == widths[0])
1342 && ["bid_count", "ask_count"]
1343 .iter()
1344 .all(|name| fixed_list_matches(name, &DataType::UInt32))
1345 && matches!(order_id_fields, 0 | 2)
1346 && (order_id_fields == 0
1347 || ["bid_order_id", "ask_order_id"]
1348 .iter()
1349 .all(|name| fixed_list_matches(name, &DataType::UInt64)))
1350}
1351
1352#[must_use]
1354pub fn is_timestamp_field(name: &str) -> bool {
1355 name.starts_with("ts_")
1356 || matches!(
1357 name,
1358 "activation_ns"
1359 | "expiration_ns"
1360 | "next_funding_ns"
1361 | "expire_time"
1362 | "event_open_date"
1363 | "market_start_time"
1364 )
1365}
1366
1367#[must_use]
1369pub fn is_legacy_enum_field(name: &str) -> bool {
1370 matches!(name, "action" | "side" | "aggressor_side" | "close_type")
1371}
1372
1373fn legacy_enum_name(name: &str, value: u8) -> Result<String, ArrowError> {
1374 let name_value = match name {
1375 "action" => BookAction::from_u8(value).map(|value| value.to_string()),
1376 "side" => match value {
1377 0 => Some("NO_ORDER_SIDE".to_string()),
1378 1 => Some(OrderSide::Buy.to_string()),
1379 2 => Some(OrderSide::Sell.to_string()),
1380 _ => None,
1381 },
1382 "aggressor_side" => AggressorSide::from_u8(value).map(|value| value.to_string()),
1383 "close_type" => InstrumentCloseType::from_u8(value).map(|value| value.to_string()),
1384 _ => None,
1385 };
1386 name_value.ok_or_else(|| {
1387 ArrowError::CastError(format!(
1388 "Invalid legacy enum value {value} for column '{name}'"
1389 ))
1390 })
1391}
1392
1393const LEGACY_QUANTITY_FIELDS: &[&str] = &[
1395 "size",
1396 "quantity",
1397 "qty",
1398 "volume",
1399 "bid_size",
1400 "ask_size",
1401 "bid_size_0",
1402 "bid_size_1",
1403 "bid_size_2",
1404 "bid_size_3",
1405 "bid_size_4",
1406 "bid_size_5",
1407 "bid_size_6",
1408 "bid_size_7",
1409 "bid_size_8",
1410 "bid_size_9",
1411 "ask_size_0",
1412 "ask_size_1",
1413 "ask_size_2",
1414 "ask_size_3",
1415 "ask_size_4",
1416 "ask_size_5",
1417 "ask_size_6",
1418 "ask_size_7",
1419 "ask_size_8",
1420 "ask_size_9",
1421];
1422
1423fn is_legacy_quantity_field(name: &str) -> bool {
1424 LEGACY_QUANTITY_FIELDS.contains(&name)
1425}
1426
1427#[cfg(test)]
1428mod tests {
1429 use std::collections::HashMap;
1430
1431 use arrow::{
1432 array::{BinaryArray, BooleanArray},
1433 datatypes::Schema,
1434 };
1435 use nautilus_model::{
1436 data::{FundingRateUpdate, InstrumentClose},
1437 enums::{InstrumentCloseType, MarketStatusAction},
1438 types::fixed::{FIXED_PRECISION, FIXED_PRECISION_STANDARD},
1439 };
1440 use rstest::rstest;
1441
1442 use super::*;
1443 use crate::arrow::{
1444 DecodeFromRecordBatch, DecodeTypedFromRecordBatch, EncodeToRecordBatch, StringColumnRef,
1445 decode_decimal_price, decode_decimal_quantity,
1446 };
1447
1448 #[rstest]
1449 #[case("price", false)]
1450 #[case("size", true)]
1451 fn legacy_fixed_conversion_preserves_spurious_digits_for_checked_rejection(
1452 #[case] field: &'static str,
1453 #[case] quantity: bool,
1454 #[values(8, 16)] width: i32,
1455 ) {
1456 let bytes = if width == 8 {
1457 1_i64.to_le_bytes().to_vec()
1458 } else {
1459 STANDARD_TO_DECIMAL_SCALE.to_le_bytes().to_vec()
1460 };
1461
1462 let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1463 [Some(bytes.as_slice())].into_iter(),
1464 width,
1465 )
1466 .unwrap();
1467 let normalized = normalize_legacy_fixed_array(field, &array, width).unwrap();
1468 let error = if quantity {
1469 decode_decimal_quantity(&normalized, 0, field, 0).unwrap_err()
1470 } else {
1471 decode_decimal_price(&normalized, 0, field, 0).unwrap_err()
1472 };
1473
1474 let raw = if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
1475 1
1476 } else {
1477 STANDARD_TO_DECIMAL_SCALE
1478 };
1479
1480 let scale = 10_i128.pow(u32::from(FIXED_PRECISION));
1481
1482 assert_eq!(normalized.value(0), STANDARD_TO_DECIMAL_SCALE);
1483 assert_eq!(
1484 error.to_string(),
1485 format!(
1486 "Error parsing `{field}`: row 0: Invalid fixed-point raw value {raw} for precision 0: \
1487 remainder {raw} when divided by scale {scale}. \
1488 Raw value should be a multiple of {scale}. \
1489 This indicates data corruption or incorrect precision/scaling upstream"
1490 )
1491 );
1492 }
1493
1494 #[rstest]
1495 #[case("size", true)]
1496 #[case("volume", true)]
1497 #[case("bid_size_0", true)]
1498 #[case("ask_size_9", true)]
1499 #[case("price", false)]
1500 #[case("quantity_hint", false)]
1501 #[case("bid_size_10", false)]
1502 fn legacy_quantity_fields_use_exact_schema_names(#[case] name: &str, #[case] expected: bool) {
1503 assert_eq!(is_legacy_quantity_field(name), expected);
1504 }
1505
1506 #[rstest]
1507 fn legacy_market_columns_normalize_enums_without_changing_flags() {
1508 let price = 1_i64.to_le_bytes();
1509 let size = 2_u64.to_le_bytes();
1510 let batch = RecordBatch::try_new(
1511 Arc::new(Schema::new(vec![
1512 Field::new("action", DataType::UInt8, false),
1513 Field::new("side", DataType::UInt8, false),
1514 Field::new("price", DataType::FixedSizeBinary(8), false),
1515 Field::new("size", DataType::FixedSizeBinary(8), false),
1516 Field::new("order_id", DataType::UInt64, false),
1517 Field::new("flags", DataType::UInt8, false),
1518 Field::new("sequence", DataType::UInt64, false),
1519 Field::new("ts_event", DataType::UInt64, false),
1520 Field::new("ts_init", DataType::UInt64, false),
1521 ])),
1522 vec![
1523 Arc::new(UInt8Array::from(vec![BookAction::Add as u8])),
1524 Arc::new(UInt8Array::from(vec![OrderSide::Buy as u8])),
1525 Arc::new(
1526 FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1527 [Some(price.as_slice())].into_iter(),
1528 8,
1529 )
1530 .unwrap(),
1531 ),
1532 Arc::new(
1533 FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1534 [Some(size.as_slice())].into_iter(),
1535 8,
1536 )
1537 .unwrap(),
1538 ),
1539 Arc::new(UInt64Array::from(vec![11])),
1540 Arc::new(UInt8Array::from(vec![7])),
1541 Arc::new(UInt64Array::from(vec![12])),
1542 Arc::new(UInt64Array::from(vec![13])),
1543 Arc::new(UInt64Array::from(vec![1])),
1544 ],
1545 )
1546 .unwrap();
1547
1548 let normalized = normalize_legacy_fixed_columns(&batch).unwrap();
1549 let action =
1550 StringColumnRef::try_from_array(normalized.column_by_name("action").unwrap().as_ref())
1551 .unwrap();
1552 let side =
1553 StringColumnRef::try_from_array(normalized.column_by_name("side").unwrap().as_ref())
1554 .unwrap();
1555
1556 assert_eq!(action.value(0), "ADD");
1557 assert_eq!(side.value(0), "BUY");
1558 assert_eq!(
1559 normalized.column_by_name("price").unwrap().data_type(),
1560 &fixed_decimal_data_type(),
1561 );
1562 assert_eq!(
1563 normalized.column_by_name("size").unwrap().data_type(),
1564 &fixed_decimal_data_type(),
1565 );
1566 assert_eq!(
1567 normalized.column_by_name("flags").unwrap().data_type(),
1568 &DataType::UInt8,
1569 );
1570 assert_eq!(
1571 normalized.column_by_name("ts_init").unwrap().data_type(),
1572 ×tamp_data_type(),
1573 );
1574 }
1575
1576 #[rstest]
1577 fn legacy_enum_normalization_rejects_null_with_field_and_row() {
1578 let field = Field::new("aggressor_side", DataType::UInt8, true);
1579 let values = UInt8Array::from(vec![Some(AggressorSide::Buy as u8), None]);
1580
1581 let error = legacy_enum_dictionary_column(&field, &values).unwrap_err();
1582
1583 assert_eq!(
1584 error.to_string(),
1585 "Cast error: Legacy enum column 'aggressor_side' is null at row 1"
1586 );
1587 }
1588
1589 #[rstest]
1590 fn unrelated_named_columns_are_not_retyped() {
1591 let batch = RecordBatch::try_new(
1592 Arc::new(Schema::new(vec![
1593 Field::new("side", DataType::UInt8, false),
1594 Field::new("ts_recv", DataType::UInt64, false),
1595 ])),
1596 vec![
1597 Arc::new(UInt8Array::from(vec![127])),
1598 Arc::new(UInt64Array::from(vec![u64::MAX])),
1599 ],
1600 )
1601 .unwrap();
1602
1603 let normalized = normalize_legacy_fixed_columns(&batch).unwrap();
1604
1605 assert_eq!(normalized, batch);
1606 }
1607
1608 #[rstest]
1609 fn unrelated_fixed_binary_column_is_not_retyped() {
1610 let value = i64::MIN.to_le_bytes();
1611 let batch = RecordBatch::try_new(
1612 Arc::new(Schema::new(vec![Field::new(
1613 "price",
1614 DataType::FixedSizeBinary(8),
1615 false,
1616 )])),
1617 vec![Arc::new(
1618 FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1619 [Some(value.as_slice())].into_iter(),
1620 8,
1621 )
1622 .unwrap(),
1623 )],
1624 )
1625 .unwrap();
1626
1627 let normalized = normalize_legacy_fixed_columns(&batch).unwrap();
1628
1629 assert_eq!(normalized, batch);
1630 }
1631
1632 #[rstest]
1633 fn legacy_family_fingerprints_cover_unpinned_schemas() {
1634 let fixed = DataType::FixedSizeBinary(8);
1635 let schemas = [
1636 Schema::new(vec![
1637 Field::new("close_price", fixed.clone(), false),
1638 Field::new("close_type", DataType::UInt8, false),
1639 Field::new("ts_event", DataType::UInt64, false),
1640 Field::new("ts_init", DataType::UInt64, false),
1641 ]),
1642 Schema::new(vec![
1643 Field::new("instrument_id", DataType::Utf8, true),
1644 Field::new("close_type", DataType::Utf8, true),
1645 Field::new("close_price", DataType::Utf8, true),
1646 Field::new("ts_event", DataType::UInt64, true),
1647 Field::new("ts_init", DataType::UInt64, true),
1648 ]),
1649 Schema::new(vec![
1650 Field::new("value", fixed, false),
1651 Field::new("ts_event", DataType::UInt64, false),
1652 Field::new("ts_init", DataType::UInt64, false),
1653 ]),
1654 Schema::new(vec![
1655 Field::new("rate", DataType::Binary, false),
1656 Field::new("interval", DataType::UInt16, true),
1657 Field::new("next_funding_ns", DataType::UInt64, true),
1658 Field::new("ts_event", DataType::UInt64, false),
1659 Field::new("ts_init", DataType::UInt64, false),
1660 ]),
1661 Schema::new(vec![
1662 Field::new("instrument_id", DataType::Utf8, true),
1663 Field::new("action", DataType::Utf8, true),
1664 Field::new("reason", DataType::Utf8, true),
1665 Field::new("trading_event", DataType::Utf8, true),
1666 Field::new("is_trading", DataType::Boolean, true),
1667 Field::new("is_quoting", DataType::Boolean, true),
1668 Field::new("is_short_sell_restricted", DataType::Boolean, true),
1669 Field::new("ts_event", DataType::UInt64, true),
1670 Field::new("ts_init", DataType::UInt64, true),
1671 ]),
1672 ];
1673
1674 assert!(schemas.iter().all(is_nautilus_legacy_schema));
1675 }
1676
1677 #[rstest]
1678 fn legacy_family_fingerprint_accepts_dictionary_string_fields() {
1679 let schema = Schema::new(vec![
1680 Field::new("price", DataType::FixedSizeBinary(8), false),
1681 Field::new("size", DataType::FixedSizeBinary(8), false),
1682 Field::new("aggressor_side", DataType::UInt8, false),
1683 Field::new(
1684 "trade_id",
1685 DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
1686 false,
1687 ),
1688 Field::new("ts_event", DataType::UInt64, false),
1689 Field::new("ts_init", DataType::UInt64, false),
1690 Field::new(
1691 KEY_IDENTIFIER,
1692 DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
1693 false,
1694 ),
1695 ]);
1696
1697 assert!(is_nautilus_legacy_schema(&schema));
1698 }
1699
1700 #[rstest]
1701 fn legacy_flat_depth_fingerprint_accepts_all_nullable_fields() {
1702 let fixed = DataType::FixedSizeBinary(8);
1703 let mut fields = Vec::new();
1704
1705 for side in ["bid", "ask"] {
1706 for level in 0..DEPTH10_LEN {
1707 fields.extend([
1708 Field::new(format!("{side}_price_{level}"), fixed.clone(), true),
1709 Field::new(format!("{side}_size_{level}"), fixed.clone(), true),
1710 Field::new(format!("{side}_count_{level}"), DataType::UInt32, true),
1711 ]);
1712 }
1713 }
1714 fields.extend([
1715 Field::new("flags", DataType::UInt8, true),
1716 Field::new("sequence", DataType::UInt64, true),
1717 Field::new("ts_event", DataType::UInt64, true),
1718 Field::new("ts_init", DataType::UInt64, true),
1719 ]);
1720 let schema = Schema::new(fields);
1721
1722 assert!(super::legacy_flat_depth_fingerprint_matches(&schema));
1723 }
1724
1725 #[rstest]
1726 fn legacy_fixed_list_depth_fingerprint_accepts_missing_order_ids() {
1727 let fixed = Arc::new(Field::new("item", DataType::FixedSizeBinary(8), false));
1728 let count = Arc::new(Field::new("item", DataType::UInt32, false));
1729 let schema = Schema::new(vec![
1730 Field::new(
1731 "bid_price",
1732 DataType::FixedSizeList(Arc::clone(&fixed), 10),
1733 false,
1734 ),
1735 Field::new(
1736 "ask_price",
1737 DataType::FixedSizeList(Arc::clone(&fixed), 10),
1738 false,
1739 ),
1740 Field::new(
1741 "bid_size",
1742 DataType::FixedSizeList(Arc::clone(&fixed), 10),
1743 false,
1744 ),
1745 Field::new("ask_size", DataType::FixedSizeList(fixed, 10), false),
1746 Field::new(
1747 "bid_count",
1748 DataType::FixedSizeList(Arc::clone(&count), 10),
1749 false,
1750 ),
1751 Field::new("ask_count", DataType::FixedSizeList(count, 10), false),
1752 ]);
1753
1754 assert!(is_nautilus_legacy_schema(&schema));
1755 }
1756
1757 #[rstest]
1758 fn metadata_only_schema_is_not_a_legacy_family() {
1759 let schema = Schema::new_with_metadata(
1760 vec![
1761 Field::new("side", DataType::UInt8, false),
1762 Field::new("ts_recv", DataType::UInt64, false),
1763 ],
1764 [("type_name".to_string(), "CustomData".to_string())].into(),
1765 );
1766
1767 assert!(!is_nautilus_legacy_schema(&schema));
1768 }
1769
1770 #[rstest]
1771 fn known_metadata_families_require_and_normalize_their_legacy_shape() {
1772 let schemas = [
1773 Schema::new_with_metadata(
1774 vec![
1775 Field::new("rate", DataType::Binary, false),
1776 Field::new("interval", DataType::UInt16, true),
1777 Field::new("next_funding_ns", DataType::UInt64, true),
1778 Field::new("ts_event", DataType::UInt64, false),
1779 Field::new("ts_init", DataType::UInt64, false),
1780 ],
1781 [("type".to_string(), "FundingRateUpdate".to_string())].into(),
1782 ),
1783 Schema::new_with_metadata(
1784 vec![
1785 Field::new("instrument_id", DataType::Utf8, true),
1786 Field::new("action", DataType::Utf8, true),
1787 Field::new("reason", DataType::Utf8, true),
1788 Field::new("trading_event", DataType::Utf8, true),
1789 Field::new("is_trading", DataType::Boolean, true),
1790 Field::new("is_quoting", DataType::Boolean, true),
1791 Field::new("is_short_sell_restricted", DataType::Boolean, true),
1792 Field::new("ts_event", DataType::UInt64, true),
1793 Field::new("ts_init", DataType::UInt64, true),
1794 ],
1795 [("type".to_string(), "InstrumentStatus".to_string())].into(),
1796 ),
1797 Schema::new_with_metadata(
1798 vec![
1799 Field::new("instrument_id", DataType::Utf8, false),
1800 Field::new("delta", DataType::Float64, false),
1801 Field::new("gamma", DataType::Float64, false),
1802 Field::new("vega", DataType::Float64, false),
1803 Field::new("theta", DataType::Float64, false),
1804 Field::new("rho", DataType::Float64, false),
1805 Field::new("mark_iv", DataType::Float64, true),
1806 Field::new("bid_iv", DataType::Float64, true),
1807 Field::new("ask_iv", DataType::Float64, true),
1808 Field::new("underlying_price", DataType::Float64, true),
1809 Field::new("open_interest", DataType::Float64, true),
1810 Field::new("ts_event", DataType::UInt64, false),
1811 Field::new("ts_init", DataType::UInt64, false),
1812 Field::new("convention", DataType::Utf8, false),
1813 ],
1814 [("type".to_string(), "OptionGreeks".to_string())].into(),
1815 ),
1816 ];
1817
1818 for schema in schemas {
1819 assert!(is_nautilus_legacy_schema(&schema));
1820 let normalized =
1821 normalize_legacy_fixed_columns(&RecordBatch::new_empty(Arc::new(schema))).unwrap();
1822
1823 assert_eq!(
1824 normalized
1825 .schema()
1826 .field_with_name("ts_init")
1827 .unwrap()
1828 .data_type(),
1829 ×tamp_data_type(),
1830 );
1831 }
1832 }
1833
1834 #[rstest]
1835 fn near_match_with_foreign_field_type_is_not_retyped() {
1836 let schema = Arc::new(Schema::new(vec![
1837 Field::new("action", DataType::UInt8, false),
1838 Field::new("side", DataType::UInt8, false),
1839 Field::new("price", DataType::FixedSizeBinary(8), false),
1840 Field::new("size", DataType::FixedSizeBinary(8), false),
1841 Field::new("order_id", DataType::UInt64, false),
1842 Field::new("flags", DataType::UInt8, false),
1843 Field::new("sequence", DataType::Int64, false),
1844 Field::new("ts_event", DataType::UInt64, false),
1845 Field::new("ts_init", DataType::UInt64, false),
1846 ]));
1847 let batch = RecordBatch::new_empty(schema);
1848
1849 let normalized = normalize_legacy_fixed_columns(&batch).unwrap();
1850
1851 assert_eq!(normalized, batch);
1852 }
1853
1854 #[rstest]
1855 fn legacy_price_error_fails_with_field_and_row() {
1856 let fixed = |raw: [u8; 8]| -> ArrayRef {
1857 Arc::new(
1858 FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1859 [Some(raw.as_slice())].into_iter(),
1860 8,
1861 )
1862 .unwrap(),
1863 )
1864 };
1865 let batch = RecordBatch::try_new(
1866 Arc::new(Schema::new_with_metadata(
1867 vec![
1868 Field::new("bid_price", DataType::FixedSizeBinary(8), false),
1869 Field::new("ask_price", DataType::FixedSizeBinary(8), false),
1870 Field::new("bid_size", DataType::FixedSizeBinary(8), false),
1871 Field::new("ask_size", DataType::FixedSizeBinary(8), false),
1872 Field::new("ts_event", DataType::UInt64, false),
1873 Field::new("ts_init", DataType::UInt64, false),
1874 ],
1875 [("type".to_string(), "QuoteTick".to_string())].into(),
1876 )),
1877 vec![
1878 fixed(i64::MIN.to_le_bytes()),
1879 fixed(2_i64.to_le_bytes()),
1880 fixed(3_u64.to_le_bytes()),
1881 fixed(4_u64.to_le_bytes()),
1882 Arc::new(UInt64Array::from(vec![5])),
1883 Arc::new(UInt64Array::from(vec![6])),
1884 ],
1885 )
1886 .unwrap();
1887
1888 let error = normalize_legacy_fixed_columns(&batch).unwrap_err();
1889
1890 assert_eq!(
1891 error.to_string(),
1892 format!(
1893 "Cast error: Legacy price column 'bid_price' contains PRICE_ERROR raw value {} at row 0",
1894 i64::MIN,
1895 ),
1896 );
1897 }
1898
1899 #[rstest]
1900 fn metadata_family_near_match_passes_through_unchanged() {
1901 let raw = i64::MIN.to_le_bytes();
1902 let batch = RecordBatch::try_new(
1903 Arc::new(Schema::new_with_metadata(
1904 vec![Field::new("bid_price", DataType::FixedSizeBinary(8), false)],
1905 [("type".to_string(), "QuoteTick".to_string())].into(),
1906 )),
1907 vec![Arc::new(
1908 FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1909 [Some(raw.as_slice())].into_iter(),
1910 8,
1911 )
1912 .unwrap(),
1913 )],
1914 )
1915 .unwrap();
1916
1917 let normalized = normalize_legacy_fixed_columns(&batch).unwrap();
1918
1919 assert_eq!(normalized, batch);
1920 }
1921
1922 #[rstest]
1923 fn schema_fingerprint_ignores_metadata_and_includes_nullability() {
1924 let fields = vec![Field::new("value", DataType::Utf8, false)];
1925 let schema1 = Schema::new_with_metadata(
1926 fields.clone(),
1927 HashMap::from([("a".to_string(), "1".to_string())]),
1928 );
1929 let schema2 =
1930 Schema::new_with_metadata(fields, HashMap::from([("b".to_string(), "2".to_string())]));
1931 let nullable = Schema::new(vec![Field::new("value", DataType::Utf8, true)]);
1932
1933 assert_eq!(schema_fingerprint(&schema1), schema_fingerprint(&schema2));
1934 assert_ne!(schema_fingerprint(&schema1), schema_fingerprint(&nullable));
1935 }
1936
1937 #[rstest]
1938 fn current_schema_passes_through() {
1939 let batch = RecordBatch::new_empty(Arc::new(QuoteTick::get_schema(None)));
1940 let result = transcode_legacy_record_batch("quotes", "quotes.parquet", batch).unwrap();
1941
1942 assert_eq!(result.kind, LegacyTranscodeKind::PassThrough);
1943 assert_eq!(result.batches.len(), 1);
1944 }
1945
1946 #[rstest]
1947 fn parquet_schema_with_plain_dictionary_strings_passes_through() {
1948 let fields = TradeTick::get_schema(None)
1949 .fields()
1950 .iter()
1951 .filter(|field| field.name() != KEY_IDENTIFIER)
1952 .map(|field| match field.data_type() {
1953 DataType::Dictionary(..) => field.as_ref().clone().with_data_type(DataType::Utf8),
1954 _ => field.as_ref().clone(),
1955 })
1956 .collect::<Vec<_>>();
1957
1958 let batch = RecordBatch::new_empty(Arc::new(Schema::new(fields)));
1959 let result = transcode_legacy_record_batch("trades", "trades.parquet", batch).unwrap();
1960
1961 assert_eq!(result.kind, LegacyTranscodeKind::PassThrough);
1962 assert_eq!(result.batches.len(), 1);
1963 }
1964
1965 #[rstest]
1966 fn reordered_nullable_schema_with_identity_column_passes_through() {
1967 let mut fields = QuoteTick::get_schema(None)
1968 .fields()
1969 .iter()
1970 .rev()
1971 .map(|field| field.as_ref().clone().with_nullable(true))
1972 .collect::<Vec<_>>();
1973 fields.push(Field::new(KEY_INSTRUMENT_ID, DataType::Utf8, false));
1974 let batch = RecordBatch::new_empty(Arc::new(Schema::new(fields)));
1975 let result = transcode_legacy_record_batch("quotes", "quotes.parquet", batch).unwrap();
1976
1977 assert_eq!(result.kind, LegacyTranscodeKind::PassThrough);
1978 assert_eq!(result.batches.len(), 1);
1979 }
1980
1981 #[rstest]
1982 #[case::extra_column(Some(Field::new("venue", DataType::Utf8, true)), None)]
1983 #[case::missing_column(None, Some("ask_size"))]
1984 fn schema_with_unexpected_columns_is_rejected(
1985 #[case] extra: Option<Field>,
1986 #[case] missing: Option<&str>,
1987 ) {
1988 let mut fields = QuoteTick::get_schema(None)
1989 .fields()
1990 .iter()
1991 .filter(|field| Some(field.name().as_str()) != missing)
1992 .map(|field| field.as_ref().clone())
1993 .collect::<Vec<_>>();
1994 fields.extend(extra);
1995 let batch = RecordBatch::new_empty(Arc::new(Schema::new(fields)));
1996 let error = transcode_legacy_record_batch("quotes", "quotes.parquet", batch).unwrap_err();
1997
1998 assert!(matches!(error, LegacyArrowError::UnknownSchema { .. }));
1999 }
2000
2001 #[rstest]
2002 fn unknown_registered_schema_is_rejected() {
2003 let batch = RecordBatch::new_empty(Arc::new(Schema::new(vec![Field::new(
2004 "unexpected",
2005 DataType::UInt64,
2006 false,
2007 )])));
2008 let error = transcode_legacy_record_batch("quotes", "quotes.parquet", batch).unwrap_err();
2009
2010 assert!(matches!(error, LegacyArrowError::UnknownSchema { .. }));
2011 assert!(error.to_string().contains("quotes.parquet"));
2012 }
2013
2014 #[rstest]
2015 #[case(false)]
2016 #[case(true)]
2017 fn instrument_status_fields_are_reordered(#[case] normalized: bool) {
2018 let schema = Arc::new(legacy_instrument_status_schema());
2019 let batch = RecordBatch::try_new(
2020 schema,
2021 vec![
2022 Arc::new(StringArray::from(vec!["AAPL.XNAS"])),
2023 Arc::new(StringArray::from(vec!["TRADING"])),
2024 Arc::new(StringArray::from(vec![Some("Normal")])),
2025 Arc::new(StringArray::from(vec![Some("OPEN")])),
2026 Arc::new(BooleanArray::from(vec![Some(true)])),
2027 Arc::new(BooleanArray::from(vec![Some(true)])),
2028 Arc::new(BooleanArray::from(vec![Some(false)])),
2029 Arc::new(UInt64Array::from(vec![1])),
2030 Arc::new(UInt64Array::from(vec![2])),
2031 ],
2032 )
2033 .unwrap();
2034
2035 let batch = if normalized {
2037 normalize_legacy_fixed_columns(&batch).unwrap()
2038 } else {
2039 batch
2040 };
2041
2042 let result =
2043 transcode_legacy_record_batch("instrument_status", "status.parquet", batch).unwrap();
2044 let output = &result.batches[0];
2045
2046 assert_eq!(result.kind, LegacyTranscodeKind::InstrumentStatusV1);
2047 assert_eq!(
2048 output
2049 .schema()
2050 .fields()
2051 .iter()
2052 .map(|field| field.name().as_str())
2053 .collect::<Vec<_>>(),
2054 vec![
2055 "action",
2056 "ts_event",
2057 "ts_init",
2058 "reason",
2059 "trading_event",
2060 "is_trading",
2061 "is_quoting",
2062 "is_short_sell_restricted",
2063 "identifier",
2064 ]
2065 );
2066 assert_eq!(
2067 output
2068 .schema()
2069 .field_with_name("action")
2070 .unwrap()
2071 .data_type(),
2072 &enum_dictionary_data_type()
2073 );
2074 let identifiers = output
2075 .column_by_name(KEY_IDENTIFIER)
2076 .unwrap()
2077 .as_any()
2078 .downcast_ref::<StringArray>()
2079 .unwrap();
2080 assert_eq!(identifiers.value(0), "AAPL.XNAS");
2081
2082 let mut metadata = output.schema().metadata().clone();
2084 metadata.insert(
2085 KEY_INSTRUMENT_ID.to_string(),
2086 identifiers.value(0).to_string(),
2087 );
2088 let decoded = InstrumentStatus::decode_typed_batch(&metadata, output.clone()).unwrap();
2089 assert_eq!(decoded[0].action, MarketStatusAction::Trading);
2090 assert_eq!(decoded[0].instrument_id.to_string(), "AAPL.XNAS");
2091 assert_eq!(decoded[0].ts_event.as_u64(), 1);
2092 assert_eq!(decoded[0].ts_init.as_u64(), 2);
2093 }
2094
2095 #[rstest]
2096 fn funding_rate_is_decoded_and_interval_is_widened() {
2097 let schema = Arc::new(Schema::new_with_metadata(
2098 legacy_funding_rate_schema()
2099 .fields()
2100 .iter()
2101 .cloned()
2102 .collect::<Vec<_>>(),
2103 HashMap::from([(
2104 KEY_INSTRUMENT_ID.to_string(),
2105 "BTCUSDT-PERP.BINANCE".to_string(),
2106 )]),
2107 ));
2108 let batch = RecordBatch::try_new(
2109 schema,
2110 vec![
2111 Arc::new(BinaryArray::from(vec![b"\"0.0001\"".as_slice()])),
2112 Arc::new(UInt16Array::from(vec![Some(480)])),
2113 Arc::new(UInt64Array::from(vec![Some(9)])),
2114 Arc::new(UInt64Array::from(vec![1])),
2115 Arc::new(UInt64Array::from(vec![2])),
2116 ],
2117 )
2118 .unwrap();
2119
2120 let result =
2121 transcode_legacy_record_batch("funding_rates", "funding.parquet", batch).unwrap();
2122 let output = &result.batches[0];
2123 let decoded =
2124 FundingRateUpdate::decode_typed_batch(output.schema().metadata(), output.clone())
2125 .unwrap();
2126
2127 assert_eq!(result.kind, LegacyTranscodeKind::FundingRateUpdateV1);
2128 assert_eq!(decoded[0].instrument_id.to_string(), "BTCUSDT-PERP.BINANCE");
2129 assert_eq!(decoded[0].rate.to_string(), "0.0001");
2130 assert_eq!(decoded[0].interval, Some(480));
2131 assert_eq!(
2132 decoded[0].next_funding_ns.map(|value| value.as_u64()),
2133 Some(9)
2134 );
2135 }
2136
2137 #[rstest]
2138 #[case(false)]
2139 #[case(true)]
2140 fn instrument_close_is_split_and_decoded(#[case] normalized: bool) {
2141 let schema = Arc::new(legacy_instrument_close_schema());
2142 let batch = RecordBatch::try_new(
2143 schema,
2144 vec![
2145 Arc::new(StringArray::from(vec![
2146 "AUD/USD.SIM",
2147 "GBP/USD.SIM",
2148 "AUD/USD.SIM",
2149 ])),
2150 Arc::new(StringArray::from(vec![
2151 "END_OF_SESSION",
2152 "CONTRACT_EXPIRED",
2153 "END_OF_SESSION",
2154 ])),
2155 Arc::new(StringArray::from(vec!["1.0500", "2.1000", "1.0600"])),
2156 Arc::new(UInt64Array::from(vec![1, 2, 3])),
2157 Arc::new(UInt64Array::from(vec![4, 5, 6])),
2158 ],
2159 )
2160 .unwrap();
2161
2162 let batch = if normalized {
2163 normalize_legacy_fixed_columns(&batch).unwrap()
2164 } else {
2165 batch
2166 };
2167
2168 let result =
2169 transcode_legacy_record_batch("instrument_closes", "close.parquet", batch).unwrap();
2170 let decoded = result
2171 .batches
2172 .iter()
2173 .flat_map(|batch| {
2174 InstrumentClose::decode_batch(batch.schema().metadata(), batch.clone()).unwrap()
2175 })
2176 .collect::<Vec<_>>();
2177
2178 assert_eq!(
2179 decoded
2180 .iter()
2181 .map(|close| (close.ts_event.as_u64(), close.ts_init.as_u64()))
2182 .collect::<Vec<_>>(),
2183 vec![(1, 4), (3, 6), (2, 5)]
2184 );
2185 assert_eq!(result.kind, LegacyTranscodeKind::InstrumentCloseV1);
2186 assert_eq!(result.batches.len(), 2);
2187 assert_eq!(
2188 decoded
2189 .iter()
2190 .map(|close| close.instrument_id.to_string())
2191 .collect::<Vec<_>>(),
2192 vec!["AUD/USD.SIM", "AUD/USD.SIM", "GBP/USD.SIM"]
2193 );
2194 assert_eq!(
2195 decoded
2196 .iter()
2197 .map(|close| close.close_price.to_string())
2198 .collect::<Vec<_>>(),
2199 vec!["1.0500", "1.0600", "2.1000"]
2200 );
2201 assert_eq!(
2202 decoded
2203 .iter()
2204 .map(|close| close.close_type)
2205 .collect::<Vec<_>>(),
2206 vec![
2207 InstrumentCloseType::EndOfSession,
2208 InstrumentCloseType::EndOfSession,
2209 InstrumentCloseType::ContractExpired,
2210 ]
2211 );
2212
2213 for batch in &result.batches {
2215 let schema = batch.schema();
2216 let closes = InstrumentClose::decode_batch(schema.metadata(), batch.clone()).unwrap();
2217 let fresh = InstrumentClose::encode_batch(&closes[0].metadata(), &closes).unwrap();
2218 assert_eq!(fresh.schema(), schema);
2219 }
2220 }
2221
2222 #[rstest]
2223 fn instrument_close_rejects_mixed_precision() {
2224 let batch = RecordBatch::try_new(
2225 Arc::new(legacy_instrument_close_schema()),
2226 vec![
2227 Arc::new(StringArray::from(vec!["AUD/USD.SIM", "AUD/USD.SIM"])),
2228 Arc::new(StringArray::from(vec!["END_OF_SESSION", "END_OF_SESSION"])),
2229 Arc::new(StringArray::from(vec!["1.0500", "1.060"])),
2230 Arc::new(UInt64Array::from(vec![1, 2])),
2231 Arc::new(UInt64Array::from(vec![3, 4])),
2232 ],
2233 )
2234 .unwrap();
2235
2236 let error =
2237 transcode_legacy_record_batch("instrument_closes", "close.parquet", batch).unwrap_err();
2238
2239 assert!(error.to_string().contains("close.parquet"));
2240 assert!(error.to_string().contains("precision conflict at row 1"));
2241 }
2242
2243 #[rstest]
2244 fn instrument_close_rejects_unknown_close_type() {
2245 let batch = RecordBatch::try_new(
2246 Arc::new(legacy_instrument_close_schema()),
2247 vec![
2248 Arc::new(StringArray::from(vec!["AUD/USD.SIM"])),
2249 Arc::new(StringArray::from(vec!["UNKNOWN"])),
2250 Arc::new(StringArray::from(vec!["1.0500"])),
2251 Arc::new(UInt64Array::from(vec![1])),
2252 Arc::new(UInt64Array::from(vec![2])),
2253 ],
2254 )
2255 .unwrap();
2256
2257 let error =
2258 transcode_legacy_record_batch("instrument_closes", "close.parquet", batch).unwrap_err();
2259
2260 assert!(error.to_string().contains("close.parquet"));
2261 assert!(
2262 error
2263 .to_string()
2264 .contains("unknown close_type \"UNKNOWN\" at row 0")
2265 );
2266 }
2267}