1use std::{collections::HashMap, str::FromStr, sync::Arc};
23
24use arrow::{
25 array::{
26 Array, ArrayRef, BooleanArray, Decimal128Array, Float64Array, Float64Builder, Int8Array,
27 Int16Array, ListArray, StringArray, StringBuilder, StructArray, TimestampNanosecondArray,
28 TimestampNanosecondBuilder, UInt8Array, UInt8Builder, UInt32Array, UInt64Array,
29 UInt64Builder,
30 },
31 compute::cast,
32 datatypes::{DataType, Field, Schema},
33 record_batch::RecordBatch,
34};
35use nautilus_model::{
36 data::{
37 Bar, BarType, FundingRateUpdate, IndexPriceUpdate, InstrumentClose, InstrumentStatus,
38 MarkPriceUpdate, NautilusDataType, OptionGreeks, OrderBookDelta, OrderBookDepth, QuoteTick,
39 TradeTick, get_arrow_schema,
40 },
41 enums::{AggressorSide, BookAction, InstrumentCloseType, OrderSide},
42 instruments::InstrumentAny,
43 types::{Price, Quantity},
44};
45use rust_decimal::{Decimal, prelude::ToPrimitive};
46
47use super::{
48 ArrowSchemaProvider, EncodingError, KEY_BAR_TYPE, KEY_IDENTIFIER, KEY_INSTRUMENT_ID,
49 KEY_PRICE_PRECISION, KEY_SIZE_PRECISION, U64ColumnRef,
50 custom::CustomDataDecoder,
51 decode_decimal_price, decode_decimal_quantity,
52 depth_display::{DepthSideBuilder, schema as depth_schema},
53 display_conversion::{
54 float64_field, price_to_f64, quantity_to_f64, timestamp_field, utf8_field,
55 },
56 extract_column, extract_column_string, fixed_decimal_data_type,
57};
58use crate::arrow::timestamp_data_type;
59
60struct CatalogDisplayFns {
61 schema: fn() -> Schema,
62 convert: fn(&HashMap<String, String>, &RecordBatch) -> Result<RecordBatch, EncodingError>,
63}
64
65trait CatalogDisplay {
66 const FUNCTIONS: Option<CatalogDisplayFns>;
67}
68
69impl CatalogDisplay for InstrumentAny {
70 const FUNCTIONS: Option<CatalogDisplayFns> = None;
71}
72
73impl CatalogDisplay for QuoteTick {
74 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
75 schema: quotes_schema,
76 convert: convert_quotes,
77 });
78}
79
80impl CatalogDisplay for TradeTick {
81 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
82 schema: trades_schema,
83 convert: convert_trades,
84 });
85}
86
87impl CatalogDisplay for Bar {
88 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
89 schema: bars_schema,
90 convert: convert_bars,
91 });
92}
93
94impl CatalogDisplay for OrderBookDelta {
95 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
96 schema: deltas_schema,
97 convert: convert_deltas,
98 });
99}
100
101impl CatalogDisplay for OrderBookDepth {
102 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
103 schema: depth_schema,
104 convert: convert_depths,
105 });
106}
107
108impl CatalogDisplay for MarkPriceUpdate {
109 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
110 schema: price_update_schema,
111 convert: convert_price_updates,
112 });
113}
114
115impl CatalogDisplay for IndexPriceUpdate {
116 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
117 schema: price_update_schema,
118 convert: convert_price_updates,
119 });
120}
121
122impl CatalogDisplay for FundingRateUpdate {
123 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
124 schema: funding_rate_schema,
125 convert: convert_funding_rates,
126 });
127}
128
129impl CatalogDisplay for InstrumentStatus {
130 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
131 schema: instrument_status_schema,
132 convert: convert_instrument_status,
133 });
134}
135
136impl CatalogDisplay for OptionGreeks {
137 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
138 schema: option_greeks_schema,
139 convert: convert_option_greeks,
140 });
141}
142
143impl CatalogDisplay for InstrumentClose {
144 const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
145 schema: instrument_closes_schema,
146 convert: convert_instrument_closes,
147 });
148}
149
150macro_rules! define_catalog_display_lookup {
151 ($(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?) => {
152 fn builtin_catalog_display(data_type: &NautilusDataType) -> Option<CatalogDisplayFns> {
153 match data_type {
154 $(
155 NautilusDataType::$variant => <$type as CatalogDisplay>::FUNCTIONS,
156 )+
157 _ => None,
158 }
159 }
160 };
161}
162
163nautilus_model::for_each_data_type!(define_catalog_display_lookup);
164
165macro_rules! define_catalog_raw_schema_lookup {
166 (
167 ($_instrument_variant:ident, $_instrument_type:ident, $_instrument_data:ident, $_instrument_batch:ident, $_instrument_prefix:literal),
168 $(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?
169 ) => {
170 fn builtin_catalog_raw_schema(data_type: &NautilusDataType) -> Option<Schema> {
171 match data_type {
172 $(
173 NautilusDataType::$variant => {
174 Some(<$type as ArrowSchemaProvider>::get_schema(None))
175 }
176 )+
177 _ => None,
178 }
179 }
180 };
181}
182
183nautilus_model::for_each_data_type!(define_catalog_raw_schema_lookup);
184
185pub fn catalog_raw_schema(data_type: &NautilusDataType) -> Result<Schema, EncodingError> {
191 if let NautilusDataType::Custom { type_name } = data_type {
192 if get_arrow_schema(type_name).is_none() {
193 return Err(unsupported_display_type(data_type));
194 }
195 let metadata = HashMap::from([("type_name".to_string(), type_name.clone())]);
196 return Ok(super::schema_with_identifier_column(
197 &CustomDataDecoder::get_schema(Some(metadata)),
198 ));
199 }
200
201 builtin_catalog_raw_schema(data_type).ok_or_else(|| unsupported_display_type(data_type))
202}
203
204#[allow(
210 clippy::match_wildcard_for_single_variants,
211 unreachable_patterns,
212 reason = "NautilusDataType::Defi is controlled by nautilus-model features, not this crate"
213)]
214pub fn catalog_display_schema(data_type: &NautilusDataType) -> Result<Schema, EncodingError> {
215 if matches!(data_type, NautilusDataType::Custom { .. }) {
216 let schema = catalog_raw_schema(data_type)?;
217 let batch = RecordBatch::new_empty(Arc::new(schema));
218 return catalog_record_batch_to_display(data_type, batch.schema().metadata(), &batch)
219 .map(|batch| batch.schema().as_ref().clone());
220 }
221
222 builtin_catalog_display(data_type)
223 .map(|functions| (functions.schema)())
224 .ok_or_else(|| unsupported_display_type(data_type))
225}
226
227#[allow(
234 clippy::match_wildcard_for_single_variants,
235 unreachable_patterns,
236 reason = "NautilusDataType::Defi is controlled by nautilus-model features, not this crate"
237)]
238pub fn catalog_record_batch_to_display(
239 data_type: &NautilusDataType,
240 metadata: &HashMap<String, String>,
241 record_batch: &RecordBatch,
242) -> Result<RecordBatch, EncodingError> {
243 let display_batch = if matches!(data_type, NautilusDataType::Custom { .. }) {
244 convert_custom(record_batch)
245 } else if let Some(functions) = builtin_catalog_display(data_type) {
246 (functions.convert)(metadata, record_batch)
247 } else {
248 Err(unsupported_display_type(data_type))
249 }?;
250
251 append_identifier_column_if_present(display_batch, record_batch)
252}
253
254fn unsupported_display_type(data_type: &NautilusDataType) -> EncodingError {
255 EncodingError::ParseError(
256 "data_type",
257 format!("unsupported display catalog data type `{data_type}`"),
258 )
259}
260
261fn append_identifier_column_if_present(
262 display_batch: RecordBatch,
263 catalog_batch: &RecordBatch,
264) -> Result<RecordBatch, EncodingError> {
265 if display_batch.schema().index_of(KEY_IDENTIFIER).is_ok() {
266 return Ok(display_batch);
267 }
268
269 let Ok(identifier_index) = catalog_batch.schema().index_of(KEY_IDENTIFIER) else {
270 return Ok(display_batch);
271 };
272
273 let catalog_schema = catalog_batch.schema();
274 let identifier_field = catalog_schema.field(identifier_index);
275 let (identifier_field, identifier_column) =
276 if identifier_field.data_type() == &DataType::Utf8View {
277 (
278 Field::new(
279 identifier_field.name(),
280 DataType::Utf8,
281 identifier_field.is_nullable(),
282 ),
283 cast(
284 catalog_batch.column(identifier_index).as_ref(),
285 &DataType::Utf8,
286 )?,
287 )
288 } else {
289 (
290 identifier_field.clone(),
291 Arc::clone(catalog_batch.column(identifier_index)),
292 )
293 };
294
295 let mut fields = display_batch.schema().fields().to_vec();
296 fields.push(Arc::new(identifier_field));
297 let mut columns = display_batch.columns().to_vec();
298 columns.push(identifier_column);
299
300 RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).map_err(EncodingError::from)
301}
302
303fn parse_price_precision(metadata: &HashMap<String, String>) -> Result<u8, EncodingError> {
304 parse_precision(metadata, KEY_PRICE_PRECISION)
305}
306
307fn parse_size_precision(metadata: &HashMap<String, String>) -> Result<u8, EncodingError> {
308 parse_precision(metadata, KEY_SIZE_PRECISION)
309}
310
311fn parse_precision(
312 metadata: &HashMap<String, String>,
313 key: &'static str,
314) -> Result<u8, EncodingError> {
315 metadata
316 .get(key)
317 .ok_or(EncodingError::MissingMetadata(key))?
318 .parse::<u8>()
319 .map_err(|e| EncodingError::ParseError(key, e.to_string()))
320}
321
322fn instrument_id(metadata: &HashMap<String, String>) -> Result<&str, EncodingError> {
323 metadata
324 .get(KEY_INSTRUMENT_ID)
325 .map(String::as_str)
326 .ok_or_else(|| EncodingError::MissingMetadata(KEY_INSTRUMENT_ID))
327}
328
329fn bar_type(metadata: &HashMap<String, String>) -> Result<BarType, EncodingError> {
330 let value = metadata
331 .get(KEY_BAR_TYPE)
332 .ok_or_else(|| EncodingError::MissingMetadata(KEY_BAR_TYPE))?;
333 BarType::from_str(value).map_err(|e| EncodingError::ParseError(KEY_BAR_TYPE, e.to_string()))
334}
335
336fn constant_string_column(value: &str, len: usize) -> ArrayRef {
337 Arc::new(StringArray::from_iter_values(std::iter::repeat_n(
338 value, len,
339 )))
340}
341
342struct FixedPrecisionColumn<'a>(&'a Decimal128Array);
343
344impl FixedPrecisionColumn<'_> {
345 fn len(&self) -> usize {
346 self.0.len()
347 }
348 fn is_null(&self, row: usize) -> bool {
349 self.0.is_null(row)
350 }
351 fn price(
352 &self,
353 precision: u8,
354 field: &'static str,
355 row: usize,
356 ) -> Result<Price, EncodingError> {
357 decode_decimal_price(self.0, precision, field, row)
358 }
359 fn quantity(
360 &self,
361 precision: u8,
362 field: &'static str,
363 row: usize,
364 ) -> Result<Quantity, EncodingError> {
365 decode_decimal_quantity(self.0, precision, field, row)
366 }
367}
368
369struct NanosColumn<'a>(&'a TimestampNanosecondArray);
370
371impl NanosColumn<'_> {
372 fn len(&self) -> usize {
373 self.0.len()
374 }
375 fn is_null(&self, row: usize) -> bool {
376 self.0.is_null(row)
377 }
378 fn value(&self, row: usize) -> i64 {
379 self.0.value(row)
380 }
381}
382
383enum U8Column<'a> {
384 UInt8(&'a UInt8Array),
385 Int8(&'a Int8Array),
386 Int16(&'a Int16Array),
387}
388
389impl U8Column<'_> {
390 fn value(&self, row: usize, field: &'static str) -> Result<u8, EncodingError> {
391 match self {
392 Self::UInt8(values) => Ok(values.value(row)),
393 Self::Int8(values) => u8::try_from(values.value(row)).map_err(|_| {
394 EncodingError::ParseError(field, format!("Invalid negative value at row {row}"))
395 }),
396 Self::Int16(values) => u8::try_from(values.value(row)).map_err(|_| {
397 EncodingError::ParseError(field, format!("Value out of u8 range at row {row}"))
398 }),
399 }
400 }
401}
402
403fn fixed_col<'a>(
404 batch: &'a RecordBatch,
405 name: &'static str,
406) -> Result<FixedPrecisionColumn<'a>, EncodingError> {
407 Ok(FixedPrecisionColumn(super::extract_decimal_column(
408 batch, name,
409 )?))
410}
411
412fn u8_col<'a>(batch: &'a RecordBatch, name: &'static str) -> Result<U8Column<'a>, EncodingError> {
413 let index = batch.schema().index_of(name)?;
414 let column = batch.column(index);
415 match column.data_type() {
416 DataType::UInt8 => Ok(U8Column::UInt8(extract_column::<UInt8Array>(
417 batch.columns(),
418 name,
419 index,
420 DataType::UInt8,
421 )?)),
422 DataType::Int8 => Ok(U8Column::Int8(extract_column::<Int8Array>(
423 batch.columns(),
424 name,
425 index,
426 DataType::Int8,
427 )?)),
428 DataType::Int16 => Ok(U8Column::Int16(extract_column::<Int16Array>(
429 batch.columns(),
430 name,
431 index,
432 DataType::Int16,
433 )?)),
434 data_type => Err(EncodingError::InvalidColumnType(
435 name,
436 index,
437 DataType::UInt8,
438 data_type.clone(),
439 )),
440 }
441}
442
443fn u64_value(
444 values: &U64ColumnRef<'_>,
445 row: usize,
446 field: &'static str,
447) -> Result<u64, EncodingError> {
448 values.value(row).ok_or_else(|| {
449 EncodingError::ParseError(field, format!("Invalid negative value at row {row}"))
450 })
451}
452
453fn u64_col<'a>(
454 batch: &'a RecordBatch,
455 name: &'static str,
456) -> Result<U64ColumnRef<'a>, EncodingError> {
457 let index = batch.schema().index_of(name)?;
458 let column = batch.column(index);
459 U64ColumnRef::try_from_array(column.as_ref()).ok_or_else(|| {
460 EncodingError::InvalidColumnType(name, index, DataType::UInt64, column.data_type().clone())
461 })
462}
463
464fn nanos_col<'a>(
465 batch: &'a RecordBatch,
466 name: &'static str,
467) -> Result<NanosColumn<'a>, EncodingError> {
468 let index = batch.schema().index_of(name)?;
469 let column = batch.column(index);
470 let expected = timestamp_data_type();
471 if column.data_type() != &expected {
472 return Err(EncodingError::InvalidColumnType(
473 name,
474 index,
475 expected,
476 column.data_type().clone(),
477 ));
478 }
479 Ok(NanosColumn(extract_column::<TimestampNanosecondArray>(
480 batch.columns(),
481 name,
482 index,
483 expected,
484 )?))
485}
486
487fn f64_col<'a>(
488 batch: &'a RecordBatch,
489 name: &'static str,
490) -> Result<&'a Float64Array, EncodingError> {
491 let index = batch.schema().index_of(name)?;
492 extract_column::<Float64Array>(batch.columns(), name, index, DataType::Float64)
493}
494
495fn bool_col<'a>(
496 batch: &'a RecordBatch,
497 name: &'static str,
498) -> Result<&'a BooleanArray, EncodingError> {
499 let index = batch.schema().index_of(name)?;
500 extract_column::<BooleanArray>(batch.columns(), name, index, DataType::Boolean)
501}
502
503fn timestamp_array_from_nanos(values: &NanosColumn<'_>) -> ArrayRef {
504 let mut builder = TimestampNanosecondBuilder::with_capacity(values.len())
505 .with_data_type(timestamp_data_type());
506
507 for row in 0..values.len() {
508 append_timestamp(&mut builder, values, row);
509 }
510 Arc::new(builder.finish())
511}
512
513fn append_timestamp(
514 builder: &mut TimestampNanosecondBuilder,
515 values: &NanosColumn<'_>,
516 row: usize,
517) {
518 if values.is_null(row) {
519 builder.append_null();
520 } else {
521 builder.append_value(values.value(row));
522 }
523}
524
525fn timestamp_field_from(field: &Field) -> Field {
526 Field::new(field.name(), timestamp_data_type(), field.is_nullable())
527}
528
529fn fixed_precision_field_to_f64(
530 values: &FixedPrecisionColumn<'_>,
531 precision: u8,
532 is_price: bool,
533) -> Result<ArrayRef, EncodingError> {
534 let mut builder = Float64Builder::with_capacity(values.len());
535 for row in 0..values.len() {
536 if values.is_null(row) {
537 builder.append_null();
538 } else if is_price {
539 append_price(&mut builder, values, precision, "custom", row)?;
540 } else {
541 append_quantity(&mut builder, values, precision, "custom", row)?;
542 }
543 }
544
545 Ok(Arc::new(builder.finish()))
546}
547
548fn custom_fixed_precision(
549 field_name: &str,
550 metadata: &HashMap<String, String>,
551) -> Option<(u8, bool)> {
552 let normalized = field_name.to_ascii_lowercase();
553 let field_precision = metadata
554 .get(&format!("{field_name}_precision"))
555 .and_then(|value| value.parse::<u8>().ok());
556
557 match metadata
558 .get(&format!("{field_name}_kind"))
559 .map(String::as_str)
560 {
561 Some("price") => return field_precision.map(|precision| (precision, true)),
562 Some("quantity") => return field_precision.map(|precision| (precision, false)),
563 _ => {}
564 }
565 let is_quantity = matches!(
566 normalized.as_str(),
567 "size" | "quantity" | "qty" | "volume" | "bid_size" | "ask_size"
568 ) || normalized.ends_with("_size")
569 || normalized.ends_with("_quantity")
570 || normalized.ends_with("_qty")
571 || normalized.ends_with("_volume");
572
573 if is_quantity {
574 return field_precision
575 .or_else(|| {
576 metadata
577 .get(KEY_SIZE_PRECISION)
578 .and_then(|value| value.parse::<u8>().ok())
579 })
580 .map(|precision| (precision, false));
581 }
582
583 if let Some(precision) = field_precision {
584 return Some((precision, true));
585 }
586
587 let is_price = matches!(
588 normalized.as_str(),
589 "price"
590 | "value"
591 | "bid"
592 | "ask"
593 | "bid_price"
594 | "ask_price"
595 | "open"
596 | "high"
597 | "low"
598 | "close"
599 ) || normalized.ends_with("_price")
600 || normalized.ends_with("_value")
601 || normalized.ends_with("_bid")
602 || normalized.ends_with("_ask")
603 || normalized.ends_with("_open")
604 || normalized.ends_with("_high")
605 || normalized.ends_with("_low")
606 || normalized.ends_with("_close");
607
608 if is_price {
609 return metadata
610 .get(KEY_PRICE_PRECISION)
611 .and_then(|value| value.parse::<u8>().ok())
612 .map(|precision| (precision, true));
613 }
614
615 None
616}
617
618fn append_price(
619 builder: &mut Float64Builder,
620 values: &FixedPrecisionColumn<'_>,
621 precision: u8,
622 field: &'static str,
623 row: usize,
624) -> Result<(), EncodingError> {
625 let price = values.price(precision, field, row)?;
626 builder.append_value(price_to_f64(&price));
627 Ok(())
628}
629
630fn append_price_with_sentinel(
631 builder: &mut Float64Builder,
632 values: &FixedPrecisionColumn<'_>,
633 precision: u8,
634 field: &'static str,
635 row: usize,
636) -> Result<(), EncodingError> {
637 let price = values.price(precision, field, row)?;
638 builder.append_value(price_to_f64(&price));
639 Ok(())
640}
641
642fn append_optional_price_with_sentinel(
643 builder: &mut Float64Builder,
644 values: &FixedPrecisionColumn<'_>,
645 precision: u8,
646 field: &'static str,
647 row: usize,
648) -> Result<(), EncodingError> {
649 let price = values.price(precision, field, row)?;
650
651 if price.is_undefined() {
652 builder.append_null();
653 } else {
654 builder.append_value(price_to_f64(&price));
655 }
656 Ok(())
657}
658
659fn append_quantity(
660 builder: &mut Float64Builder,
661 values: &FixedPrecisionColumn<'_>,
662 precision: u8,
663 field: &'static str,
664 row: usize,
665) -> Result<(), EncodingError> {
666 let quantity = values.quantity(precision, field, row)?;
667 builder.append_value(quantity_to_f64(&quantity));
668 Ok(())
669}
670
671fn append_quantity_with_sentinel(
672 builder: &mut Float64Builder,
673 values: &FixedPrecisionColumn<'_>,
674 precision: u8,
675 field: &'static str,
676 row: usize,
677) -> Result<(), EncodingError> {
678 let quantity = values.quantity(precision, field, row)?;
679 builder.append_value(quantity_to_f64(&quantity));
680 Ok(())
681}
682
683fn append_optional_quantity_with_sentinel(
684 builder: &mut Float64Builder,
685 values: &FixedPrecisionColumn<'_>,
686 precision: u8,
687 field: &'static str,
688 row: usize,
689) -> Result<(), EncodingError> {
690 let quantity = values.quantity(precision, field, row)?;
691
692 if quantity.is_undefined() {
693 builder.append_null();
694 } else {
695 builder.append_value(quantity_to_f64(&quantity));
696 }
697 Ok(())
698}
699
700fn bars_schema() -> Schema {
701 Schema::new(vec![
702 utf8_field("instrument_id", false),
703 utf8_field("bar_type", false),
704 float64_field("open", false),
705 float64_field("high", false),
706 float64_field("low", false),
707 float64_field("close", false),
708 float64_field("volume", false),
709 timestamp_field("ts_event", false),
710 timestamp_field("ts_init", false),
711 ])
712}
713
714fn convert_bars(
715 metadata: &HashMap<String, String>,
716 batch: &RecordBatch,
717) -> Result<RecordBatch, EncodingError> {
718 let bar_type = bar_type(metadata)?;
719 let bar_type_string = bar_type.to_string();
720 let instrument_id_string = bar_type.instrument_id().to_string();
721 let price_precision = parse_price_precision(metadata)?;
722 let size_precision = parse_size_precision(metadata)?;
723 let open = fixed_col(batch, "open")?;
724 let high = fixed_col(batch, "high")?;
725 let low = fixed_col(batch, "low")?;
726 let close = fixed_col(batch, "close")?;
727 let volume = fixed_col(batch, "volume")?;
728 let ts_event = nanos_col(batch, "ts_event")?;
729 let ts_init = nanos_col(batch, "ts_init")?;
730
731 let len = batch.num_rows();
732 let mut open_builder = Float64Builder::with_capacity(len);
733 let mut high_builder = Float64Builder::with_capacity(len);
734 let mut low_builder = Float64Builder::with_capacity(len);
735 let mut close_builder = Float64Builder::with_capacity(len);
736 let mut volume_builder = Float64Builder::with_capacity(len);
737 let mut ts_event_builder =
738 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
739 let mut ts_init_builder =
740 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
741
742 for row in 0..len {
743 append_price(&mut open_builder, &open, price_precision, "open", row)?;
744 append_price(&mut high_builder, &high, price_precision, "high", row)?;
745 append_price(&mut low_builder, &low, price_precision, "low", row)?;
746 append_price(&mut close_builder, &close, price_precision, "close", row)?;
747 append_quantity(&mut volume_builder, &volume, size_precision, "volume", row)?;
748 append_timestamp(&mut ts_event_builder, &ts_event, row);
749 append_timestamp(&mut ts_init_builder, &ts_init, row);
750 }
751
752 RecordBatch::try_new(
753 Arc::new(bars_schema()),
754 vec![
755 constant_string_column(&instrument_id_string, len),
756 constant_string_column(&bar_type_string, len),
757 Arc::new(open_builder.finish()),
758 Arc::new(high_builder.finish()),
759 Arc::new(low_builder.finish()),
760 Arc::new(close_builder.finish()),
761 Arc::new(volume_builder.finish()),
762 Arc::new(ts_event_builder.finish()),
763 Arc::new(ts_init_builder.finish()),
764 ],
765 )
766 .map_err(EncodingError::from)
767}
768
769fn quotes_schema() -> Schema {
770 Schema::new(vec![
771 utf8_field("instrument_id", false),
772 float64_field("bid_price", false),
773 float64_field("ask_price", false),
774 float64_field("bid_size", false),
775 float64_field("ask_size", false),
776 timestamp_field("ts_event", false),
777 timestamp_field("ts_init", false),
778 ])
779}
780
781fn convert_quotes(
782 metadata: &HashMap<String, String>,
783 batch: &RecordBatch,
784) -> Result<RecordBatch, EncodingError> {
785 let instrument_id = instrument_id(metadata)?;
786 let price_precision = parse_price_precision(metadata)?;
787 let size_precision = parse_size_precision(metadata)?;
788 let bid_price = fixed_col(batch, "bid_price")?;
789 let ask_price = fixed_col(batch, "ask_price")?;
790 let bid_size = fixed_col(batch, "bid_size")?;
791 let ask_size = fixed_col(batch, "ask_size")?;
792 let ts_event = nanos_col(batch, "ts_event")?;
793 let ts_init = nanos_col(batch, "ts_init")?;
794
795 let len = batch.num_rows();
796 let mut bid_price_builder = Float64Builder::with_capacity(len);
797 let mut ask_price_builder = Float64Builder::with_capacity(len);
798 let mut bid_size_builder = Float64Builder::with_capacity(len);
799 let mut ask_size_builder = Float64Builder::with_capacity(len);
800 let mut ts_event_builder =
801 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
802 let mut ts_init_builder =
803 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
804
805 for row in 0..len {
806 append_price(
807 &mut bid_price_builder,
808 &bid_price,
809 price_precision,
810 "bid_price",
811 row,
812 )?;
813 append_price(
814 &mut ask_price_builder,
815 &ask_price,
816 price_precision,
817 "ask_price",
818 row,
819 )?;
820 append_quantity(
821 &mut bid_size_builder,
822 &bid_size,
823 size_precision,
824 "bid_size",
825 row,
826 )?;
827 append_quantity(
828 &mut ask_size_builder,
829 &ask_size,
830 size_precision,
831 "ask_size",
832 row,
833 )?;
834 append_timestamp(&mut ts_event_builder, &ts_event, row);
835 append_timestamp(&mut ts_init_builder, &ts_init, row);
836 }
837
838 RecordBatch::try_new(
839 Arc::new(quotes_schema()),
840 vec![
841 constant_string_column(instrument_id, len),
842 Arc::new(bid_price_builder.finish()),
843 Arc::new(ask_price_builder.finish()),
844 Arc::new(bid_size_builder.finish()),
845 Arc::new(ask_size_builder.finish()),
846 Arc::new(ts_event_builder.finish()),
847 Arc::new(ts_init_builder.finish()),
848 ],
849 )
850 .map_err(EncodingError::from)
851}
852
853fn trades_schema() -> Schema {
854 Schema::new(vec![
855 utf8_field("instrument_id", false),
856 float64_field("price", false),
857 float64_field("size", false),
858 utf8_field("aggressor_side", false),
859 utf8_field("trade_id", false),
860 timestamp_field("ts_event", false),
861 timestamp_field("ts_init", false),
862 ])
863}
864
865fn convert_trades(
866 metadata: &HashMap<String, String>,
867 batch: &RecordBatch,
868) -> Result<RecordBatch, EncodingError> {
869 let instrument_id = instrument_id(metadata)?;
870 let price_precision = parse_price_precision(metadata)?;
871 let size_precision = parse_size_precision(metadata)?;
872 let price = fixed_col(batch, "price")?;
873 let size = fixed_col(batch, "size")?;
874 let aggressor_side_index = batch.schema().index_of("aggressor_side")?;
875 let aggressor_side =
876 extract_column_string(batch.columns(), "aggressor_side", aggressor_side_index)?;
877 let trade_id_index = batch.schema().index_of("trade_id")?;
878 let trade_id = extract_column_string(batch.columns(), "trade_id", trade_id_index)?;
879 let ts_event = nanos_col(batch, "ts_event")?;
880 let ts_init = nanos_col(batch, "ts_init")?;
881
882 let len = batch.num_rows();
883 let mut price_builder = Float64Builder::with_capacity(len);
884 let mut size_builder = Float64Builder::with_capacity(len);
885 let mut aggressor_side_builder = StringBuilder::new();
886 let mut trade_id_builder = StringBuilder::new();
887 let mut ts_event_builder =
888 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
889 let mut ts_init_builder =
890 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
891
892 for row in 0..len {
893 let aggressor_side_value = aggressor_side.value(row);
894 let side = AggressorSide::from_str(aggressor_side_value)
895 .map_err(|e| EncodingError::ParseError(stringify!(AggressorSide), e.to_string()))?;
896 append_price(&mut price_builder, &price, price_precision, "price", row)?;
897 append_quantity(&mut size_builder, &size, size_precision, "size", row)?;
898 aggressor_side_builder.append_value(side.as_ref());
899 trade_id_builder.append_value(trade_id.value(row));
900 append_timestamp(&mut ts_event_builder, &ts_event, row);
901 append_timestamp(&mut ts_init_builder, &ts_init, row);
902 }
903
904 RecordBatch::try_new(
905 Arc::new(trades_schema()),
906 vec![
907 constant_string_column(instrument_id, len),
908 Arc::new(price_builder.finish()),
909 Arc::new(size_builder.finish()),
910 Arc::new(aggressor_side_builder.finish()),
911 Arc::new(trade_id_builder.finish()),
912 Arc::new(ts_event_builder.finish()),
913 Arc::new(ts_init_builder.finish()),
914 ],
915 )
916 .map_err(EncodingError::from)
917}
918
919fn deltas_schema() -> Schema {
920 Schema::new(vec![
921 utf8_field("instrument_id", false),
922 utf8_field("action", false),
923 utf8_field("side", false),
924 float64_field("price", false),
925 float64_field("size", false),
926 utf8_field("order_id", false),
927 Field::new("flags", DataType::UInt8, false),
928 Field::new("sequence", DataType::UInt64, false),
929 timestamp_field("ts_event", false),
930 timestamp_field("ts_init", false),
931 ])
932}
933
934fn convert_deltas(
935 metadata: &HashMap<String, String>,
936 batch: &RecordBatch,
937) -> Result<RecordBatch, EncodingError> {
938 const NO_ORDER_SIDE: &str = "NO_ORDER_SIDE";
939
940 let instrument_id = instrument_id(metadata)?;
941 let price_precision = parse_price_precision(metadata)?;
942 let size_precision = parse_size_precision(metadata)?;
943 let action_index = batch.schema().index_of("action")?;
944 let action = extract_column_string(batch.columns(), "action", action_index)?;
945 let side_index = batch.schema().index_of("side")?;
946 let side = extract_column_string(batch.columns(), "side", side_index)?;
947 let price = fixed_col(batch, "price")?;
948 let size = fixed_col(batch, "size")?;
949 let order_id = u64_col(batch, "order_id")?;
950 let flags = u8_col(batch, "flags")?;
951 let sequence = u64_col(batch, "sequence")?;
952 let ts_event = nanos_col(batch, "ts_event")?;
953 let ts_init = nanos_col(batch, "ts_init")?;
954
955 let len = batch.num_rows();
956 let mut action_builder = StringBuilder::new();
957 let mut side_builder = StringBuilder::new();
958 let mut price_builder = Float64Builder::with_capacity(len);
959 let mut size_builder = Float64Builder::with_capacity(len);
960 let mut order_id_builder = StringBuilder::new();
961 let mut flags_builder = UInt8Builder::with_capacity(len);
962 let mut sequence_builder = UInt64Builder::with_capacity(len);
963 let mut ts_event_builder =
964 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
965 let mut ts_init_builder =
966 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
967
968 for row in 0..len {
969 let action_value = action.value(row);
970 let action = BookAction::from_str(action_value)
971 .map_err(|e| EncodingError::ParseError(stringify!(BookAction), e.to_string()))?;
972 let side_value = side.value(row);
973
974 let side = if side_value.eq_ignore_ascii_case(NO_ORDER_SIDE) {
975 None
976 } else {
977 Some(
978 OrderSide::from_str(side_value)
979 .map_err(|e| EncodingError::ParseError(stringify!(OrderSide), e.to_string()))?,
980 )
981 };
982
983 action_builder.append_value(action.as_ref());
984 side_builder.append_value(side.as_ref().map_or(NO_ORDER_SIDE, AsRef::as_ref));
985
986 if action == BookAction::Clear {
987 price_builder.append_value(f64::NAN);
988 size_builder.append_value(f64::NAN);
989 } else {
990 append_price_with_sentinel(&mut price_builder, &price, price_precision, "price", row)?;
991 append_quantity_with_sentinel(&mut size_builder, &size, size_precision, "size", row)?;
992 }
993 order_id_builder.append_value(u64_value(&order_id, row, "order_id")?.to_string());
994 flags_builder.append_value(flags.value(row, "flags")?);
995 sequence_builder.append_value(u64_value(&sequence, row, "sequence")?);
996 append_timestamp(&mut ts_event_builder, &ts_event, row);
997 append_timestamp(&mut ts_init_builder, &ts_init, row);
998 }
999
1000 RecordBatch::try_new(
1001 Arc::new(deltas_schema()),
1002 vec![
1003 constant_string_column(instrument_id, len),
1004 Arc::new(action_builder.finish()),
1005 Arc::new(side_builder.finish()),
1006 Arc::new(price_builder.finish()),
1007 Arc::new(size_builder.finish()),
1008 Arc::new(order_id_builder.finish()),
1009 Arc::new(flags_builder.finish()),
1010 Arc::new(sequence_builder.finish()),
1011 Arc::new(ts_event_builder.finish()),
1012 Arc::new(ts_init_builder.finish()),
1013 ],
1014 )
1015 .map_err(EncodingError::from)
1016}
1017
1018fn convert_depths(
1019 metadata: &HashMap<String, String>,
1020 batch: &RecordBatch,
1021) -> Result<RecordBatch, EncodingError> {
1022 let instrument_id = instrument_id(metadata)?;
1023 let price_precision = parse_price_precision(metadata)?;
1024 let size_precision = parse_size_precision(metadata)?;
1025 let bids = depth_struct_values(batch, "bids")?;
1026 let asks = depth_struct_values(batch, "asks")?;
1027 let flags = u8_col(batch, "flags")?;
1028 let sequence = u64_col(batch, "sequence")?;
1029 let ts_event = nanos_col(batch, "ts_event")?;
1030 let ts_init = nanos_col(batch, "ts_init")?;
1031 let len = batch.num_rows();
1032 let mut bids_display = DepthSideBuilder::new();
1033 let mut asks_display = DepthSideBuilder::new();
1034 let mut flags_builder = UInt8Builder::with_capacity(len);
1035 let mut sequence_builder = UInt64Builder::with_capacity(len);
1036 let mut ts_event_builder =
1037 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1038 let mut ts_init_builder =
1039 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1040 let bid_prices = FixedPrecisionColumn(&bids.prices);
1041 let ask_prices = FixedPrecisionColumn(&asks.prices);
1042 let bid_sizes = FixedPrecisionColumn(&bids.sizes);
1043 let ask_sizes = FixedPrecisionColumn(&asks.sizes);
1044
1045 for row in 0..len {
1046 let (bid_start, bid_end) = depth_struct_range(&bids.list, "bids", row)?;
1047 let (ask_start, ask_end) = depth_struct_range(&asks.list, "asks", row)?;
1048
1049 for value_index in bid_start..bid_end {
1050 append_optional_price_with_sentinel(
1051 &mut bids_display.prices,
1052 &bid_prices,
1053 price_precision,
1054 "bids.price",
1055 value_index,
1056 )?;
1057 append_optional_quantity_with_sentinel(
1058 &mut bids_display.sizes,
1059 &bid_sizes,
1060 size_precision,
1061 "bids.size",
1062 value_index,
1063 )?;
1064
1065 if bids.counts.is_null(value_index) || bids.order_ids.is_null(value_index) {
1066 return Err(EncodingError::ParseError(
1067 "bids",
1068 "count and order_id must not be null".to_string(),
1069 ));
1070 }
1071 bids_display
1072 .counts
1073 .append_value(bids.counts.value(value_index));
1074 bids_display
1075 .order_ids
1076 .append_value(bids.order_ids.value(value_index));
1077 }
1078
1079 for value_index in ask_start..ask_end {
1080 append_optional_price_with_sentinel(
1081 &mut asks_display.prices,
1082 &ask_prices,
1083 price_precision,
1084 "asks.price",
1085 value_index,
1086 )?;
1087 append_optional_quantity_with_sentinel(
1088 &mut asks_display.sizes,
1089 &ask_sizes,
1090 size_precision,
1091 "asks.size",
1092 value_index,
1093 )?;
1094
1095 if asks.counts.is_null(value_index) || asks.order_ids.is_null(value_index) {
1096 return Err(EncodingError::ParseError(
1097 "asks",
1098 "count and order_id must not be null".to_string(),
1099 ));
1100 }
1101 asks_display
1102 .counts
1103 .append_value(asks.counts.value(value_index));
1104 asks_display
1105 .order_ids
1106 .append_value(asks.order_ids.value(value_index));
1107 }
1108 bids_display.finish_row()?;
1109 asks_display.finish_row()?;
1110 flags_builder.append_value(flags.value(row, "flags")?);
1111 sequence_builder.append_value(u64_value(&sequence, row, "sequence")?);
1112 append_timestamp(&mut ts_event_builder, &ts_event, row);
1113 append_timestamp(&mut ts_init_builder, &ts_init, row);
1114 }
1115
1116 finish_depth_display(
1117 instrument_id,
1118 len,
1119 bids_display,
1120 asks_display,
1121 flags_builder,
1122 sequence_builder,
1123 ts_event_builder,
1124 ts_init_builder,
1125 )
1126}
1127
1128#[expect(
1129 clippy::too_many_arguments,
1130 reason = "the two nested sides and four scalar builders form one batch"
1131)]
1132fn finish_depth_display(
1133 instrument_id: &str,
1134 len: usize,
1135 bids: DepthSideBuilder,
1136 asks: DepthSideBuilder,
1137 mut flags: UInt8Builder,
1138 mut sequence: UInt64Builder,
1139 mut ts_event: TimestampNanosecondBuilder,
1140 mut ts_init: TimestampNanosecondBuilder,
1141) -> Result<RecordBatch, EncodingError> {
1142 RecordBatch::try_new(
1143 Arc::new(depth_schema()),
1144 vec![
1145 constant_string_column(instrument_id, len),
1146 Arc::new(bids.finish()?),
1147 Arc::new(asks.finish()?),
1148 Arc::new(flags.finish()),
1149 Arc::new(sequence.finish()),
1150 Arc::new(ts_event.finish()),
1151 Arc::new(ts_init.finish()),
1152 ],
1153 )
1154 .map_err(EncodingError::from)
1155}
1156
1157struct DepthStructColumns {
1158 list: ListArray,
1159 prices: Decimal128Array,
1160 sizes: Decimal128Array,
1161 counts: UInt32Array,
1162 order_ids: UInt64Array,
1163}
1164
1165fn depth_struct_values(
1166 batch: &RecordBatch,
1167 name: &'static str,
1168) -> Result<DepthStructColumns, EncodingError> {
1169 let list = batch
1170 .column_by_name(name)
1171 .and_then(|column| column.as_any().downcast_ref::<ListArray>())
1172 .cloned()
1173 .ok_or_else(|| EncodingError::ParseError(name, "expected List<Struct>".to_string()))?;
1174 let levels = list
1175 .values()
1176 .as_any()
1177 .downcast_ref::<StructArray>()
1178 .ok_or_else(|| EncodingError::ParseError(name, "expected List<Struct>".to_string()))?;
1179 let child = |field: &'static str| {
1180 levels.column_by_name(field).cloned().ok_or_else(|| {
1181 EncodingError::ParseError(name, format!("missing struct field '{field}'"))
1182 })
1183 };
1184
1185 for field in ["price", "size"] {
1186 if child(field)?.data_type() != &fixed_decimal_data_type() {
1187 return Err(EncodingError::ParseError(
1188 name,
1189 format!("field '{field}' must use Decimal128(38, 16)"),
1190 ));
1191 }
1192 }
1193 let prices = child("price")?
1194 .as_any()
1195 .downcast_ref::<Decimal128Array>()
1196 .cloned()
1197 .ok_or_else(|| {
1198 EncodingError::ParseError(name, "field 'price' must be Decimal128".to_string())
1199 })?;
1200 let sizes = child("size")?
1201 .as_any()
1202 .downcast_ref::<Decimal128Array>()
1203 .cloned()
1204 .ok_or_else(|| {
1205 EncodingError::ParseError(name, "field 'size' must be Decimal128".to_string())
1206 })?;
1207 let counts = child("count")?
1208 .as_any()
1209 .downcast_ref::<UInt32Array>()
1210 .cloned()
1211 .ok_or_else(|| {
1212 EncodingError::ParseError(name, "field 'count' must be UInt32".to_string())
1213 })?;
1214 let order_ids = child("order_id")?
1215 .as_any()
1216 .downcast_ref::<UInt64Array>()
1217 .cloned()
1218 .ok_or_else(|| {
1219 EncodingError::ParseError(name, "field 'order_id' must be UInt64".to_string())
1220 })?;
1221 Ok(DepthStructColumns {
1222 list,
1223 prices,
1224 sizes,
1225 counts,
1226 order_ids,
1227 })
1228}
1229
1230fn depth_struct_range(
1231 list: &ListArray,
1232 name: &'static str,
1233 row: usize,
1234) -> Result<(usize, usize), EncodingError> {
1235 if list.is_null(row) {
1236 return Err(EncodingError::ParseError(
1237 name,
1238 format!("side is null at row {row}"),
1239 ));
1240 }
1241 let offsets = list.value_offsets();
1242 let start = usize::try_from(offsets[row])
1243 .map_err(|e| EncodingError::ParseError(name, e.to_string()))?;
1244 let end = usize::try_from(offsets[row + 1])
1245 .map_err(|e| EncodingError::ParseError(name, e.to_string()))?;
1246 Ok((start, end))
1247}
1248
1249fn price_update_schema() -> Schema {
1250 Schema::new(vec![
1251 utf8_field("instrument_id", false),
1252 float64_field("value", false),
1253 timestamp_field("ts_event", false),
1254 timestamp_field("ts_init", false),
1255 ])
1256}
1257
1258fn convert_price_updates(
1259 metadata: &HashMap<String, String>,
1260 batch: &RecordBatch,
1261) -> Result<RecordBatch, EncodingError> {
1262 let instrument_id = instrument_id(metadata)?;
1263 let price_precision = parse_price_precision(metadata)?;
1264 let value = fixed_col(batch, "value")?;
1265 let ts_event = nanos_col(batch, "ts_event")?;
1266 let ts_init = nanos_col(batch, "ts_init")?;
1267 let len = batch.num_rows();
1268 let mut value_builder = Float64Builder::with_capacity(len);
1269 let mut ts_event_builder =
1270 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1271 let mut ts_init_builder =
1272 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1273
1274 for row in 0..len {
1275 append_price(&mut value_builder, &value, price_precision, "value", row)?;
1276 append_timestamp(&mut ts_event_builder, &ts_event, row);
1277 append_timestamp(&mut ts_init_builder, &ts_init, row);
1278 }
1279
1280 RecordBatch::try_new(
1281 Arc::new(price_update_schema()),
1282 vec![
1283 constant_string_column(instrument_id, len),
1284 Arc::new(value_builder.finish()),
1285 Arc::new(ts_event_builder.finish()),
1286 Arc::new(ts_init_builder.finish()),
1287 ],
1288 )
1289 .map_err(EncodingError::from)
1290}
1291
1292fn funding_rate_schema() -> Schema {
1293 Schema::new(vec![
1294 utf8_field("instrument_id", false),
1295 float64_field("rate", false),
1296 Field::new("interval", DataType::UInt64, true),
1297 timestamp_field("next_funding_ns", true),
1298 timestamp_field("ts_event", false),
1299 timestamp_field("ts_init", false),
1300 ])
1301}
1302
1303fn convert_funding_rates(
1304 metadata: &HashMap<String, String>,
1305 batch: &RecordBatch,
1306) -> Result<RecordBatch, EncodingError> {
1307 let instrument_id = instrument_id(metadata)?;
1308 let rate_index = batch.schema().index_of("rate")?;
1309 let rate = extract_column_string(batch.columns(), "rate", rate_index)?;
1310 let interval = u64_col(batch, "interval")?;
1311 let next_funding_ns = nanos_col(batch, "next_funding_ns")?;
1312 let ts_event = nanos_col(batch, "ts_event")?;
1313 let ts_init = nanos_col(batch, "ts_init")?;
1314 let len = batch.num_rows();
1315
1316 let mut rate_builder = Float64Builder::with_capacity(len);
1317 let mut interval_builder = UInt64Builder::with_capacity(len);
1318 let mut next_funding_ns_builder =
1319 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1320 let mut ts_event_builder =
1321 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1322 let mut ts_init_builder =
1323 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1324
1325 for row in 0..len {
1326 let value = Decimal::from_str(rate.value(row))
1327 .map_err(|e| EncodingError::ParseError("rate", e.to_string()))?;
1328 rate_builder.append_value(value.to_f64().unwrap_or(f64::NAN));
1329 append_optional_u64(&mut interval_builder, &interval, "interval", row)?;
1330 append_timestamp(&mut next_funding_ns_builder, &next_funding_ns, row);
1331 append_timestamp(&mut ts_event_builder, &ts_event, row);
1332 append_timestamp(&mut ts_init_builder, &ts_init, row);
1333 }
1334
1335 RecordBatch::try_new(
1336 Arc::new(funding_rate_schema()),
1337 vec![
1338 constant_string_column(instrument_id, len),
1339 Arc::new(rate_builder.finish()),
1340 Arc::new(interval_builder.finish()),
1341 Arc::new(next_funding_ns_builder.finish()),
1342 Arc::new(ts_event_builder.finish()),
1343 Arc::new(ts_init_builder.finish()),
1344 ],
1345 )
1346 .map_err(EncodingError::from)
1347}
1348
1349fn instrument_closes_schema() -> Schema {
1350 Schema::new(vec![
1351 utf8_field("instrument_id", false),
1352 float64_field("close_price", false),
1353 utf8_field("close_type", false),
1354 timestamp_field("ts_event", false),
1355 timestamp_field("ts_init", false),
1356 ])
1357}
1358
1359fn convert_instrument_closes(
1360 metadata: &HashMap<String, String>,
1361 batch: &RecordBatch,
1362) -> Result<RecordBatch, EncodingError> {
1363 let instrument_id = instrument_id(metadata)?;
1364 let price_precision = parse_price_precision(metadata)?;
1365 let close_price = fixed_col(batch, "close_price")?;
1366 let close_type_index = batch.schema().index_of("close_type")?;
1367 let close_type = extract_column_string(batch.columns(), "close_type", close_type_index)?;
1368 let ts_event = nanos_col(batch, "ts_event")?;
1369 let ts_init = nanos_col(batch, "ts_init")?;
1370 let len = batch.num_rows();
1371 let mut close_price_builder = Float64Builder::with_capacity(len);
1372 let mut close_type_builder = StringBuilder::new();
1373 let mut ts_event_builder =
1374 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1375 let mut ts_init_builder =
1376 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1377
1378 for row in 0..len {
1379 let close_type_value = close_type.value(row);
1380 let close_type = InstrumentCloseType::from_str(close_type_value).map_err(|e| {
1381 EncodingError::ParseError(stringify!(InstrumentCloseType), e.to_string())
1382 })?;
1383 append_price(
1384 &mut close_price_builder,
1385 &close_price,
1386 price_precision,
1387 "close_price",
1388 row,
1389 )?;
1390 close_type_builder.append_value(close_type.as_ref());
1391 append_timestamp(&mut ts_event_builder, &ts_event, row);
1392 append_timestamp(&mut ts_init_builder, &ts_init, row);
1393 }
1394
1395 RecordBatch::try_new(
1396 Arc::new(instrument_closes_schema()),
1397 vec![
1398 constant_string_column(instrument_id, len),
1399 Arc::new(close_price_builder.finish()),
1400 Arc::new(close_type_builder.finish()),
1401 Arc::new(ts_event_builder.finish()),
1402 Arc::new(ts_init_builder.finish()),
1403 ],
1404 )
1405 .map_err(EncodingError::from)
1406}
1407
1408fn option_greeks_schema() -> Schema {
1409 Schema::new(vec![
1410 utf8_field("instrument_id", false),
1411 float64_field("delta", false),
1412 float64_field("gamma", false),
1413 float64_field("vega", false),
1414 float64_field("theta", false),
1415 float64_field("rho", false),
1416 float64_field("mark_iv", true),
1417 float64_field("bid_iv", true),
1418 float64_field("ask_iv", true),
1419 float64_field("underlying_price", true),
1420 float64_field("open_interest", true),
1421 timestamp_field("ts_event", false),
1422 timestamp_field("ts_init", false),
1423 utf8_field("convention", false),
1424 ])
1425}
1426
1427fn append_optional_f64(builder: &mut Float64Builder, values: &Float64Array, row: usize) {
1428 if values.is_null(row) {
1429 builder.append_null();
1430 } else {
1431 builder.append_value(values.value(row));
1432 }
1433}
1434
1435fn convert_option_greeks(
1436 metadata: &HashMap<String, String>,
1437 batch: &RecordBatch,
1438) -> Result<RecordBatch, EncodingError> {
1439 let instrument_id = instrument_id(metadata)?;
1440 let delta = f64_col(batch, "delta")?;
1441 let gamma = f64_col(batch, "gamma")?;
1442 let vega = f64_col(batch, "vega")?;
1443 let theta = f64_col(batch, "theta")?;
1444 let rho = f64_col(batch, "rho")?;
1445 let mark_iv = f64_col(batch, "mark_iv")?;
1446 let bid_iv = f64_col(batch, "bid_iv")?;
1447 let ask_iv = f64_col(batch, "ask_iv")?;
1448 let underlying_price = f64_col(batch, "underlying_price")?;
1449 let open_interest = f64_col(batch, "open_interest")?;
1450 let ts_event = nanos_col(batch, "ts_event")?;
1451 let ts_init = nanos_col(batch, "ts_init")?;
1452 let convention_index = batch.schema().index_of("convention")?;
1453 let convention = extract_column_string(batch.columns(), "convention", convention_index)?;
1454 let len = batch.num_rows();
1455
1456 let mut delta_builder = Float64Builder::with_capacity(len);
1457 let mut gamma_builder = Float64Builder::with_capacity(len);
1458 let mut vega_builder = Float64Builder::with_capacity(len);
1459 let mut theta_builder = Float64Builder::with_capacity(len);
1460 let mut rho_builder = Float64Builder::with_capacity(len);
1461 let mut mark_iv_builder = Float64Builder::with_capacity(len);
1462 let mut bid_iv_builder = Float64Builder::with_capacity(len);
1463 let mut ask_iv_builder = Float64Builder::with_capacity(len);
1464 let mut underlying_price_builder = Float64Builder::with_capacity(len);
1465 let mut open_interest_builder = Float64Builder::with_capacity(len);
1466 let mut ts_event_builder =
1467 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1468 let mut ts_init_builder =
1469 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1470 let mut convention_builder = StringBuilder::new();
1471
1472 for row in 0..len {
1473 delta_builder.append_value(delta.value(row));
1474 gamma_builder.append_value(gamma.value(row));
1475 vega_builder.append_value(vega.value(row));
1476 theta_builder.append_value(theta.value(row));
1477 rho_builder.append_value(rho.value(row));
1478 append_optional_f64(&mut mark_iv_builder, mark_iv, row);
1479 append_optional_f64(&mut bid_iv_builder, bid_iv, row);
1480 append_optional_f64(&mut ask_iv_builder, ask_iv, row);
1481 append_optional_f64(&mut underlying_price_builder, underlying_price, row);
1482 append_optional_f64(&mut open_interest_builder, open_interest, row);
1483 append_timestamp(&mut ts_event_builder, &ts_event, row);
1484 append_timestamp(&mut ts_init_builder, &ts_init, row);
1485 convention_builder.append_value(convention.value(row));
1486 }
1487
1488 RecordBatch::try_new(
1489 Arc::new(option_greeks_schema()),
1490 vec![
1491 constant_string_column(instrument_id, len),
1492 Arc::new(delta_builder.finish()),
1493 Arc::new(gamma_builder.finish()),
1494 Arc::new(vega_builder.finish()),
1495 Arc::new(theta_builder.finish()),
1496 Arc::new(rho_builder.finish()),
1497 Arc::new(mark_iv_builder.finish()),
1498 Arc::new(bid_iv_builder.finish()),
1499 Arc::new(ask_iv_builder.finish()),
1500 Arc::new(underlying_price_builder.finish()),
1501 Arc::new(open_interest_builder.finish()),
1502 Arc::new(ts_event_builder.finish()),
1503 Arc::new(ts_init_builder.finish()),
1504 Arc::new(convention_builder.finish()),
1505 ],
1506 )
1507 .map_err(EncodingError::from)
1508}
1509
1510fn instrument_status_schema() -> Schema {
1511 Schema::new(vec![
1512 utf8_field("instrument_id", false),
1513 utf8_field("action", false),
1514 timestamp_field("ts_event", false),
1515 timestamp_field("ts_init", false),
1516 utf8_field("reason", true),
1517 utf8_field("trading_event", true),
1518 Field::new("is_trading", DataType::Boolean, true),
1519 Field::new("is_quoting", DataType::Boolean, true),
1520 Field::new("is_short_sell_restricted", DataType::Boolean, true),
1521 ])
1522}
1523
1524fn append_optional_string(
1525 builder: &mut StringBuilder,
1526 values: &super::StringColumnRef<'_>,
1527 row: usize,
1528 array: &dyn Array,
1529) {
1530 if array.is_null(row) {
1531 builder.append_null();
1532 } else {
1533 builder.append_value(values.value(row));
1534 }
1535}
1536
1537fn append_optional_bool(
1538 builder: &mut arrow::array::BooleanBuilder,
1539 values: &BooleanArray,
1540 row: usize,
1541) {
1542 if values.is_null(row) {
1543 builder.append_null();
1544 } else {
1545 builder.append_value(values.value(row));
1546 }
1547}
1548
1549fn append_optional_u64(
1550 builder: &mut UInt64Builder,
1551 values: &U64ColumnRef<'_>,
1552 field: &'static str,
1553 row: usize,
1554) -> Result<(), EncodingError> {
1555 if values.is_null(row) {
1556 builder.append_null();
1557 } else {
1558 builder.append_value(u64_value(values, row, field)?);
1559 }
1560 Ok(())
1561}
1562
1563fn convert_instrument_status(
1564 metadata: &HashMap<String, String>,
1565 batch: &RecordBatch,
1566) -> Result<RecordBatch, EncodingError> {
1567 let instrument_id = instrument_id(metadata)?;
1568 let action_index = batch.schema().index_of("action")?;
1569 let action = extract_column_string(batch.columns(), "action", action_index)?;
1570 let ts_event = nanos_col(batch, "ts_event")?;
1571 let ts_init = nanos_col(batch, "ts_init")?;
1572 let reason_index = batch.schema().index_of("reason")?;
1573 let reason = extract_column_string(batch.columns(), "reason", reason_index)?;
1574 let trading_event_index = batch.schema().index_of("trading_event")?;
1575 let trading_event =
1576 extract_column_string(batch.columns(), "trading_event", trading_event_index)?;
1577 let is_trading = bool_col(batch, "is_trading")?;
1578 let is_quoting = bool_col(batch, "is_quoting")?;
1579 let is_short_sell_restricted = bool_col(batch, "is_short_sell_restricted")?;
1580 let len = batch.num_rows();
1581
1582 let mut action_builder = StringBuilder::new();
1583 let mut ts_event_builder =
1584 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1585 let mut ts_init_builder =
1586 TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1587 let mut reason_builder = StringBuilder::new();
1588 let mut trading_event_builder = StringBuilder::new();
1589 let mut is_trading_builder = arrow::array::BooleanBuilder::with_capacity(len);
1590 let mut is_quoting_builder = arrow::array::BooleanBuilder::with_capacity(len);
1591 let mut is_short_sell_restricted_builder = arrow::array::BooleanBuilder::with_capacity(len);
1592
1593 for row in 0..len {
1594 action_builder.append_value(action.value(row));
1595 append_timestamp(&mut ts_event_builder, &ts_event, row);
1596 append_timestamp(&mut ts_init_builder, &ts_init, row);
1597 append_optional_string(
1598 &mut reason_builder,
1599 &reason,
1600 row,
1601 batch.column(reason_index).as_ref(),
1602 );
1603 append_optional_string(
1604 &mut trading_event_builder,
1605 &trading_event,
1606 row,
1607 batch.column(trading_event_index).as_ref(),
1608 );
1609 append_optional_bool(&mut is_trading_builder, is_trading, row);
1610 append_optional_bool(&mut is_quoting_builder, is_quoting, row);
1611 append_optional_bool(
1612 &mut is_short_sell_restricted_builder,
1613 is_short_sell_restricted,
1614 row,
1615 );
1616 }
1617
1618 RecordBatch::try_new(
1619 Arc::new(instrument_status_schema()),
1620 vec![
1621 constant_string_column(instrument_id, len),
1622 Arc::new(action_builder.finish()),
1623 Arc::new(ts_event_builder.finish()),
1624 Arc::new(ts_init_builder.finish()),
1625 Arc::new(reason_builder.finish()),
1626 Arc::new(trading_event_builder.finish()),
1627 Arc::new(is_trading_builder.finish()),
1628 Arc::new(is_quoting_builder.finish()),
1629 Arc::new(is_short_sell_restricted_builder.finish()),
1630 ],
1631 )
1632 .map_err(EncodingError::from)
1633}
1634
1635fn convert_custom(batch: &RecordBatch) -> Result<RecordBatch, EncodingError> {
1640 let batch_schema = batch.schema();
1641 let metadata = batch_schema.metadata();
1642 let mut fields = Vec::with_capacity(batch.num_columns());
1643 let mut columns = Vec::with_capacity(batch.num_columns());
1644
1645 for (field, column) in batch_schema.fields().iter().zip(batch.columns()) {
1646 if matches!(field.name().as_str(), "ts_event" | "ts_init") {
1647 let name = if field.name() == "ts_event" {
1648 "ts_event"
1649 } else {
1650 "ts_init"
1651 };
1652 let values = nanos_col(batch, name)?;
1653 fields.push(Arc::new(timestamp_field_from(field)));
1654 columns.push(timestamp_array_from_nanos(&values));
1655 } else if let Some((precision, is_price)) = custom_fixed_precision(field.name(), metadata)
1656 && matches!(
1657 field.data_type(),
1658 DataType::Decimal128(_, _)
1659 | DataType::FixedSizeBinary(_)
1660 | DataType::Binary
1661 | DataType::BinaryView
1662 )
1663 {
1664 if field.data_type() != &fixed_decimal_data_type() {
1665 return Err(EncodingError::ParseError(
1666 "custom",
1667 format!(
1668 "{} must use Decimal128(38, 16); migrate legacy data before display",
1669 field.name()
1670 ),
1671 ));
1672 }
1673 let values = FixedPrecisionColumn(
1674 column
1675 .as_any()
1676 .downcast_ref::<Decimal128Array>()
1677 .expect("decimal field type checked above"),
1678 );
1679 fields.push(Arc::new(Field::new(
1680 field.name(),
1681 DataType::Float64,
1682 field.is_nullable(),
1683 )));
1684 columns.push(fixed_precision_field_to_f64(&values, precision, is_price)?);
1685 } else if field.data_type() == &DataType::Utf8View {
1686 fields.push(Arc::new(Field::new(
1687 field.name(),
1688 DataType::Utf8,
1689 field.is_nullable(),
1690 )));
1691 columns.push(cast(column.as_ref(), &DataType::Utf8)?);
1692 } else {
1693 fields.push(Arc::clone(field));
1694 columns.push(Arc::clone(column));
1695 }
1696 }
1697
1698 let metadata = if let Some(type_name) = metadata.get("type_name")
1699 && get_arrow_schema(type_name).is_some()
1700 {
1701 let canonical_metadata = HashMap::from([("type_name".to_string(), type_name.clone())]);
1702 let canonical_schema = super::schema_with_identifier_column(
1703 &CustomDataDecoder::get_schema(Some(canonical_metadata)),
1704 );
1705
1706 for field in &mut fields {
1707 if let Ok(canonical_field) = canonical_schema.field_with_name(field.name()) {
1708 *field = Arc::new(
1709 field
1710 .as_ref()
1711 .clone()
1712 .with_nullable(canonical_field.is_nullable()),
1713 );
1714 }
1715 }
1716 canonical_schema.metadata().clone()
1717 } else {
1718 batch_schema.metadata().clone()
1719 };
1720 let schema = Schema::new_with_metadata(fields, metadata);
1721 RecordBatch::try_new(Arc::new(schema), columns).map_err(EncodingError::from)
1722}
1723
1724#[cfg(test)]
1725mod tests {
1726 use std::{collections::HashMap, str::FromStr};
1727
1728 use arrow::{
1729 array::{
1730 Array, FixedSizeBinaryArray, FixedSizeListArray, Float64Array, StringArray,
1731 StringViewArray, UInt64Array,
1732 },
1733 datatypes::{Field, TimeUnit},
1734 };
1735 use nautilus_core::UnixNanos;
1736 #[cfg(feature = "arrow-display")]
1737 use nautilus_model::data::stubs::{
1738 stub_bar, stub_delta, stub_instrument_close, stub_trade_ethusdt_buy,
1739 };
1740 use nautilus_model::{
1741 data::{
1742 FundingRateUpdate, OrderBookDepth, QuoteTick, ensure_arrow_registered,
1743 order::BookOrder, stubs::stub_depth10,
1744 },
1745 identifiers::InstrumentId,
1746 types::{Price, Quantity},
1747 };
1748 use rstest::rstest;
1749 use rust_decimal::Decimal;
1750
1751 use super::*;
1752 use crate::arrow::{
1753 EncodeToRecordBatch, KEY_INSTRUMENT_ID, KEY_PRICE_PRECISION, KEY_SIZE_PRECISION,
1754 book_deltas_to_arrow_record_batch_bytes, enum_dictionary_array,
1755 };
1756 #[cfg(feature = "arrow-display")]
1757 use crate::arrow::{
1758 display::{
1759 bar::encode_bars, close::encode_instrument_closes, delta::encode_deltas,
1760 trade::encode_trades,
1761 },
1762 record_batch_without_identifier_column,
1763 };
1764
1765 #[rstest]
1766 #[case::clear_only(false)]
1767 #[case::mixed(true)]
1768 fn test_catalog_display_converts_clear_deltas(#[case] mixed: bool) {
1769 let instrument_id = InstrumentId::from("AAPL.XNAS");
1770 let clear = OrderBookDelta::clear(instrument_id, 41, 101.into(), 102.into());
1771 let mut deltas = vec![clear];
1772 if mixed {
1773 deltas.extend([
1774 OrderBookDelta::new(
1775 instrument_id,
1776 BookAction::Add,
1777 BookOrder::new(
1778 OrderSide::Buy,
1779 Price::from("100.25"),
1780 Quantity::from("2.50"),
1781 9_007_199_254_740_993,
1782 ),
1783 128,
1784 42,
1785 103.into(),
1786 104.into(),
1787 ),
1788 OrderBookDelta::new(
1789 instrument_id,
1790 BookAction::Update,
1791 BookOrder::new(
1792 OrderSide::Sell,
1793 Price::from("101.75"),
1794 Quantity::from("3.25"),
1795 73,
1796 ),
1797 64,
1798 43,
1799 105.into(),
1800 106.into(),
1801 ),
1802 ]);
1803 }
1804
1805 let raw = book_deltas_to_arrow_record_batch_bytes(&deltas).unwrap();
1806 let display = catalog_record_batch_to_display(
1807 &NautilusDataType::OrderBookDelta,
1808 raw.schema().metadata(),
1809 &raw,
1810 )
1811 .unwrap();
1812 let len = deltas.len();
1813 let mut fields = deltas_schema().fields().to_vec();
1814 fields.push(Arc::new(utf8_field(KEY_IDENTIFIER, true)));
1815 let expected_schema = Schema::new(fields);
1816 let expected: Vec<ArrayRef> = vec![
1817 Arc::new(StringArray::from(vec!["AAPL.XNAS"; len])),
1818 Arc::new(StringArray::from(
1819 ["CLEAR", "ADD", "UPDATE"][..len].to_vec(),
1820 )),
1821 Arc::new(StringArray::from(
1822 ["NO_ORDER_SIDE", "BUY", "SELL"][..len].to_vec(),
1823 )),
1824 Arc::new(Float64Array::from(
1825 [f64::NAN, 100.25, 101.75][..len].to_vec(),
1826 )),
1827 Arc::new(Float64Array::from([f64::NAN, 2.50, 3.25][..len].to_vec())),
1828 Arc::new(StringArray::from(
1829 ["0", "9007199254740993", "73"][..len].to_vec(),
1830 )),
1831 Arc::new(UInt8Array::from([32, 128, 64][..len].to_vec())),
1832 Arc::new(UInt64Array::from([41, 42, 43][..len].to_vec())),
1833 Arc::new(
1834 TimestampNanosecondArray::from([101, 103, 105][..len].to_vec())
1835 .with_timezone("UTC"),
1836 ),
1837 Arc::new(
1838 TimestampNanosecondArray::from([102, 104, 106][..len].to_vec())
1839 .with_timezone("UTC"),
1840 ),
1841 Arc::new(StringArray::from(vec!["AAPL.XNAS"; len])),
1842 ];
1843
1844 assert_eq!(display.schema().as_ref(), &expected_schema);
1845 assert_eq!(display.num_rows(), len);
1846 assert_eq!(display.num_columns(), expected.len());
1847
1848 for (actual, expected) in display.columns().iter().zip(&expected) {
1849 if let Some(actual) = actual.as_any().downcast_ref::<Float64Array>() {
1850 let expected = expected.as_any().downcast_ref::<Float64Array>().unwrap();
1851 assert_eq!(actual.null_count(), 0);
1852 assert!(actual.value(0).is_nan());
1853 assert_eq!(&actual.values()[1..], &expected.values()[1..]);
1854 } else {
1855 assert_eq!(actual, expected);
1856 }
1857 }
1858 }
1859
1860 #[rstest]
1861 #[case("no_order_side", "NO_ORDER_SIDE")]
1862 #[case("buy", "BUY")]
1863 #[case("sell", "SELL")]
1864 fn test_catalog_display_normalizes_delta_side(#[case] side: &str, #[case] expected: &str) {
1865 let delta =
1866 OrderBookDelta::clear(InstrumentId::from("AAPL.XNAS"), 41, 101.into(), 102.into());
1867 let raw = book_deltas_to_arrow_record_batch_bytes(&[delta]).unwrap();
1868 let mut columns = raw.columns().to_vec();
1869 let side_index = raw.schema().index_of("side").unwrap();
1870 let side = enum_dictionary_array([side]).unwrap();
1871 columns[side_index] = Arc::new(side);
1872 let raw = RecordBatch::try_new(raw.schema(), columns).unwrap();
1873
1874 let display = catalog_record_batch_to_display(
1875 &NautilusDataType::OrderBookDelta,
1876 raw.schema().metadata(),
1877 &raw,
1878 )
1879 .unwrap();
1880
1881 assert_eq!(
1882 display.column_by_name("side").unwrap().as_ref(),
1883 &StringArray::from(vec![expected])
1884 );
1885 }
1886
1887 #[rstest]
1888 #[case::clear(BookAction::Clear)]
1889 #[case::add(BookAction::Add)]
1890 fn test_catalog_display_rejects_invalid_delta_side(#[case] action: BookAction) {
1891 let instrument_id = InstrumentId::from("AAPL.XNAS");
1892 let mut delta = OrderBookDelta::clear(instrument_id, 41, 101.into(), 102.into());
1893 delta.action = action;
1894 let raw = book_deltas_to_arrow_record_batch_bytes(&[delta]).unwrap();
1895 let mut columns = raw.columns().to_vec();
1896 let side_index = raw.schema().index_of("side").unwrap();
1897 let side = enum_dictionary_array(["INVALID"]).unwrap();
1898 columns[side_index] = Arc::new(side);
1899 let raw = RecordBatch::try_new(raw.schema(), columns).unwrap();
1900
1901 let error = catalog_record_batch_to_display(
1902 &NautilusDataType::OrderBookDelta,
1903 raw.schema().metadata(),
1904 &raw,
1905 )
1906 .unwrap_err();
1907
1908 assert!(
1909 matches!(error, EncodingError::ParseError("OrderSide", message) if message == "Matching variant not found")
1910 );
1911 }
1912
1913 #[rstest]
1914 fn test_catalog_record_batch_to_display_converts_quote_prices_and_metadata() {
1915 let instrument_id = InstrumentId::from("AUD/USD.SIM");
1916 let metadata = HashMap::from([
1917 (KEY_INSTRUMENT_ID.to_string(), instrument_id.to_string()),
1918 (KEY_PRICE_PRECISION.to_string(), "5".to_string()),
1919 (KEY_SIZE_PRECISION.to_string(), "0".to_string()),
1920 ]);
1921 let quote = QuoteTick {
1922 instrument_id,
1923 bid_price: Price::from("1.00001"),
1924 ask_price: Price::from("1.00003"),
1925 bid_size: Quantity::from(1_000),
1926 ask_size: Quantity::from(2_000),
1927 ts_event: 10.into(),
1928 ts_init: 11.into(),
1929 };
1930 let raw_batch = QuoteTick::encode_batch(&metadata, &[quote]).unwrap();
1931
1932 let display_batch =
1933 catalog_record_batch_to_display(&NautilusDataType::QuoteTick, &metadata, &raw_batch)
1934 .unwrap();
1935
1936 let mut expected_fields = quotes_schema().fields().to_vec();
1937 expected_fields.push(Arc::new(utf8_field(KEY_IDENTIFIER, true)));
1938 assert_eq!(
1939 display_batch.schema(),
1940 Arc::new(Schema::new(expected_fields))
1941 );
1942 let instrument_ids = display_batch
1943 .column(0)
1944 .as_any()
1945 .downcast_ref::<StringArray>()
1946 .unwrap();
1947 let bid_prices = display_batch
1948 .column(1)
1949 .as_any()
1950 .downcast_ref::<Float64Array>()
1951 .unwrap();
1952 let ask_sizes = display_batch
1953 .column(4)
1954 .as_any()
1955 .downcast_ref::<Float64Array>()
1956 .unwrap();
1957 let ts_init = display_batch
1958 .column(6)
1959 .as_any()
1960 .downcast_ref::<arrow::array::TimestampNanosecondArray>()
1961 .unwrap();
1962 let identifiers = display_batch
1963 .column(7)
1964 .as_any()
1965 .downcast_ref::<StringArray>()
1966 .unwrap();
1967
1968 assert_eq!(instrument_ids.value(0), "AUD/USD.SIM");
1969 assert_eq!(bid_prices.value(0), 1.00001);
1970 assert_eq!(ask_sizes.value(0), 2_000.0);
1971 assert_eq!(ts_init.value(0), 11);
1972 assert_eq!(identifiers.value(0), "AUD/USD.SIM");
1973 }
1974
1975 #[rstest]
1976 #[case(1)]
1977 #[case(u64::MAX)]
1978 fn test_catalog_record_batch_to_display_rejects_legacy_timestamps(#[case] timestamp: u64) {
1979 let instrument_id = InstrumentId::from("AUD/USD.SIM");
1980 let metadata = HashMap::from([
1981 (KEY_INSTRUMENT_ID.to_string(), instrument_id.to_string()),
1982 (KEY_PRICE_PRECISION.to_string(), "5".to_string()),
1983 (KEY_SIZE_PRECISION.to_string(), "0".to_string()),
1984 ]);
1985 let quote = QuoteTick {
1986 instrument_id,
1987 bid_price: Price::from("1.00001"),
1988 ask_price: Price::from("1.00003"),
1989 bid_size: Quantity::from(1_000),
1990 ask_size: Quantity::from(2_000),
1991 ts_event: 10.into(),
1992 ts_init: 11.into(),
1993 };
1994 let raw_batch = QuoteTick::encode_batch(&metadata, &[quote]).unwrap();
1995 let mut fields = raw_batch.schema().fields().to_vec();
1996 let ts_init_index = raw_batch.schema().index_of("ts_init").unwrap();
1997 fields[ts_init_index] = Arc::new(Field::new("ts_init", DataType::UInt64, false));
1998 let mut columns = raw_batch.columns().to_vec();
1999 columns[ts_init_index] = Arc::new(UInt64Array::from(vec![timestamp]));
2000 let raw_batch = RecordBatch::try_new(
2001 Arc::new(Schema::new_with_metadata(fields, metadata.clone())),
2002 columns,
2003 )
2004 .unwrap();
2005
2006 let error =
2007 catalog_record_batch_to_display(&NautilusDataType::QuoteTick, &metadata, &raw_batch)
2008 .unwrap_err();
2009
2010 assert!(matches!(
2011 error,
2012 EncodingError::InvalidColumnType(
2013 "ts_init",
2014 _,
2015 DataType::Timestamp(TimeUnit::Nanosecond, Some(timezone)),
2016 DataType::UInt64
2017 ) if timezone.as_ref() == "UTC"
2018 ));
2019 }
2020
2021 #[rstest]
2022 #[case(38, 15)]
2023 #[case(37, 16)]
2024 fn test_catalog_display_rejects_noncanonical_decimal_schema(
2025 #[case] precision: u8,
2026 #[case] scale: i8,
2027 ) {
2028 let quote = QuoteTick::new(
2029 InstrumentId::from("AUD/USD.SIM"),
2030 Price::from("1.25"),
2031 Price::from("1.50"),
2032 Quantity::from(10),
2033 Quantity::from(20),
2034 10.into(),
2035 11.into(),
2036 );
2037 let metadata = quote.metadata();
2038 let raw = QuoteTick::encode_batch(&metadata, &[quote]).unwrap();
2039 let index = raw.schema().index_of("bid_price").unwrap();
2040 let mut fields = raw.schema().fields().to_vec();
2041 let mut columns = raw.columns().to_vec();
2042 let values = columns[index]
2043 .as_any()
2044 .downcast_ref::<Decimal128Array>()
2045 .unwrap()
2046 .clone()
2047 .with_precision_and_scale(precision, scale)
2048 .unwrap();
2049 fields[index] = Arc::new(
2050 fields[index]
2051 .as_ref()
2052 .clone()
2053 .with_data_type(values.data_type().clone()),
2054 );
2055 columns[index] = Arc::new(values);
2056 let batch = RecordBatch::try_new(
2057 Arc::new(Schema::new_with_metadata(fields, metadata.clone())),
2058 columns,
2059 )
2060 .unwrap();
2061 let error =
2062 catalog_record_batch_to_display(&NautilusDataType::QuoteTick, &metadata, &batch)
2063 .unwrap_err();
2064
2065 assert!(
2066 matches!(error, EncodingError::InvalidColumnType("bid_price", i, DataType::Decimal128(38, 16), actual) if i == index && actual == DataType::Decimal128(precision, scale))
2067 );
2068 }
2069
2070 #[rstest]
2071 #[case(0)]
2072 #[case(5)]
2073 #[case(10)]
2074 #[case(25)]
2075 fn test_depth_display_preserves_all_levels(#[case] levels: usize) {
2076 let mut depth = stub_depth10();
2077 depth.bids.resize(levels, depth.bids[0]);
2078 depth.bid_counts.resize(levels, 0);
2079 depth.asks.clear();
2080 depth.ask_counts.clear();
2081 for (i, (order, count)) in depth.bids.iter_mut().zip(&mut depth.bid_counts).enumerate() {
2082 order.price = format!("{}.25", 100 + i).parse().unwrap();
2083 order.size = format!("{}.5", 200 + i).parse().unwrap();
2084 order.order_id = u64::MAX - u64::try_from(i).unwrap();
2085 *count = 300 + u32::try_from(i).unwrap();
2086 }
2087 let metadata = depth.metadata();
2088 let raw = OrderBookDepth::encode_batch(&metadata, &[depth]).unwrap();
2089 let display =
2090 catalog_record_batch_to_display(&NautilusDataType::OrderBookDepth, &metadata, &raw)
2091 .unwrap();
2092 let list = display
2093 .column_by_name("bids")
2094 .unwrap()
2095 .as_any()
2096 .downcast_ref::<ListArray>()
2097 .unwrap();
2098 let values = list.value(0);
2099 let structs = values.as_any().downcast_ref::<StructArray>().unwrap();
2100 let prices = structs
2101 .column_by_name("price")
2102 .unwrap()
2103 .as_any()
2104 .downcast_ref::<Float64Array>()
2105 .unwrap();
2106 let sizes = structs
2107 .column_by_name("size")
2108 .unwrap()
2109 .as_any()
2110 .downcast_ref::<Float64Array>()
2111 .unwrap();
2112 let counts = structs
2113 .column_by_name("count")
2114 .unwrap()
2115 .as_any()
2116 .downcast_ref::<UInt32Array>()
2117 .unwrap();
2118 let ids = structs
2119 .column_by_name("order_id")
2120 .unwrap()
2121 .as_any()
2122 .downcast_ref::<UInt64Array>()
2123 .unwrap();
2124 let asks = display
2125 .column_by_name("asks")
2126 .unwrap()
2127 .as_any()
2128 .downcast_ref::<ListArray>()
2129 .unwrap();
2130 let empty = catalog_record_batch_to_display(
2131 &NautilusDataType::OrderBookDepth,
2132 &metadata,
2133 &raw.slice(0, 0),
2134 )
2135 .unwrap();
2136
2137 assert_eq!(
2138 display.schema().as_ref(),
2139 &crate::arrow::schema_with_identifier_column(&depth_schema())
2140 );
2141 assert_eq!(empty.schema(), display.schema());
2142 assert_eq!(empty.num_rows(), 0);
2143 assert_eq!(display.num_rows(), 1);
2144 assert_eq!(list.value_length(0), i32::try_from(levels).unwrap());
2145 assert_eq!(asks.value_length(0), 0);
2146 assert!(!asks.is_null(0));
2147
2148 for i in 0..levels {
2149 assert_eq!(
2150 prices.value(i),
2151 f64::from(u32::try_from(100 + i).unwrap()) + 0.25
2152 );
2153 assert_eq!(
2154 sizes.value(i),
2155 f64::from(u32::try_from(200 + i).unwrap()) + 0.5
2156 );
2157 assert_eq!(counts.value(i), 300 + u32::try_from(i).unwrap());
2158 assert_eq!(ids.value(i), u64::MAX - u64::try_from(i).unwrap());
2159 }
2160 }
2161
2162 #[rstest]
2163 #[case(false, false)]
2164 #[case(false, true)]
2165 #[case(true, false)]
2166 #[case(true, true)]
2167 fn test_depth_display_rejects_legacy_shapes(#[case] lists: bool, #[case] with_ids: bool) {
2168 let depth = stub_depth10();
2169 let metadata = depth.metadata();
2170 let raw = OrderBookDepth::encode_batch(&metadata, &[depth]).unwrap();
2171 let mut columns = Vec::<ArrayRef>::new();
2172 let mut fields = Vec::<Field>::new();
2173
2174 for (side, prefix) in [("bids", "bid"), ("asks", "ask")] {
2175 let list = raw
2176 .column_by_name(side)
2177 .unwrap()
2178 .as_any()
2179 .downcast_ref::<ListArray>()
2180 .unwrap();
2181 let levels = list
2182 .values()
2183 .as_any()
2184 .downcast_ref::<StructArray>()
2185 .unwrap();
2186
2187 for name in ["price", "size", "count", "order_id"] {
2188 if name == "order_id" && !with_ids {
2189 continue;
2190 }
2191 let values = levels.column_by_name(name).unwrap();
2192 if lists {
2193 let item = Arc::new(Field::new("item", values.data_type().clone(), true));
2194 let column =
2195 FixedSizeListArray::try_new(item, 10, Arc::clone(values), None).unwrap();
2196 fields.push(Field::new(
2197 format!("{prefix}_{name}"),
2198 column.data_type().clone(),
2199 false,
2200 ));
2201 columns.push(Arc::new(column));
2202 } else {
2203 for i in 0..10 {
2204 fields.push(Field::new(
2205 format!("{prefix}_{name}_{i}"),
2206 values.data_type().clone(),
2207 true,
2208 ));
2209 columns.push(values.slice(i, 1));
2210 }
2211 }
2212 }
2213 }
2214
2215 for name in ["flags", "sequence", "ts_event", "ts_init"] {
2216 fields.push(raw.schema().field_with_name(name).unwrap().clone());
2217 columns.push(Arc::clone(raw.column_by_name(name).unwrap()));
2218 }
2219 let legacy = RecordBatch::try_new(
2220 Arc::new(Schema::new_with_metadata(fields, metadata.clone())),
2221 columns,
2222 )
2223 .unwrap();
2224 let error =
2225 catalog_record_batch_to_display(&NautilusDataType::OrderBookDepth, &metadata, &legacy)
2226 .unwrap_err();
2227 assert!(
2228 matches!(error, EncodingError::ParseError("bids", ref message) if message == "expected List<Struct>")
2229 );
2230 }
2231
2232 #[rstest]
2233 fn test_catalog_display_schema_uses_data_type() {
2234 let schema = catalog_display_schema(&NautilusDataType::QuoteTick).unwrap();
2235
2236 assert_eq!(schema.fields()[0].name(), "instrument_id");
2237 assert_eq!(schema.fields()[1].data_type(), &DataType::Float64);
2238 assert_eq!(schema.fields().last().unwrap().name(), "ts_init");
2239 }
2240
2241 #[rstest]
2242 fn test_custom_catalog_display_schema_uses_registered_schema() {
2243 let type_name = "CatalogDisplaySchemaCustom";
2244 let schema = Schema::new(vec![
2245 Field::new("value", DataType::Float64, false),
2246 Field::new(
2247 "ts_init",
2248 DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
2249 false,
2250 ),
2251 ]);
2252 ensure_arrow_registered(
2253 type_name,
2254 Arc::new(schema),
2255 Box::new(|_| unreachable!()),
2256 Box::new(|_, _| unreachable!()),
2257 )
2258 .unwrap();
2259
2260 let data_type = NautilusDataType::Custom {
2261 type_name: type_name.to_string(),
2262 };
2263 let raw_schema = catalog_raw_schema(&data_type).unwrap();
2264 let schema = catalog_display_schema(&data_type).unwrap();
2265 let fields = schema
2266 .fields()
2267 .iter()
2268 .map(|field| (field.name().as_str(), field.data_type()))
2269 .collect::<Vec<_>>();
2270
2271 assert_eq!(
2272 fields,
2273 [
2274 ("value", &DataType::Float64),
2275 (
2276 "ts_init",
2277 &DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
2278 ),
2279 ("data_type", &DataType::Utf8),
2280 ("identifier", &DataType::Utf8),
2281 ]
2282 );
2283 assert_eq!(raw_schema.fields().last().unwrap().name(), "identifier");
2284 }
2285
2286 #[rstest]
2287 fn test_custom_catalog_schema_rejects_unregistered_type() {
2288 let data_type = NautilusDataType::Custom {
2289 type_name: "UnregisteredCatalogDisplayCustom".to_string(),
2290 };
2291
2292 assert!(catalog_raw_schema(&data_type).is_err());
2293 assert!(catalog_display_schema(&data_type).is_err());
2294 }
2295
2296 #[rstest]
2297 fn test_custom_display_normalizes_utf8_view_to_utf8() {
2298 let batch = RecordBatch::try_new(
2299 Arc::new(Schema::new(vec![Field::new(
2300 "label",
2301 DataType::Utf8View,
2302 false,
2303 )])),
2304 vec![Arc::new(StringViewArray::from(vec!["value"]))],
2305 )
2306 .unwrap();
2307 let display = catalog_record_batch_to_display(
2308 &NautilusDataType::Custom {
2309 type_name: "ViewCustom".to_string(),
2310 },
2311 batch.schema().metadata(),
2312 &batch,
2313 )
2314 .unwrap();
2315
2316 assert_eq!(display.schema().field(0).data_type(), &DataType::Utf8);
2317 assert_eq!(
2318 display
2319 .column(0)
2320 .as_any()
2321 .downcast_ref::<StringArray>()
2322 .unwrap()
2323 .value(0),
2324 "value"
2325 );
2326 }
2327
2328 #[rstest]
2329 fn test_catalog_record_batch_to_display_converts_funding_rates() {
2330 let instrument_id = InstrumentId::from("BTCUSDT-PERP.BINANCE");
2331 let funding_rate = FundingRateUpdate::new(
2332 instrument_id,
2333 Decimal::from_str("0.000125").unwrap(),
2334 Some(480),
2335 None,
2336 UnixNanos::from(10),
2337 UnixNanos::from(11),
2338 );
2339 let metadata = funding_rate.metadata();
2340 let raw_batch = FundingRateUpdate::encode_batch(&metadata, &[funding_rate]).unwrap();
2341
2342 let display_batch = catalog_record_batch_to_display(
2343 &NautilusDataType::FundingRateUpdate,
2344 &metadata,
2345 &raw_batch,
2346 )
2347 .unwrap();
2348
2349 let mut expected_fields = funding_rate_schema().fields().to_vec();
2350 expected_fields.push(Arc::new(utf8_field(KEY_IDENTIFIER, true)));
2351 assert_eq!(
2352 display_batch.schema(),
2353 Arc::new(Schema::new(expected_fields))
2354 );
2355 let instrument_ids = display_batch
2356 .column(0)
2357 .as_any()
2358 .downcast_ref::<StringArray>()
2359 .unwrap();
2360 let rates = display_batch
2361 .column(1)
2362 .as_any()
2363 .downcast_ref::<Float64Array>()
2364 .unwrap();
2365 let intervals = display_batch
2366 .column(2)
2367 .as_any()
2368 .downcast_ref::<UInt64Array>()
2369 .unwrap();
2370 let next_funding_ns = display_batch
2371 .column(3)
2372 .as_any()
2373 .downcast_ref::<TimestampNanosecondArray>()
2374 .unwrap();
2375 let identifiers = display_batch
2376 .column(6)
2377 .as_any()
2378 .downcast_ref::<StringArray>()
2379 .unwrap();
2380
2381 assert_eq!(instrument_ids.value(0), "BTCUSDT-PERP.BINANCE");
2382 assert_eq!(rates.value(0), 0.000_125);
2383 assert_eq!(intervals.value(0), 480);
2384 assert!(next_funding_ns.is_null(0));
2385 assert_eq!(identifiers.value(0), "BTCUSDT-PERP.BINANCE");
2386 }
2387
2388 #[rstest]
2389 fn test_catalog_record_batch_to_display_rejects_precision_mismatch() {
2390 let metadata = HashMap::from([
2391 (KEY_INSTRUMENT_ID.to_string(), "AUD/USD.SIM".to_string()),
2392 (KEY_PRICE_PRECISION.to_string(), "5".to_string()),
2393 (KEY_SIZE_PRECISION.to_string(), "0".to_string()),
2394 ]);
2395 let schema = Arc::new(Schema::new_with_metadata(
2396 vec![
2397 Field::new("bid_price", DataType::FixedSizeBinary(1), false),
2398 Field::new("ask_price", DataType::FixedSizeBinary(1), false),
2399 Field::new("bid_size", DataType::FixedSizeBinary(1), false),
2400 Field::new("ask_size", DataType::FixedSizeBinary(1), false),
2401 Field::new("ts_event", DataType::UInt64, false),
2402 Field::new("ts_init", DataType::UInt64, false),
2403 ],
2404 metadata.clone(),
2405 ));
2406 let batch = RecordBatch::try_new(
2407 schema,
2408 vec![
2409 Arc::new(
2410 FixedSizeBinaryArray::try_from_sparse_iter_with_size(
2411 vec![Some([0_u8].as_slice())].into_iter(),
2412 1,
2413 )
2414 .unwrap(),
2415 ),
2416 Arc::new(
2417 FixedSizeBinaryArray::try_from_sparse_iter_with_size(
2418 vec![Some([0_u8].as_slice())].into_iter(),
2419 1,
2420 )
2421 .unwrap(),
2422 ),
2423 Arc::new(
2424 FixedSizeBinaryArray::try_from_sparse_iter_with_size(
2425 vec![Some([0_u8].as_slice())].into_iter(),
2426 1,
2427 )
2428 .unwrap(),
2429 ),
2430 Arc::new(
2431 FixedSizeBinaryArray::try_from_sparse_iter_with_size(
2432 vec![Some([0_u8].as_slice())].into_iter(),
2433 1,
2434 )
2435 .unwrap(),
2436 ),
2437 Arc::new(UInt64Array::from(vec![1])),
2438 Arc::new(UInt64Array::from(vec![1])),
2439 ],
2440 )
2441 .unwrap();
2442
2443 let error =
2444 catalog_record_batch_to_display(&NautilusDataType::QuoteTick, &metadata, &batch)
2445 .unwrap_err();
2446
2447 assert!(matches!(
2448 error,
2449 EncodingError::InvalidColumnType(
2450 "bid_price",
2451 _,
2452 DataType::Decimal128(38, 16),
2453 DataType::FixedSizeBinary(1)
2454 )
2455 ));
2456 }
2457
2458 #[cfg(feature = "arrow-display")]
2459 #[rstest]
2460 #[case::bar(NautilusDataType::Bar)]
2461 #[case::trade(NautilusDataType::TradeTick)]
2462 #[case::delta(NautilusDataType::OrderBookDelta)]
2463 #[case::close(NautilusDataType::InstrumentClose)]
2464 fn test_catalog_display_matches_typed_encoder(#[case] data_type: NautilusDataType) {
2465 let (raw, expected) = match data_type {
2466 NautilusDataType::Bar => {
2467 let value = stub_bar();
2468 (
2469 Bar::encode_batch(&value.metadata(), &[value]).unwrap(),
2470 encode_bars(&[value]).unwrap(),
2471 )
2472 }
2473 NautilusDataType::TradeTick => {
2474 let value = stub_trade_ethusdt_buy();
2475 (
2476 TradeTick::encode_batch(&value.metadata(), &[value]).unwrap(),
2477 encode_trades(&[value]).unwrap(),
2478 )
2479 }
2480 NautilusDataType::OrderBookDelta => {
2481 let value = stub_delta();
2482 (
2483 OrderBookDelta::encode_batch(&value.metadata(), &[value]).unwrap(),
2484 encode_deltas(&[value]).unwrap(),
2485 )
2486 }
2487 NautilusDataType::InstrumentClose => {
2488 let value = stub_instrument_close();
2489 (
2490 InstrumentClose::encode_batch(&value.metadata(), &[value]).unwrap(),
2491 encode_instrument_closes(&[value]).unwrap(),
2492 )
2493 }
2494 _ => unreachable!(),
2495 };
2496
2497 let displayed =
2498 catalog_record_batch_to_display(&data_type, raw.schema().metadata(), &raw).unwrap();
2499 let identifier = Arc::clone(displayed.column_by_name(KEY_IDENTIFIER).unwrap());
2500 let without_identifier = record_batch_without_identifier_column(displayed).unwrap();
2501
2502 assert_eq!(&identifier, raw.column_by_name(KEY_IDENTIFIER).unwrap());
2503 assert_eq!(without_identifier, expected);
2504 }
2505}