Skip to main content

nautilus_serialization/arrow/
catalog_display.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Converts raw catalog Arrow batches into display-friendly Arrow batches.
17//!
18//! Catalog batches store prices and quantities as scale-16 decimals plus schema metadata. This
19//! module preserves the raw catalog query path while converting the values to named `Float64`
20//! columns suitable for PyArrow/Polars. Migrate legacy catalog files before requesting display output.
21
22use std::{collections::HashMap, str::FromStr, sync::Arc};
23
24use arrow::{
25    array::{
26        Array, ArrayRef, BooleanArray, Decimal128Array, Float64Array, Float64Builder, Int8Array,
27        Int16Array, ListArray, StringArray, StringBuilder, StructArray, TimestampNanosecondArray,
28        TimestampNanosecondBuilder, UInt8Array, UInt8Builder, UInt32Array, UInt64Array,
29        UInt64Builder,
30    },
31    compute::cast,
32    datatypes::{DataType, Field, Schema},
33    record_batch::RecordBatch,
34};
35use nautilus_model::{
36    data::{
37        Bar, BarType, FundingRateUpdate, IndexPriceUpdate, InstrumentClose, InstrumentStatus,
38        MarkPriceUpdate, NautilusDataType, OptionGreeks, OrderBookDelta, OrderBookDepth, QuoteTick,
39        TradeTick, get_arrow_schema,
40    },
41    enums::{AggressorSide, BookAction, InstrumentCloseType, OrderSide},
42    instruments::InstrumentAny,
43    types::{Price, Quantity},
44};
45use rust_decimal::{Decimal, prelude::ToPrimitive};
46
47use super::{
48    ArrowSchemaProvider, EncodingError, KEY_BAR_TYPE, KEY_IDENTIFIER, KEY_INSTRUMENT_ID,
49    KEY_PRICE_PRECISION, KEY_SIZE_PRECISION, U64ColumnRef,
50    custom::CustomDataDecoder,
51    decode_decimal_price, decode_decimal_quantity,
52    depth_display::{DepthSideBuilder, schema as depth_schema},
53    display_conversion::{
54        float64_field, price_to_f64, quantity_to_f64, timestamp_field, utf8_field,
55    },
56    extract_column, extract_column_string, fixed_decimal_data_type,
57};
58use crate::arrow::timestamp_data_type;
59
60struct CatalogDisplayFns {
61    schema: fn() -> Schema,
62    convert: fn(&HashMap<String, String>, &RecordBatch) -> Result<RecordBatch, EncodingError>,
63}
64
65trait CatalogDisplay {
66    const FUNCTIONS: Option<CatalogDisplayFns>;
67}
68
69impl CatalogDisplay for InstrumentAny {
70    const FUNCTIONS: Option<CatalogDisplayFns> = None;
71}
72
73impl CatalogDisplay for QuoteTick {
74    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
75        schema: quotes_schema,
76        convert: convert_quotes,
77    });
78}
79
80impl CatalogDisplay for TradeTick {
81    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
82        schema: trades_schema,
83        convert: convert_trades,
84    });
85}
86
87impl CatalogDisplay for Bar {
88    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
89        schema: bars_schema,
90        convert: convert_bars,
91    });
92}
93
94impl CatalogDisplay for OrderBookDelta {
95    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
96        schema: deltas_schema,
97        convert: convert_deltas,
98    });
99}
100
101impl CatalogDisplay for OrderBookDepth {
102    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
103        schema: depth_schema,
104        convert: convert_depths,
105    });
106}
107
108impl CatalogDisplay for MarkPriceUpdate {
109    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
110        schema: price_update_schema,
111        convert: convert_price_updates,
112    });
113}
114
115impl CatalogDisplay for IndexPriceUpdate {
116    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
117        schema: price_update_schema,
118        convert: convert_price_updates,
119    });
120}
121
122impl CatalogDisplay for FundingRateUpdate {
123    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
124        schema: funding_rate_schema,
125        convert: convert_funding_rates,
126    });
127}
128
129impl CatalogDisplay for InstrumentStatus {
130    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
131        schema: instrument_status_schema,
132        convert: convert_instrument_status,
133    });
134}
135
136impl CatalogDisplay for OptionGreeks {
137    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
138        schema: option_greeks_schema,
139        convert: convert_option_greeks,
140    });
141}
142
143impl CatalogDisplay for InstrumentClose {
144    const FUNCTIONS: Option<CatalogDisplayFns> = Some(CatalogDisplayFns {
145        schema: instrument_closes_schema,
146        convert: convert_instrument_closes,
147    });
148}
149
150macro_rules! define_catalog_display_lookup {
151    ($(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?) => {
152        fn builtin_catalog_display(data_type: &NautilusDataType) -> Option<CatalogDisplayFns> {
153            match data_type {
154                $(
155                    NautilusDataType::$variant => <$type as CatalogDisplay>::FUNCTIONS,
156                )+
157                _ => None,
158            }
159        }
160    };
161}
162
163nautilus_model::for_each_data_type!(define_catalog_display_lookup);
164
165macro_rules! define_catalog_raw_schema_lookup {
166    (
167        ($_instrument_variant:ident, $_instrument_type:ident, $_instrument_data:ident, $_instrument_batch:ident, $_instrument_prefix:literal),
168        $(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?
169    ) => {
170        fn builtin_catalog_raw_schema(data_type: &NautilusDataType) -> Option<Schema> {
171            match data_type {
172                $(
173                    NautilusDataType::$variant => {
174                        Some(<$type as ArrowSchemaProvider>::get_schema(None))
175                    }
176                )+
177                _ => None,
178            }
179        }
180    };
181}
182
183nautilus_model::for_each_data_type!(define_catalog_raw_schema_lookup);
184
185/// Returns the raw Arrow schema for a catalog data type.
186///
187/// # Errors
188///
189/// Returns an error if the data type is unsupported.
190pub fn catalog_raw_schema(data_type: &NautilusDataType) -> Result<Schema, EncodingError> {
191    if let NautilusDataType::Custom { type_name } = data_type {
192        if get_arrow_schema(type_name).is_none() {
193            return Err(unsupported_display_type(data_type));
194        }
195        let metadata = HashMap::from([("type_name".to_string(), type_name.clone())]);
196        return Ok(super::schema_with_identifier_column(
197            &CustomDataDecoder::get_schema(Some(metadata)),
198        ));
199    }
200
201    builtin_catalog_raw_schema(data_type).ok_or_else(|| unsupported_display_type(data_type))
202}
203
204/// Returns a display-friendly Arrow schema for a catalog data type.
205///
206/// # Errors
207///
208/// Returns an error if the data type is unsupported.
209#[allow(
210    clippy::match_wildcard_for_single_variants,
211    unreachable_patterns,
212    reason = "NautilusDataType::Defi is controlled by nautilus-model features, not this crate"
213)]
214pub fn catalog_display_schema(data_type: &NautilusDataType) -> Result<Schema, EncodingError> {
215    if matches!(data_type, NautilusDataType::Custom { .. }) {
216        let schema = catalog_raw_schema(data_type)?;
217        let batch = RecordBatch::new_empty(Arc::new(schema));
218        return catalog_record_batch_to_display(data_type, batch.schema().metadata(), &batch)
219            .map(|batch| batch.schema().as_ref().clone());
220    }
221
222    builtin_catalog_display(data_type)
223        .map(|functions| (functions.schema)())
224        .ok_or_else(|| unsupported_display_type(data_type))
225}
226
227/// Converts a raw catalog record batch into display Arrow without decoding full data objects.
228///
229/// # Errors
230///
231/// Returns an error if required metadata or columns are missing, a columns do not use the current
232/// Arrow format, or enum values are invalid.
233#[allow(
234    clippy::match_wildcard_for_single_variants,
235    unreachable_patterns,
236    reason = "NautilusDataType::Defi is controlled by nautilus-model features, not this crate"
237)]
238pub fn catalog_record_batch_to_display(
239    data_type: &NautilusDataType,
240    metadata: &HashMap<String, String>,
241    record_batch: &RecordBatch,
242) -> Result<RecordBatch, EncodingError> {
243    let display_batch = if matches!(data_type, NautilusDataType::Custom { .. }) {
244        convert_custom(record_batch)
245    } else if let Some(functions) = builtin_catalog_display(data_type) {
246        (functions.convert)(metadata, record_batch)
247    } else {
248        Err(unsupported_display_type(data_type))
249    }?;
250
251    append_identifier_column_if_present(display_batch, record_batch)
252}
253
254fn unsupported_display_type(data_type: &NautilusDataType) -> EncodingError {
255    EncodingError::ParseError(
256        "data_type",
257        format!("unsupported display catalog data type `{data_type}`"),
258    )
259}
260
261fn append_identifier_column_if_present(
262    display_batch: RecordBatch,
263    catalog_batch: &RecordBatch,
264) -> Result<RecordBatch, EncodingError> {
265    if display_batch.schema().index_of(KEY_IDENTIFIER).is_ok() {
266        return Ok(display_batch);
267    }
268
269    let Ok(identifier_index) = catalog_batch.schema().index_of(KEY_IDENTIFIER) else {
270        return Ok(display_batch);
271    };
272
273    let catalog_schema = catalog_batch.schema();
274    let identifier_field = catalog_schema.field(identifier_index);
275    let (identifier_field, identifier_column) =
276        if identifier_field.data_type() == &DataType::Utf8View {
277            (
278                Field::new(
279                    identifier_field.name(),
280                    DataType::Utf8,
281                    identifier_field.is_nullable(),
282                ),
283                cast(
284                    catalog_batch.column(identifier_index).as_ref(),
285                    &DataType::Utf8,
286                )?,
287            )
288        } else {
289            (
290                identifier_field.clone(),
291                Arc::clone(catalog_batch.column(identifier_index)),
292            )
293        };
294
295    let mut fields = display_batch.schema().fields().to_vec();
296    fields.push(Arc::new(identifier_field));
297    let mut columns = display_batch.columns().to_vec();
298    columns.push(identifier_column);
299
300    RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).map_err(EncodingError::from)
301}
302
303fn parse_price_precision(metadata: &HashMap<String, String>) -> Result<u8, EncodingError> {
304    parse_precision(metadata, KEY_PRICE_PRECISION)
305}
306
307fn parse_size_precision(metadata: &HashMap<String, String>) -> Result<u8, EncodingError> {
308    parse_precision(metadata, KEY_SIZE_PRECISION)
309}
310
311fn parse_precision(
312    metadata: &HashMap<String, String>,
313    key: &'static str,
314) -> Result<u8, EncodingError> {
315    metadata
316        .get(key)
317        .ok_or(EncodingError::MissingMetadata(key))?
318        .parse::<u8>()
319        .map_err(|e| EncodingError::ParseError(key, e.to_string()))
320}
321
322fn instrument_id(metadata: &HashMap<String, String>) -> Result<&str, EncodingError> {
323    metadata
324        .get(KEY_INSTRUMENT_ID)
325        .map(String::as_str)
326        .ok_or_else(|| EncodingError::MissingMetadata(KEY_INSTRUMENT_ID))
327}
328
329fn bar_type(metadata: &HashMap<String, String>) -> Result<BarType, EncodingError> {
330    let value = metadata
331        .get(KEY_BAR_TYPE)
332        .ok_or_else(|| EncodingError::MissingMetadata(KEY_BAR_TYPE))?;
333    BarType::from_str(value).map_err(|e| EncodingError::ParseError(KEY_BAR_TYPE, e.to_string()))
334}
335
336fn constant_string_column(value: &str, len: usize) -> ArrayRef {
337    Arc::new(StringArray::from_iter_values(std::iter::repeat_n(
338        value, len,
339    )))
340}
341
342struct FixedPrecisionColumn<'a>(&'a Decimal128Array);
343
344impl FixedPrecisionColumn<'_> {
345    fn len(&self) -> usize {
346        self.0.len()
347    }
348    fn is_null(&self, row: usize) -> bool {
349        self.0.is_null(row)
350    }
351    fn price(
352        &self,
353        precision: u8,
354        field: &'static str,
355        row: usize,
356    ) -> Result<Price, EncodingError> {
357        decode_decimal_price(self.0, precision, field, row)
358    }
359    fn quantity(
360        &self,
361        precision: u8,
362        field: &'static str,
363        row: usize,
364    ) -> Result<Quantity, EncodingError> {
365        decode_decimal_quantity(self.0, precision, field, row)
366    }
367}
368
369struct NanosColumn<'a>(&'a TimestampNanosecondArray);
370
371impl NanosColumn<'_> {
372    fn len(&self) -> usize {
373        self.0.len()
374    }
375    fn is_null(&self, row: usize) -> bool {
376        self.0.is_null(row)
377    }
378    fn value(&self, row: usize) -> i64 {
379        self.0.value(row)
380    }
381}
382
383enum U8Column<'a> {
384    UInt8(&'a UInt8Array),
385    Int8(&'a Int8Array),
386    Int16(&'a Int16Array),
387}
388
389impl U8Column<'_> {
390    fn value(&self, row: usize, field: &'static str) -> Result<u8, EncodingError> {
391        match self {
392            Self::UInt8(values) => Ok(values.value(row)),
393            Self::Int8(values) => u8::try_from(values.value(row)).map_err(|_| {
394                EncodingError::ParseError(field, format!("Invalid negative value at row {row}"))
395            }),
396            Self::Int16(values) => u8::try_from(values.value(row)).map_err(|_| {
397                EncodingError::ParseError(field, format!("Value out of u8 range at row {row}"))
398            }),
399        }
400    }
401}
402
403fn fixed_col<'a>(
404    batch: &'a RecordBatch,
405    name: &'static str,
406) -> Result<FixedPrecisionColumn<'a>, EncodingError> {
407    Ok(FixedPrecisionColumn(super::extract_decimal_column(
408        batch, name,
409    )?))
410}
411
412fn u8_col<'a>(batch: &'a RecordBatch, name: &'static str) -> Result<U8Column<'a>, EncodingError> {
413    let index = batch.schema().index_of(name)?;
414    let column = batch.column(index);
415    match column.data_type() {
416        DataType::UInt8 => Ok(U8Column::UInt8(extract_column::<UInt8Array>(
417            batch.columns(),
418            name,
419            index,
420            DataType::UInt8,
421        )?)),
422        DataType::Int8 => Ok(U8Column::Int8(extract_column::<Int8Array>(
423            batch.columns(),
424            name,
425            index,
426            DataType::Int8,
427        )?)),
428        DataType::Int16 => Ok(U8Column::Int16(extract_column::<Int16Array>(
429            batch.columns(),
430            name,
431            index,
432            DataType::Int16,
433        )?)),
434        data_type => Err(EncodingError::InvalidColumnType(
435            name,
436            index,
437            DataType::UInt8,
438            data_type.clone(),
439        )),
440    }
441}
442
443fn u64_value(
444    values: &U64ColumnRef<'_>,
445    row: usize,
446    field: &'static str,
447) -> Result<u64, EncodingError> {
448    values.value(row).ok_or_else(|| {
449        EncodingError::ParseError(field, format!("Invalid negative value at row {row}"))
450    })
451}
452
453fn u64_col<'a>(
454    batch: &'a RecordBatch,
455    name: &'static str,
456) -> Result<U64ColumnRef<'a>, EncodingError> {
457    let index = batch.schema().index_of(name)?;
458    let column = batch.column(index);
459    U64ColumnRef::try_from_array(column.as_ref()).ok_or_else(|| {
460        EncodingError::InvalidColumnType(name, index, DataType::UInt64, column.data_type().clone())
461    })
462}
463
464fn nanos_col<'a>(
465    batch: &'a RecordBatch,
466    name: &'static str,
467) -> Result<NanosColumn<'a>, EncodingError> {
468    let index = batch.schema().index_of(name)?;
469    let column = batch.column(index);
470    let expected = timestamp_data_type();
471    if column.data_type() != &expected {
472        return Err(EncodingError::InvalidColumnType(
473            name,
474            index,
475            expected,
476            column.data_type().clone(),
477        ));
478    }
479    Ok(NanosColumn(extract_column::<TimestampNanosecondArray>(
480        batch.columns(),
481        name,
482        index,
483        expected,
484    )?))
485}
486
487fn f64_col<'a>(
488    batch: &'a RecordBatch,
489    name: &'static str,
490) -> Result<&'a Float64Array, EncodingError> {
491    let index = batch.schema().index_of(name)?;
492    extract_column::<Float64Array>(batch.columns(), name, index, DataType::Float64)
493}
494
495fn bool_col<'a>(
496    batch: &'a RecordBatch,
497    name: &'static str,
498) -> Result<&'a BooleanArray, EncodingError> {
499    let index = batch.schema().index_of(name)?;
500    extract_column::<BooleanArray>(batch.columns(), name, index, DataType::Boolean)
501}
502
503fn timestamp_array_from_nanos(values: &NanosColumn<'_>) -> ArrayRef {
504    let mut builder = TimestampNanosecondBuilder::with_capacity(values.len())
505        .with_data_type(timestamp_data_type());
506
507    for row in 0..values.len() {
508        append_timestamp(&mut builder, values, row);
509    }
510    Arc::new(builder.finish())
511}
512
513fn append_timestamp(
514    builder: &mut TimestampNanosecondBuilder,
515    values: &NanosColumn<'_>,
516    row: usize,
517) {
518    if values.is_null(row) {
519        builder.append_null();
520    } else {
521        builder.append_value(values.value(row));
522    }
523}
524
525fn timestamp_field_from(field: &Field) -> Field {
526    Field::new(field.name(), timestamp_data_type(), field.is_nullable())
527}
528
529fn fixed_precision_field_to_f64(
530    values: &FixedPrecisionColumn<'_>,
531    precision: u8,
532    is_price: bool,
533) -> Result<ArrayRef, EncodingError> {
534    let mut builder = Float64Builder::with_capacity(values.len());
535    for row in 0..values.len() {
536        if values.is_null(row) {
537            builder.append_null();
538        } else if is_price {
539            append_price(&mut builder, values, precision, "custom", row)?;
540        } else {
541            append_quantity(&mut builder, values, precision, "custom", row)?;
542        }
543    }
544
545    Ok(Arc::new(builder.finish()))
546}
547
548fn custom_fixed_precision(
549    field_name: &str,
550    metadata: &HashMap<String, String>,
551) -> Option<(u8, bool)> {
552    let normalized = field_name.to_ascii_lowercase();
553    let field_precision = metadata
554        .get(&format!("{field_name}_precision"))
555        .and_then(|value| value.parse::<u8>().ok());
556
557    match metadata
558        .get(&format!("{field_name}_kind"))
559        .map(String::as_str)
560    {
561        Some("price") => return field_precision.map(|precision| (precision, true)),
562        Some("quantity") => return field_precision.map(|precision| (precision, false)),
563        _ => {}
564    }
565    let is_quantity = matches!(
566        normalized.as_str(),
567        "size" | "quantity" | "qty" | "volume" | "bid_size" | "ask_size"
568    ) || normalized.ends_with("_size")
569        || normalized.ends_with("_quantity")
570        || normalized.ends_with("_qty")
571        || normalized.ends_with("_volume");
572
573    if is_quantity {
574        return field_precision
575            .or_else(|| {
576                metadata
577                    .get(KEY_SIZE_PRECISION)
578                    .and_then(|value| value.parse::<u8>().ok())
579            })
580            .map(|precision| (precision, false));
581    }
582
583    if let Some(precision) = field_precision {
584        return Some((precision, true));
585    }
586
587    let is_price = matches!(
588        normalized.as_str(),
589        "price"
590            | "value"
591            | "bid"
592            | "ask"
593            | "bid_price"
594            | "ask_price"
595            | "open"
596            | "high"
597            | "low"
598            | "close"
599    ) || normalized.ends_with("_price")
600        || normalized.ends_with("_value")
601        || normalized.ends_with("_bid")
602        || normalized.ends_with("_ask")
603        || normalized.ends_with("_open")
604        || normalized.ends_with("_high")
605        || normalized.ends_with("_low")
606        || normalized.ends_with("_close");
607
608    if is_price {
609        return metadata
610            .get(KEY_PRICE_PRECISION)
611            .and_then(|value| value.parse::<u8>().ok())
612            .map(|precision| (precision, true));
613    }
614
615    None
616}
617
618fn append_price(
619    builder: &mut Float64Builder,
620    values: &FixedPrecisionColumn<'_>,
621    precision: u8,
622    field: &'static str,
623    row: usize,
624) -> Result<(), EncodingError> {
625    let price = values.price(precision, field, row)?;
626    builder.append_value(price_to_f64(&price));
627    Ok(())
628}
629
630fn append_price_with_sentinel(
631    builder: &mut Float64Builder,
632    values: &FixedPrecisionColumn<'_>,
633    precision: u8,
634    field: &'static str,
635    row: usize,
636) -> Result<(), EncodingError> {
637    let price = values.price(precision, field, row)?;
638    builder.append_value(price_to_f64(&price));
639    Ok(())
640}
641
642fn append_optional_price_with_sentinel(
643    builder: &mut Float64Builder,
644    values: &FixedPrecisionColumn<'_>,
645    precision: u8,
646    field: &'static str,
647    row: usize,
648) -> Result<(), EncodingError> {
649    let price = values.price(precision, field, row)?;
650
651    if price.is_undefined() {
652        builder.append_null();
653    } else {
654        builder.append_value(price_to_f64(&price));
655    }
656    Ok(())
657}
658
659fn append_quantity(
660    builder: &mut Float64Builder,
661    values: &FixedPrecisionColumn<'_>,
662    precision: u8,
663    field: &'static str,
664    row: usize,
665) -> Result<(), EncodingError> {
666    let quantity = values.quantity(precision, field, row)?;
667    builder.append_value(quantity_to_f64(&quantity));
668    Ok(())
669}
670
671fn append_quantity_with_sentinel(
672    builder: &mut Float64Builder,
673    values: &FixedPrecisionColumn<'_>,
674    precision: u8,
675    field: &'static str,
676    row: usize,
677) -> Result<(), EncodingError> {
678    let quantity = values.quantity(precision, field, row)?;
679    builder.append_value(quantity_to_f64(&quantity));
680    Ok(())
681}
682
683fn append_optional_quantity_with_sentinel(
684    builder: &mut Float64Builder,
685    values: &FixedPrecisionColumn<'_>,
686    precision: u8,
687    field: &'static str,
688    row: usize,
689) -> Result<(), EncodingError> {
690    let quantity = values.quantity(precision, field, row)?;
691
692    if quantity.is_undefined() {
693        builder.append_null();
694    } else {
695        builder.append_value(quantity_to_f64(&quantity));
696    }
697    Ok(())
698}
699
700fn bars_schema() -> Schema {
701    Schema::new(vec![
702        utf8_field("instrument_id", false),
703        utf8_field("bar_type", false),
704        float64_field("open", false),
705        float64_field("high", false),
706        float64_field("low", false),
707        float64_field("close", false),
708        float64_field("volume", false),
709        timestamp_field("ts_event", false),
710        timestamp_field("ts_init", false),
711    ])
712}
713
714fn convert_bars(
715    metadata: &HashMap<String, String>,
716    batch: &RecordBatch,
717) -> Result<RecordBatch, EncodingError> {
718    let bar_type = bar_type(metadata)?;
719    let bar_type_string = bar_type.to_string();
720    let instrument_id_string = bar_type.instrument_id().to_string();
721    let price_precision = parse_price_precision(metadata)?;
722    let size_precision = parse_size_precision(metadata)?;
723    let open = fixed_col(batch, "open")?;
724    let high = fixed_col(batch, "high")?;
725    let low = fixed_col(batch, "low")?;
726    let close = fixed_col(batch, "close")?;
727    let volume = fixed_col(batch, "volume")?;
728    let ts_event = nanos_col(batch, "ts_event")?;
729    let ts_init = nanos_col(batch, "ts_init")?;
730
731    let len = batch.num_rows();
732    let mut open_builder = Float64Builder::with_capacity(len);
733    let mut high_builder = Float64Builder::with_capacity(len);
734    let mut low_builder = Float64Builder::with_capacity(len);
735    let mut close_builder = Float64Builder::with_capacity(len);
736    let mut volume_builder = Float64Builder::with_capacity(len);
737    let mut ts_event_builder =
738        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
739    let mut ts_init_builder =
740        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
741
742    for row in 0..len {
743        append_price(&mut open_builder, &open, price_precision, "open", row)?;
744        append_price(&mut high_builder, &high, price_precision, "high", row)?;
745        append_price(&mut low_builder, &low, price_precision, "low", row)?;
746        append_price(&mut close_builder, &close, price_precision, "close", row)?;
747        append_quantity(&mut volume_builder, &volume, size_precision, "volume", row)?;
748        append_timestamp(&mut ts_event_builder, &ts_event, row);
749        append_timestamp(&mut ts_init_builder, &ts_init, row);
750    }
751
752    RecordBatch::try_new(
753        Arc::new(bars_schema()),
754        vec![
755            constant_string_column(&instrument_id_string, len),
756            constant_string_column(&bar_type_string, len),
757            Arc::new(open_builder.finish()),
758            Arc::new(high_builder.finish()),
759            Arc::new(low_builder.finish()),
760            Arc::new(close_builder.finish()),
761            Arc::new(volume_builder.finish()),
762            Arc::new(ts_event_builder.finish()),
763            Arc::new(ts_init_builder.finish()),
764        ],
765    )
766    .map_err(EncodingError::from)
767}
768
769fn quotes_schema() -> Schema {
770    Schema::new(vec![
771        utf8_field("instrument_id", false),
772        float64_field("bid_price", false),
773        float64_field("ask_price", false),
774        float64_field("bid_size", false),
775        float64_field("ask_size", false),
776        timestamp_field("ts_event", false),
777        timestamp_field("ts_init", false),
778    ])
779}
780
781fn convert_quotes(
782    metadata: &HashMap<String, String>,
783    batch: &RecordBatch,
784) -> Result<RecordBatch, EncodingError> {
785    let instrument_id = instrument_id(metadata)?;
786    let price_precision = parse_price_precision(metadata)?;
787    let size_precision = parse_size_precision(metadata)?;
788    let bid_price = fixed_col(batch, "bid_price")?;
789    let ask_price = fixed_col(batch, "ask_price")?;
790    let bid_size = fixed_col(batch, "bid_size")?;
791    let ask_size = fixed_col(batch, "ask_size")?;
792    let ts_event = nanos_col(batch, "ts_event")?;
793    let ts_init = nanos_col(batch, "ts_init")?;
794
795    let len = batch.num_rows();
796    let mut bid_price_builder = Float64Builder::with_capacity(len);
797    let mut ask_price_builder = Float64Builder::with_capacity(len);
798    let mut bid_size_builder = Float64Builder::with_capacity(len);
799    let mut ask_size_builder = Float64Builder::with_capacity(len);
800    let mut ts_event_builder =
801        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
802    let mut ts_init_builder =
803        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
804
805    for row in 0..len {
806        append_price(
807            &mut bid_price_builder,
808            &bid_price,
809            price_precision,
810            "bid_price",
811            row,
812        )?;
813        append_price(
814            &mut ask_price_builder,
815            &ask_price,
816            price_precision,
817            "ask_price",
818            row,
819        )?;
820        append_quantity(
821            &mut bid_size_builder,
822            &bid_size,
823            size_precision,
824            "bid_size",
825            row,
826        )?;
827        append_quantity(
828            &mut ask_size_builder,
829            &ask_size,
830            size_precision,
831            "ask_size",
832            row,
833        )?;
834        append_timestamp(&mut ts_event_builder, &ts_event, row);
835        append_timestamp(&mut ts_init_builder, &ts_init, row);
836    }
837
838    RecordBatch::try_new(
839        Arc::new(quotes_schema()),
840        vec![
841            constant_string_column(instrument_id, len),
842            Arc::new(bid_price_builder.finish()),
843            Arc::new(ask_price_builder.finish()),
844            Arc::new(bid_size_builder.finish()),
845            Arc::new(ask_size_builder.finish()),
846            Arc::new(ts_event_builder.finish()),
847            Arc::new(ts_init_builder.finish()),
848        ],
849    )
850    .map_err(EncodingError::from)
851}
852
853fn trades_schema() -> Schema {
854    Schema::new(vec![
855        utf8_field("instrument_id", false),
856        float64_field("price", false),
857        float64_field("size", false),
858        utf8_field("aggressor_side", false),
859        utf8_field("trade_id", false),
860        timestamp_field("ts_event", false),
861        timestamp_field("ts_init", false),
862    ])
863}
864
865fn convert_trades(
866    metadata: &HashMap<String, String>,
867    batch: &RecordBatch,
868) -> Result<RecordBatch, EncodingError> {
869    let instrument_id = instrument_id(metadata)?;
870    let price_precision = parse_price_precision(metadata)?;
871    let size_precision = parse_size_precision(metadata)?;
872    let price = fixed_col(batch, "price")?;
873    let size = fixed_col(batch, "size")?;
874    let aggressor_side_index = batch.schema().index_of("aggressor_side")?;
875    let aggressor_side =
876        extract_column_string(batch.columns(), "aggressor_side", aggressor_side_index)?;
877    let trade_id_index = batch.schema().index_of("trade_id")?;
878    let trade_id = extract_column_string(batch.columns(), "trade_id", trade_id_index)?;
879    let ts_event = nanos_col(batch, "ts_event")?;
880    let ts_init = nanos_col(batch, "ts_init")?;
881
882    let len = batch.num_rows();
883    let mut price_builder = Float64Builder::with_capacity(len);
884    let mut size_builder = Float64Builder::with_capacity(len);
885    let mut aggressor_side_builder = StringBuilder::new();
886    let mut trade_id_builder = StringBuilder::new();
887    let mut ts_event_builder =
888        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
889    let mut ts_init_builder =
890        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
891
892    for row in 0..len {
893        let aggressor_side_value = aggressor_side.value(row);
894        let side = AggressorSide::from_str(aggressor_side_value)
895            .map_err(|e| EncodingError::ParseError(stringify!(AggressorSide), e.to_string()))?;
896        append_price(&mut price_builder, &price, price_precision, "price", row)?;
897        append_quantity(&mut size_builder, &size, size_precision, "size", row)?;
898        aggressor_side_builder.append_value(side.as_ref());
899        trade_id_builder.append_value(trade_id.value(row));
900        append_timestamp(&mut ts_event_builder, &ts_event, row);
901        append_timestamp(&mut ts_init_builder, &ts_init, row);
902    }
903
904    RecordBatch::try_new(
905        Arc::new(trades_schema()),
906        vec![
907            constant_string_column(instrument_id, len),
908            Arc::new(price_builder.finish()),
909            Arc::new(size_builder.finish()),
910            Arc::new(aggressor_side_builder.finish()),
911            Arc::new(trade_id_builder.finish()),
912            Arc::new(ts_event_builder.finish()),
913            Arc::new(ts_init_builder.finish()),
914        ],
915    )
916    .map_err(EncodingError::from)
917}
918
919fn deltas_schema() -> Schema {
920    Schema::new(vec![
921        utf8_field("instrument_id", false),
922        utf8_field("action", false),
923        utf8_field("side", false),
924        float64_field("price", false),
925        float64_field("size", false),
926        utf8_field("order_id", false),
927        Field::new("flags", DataType::UInt8, false),
928        Field::new("sequence", DataType::UInt64, false),
929        timestamp_field("ts_event", false),
930        timestamp_field("ts_init", false),
931    ])
932}
933
934fn convert_deltas(
935    metadata: &HashMap<String, String>,
936    batch: &RecordBatch,
937) -> Result<RecordBatch, EncodingError> {
938    const NO_ORDER_SIDE: &str = "NO_ORDER_SIDE";
939
940    let instrument_id = instrument_id(metadata)?;
941    let price_precision = parse_price_precision(metadata)?;
942    let size_precision = parse_size_precision(metadata)?;
943    let action_index = batch.schema().index_of("action")?;
944    let action = extract_column_string(batch.columns(), "action", action_index)?;
945    let side_index = batch.schema().index_of("side")?;
946    let side = extract_column_string(batch.columns(), "side", side_index)?;
947    let price = fixed_col(batch, "price")?;
948    let size = fixed_col(batch, "size")?;
949    let order_id = u64_col(batch, "order_id")?;
950    let flags = u8_col(batch, "flags")?;
951    let sequence = u64_col(batch, "sequence")?;
952    let ts_event = nanos_col(batch, "ts_event")?;
953    let ts_init = nanos_col(batch, "ts_init")?;
954
955    let len = batch.num_rows();
956    let mut action_builder = StringBuilder::new();
957    let mut side_builder = StringBuilder::new();
958    let mut price_builder = Float64Builder::with_capacity(len);
959    let mut size_builder = Float64Builder::with_capacity(len);
960    let mut order_id_builder = StringBuilder::new();
961    let mut flags_builder = UInt8Builder::with_capacity(len);
962    let mut sequence_builder = UInt64Builder::with_capacity(len);
963    let mut ts_event_builder =
964        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
965    let mut ts_init_builder =
966        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
967
968    for row in 0..len {
969        let action_value = action.value(row);
970        let action = BookAction::from_str(action_value)
971            .map_err(|e| EncodingError::ParseError(stringify!(BookAction), e.to_string()))?;
972        let side_value = side.value(row);
973
974        let side = if side_value.eq_ignore_ascii_case(NO_ORDER_SIDE) {
975            None
976        } else {
977            Some(
978                OrderSide::from_str(side_value)
979                    .map_err(|e| EncodingError::ParseError(stringify!(OrderSide), e.to_string()))?,
980            )
981        };
982
983        action_builder.append_value(action.as_ref());
984        side_builder.append_value(side.as_ref().map_or(NO_ORDER_SIDE, AsRef::as_ref));
985
986        if action == BookAction::Clear {
987            price_builder.append_value(f64::NAN);
988            size_builder.append_value(f64::NAN);
989        } else {
990            append_price_with_sentinel(&mut price_builder, &price, price_precision, "price", row)?;
991            append_quantity_with_sentinel(&mut size_builder, &size, size_precision, "size", row)?;
992        }
993        order_id_builder.append_value(u64_value(&order_id, row, "order_id")?.to_string());
994        flags_builder.append_value(flags.value(row, "flags")?);
995        sequence_builder.append_value(u64_value(&sequence, row, "sequence")?);
996        append_timestamp(&mut ts_event_builder, &ts_event, row);
997        append_timestamp(&mut ts_init_builder, &ts_init, row);
998    }
999
1000    RecordBatch::try_new(
1001        Arc::new(deltas_schema()),
1002        vec![
1003            constant_string_column(instrument_id, len),
1004            Arc::new(action_builder.finish()),
1005            Arc::new(side_builder.finish()),
1006            Arc::new(price_builder.finish()),
1007            Arc::new(size_builder.finish()),
1008            Arc::new(order_id_builder.finish()),
1009            Arc::new(flags_builder.finish()),
1010            Arc::new(sequence_builder.finish()),
1011            Arc::new(ts_event_builder.finish()),
1012            Arc::new(ts_init_builder.finish()),
1013        ],
1014    )
1015    .map_err(EncodingError::from)
1016}
1017
1018fn convert_depths(
1019    metadata: &HashMap<String, String>,
1020    batch: &RecordBatch,
1021) -> Result<RecordBatch, EncodingError> {
1022    let instrument_id = instrument_id(metadata)?;
1023    let price_precision = parse_price_precision(metadata)?;
1024    let size_precision = parse_size_precision(metadata)?;
1025    let bids = depth_struct_values(batch, "bids")?;
1026    let asks = depth_struct_values(batch, "asks")?;
1027    let flags = u8_col(batch, "flags")?;
1028    let sequence = u64_col(batch, "sequence")?;
1029    let ts_event = nanos_col(batch, "ts_event")?;
1030    let ts_init = nanos_col(batch, "ts_init")?;
1031    let len = batch.num_rows();
1032    let mut bids_display = DepthSideBuilder::new();
1033    let mut asks_display = DepthSideBuilder::new();
1034    let mut flags_builder = UInt8Builder::with_capacity(len);
1035    let mut sequence_builder = UInt64Builder::with_capacity(len);
1036    let mut ts_event_builder =
1037        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1038    let mut ts_init_builder =
1039        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1040    let bid_prices = FixedPrecisionColumn(&bids.prices);
1041    let ask_prices = FixedPrecisionColumn(&asks.prices);
1042    let bid_sizes = FixedPrecisionColumn(&bids.sizes);
1043    let ask_sizes = FixedPrecisionColumn(&asks.sizes);
1044
1045    for row in 0..len {
1046        let (bid_start, bid_end) = depth_struct_range(&bids.list, "bids", row)?;
1047        let (ask_start, ask_end) = depth_struct_range(&asks.list, "asks", row)?;
1048
1049        for value_index in bid_start..bid_end {
1050            append_optional_price_with_sentinel(
1051                &mut bids_display.prices,
1052                &bid_prices,
1053                price_precision,
1054                "bids.price",
1055                value_index,
1056            )?;
1057            append_optional_quantity_with_sentinel(
1058                &mut bids_display.sizes,
1059                &bid_sizes,
1060                size_precision,
1061                "bids.size",
1062                value_index,
1063            )?;
1064
1065            if bids.counts.is_null(value_index) || bids.order_ids.is_null(value_index) {
1066                return Err(EncodingError::ParseError(
1067                    "bids",
1068                    "count and order_id must not be null".to_string(),
1069                ));
1070            }
1071            bids_display
1072                .counts
1073                .append_value(bids.counts.value(value_index));
1074            bids_display
1075                .order_ids
1076                .append_value(bids.order_ids.value(value_index));
1077        }
1078
1079        for value_index in ask_start..ask_end {
1080            append_optional_price_with_sentinel(
1081                &mut asks_display.prices,
1082                &ask_prices,
1083                price_precision,
1084                "asks.price",
1085                value_index,
1086            )?;
1087            append_optional_quantity_with_sentinel(
1088                &mut asks_display.sizes,
1089                &ask_sizes,
1090                size_precision,
1091                "asks.size",
1092                value_index,
1093            )?;
1094
1095            if asks.counts.is_null(value_index) || asks.order_ids.is_null(value_index) {
1096                return Err(EncodingError::ParseError(
1097                    "asks",
1098                    "count and order_id must not be null".to_string(),
1099                ));
1100            }
1101            asks_display
1102                .counts
1103                .append_value(asks.counts.value(value_index));
1104            asks_display
1105                .order_ids
1106                .append_value(asks.order_ids.value(value_index));
1107        }
1108        bids_display.finish_row()?;
1109        asks_display.finish_row()?;
1110        flags_builder.append_value(flags.value(row, "flags")?);
1111        sequence_builder.append_value(u64_value(&sequence, row, "sequence")?);
1112        append_timestamp(&mut ts_event_builder, &ts_event, row);
1113        append_timestamp(&mut ts_init_builder, &ts_init, row);
1114    }
1115
1116    finish_depth_display(
1117        instrument_id,
1118        len,
1119        bids_display,
1120        asks_display,
1121        flags_builder,
1122        sequence_builder,
1123        ts_event_builder,
1124        ts_init_builder,
1125    )
1126}
1127
1128#[expect(
1129    clippy::too_many_arguments,
1130    reason = "the two nested sides and four scalar builders form one batch"
1131)]
1132fn finish_depth_display(
1133    instrument_id: &str,
1134    len: usize,
1135    bids: DepthSideBuilder,
1136    asks: DepthSideBuilder,
1137    mut flags: UInt8Builder,
1138    mut sequence: UInt64Builder,
1139    mut ts_event: TimestampNanosecondBuilder,
1140    mut ts_init: TimestampNanosecondBuilder,
1141) -> Result<RecordBatch, EncodingError> {
1142    RecordBatch::try_new(
1143        Arc::new(depth_schema()),
1144        vec![
1145            constant_string_column(instrument_id, len),
1146            Arc::new(bids.finish()?),
1147            Arc::new(asks.finish()?),
1148            Arc::new(flags.finish()),
1149            Arc::new(sequence.finish()),
1150            Arc::new(ts_event.finish()),
1151            Arc::new(ts_init.finish()),
1152        ],
1153    )
1154    .map_err(EncodingError::from)
1155}
1156
1157struct DepthStructColumns {
1158    list: ListArray,
1159    prices: Decimal128Array,
1160    sizes: Decimal128Array,
1161    counts: UInt32Array,
1162    order_ids: UInt64Array,
1163}
1164
1165fn depth_struct_values(
1166    batch: &RecordBatch,
1167    name: &'static str,
1168) -> Result<DepthStructColumns, EncodingError> {
1169    let list = batch
1170        .column_by_name(name)
1171        .and_then(|column| column.as_any().downcast_ref::<ListArray>())
1172        .cloned()
1173        .ok_or_else(|| EncodingError::ParseError(name, "expected List<Struct>".to_string()))?;
1174    let levels = list
1175        .values()
1176        .as_any()
1177        .downcast_ref::<StructArray>()
1178        .ok_or_else(|| EncodingError::ParseError(name, "expected List<Struct>".to_string()))?;
1179    let child = |field: &'static str| {
1180        levels.column_by_name(field).cloned().ok_or_else(|| {
1181            EncodingError::ParseError(name, format!("missing struct field '{field}'"))
1182        })
1183    };
1184
1185    for field in ["price", "size"] {
1186        if child(field)?.data_type() != &fixed_decimal_data_type() {
1187            return Err(EncodingError::ParseError(
1188                name,
1189                format!("field '{field}' must use Decimal128(38, 16)"),
1190            ));
1191        }
1192    }
1193    let prices = child("price")?
1194        .as_any()
1195        .downcast_ref::<Decimal128Array>()
1196        .cloned()
1197        .ok_or_else(|| {
1198            EncodingError::ParseError(name, "field 'price' must be Decimal128".to_string())
1199        })?;
1200    let sizes = child("size")?
1201        .as_any()
1202        .downcast_ref::<Decimal128Array>()
1203        .cloned()
1204        .ok_or_else(|| {
1205            EncodingError::ParseError(name, "field 'size' must be Decimal128".to_string())
1206        })?;
1207    let counts = child("count")?
1208        .as_any()
1209        .downcast_ref::<UInt32Array>()
1210        .cloned()
1211        .ok_or_else(|| {
1212            EncodingError::ParseError(name, "field 'count' must be UInt32".to_string())
1213        })?;
1214    let order_ids = child("order_id")?
1215        .as_any()
1216        .downcast_ref::<UInt64Array>()
1217        .cloned()
1218        .ok_or_else(|| {
1219            EncodingError::ParseError(name, "field 'order_id' must be UInt64".to_string())
1220        })?;
1221    Ok(DepthStructColumns {
1222        list,
1223        prices,
1224        sizes,
1225        counts,
1226        order_ids,
1227    })
1228}
1229
1230fn depth_struct_range(
1231    list: &ListArray,
1232    name: &'static str,
1233    row: usize,
1234) -> Result<(usize, usize), EncodingError> {
1235    if list.is_null(row) {
1236        return Err(EncodingError::ParseError(
1237            name,
1238            format!("side is null at row {row}"),
1239        ));
1240    }
1241    let offsets = list.value_offsets();
1242    let start = usize::try_from(offsets[row])
1243        .map_err(|e| EncodingError::ParseError(name, e.to_string()))?;
1244    let end = usize::try_from(offsets[row + 1])
1245        .map_err(|e| EncodingError::ParseError(name, e.to_string()))?;
1246    Ok((start, end))
1247}
1248
1249fn price_update_schema() -> Schema {
1250    Schema::new(vec![
1251        utf8_field("instrument_id", false),
1252        float64_field("value", false),
1253        timestamp_field("ts_event", false),
1254        timestamp_field("ts_init", false),
1255    ])
1256}
1257
1258fn convert_price_updates(
1259    metadata: &HashMap<String, String>,
1260    batch: &RecordBatch,
1261) -> Result<RecordBatch, EncodingError> {
1262    let instrument_id = instrument_id(metadata)?;
1263    let price_precision = parse_price_precision(metadata)?;
1264    let value = fixed_col(batch, "value")?;
1265    let ts_event = nanos_col(batch, "ts_event")?;
1266    let ts_init = nanos_col(batch, "ts_init")?;
1267    let len = batch.num_rows();
1268    let mut value_builder = Float64Builder::with_capacity(len);
1269    let mut ts_event_builder =
1270        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1271    let mut ts_init_builder =
1272        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1273
1274    for row in 0..len {
1275        append_price(&mut value_builder, &value, price_precision, "value", row)?;
1276        append_timestamp(&mut ts_event_builder, &ts_event, row);
1277        append_timestamp(&mut ts_init_builder, &ts_init, row);
1278    }
1279
1280    RecordBatch::try_new(
1281        Arc::new(price_update_schema()),
1282        vec![
1283            constant_string_column(instrument_id, len),
1284            Arc::new(value_builder.finish()),
1285            Arc::new(ts_event_builder.finish()),
1286            Arc::new(ts_init_builder.finish()),
1287        ],
1288    )
1289    .map_err(EncodingError::from)
1290}
1291
1292fn funding_rate_schema() -> Schema {
1293    Schema::new(vec![
1294        utf8_field("instrument_id", false),
1295        float64_field("rate", false),
1296        Field::new("interval", DataType::UInt64, true),
1297        timestamp_field("next_funding_ns", true),
1298        timestamp_field("ts_event", false),
1299        timestamp_field("ts_init", false),
1300    ])
1301}
1302
1303fn convert_funding_rates(
1304    metadata: &HashMap<String, String>,
1305    batch: &RecordBatch,
1306) -> Result<RecordBatch, EncodingError> {
1307    let instrument_id = instrument_id(metadata)?;
1308    let rate_index = batch.schema().index_of("rate")?;
1309    let rate = extract_column_string(batch.columns(), "rate", rate_index)?;
1310    let interval = u64_col(batch, "interval")?;
1311    let next_funding_ns = nanos_col(batch, "next_funding_ns")?;
1312    let ts_event = nanos_col(batch, "ts_event")?;
1313    let ts_init = nanos_col(batch, "ts_init")?;
1314    let len = batch.num_rows();
1315
1316    let mut rate_builder = Float64Builder::with_capacity(len);
1317    let mut interval_builder = UInt64Builder::with_capacity(len);
1318    let mut next_funding_ns_builder =
1319        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1320    let mut ts_event_builder =
1321        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1322    let mut ts_init_builder =
1323        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1324
1325    for row in 0..len {
1326        let value = Decimal::from_str(rate.value(row))
1327            .map_err(|e| EncodingError::ParseError("rate", e.to_string()))?;
1328        rate_builder.append_value(value.to_f64().unwrap_or(f64::NAN));
1329        append_optional_u64(&mut interval_builder, &interval, "interval", row)?;
1330        append_timestamp(&mut next_funding_ns_builder, &next_funding_ns, row);
1331        append_timestamp(&mut ts_event_builder, &ts_event, row);
1332        append_timestamp(&mut ts_init_builder, &ts_init, row);
1333    }
1334
1335    RecordBatch::try_new(
1336        Arc::new(funding_rate_schema()),
1337        vec![
1338            constant_string_column(instrument_id, len),
1339            Arc::new(rate_builder.finish()),
1340            Arc::new(interval_builder.finish()),
1341            Arc::new(next_funding_ns_builder.finish()),
1342            Arc::new(ts_event_builder.finish()),
1343            Arc::new(ts_init_builder.finish()),
1344        ],
1345    )
1346    .map_err(EncodingError::from)
1347}
1348
1349fn instrument_closes_schema() -> Schema {
1350    Schema::new(vec![
1351        utf8_field("instrument_id", false),
1352        float64_field("close_price", false),
1353        utf8_field("close_type", false),
1354        timestamp_field("ts_event", false),
1355        timestamp_field("ts_init", false),
1356    ])
1357}
1358
1359fn convert_instrument_closes(
1360    metadata: &HashMap<String, String>,
1361    batch: &RecordBatch,
1362) -> Result<RecordBatch, EncodingError> {
1363    let instrument_id = instrument_id(metadata)?;
1364    let price_precision = parse_price_precision(metadata)?;
1365    let close_price = fixed_col(batch, "close_price")?;
1366    let close_type_index = batch.schema().index_of("close_type")?;
1367    let close_type = extract_column_string(batch.columns(), "close_type", close_type_index)?;
1368    let ts_event = nanos_col(batch, "ts_event")?;
1369    let ts_init = nanos_col(batch, "ts_init")?;
1370    let len = batch.num_rows();
1371    let mut close_price_builder = Float64Builder::with_capacity(len);
1372    let mut close_type_builder = StringBuilder::new();
1373    let mut ts_event_builder =
1374        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1375    let mut ts_init_builder =
1376        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1377
1378    for row in 0..len {
1379        let close_type_value = close_type.value(row);
1380        let close_type = InstrumentCloseType::from_str(close_type_value).map_err(|e| {
1381            EncodingError::ParseError(stringify!(InstrumentCloseType), e.to_string())
1382        })?;
1383        append_price(
1384            &mut close_price_builder,
1385            &close_price,
1386            price_precision,
1387            "close_price",
1388            row,
1389        )?;
1390        close_type_builder.append_value(close_type.as_ref());
1391        append_timestamp(&mut ts_event_builder, &ts_event, row);
1392        append_timestamp(&mut ts_init_builder, &ts_init, row);
1393    }
1394
1395    RecordBatch::try_new(
1396        Arc::new(instrument_closes_schema()),
1397        vec![
1398            constant_string_column(instrument_id, len),
1399            Arc::new(close_price_builder.finish()),
1400            Arc::new(close_type_builder.finish()),
1401            Arc::new(ts_event_builder.finish()),
1402            Arc::new(ts_init_builder.finish()),
1403        ],
1404    )
1405    .map_err(EncodingError::from)
1406}
1407
1408fn option_greeks_schema() -> Schema {
1409    Schema::new(vec![
1410        utf8_field("instrument_id", false),
1411        float64_field("delta", false),
1412        float64_field("gamma", false),
1413        float64_field("vega", false),
1414        float64_field("theta", false),
1415        float64_field("rho", false),
1416        float64_field("mark_iv", true),
1417        float64_field("bid_iv", true),
1418        float64_field("ask_iv", true),
1419        float64_field("underlying_price", true),
1420        float64_field("open_interest", true),
1421        timestamp_field("ts_event", false),
1422        timestamp_field("ts_init", false),
1423        utf8_field("convention", false),
1424    ])
1425}
1426
1427fn append_optional_f64(builder: &mut Float64Builder, values: &Float64Array, row: usize) {
1428    if values.is_null(row) {
1429        builder.append_null();
1430    } else {
1431        builder.append_value(values.value(row));
1432    }
1433}
1434
1435fn convert_option_greeks(
1436    metadata: &HashMap<String, String>,
1437    batch: &RecordBatch,
1438) -> Result<RecordBatch, EncodingError> {
1439    let instrument_id = instrument_id(metadata)?;
1440    let delta = f64_col(batch, "delta")?;
1441    let gamma = f64_col(batch, "gamma")?;
1442    let vega = f64_col(batch, "vega")?;
1443    let theta = f64_col(batch, "theta")?;
1444    let rho = f64_col(batch, "rho")?;
1445    let mark_iv = f64_col(batch, "mark_iv")?;
1446    let bid_iv = f64_col(batch, "bid_iv")?;
1447    let ask_iv = f64_col(batch, "ask_iv")?;
1448    let underlying_price = f64_col(batch, "underlying_price")?;
1449    let open_interest = f64_col(batch, "open_interest")?;
1450    let ts_event = nanos_col(batch, "ts_event")?;
1451    let ts_init = nanos_col(batch, "ts_init")?;
1452    let convention_index = batch.schema().index_of("convention")?;
1453    let convention = extract_column_string(batch.columns(), "convention", convention_index)?;
1454    let len = batch.num_rows();
1455
1456    let mut delta_builder = Float64Builder::with_capacity(len);
1457    let mut gamma_builder = Float64Builder::with_capacity(len);
1458    let mut vega_builder = Float64Builder::with_capacity(len);
1459    let mut theta_builder = Float64Builder::with_capacity(len);
1460    let mut rho_builder = Float64Builder::with_capacity(len);
1461    let mut mark_iv_builder = Float64Builder::with_capacity(len);
1462    let mut bid_iv_builder = Float64Builder::with_capacity(len);
1463    let mut ask_iv_builder = Float64Builder::with_capacity(len);
1464    let mut underlying_price_builder = Float64Builder::with_capacity(len);
1465    let mut open_interest_builder = Float64Builder::with_capacity(len);
1466    let mut ts_event_builder =
1467        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1468    let mut ts_init_builder =
1469        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1470    let mut convention_builder = StringBuilder::new();
1471
1472    for row in 0..len {
1473        delta_builder.append_value(delta.value(row));
1474        gamma_builder.append_value(gamma.value(row));
1475        vega_builder.append_value(vega.value(row));
1476        theta_builder.append_value(theta.value(row));
1477        rho_builder.append_value(rho.value(row));
1478        append_optional_f64(&mut mark_iv_builder, mark_iv, row);
1479        append_optional_f64(&mut bid_iv_builder, bid_iv, row);
1480        append_optional_f64(&mut ask_iv_builder, ask_iv, row);
1481        append_optional_f64(&mut underlying_price_builder, underlying_price, row);
1482        append_optional_f64(&mut open_interest_builder, open_interest, row);
1483        append_timestamp(&mut ts_event_builder, &ts_event, row);
1484        append_timestamp(&mut ts_init_builder, &ts_init, row);
1485        convention_builder.append_value(convention.value(row));
1486    }
1487
1488    RecordBatch::try_new(
1489        Arc::new(option_greeks_schema()),
1490        vec![
1491            constant_string_column(instrument_id, len),
1492            Arc::new(delta_builder.finish()),
1493            Arc::new(gamma_builder.finish()),
1494            Arc::new(vega_builder.finish()),
1495            Arc::new(theta_builder.finish()),
1496            Arc::new(rho_builder.finish()),
1497            Arc::new(mark_iv_builder.finish()),
1498            Arc::new(bid_iv_builder.finish()),
1499            Arc::new(ask_iv_builder.finish()),
1500            Arc::new(underlying_price_builder.finish()),
1501            Arc::new(open_interest_builder.finish()),
1502            Arc::new(ts_event_builder.finish()),
1503            Arc::new(ts_init_builder.finish()),
1504            Arc::new(convention_builder.finish()),
1505        ],
1506    )
1507    .map_err(EncodingError::from)
1508}
1509
1510fn instrument_status_schema() -> Schema {
1511    Schema::new(vec![
1512        utf8_field("instrument_id", false),
1513        utf8_field("action", false),
1514        timestamp_field("ts_event", false),
1515        timestamp_field("ts_init", false),
1516        utf8_field("reason", true),
1517        utf8_field("trading_event", true),
1518        Field::new("is_trading", DataType::Boolean, true),
1519        Field::new("is_quoting", DataType::Boolean, true),
1520        Field::new("is_short_sell_restricted", DataType::Boolean, true),
1521    ])
1522}
1523
1524fn append_optional_string(
1525    builder: &mut StringBuilder,
1526    values: &super::StringColumnRef<'_>,
1527    row: usize,
1528    array: &dyn Array,
1529) {
1530    if array.is_null(row) {
1531        builder.append_null();
1532    } else {
1533        builder.append_value(values.value(row));
1534    }
1535}
1536
1537fn append_optional_bool(
1538    builder: &mut arrow::array::BooleanBuilder,
1539    values: &BooleanArray,
1540    row: usize,
1541) {
1542    if values.is_null(row) {
1543        builder.append_null();
1544    } else {
1545        builder.append_value(values.value(row));
1546    }
1547}
1548
1549fn append_optional_u64(
1550    builder: &mut UInt64Builder,
1551    values: &U64ColumnRef<'_>,
1552    field: &'static str,
1553    row: usize,
1554) -> Result<(), EncodingError> {
1555    if values.is_null(row) {
1556        builder.append_null();
1557    } else {
1558        builder.append_value(u64_value(values, row, field)?);
1559    }
1560    Ok(())
1561}
1562
1563fn convert_instrument_status(
1564    metadata: &HashMap<String, String>,
1565    batch: &RecordBatch,
1566) -> Result<RecordBatch, EncodingError> {
1567    let instrument_id = instrument_id(metadata)?;
1568    let action_index = batch.schema().index_of("action")?;
1569    let action = extract_column_string(batch.columns(), "action", action_index)?;
1570    let ts_event = nanos_col(batch, "ts_event")?;
1571    let ts_init = nanos_col(batch, "ts_init")?;
1572    let reason_index = batch.schema().index_of("reason")?;
1573    let reason = extract_column_string(batch.columns(), "reason", reason_index)?;
1574    let trading_event_index = batch.schema().index_of("trading_event")?;
1575    let trading_event =
1576        extract_column_string(batch.columns(), "trading_event", trading_event_index)?;
1577    let is_trading = bool_col(batch, "is_trading")?;
1578    let is_quoting = bool_col(batch, "is_quoting")?;
1579    let is_short_sell_restricted = bool_col(batch, "is_short_sell_restricted")?;
1580    let len = batch.num_rows();
1581
1582    let mut action_builder = StringBuilder::new();
1583    let mut ts_event_builder =
1584        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1585    let mut ts_init_builder =
1586        TimestampNanosecondBuilder::with_capacity(len).with_data_type(timestamp_data_type());
1587    let mut reason_builder = StringBuilder::new();
1588    let mut trading_event_builder = StringBuilder::new();
1589    let mut is_trading_builder = arrow::array::BooleanBuilder::with_capacity(len);
1590    let mut is_quoting_builder = arrow::array::BooleanBuilder::with_capacity(len);
1591    let mut is_short_sell_restricted_builder = arrow::array::BooleanBuilder::with_capacity(len);
1592
1593    for row in 0..len {
1594        action_builder.append_value(action.value(row));
1595        append_timestamp(&mut ts_event_builder, &ts_event, row);
1596        append_timestamp(&mut ts_init_builder, &ts_init, row);
1597        append_optional_string(
1598            &mut reason_builder,
1599            &reason,
1600            row,
1601            batch.column(reason_index).as_ref(),
1602        );
1603        append_optional_string(
1604            &mut trading_event_builder,
1605            &trading_event,
1606            row,
1607            batch.column(trading_event_index).as_ref(),
1608        );
1609        append_optional_bool(&mut is_trading_builder, is_trading, row);
1610        append_optional_bool(&mut is_quoting_builder, is_quoting, row);
1611        append_optional_bool(
1612            &mut is_short_sell_restricted_builder,
1613            is_short_sell_restricted,
1614            row,
1615        );
1616    }
1617
1618    RecordBatch::try_new(
1619        Arc::new(instrument_status_schema()),
1620        vec![
1621            constant_string_column(instrument_id, len),
1622            Arc::new(action_builder.finish()),
1623            Arc::new(ts_event_builder.finish()),
1624            Arc::new(ts_init_builder.finish()),
1625            Arc::new(reason_builder.finish()),
1626            Arc::new(trading_event_builder.finish()),
1627            Arc::new(is_trading_builder.finish()),
1628            Arc::new(is_quoting_builder.finish()),
1629            Arc::new(is_short_sell_restricted_builder.finish()),
1630        ],
1631    )
1632    .map_err(EncodingError::from)
1633}
1634
1635/// Converts custom data to display types.
1636///
1637/// For a registered custom type, the output uses its canonical registered schema metadata.
1638/// User-provided `DataType` metadata keys are therefore not retained in the display schema.
1639fn convert_custom(batch: &RecordBatch) -> Result<RecordBatch, EncodingError> {
1640    let batch_schema = batch.schema();
1641    let metadata = batch_schema.metadata();
1642    let mut fields = Vec::with_capacity(batch.num_columns());
1643    let mut columns = Vec::with_capacity(batch.num_columns());
1644
1645    for (field, column) in batch_schema.fields().iter().zip(batch.columns()) {
1646        if matches!(field.name().as_str(), "ts_event" | "ts_init") {
1647            let name = if field.name() == "ts_event" {
1648                "ts_event"
1649            } else {
1650                "ts_init"
1651            };
1652            let values = nanos_col(batch, name)?;
1653            fields.push(Arc::new(timestamp_field_from(field)));
1654            columns.push(timestamp_array_from_nanos(&values));
1655        } else if let Some((precision, is_price)) = custom_fixed_precision(field.name(), metadata)
1656            && matches!(
1657                field.data_type(),
1658                DataType::Decimal128(_, _)
1659                    | DataType::FixedSizeBinary(_)
1660                    | DataType::Binary
1661                    | DataType::BinaryView
1662            )
1663        {
1664            if field.data_type() != &fixed_decimal_data_type() {
1665                return Err(EncodingError::ParseError(
1666                    "custom",
1667                    format!(
1668                        "{} must use Decimal128(38, 16); migrate legacy data before display",
1669                        field.name()
1670                    ),
1671                ));
1672            }
1673            let values = FixedPrecisionColumn(
1674                column
1675                    .as_any()
1676                    .downcast_ref::<Decimal128Array>()
1677                    .expect("decimal field type checked above"),
1678            );
1679            fields.push(Arc::new(Field::new(
1680                field.name(),
1681                DataType::Float64,
1682                field.is_nullable(),
1683            )));
1684            columns.push(fixed_precision_field_to_f64(&values, precision, is_price)?);
1685        } else if field.data_type() == &DataType::Utf8View {
1686            fields.push(Arc::new(Field::new(
1687                field.name(),
1688                DataType::Utf8,
1689                field.is_nullable(),
1690            )));
1691            columns.push(cast(column.as_ref(), &DataType::Utf8)?);
1692        } else {
1693            fields.push(Arc::clone(field));
1694            columns.push(Arc::clone(column));
1695        }
1696    }
1697
1698    let metadata = if let Some(type_name) = metadata.get("type_name")
1699        && get_arrow_schema(type_name).is_some()
1700    {
1701        let canonical_metadata = HashMap::from([("type_name".to_string(), type_name.clone())]);
1702        let canonical_schema = super::schema_with_identifier_column(
1703            &CustomDataDecoder::get_schema(Some(canonical_metadata)),
1704        );
1705
1706        for field in &mut fields {
1707            if let Ok(canonical_field) = canonical_schema.field_with_name(field.name()) {
1708                *field = Arc::new(
1709                    field
1710                        .as_ref()
1711                        .clone()
1712                        .with_nullable(canonical_field.is_nullable()),
1713                );
1714            }
1715        }
1716        canonical_schema.metadata().clone()
1717    } else {
1718        batch_schema.metadata().clone()
1719    };
1720    let schema = Schema::new_with_metadata(fields, metadata);
1721    RecordBatch::try_new(Arc::new(schema), columns).map_err(EncodingError::from)
1722}
1723
1724#[cfg(test)]
1725mod tests {
1726    use std::{collections::HashMap, str::FromStr};
1727
1728    use arrow::{
1729        array::{
1730            Array, FixedSizeBinaryArray, FixedSizeListArray, Float64Array, StringArray,
1731            StringViewArray, UInt64Array,
1732        },
1733        datatypes::{Field, TimeUnit},
1734    };
1735    use nautilus_core::UnixNanos;
1736    #[cfg(feature = "arrow-display")]
1737    use nautilus_model::data::stubs::{
1738        stub_bar, stub_delta, stub_instrument_close, stub_trade_ethusdt_buy,
1739    };
1740    use nautilus_model::{
1741        data::{
1742            FundingRateUpdate, OrderBookDepth, QuoteTick, ensure_arrow_registered,
1743            order::BookOrder, stubs::stub_depth10,
1744        },
1745        identifiers::InstrumentId,
1746        types::{Price, Quantity},
1747    };
1748    use rstest::rstest;
1749    use rust_decimal::Decimal;
1750
1751    use super::*;
1752    use crate::arrow::{
1753        EncodeToRecordBatch, KEY_INSTRUMENT_ID, KEY_PRICE_PRECISION, KEY_SIZE_PRECISION,
1754        book_deltas_to_arrow_record_batch_bytes, enum_dictionary_array,
1755    };
1756    #[cfg(feature = "arrow-display")]
1757    use crate::arrow::{
1758        display::{
1759            bar::encode_bars, close::encode_instrument_closes, delta::encode_deltas,
1760            trade::encode_trades,
1761        },
1762        record_batch_without_identifier_column,
1763    };
1764
1765    #[rstest]
1766    #[case::clear_only(false)]
1767    #[case::mixed(true)]
1768    fn test_catalog_display_converts_clear_deltas(#[case] mixed: bool) {
1769        let instrument_id = InstrumentId::from("AAPL.XNAS");
1770        let clear = OrderBookDelta::clear(instrument_id, 41, 101.into(), 102.into());
1771        let mut deltas = vec![clear];
1772        if mixed {
1773            deltas.extend([
1774                OrderBookDelta::new(
1775                    instrument_id,
1776                    BookAction::Add,
1777                    BookOrder::new(
1778                        OrderSide::Buy,
1779                        Price::from("100.25"),
1780                        Quantity::from("2.50"),
1781                        9_007_199_254_740_993,
1782                    ),
1783                    128,
1784                    42,
1785                    103.into(),
1786                    104.into(),
1787                ),
1788                OrderBookDelta::new(
1789                    instrument_id,
1790                    BookAction::Update,
1791                    BookOrder::new(
1792                        OrderSide::Sell,
1793                        Price::from("101.75"),
1794                        Quantity::from("3.25"),
1795                        73,
1796                    ),
1797                    64,
1798                    43,
1799                    105.into(),
1800                    106.into(),
1801                ),
1802            ]);
1803        }
1804
1805        let raw = book_deltas_to_arrow_record_batch_bytes(&deltas).unwrap();
1806        let display = catalog_record_batch_to_display(
1807            &NautilusDataType::OrderBookDelta,
1808            raw.schema().metadata(),
1809            &raw,
1810        )
1811        .unwrap();
1812        let len = deltas.len();
1813        let mut fields = deltas_schema().fields().to_vec();
1814        fields.push(Arc::new(utf8_field(KEY_IDENTIFIER, true)));
1815        let expected_schema = Schema::new(fields);
1816        let expected: Vec<ArrayRef> = vec![
1817            Arc::new(StringArray::from(vec!["AAPL.XNAS"; len])),
1818            Arc::new(StringArray::from(
1819                ["CLEAR", "ADD", "UPDATE"][..len].to_vec(),
1820            )),
1821            Arc::new(StringArray::from(
1822                ["NO_ORDER_SIDE", "BUY", "SELL"][..len].to_vec(),
1823            )),
1824            Arc::new(Float64Array::from(
1825                [f64::NAN, 100.25, 101.75][..len].to_vec(),
1826            )),
1827            Arc::new(Float64Array::from([f64::NAN, 2.50, 3.25][..len].to_vec())),
1828            Arc::new(StringArray::from(
1829                ["0", "9007199254740993", "73"][..len].to_vec(),
1830            )),
1831            Arc::new(UInt8Array::from([32, 128, 64][..len].to_vec())),
1832            Arc::new(UInt64Array::from([41, 42, 43][..len].to_vec())),
1833            Arc::new(
1834                TimestampNanosecondArray::from([101, 103, 105][..len].to_vec())
1835                    .with_timezone("UTC"),
1836            ),
1837            Arc::new(
1838                TimestampNanosecondArray::from([102, 104, 106][..len].to_vec())
1839                    .with_timezone("UTC"),
1840            ),
1841            Arc::new(StringArray::from(vec!["AAPL.XNAS"; len])),
1842        ];
1843
1844        assert_eq!(display.schema().as_ref(), &expected_schema);
1845        assert_eq!(display.num_rows(), len);
1846        assert_eq!(display.num_columns(), expected.len());
1847
1848        for (actual, expected) in display.columns().iter().zip(&expected) {
1849            if let Some(actual) = actual.as_any().downcast_ref::<Float64Array>() {
1850                let expected = expected.as_any().downcast_ref::<Float64Array>().unwrap();
1851                assert_eq!(actual.null_count(), 0);
1852                assert!(actual.value(0).is_nan());
1853                assert_eq!(&actual.values()[1..], &expected.values()[1..]);
1854            } else {
1855                assert_eq!(actual, expected);
1856            }
1857        }
1858    }
1859
1860    #[rstest]
1861    #[case("no_order_side", "NO_ORDER_SIDE")]
1862    #[case("buy", "BUY")]
1863    #[case("sell", "SELL")]
1864    fn test_catalog_display_normalizes_delta_side(#[case] side: &str, #[case] expected: &str) {
1865        let delta =
1866            OrderBookDelta::clear(InstrumentId::from("AAPL.XNAS"), 41, 101.into(), 102.into());
1867        let raw = book_deltas_to_arrow_record_batch_bytes(&[delta]).unwrap();
1868        let mut columns = raw.columns().to_vec();
1869        let side_index = raw.schema().index_of("side").unwrap();
1870        let side = enum_dictionary_array([side]).unwrap();
1871        columns[side_index] = Arc::new(side);
1872        let raw = RecordBatch::try_new(raw.schema(), columns).unwrap();
1873
1874        let display = catalog_record_batch_to_display(
1875            &NautilusDataType::OrderBookDelta,
1876            raw.schema().metadata(),
1877            &raw,
1878        )
1879        .unwrap();
1880
1881        assert_eq!(
1882            display.column_by_name("side").unwrap().as_ref(),
1883            &StringArray::from(vec![expected])
1884        );
1885    }
1886
1887    #[rstest]
1888    #[case::clear(BookAction::Clear)]
1889    #[case::add(BookAction::Add)]
1890    fn test_catalog_display_rejects_invalid_delta_side(#[case] action: BookAction) {
1891        let instrument_id = InstrumentId::from("AAPL.XNAS");
1892        let mut delta = OrderBookDelta::clear(instrument_id, 41, 101.into(), 102.into());
1893        delta.action = action;
1894        let raw = book_deltas_to_arrow_record_batch_bytes(&[delta]).unwrap();
1895        let mut columns = raw.columns().to_vec();
1896        let side_index = raw.schema().index_of("side").unwrap();
1897        let side = enum_dictionary_array(["INVALID"]).unwrap();
1898        columns[side_index] = Arc::new(side);
1899        let raw = RecordBatch::try_new(raw.schema(), columns).unwrap();
1900
1901        let error = catalog_record_batch_to_display(
1902            &NautilusDataType::OrderBookDelta,
1903            raw.schema().metadata(),
1904            &raw,
1905        )
1906        .unwrap_err();
1907
1908        assert!(
1909            matches!(error, EncodingError::ParseError("OrderSide", message) if message == "Matching variant not found")
1910        );
1911    }
1912
1913    #[rstest]
1914    fn test_catalog_record_batch_to_display_converts_quote_prices_and_metadata() {
1915        let instrument_id = InstrumentId::from("AUD/USD.SIM");
1916        let metadata = HashMap::from([
1917            (KEY_INSTRUMENT_ID.to_string(), instrument_id.to_string()),
1918            (KEY_PRICE_PRECISION.to_string(), "5".to_string()),
1919            (KEY_SIZE_PRECISION.to_string(), "0".to_string()),
1920        ]);
1921        let quote = QuoteTick {
1922            instrument_id,
1923            bid_price: Price::from("1.00001"),
1924            ask_price: Price::from("1.00003"),
1925            bid_size: Quantity::from(1_000),
1926            ask_size: Quantity::from(2_000),
1927            ts_event: 10.into(),
1928            ts_init: 11.into(),
1929        };
1930        let raw_batch = QuoteTick::encode_batch(&metadata, &[quote]).unwrap();
1931
1932        let display_batch =
1933            catalog_record_batch_to_display(&NautilusDataType::QuoteTick, &metadata, &raw_batch)
1934                .unwrap();
1935
1936        let mut expected_fields = quotes_schema().fields().to_vec();
1937        expected_fields.push(Arc::new(utf8_field(KEY_IDENTIFIER, true)));
1938        assert_eq!(
1939            display_batch.schema(),
1940            Arc::new(Schema::new(expected_fields))
1941        );
1942        let instrument_ids = display_batch
1943            .column(0)
1944            .as_any()
1945            .downcast_ref::<StringArray>()
1946            .unwrap();
1947        let bid_prices = display_batch
1948            .column(1)
1949            .as_any()
1950            .downcast_ref::<Float64Array>()
1951            .unwrap();
1952        let ask_sizes = display_batch
1953            .column(4)
1954            .as_any()
1955            .downcast_ref::<Float64Array>()
1956            .unwrap();
1957        let ts_init = display_batch
1958            .column(6)
1959            .as_any()
1960            .downcast_ref::<arrow::array::TimestampNanosecondArray>()
1961            .unwrap();
1962        let identifiers = display_batch
1963            .column(7)
1964            .as_any()
1965            .downcast_ref::<StringArray>()
1966            .unwrap();
1967
1968        assert_eq!(instrument_ids.value(0), "AUD/USD.SIM");
1969        assert_eq!(bid_prices.value(0), 1.00001);
1970        assert_eq!(ask_sizes.value(0), 2_000.0);
1971        assert_eq!(ts_init.value(0), 11);
1972        assert_eq!(identifiers.value(0), "AUD/USD.SIM");
1973    }
1974
1975    #[rstest]
1976    #[case(1)]
1977    #[case(u64::MAX)]
1978    fn test_catalog_record_batch_to_display_rejects_legacy_timestamps(#[case] timestamp: u64) {
1979        let instrument_id = InstrumentId::from("AUD/USD.SIM");
1980        let metadata = HashMap::from([
1981            (KEY_INSTRUMENT_ID.to_string(), instrument_id.to_string()),
1982            (KEY_PRICE_PRECISION.to_string(), "5".to_string()),
1983            (KEY_SIZE_PRECISION.to_string(), "0".to_string()),
1984        ]);
1985        let quote = QuoteTick {
1986            instrument_id,
1987            bid_price: Price::from("1.00001"),
1988            ask_price: Price::from("1.00003"),
1989            bid_size: Quantity::from(1_000),
1990            ask_size: Quantity::from(2_000),
1991            ts_event: 10.into(),
1992            ts_init: 11.into(),
1993        };
1994        let raw_batch = QuoteTick::encode_batch(&metadata, &[quote]).unwrap();
1995        let mut fields = raw_batch.schema().fields().to_vec();
1996        let ts_init_index = raw_batch.schema().index_of("ts_init").unwrap();
1997        fields[ts_init_index] = Arc::new(Field::new("ts_init", DataType::UInt64, false));
1998        let mut columns = raw_batch.columns().to_vec();
1999        columns[ts_init_index] = Arc::new(UInt64Array::from(vec![timestamp]));
2000        let raw_batch = RecordBatch::try_new(
2001            Arc::new(Schema::new_with_metadata(fields, metadata.clone())),
2002            columns,
2003        )
2004        .unwrap();
2005
2006        let error =
2007            catalog_record_batch_to_display(&NautilusDataType::QuoteTick, &metadata, &raw_batch)
2008                .unwrap_err();
2009
2010        assert!(matches!(
2011            error,
2012            EncodingError::InvalidColumnType(
2013                "ts_init",
2014                _,
2015                DataType::Timestamp(TimeUnit::Nanosecond, Some(timezone)),
2016                DataType::UInt64
2017            ) if timezone.as_ref() == "UTC"
2018        ));
2019    }
2020
2021    #[rstest]
2022    #[case(38, 15)]
2023    #[case(37, 16)]
2024    fn test_catalog_display_rejects_noncanonical_decimal_schema(
2025        #[case] precision: u8,
2026        #[case] scale: i8,
2027    ) {
2028        let quote = QuoteTick::new(
2029            InstrumentId::from("AUD/USD.SIM"),
2030            Price::from("1.25"),
2031            Price::from("1.50"),
2032            Quantity::from(10),
2033            Quantity::from(20),
2034            10.into(),
2035            11.into(),
2036        );
2037        let metadata = quote.metadata();
2038        let raw = QuoteTick::encode_batch(&metadata, &[quote]).unwrap();
2039        let index = raw.schema().index_of("bid_price").unwrap();
2040        let mut fields = raw.schema().fields().to_vec();
2041        let mut columns = raw.columns().to_vec();
2042        let values = columns[index]
2043            .as_any()
2044            .downcast_ref::<Decimal128Array>()
2045            .unwrap()
2046            .clone()
2047            .with_precision_and_scale(precision, scale)
2048            .unwrap();
2049        fields[index] = Arc::new(
2050            fields[index]
2051                .as_ref()
2052                .clone()
2053                .with_data_type(values.data_type().clone()),
2054        );
2055        columns[index] = Arc::new(values);
2056        let batch = RecordBatch::try_new(
2057            Arc::new(Schema::new_with_metadata(fields, metadata.clone())),
2058            columns,
2059        )
2060        .unwrap();
2061        let error =
2062            catalog_record_batch_to_display(&NautilusDataType::QuoteTick, &metadata, &batch)
2063                .unwrap_err();
2064
2065        assert!(
2066            matches!(error, EncodingError::InvalidColumnType("bid_price", i, DataType::Decimal128(38, 16), actual) if i == index && actual == DataType::Decimal128(precision, scale))
2067        );
2068    }
2069
2070    #[rstest]
2071    #[case(0)]
2072    #[case(5)]
2073    #[case(10)]
2074    #[case(25)]
2075    fn test_depth_display_preserves_all_levels(#[case] levels: usize) {
2076        let mut depth = stub_depth10();
2077        depth.bids.resize(levels, depth.bids[0]);
2078        depth.bid_counts.resize(levels, 0);
2079        depth.asks.clear();
2080        depth.ask_counts.clear();
2081        for (i, (order, count)) in depth.bids.iter_mut().zip(&mut depth.bid_counts).enumerate() {
2082            order.price = format!("{}.25", 100 + i).parse().unwrap();
2083            order.size = format!("{}.5", 200 + i).parse().unwrap();
2084            order.order_id = u64::MAX - u64::try_from(i).unwrap();
2085            *count = 300 + u32::try_from(i).unwrap();
2086        }
2087        let metadata = depth.metadata();
2088        let raw = OrderBookDepth::encode_batch(&metadata, &[depth]).unwrap();
2089        let display =
2090            catalog_record_batch_to_display(&NautilusDataType::OrderBookDepth, &metadata, &raw)
2091                .unwrap();
2092        let list = display
2093            .column_by_name("bids")
2094            .unwrap()
2095            .as_any()
2096            .downcast_ref::<ListArray>()
2097            .unwrap();
2098        let values = list.value(0);
2099        let structs = values.as_any().downcast_ref::<StructArray>().unwrap();
2100        let prices = structs
2101            .column_by_name("price")
2102            .unwrap()
2103            .as_any()
2104            .downcast_ref::<Float64Array>()
2105            .unwrap();
2106        let sizes = structs
2107            .column_by_name("size")
2108            .unwrap()
2109            .as_any()
2110            .downcast_ref::<Float64Array>()
2111            .unwrap();
2112        let counts = structs
2113            .column_by_name("count")
2114            .unwrap()
2115            .as_any()
2116            .downcast_ref::<UInt32Array>()
2117            .unwrap();
2118        let ids = structs
2119            .column_by_name("order_id")
2120            .unwrap()
2121            .as_any()
2122            .downcast_ref::<UInt64Array>()
2123            .unwrap();
2124        let asks = display
2125            .column_by_name("asks")
2126            .unwrap()
2127            .as_any()
2128            .downcast_ref::<ListArray>()
2129            .unwrap();
2130        let empty = catalog_record_batch_to_display(
2131            &NautilusDataType::OrderBookDepth,
2132            &metadata,
2133            &raw.slice(0, 0),
2134        )
2135        .unwrap();
2136
2137        assert_eq!(
2138            display.schema().as_ref(),
2139            &crate::arrow::schema_with_identifier_column(&depth_schema())
2140        );
2141        assert_eq!(empty.schema(), display.schema());
2142        assert_eq!(empty.num_rows(), 0);
2143        assert_eq!(display.num_rows(), 1);
2144        assert_eq!(list.value_length(0), i32::try_from(levels).unwrap());
2145        assert_eq!(asks.value_length(0), 0);
2146        assert!(!asks.is_null(0));
2147
2148        for i in 0..levels {
2149            assert_eq!(
2150                prices.value(i),
2151                f64::from(u32::try_from(100 + i).unwrap()) + 0.25
2152            );
2153            assert_eq!(
2154                sizes.value(i),
2155                f64::from(u32::try_from(200 + i).unwrap()) + 0.5
2156            );
2157            assert_eq!(counts.value(i), 300 + u32::try_from(i).unwrap());
2158            assert_eq!(ids.value(i), u64::MAX - u64::try_from(i).unwrap());
2159        }
2160    }
2161
2162    #[rstest]
2163    #[case(false, false)]
2164    #[case(false, true)]
2165    #[case(true, false)]
2166    #[case(true, true)]
2167    fn test_depth_display_rejects_legacy_shapes(#[case] lists: bool, #[case] with_ids: bool) {
2168        let depth = stub_depth10();
2169        let metadata = depth.metadata();
2170        let raw = OrderBookDepth::encode_batch(&metadata, &[depth]).unwrap();
2171        let mut columns = Vec::<ArrayRef>::new();
2172        let mut fields = Vec::<Field>::new();
2173
2174        for (side, prefix) in [("bids", "bid"), ("asks", "ask")] {
2175            let list = raw
2176                .column_by_name(side)
2177                .unwrap()
2178                .as_any()
2179                .downcast_ref::<ListArray>()
2180                .unwrap();
2181            let levels = list
2182                .values()
2183                .as_any()
2184                .downcast_ref::<StructArray>()
2185                .unwrap();
2186
2187            for name in ["price", "size", "count", "order_id"] {
2188                if name == "order_id" && !with_ids {
2189                    continue;
2190                }
2191                let values = levels.column_by_name(name).unwrap();
2192                if lists {
2193                    let item = Arc::new(Field::new("item", values.data_type().clone(), true));
2194                    let column =
2195                        FixedSizeListArray::try_new(item, 10, Arc::clone(values), None).unwrap();
2196                    fields.push(Field::new(
2197                        format!("{prefix}_{name}"),
2198                        column.data_type().clone(),
2199                        false,
2200                    ));
2201                    columns.push(Arc::new(column));
2202                } else {
2203                    for i in 0..10 {
2204                        fields.push(Field::new(
2205                            format!("{prefix}_{name}_{i}"),
2206                            values.data_type().clone(),
2207                            true,
2208                        ));
2209                        columns.push(values.slice(i, 1));
2210                    }
2211                }
2212            }
2213        }
2214
2215        for name in ["flags", "sequence", "ts_event", "ts_init"] {
2216            fields.push(raw.schema().field_with_name(name).unwrap().clone());
2217            columns.push(Arc::clone(raw.column_by_name(name).unwrap()));
2218        }
2219        let legacy = RecordBatch::try_new(
2220            Arc::new(Schema::new_with_metadata(fields, metadata.clone())),
2221            columns,
2222        )
2223        .unwrap();
2224        let error =
2225            catalog_record_batch_to_display(&NautilusDataType::OrderBookDepth, &metadata, &legacy)
2226                .unwrap_err();
2227        assert!(
2228            matches!(error, EncodingError::ParseError("bids", ref message) if message == "expected List<Struct>")
2229        );
2230    }
2231
2232    #[rstest]
2233    fn test_catalog_display_schema_uses_data_type() {
2234        let schema = catalog_display_schema(&NautilusDataType::QuoteTick).unwrap();
2235
2236        assert_eq!(schema.fields()[0].name(), "instrument_id");
2237        assert_eq!(schema.fields()[1].data_type(), &DataType::Float64);
2238        assert_eq!(schema.fields().last().unwrap().name(), "ts_init");
2239    }
2240
2241    #[rstest]
2242    fn test_custom_catalog_display_schema_uses_registered_schema() {
2243        let type_name = "CatalogDisplaySchemaCustom";
2244        let schema = Schema::new(vec![
2245            Field::new("value", DataType::Float64, false),
2246            Field::new(
2247                "ts_init",
2248                DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
2249                false,
2250            ),
2251        ]);
2252        ensure_arrow_registered(
2253            type_name,
2254            Arc::new(schema),
2255            Box::new(|_| unreachable!()),
2256            Box::new(|_, _| unreachable!()),
2257        )
2258        .unwrap();
2259
2260        let data_type = NautilusDataType::Custom {
2261            type_name: type_name.to_string(),
2262        };
2263        let raw_schema = catalog_raw_schema(&data_type).unwrap();
2264        let schema = catalog_display_schema(&data_type).unwrap();
2265        let fields = schema
2266            .fields()
2267            .iter()
2268            .map(|field| (field.name().as_str(), field.data_type()))
2269            .collect::<Vec<_>>();
2270
2271        assert_eq!(
2272            fields,
2273            [
2274                ("value", &DataType::Float64),
2275                (
2276                    "ts_init",
2277                    &DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
2278                ),
2279                ("data_type", &DataType::Utf8),
2280                ("identifier", &DataType::Utf8),
2281            ]
2282        );
2283        assert_eq!(raw_schema.fields().last().unwrap().name(), "identifier");
2284    }
2285
2286    #[rstest]
2287    fn test_custom_catalog_schema_rejects_unregistered_type() {
2288        let data_type = NautilusDataType::Custom {
2289            type_name: "UnregisteredCatalogDisplayCustom".to_string(),
2290        };
2291
2292        assert!(catalog_raw_schema(&data_type).is_err());
2293        assert!(catalog_display_schema(&data_type).is_err());
2294    }
2295
2296    #[rstest]
2297    fn test_custom_display_normalizes_utf8_view_to_utf8() {
2298        let batch = RecordBatch::try_new(
2299            Arc::new(Schema::new(vec![Field::new(
2300                "label",
2301                DataType::Utf8View,
2302                false,
2303            )])),
2304            vec![Arc::new(StringViewArray::from(vec!["value"]))],
2305        )
2306        .unwrap();
2307        let display = catalog_record_batch_to_display(
2308            &NautilusDataType::Custom {
2309                type_name: "ViewCustom".to_string(),
2310            },
2311            batch.schema().metadata(),
2312            &batch,
2313        )
2314        .unwrap();
2315
2316        assert_eq!(display.schema().field(0).data_type(), &DataType::Utf8);
2317        assert_eq!(
2318            display
2319                .column(0)
2320                .as_any()
2321                .downcast_ref::<StringArray>()
2322                .unwrap()
2323                .value(0),
2324            "value"
2325        );
2326    }
2327
2328    #[rstest]
2329    fn test_catalog_record_batch_to_display_converts_funding_rates() {
2330        let instrument_id = InstrumentId::from("BTCUSDT-PERP.BINANCE");
2331        let funding_rate = FundingRateUpdate::new(
2332            instrument_id,
2333            Decimal::from_str("0.000125").unwrap(),
2334            Some(480),
2335            None,
2336            UnixNanos::from(10),
2337            UnixNanos::from(11),
2338        );
2339        let metadata = funding_rate.metadata();
2340        let raw_batch = FundingRateUpdate::encode_batch(&metadata, &[funding_rate]).unwrap();
2341
2342        let display_batch = catalog_record_batch_to_display(
2343            &NautilusDataType::FundingRateUpdate,
2344            &metadata,
2345            &raw_batch,
2346        )
2347        .unwrap();
2348
2349        let mut expected_fields = funding_rate_schema().fields().to_vec();
2350        expected_fields.push(Arc::new(utf8_field(KEY_IDENTIFIER, true)));
2351        assert_eq!(
2352            display_batch.schema(),
2353            Arc::new(Schema::new(expected_fields))
2354        );
2355        let instrument_ids = display_batch
2356            .column(0)
2357            .as_any()
2358            .downcast_ref::<StringArray>()
2359            .unwrap();
2360        let rates = display_batch
2361            .column(1)
2362            .as_any()
2363            .downcast_ref::<Float64Array>()
2364            .unwrap();
2365        let intervals = display_batch
2366            .column(2)
2367            .as_any()
2368            .downcast_ref::<UInt64Array>()
2369            .unwrap();
2370        let next_funding_ns = display_batch
2371            .column(3)
2372            .as_any()
2373            .downcast_ref::<TimestampNanosecondArray>()
2374            .unwrap();
2375        let identifiers = display_batch
2376            .column(6)
2377            .as_any()
2378            .downcast_ref::<StringArray>()
2379            .unwrap();
2380
2381        assert_eq!(instrument_ids.value(0), "BTCUSDT-PERP.BINANCE");
2382        assert_eq!(rates.value(0), 0.000_125);
2383        assert_eq!(intervals.value(0), 480);
2384        assert!(next_funding_ns.is_null(0));
2385        assert_eq!(identifiers.value(0), "BTCUSDT-PERP.BINANCE");
2386    }
2387
2388    #[rstest]
2389    fn test_catalog_record_batch_to_display_rejects_precision_mismatch() {
2390        let metadata = HashMap::from([
2391            (KEY_INSTRUMENT_ID.to_string(), "AUD/USD.SIM".to_string()),
2392            (KEY_PRICE_PRECISION.to_string(), "5".to_string()),
2393            (KEY_SIZE_PRECISION.to_string(), "0".to_string()),
2394        ]);
2395        let schema = Arc::new(Schema::new_with_metadata(
2396            vec![
2397                Field::new("bid_price", DataType::FixedSizeBinary(1), false),
2398                Field::new("ask_price", DataType::FixedSizeBinary(1), false),
2399                Field::new("bid_size", DataType::FixedSizeBinary(1), false),
2400                Field::new("ask_size", DataType::FixedSizeBinary(1), false),
2401                Field::new("ts_event", DataType::UInt64, false),
2402                Field::new("ts_init", DataType::UInt64, false),
2403            ],
2404            metadata.clone(),
2405        ));
2406        let batch = RecordBatch::try_new(
2407            schema,
2408            vec![
2409                Arc::new(
2410                    FixedSizeBinaryArray::try_from_sparse_iter_with_size(
2411                        vec![Some([0_u8].as_slice())].into_iter(),
2412                        1,
2413                    )
2414                    .unwrap(),
2415                ),
2416                Arc::new(
2417                    FixedSizeBinaryArray::try_from_sparse_iter_with_size(
2418                        vec![Some([0_u8].as_slice())].into_iter(),
2419                        1,
2420                    )
2421                    .unwrap(),
2422                ),
2423                Arc::new(
2424                    FixedSizeBinaryArray::try_from_sparse_iter_with_size(
2425                        vec![Some([0_u8].as_slice())].into_iter(),
2426                        1,
2427                    )
2428                    .unwrap(),
2429                ),
2430                Arc::new(
2431                    FixedSizeBinaryArray::try_from_sparse_iter_with_size(
2432                        vec![Some([0_u8].as_slice())].into_iter(),
2433                        1,
2434                    )
2435                    .unwrap(),
2436                ),
2437                Arc::new(UInt64Array::from(vec![1])),
2438                Arc::new(UInt64Array::from(vec![1])),
2439            ],
2440        )
2441        .unwrap();
2442
2443        let error =
2444            catalog_record_batch_to_display(&NautilusDataType::QuoteTick, &metadata, &batch)
2445                .unwrap_err();
2446
2447        assert!(matches!(
2448            error,
2449            EncodingError::InvalidColumnType(
2450                "bid_price",
2451                _,
2452                DataType::Decimal128(38, 16),
2453                DataType::FixedSizeBinary(1)
2454            )
2455        ));
2456    }
2457
2458    #[cfg(feature = "arrow-display")]
2459    #[rstest]
2460    #[case::bar(NautilusDataType::Bar)]
2461    #[case::trade(NautilusDataType::TradeTick)]
2462    #[case::delta(NautilusDataType::OrderBookDelta)]
2463    #[case::close(NautilusDataType::InstrumentClose)]
2464    fn test_catalog_display_matches_typed_encoder(#[case] data_type: NautilusDataType) {
2465        let (raw, expected) = match data_type {
2466            NautilusDataType::Bar => {
2467                let value = stub_bar();
2468                (
2469                    Bar::encode_batch(&value.metadata(), &[value]).unwrap(),
2470                    encode_bars(&[value]).unwrap(),
2471                )
2472            }
2473            NautilusDataType::TradeTick => {
2474                let value = stub_trade_ethusdt_buy();
2475                (
2476                    TradeTick::encode_batch(&value.metadata(), &[value]).unwrap(),
2477                    encode_trades(&[value]).unwrap(),
2478                )
2479            }
2480            NautilusDataType::OrderBookDelta => {
2481                let value = stub_delta();
2482                (
2483                    OrderBookDelta::encode_batch(&value.metadata(), &[value]).unwrap(),
2484                    encode_deltas(&[value]).unwrap(),
2485                )
2486            }
2487            NautilusDataType::InstrumentClose => {
2488                let value = stub_instrument_close();
2489                (
2490                    InstrumentClose::encode_batch(&value.metadata(), &[value]).unwrap(),
2491                    encode_instrument_closes(&[value]).unwrap(),
2492                )
2493            }
2494            _ => unreachable!(),
2495        };
2496
2497        let displayed =
2498            catalog_record_batch_to_display(&data_type, raw.schema().metadata(), &raw).unwrap();
2499        let identifier = Arc::clone(displayed.column_by_name(KEY_IDENTIFIER).unwrap());
2500        let without_identifier = record_batch_without_identifier_column(displayed).unwrap();
2501
2502        assert_eq!(&identifier, raw.column_by_name(KEY_IDENTIFIER).unwrap());
2503        assert_eq!(without_identifier, expected);
2504    }
2505}