Skip to main content

nautilus_serialization/arrow/
mod.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Defines the Apache Arrow schema for Nautilus types.
17
18pub mod account_state;
19pub mod bar;
20pub mod catalog_display;
21pub mod close;
22pub mod custom;
23pub mod delta;
24pub mod depth;
25pub mod funding;
26pub mod greeks;
27pub mod index_price;
28pub mod instrument;
29pub mod instrument_status;
30pub mod json;
31pub mod legacy;
32pub mod mark_price;
33pub mod option_greeks;
34pub mod order_event;
35pub mod position_event;
36pub mod quote;
37pub mod report;
38pub mod snapshot;
39pub mod trade;
40
41mod depth_display;
42mod display_conversion;
43
44#[cfg(feature = "arrow-display")]
45pub mod display;
46
47#[cfg(test)]
48pub(crate) mod test_support;
49
50use std::{
51    borrow::Borrow,
52    collections::HashMap,
53    fmt::{Display, Write as FmtWrite},
54    io::{self, Write},
55    str::FromStr,
56    sync::Arc,
57};
58
59use arrow::{
60    array::{
61        Array, ArrayRef, BinaryArray, BinaryViewArray, Decimal128Array, DictionaryArray,
62        Int32Array, Int64Array, StringArray, StringBuilder, StringDictionaryBuilder,
63        StringViewArray, StructArray, TimestampNanosecondArray, UInt8Array, UInt32Array,
64        UInt64Array,
65    },
66    buffer::NullBuffer,
67    datatypes::{DataType, Field, Int8Type, Int32Type, Schema, TimeUnit},
68    error::ArrowError,
69    ipc::writer::StreamWriter,
70    record_batch::RecordBatch,
71};
72use nautilus_core::UnixNanos;
73use nautilus_model::{
74    data::{
75        Data, IndexPriceUpdate, InstrumentStatus, MarkPriceUpdate, bar::Bar,
76        close::InstrumentClose, delta::OrderBookDelta, depth::OrderBookDepth,
77        option_chain::OptionGreeks, quote::QuoteTick, trade::TradeTick,
78    },
79    enums::BookAction,
80    identifiers::InstrumentId,
81    types::{
82        Currency, Money, PRICE_ERROR, PRICE_UNDEF, Price, QUANTITY_UNDEF, Quantity,
83        fixed::{FIXED_PRECISION, FIXED_PRECISION_STANDARD},
84        money::MoneyRaw,
85        price::PriceRaw,
86        quantity::{QUANTITY_RAW_MAX, QuantityRaw},
87    },
88};
89#[cfg(feature = "python")]
90use pyo3::prelude::*;
91use rust_decimal::Decimal;
92use ustr::Ustr;
93
94pub use self::legacy::{
95    is_nautilus_legacy_schema, is_nautilus_timestamp_schema, is_timestamp_field,
96    normalize_legacy_fixed_columns, normalized_legacy_data_type, normalized_timestamp_type,
97};
98
99// Define metadata key constants
100pub const KEY_BAR_TYPE: &str = "bar_type";
101pub const KEY_IDENTIFIER: &str = "identifier";
102pub const KEY_INSTRUMENT_ID: &str = "instrument_id";
103pub const KEY_PRICE_PRECISION: &str = "price_precision";
104pub const KEY_SIZE_PRECISION: &str = "size_precision";
105
106pub(crate) fn parse_metadata(
107    metadata: &HashMap<String, String>,
108) -> Result<(InstrumentId, u8, u8), EncodingError> {
109    let instrument_id = metadata
110        .get(KEY_INSTRUMENT_ID)
111        .ok_or(EncodingError::MissingMetadata(KEY_INSTRUMENT_ID))?
112        .parse::<InstrumentId>()
113        .map_err(|e| EncodingError::ParseError(KEY_INSTRUMENT_ID, e.to_string()))?;
114    let price_precision = metadata
115        .get(KEY_PRICE_PRECISION)
116        .ok_or(EncodingError::MissingMetadata(KEY_PRICE_PRECISION))?
117        .parse::<u8>()
118        .map_err(|e| EncodingError::ParseError(KEY_PRICE_PRECISION, e.to_string()))?;
119    let size_precision = metadata
120        .get(KEY_SIZE_PRECISION)
121        .ok_or(EncodingError::MissingMetadata(KEY_SIZE_PRECISION))?
122        .parse::<u8>()
123        .map_err(|e| EncodingError::ParseError(KEY_SIZE_PRECISION, e.to_string()))?;
124    Ok((instrument_id, price_precision, size_precision))
125}
126pub const KEY_TYPE_NAME: &str = "type_name";
127pub const FIXED_DECIMAL_PRECISION: u8 = 38;
128pub const FIXED_DECIMAL_SCALE: i8 = 16;
129pub(crate) const EMPTY_DEPTH_PRECISION: (u8, u8) = (0, 0);
130
131/// Returns `metadata` with the catalog type discriminator set to `type_name`.
132#[must_use]
133pub fn metadata_with_type_name(
134    type_name: &str,
135    metadata: Option<HashMap<String, String>>,
136) -> HashMap<String, String> {
137    let mut metadata = metadata.unwrap_or_default();
138    metadata.insert(KEY_TYPE_NAME.to_string(), type_name.to_string());
139    metadata
140}
141
142/// Returns the open Arrow data type used for nanosecond instants.
143#[must_use]
144pub fn timestamp_data_type() -> DataType {
145    DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
146}
147
148/// Builds a UTC nanosecond timestamp array from model timestamp values.
149///
150/// # Errors
151///
152/// Returns an error if a value exceeds Arrow's signed timestamp range.
153pub fn timestamp_array(
154    values: impl IntoIterator<Item = u64>,
155) -> Result<TimestampNanosecondArray, ArrowError> {
156    optional_timestamp_array(values.into_iter().map(Some))
157}
158
159/// Builds a nullable UTC nanosecond timestamp array from model timestamp values.
160///
161/// # Errors
162///
163/// Returns an error if a value exceeds Arrow's signed timestamp range.
164pub fn optional_timestamp_array(
165    values: impl IntoIterator<Item = Option<u64>>,
166) -> Result<TimestampNanosecondArray, ArrowError> {
167    let values = values
168        .into_iter()
169        .map(|value| {
170            value
171                .map(|value| {
172                    i64::try_from(value).map_err(|_| {
173                        ArrowError::InvalidArgumentError(format!(
174                            "Nanosecond timestamp {value} exceeds Arrow's signed timestamp range"
175                        ))
176                    })
177                })
178                .transpose()
179        })
180        .collect::<Result<Vec<_>, _>>()?;
181    Ok(TimestampNanosecondArray::from(values).with_data_type(timestamp_data_type()))
182}
183
184/// Reads a non-negative Arrow nanosecond timestamp as the model's unsigned representation.
185///
186/// # Errors
187///
188/// Returns an error if the value is negative.
189pub fn decode_timestamp(
190    values: &TimestampNanosecondArray,
191    name: &'static str,
192    row: usize,
193) -> Result<u64, EncodingError> {
194    u64::try_from(values.value(row)).map_err(|_| {
195        EncodingError::ParseError(
196            name,
197            format!(
198                "row {row}: negative nanosecond timestamp {}",
199                values.value(row)
200            ),
201        )
202    })
203}
204
205/// Builds a record batch, converting unsigned nanosecond inputs for timestamp schema fields.
206///
207/// This keeps model encoders simple while ensuring their public Arrow batches use logical
208/// timestamp columns.
209///
210/// # Errors
211///
212/// Returns an error if a timestamp exceeds the signed Arrow range or the batch is otherwise
213/// invalid.
214pub fn record_batch_with_timestamps(
215    schema: Arc<Schema>,
216    columns: Vec<ArrayRef>,
217) -> Result<RecordBatch, ArrowError> {
218    validate_encode_precisions(schema.metadata())?;
219
220    if columns.len() != schema.fields().len() {
221        return Err(ArrowError::InvalidArgumentError(format!(
222            "number of columns({}) must match number of fields({}) in schema",
223            columns.len(),
224            schema.fields().len()
225        )));
226    }
227
228    let columns = schema
229        .fields()
230        .iter()
231        .zip(columns)
232        .map(|(field, column)| {
233            if field.data_type() != &timestamp_data_type()
234                || column.data_type() != &DataType::UInt64
235            {
236                return Ok(column);
237            }
238            timestamp_column(field, column.as_ref())
239        })
240        .collect::<Result<Vec<_>, ArrowError>>()?;
241    RecordBatch::try_new(schema, columns)
242}
243
244// Rejects batch metadata whose precisions exceed the catalog's uniform decimal scale. Defi
245// precisions (for example wei at 17 or 18) store raws at their own native scale, so encoding
246// them bit-for-bit into `Decimal128(38, 16)` columns would inflate the externally visible
247// values; mirror the SBE and custom-data macro encode guards and fail the write instead.
248fn validate_encode_precisions(metadata: &HashMap<String, String>) -> Result<(), ArrowError> {
249    for key in [KEY_PRICE_PRECISION, KEY_SIZE_PRECISION] {
250        if let Some(value) = metadata.get(key)
251            && let Ok(precision) = value.parse::<u8>()
252            && precision > FIXED_DECIMAL_SCALE as u8
253        {
254            return Err(ArrowError::InvalidArgumentError(format!(
255                "Metadata '{key}' is {precision}, maximum supported catalog scale is {FIXED_DECIMAL_SCALE}"
256            )));
257        }
258    }
259    Ok(())
260}
261
262fn timestamp_column(field: &Field, column: &dyn Array) -> Result<ArrayRef, ArrowError> {
263    let values = column
264        .as_any()
265        .downcast_ref::<UInt64Array>()
266        .ok_or_else(|| ArrowError::CastError(format!("Column '{}' is not UInt64", field.name())))?;
267    let timestamps = optional_timestamp_array(
268        (0..values.len()).map(|row| (!values.is_null(row)).then(|| values.value(row))),
269    )?;
270    Ok(Arc::new(timestamps) as ArrayRef)
271}
272
273/// Converts logical timestamp columns to unsigned nanoseconds for existing model decoders.
274///
275/// # Errors
276///
277/// Returns an error if timestamps are negative or arrays are invalid.
278pub fn record_batch_with_u64_timestamps(batch: &RecordBatch) -> Result<RecordBatch, EncodingError> {
279    let mut changed = false;
280    let mut fields = Vec::with_capacity(batch.num_columns());
281    let mut columns = Vec::with_capacity(batch.num_columns());
282
283    for (field, column) in batch.schema().fields().iter().zip(batch.columns()) {
284        if field.data_type() != &timestamp_data_type() {
285            fields.push(Arc::clone(field));
286            columns.push(Arc::clone(column));
287            continue;
288        }
289        let values = column
290            .as_any()
291            .downcast_ref::<TimestampNanosecondArray>()
292            .ok_or_else(|| {
293                EncodingError::ParseError(
294                    "timestamp",
295                    format!("Column '{}' is not TimestampNanosecond", field.name()),
296                )
297            })?;
298        let values = (0..values.len())
299            .map(|row| {
300                if values.is_null(row) {
301                    Ok(None)
302                } else {
303                    decode_timestamp(values, "timestamp", row).map(Some)
304                }
305            })
306            .collect::<Result<Vec<_>, _>>()?;
307        fields.push(Arc::new(
308            field.as_ref().clone().with_data_type(DataType::UInt64),
309        ));
310        columns.push(Arc::new(UInt64Array::from(values)) as ArrayRef);
311        changed = true;
312    }
313
314    if !changed {
315        return Ok(batch.clone());
316    }
317    RecordBatch::try_new(
318        Arc::new(Schema::new_with_metadata(
319            fields,
320            batch.schema().metadata().clone(),
321        )),
322        columns,
323    )
324    .map_err(EncodingError::from)
325}
326
327/// Returns the open Arrow data type used for enum-valued catalog columns.
328#[must_use]
329pub fn enum_dictionary_data_type() -> DataType {
330    DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8))
331}
332
333/// Builds a compact dictionary array containing enum display names.
334///
335/// # Errors
336///
337/// Returns an error if the number of distinct values exceeds the `Int8` key range.
338pub fn enum_dictionary_array(
339    values: impl IntoIterator<Item = impl Display>,
340) -> Result<DictionaryArray<Int8Type>, ArrowError> {
341    let mut builder = StringDictionaryBuilder::<Int8Type>::new();
342    for value in values {
343        builder.append(value.to_string())?;
344    }
345    Ok(builder.finish())
346}
347
348/// Returns the open Arrow data type used for monetary values.
349#[must_use]
350pub fn money_data_type() -> DataType {
351    DataType::Struct(
352        vec![
353            Field::new("amount", fixed_decimal_data_type(), false),
354            Field::new(
355                "currency",
356                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
357                false,
358            ),
359        ]
360        .into(),
361    )
362}
363
364/// Builds a nullable struct array containing monetary amounts and currencies.
365///
366/// # Errors
367///
368/// Returns an error if an amount or dictionary value cannot be represented.
369pub fn money_array(
370    values: impl IntoIterator<Item = Option<Money>>,
371) -> Result<StructArray, ArrowError> {
372    let mut amounts = Vec::new();
373    let mut currencies = StringDictionaryBuilder::<Int32Type>::new();
374    let mut validity = Vec::new();
375
376    for value in values {
377        if let Some(value) = value {
378            if value.currency.precision > FIXED_DECIMAL_SCALE as u8 {
379                return Err(ArrowError::InvalidArgumentError(format!(
380                    "Money currency precision {} exceeds catalog scale {FIXED_DECIMAL_SCALE}",
381                    value.currency.precision,
382                )));
383            }
384
385            amounts.push(money_raw_to_decimal(value.raw()));
386            currencies.append(value.currency.to_string())?;
387            validity.push(true);
388        } else {
389            amounts.push(0);
390            currencies.append_null();
391            validity.push(false);
392        }
393    }
394    let amounts = Decimal128Array::from(amounts)
395        .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)?;
396    StructArray::try_new(
397        match money_data_type() {
398            DataType::Struct(fields) => fields,
399            _ => unreachable!("money data type is a struct"),
400        },
401        vec![Arc::new(amounts), Arc::new(currencies.finish())],
402        Some(NullBuffer::from(validity)),
403    )
404}
405
406/// Encodes a Rust decimal at the catalog's scale.
407///
408/// # Errors
409///
410/// Returns an error if the value has more than 16 decimal places or cannot be rescaled exactly. The
411/// error names `field`.
412pub fn decimal_to_arrow(value: &Decimal, field: &'static str) -> Result<i128, ArrowError> {
413    let value = value.normalize();
414    let scale = value.scale();
415    if scale > FIXED_DECIMAL_SCALE as u32 {
416        return Err(ArrowError::InvalidArgumentError(format!(
417            "Decimal field '{field}' has scale {scale}, maximum supported scale is {FIXED_DECIMAL_SCALE}"
418        )));
419    }
420    let rescaled = value
421        .mantissa()
422        .checked_mul(10_i128.pow(FIXED_DECIMAL_SCALE as u32 - scale))
423        .ok_or_else(|| {
424            ArrowError::InvalidArgumentError(format!(
425                "Decimal field '{field}' cannot be represented as Decimal128(38, 16)"
426            ))
427        })?;
428    let max = Decimal::MAX.mantissa();
429    if rescaled < -max || rescaled > max {
430        return Err(ArrowError::InvalidArgumentError(format!(
431            "Decimal field '{field}' exceeds the rust_decimal 96-bit range after rescaling"
432        )));
433    }
434    Ok(rescaled)
435}
436
437/// Decodes a Rust decimal from the catalog's scale.
438///
439/// # Errors
440///
441/// Returns an error if the value is NULL or outside `rust_decimal`'s range.
442pub fn decode_decimal(
443    values: &Decimal128Array,
444    field: &'static str,
445    row: usize,
446) -> Result<Decimal, EncodingError> {
447    if values.is_null(row) {
448        return Err(EncodingError::ParseError(
449            field,
450            format!("row {row}: required decimal is null"),
451        ));
452    }
453    Decimal::try_from_i128_with_scale(values.value(row), FIXED_DECIMAL_SCALE as u32)
454        .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")))
455}
456
457/// Decodes a monetary value from its Arrow struct representation.
458///
459/// # Errors
460///
461/// Returns an error if the struct is NULL, malformed, or outside the model range.
462pub fn decode_money(
463    values: &StructArray,
464    field: &'static str,
465    row: usize,
466) -> Result<Money, EncodingError> {
467    if values.is_null(row) {
468        return Err(EncodingError::ParseError(
469            field,
470            format!("row {row}: required money is null"),
471        ));
472    }
473    let amounts = values
474        .column_by_name("amount")
475        .and_then(|array| array.as_any().downcast_ref::<Decimal128Array>())
476        .ok_or_else(|| {
477            EncodingError::ParseError(field, "money amount must be Decimal128(38, 16)".to_string())
478        })?;
479    let currencies = values
480        .column_by_name("currency")
481        .and_then(|array| StringColumnRef::try_from_array(array.as_ref()))
482        .ok_or_else(|| {
483            EncodingError::ParseError(
484                field,
485                "money currency must be Dictionary<Int32, Utf8>".to_string(),
486            )
487        })?;
488    let raw = decimal_to_money_raw(amounts.value(row), field, row)?;
489    let currency_code = currencies.value(row);
490    let currency = Currency::from_str(currency_code).map_err(|e| {
491        EncodingError::ParseError(
492            field,
493            format!(
494                "row {row}: currency '{currency_code}' must be registered before decoding Money: {e}"
495            ),
496        )
497    })?;
498
499    if currency.precision > FIXED_DECIMAL_SCALE as u8 {
500        return Err(EncodingError::ParseError(
501            field,
502            format!(
503                "row {row}: Money currency precision {} exceeds catalog scale {FIXED_DECIMAL_SCALE}",
504                currency.precision,
505            ),
506        ));
507    }
508
509    Money::from_raw_checked(raw, currency)
510        .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")))
511}
512
513#[allow(
514    clippy::useless_conversion,
515    reason = "MoneyRaw is i64 or i128 depending on model feature unification"
516)]
517fn money_raw_to_decimal(raw: MoneyRaw) -> i128 {
518    let mut decimal = i128::from(raw);
519    if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
520        decimal *= STANDARD_TO_DECIMAL_SCALE;
521    }
522    decimal
523}
524
525fn decimal_to_money_raw(
526    value: i128,
527    field: &'static str,
528    row: usize,
529) -> Result<MoneyRaw, EncodingError> {
530    decimal_to_raw(value, field, row, "MoneyRaw")
531}
532
533const STANDARD_TO_DECIMAL_SCALE: i128 =
534    10_i128.pow((FIXED_DECIMAL_SCALE as u8 - FIXED_PRECISION_STANDARD) as u32);
535
536#[derive(thiserror::Error, Debug)]
537pub enum DataStreamingError {
538    #[error("I/O error: {0}")]
539    IoError(#[from] io::Error),
540    #[error("Arrow error: {0}")]
541    ArrowError(#[from] arrow::error::ArrowError),
542    #[cfg(feature = "python")]
543    #[error("Python error: {0}")]
544    PythonError(#[from] PyErr),
545}
546
547#[derive(thiserror::Error, Debug)]
548pub enum EncodingError {
549    #[error("Empty data")]
550    EmptyData,
551    #[error(
552        "Mixed metadata at row {index}; encode each instrument, bar type, or precision separately"
553    )]
554    MixedMetadata { index: usize },
555    #[error("Missing metadata key: `{0}`")]
556    MissingMetadata(&'static str),
557    #[error("Missing data column: `{0}` at index {1}")]
558    MissingColumn(&'static str, usize),
559    #[error("Error parsing `{0}`: {1}")]
560    ParseError(&'static str, String),
561    #[error("Invalid column type `{0}` at index {1}: expected {2}, found {3}")]
562    InvalidColumnType(&'static str, usize, DataType, DataType),
563    #[error("Arrow error: {0}")]
564    ArrowError(#[from] arrow::error::ArrowError),
565}
566
567/// Returns the open fixed-point Arrow data type used by catalog write schemas.
568#[must_use]
569pub const fn fixed_decimal_data_type() -> DataType {
570    DataType::Decimal128(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
571}
572
573/// Returns a UTF-8 field annotated with the canonical Arrow JSON extension.
574#[must_use]
575pub fn json_string_field(name: impl Into<String>, nullable: bool) -> Field {
576    Field::new(name, DataType::Utf8, nullable).with_metadata(HashMap::from([
577        ("ARROW:extension:name".to_string(), "arrow.json".to_string()),
578        ("ARROW:extension:metadata".to_string(), String::new()),
579    ]))
580}
581
582/// Returns whether a field carries the canonical Arrow JSON extension.
583#[must_use]
584pub fn is_json_string_field(field: &Field) -> bool {
585    field.extension_type_name() == Some("arrow.json")
586}
587
588/// Encodes a model price raw value at the catalog's uniform decimal scale.
589///
590/// # Errors
591///
592/// Returns an error if `raw` is [`PRICE_ERROR`].
593pub fn price_raw_to_decimal(
594    raw: PriceRaw,
595    field: &'static str,
596) -> Result<Option<i128>, ArrowError> {
597    if raw == PRICE_UNDEF {
598        return Ok(None);
599    }
600
601    if raw == PRICE_ERROR {
602        return Err(ArrowError::InvalidArgumentError(format!(
603            "Price field '{field}' contains PRICE_ERROR raw value {raw}"
604        )));
605    }
606
607    #[allow(
608        clippy::useless_conversion,
609        reason = "PriceRaw is i64 or i128 depending on model feature unification"
610    )]
611    let mut decimal = i128::from(raw);
612    if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
613        decimal *= STANDARD_TO_DECIMAL_SCALE;
614    }
615
616    Ok(Some(decimal))
617}
618
619/// Encodes a model quantity raw value at the catalog's uniform decimal scale.
620///
621/// # Errors
622///
623/// Returns an error if a non-sentinel quantity does not fit in Arrow's signed decimal
624/// representation.
625pub fn quantity_raw_to_decimal(
626    raw: QuantityRaw,
627    field: &'static str,
628) -> Result<Option<i128>, ArrowError> {
629    if raw == QUANTITY_UNDEF {
630        return Ok(None);
631    }
632
633    if raw > QUANTITY_RAW_MAX {
634        return Err(ArrowError::InvalidArgumentError(format!(
635            "Quantity field '{field}' raw value {raw} exceeds QUANTITY_RAW_MAX={QUANTITY_RAW_MAX}"
636        )));
637    }
638
639    #[allow(
640        clippy::unnecessary_fallible_conversions,
641        reason = "QuantityRaw is u64 or u128 depending on model feature unification"
642    )]
643    let mut decimal = i128::try_from(raw).map_err(|_| {
644        ArrowError::InvalidArgumentError(format!(
645            "Quantity field '{field}' raw value {raw} exceeds Decimal128 range"
646        ))
647    })?;
648
649    if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
650        decimal *= STANDARD_TO_DECIMAL_SCALE;
651    }
652
653    Ok(Some(decimal))
654}
655
656/// Builds a scale-16 decimal array from model price raw values.
657///
658/// # Errors
659///
660/// Returns an error if the values do not fit the declared decimal type.
661pub fn price_decimal_array(
662    values: impl IntoIterator<Item = PriceRaw>,
663    field: &'static str,
664) -> Result<Decimal128Array, ArrowError> {
665    let values = values
666        .into_iter()
667        .map(|raw| price_raw_to_decimal(raw, field))
668        .collect::<Result<Vec<_>, _>>()?;
669    Decimal128Array::from(values)
670        .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
671}
672
673/// Builds a scale-16 decimal array from model quantity raw values.
674///
675/// # Errors
676///
677/// Returns an error if a value does not fit or the declared decimal type is invalid.
678pub fn quantity_decimal_array(
679    values: impl IntoIterator<Item = QuantityRaw>,
680    field: &'static str,
681) -> Result<Decimal128Array, ArrowError> {
682    let values = values
683        .into_iter()
684        .map(|raw| quantity_raw_to_decimal(raw, field))
685        .collect::<Result<Vec<_>, _>>()?;
686    Decimal128Array::from(values)
687        .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
688}
689
690/// Builds a scale-16 decimal array from price raw values of a type with no NULL sentinel.
691///
692/// # Errors
693///
694/// Returns an error if a value is `PRICE_UNDEF` or does not fit the declared decimal type, so the
695/// required decoders can read back every written value.
696pub fn required_price_decimal_array(
697    values: impl IntoIterator<Item = PriceRaw>,
698    field: &'static str,
699) -> Result<Decimal128Array, ArrowError> {
700    let values = values
701        .into_iter()
702        .map(|raw| {
703            price_raw_to_decimal(raw, field)?.ok_or_else(|| {
704                ArrowError::InvalidArgumentError(format!(
705                    "Price field '{field}' contains PRICE_UNDEF, which has no sentinel encoding for this type"
706                ))
707            })
708        })
709        .collect::<Result<Vec<_>, _>>()?;
710    Decimal128Array::from(values)
711        .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
712}
713
714/// Builds a scale-16 decimal array from quantity raw values of a type with no NULL sentinel.
715///
716/// # Errors
717///
718/// Returns an error if a value is `QUANTITY_UNDEF` or does not fit the declared decimal type, so
719/// the required decoders can read back every written value.
720pub fn required_quantity_decimal_array(
721    values: impl IntoIterator<Item = QuantityRaw>,
722    field: &'static str,
723) -> Result<Decimal128Array, ArrowError> {
724    let values = values
725        .into_iter()
726        .map(|raw| {
727            quantity_raw_to_decimal(raw, field)?.ok_or_else(|| {
728                ArrowError::InvalidArgumentError(format!(
729                    "Quantity field '{field}' contains QUANTITY_UNDEF, which has no sentinel encoding for this type"
730                ))
731            })
732        })
733        .collect::<Result<Vec<_>, _>>()?;
734    Decimal128Array::from(values)
735        .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
736}
737
738fn decimal_to_price_raw(
739    value: i128,
740    field: &'static str,
741    row: usize,
742) -> Result<PriceRaw, EncodingError> {
743    decimal_to_raw(value, field, row, "PriceRaw")
744}
745
746fn decimal_to_quantity_raw(
747    value: i128,
748    field: &'static str,
749    row: usize,
750) -> Result<QuantityRaw, EncodingError> {
751    decimal_to_raw(value, field, row, "QuantityRaw")
752}
753
754fn decimal_to_raw<T: TryFrom<i128>>(
755    value: i128,
756    field: &'static str,
757    row: usize,
758    raw_type: &'static str,
759) -> Result<T, EncodingError> {
760    let raw_value = if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
761        if value % STANDARD_TO_DECIMAL_SCALE != 0 {
762            return Err(EncodingError::ParseError(
763                field,
764                format!(
765                    "row {row}: decimal value {value} has nonzero digits beyond build precision 9"
766                ),
767            ));
768        }
769        value / STANDARD_TO_DECIMAL_SCALE
770    } else {
771        value
772    };
773
774    T::try_from(raw_value).map_err(|_| {
775        EncodingError::ParseError(
776            field,
777            format!("row {row}: decimal value {value} exceeds {raw_type} range"),
778        )
779    })
780}
781
782/// Decodes a price from a nullable scale-16 decimal column.
783///
784/// # Errors
785///
786/// Returns an error if the value cannot be represented by this build.
787pub fn decode_decimal_price(
788    values: &Decimal128Array,
789    precision: u8,
790    field: &'static str,
791    row: usize,
792) -> Result<Price, EncodingError> {
793    if values.is_null(row) {
794        return Price::from_raw_checked(PRICE_UNDEF, 0)
795            .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")));
796    }
797
798    let raw = decimal_to_price_raw(values.value(row), field, row)?;
799    Price::from_raw_checked(raw, precision)
800        .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")))
801}
802
803/// Decodes a required price from a scale-16 decimal column.
804///
805/// # Errors
806///
807/// Returns an error if the value is null or cannot be represented by this build.
808pub fn decode_required_decimal_price(
809    values: &Decimal128Array,
810    precision: u8,
811    field: &'static str,
812    row: usize,
813) -> Result<Price, EncodingError> {
814    if values.is_null(row) {
815        return Err(EncodingError::ParseError(
816            field,
817            format!("row {row}: required price is null"),
818        ));
819    }
820    decode_decimal_price(values, precision, field, row)
821}
822
823/// Decodes a quantity from a nullable scale-16 decimal column.
824///
825/// # Errors
826///
827/// Returns an error if the value cannot be represented by this build.
828pub fn decode_decimal_quantity(
829    values: &Decimal128Array,
830    precision: u8,
831    field: &'static str,
832    row: usize,
833) -> Result<Quantity, EncodingError> {
834    if values.is_null(row) {
835        return Quantity::from_raw_checked(QUANTITY_UNDEF, 0)
836            .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")));
837    }
838
839    let raw = decimal_to_quantity_raw(values.value(row), field, row)?;
840    Quantity::from_raw_checked(raw, precision)
841        .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")))
842}
843
844/// Decodes a required quantity from a scale-16 decimal column.
845///
846/// # Errors
847///
848/// Returns an error if the value is null or cannot be represented by this build.
849pub fn decode_required_decimal_quantity(
850    values: &Decimal128Array,
851    precision: u8,
852    field: &'static str,
853    row: usize,
854) -> Result<Quantity, EncodingError> {
855    if values.is_null(row) {
856        return Err(EncodingError::ParseError(
857            field,
858            format!("row {row}: required quantity is null"),
859        ));
860    }
861    decode_decimal_quantity(values, precision, field, row)
862}
863
864/// Returns a required timestamp value, naming the field and row on NULL.
865///
866/// # Errors
867///
868/// Returns an error if the value is null.
869pub fn decode_required_timestamp(
870    values: &UInt64Array,
871    field: &'static str,
872    row: usize,
873) -> Result<UnixNanos, EncodingError> {
874    if values.is_null(row) {
875        return Err(EncodingError::ParseError(
876            field,
877            format!("row {row}: required timestamp is null"),
878        ));
879    }
880    Ok(values.value(row).into())
881}
882
883pub(crate) fn decode_required_u64(
884    values: &UInt64Array,
885    field: &'static str,
886    row: usize,
887) -> Result<u64, EncodingError> {
888    if values.is_null(row) {
889        return Err(EncodingError::ParseError(
890            field,
891            format!("row {row}: required integer is null"),
892        ));
893    }
894    Ok(values.value(row))
895}
896
897pub(crate) fn decode_required_u8(
898    values: &UInt8Array,
899    field: &'static str,
900    row: usize,
901) -> Result<u8, EncodingError> {
902    if values.is_null(row) {
903        return Err(EncodingError::ParseError(
904            field,
905            format!("row {row}: required integer is null"),
906        ));
907    }
908    Ok(values.value(row))
909}
910
911pub(crate) fn validate_required_column(
912    values: &dyn Array,
913    field: &'static str,
914) -> Result<(), EncodingError> {
915    if let Some(row) = first_logical_null(values) {
916        return Err(EncodingError::ParseError(
917            field,
918            format!("row {row}: required value is null"),
919        ));
920    }
921
922    Ok(())
923}
924
925pub(crate) fn first_logical_null(values: &dyn Array) -> Option<usize> {
926    values
927        .logical_nulls()
928        .and_then(|nulls| nulls.iter().position(|valid| !valid))
929}
930
931#[cfg(test)]
932fn get_raw_price(value: i128) -> PriceRaw {
933    decimal_to_price_raw(value, "test", 0).expect("Decimal price must fit the current build")
934}
935
936#[cfg(test)]
937fn get_raw_quantity(value: i128) -> QuantityRaw {
938    decimal_to_quantity_raw(value, "test", 0).expect("Decimal quantity must fit the current build")
939}
940
941/// Provides Apache Arrow schema definitions for data types.
942pub trait ArrowSchemaProvider {
943    /// Returns the Arrow schema for this type with optional metadata.
944    fn get_schema(metadata: Option<HashMap<String, String>>) -> Schema;
945
946    /// Returns a map of field names to their Arrow data types.
947    #[must_use]
948    fn get_schema_map() -> HashMap<String, String> {
949        let schema = Self::get_schema(None);
950        let mut map = HashMap::new();
951
952        for field in schema.fields() {
953            let name = field.name().clone();
954            let data_type = format!("{:?}", field.data_type());
955            map.insert(name, data_type);
956        }
957        map
958    }
959}
960
961/// Encodes data types to Apache Arrow RecordBatch format.
962pub trait EncodeToRecordBatch
963where
964    Self: Sized + ArrowSchemaProvider,
965{
966    /// Encodes a batch of values into an Arrow `RecordBatch` using the provided metadata.
967    ///
968    /// # Errors
969    ///
970    /// Returns an error if the encoding fails.
971    fn encode_batch<T>(
972        metadata: &HashMap<String, String>,
973        data: &[T],
974    ) -> Result<RecordBatch, ArrowError>
975    where
976        T: Borrow<Self>;
977
978    /// Returns the metadata for this data element.
979    fn metadata(&self) -> HashMap<String, String>;
980
981    /// Returns the metadata selected for a chunk.
982    ///
983    /// The default uses the first element. Implementations may override this when leading sentinel
984    /// values do not carry meaningful metadata.
985    ///
986    /// # Panics
987    ///
988    /// Panics if `chunk` is empty.
989    fn chunk_metadata<T>(chunk: &[T]) -> HashMap<String, String>
990    where
991        T: Borrow<Self>,
992    {
993        chunk
994            .first()
995            .map(|item| item.borrow().metadata())
996            .expect("Chunk must contain at least one element to encode")
997    }
998
999    /// Returns whether this element is compatible with metadata selected for its chunk.
1000    fn matches_chunk_metadata(&self, metadata: &HashMap<String, String>) -> bool {
1001        self.metadata() == *metadata
1002    }
1003}
1004
1005/// Returns the catalog row identifier from Arrow schema metadata.
1006///
1007/// Bars use `bar_type`; all other built-in catalog types use `instrument_id`.
1008/// Custom data can pass an explicit identifier to [`record_batch_with_identifier_column`].
1009#[must_use]
1010pub fn catalog_identifier_from_metadata(metadata: &HashMap<String, String>) -> Option<String> {
1011    metadata
1012        .get(KEY_BAR_TYPE)
1013        .cloned()
1014        .or_else(|| metadata.get(KEY_INSTRUMENT_ID).cloned())
1015}
1016
1017/// Builds a schema with the catalog `identifier` column appended if absent.
1018#[must_use]
1019pub fn schema_with_identifier_column(schema: &Schema) -> Schema {
1020    if schema.index_of(KEY_IDENTIFIER).is_ok() {
1021        return schema.clone();
1022    }
1023
1024    let mut fields = schema.fields().iter().cloned().collect::<Vec<_>>();
1025    fields.push(Arc::new(Field::new(KEY_IDENTIFIER, DataType::Utf8, true)));
1026
1027    Schema::new_with_metadata(fields, schema.metadata().clone())
1028}
1029
1030/// Builds a schema without the catalog `identifier` column.
1031#[must_use]
1032pub fn schema_without_identifier_column(schema: &Schema) -> Schema {
1033    let Ok(identifier_index) = schema.index_of(KEY_IDENTIFIER) else {
1034        return schema.clone();
1035    };
1036
1037    let fields = schema
1038        .fields()
1039        .iter()
1040        .enumerate()
1041        .filter_map(|(index, field)| (index != identifier_index).then_some(Arc::clone(field)))
1042        .collect::<Vec<_>>();
1043
1044    Schema::new_with_metadata(fields, schema.metadata().clone())
1045}
1046
1047/// Adds the catalog `identifier` column used by table-oriented catalog storage.
1048///
1049/// The column is nullable so custom data without a `DataType.identifier()` can
1050/// still be stored in the same type table.
1051///
1052/// # Errors
1053///
1054/// Returns an error if the record batch cannot be rebuilt.
1055pub fn record_batch_with_identifier_column(
1056    batch: RecordBatch,
1057    identifier: Option<&str>,
1058) -> Result<RecordBatch, ArrowError> {
1059    let identifier_values = vec![identifier.map(ToString::to_string); batch.num_rows()];
1060    record_batch_with_identifier_values(batch, identifier_values)
1061}
1062
1063/// Adds the catalog `identifier` column with one identifier value per row.
1064///
1065/// # Errors
1066///
1067/// Returns an error if the number of identifier values differs from the record batch row count or
1068/// the record batch cannot be rebuilt.
1069pub fn record_batch_with_identifier_values(
1070    batch: RecordBatch,
1071    identifier_values: Vec<Option<String>>,
1072) -> Result<RecordBatch, ArrowError> {
1073    if batch.schema().index_of(KEY_IDENTIFIER).is_ok() {
1074        return Ok(batch);
1075    }
1076
1077    if identifier_values.len() != batch.num_rows() {
1078        return Err(ArrowError::InvalidArgumentError(format!(
1079            "identifier values length {} does not match record batch row count {}",
1080            identifier_values.len(),
1081            batch.num_rows()
1082        )));
1083    }
1084
1085    let schema = schema_with_identifier_column(batch.schema().as_ref());
1086    let mut columns = batch.columns().to_vec();
1087    columns.push(Arc::new(StringArray::from(identifier_values)));
1088
1089    RecordBatch::try_new(Arc::new(schema), columns)
1090}
1091
1092/// Builds a string array for catalog identifiers without collecting intermediate strings.
1093#[must_use]
1094pub fn identifier_array_from_display<T: Display>(
1095    identifiers: impl IntoIterator<Item = T>,
1096) -> StringArray {
1097    let mut builder = StringBuilder::new();
1098    let mut scratch = String::new();
1099
1100    for identifier in identifiers {
1101        scratch.clear();
1102        write!(&mut scratch, "{identifier}").expect("writing to String should not fail");
1103        builder.append_value(scratch.as_str());
1104    }
1105
1106    builder.finish()
1107}
1108
1109/// Drops the catalog `identifier` column when writing legacy per-identifier formats.
1110///
1111/// # Errors
1112///
1113/// Returns an error if the record batch cannot be rebuilt.
1114pub fn record_batch_without_identifier_column(
1115    mut batch: RecordBatch,
1116) -> Result<RecordBatch, ArrowError> {
1117    let Ok(identifier_index) = batch.schema().index_of(KEY_IDENTIFIER) else {
1118        return Ok(batch);
1119    };
1120
1121    batch.remove_column(identifier_index);
1122    Ok(batch)
1123}
1124
1125/// Decodes data types from Apache Arrow RecordBatch format.
1126pub trait DecodeFromRecordBatch
1127where
1128    Self: Sized + Into<Data> + ArrowSchemaProvider,
1129{
1130    /// Decodes a `RecordBatch` into a vector of values of the implementing type, using the provided metadata.
1131    ///
1132    /// # Errors
1133    ///
1134    /// Returns an error if the decoding fails.
1135    fn decode_batch(
1136        metadata: &HashMap<String, String>,
1137        record_batch: RecordBatch,
1138    ) -> Result<Vec<Self>, EncodingError>;
1139}
1140
1141/// Decodes strongly typed values from Apache Arrow RecordBatch format.
1142pub trait DecodeTypedFromRecordBatch
1143where
1144    Self: Sized + ArrowSchemaProvider,
1145{
1146    /// Decodes a `RecordBatch` into a vector of values of the implementing type.
1147    ///
1148    /// # Errors
1149    ///
1150    /// Returns an error if the decoding fails.
1151    fn decode_typed_batch(
1152        metadata: &HashMap<String, String>,
1153        record_batch: RecordBatch,
1154    ) -> Result<Vec<Self>, EncodingError>;
1155}
1156
1157impl<T> DecodeTypedFromRecordBatch for T
1158where
1159    T: DecodeFromRecordBatch,
1160{
1161    fn decode_typed_batch(
1162        metadata: &HashMap<String, String>,
1163        record_batch: RecordBatch,
1164    ) -> Result<Vec<Self>, EncodingError> {
1165        Self::decode_batch(metadata, record_batch)
1166    }
1167}
1168
1169/// Decodes raw Data objects from Apache Arrow RecordBatch format.
1170pub trait DecodeDataFromRecordBatch
1171where
1172    Self: Sized + ArrowSchemaProvider,
1173{
1174    /// Decodes a `RecordBatch` into raw `Data` values, using the provided metadata.
1175    ///
1176    /// # Errors
1177    ///
1178    /// Returns an error if the decoding fails.
1179    fn decode_data_batch(
1180        metadata: &HashMap<String, String>,
1181        record_batch: RecordBatch,
1182    ) -> Result<Vec<Data>, EncodingError>;
1183}
1184
1185/// Writes RecordBatch data to output streams.
1186pub trait WriteStream {
1187    /// Writes a `RecordBatch` to the implementing output stream.
1188    ///
1189    /// # Errors
1190    ///
1191    /// Returns an error if writing or finishing the stream fails.
1192    fn write(&mut self, record_batch: &RecordBatch) -> Result<(), DataStreamingError>;
1193}
1194
1195impl<T: Write> WriteStream for T {
1196    fn write(&mut self, record_batch: &RecordBatch) -> Result<(), DataStreamingError> {
1197        let mut writer = StreamWriter::try_new(self, &record_batch.schema())?;
1198        writer.write(record_batch)?;
1199        writer.finish()?;
1200        Ok(())
1201    }
1202}
1203
1204/// Extracts a string column, accepting both Utf8 (`StringArray`) and Utf8View (`StringViewArray`).
1205/// Parquet may return Utf8View when reading, so this handles both formats.
1206///
1207/// # Errors
1208///
1209/// Returns an error if:
1210/// - `column_index` is out of range.
1211/// - The column type is neither Utf8 nor Utf8View.
1212pub fn extract_column_string<'a>(
1213    cols: &'a [ArrayRef],
1214    column_key: &'static str,
1215    column_index: usize,
1216) -> Result<StringColumnRef<'a>, EncodingError> {
1217    let column_values = cols
1218        .get(column_index)
1219        .ok_or(EncodingError::MissingColumn(column_key, column_index))?;
1220    StringColumnRef::try_from_array(column_values.as_ref()).ok_or_else(|| {
1221        EncodingError::InvalidColumnType(
1222            column_key,
1223            column_index,
1224            DataType::Utf8,
1225            column_values.data_type().clone(),
1226        )
1227    })
1228}
1229
1230/// Reference to a string column in a supported Arrow string representation.
1231#[derive(Debug)]
1232pub enum StringColumnRef<'a> {
1233    DictionaryInt8(&'a DictionaryArray<Int8Type>, &'a StringArray),
1234    DictionaryInt32(&'a DictionaryArray<Int32Type>, &'a StringArray),
1235    Utf8(&'a StringArray),
1236    Utf8View(&'a StringViewArray),
1237}
1238
1239impl StringColumnRef<'_> {
1240    /// Returns the number of rows.
1241    #[must_use]
1242    pub fn len(&self) -> usize {
1243        match self {
1244            Self::DictionaryInt8(array, _) => array.len(),
1245            Self::DictionaryInt32(array, _) => array.len(),
1246            Self::Utf8(array) => array.len(),
1247            Self::Utf8View(array) => array.len(),
1248        }
1249    }
1250
1251    /// Returns whether the column contains no rows.
1252    #[must_use]
1253    pub fn is_empty(&self) -> bool {
1254        self.len() == 0
1255    }
1256
1257    /// Returns a string view when `array` uses `Utf8` or `Utf8View` encoding.
1258    #[must_use]
1259    pub fn try_from_array(array: &dyn Array) -> Option<StringColumnRef<'_>> {
1260        if let Some(array) = array.as_any().downcast_ref::<DictionaryArray<Int8Type>>()
1261            && let Some(values) = array.values().as_any().downcast_ref::<StringArray>()
1262        {
1263            return Some(StringColumnRef::DictionaryInt8(array, values));
1264        }
1265
1266        if let Some(array) = array.as_any().downcast_ref::<DictionaryArray<Int32Type>>()
1267            && let Some(values) = array.values().as_any().downcast_ref::<StringArray>()
1268        {
1269            return Some(StringColumnRef::DictionaryInt32(array, values));
1270        }
1271
1272        if let Some(array) = array.as_any().downcast_ref::<StringArray>() {
1273            return Some(StringColumnRef::Utf8(array));
1274        }
1275
1276        array
1277            .as_any()
1278            .downcast_ref::<StringViewArray>()
1279            .map(StringColumnRef::Utf8View)
1280    }
1281
1282    /// Returns the string value at row `i`.
1283    ///
1284    /// # Panics
1285    ///
1286    /// Panics if `i` is out of bounds or a dictionary column contains an invalid key.
1287    #[inline]
1288    #[must_use]
1289    pub fn value(&self, i: usize) -> &str {
1290        match self {
1291            Self::DictionaryInt8(array, values) => {
1292                let key = usize::try_from(array.keys().value(i)).expect("Int8 key is non-negative");
1293                values.value(key)
1294            }
1295            Self::DictionaryInt32(array, values) => {
1296                let key =
1297                    usize::try_from(array.keys().value(i)).expect("Int32 key is non-negative");
1298                values.value(key)
1299            }
1300            Self::Utf8(arr) => arr.value(i),
1301            Self::Utf8View(arr) => arr.value(i),
1302        }
1303    }
1304
1305    /// Returns whether the value at row `i` is null.
1306    #[inline]
1307    #[must_use]
1308    pub fn is_null(&self, i: usize) -> bool {
1309        match self {
1310            Self::DictionaryInt8(array, _) => array.is_null(i),
1311            Self::DictionaryInt32(array, _) => array.is_null(i),
1312            Self::Utf8(arr) => arr.is_null(i),
1313            Self::Utf8View(arr) => arr.is_null(i),
1314        }
1315    }
1316
1317    /// Returns the string value at row `i`, or `None` when it is null.
1318    #[must_use]
1319    pub fn value_opt(&self, i: usize) -> Option<&str> {
1320        (!self.is_null(i)).then(|| self.value(i))
1321    }
1322}
1323
1324/// Reference to an unsigned 64-bit value stored as an integer or UTC nanosecond timestamp.
1325#[derive(Debug)]
1326pub enum U64ColumnRef<'a> {
1327    UInt64(&'a UInt64Array),
1328    Int64(&'a Int64Array),
1329    TimestampNanosecond(&'a TimestampNanosecondArray),
1330}
1331
1332impl U64ColumnRef<'_> {
1333    /// Returns a compatible unsigned 64-bit view for `array`.
1334    #[must_use]
1335    pub fn try_from_array(array: &dyn Array) -> Option<U64ColumnRef<'_>> {
1336        if let Some(array) = array.as_any().downcast_ref::<UInt64Array>() {
1337            return Some(U64ColumnRef::UInt64(array));
1338        }
1339
1340        if let Some(array) = array.as_any().downcast_ref::<Int64Array>() {
1341            return Some(U64ColumnRef::Int64(array));
1342        }
1343
1344        array
1345            .as_any()
1346            .downcast_ref::<TimestampNanosecondArray>()
1347            .map(U64ColumnRef::TimestampNanosecond)
1348    }
1349
1350    /// Returns the number of values in the column.
1351    #[must_use]
1352    pub fn len(&self) -> usize {
1353        match self {
1354            Self::UInt64(array) => array.len(),
1355            Self::Int64(array) => array.len(),
1356            Self::TimestampNanosecond(array) => array.len(),
1357        }
1358    }
1359
1360    /// Returns whether the column has no values.
1361    #[must_use]
1362    pub fn is_empty(&self) -> bool {
1363        self.len() == 0
1364    }
1365
1366    /// Returns whether the value at row `i` is null.
1367    #[must_use]
1368    pub fn is_null(&self, i: usize) -> bool {
1369        match self {
1370            Self::UInt64(array) => array.is_null(i),
1371            Self::Int64(array) => array.is_null(i),
1372            Self::TimestampNanosecond(array) => array.is_null(i),
1373        }
1374    }
1375
1376    /// Returns the value at row `i`, or `None` when a signed representation is negative.
1377    #[must_use]
1378    pub fn value(&self, i: usize) -> Option<u64> {
1379        match self {
1380            Self::UInt64(array) => Some(array.value(i)),
1381            Self::Int64(array) => u64::try_from(array.value(i)).ok(),
1382            Self::TimestampNanosecond(array) => u64::try_from(array.value(i)).ok(),
1383        }
1384    }
1385}
1386
1387/// Reference to an unsigned 32-bit column stored as `UInt32`, `Int32`, or `Int64`.
1388#[derive(Debug)]
1389pub enum U32ColumnRef<'a> {
1390    UInt32(&'a UInt32Array),
1391    Int32(&'a Int32Array),
1392    Int64(&'a Int64Array),
1393}
1394
1395impl U32ColumnRef<'_> {
1396    /// Returns a compatible unsigned 32-bit view for `array`.
1397    #[must_use]
1398    pub fn try_from_array(array: &dyn Array) -> Option<U32ColumnRef<'_>> {
1399        if let Some(array) = array.as_any().downcast_ref::<UInt32Array>() {
1400            return Some(U32ColumnRef::UInt32(array));
1401        }
1402
1403        if let Some(array) = array.as_any().downcast_ref::<Int32Array>() {
1404            return Some(U32ColumnRef::Int32(array));
1405        }
1406
1407        array
1408            .as_any()
1409            .downcast_ref::<Int64Array>()
1410            .map(U32ColumnRef::Int64)
1411    }
1412}
1413
1414/// Extracts a binary column, accepting both Binary (`BinaryArray`) and BinaryView
1415/// (`BinaryViewArray`).
1416/// DataFusion may return BinaryView when reading Parquet, so this handles both formats.
1417///
1418/// # Errors
1419///
1420/// Returns an error if:
1421/// - `column_index` is out of range.
1422/// - The column type is neither Binary nor BinaryView.
1423pub fn extract_column_binary<'a>(
1424    cols: &'a [ArrayRef],
1425    column_key: &'static str,
1426    column_index: usize,
1427) -> Result<BinaryColumnRef<'a>, EncodingError> {
1428    let column_values = cols
1429        .get(column_index)
1430        .ok_or(EncodingError::MissingColumn(column_key, column_index))?;
1431    let dt = column_values.data_type();
1432    if let Some(arr) = column_values.as_any().downcast_ref::<BinaryArray>() {
1433        Ok(BinaryColumnRef::Binary(arr))
1434    } else if let Some(arr) = column_values.as_any().downcast_ref::<BinaryViewArray>() {
1435        Ok(BinaryColumnRef::BinaryView(arr))
1436    } else {
1437        Err(EncodingError::InvalidColumnType(
1438            column_key,
1439            column_index,
1440            DataType::Binary,
1441            dt.clone(),
1442        ))
1443    }
1444}
1445
1446/// Reference to a binary column, either Binary or BinaryView.
1447#[derive(Debug)]
1448pub enum BinaryColumnRef<'a> {
1449    Binary(&'a BinaryArray),
1450    BinaryView(&'a BinaryViewArray),
1451}
1452
1453impl BinaryColumnRef<'_> {
1454    /// Returns whether the row contains a null value.
1455    #[inline]
1456    #[must_use]
1457    pub fn is_null(&self, i: usize) -> bool {
1458        match self {
1459            Self::Binary(arr) => arr.is_null(i),
1460            Self::BinaryView(arr) => arr.is_null(i),
1461        }
1462    }
1463
1464    /// Returns the bytes at row `i`.
1465    #[inline]
1466    #[must_use]
1467    pub fn value(&self, i: usize) -> &[u8] {
1468        match self {
1469            Self::Binary(arr) => arr.value(i),
1470            Self::BinaryView(arr) => arr.value(i),
1471        }
1472    }
1473}
1474
1475/// Extracts and downcasts the specified `column_key` column from an Arrow array slice.
1476///
1477/// # Errors
1478///
1479/// Returns an error if:
1480/// - `column_index` is out of range.
1481/// - The column type does not match `expected_type`.
1482pub fn extract_column<'a, T: Array + 'static>(
1483    cols: &'a [ArrayRef],
1484    column_key: &'static str,
1485    column_index: usize,
1486    expected_type: DataType,
1487) -> Result<&'a T, EncodingError> {
1488    let column_values = cols
1489        .get(column_index)
1490        .ok_or(EncodingError::MissingColumn(column_key, column_index))?;
1491
1492    let downcasted_values = column_values
1493        .as_any()
1494        .downcast_ref::<T>()
1495        .filter(|_| column_values.data_type() == &expected_type)
1496        .ok_or(EncodingError::InvalidColumnType(
1497            column_key,
1498            column_index,
1499            expected_type,
1500            column_values.data_type().clone(),
1501        ))?;
1502    Ok(downcasted_values)
1503}
1504
1505/// Extracts a column by name.
1506///
1507/// # Errors
1508///
1509/// Returns an error if the column is missing or has the wrong type.
1510pub fn extract_column_by_name<'a, T: Array + 'static>(
1511    record_batch: &'a RecordBatch,
1512    column_key: &'static str,
1513    expected_type: DataType,
1514) -> Result<&'a T, EncodingError> {
1515    let column_index = record_batch.schema().index_of(column_key)?;
1516    extract_column::<T>(
1517        record_batch.columns(),
1518        column_key,
1519        column_index,
1520        expected_type,
1521    )
1522}
1523
1524/// Extracts a decimal column by its schema name.
1525///
1526/// # Errors
1527///
1528/// Returns an error if the named column is missing or is not `Decimal128(38, 16)`.
1529pub fn extract_decimal_column<'a>(
1530    record_batch: &'a RecordBatch,
1531    column_key: &'static str,
1532) -> Result<&'a Decimal128Array, EncodingError> {
1533    let column_index = record_batch.schema().index_of(column_key)?;
1534    extract_column(
1535        record_batch.columns(),
1536        column_key,
1537        column_index,
1538        fixed_decimal_data_type(),
1539    )
1540}
1541
1542/// Extracts an optional UTF-8 column by name.
1543///
1544/// # Errors
1545///
1546/// Returns an error if the column exists but is not UTF-8.
1547pub fn extract_optional_string_column_by_name<'a>(
1548    record_batch: &'a RecordBatch,
1549    column_key: &'static str,
1550) -> Result<Option<&'a StringArray>, EncodingError> {
1551    let Ok(column_index) = record_batch.schema().index_of(column_key) else {
1552        return Ok(None);
1553    };
1554    let column_values = record_batch
1555        .columns()
1556        .get(column_index)
1557        .ok_or(EncodingError::MissingColumn(column_key, column_index))?;
1558    let downcasted_values = column_values.as_any().downcast_ref::<StringArray>().ok_or(
1559        EncodingError::InvalidColumnType(
1560            column_key,
1561            column_index,
1562            DataType::Utf8,
1563            column_values.data_type().clone(),
1564        ),
1565    )?;
1566    Ok(Some(downcasted_values))
1567}
1568
1569/// Returns an optional [`Ustr`] value from an optional string column.
1570#[must_use]
1571pub fn optional_ustr_value(values: Option<&StringArray>, row: usize) -> Option<Ustr> {
1572    values.and_then(|column| (!column.is_null(row)).then(|| Ustr::from(column.value(row))))
1573}
1574
1575/// Converts a vector of `OrderBookDelta` into an Arrow `RecordBatch`.
1576///
1577/// # Errors
1578///
1579/// Returns an error if:
1580/// - `data` is empty.
1581/// - Instrument IDs differ, or non-clear precision metadata differs.
1582/// - Encoding fails.
1583pub fn book_deltas_to_arrow_record_batch_bytes(
1584    data: &[OrderBookDelta],
1585) -> Result<RecordBatch, EncodingError> {
1586    let Some(first) = data.first() else {
1587        return Err(EncodingError::EmptyData);
1588    };
1589
1590    let metadata = OrderBookDelta::chunk_metadata(data);
1591    let instrument_id = data
1592        .iter()
1593        .find(|delta| delta.action != BookAction::Clear)
1594        .unwrap_or(first)
1595        .instrument_id;
1596
1597    if let Some(index) = data.iter().position(|delta| {
1598        delta.instrument_id != instrument_id
1599            || (delta.action != BookAction::Clear && delta.metadata() != metadata)
1600    }) {
1601        return Err(EncodingError::MixedMetadata { index });
1602    }
1603
1604    OrderBookDelta::encode_batch(&metadata, data).map_err(EncodingError::ArrowError)
1605}
1606
1607/// Converts a vector of `OrderBookDepth` into an Arrow `RecordBatch`.
1608///
1609/// # Errors
1610///
1611/// Returns an error if:
1612/// - `data` is empty.
1613/// - Metadata differs between rows.
1614/// - Encoding fails.
1615pub fn book_depths_to_arrow_record_batch_bytes(
1616    data: &[OrderBookDepth],
1617) -> Result<RecordBatch, EncodingError> {
1618    if data.is_empty() {
1619        return Err(EncodingError::EmptyData);
1620    }
1621    let metadata = OrderBookDepth::chunk_metadata(data);
1622
1623    if let Some(index) = data
1624        .iter()
1625        .position(|depth| !depth.matches_chunk_metadata(&metadata))
1626    {
1627        return Err(EncodingError::MixedMetadata { index });
1628    }
1629
1630    OrderBookDepth::encode_batch(&metadata, data).map_err(EncodingError::ArrowError)
1631}
1632
1633/// Converts a vector of `QuoteTick` into an Arrow `RecordBatch`.
1634///
1635/// # Errors
1636///
1637/// Returns an error if:
1638/// - `data` is empty.
1639/// - Metadata differs between rows.
1640/// - Encoding fails.
1641pub fn quotes_to_arrow_record_batch_bytes(
1642    data: &[QuoteTick],
1643) -> Result<RecordBatch, EncodingError> {
1644    encode_batch_with_metadata(data)
1645}
1646
1647/// Converts a vector of `TradeTick` into an Arrow `RecordBatch`.
1648///
1649/// # Errors
1650///
1651/// Returns an error if:
1652/// - `data` is empty.
1653/// - Metadata differs between rows.
1654/// - Encoding fails.
1655pub fn trades_to_arrow_record_batch_bytes(
1656    data: &[TradeTick],
1657) -> Result<RecordBatch, EncodingError> {
1658    encode_batch_with_metadata(data)
1659}
1660
1661/// Converts a vector of `Bar` into an Arrow `RecordBatch`.
1662///
1663/// # Errors
1664///
1665/// Returns an error if:
1666/// - `data` is empty.
1667/// - Metadata differs between rows.
1668/// - Encoding fails.
1669pub fn bars_to_arrow_record_batch_bytes(data: &[Bar]) -> Result<RecordBatch, EncodingError> {
1670    encode_batch_with_metadata(data)
1671}
1672
1673/// Converts a vector of `MarkPriceUpdate` into an Arrow `RecordBatch`.
1674///
1675/// # Errors
1676///
1677/// Returns an error if:
1678/// - `data` is empty.
1679/// - Metadata differs between rows.
1680/// - Encoding fails.
1681pub fn mark_prices_to_arrow_record_batch_bytes(
1682    data: &[MarkPriceUpdate],
1683) -> Result<RecordBatch, EncodingError> {
1684    encode_batch_with_metadata(data)
1685}
1686
1687/// Converts a vector of `IndexPriceUpdate` into an Arrow `RecordBatch`.
1688///
1689/// # Errors
1690///
1691/// Returns an error if:
1692/// - `data` is empty.
1693/// - Metadata differs between rows.
1694/// - Encoding fails.
1695pub fn index_prices_to_arrow_record_batch_bytes(
1696    data: &[IndexPriceUpdate],
1697) -> Result<RecordBatch, EncodingError> {
1698    encode_batch_with_metadata(data)
1699}
1700
1701/// Converts a vector of `InstrumentStatus` into an Arrow `RecordBatch`.
1702///
1703/// # Errors
1704///
1705/// Returns an error if:
1706/// - `data` is empty.
1707/// - Metadata differs between rows.
1708/// - Encoding fails.
1709pub fn instrument_status_to_arrow_record_batch_bytes(
1710    data: &[InstrumentStatus],
1711) -> Result<RecordBatch, EncodingError> {
1712    encode_batch_with_metadata(data)
1713}
1714
1715/// Converts a vector of `OptionGreeks` into an Arrow `RecordBatch`.
1716///
1717/// # Errors
1718///
1719/// Returns an error if:
1720/// - `data` is empty.
1721/// - Metadata differs between rows.
1722/// - Encoding fails.
1723pub fn option_greeks_to_arrow_record_batch_bytes(
1724    data: &[OptionGreeks],
1725) -> Result<RecordBatch, EncodingError> {
1726    encode_batch_with_metadata(data)
1727}
1728
1729/// Converts a vector of `InstrumentClose` into an Arrow `RecordBatch`.
1730///
1731/// # Errors
1732///
1733/// Returns an error if:
1734/// - `data` is empty.
1735/// - Metadata differs between rows.
1736/// - Encoding fails.
1737pub fn instrument_closes_to_arrow_record_batch_bytes(
1738    data: &[InstrumentClose],
1739) -> Result<RecordBatch, EncodingError> {
1740    encode_batch_with_metadata(data)
1741}
1742
1743pub(crate) fn encode_batch_with_metadata<T>(data: &[T]) -> Result<RecordBatch, EncodingError>
1744where
1745    T: EncodeToRecordBatch,
1746{
1747    if data.is_empty() {
1748        return Err(EncodingError::EmptyData);
1749    }
1750
1751    let metadata = T::chunk_metadata(data);
1752    if let Some(index) = data.iter().position(|value| value.metadata() != metadata) {
1753        return Err(EncodingError::MixedMetadata { index });
1754    }
1755
1756    T::encode_batch(&metadata, data).map_err(EncodingError::ArrowError)
1757}
1758
1759#[cfg(test)]
1760mod tests {
1761    use nautilus_model::{
1762        data::{
1763            Bar, BarSpecification, BarType, BookOrder, DEPTH10_LEN, OrderBookDelta, OrderBookDepth,
1764            QuoteTick, order::NULL_ORDER,
1765        },
1766        enums::{
1767            AggregationSource, BarAggregation, BookAction, MarketStatusAction, OrderSide, PriceType,
1768        },
1769        identifiers::InstrumentId,
1770        types::{Price, Quantity},
1771    };
1772    use rstest::rstest;
1773
1774    use super::*;
1775
1776    #[cfg(feature = "high-precision")]
1777    #[rstest]
1778    fn test_money_rejects_currency_precision_above_catalog_scale() {
1779        use nautilus_model::enums::CurrencyType;
1780
1781        if nautilus_model::types::fixed::check_fixed_precision(18).is_err() {
1782            return;
1783        }
1784
1785        let currency = Currency::new("TST18", 18, 0, "Test token", CurrencyType::Crypto);
1786        let value = Money::from_raw(1_000_000_000_000_000_000, currency);
1787        let error = money_array([Some(value)]).unwrap_err();
1788
1789        assert_eq!(
1790            error.to_string(),
1791            "Invalid argument error: Money currency precision 18 exceeds catalog scale 16"
1792        );
1793    }
1794
1795    #[rstest]
1796    fn test_encode_rejects_defi_precision_metadata() {
1797        let mut metadata = QuoteTick::get_metadata(
1798            &InstrumentId::from("WETH-USDC.UNISWAP"),
1799            FIXED_DECIMAL_SCALE as u8,
1800            0,
1801        );
1802        metadata.insert(KEY_PRICE_PRECISION.to_string(), "17".to_string());
1803        let schema = Arc::new(QuoteTick::get_schema(Some(metadata)));
1804
1805        let err = record_batch_with_timestamps(schema, vec![]).unwrap_err();
1806
1807        assert_eq!(
1808            err.to_string(),
1809            "Invalid argument error: Metadata 'price_precision' is 17, maximum supported catalog scale is 16"
1810        );
1811    }
1812
1813    #[rstest]
1814    fn test_record_batch_with_timestamps_rejects_surplus_columns() {
1815        let schema = Arc::new(Schema::new(vec![Field::new(
1816            "ts_event",
1817            timestamp_data_type(),
1818            false,
1819        )]));
1820        let columns = vec![
1821            Arc::new(UInt64Array::from(vec![1])) as ArrayRef,
1822            Arc::new(UInt64Array::from(vec![2])) as ArrayRef,
1823        ];
1824
1825        let error = record_batch_with_timestamps(schema, columns).unwrap_err();
1826
1827        assert_eq!(
1828            error.to_string(),
1829            "Invalid argument error: number of columns(2) must match number of fields(1) in schema"
1830        );
1831    }
1832
1833    #[rstest]
1834    fn test_timestamp_arrays_preserve_utc_nanoseconds_and_nulls() {
1835        let values = [Some(1_788_652_800_123_456_789), None, Some(i64::MAX as u64)];
1836        let array = optional_timestamp_array(values).unwrap();
1837
1838        assert_eq!(
1839            timestamp_data_type(),
1840            DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
1841        );
1842        assert_eq!(array.data_type(), &timestamp_data_type());
1843        assert_eq!(
1844            array.iter().collect::<Vec<_>>(),
1845            vec![Some(1_788_652_800_123_456_789), None, Some(i64::MAX)]
1846        );
1847        assert_eq!(
1848            decode_timestamp(&array, "ts_event", 0).unwrap(),
1849            1_788_652_800_123_456_789
1850        );
1851        assert_eq!(
1852            decode_timestamp(&array, "ts_event", 2).unwrap(),
1853            i64::MAX as u64
1854        );
1855        assert!(
1856            decode_timestamp(&TimestampNanosecondArray::from(vec![-1]), "ts_event", 0).is_err()
1857        );
1858    }
1859
1860    #[rstest]
1861    fn test_timestamp_array_rejects_unsigned_overflow() {
1862        let error = timestamp_array([u64::MAX]).unwrap_err();
1863
1864        assert_eq!(
1865            error.to_string(),
1866            "Invalid argument error: Nanosecond timestamp 18446744073709551615 exceeds Arrow's signed timestamp range",
1867        );
1868    }
1869
1870    #[rstest]
1871    fn test_quotes_to_arrow_record_batch_rejects_mixed_instruments() {
1872        let first = QuoteTick::new(
1873            InstrumentId::from("AAPL.XNAS"),
1874            Price::from("100.01"),
1875            Price::from("100.02"),
1876            Quantity::from("10"),
1877            Quantity::from("11"),
1878            1.into(),
1879            1.into(),
1880        );
1881        let second = QuoteTick::new(
1882            InstrumentId::from("MSFT.XNAS"),
1883            Price::from("200.01"),
1884            Price::from("200.02"),
1885            Quantity::from("20"),
1886            Quantity::from("21"),
1887            2.into(),
1888            2.into(),
1889        );
1890
1891        let result = quotes_to_arrow_record_batch_bytes(&[first, second]);
1892
1893        assert!(matches!(
1894            result,
1895            Err(EncodingError::MixedMetadata { index: 1 })
1896        ));
1897    }
1898
1899    #[rstest]
1900    fn test_instrument_status_to_arrow_record_batch_rejects_mixed_instruments() {
1901        let status = |instrument_id: &str, ts: u64| {
1902            InstrumentStatus::new(
1903                InstrumentId::from(instrument_id),
1904                MarketStatusAction::Trading,
1905                ts.into(),
1906                ts.into(),
1907                None,
1908                None,
1909                None,
1910                None,
1911                None,
1912            )
1913        };
1914
1915        let result = instrument_status_to_arrow_record_batch_bytes(&[
1916            status("AAA.XNAS", 1),
1917            status("BBB.XNAS", 2),
1918        ]);
1919
1920        assert!(matches!(
1921            result,
1922            Err(EncodingError::MixedMetadata { index: 1 })
1923        ));
1924    }
1925
1926    #[rstest]
1927    fn test_option_greeks_to_arrow_record_batch_rejects_mixed_instruments() {
1928        let greeks = |instrument_id: &str| OptionGreeks {
1929            instrument_id: InstrumentId::from(instrument_id),
1930            ..OptionGreeks::default()
1931        };
1932
1933        let result = option_greeks_to_arrow_record_batch_bytes(&[
1934            greeks("BTC-20260529-100000-C.OKX"),
1935            greeks("BTC-20260529-110000-C.OKX"),
1936        ]);
1937
1938        assert!(matches!(
1939            result,
1940            Err(EncodingError::MixedMetadata { index: 1 })
1941        ));
1942    }
1943
1944    #[rstest]
1945    fn test_quotes_to_arrow_record_batch_rejects_mixed_precision() {
1946        let instrument_id = InstrumentId::from("AAPL.XNAS");
1947        let first = QuoteTick::new(
1948            instrument_id,
1949            Price::from("100.01"),
1950            Price::from("100.02"),
1951            Quantity::from("10.00"),
1952            Quantity::from("11.00"),
1953            1.into(),
1954            1.into(),
1955        );
1956        let second = QuoteTick::new(
1957            instrument_id,
1958            Price::from("100.010"),
1959            Price::from("100.020"),
1960            Quantity::from("10.000"),
1961            Quantity::from("11.000"),
1962            2.into(),
1963            2.into(),
1964        );
1965
1966        let result = quotes_to_arrow_record_batch_bytes(&[first, second]);
1967
1968        assert!(matches!(
1969            result,
1970            Err(EncodingError::MixedMetadata { index: 1 })
1971        ));
1972    }
1973
1974    #[rstest]
1975    fn test_bars_to_arrow_record_batch_rejects_mixed_bar_types() {
1976        let instrument_id = InstrumentId::from("AAPL.XNAS");
1977        let first_type = BarType::new(
1978            instrument_id,
1979            BarSpecification::new(1, BarAggregation::Minute, PriceType::Last),
1980            AggregationSource::Internal,
1981        );
1982        let second_type = BarType::new(
1983            instrument_id,
1984            BarSpecification::new(5, BarAggregation::Minute, PriceType::Last),
1985            AggregationSource::Internal,
1986        );
1987        let first = Bar::new(
1988            first_type,
1989            Price::from("100.01"),
1990            Price::from("100.02"),
1991            Price::from("100.00"),
1992            Price::from("100.01"),
1993            Quantity::from("10"),
1994            1.into(),
1995            1.into(),
1996        );
1997        let second = Bar::new(
1998            second_type,
1999            Price::from("100.01"),
2000            Price::from("100.02"),
2001            Price::from("100.00"),
2002            Price::from("100.01"),
2003            Quantity::from("11"),
2004            2.into(),
2005            2.into(),
2006        );
2007
2008        let result = bars_to_arrow_record_batch_bytes(&[first, second]);
2009
2010        assert!(matches!(
2011            result,
2012            Err(EncodingError::MixedMetadata { index: 1 })
2013        ));
2014    }
2015
2016    #[rstest]
2017    fn test_depths_to_arrow_record_batch_rejects_mixed_level_price_precision() {
2018        let instrument_id = InstrumentId::from("AUD/USD.SIM");
2019        let bid = BookOrder::new(
2020            OrderSide::Buy,
2021            Price::from("1.23"),
2022            Quantity::from("100.00"),
2023            1,
2024        );
2025        let ask = BookOrder::new(
2026            OrderSide::Sell,
2027            Price::from("1.24"),
2028            Quantity::from("100.00"),
2029            2,
2030        );
2031        let mut asks = [ask; DEPTH10_LEN];
2032        asks[1].price = Price::from("1.241");
2033        let depth = OrderBookDepth::new(
2034            instrument_id,
2035            [bid; DEPTH10_LEN],
2036            asks,
2037            [1; DEPTH10_LEN],
2038            [1; DEPTH10_LEN],
2039            0,
2040            1,
2041            1.into(),
2042            1.into(),
2043        );
2044
2045        let result = book_depths_to_arrow_record_batch_bytes(&[depth]);
2046
2047        assert!(matches!(
2048            result,
2049            Err(EncodingError::MixedMetadata { index: 0 })
2050        ));
2051    }
2052
2053    #[rstest]
2054    fn test_depths_to_arrow_record_batch_rejects_mixed_level_size_precision() {
2055        let instrument_id = InstrumentId::from("AUD/USD.SIM");
2056        let bid = BookOrder::new(
2057            OrderSide::Buy,
2058            Price::from("1.23"),
2059            Quantity::from("100.00"),
2060            1,
2061        );
2062        let ask = BookOrder::new(
2063            OrderSide::Sell,
2064            Price::from("1.24"),
2065            Quantity::from("100.00"),
2066            2,
2067        );
2068        let mut bids = [bid; DEPTH10_LEN];
2069        bids[1].size = Quantity::from("100.000");
2070        let depth = OrderBookDepth::new(
2071            instrument_id,
2072            bids,
2073            [ask; DEPTH10_LEN],
2074            [1; DEPTH10_LEN],
2075            [1; DEPTH10_LEN],
2076            0,
2077            1,
2078            1.into(),
2079            1.into(),
2080        );
2081
2082        let result = book_depths_to_arrow_record_batch_bytes(&[depth]);
2083
2084        assert!(matches!(
2085            result,
2086            Err(EncodingError::MixedMetadata { index: 0 })
2087        ));
2088    }
2089
2090    #[rstest]
2091    fn test_depths_to_arrow_record_batch_uses_first_defined_level_precision() {
2092        let instrument_id = InstrumentId::from("AUD/USD.SIM");
2093        let bid = BookOrder::new(
2094            OrderSide::Buy,
2095            Price::from("1.23"),
2096            Quantity::from("100.00"),
2097            1,
2098        );
2099        let ask = BookOrder::new(
2100            OrderSide::Sell,
2101            Price::from("1.24"),
2102            Quantity::from("100.00"),
2103            2,
2104        );
2105        let mut bids = [bid; DEPTH10_LEN];
2106        bids[0] = NULL_ORDER;
2107        let depth = OrderBookDepth::new(
2108            instrument_id,
2109            bids,
2110            [ask; DEPTH10_LEN],
2111            [0; DEPTH10_LEN],
2112            [1; DEPTH10_LEN],
2113            0,
2114            1,
2115            1.into(),
2116            1.into(),
2117        );
2118
2119        let result = book_depths_to_arrow_record_batch_bytes(&[depth]).unwrap();
2120
2121        assert_eq!(
2122            result.schema().metadata().get(KEY_PRICE_PRECISION).unwrap(),
2123            "2"
2124        );
2125        assert_eq!(
2126            result.schema().metadata().get(KEY_SIZE_PRECISION).unwrap(),
2127            "2"
2128        );
2129    }
2130
2131    #[rstest]
2132    fn test_deltas_to_arrow_record_batch_skips_leading_clears_for_precision() {
2133        let instrument_id = InstrumentId::from("AUD/USD.SIM");
2134        let first = OrderBookDelta::clear(instrument_id, 0, 1.into(), 1.into());
2135        let second = OrderBookDelta::clear(instrument_id, 1, 2.into(), 2.into());
2136        let third = OrderBookDelta::new(
2137            instrument_id,
2138            BookAction::Add,
2139            BookOrder::new(
2140                OrderSide::Buy,
2141                Price::from("1.23"),
2142                Quantity::from("100.000000"),
2143                1,
2144            ),
2145            0,
2146            2,
2147            3.into(),
2148            3.into(),
2149        );
2150        let expected = vec![first, second, third];
2151
2152        let batch = book_deltas_to_arrow_record_batch_bytes(&expected).unwrap();
2153        let metadata = batch.schema().metadata().clone();
2154        assert_eq!(
2155            metadata.get(KEY_PRICE_PRECISION).map(String::as_str),
2156            Some("2")
2157        );
2158        assert_eq!(
2159            metadata.get(KEY_SIZE_PRECISION).map(String::as_str),
2160            Some("6")
2161        );
2162
2163        let decoded = OrderBookDelta::decode_batch(&metadata, batch).unwrap();
2164
2165        assert_eq!(decoded, expected);
2166        assert_eq!(decoded[2].order.price.precision, 2);
2167        assert_eq!(decoded[2].order.size.precision, 6);
2168    }
2169
2170    #[rstest]
2171    fn test_deltas_to_arrow_record_batch_all_clear_roundtrip() {
2172        let instrument_id = InstrumentId::from("AUD/USD.SIM");
2173        let expected = vec![
2174            OrderBookDelta::clear(instrument_id, 0, 1.into(), 1.into()),
2175            OrderBookDelta::clear(instrument_id, 1, 2.into(), 2.into()),
2176        ];
2177
2178        let batch = book_deltas_to_arrow_record_batch_bytes(&expected).unwrap();
2179        let metadata = batch.schema().metadata().clone();
2180        let decoded = OrderBookDelta::decode_batch(&metadata, batch).unwrap();
2181
2182        assert_eq!(decoded, expected);
2183    }
2184
2185    #[rstest]
2186    fn test_deltas_to_arrow_record_batch_rejects_mixed_precision() {
2187        let instrument_id = InstrumentId::from("AUD/USD.SIM");
2188        let first = OrderBookDelta::new(
2189            instrument_id,
2190            BookAction::Add,
2191            BookOrder::new(
2192                OrderSide::Buy,
2193                Price::from("1.23"),
2194                Quantity::from("100.00"),
2195                1,
2196            ),
2197            0,
2198            1,
2199            1.into(),
2200            1.into(),
2201        );
2202        let second = OrderBookDelta::new(
2203            instrument_id,
2204            BookAction::Update,
2205            BookOrder::new(
2206                OrderSide::Buy,
2207                Price::from("1.234"),
2208                Quantity::from("100.000"),
2209                1,
2210            ),
2211            0,
2212            2,
2213            2.into(),
2214            2.into(),
2215        );
2216
2217        let result = book_deltas_to_arrow_record_batch_bytes(&[first, second]);
2218
2219        assert!(matches!(
2220            result,
2221            Err(EncodingError::MixedMetadata { index: 1 })
2222        ));
2223    }
2224
2225    #[rstest]
2226    fn test_deltas_to_arrow_record_batch_rejects_mixed_instruments() {
2227        let first = OrderBookDelta::clear(InstrumentId::from("AUD/USD.SIM"), 0, 1.into(), 1.into());
2228        let second = OrderBookDelta::new(
2229            InstrumentId::from("EUR/USD.SIM"),
2230            BookAction::Add,
2231            BookOrder::new(
2232                OrderSide::Buy,
2233                Price::from("1.23"),
2234                Quantity::from("100.00"),
2235                1,
2236            ),
2237            0,
2238            1,
2239            2.into(),
2240            2.into(),
2241        );
2242
2243        let result = book_deltas_to_arrow_record_batch_bytes(&[first, second]);
2244
2245        // The first non-clear delta supplies metadata, so the leading clear is the mismatched row.
2246        assert!(matches!(
2247            result,
2248            Err(EncodingError::MixedMetadata { index: 0 })
2249        ));
2250    }
2251}
2252
2253#[cfg(test)]
2254mod schema_invariant_tests {
2255    use arrow::{
2256        array::{Array, Decimal128Array},
2257        datatypes::{DataType, Field, Schema},
2258    };
2259    use nautilus_model::{
2260        data::{
2261            BookOrder, FundingRateUpdate, IndexPriceUpdate, InstrumentStatus, MarkPriceUpdate,
2262            OptionGreeks,
2263            bar::Bar,
2264            close::InstrumentClose,
2265            delta::OrderBookDelta,
2266            depth::OrderBookDepth,
2267            quote::QuoteTick,
2268            stubs::{stub_bar, stub_depth10},
2269            trade::TradeTick,
2270        },
2271        enums::{AggressorSide, BookAction, InstrumentCloseType, OrderSide},
2272        events::{
2273            AccountState, OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled,
2274            OrderDenied, OrderEmulated, OrderExpired, OrderFillVoided, OrderFilled,
2275            OrderInitialized, OrderModifyDenied, OrderModifyRejected, OrderPendingCancel,
2276            OrderPendingUpdate, OrderRejected, OrderReleased, OrderSnapshot, OrderSubmitted,
2277            OrderTriggered, OrderUpdated, PositionAdjusted, PositionChanged, PositionClosed,
2278            PositionOpened, PositionSnapshot,
2279        },
2280        identifiers::{InstrumentId, TradeId},
2281        instruments::{
2282            InstrumentAny, betting::BettingInstrument, binary_option::BinaryOption, cfd::Cfd,
2283            commodity::Commodity, crypto_future::CryptoFuture,
2284            crypto_futures_spread::CryptoFuturesSpread, crypto_option::CryptoOption,
2285            crypto_option_spread::CryptoOptionSpread, crypto_perpetual::CryptoPerpetual,
2286            currency_pair::CurrencyPair, equity::Equity, futures_contract::FuturesContract,
2287            futures_spread::FuturesSpread, index_instrument::IndexInstrument,
2288            option_contract::OptionContract, option_spread::OptionSpread,
2289            perpetual_contract::PerpetualContract, tokenized_asset::TokenizedAsset,
2290        },
2291        reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
2292        types::{
2293            PRICE_ERROR, PRICE_UNDEF, Price, QUANTITY_UNDEF, Quantity,
2294            fixed::{FIXED_PRECISION, FIXED_PRECISION_STANDARD},
2295        },
2296    };
2297    use rstest::rstest;
2298    use rust_decimal::Decimal;
2299
2300    use super::{
2301        ArrowSchemaProvider, DecodeFromRecordBatch, EncodeToRecordBatch, FIXED_DECIMAL_PRECISION,
2302        FIXED_DECIMAL_SCALE, KEY_IDENTIFIER, QUANTITY_RAW_MAX, STANDARD_TO_DECIMAL_SCALE,
2303        decimal_to_arrow, decode_decimal, decode_decimal_price, decode_decimal_quantity,
2304        fixed_decimal_data_type, is_timestamp_field, price_decimal_array, quantity_decimal_array,
2305        timestamp_data_type,
2306    };
2307
2308    #[derive(Clone, Copy, Debug)]
2309    enum FixedFamily {
2310        Quote,
2311        Trade,
2312        Bar,
2313        Delta,
2314        Depth,
2315        MarkPrice,
2316        IndexPrice,
2317        Close,
2318    }
2319
2320    #[rstest]
2321    fn decimal_to_arrow_rejects_values_outside_decode_range() {
2322        let value = Decimal::from_i128_with_scale(8_000_000_000_000, 0);
2323
2324        let error = decimal_to_arrow(&value, "amount").unwrap_err();
2325
2326        assert!(error.to_string().contains("96-bit range"));
2327    }
2328
2329    #[rstest]
2330    fn decimal_to_arrow_normalizes_trailing_zero_scale() {
2331        let value = Decimal::from_i128_with_scale(10_000_000_000_000_000, 18);
2332        let encoded = decimal_to_arrow(&value, "amount").unwrap();
2333        let array = Decimal128Array::from(vec![encoded])
2334            .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
2335            .unwrap();
2336
2337        assert_eq!(
2338            decode_decimal(&array, "amount", 0).unwrap(),
2339            value.normalize()
2340        );
2341    }
2342
2343    macro_rules! collect_data_schemas {
2344        ($(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?) => {
2345            vec![
2346                $(
2347                    (stringify!($type), <$type as ArrowSchemaProvider>::get_schema(None)),
2348                )+
2349            ]
2350        };
2351    }
2352
2353    macro_rules! assert_model_field_map {
2354        // InstrumentAny is an enum over concrete instrument types and has no field map.
2355        (InstrumentAny) => {};
2356        // InstrumentStatus has no model get_fields implementation.
2357        (InstrumentStatus) => {};
2358        // OptionGreeks has no model get_fields implementation.
2359        (OptionGreeks) => {};
2360        (OrderBookDelta) => {
2361            assert_fields_match_schema(
2362                catalog_field_map(OrderBookDelta::get_fields()),
2363                &OrderBookDelta::get_schema(None),
2364                &["price", "size", KEY_IDENTIFIER],
2365            );
2366        };
2367        (OrderBookDepth) => {
2368            assert_fields_match_schema(
2369                catalog_field_map(OrderBookDepth::get_fields()),
2370                &OrderBookDepth::get_schema(None),
2371                &[KEY_IDENTIFIER],
2372            );
2373        };
2374        (QuoteTick) => {
2375            assert_fields_match_schema(
2376                catalog_field_map(QuoteTick::get_fields()),
2377                &QuoteTick::get_schema(None),
2378                &[KEY_IDENTIFIER],
2379            );
2380        };
2381        (TradeTick) => {
2382            assert_fields_match_schema(
2383                catalog_field_map(TradeTick::get_fields()),
2384                &TradeTick::get_schema(None),
2385                &[KEY_IDENTIFIER],
2386            );
2387        };
2388        (Bar) => {
2389            assert_fields_match_schema(
2390                catalog_field_map(Bar::get_fields()),
2391                &Bar::get_schema(None),
2392                &[KEY_IDENTIFIER],
2393            );
2394        };
2395        (MarkPriceUpdate) => {
2396            assert_fields_match_schema(
2397                catalog_field_map(MarkPriceUpdate::get_fields()),
2398                &MarkPriceUpdate::get_schema(None),
2399                &["value", KEY_IDENTIFIER],
2400            );
2401        };
2402        (IndexPriceUpdate) => {
2403            assert_fields_match_schema(
2404                catalog_field_map(IndexPriceUpdate::get_fields()),
2405                &IndexPriceUpdate::get_schema(None),
2406                &["value", KEY_IDENTIFIER],
2407            );
2408        };
2409        (FundingRateUpdate) => {
2410            assert_fields_match_schema(
2411                catalog_field_map(FundingRateUpdate::get_fields()),
2412                &FundingRateUpdate::get_schema(None),
2413                &["interval", "next_funding_ns", KEY_IDENTIFIER],
2414            );
2415        };
2416        (InstrumentClose) => {
2417            assert_fields_match_schema(
2418                catalog_field_map(InstrumentClose::get_fields()),
2419                &InstrumentClose::get_schema(None),
2420                &["close_price", KEY_IDENTIFIER],
2421            );
2422        };
2423    }
2424
2425    macro_rules! assert_registered_model_field_maps {
2426        ($(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?) => {
2427            $(assert_model_field_map!($type);)+
2428        };
2429    }
2430
2431    #[rstest]
2432    fn registered_write_schemas_have_no_opaque_byte_fields() {
2433        let mut schemas = nautilus_model::for_each_data_type!(collect_data_schemas);
2434        schemas.extend(instrument_schemas());
2435        schemas.extend(record_schemas());
2436
2437        for (name, schema) in schemas {
2438            assert_open_schema(name, &schema);
2439        }
2440    }
2441
2442    #[rstest]
2443    fn model_field_maps_match_encoder_schemas() {
2444        nautilus_model::for_each_data_type!(assert_registered_model_field_maps);
2445    }
2446
2447    #[rstest]
2448    fn fixed_point_storage_uses_uniform_scale() {
2449        let price = Price::from("1.23456789");
2450        let array = price_decimal_array([price.raw()], "price").unwrap();
2451
2452        assert_eq!(array.data_type(), &fixed_decimal_data_type());
2453        assert_eq!(
2454            decode_decimal_price(&array, price.precision, "price", 0).unwrap(),
2455            price,
2456        );
2457    }
2458
2459    #[rstest]
2460    #[case("price", false)]
2461    #[case("size", true)]
2462    fn decimal_decode_rejects_spurious_digits_with_row_context(
2463        #[case] field: &'static str,
2464        #[case] quantity: bool,
2465    ) {
2466        let array = Decimal128Array::from(vec![0, STANDARD_TO_DECIMAL_SCALE])
2467            .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
2468            .unwrap();
2469
2470        let error = if quantity {
2471            decode_decimal_quantity(&array, 0, field, 1).unwrap_err()
2472        } else {
2473            decode_decimal_price(&array, 0, field, 1).unwrap_err()
2474        };
2475
2476        let raw = if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
2477            1
2478        } else {
2479            STANDARD_TO_DECIMAL_SCALE
2480        };
2481
2482        let scale = 10_i128.pow(u32::from(FIXED_PRECISION));
2483
2484        assert_eq!(
2485            error.to_string(),
2486            format!(
2487                "Error parsing `{field}`: row 1: Invalid fixed-point raw value {raw} for precision 0: \
2488             remainder {raw} when divided by scale {scale}. \
2489             Raw value should be a multiple of {scale}. \
2490             This indicates data corruption or incorrect precision/scaling upstream"
2491            )
2492        );
2493    }
2494
2495    #[rstest]
2496    fn undefined_fixed_point_values_encode_as_null() {
2497        let prices = price_decimal_array([PRICE_UNDEF], "price").unwrap();
2498        let quantities = quantity_decimal_array([QUANTITY_UNDEF], "quantity").unwrap();
2499
2500        assert!(prices.is_null(0));
2501        assert!(quantities.is_null(0));
2502    }
2503
2504    #[rstest]
2505    fn price_error_fails_with_field_and_value() {
2506        let error = price_decimal_array([PRICE_ERROR], "bid_price").unwrap_err();
2507
2508        assert_eq!(
2509            error.to_string(),
2510            format!(
2511                "Invalid argument error: Price field 'bid_price' contains PRICE_ERROR raw value {PRICE_ERROR}"
2512            ),
2513        );
2514    }
2515
2516    #[rstest]
2517    fn quantity_overflow_fails_with_field_and_value() {
2518        let raw = QUANTITY_RAW_MAX + 1;
2519        let error = quantity_decimal_array([raw], "bid_size").unwrap_err();
2520
2521        assert_eq!(
2522            error.to_string(),
2523            format!(
2524                "Invalid argument error: Quantity field 'bid_size' raw value {raw} exceeds QUANTITY_RAW_MAX={QUANTITY_RAW_MAX}"
2525            ),
2526        );
2527    }
2528
2529    #[cfg(not(feature = "high-precision"))]
2530    #[rstest]
2531    fn standard_precision_decode_rejects_nonzero_scale_remainder() {
2532        let array = Decimal128Array::from(vec![1_i128])
2533            .with_precision_and_scale(38, 16)
2534            .unwrap();
2535
2536        let error = decode_decimal_price(&array, 9, "bid_price", 0).unwrap_err();
2537
2538        assert_eq!(
2539            error.to_string(),
2540            "Error parsing `bid_price`: row 0: decimal value 1 has nonzero digits beyond build \
2541             precision 9",
2542        );
2543    }
2544
2545    #[cfg(feature = "high-precision")]
2546    #[rstest]
2547    fn high_precision_decimal_payload_is_bit_exact() {
2548        let raw = 1_234_567_890_123_456_i128;
2549        let array = price_decimal_array([raw], "price").unwrap();
2550
2551        assert_eq!(array.value(0), raw);
2552        assert_eq!(
2553            decode_decimal_price(&array, 16, "price", 0).unwrap().raw(),
2554            raw
2555        );
2556    }
2557
2558    #[rstest]
2559    #[case::quote(FixedFamily::Quote)]
2560    #[case::trade(FixedFamily::Trade)]
2561    #[case::bar(FixedFamily::Bar)]
2562    #[case::delta(FixedFamily::Delta)]
2563    #[case::depth(FixedFamily::Depth)]
2564    #[case::mark_price(FixedFamily::MarkPrice)]
2565    #[case::index_price(FixedFamily::IndexPrice)]
2566    #[case::close(FixedFamily::Close)]
2567    fn fixed_family_round_trips_multiple_precisions(#[case] family: FixedFamily) {
2568        round_trip_fixed_family(family, Price::from("1"), Quantity::from("2"));
2569        round_trip_fixed_family(family, Price::from("1.23456"), Quantity::from("2.34567"));
2570        #[cfg(feature = "high-precision")]
2571        round_trip_fixed_family(
2572            family,
2573            Price::from("1.2345678901234567"),
2574            Quantity::from("2.3456789012345678"),
2575        );
2576    }
2577
2578    #[rstest]
2579    #[case::quote(FixedFamily::Quote)]
2580    #[case::trade(FixedFamily::Trade)]
2581    #[case::bar(FixedFamily::Bar)]
2582    #[case::delta(FixedFamily::Delta)]
2583    #[case::mark_price(FixedFamily::MarkPrice)]
2584    #[case::index_price(FixedFamily::IndexPrice)]
2585    #[case::close(FixedFamily::Close)]
2586    fn fixed_family_sentinel_encoding(#[case] family: FixedFamily) {
2587        let instrument_id = InstrumentId::from("SENTINEL.TEST");
2588        let price = Price::from_raw(PRICE_UNDEF, 0);
2589        let quantity = Quantity::from_raw(QUANTITY_UNDEF, 0);
2590
2591        match family {
2592            FixedFamily::Quote => {
2593                let value = QuoteTick::new(
2594                    instrument_id,
2595                    price,
2596                    Price::from("1"),
2597                    quantity,
2598                    Quantity::from("1"),
2599                    1.into(),
2600                    2.into(),
2601                );
2602                let metadata = QuoteTick::get_metadata(&instrument_id, 0, 0);
2603                let error = QuoteTick::encode_batch(&metadata, &[value]).unwrap_err();
2604                assert!(error.to_string().contains("bid_price"));
2605                assert!(error.to_string().contains("PRICE_UNDEF"));
2606            }
2607            FixedFamily::Trade => {
2608                let value = TradeTick {
2609                    instrument_id,
2610                    price,
2611                    size: quantity,
2612                    aggressor_side: AggressorSide::Buy,
2613                    trade_id: TradeId::from("sentinel"),
2614                    ts_event: 1.into(),
2615                    ts_init: 2.into(),
2616                };
2617                let metadata = TradeTick::get_metadata(&instrument_id, 0, 0);
2618                let error = TradeTick::encode_batch(&metadata, &[value]).unwrap_err();
2619                assert!(error.to_string().contains("price"));
2620                assert!(error.to_string().contains("PRICE_UNDEF"));
2621            }
2622            FixedFamily::Bar => {
2623                let mut value = stub_bar();
2624                value.open = price;
2625                value.volume = quantity;
2626                let metadata = Bar::get_metadata(&value.bar_type, 0, 0);
2627                let error = Bar::encode_batch(&metadata, &[value]).unwrap_err();
2628                assert!(error.to_string().contains("open"));
2629                assert!(error.to_string().contains("PRICE_UNDEF"));
2630            }
2631            FixedFamily::Delta => {
2632                let value = OrderBookDelta {
2633                    instrument_id,
2634                    action: BookAction::Update,
2635                    order: BookOrder {
2636                        side: OrderSide::Buy.into(),
2637                        price,
2638                        size: quantity,
2639                        order_id: 1,
2640                    },
2641                    flags: 0,
2642                    sequence: 1,
2643                    ts_event: 1.into(),
2644                    ts_init: 2.into(),
2645                };
2646                let metadata = OrderBookDelta::get_metadata(&instrument_id, 0, 0);
2647                let batch = OrderBookDelta::encode_batch(&metadata, &[value]).unwrap();
2648                assert!(batch.column_by_name("price").unwrap().is_null(0));
2649                assert!(batch.column_by_name("size").unwrap().is_null(0));
2650                assert_eq!(
2651                    OrderBookDelta::decode_batch(&metadata, batch).unwrap(),
2652                    vec![value],
2653                );
2654            }
2655            FixedFamily::Depth => unreachable!("depth sides omit absent levels"),
2656            FixedFamily::MarkPrice => {
2657                let value = MarkPriceUpdate::new(instrument_id, price, 1.into(), 2.into());
2658                let metadata = MarkPriceUpdate::get_metadata(&instrument_id, 0);
2659                let batch = MarkPriceUpdate::encode_batch(&metadata, &[value]).unwrap();
2660                assert!(batch.column_by_name("value").unwrap().is_null(0));
2661                assert_eq!(
2662                    MarkPriceUpdate::decode_batch(&metadata, batch).unwrap(),
2663                    vec![value],
2664                );
2665            }
2666            FixedFamily::IndexPrice => {
2667                let value = IndexPriceUpdate::new(instrument_id, price, 1.into(), 2.into());
2668                let metadata = IndexPriceUpdate::get_metadata(&instrument_id, 0);
2669                let batch = IndexPriceUpdate::encode_batch(&metadata, &[value]).unwrap();
2670                assert!(batch.column_by_name("value").unwrap().is_null(0));
2671                assert_eq!(
2672                    IndexPriceUpdate::decode_batch(&metadata, batch).unwrap(),
2673                    vec![value],
2674                );
2675            }
2676            FixedFamily::Close => {
2677                let value = InstrumentClose::new(
2678                    instrument_id,
2679                    price,
2680                    InstrumentCloseType::EndOfSession,
2681                    1.into(),
2682                    2.into(),
2683                );
2684                let metadata = InstrumentClose::get_metadata(&instrument_id, 0);
2685                let batch = InstrumentClose::encode_batch(&metadata, &[value]).unwrap();
2686                assert!(batch.column_by_name("close_price").unwrap().is_null(0));
2687                assert_eq!(
2688                    InstrumentClose::decode_batch(&metadata, batch).unwrap(),
2689                    vec![value],
2690                );
2691            }
2692        }
2693    }
2694
2695    fn round_trip_fixed_family(family: FixedFamily, price: Price, quantity: Quantity) {
2696        let instrument_id = InstrumentId::from("PRECISION.TEST");
2697
2698        match family {
2699            FixedFamily::Quote => {
2700                let value = QuoteTick::new(
2701                    instrument_id,
2702                    price,
2703                    price,
2704                    quantity,
2705                    quantity,
2706                    1.into(),
2707                    2.into(),
2708                );
2709                let metadata =
2710                    QuoteTick::get_metadata(&instrument_id, price.precision, quantity.precision);
2711                let batch = QuoteTick::encode_batch(&metadata, &[value]).unwrap();
2712                assert_eq!(
2713                    QuoteTick::decode_batch(&metadata, batch).unwrap(),
2714                    vec![value],
2715                );
2716            }
2717            FixedFamily::Trade => {
2718                let value = TradeTick::new(
2719                    instrument_id,
2720                    price,
2721                    quantity,
2722                    AggressorSide::Buy,
2723                    TradeId::from("precision"),
2724                    1.into(),
2725                    2.into(),
2726                );
2727                let metadata =
2728                    TradeTick::get_metadata(&instrument_id, price.precision, quantity.precision);
2729                let batch = TradeTick::encode_batch(&metadata, &[value]).unwrap();
2730                assert_eq!(
2731                    TradeTick::decode_batch(&metadata, batch).unwrap(),
2732                    vec![value],
2733                );
2734            }
2735            FixedFamily::Bar => {
2736                let mut value = stub_bar();
2737                value.open = price;
2738                value.high = price;
2739                value.low = price;
2740                value.close = price;
2741                value.volume = quantity;
2742                let metadata =
2743                    Bar::get_metadata(&value.bar_type, price.precision, quantity.precision);
2744                let batch = Bar::encode_batch(&metadata, &[value]).unwrap();
2745                assert_eq!(Bar::decode_batch(&metadata, batch).unwrap(), vec![value]);
2746            }
2747            FixedFamily::Delta => {
2748                let value = OrderBookDelta::new(
2749                    instrument_id,
2750                    BookAction::Add,
2751                    BookOrder {
2752                        side: OrderSide::Buy.into(),
2753                        price,
2754                        size: quantity,
2755                        order_id: 1,
2756                    },
2757                    0,
2758                    1,
2759                    1.into(),
2760                    2.into(),
2761                );
2762                let metadata = OrderBookDelta::get_metadata(
2763                    &instrument_id,
2764                    price.precision,
2765                    quantity.precision,
2766                );
2767                let batch = OrderBookDelta::encode_batch(&metadata, &[value]).unwrap();
2768                assert_eq!(
2769                    OrderBookDelta::decode_batch(&metadata, batch).unwrap(),
2770                    vec![value],
2771                );
2772            }
2773            FixedFamily::Depth => {
2774                let mut value = stub_depth10();
2775                for order in value.bids.iter_mut().chain(value.asks.iter_mut()) {
2776                    order.price = price;
2777                    order.size = quantity;
2778                }
2779                let metadata = OrderBookDepth::get_metadata(
2780                    &value.instrument_id,
2781                    price.precision,
2782                    quantity.precision,
2783                );
2784                let batch = OrderBookDepth::encode_batch(&metadata, &[value.clone()]).unwrap();
2785                assert_eq!(
2786                    OrderBookDepth::decode_batch(&metadata, batch).unwrap(),
2787                    vec![value],
2788                );
2789            }
2790            FixedFamily::MarkPrice => {
2791                let value = MarkPriceUpdate::new(instrument_id, price, 1.into(), 2.into());
2792                let metadata = MarkPriceUpdate::get_metadata(&instrument_id, price.precision);
2793                let batch = MarkPriceUpdate::encode_batch(&metadata, &[value]).unwrap();
2794                assert_eq!(
2795                    MarkPriceUpdate::decode_batch(&metadata, batch).unwrap(),
2796                    vec![value],
2797                );
2798            }
2799            FixedFamily::IndexPrice => {
2800                let value = IndexPriceUpdate::new(instrument_id, price, 1.into(), 2.into());
2801                let metadata = IndexPriceUpdate::get_metadata(&instrument_id, price.precision);
2802                let batch = IndexPriceUpdate::encode_batch(&metadata, &[value]).unwrap();
2803                assert_eq!(
2804                    IndexPriceUpdate::decode_batch(&metadata, batch).unwrap(),
2805                    vec![value],
2806                );
2807            }
2808            FixedFamily::Close => {
2809                let value = InstrumentClose::new(
2810                    instrument_id,
2811                    price,
2812                    InstrumentCloseType::EndOfSession,
2813                    1.into(),
2814                    2.into(),
2815                );
2816                let metadata = InstrumentClose::get_metadata(&instrument_id, price.precision);
2817                let batch = InstrumentClose::encode_batch(&metadata, &[value]).unwrap();
2818                assert_eq!(
2819                    InstrumentClose::decode_batch(&metadata, batch).unwrap(),
2820                    vec![value],
2821                );
2822            }
2823        }
2824    }
2825
2826    fn instrument_schemas() -> Vec<(&'static str, Schema)> {
2827        // Keep this list explicit until instrument types have a registry equivalent to data types.
2828        vec![
2829            schema::<BettingInstrument>(),
2830            schema::<BinaryOption>(),
2831            schema::<Cfd>(),
2832            schema::<Commodity>(),
2833            schema::<CryptoFuture>(),
2834            schema::<CryptoFuturesSpread>(),
2835            schema::<CryptoOption>(),
2836            schema::<CryptoOptionSpread>(),
2837            schema::<CryptoPerpetual>(),
2838            schema::<CurrencyPair>(),
2839            schema::<Equity>(),
2840            schema::<FuturesContract>(),
2841            schema::<FuturesSpread>(),
2842            schema::<IndexInstrument>(),
2843            schema::<OptionContract>(),
2844            schema::<OptionSpread>(),
2845            schema::<PerpetualContract>(),
2846            schema::<TokenizedAsset>(),
2847        ]
2848    }
2849
2850    fn record_schemas() -> Vec<(&'static str, Schema)> {
2851        // Keep this list explicit until record types have a registry equivalent to data types.
2852        vec![
2853            schema::<AccountState>(),
2854            schema::<OrderInitialized>(),
2855            schema::<OrderDenied>(),
2856            schema::<OrderEmulated>(),
2857            schema::<OrderSubmitted>(),
2858            schema::<OrderAccepted>(),
2859            schema::<OrderRejected>(),
2860            schema::<OrderPendingCancel>(),
2861            schema::<OrderCanceled>(),
2862            schema::<OrderCancelDenied>(),
2863            schema::<OrderCancelRejected>(),
2864            schema::<OrderExpired>(),
2865            schema::<OrderTriggered>(),
2866            schema::<OrderPendingUpdate>(),
2867            schema::<OrderReleased>(),
2868            schema::<OrderModifyDenied>(),
2869            schema::<OrderModifyRejected>(),
2870            schema::<OrderUpdated>(),
2871            schema::<OrderFilled>(),
2872            schema::<OrderFillVoided>(),
2873            schema::<PositionOpened>(),
2874            schema::<PositionChanged>(),
2875            schema::<PositionClosed>(),
2876            schema::<PositionAdjusted>(),
2877            schema::<OrderStatusReport>(),
2878            schema::<FillReport>(),
2879            schema::<PositionStatusReport>(),
2880            schema::<ExecutionMassStatus>(),
2881            schema::<OrderSnapshot>(),
2882            schema::<PositionSnapshot>(),
2883        ]
2884    }
2885
2886    fn schema<T: ArrowSchemaProvider>() -> (&'static str, Schema) {
2887        (std::any::type_name::<T>(), T::get_schema(None))
2888    }
2889
2890    fn catalog_field_map(
2891        fields: impl IntoIterator<Item = (String, String)>,
2892    ) -> Vec<(String, String)> {
2893        let mut fields = fields.into_iter().collect::<Vec<_>>();
2894        fields.push((KEY_IDENTIFIER.to_string(), "Utf8".to_string()));
2895        fields
2896    }
2897
2898    fn assert_fields_match_schema(
2899        expected: Vec<(String, String)>,
2900        schema: &Schema,
2901        nullable: &[&str],
2902    ) {
2903        let expected = expected
2904            .into_iter()
2905            .map(|(name, data_type)| {
2906                let is_nullable = nullable.contains(&name.as_str());
2907                (name, data_type, is_nullable)
2908            })
2909            .collect::<Vec<_>>();
2910        let actual: Vec<_> = schema
2911            .fields()
2912            .iter()
2913            .map(|field| {
2914                (
2915                    field.name().clone(),
2916                    arrow_type_name(field.data_type()),
2917                    field.is_nullable(),
2918                )
2919            })
2920            .collect();
2921
2922        assert_eq!(actual, expected);
2923    }
2924
2925    fn arrow_type_name(data_type: &DataType) -> String {
2926        match data_type {
2927            DataType::List(field) => format!("List({})", arrow_type_name(field.data_type())),
2928            DataType::Struct(fields) => {
2929                let fields = fields
2930                    .iter()
2931                    .map(|field| {
2932                        format!("{}: {}", field.name(), arrow_type_name(field.data_type()))
2933                    })
2934                    .collect::<Vec<_>>()
2935                    .join(", ");
2936                format!("Struct({fields})")
2937            }
2938            _ => format!("{data_type:?}"),
2939        }
2940    }
2941
2942    fn assert_open_schema(name: &str, schema: &Schema) {
2943        for field in schema.fields() {
2944            assert_open_field(name, field);
2945        }
2946    }
2947
2948    fn assert_open_field(schema_name: &str, field: &Field) {
2949        if is_timestamp_field(field.name()) {
2950            assert_eq!(
2951                field.data_type(),
2952                &timestamp_data_type(),
2953                "write schema `{schema_name}` timestamp field `{}` is not a UTC nanosecond timestamp",
2954                field.name(),
2955            );
2956        }
2957
2958        match field.data_type() {
2959            DataType::Binary
2960            | DataType::LargeBinary
2961            | DataType::BinaryView
2962            | DataType::FixedSizeBinary(_) => {
2963                panic!(
2964                    "write schema `{schema_name}` contains opaque byte field `{}`: {}",
2965                    field.name(),
2966                    field.data_type(),
2967                );
2968            }
2969            DataType::List(child)
2970            | DataType::LargeList(child)
2971            | DataType::ListView(child)
2972            | DataType::LargeListView(child)
2973            | DataType::FixedSizeList(child, _)
2974            | DataType::Map(child, _) => assert_open_field(schema_name, child),
2975            DataType::Struct(children) => {
2976                for child in children {
2977                    assert_open_field(schema_name, child);
2978                }
2979            }
2980            _ => {}
2981        }
2982    }
2983}