1pub mod account_state;
19pub mod bar;
20pub mod catalog_display;
21pub mod close;
22pub mod custom;
23pub mod delta;
24pub mod depth;
25pub mod funding;
26pub mod greeks;
27pub mod index_price;
28pub mod instrument;
29pub mod instrument_status;
30pub mod json;
31pub mod legacy;
32pub mod mark_price;
33pub mod option_greeks;
34pub mod order_event;
35pub mod position_event;
36pub mod quote;
37pub mod report;
38pub mod snapshot;
39pub mod trade;
40
41mod depth_display;
42mod display_conversion;
43
44#[cfg(feature = "arrow-display")]
45pub mod display;
46
47#[cfg(test)]
48pub(crate) mod test_support;
49
50use std::{
51 borrow::Borrow,
52 collections::HashMap,
53 fmt::{Display, Write as FmtWrite},
54 io::{self, Write},
55 str::FromStr,
56 sync::Arc,
57};
58
59use arrow::{
60 array::{
61 Array, ArrayRef, BinaryArray, BinaryViewArray, Decimal128Array, DictionaryArray,
62 Int32Array, Int64Array, StringArray, StringBuilder, StringDictionaryBuilder,
63 StringViewArray, StructArray, TimestampNanosecondArray, UInt8Array, UInt32Array,
64 UInt64Array,
65 },
66 buffer::NullBuffer,
67 datatypes::{DataType, Field, Int8Type, Int32Type, Schema, TimeUnit},
68 error::ArrowError,
69 ipc::writer::StreamWriter,
70 record_batch::RecordBatch,
71};
72use nautilus_core::UnixNanos;
73use nautilus_model::{
74 data::{
75 Data, IndexPriceUpdate, InstrumentStatus, MarkPriceUpdate, bar::Bar,
76 close::InstrumentClose, delta::OrderBookDelta, depth::OrderBookDepth,
77 option_chain::OptionGreeks, quote::QuoteTick, trade::TradeTick,
78 },
79 enums::BookAction,
80 identifiers::InstrumentId,
81 types::{
82 Currency, Money, PRICE_ERROR, PRICE_UNDEF, Price, QUANTITY_UNDEF, Quantity,
83 fixed::{FIXED_PRECISION, FIXED_PRECISION_STANDARD},
84 money::MoneyRaw,
85 price::PriceRaw,
86 quantity::{QUANTITY_RAW_MAX, QuantityRaw},
87 },
88};
89#[cfg(feature = "python")]
90use pyo3::prelude::*;
91use rust_decimal::Decimal;
92use ustr::Ustr;
93
94pub use self::legacy::{
95 is_nautilus_legacy_schema, is_nautilus_timestamp_schema, is_timestamp_field,
96 normalize_legacy_fixed_columns, normalized_legacy_data_type, normalized_timestamp_type,
97};
98
99pub const KEY_BAR_TYPE: &str = "bar_type";
101pub const KEY_IDENTIFIER: &str = "identifier";
102pub const KEY_INSTRUMENT_ID: &str = "instrument_id";
103pub const KEY_PRICE_PRECISION: &str = "price_precision";
104pub const KEY_SIZE_PRECISION: &str = "size_precision";
105
106pub(crate) fn parse_metadata(
107 metadata: &HashMap<String, String>,
108) -> Result<(InstrumentId, u8, u8), EncodingError> {
109 let instrument_id = metadata
110 .get(KEY_INSTRUMENT_ID)
111 .ok_or(EncodingError::MissingMetadata(KEY_INSTRUMENT_ID))?
112 .parse::<InstrumentId>()
113 .map_err(|e| EncodingError::ParseError(KEY_INSTRUMENT_ID, e.to_string()))?;
114 let price_precision = metadata
115 .get(KEY_PRICE_PRECISION)
116 .ok_or(EncodingError::MissingMetadata(KEY_PRICE_PRECISION))?
117 .parse::<u8>()
118 .map_err(|e| EncodingError::ParseError(KEY_PRICE_PRECISION, e.to_string()))?;
119 let size_precision = metadata
120 .get(KEY_SIZE_PRECISION)
121 .ok_or(EncodingError::MissingMetadata(KEY_SIZE_PRECISION))?
122 .parse::<u8>()
123 .map_err(|e| EncodingError::ParseError(KEY_SIZE_PRECISION, e.to_string()))?;
124 Ok((instrument_id, price_precision, size_precision))
125}
126pub const KEY_TYPE_NAME: &str = "type_name";
127pub const FIXED_DECIMAL_PRECISION: u8 = 38;
128pub const FIXED_DECIMAL_SCALE: i8 = 16;
129pub(crate) const EMPTY_DEPTH_PRECISION: (u8, u8) = (0, 0);
130
131#[must_use]
133pub fn metadata_with_type_name(
134 type_name: &str,
135 metadata: Option<HashMap<String, String>>,
136) -> HashMap<String, String> {
137 let mut metadata = metadata.unwrap_or_default();
138 metadata.insert(KEY_TYPE_NAME.to_string(), type_name.to_string());
139 metadata
140}
141
142#[must_use]
144pub fn timestamp_data_type() -> DataType {
145 DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
146}
147
148pub fn timestamp_array(
154 values: impl IntoIterator<Item = u64>,
155) -> Result<TimestampNanosecondArray, ArrowError> {
156 optional_timestamp_array(values.into_iter().map(Some))
157}
158
159pub fn optional_timestamp_array(
165 values: impl IntoIterator<Item = Option<u64>>,
166) -> Result<TimestampNanosecondArray, ArrowError> {
167 let values = values
168 .into_iter()
169 .map(|value| {
170 value
171 .map(|value| {
172 i64::try_from(value).map_err(|_| {
173 ArrowError::InvalidArgumentError(format!(
174 "Nanosecond timestamp {value} exceeds Arrow's signed timestamp range"
175 ))
176 })
177 })
178 .transpose()
179 })
180 .collect::<Result<Vec<_>, _>>()?;
181 Ok(TimestampNanosecondArray::from(values).with_data_type(timestamp_data_type()))
182}
183
184pub fn decode_timestamp(
190 values: &TimestampNanosecondArray,
191 name: &'static str,
192 row: usize,
193) -> Result<u64, EncodingError> {
194 u64::try_from(values.value(row)).map_err(|_| {
195 EncodingError::ParseError(
196 name,
197 format!(
198 "row {row}: negative nanosecond timestamp {}",
199 values.value(row)
200 ),
201 )
202 })
203}
204
205pub fn record_batch_with_timestamps(
215 schema: Arc<Schema>,
216 columns: Vec<ArrayRef>,
217) -> Result<RecordBatch, ArrowError> {
218 validate_encode_precisions(schema.metadata())?;
219
220 if columns.len() != schema.fields().len() {
221 return Err(ArrowError::InvalidArgumentError(format!(
222 "number of columns({}) must match number of fields({}) in schema",
223 columns.len(),
224 schema.fields().len()
225 )));
226 }
227
228 let columns = schema
229 .fields()
230 .iter()
231 .zip(columns)
232 .map(|(field, column)| {
233 if field.data_type() != ×tamp_data_type()
234 || column.data_type() != &DataType::UInt64
235 {
236 return Ok(column);
237 }
238 timestamp_column(field, column.as_ref())
239 })
240 .collect::<Result<Vec<_>, ArrowError>>()?;
241 RecordBatch::try_new(schema, columns)
242}
243
244fn validate_encode_precisions(metadata: &HashMap<String, String>) -> Result<(), ArrowError> {
249 for key in [KEY_PRICE_PRECISION, KEY_SIZE_PRECISION] {
250 if let Some(value) = metadata.get(key)
251 && let Ok(precision) = value.parse::<u8>()
252 && precision > FIXED_DECIMAL_SCALE as u8
253 {
254 return Err(ArrowError::InvalidArgumentError(format!(
255 "Metadata '{key}' is {precision}, maximum supported catalog scale is {FIXED_DECIMAL_SCALE}"
256 )));
257 }
258 }
259 Ok(())
260}
261
262fn timestamp_column(field: &Field, column: &dyn Array) -> Result<ArrayRef, ArrowError> {
263 let values = column
264 .as_any()
265 .downcast_ref::<UInt64Array>()
266 .ok_or_else(|| ArrowError::CastError(format!("Column '{}' is not UInt64", field.name())))?;
267 let timestamps = optional_timestamp_array(
268 (0..values.len()).map(|row| (!values.is_null(row)).then(|| values.value(row))),
269 )?;
270 Ok(Arc::new(timestamps) as ArrayRef)
271}
272
273pub fn record_batch_with_u64_timestamps(batch: &RecordBatch) -> Result<RecordBatch, EncodingError> {
279 let mut changed = false;
280 let mut fields = Vec::with_capacity(batch.num_columns());
281 let mut columns = Vec::with_capacity(batch.num_columns());
282
283 for (field, column) in batch.schema().fields().iter().zip(batch.columns()) {
284 if field.data_type() != ×tamp_data_type() {
285 fields.push(Arc::clone(field));
286 columns.push(Arc::clone(column));
287 continue;
288 }
289 let values = column
290 .as_any()
291 .downcast_ref::<TimestampNanosecondArray>()
292 .ok_or_else(|| {
293 EncodingError::ParseError(
294 "timestamp",
295 format!("Column '{}' is not TimestampNanosecond", field.name()),
296 )
297 })?;
298 let values = (0..values.len())
299 .map(|row| {
300 if values.is_null(row) {
301 Ok(None)
302 } else {
303 decode_timestamp(values, "timestamp", row).map(Some)
304 }
305 })
306 .collect::<Result<Vec<_>, _>>()?;
307 fields.push(Arc::new(
308 field.as_ref().clone().with_data_type(DataType::UInt64),
309 ));
310 columns.push(Arc::new(UInt64Array::from(values)) as ArrayRef);
311 changed = true;
312 }
313
314 if !changed {
315 return Ok(batch.clone());
316 }
317 RecordBatch::try_new(
318 Arc::new(Schema::new_with_metadata(
319 fields,
320 batch.schema().metadata().clone(),
321 )),
322 columns,
323 )
324 .map_err(EncodingError::from)
325}
326
327#[must_use]
329pub fn enum_dictionary_data_type() -> DataType {
330 DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8))
331}
332
333pub fn enum_dictionary_array(
339 values: impl IntoIterator<Item = impl Display>,
340) -> Result<DictionaryArray<Int8Type>, ArrowError> {
341 let mut builder = StringDictionaryBuilder::<Int8Type>::new();
342 for value in values {
343 builder.append(value.to_string())?;
344 }
345 Ok(builder.finish())
346}
347
348#[must_use]
350pub fn money_data_type() -> DataType {
351 DataType::Struct(
352 vec![
353 Field::new("amount", fixed_decimal_data_type(), false),
354 Field::new(
355 "currency",
356 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
357 false,
358 ),
359 ]
360 .into(),
361 )
362}
363
364pub fn money_array(
370 values: impl IntoIterator<Item = Option<Money>>,
371) -> Result<StructArray, ArrowError> {
372 let mut amounts = Vec::new();
373 let mut currencies = StringDictionaryBuilder::<Int32Type>::new();
374 let mut validity = Vec::new();
375
376 for value in values {
377 if let Some(value) = value {
378 if value.currency.precision > FIXED_DECIMAL_SCALE as u8 {
379 return Err(ArrowError::InvalidArgumentError(format!(
380 "Money currency precision {} exceeds catalog scale {FIXED_DECIMAL_SCALE}",
381 value.currency.precision,
382 )));
383 }
384
385 amounts.push(money_raw_to_decimal(value.raw()));
386 currencies.append(value.currency.to_string())?;
387 validity.push(true);
388 } else {
389 amounts.push(0);
390 currencies.append_null();
391 validity.push(false);
392 }
393 }
394 let amounts = Decimal128Array::from(amounts)
395 .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)?;
396 StructArray::try_new(
397 match money_data_type() {
398 DataType::Struct(fields) => fields,
399 _ => unreachable!("money data type is a struct"),
400 },
401 vec![Arc::new(amounts), Arc::new(currencies.finish())],
402 Some(NullBuffer::from(validity)),
403 )
404}
405
406pub fn decimal_to_arrow(value: &Decimal, field: &'static str) -> Result<i128, ArrowError> {
413 let value = value.normalize();
414 let scale = value.scale();
415 if scale > FIXED_DECIMAL_SCALE as u32 {
416 return Err(ArrowError::InvalidArgumentError(format!(
417 "Decimal field '{field}' has scale {scale}, maximum supported scale is {FIXED_DECIMAL_SCALE}"
418 )));
419 }
420 let rescaled = value
421 .mantissa()
422 .checked_mul(10_i128.pow(FIXED_DECIMAL_SCALE as u32 - scale))
423 .ok_or_else(|| {
424 ArrowError::InvalidArgumentError(format!(
425 "Decimal field '{field}' cannot be represented as Decimal128(38, 16)"
426 ))
427 })?;
428 let max = Decimal::MAX.mantissa();
429 if rescaled < -max || rescaled > max {
430 return Err(ArrowError::InvalidArgumentError(format!(
431 "Decimal field '{field}' exceeds the rust_decimal 96-bit range after rescaling"
432 )));
433 }
434 Ok(rescaled)
435}
436
437pub fn decode_decimal(
443 values: &Decimal128Array,
444 field: &'static str,
445 row: usize,
446) -> Result<Decimal, EncodingError> {
447 if values.is_null(row) {
448 return Err(EncodingError::ParseError(
449 field,
450 format!("row {row}: required decimal is null"),
451 ));
452 }
453 Decimal::try_from_i128_with_scale(values.value(row), FIXED_DECIMAL_SCALE as u32)
454 .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")))
455}
456
457pub fn decode_money(
463 values: &StructArray,
464 field: &'static str,
465 row: usize,
466) -> Result<Money, EncodingError> {
467 if values.is_null(row) {
468 return Err(EncodingError::ParseError(
469 field,
470 format!("row {row}: required money is null"),
471 ));
472 }
473 let amounts = values
474 .column_by_name("amount")
475 .and_then(|array| array.as_any().downcast_ref::<Decimal128Array>())
476 .ok_or_else(|| {
477 EncodingError::ParseError(field, "money amount must be Decimal128(38, 16)".to_string())
478 })?;
479 let currencies = values
480 .column_by_name("currency")
481 .and_then(|array| StringColumnRef::try_from_array(array.as_ref()))
482 .ok_or_else(|| {
483 EncodingError::ParseError(
484 field,
485 "money currency must be Dictionary<Int32, Utf8>".to_string(),
486 )
487 })?;
488 let raw = decimal_to_money_raw(amounts.value(row), field, row)?;
489 let currency_code = currencies.value(row);
490 let currency = Currency::from_str(currency_code).map_err(|e| {
491 EncodingError::ParseError(
492 field,
493 format!(
494 "row {row}: currency '{currency_code}' must be registered before decoding Money: {e}"
495 ),
496 )
497 })?;
498
499 if currency.precision > FIXED_DECIMAL_SCALE as u8 {
500 return Err(EncodingError::ParseError(
501 field,
502 format!(
503 "row {row}: Money currency precision {} exceeds catalog scale {FIXED_DECIMAL_SCALE}",
504 currency.precision,
505 ),
506 ));
507 }
508
509 Money::from_raw_checked(raw, currency)
510 .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")))
511}
512
513#[allow(
514 clippy::useless_conversion,
515 reason = "MoneyRaw is i64 or i128 depending on model feature unification"
516)]
517fn money_raw_to_decimal(raw: MoneyRaw) -> i128 {
518 let mut decimal = i128::from(raw);
519 if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
520 decimal *= STANDARD_TO_DECIMAL_SCALE;
521 }
522 decimal
523}
524
525fn decimal_to_money_raw(
526 value: i128,
527 field: &'static str,
528 row: usize,
529) -> Result<MoneyRaw, EncodingError> {
530 decimal_to_raw(value, field, row, "MoneyRaw")
531}
532
533const STANDARD_TO_DECIMAL_SCALE: i128 =
534 10_i128.pow((FIXED_DECIMAL_SCALE as u8 - FIXED_PRECISION_STANDARD) as u32);
535
536#[derive(thiserror::Error, Debug)]
537pub enum DataStreamingError {
538 #[error("I/O error: {0}")]
539 IoError(#[from] io::Error),
540 #[error("Arrow error: {0}")]
541 ArrowError(#[from] arrow::error::ArrowError),
542 #[cfg(feature = "python")]
543 #[error("Python error: {0}")]
544 PythonError(#[from] PyErr),
545}
546
547#[derive(thiserror::Error, Debug)]
548pub enum EncodingError {
549 #[error("Empty data")]
550 EmptyData,
551 #[error(
552 "Mixed metadata at row {index}; encode each instrument, bar type, or precision separately"
553 )]
554 MixedMetadata { index: usize },
555 #[error("Missing metadata key: `{0}`")]
556 MissingMetadata(&'static str),
557 #[error("Missing data column: `{0}` at index {1}")]
558 MissingColumn(&'static str, usize),
559 #[error("Error parsing `{0}`: {1}")]
560 ParseError(&'static str, String),
561 #[error("Invalid column type `{0}` at index {1}: expected {2}, found {3}")]
562 InvalidColumnType(&'static str, usize, DataType, DataType),
563 #[error("Arrow error: {0}")]
564 ArrowError(#[from] arrow::error::ArrowError),
565}
566
567#[must_use]
569pub const fn fixed_decimal_data_type() -> DataType {
570 DataType::Decimal128(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
571}
572
573#[must_use]
575pub fn json_string_field(name: impl Into<String>, nullable: bool) -> Field {
576 Field::new(name, DataType::Utf8, nullable).with_metadata(HashMap::from([
577 ("ARROW:extension:name".to_string(), "arrow.json".to_string()),
578 ("ARROW:extension:metadata".to_string(), String::new()),
579 ]))
580}
581
582#[must_use]
584pub fn is_json_string_field(field: &Field) -> bool {
585 field.extension_type_name() == Some("arrow.json")
586}
587
588pub fn price_raw_to_decimal(
594 raw: PriceRaw,
595 field: &'static str,
596) -> Result<Option<i128>, ArrowError> {
597 if raw == PRICE_UNDEF {
598 return Ok(None);
599 }
600
601 if raw == PRICE_ERROR {
602 return Err(ArrowError::InvalidArgumentError(format!(
603 "Price field '{field}' contains PRICE_ERROR raw value {raw}"
604 )));
605 }
606
607 #[allow(
608 clippy::useless_conversion,
609 reason = "PriceRaw is i64 or i128 depending on model feature unification"
610 )]
611 let mut decimal = i128::from(raw);
612 if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
613 decimal *= STANDARD_TO_DECIMAL_SCALE;
614 }
615
616 Ok(Some(decimal))
617}
618
619pub fn quantity_raw_to_decimal(
626 raw: QuantityRaw,
627 field: &'static str,
628) -> Result<Option<i128>, ArrowError> {
629 if raw == QUANTITY_UNDEF {
630 return Ok(None);
631 }
632
633 if raw > QUANTITY_RAW_MAX {
634 return Err(ArrowError::InvalidArgumentError(format!(
635 "Quantity field '{field}' raw value {raw} exceeds QUANTITY_RAW_MAX={QUANTITY_RAW_MAX}"
636 )));
637 }
638
639 #[allow(
640 clippy::unnecessary_fallible_conversions,
641 reason = "QuantityRaw is u64 or u128 depending on model feature unification"
642 )]
643 let mut decimal = i128::try_from(raw).map_err(|_| {
644 ArrowError::InvalidArgumentError(format!(
645 "Quantity field '{field}' raw value {raw} exceeds Decimal128 range"
646 ))
647 })?;
648
649 if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
650 decimal *= STANDARD_TO_DECIMAL_SCALE;
651 }
652
653 Ok(Some(decimal))
654}
655
656pub fn price_decimal_array(
662 values: impl IntoIterator<Item = PriceRaw>,
663 field: &'static str,
664) -> Result<Decimal128Array, ArrowError> {
665 let values = values
666 .into_iter()
667 .map(|raw| price_raw_to_decimal(raw, field))
668 .collect::<Result<Vec<_>, _>>()?;
669 Decimal128Array::from(values)
670 .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
671}
672
673pub fn quantity_decimal_array(
679 values: impl IntoIterator<Item = QuantityRaw>,
680 field: &'static str,
681) -> Result<Decimal128Array, ArrowError> {
682 let values = values
683 .into_iter()
684 .map(|raw| quantity_raw_to_decimal(raw, field))
685 .collect::<Result<Vec<_>, _>>()?;
686 Decimal128Array::from(values)
687 .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
688}
689
690pub fn required_price_decimal_array(
697 values: impl IntoIterator<Item = PriceRaw>,
698 field: &'static str,
699) -> Result<Decimal128Array, ArrowError> {
700 let values = values
701 .into_iter()
702 .map(|raw| {
703 price_raw_to_decimal(raw, field)?.ok_or_else(|| {
704 ArrowError::InvalidArgumentError(format!(
705 "Price field '{field}' contains PRICE_UNDEF, which has no sentinel encoding for this type"
706 ))
707 })
708 })
709 .collect::<Result<Vec<_>, _>>()?;
710 Decimal128Array::from(values)
711 .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
712}
713
714pub fn required_quantity_decimal_array(
721 values: impl IntoIterator<Item = QuantityRaw>,
722 field: &'static str,
723) -> Result<Decimal128Array, ArrowError> {
724 let values = values
725 .into_iter()
726 .map(|raw| {
727 quantity_raw_to_decimal(raw, field)?.ok_or_else(|| {
728 ArrowError::InvalidArgumentError(format!(
729 "Quantity field '{field}' contains QUANTITY_UNDEF, which has no sentinel encoding for this type"
730 ))
731 })
732 })
733 .collect::<Result<Vec<_>, _>>()?;
734 Decimal128Array::from(values)
735 .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
736}
737
738fn decimal_to_price_raw(
739 value: i128,
740 field: &'static str,
741 row: usize,
742) -> Result<PriceRaw, EncodingError> {
743 decimal_to_raw(value, field, row, "PriceRaw")
744}
745
746fn decimal_to_quantity_raw(
747 value: i128,
748 field: &'static str,
749 row: usize,
750) -> Result<QuantityRaw, EncodingError> {
751 decimal_to_raw(value, field, row, "QuantityRaw")
752}
753
754fn decimal_to_raw<T: TryFrom<i128>>(
755 value: i128,
756 field: &'static str,
757 row: usize,
758 raw_type: &'static str,
759) -> Result<T, EncodingError> {
760 let raw_value = if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
761 if value % STANDARD_TO_DECIMAL_SCALE != 0 {
762 return Err(EncodingError::ParseError(
763 field,
764 format!(
765 "row {row}: decimal value {value} has nonzero digits beyond build precision 9"
766 ),
767 ));
768 }
769 value / STANDARD_TO_DECIMAL_SCALE
770 } else {
771 value
772 };
773
774 T::try_from(raw_value).map_err(|_| {
775 EncodingError::ParseError(
776 field,
777 format!("row {row}: decimal value {value} exceeds {raw_type} range"),
778 )
779 })
780}
781
782pub fn decode_decimal_price(
788 values: &Decimal128Array,
789 precision: u8,
790 field: &'static str,
791 row: usize,
792) -> Result<Price, EncodingError> {
793 if values.is_null(row) {
794 return Price::from_raw_checked(PRICE_UNDEF, 0)
795 .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")));
796 }
797
798 let raw = decimal_to_price_raw(values.value(row), field, row)?;
799 Price::from_raw_checked(raw, precision)
800 .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")))
801}
802
803pub fn decode_required_decimal_price(
809 values: &Decimal128Array,
810 precision: u8,
811 field: &'static str,
812 row: usize,
813) -> Result<Price, EncodingError> {
814 if values.is_null(row) {
815 return Err(EncodingError::ParseError(
816 field,
817 format!("row {row}: required price is null"),
818 ));
819 }
820 decode_decimal_price(values, precision, field, row)
821}
822
823pub fn decode_decimal_quantity(
829 values: &Decimal128Array,
830 precision: u8,
831 field: &'static str,
832 row: usize,
833) -> Result<Quantity, EncodingError> {
834 if values.is_null(row) {
835 return Quantity::from_raw_checked(QUANTITY_UNDEF, 0)
836 .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")));
837 }
838
839 let raw = decimal_to_quantity_raw(values.value(row), field, row)?;
840 Quantity::from_raw_checked(raw, precision)
841 .map_err(|e| EncodingError::ParseError(field, format!("row {row}: {e}")))
842}
843
844pub fn decode_required_decimal_quantity(
850 values: &Decimal128Array,
851 precision: u8,
852 field: &'static str,
853 row: usize,
854) -> Result<Quantity, EncodingError> {
855 if values.is_null(row) {
856 return Err(EncodingError::ParseError(
857 field,
858 format!("row {row}: required quantity is null"),
859 ));
860 }
861 decode_decimal_quantity(values, precision, field, row)
862}
863
864pub fn decode_required_timestamp(
870 values: &UInt64Array,
871 field: &'static str,
872 row: usize,
873) -> Result<UnixNanos, EncodingError> {
874 if values.is_null(row) {
875 return Err(EncodingError::ParseError(
876 field,
877 format!("row {row}: required timestamp is null"),
878 ));
879 }
880 Ok(values.value(row).into())
881}
882
883pub(crate) fn decode_required_u64(
884 values: &UInt64Array,
885 field: &'static str,
886 row: usize,
887) -> Result<u64, EncodingError> {
888 if values.is_null(row) {
889 return Err(EncodingError::ParseError(
890 field,
891 format!("row {row}: required integer is null"),
892 ));
893 }
894 Ok(values.value(row))
895}
896
897pub(crate) fn decode_required_u8(
898 values: &UInt8Array,
899 field: &'static str,
900 row: usize,
901) -> Result<u8, EncodingError> {
902 if values.is_null(row) {
903 return Err(EncodingError::ParseError(
904 field,
905 format!("row {row}: required integer is null"),
906 ));
907 }
908 Ok(values.value(row))
909}
910
911pub(crate) fn validate_required_column(
912 values: &dyn Array,
913 field: &'static str,
914) -> Result<(), EncodingError> {
915 if let Some(row) = first_logical_null(values) {
916 return Err(EncodingError::ParseError(
917 field,
918 format!("row {row}: required value is null"),
919 ));
920 }
921
922 Ok(())
923}
924
925pub(crate) fn first_logical_null(values: &dyn Array) -> Option<usize> {
926 values
927 .logical_nulls()
928 .and_then(|nulls| nulls.iter().position(|valid| !valid))
929}
930
931#[cfg(test)]
932fn get_raw_price(value: i128) -> PriceRaw {
933 decimal_to_price_raw(value, "test", 0).expect("Decimal price must fit the current build")
934}
935
936#[cfg(test)]
937fn get_raw_quantity(value: i128) -> QuantityRaw {
938 decimal_to_quantity_raw(value, "test", 0).expect("Decimal quantity must fit the current build")
939}
940
941pub trait ArrowSchemaProvider {
943 fn get_schema(metadata: Option<HashMap<String, String>>) -> Schema;
945
946 #[must_use]
948 fn get_schema_map() -> HashMap<String, String> {
949 let schema = Self::get_schema(None);
950 let mut map = HashMap::new();
951
952 for field in schema.fields() {
953 let name = field.name().clone();
954 let data_type = format!("{:?}", field.data_type());
955 map.insert(name, data_type);
956 }
957 map
958 }
959}
960
961pub trait EncodeToRecordBatch
963where
964 Self: Sized + ArrowSchemaProvider,
965{
966 fn encode_batch<T>(
972 metadata: &HashMap<String, String>,
973 data: &[T],
974 ) -> Result<RecordBatch, ArrowError>
975 where
976 T: Borrow<Self>;
977
978 fn metadata(&self) -> HashMap<String, String>;
980
981 fn chunk_metadata<T>(chunk: &[T]) -> HashMap<String, String>
990 where
991 T: Borrow<Self>,
992 {
993 chunk
994 .first()
995 .map(|item| item.borrow().metadata())
996 .expect("Chunk must contain at least one element to encode")
997 }
998
999 fn matches_chunk_metadata(&self, metadata: &HashMap<String, String>) -> bool {
1001 self.metadata() == *metadata
1002 }
1003}
1004
1005#[must_use]
1010pub fn catalog_identifier_from_metadata(metadata: &HashMap<String, String>) -> Option<String> {
1011 metadata
1012 .get(KEY_BAR_TYPE)
1013 .cloned()
1014 .or_else(|| metadata.get(KEY_INSTRUMENT_ID).cloned())
1015}
1016
1017#[must_use]
1019pub fn schema_with_identifier_column(schema: &Schema) -> Schema {
1020 if schema.index_of(KEY_IDENTIFIER).is_ok() {
1021 return schema.clone();
1022 }
1023
1024 let mut fields = schema.fields().iter().cloned().collect::<Vec<_>>();
1025 fields.push(Arc::new(Field::new(KEY_IDENTIFIER, DataType::Utf8, true)));
1026
1027 Schema::new_with_metadata(fields, schema.metadata().clone())
1028}
1029
1030#[must_use]
1032pub fn schema_without_identifier_column(schema: &Schema) -> Schema {
1033 let Ok(identifier_index) = schema.index_of(KEY_IDENTIFIER) else {
1034 return schema.clone();
1035 };
1036
1037 let fields = schema
1038 .fields()
1039 .iter()
1040 .enumerate()
1041 .filter_map(|(index, field)| (index != identifier_index).then_some(Arc::clone(field)))
1042 .collect::<Vec<_>>();
1043
1044 Schema::new_with_metadata(fields, schema.metadata().clone())
1045}
1046
1047pub fn record_batch_with_identifier_column(
1056 batch: RecordBatch,
1057 identifier: Option<&str>,
1058) -> Result<RecordBatch, ArrowError> {
1059 let identifier_values = vec![identifier.map(ToString::to_string); batch.num_rows()];
1060 record_batch_with_identifier_values(batch, identifier_values)
1061}
1062
1063pub fn record_batch_with_identifier_values(
1070 batch: RecordBatch,
1071 identifier_values: Vec<Option<String>>,
1072) -> Result<RecordBatch, ArrowError> {
1073 if batch.schema().index_of(KEY_IDENTIFIER).is_ok() {
1074 return Ok(batch);
1075 }
1076
1077 if identifier_values.len() != batch.num_rows() {
1078 return Err(ArrowError::InvalidArgumentError(format!(
1079 "identifier values length {} does not match record batch row count {}",
1080 identifier_values.len(),
1081 batch.num_rows()
1082 )));
1083 }
1084
1085 let schema = schema_with_identifier_column(batch.schema().as_ref());
1086 let mut columns = batch.columns().to_vec();
1087 columns.push(Arc::new(StringArray::from(identifier_values)));
1088
1089 RecordBatch::try_new(Arc::new(schema), columns)
1090}
1091
1092#[must_use]
1094pub fn identifier_array_from_display<T: Display>(
1095 identifiers: impl IntoIterator<Item = T>,
1096) -> StringArray {
1097 let mut builder = StringBuilder::new();
1098 let mut scratch = String::new();
1099
1100 for identifier in identifiers {
1101 scratch.clear();
1102 write!(&mut scratch, "{identifier}").expect("writing to String should not fail");
1103 builder.append_value(scratch.as_str());
1104 }
1105
1106 builder.finish()
1107}
1108
1109pub fn record_batch_without_identifier_column(
1115 mut batch: RecordBatch,
1116) -> Result<RecordBatch, ArrowError> {
1117 let Ok(identifier_index) = batch.schema().index_of(KEY_IDENTIFIER) else {
1118 return Ok(batch);
1119 };
1120
1121 batch.remove_column(identifier_index);
1122 Ok(batch)
1123}
1124
1125pub trait DecodeFromRecordBatch
1127where
1128 Self: Sized + Into<Data> + ArrowSchemaProvider,
1129{
1130 fn decode_batch(
1136 metadata: &HashMap<String, String>,
1137 record_batch: RecordBatch,
1138 ) -> Result<Vec<Self>, EncodingError>;
1139}
1140
1141pub trait DecodeTypedFromRecordBatch
1143where
1144 Self: Sized + ArrowSchemaProvider,
1145{
1146 fn decode_typed_batch(
1152 metadata: &HashMap<String, String>,
1153 record_batch: RecordBatch,
1154 ) -> Result<Vec<Self>, EncodingError>;
1155}
1156
1157impl<T> DecodeTypedFromRecordBatch for T
1158where
1159 T: DecodeFromRecordBatch,
1160{
1161 fn decode_typed_batch(
1162 metadata: &HashMap<String, String>,
1163 record_batch: RecordBatch,
1164 ) -> Result<Vec<Self>, EncodingError> {
1165 Self::decode_batch(metadata, record_batch)
1166 }
1167}
1168
1169pub trait DecodeDataFromRecordBatch
1171where
1172 Self: Sized + ArrowSchemaProvider,
1173{
1174 fn decode_data_batch(
1180 metadata: &HashMap<String, String>,
1181 record_batch: RecordBatch,
1182 ) -> Result<Vec<Data>, EncodingError>;
1183}
1184
1185pub trait WriteStream {
1187 fn write(&mut self, record_batch: &RecordBatch) -> Result<(), DataStreamingError>;
1193}
1194
1195impl<T: Write> WriteStream for T {
1196 fn write(&mut self, record_batch: &RecordBatch) -> Result<(), DataStreamingError> {
1197 let mut writer = StreamWriter::try_new(self, &record_batch.schema())?;
1198 writer.write(record_batch)?;
1199 writer.finish()?;
1200 Ok(())
1201 }
1202}
1203
1204pub fn extract_column_string<'a>(
1213 cols: &'a [ArrayRef],
1214 column_key: &'static str,
1215 column_index: usize,
1216) -> Result<StringColumnRef<'a>, EncodingError> {
1217 let column_values = cols
1218 .get(column_index)
1219 .ok_or(EncodingError::MissingColumn(column_key, column_index))?;
1220 StringColumnRef::try_from_array(column_values.as_ref()).ok_or_else(|| {
1221 EncodingError::InvalidColumnType(
1222 column_key,
1223 column_index,
1224 DataType::Utf8,
1225 column_values.data_type().clone(),
1226 )
1227 })
1228}
1229
1230#[derive(Debug)]
1232pub enum StringColumnRef<'a> {
1233 DictionaryInt8(&'a DictionaryArray<Int8Type>, &'a StringArray),
1234 DictionaryInt32(&'a DictionaryArray<Int32Type>, &'a StringArray),
1235 Utf8(&'a StringArray),
1236 Utf8View(&'a StringViewArray),
1237}
1238
1239impl StringColumnRef<'_> {
1240 #[must_use]
1242 pub fn len(&self) -> usize {
1243 match self {
1244 Self::DictionaryInt8(array, _) => array.len(),
1245 Self::DictionaryInt32(array, _) => array.len(),
1246 Self::Utf8(array) => array.len(),
1247 Self::Utf8View(array) => array.len(),
1248 }
1249 }
1250
1251 #[must_use]
1253 pub fn is_empty(&self) -> bool {
1254 self.len() == 0
1255 }
1256
1257 #[must_use]
1259 pub fn try_from_array(array: &dyn Array) -> Option<StringColumnRef<'_>> {
1260 if let Some(array) = array.as_any().downcast_ref::<DictionaryArray<Int8Type>>()
1261 && let Some(values) = array.values().as_any().downcast_ref::<StringArray>()
1262 {
1263 return Some(StringColumnRef::DictionaryInt8(array, values));
1264 }
1265
1266 if let Some(array) = array.as_any().downcast_ref::<DictionaryArray<Int32Type>>()
1267 && let Some(values) = array.values().as_any().downcast_ref::<StringArray>()
1268 {
1269 return Some(StringColumnRef::DictionaryInt32(array, values));
1270 }
1271
1272 if let Some(array) = array.as_any().downcast_ref::<StringArray>() {
1273 return Some(StringColumnRef::Utf8(array));
1274 }
1275
1276 array
1277 .as_any()
1278 .downcast_ref::<StringViewArray>()
1279 .map(StringColumnRef::Utf8View)
1280 }
1281
1282 #[inline]
1288 #[must_use]
1289 pub fn value(&self, i: usize) -> &str {
1290 match self {
1291 Self::DictionaryInt8(array, values) => {
1292 let key = usize::try_from(array.keys().value(i)).expect("Int8 key is non-negative");
1293 values.value(key)
1294 }
1295 Self::DictionaryInt32(array, values) => {
1296 let key =
1297 usize::try_from(array.keys().value(i)).expect("Int32 key is non-negative");
1298 values.value(key)
1299 }
1300 Self::Utf8(arr) => arr.value(i),
1301 Self::Utf8View(arr) => arr.value(i),
1302 }
1303 }
1304
1305 #[inline]
1307 #[must_use]
1308 pub fn is_null(&self, i: usize) -> bool {
1309 match self {
1310 Self::DictionaryInt8(array, _) => array.is_null(i),
1311 Self::DictionaryInt32(array, _) => array.is_null(i),
1312 Self::Utf8(arr) => arr.is_null(i),
1313 Self::Utf8View(arr) => arr.is_null(i),
1314 }
1315 }
1316
1317 #[must_use]
1319 pub fn value_opt(&self, i: usize) -> Option<&str> {
1320 (!self.is_null(i)).then(|| self.value(i))
1321 }
1322}
1323
1324#[derive(Debug)]
1326pub enum U64ColumnRef<'a> {
1327 UInt64(&'a UInt64Array),
1328 Int64(&'a Int64Array),
1329 TimestampNanosecond(&'a TimestampNanosecondArray),
1330}
1331
1332impl U64ColumnRef<'_> {
1333 #[must_use]
1335 pub fn try_from_array(array: &dyn Array) -> Option<U64ColumnRef<'_>> {
1336 if let Some(array) = array.as_any().downcast_ref::<UInt64Array>() {
1337 return Some(U64ColumnRef::UInt64(array));
1338 }
1339
1340 if let Some(array) = array.as_any().downcast_ref::<Int64Array>() {
1341 return Some(U64ColumnRef::Int64(array));
1342 }
1343
1344 array
1345 .as_any()
1346 .downcast_ref::<TimestampNanosecondArray>()
1347 .map(U64ColumnRef::TimestampNanosecond)
1348 }
1349
1350 #[must_use]
1352 pub fn len(&self) -> usize {
1353 match self {
1354 Self::UInt64(array) => array.len(),
1355 Self::Int64(array) => array.len(),
1356 Self::TimestampNanosecond(array) => array.len(),
1357 }
1358 }
1359
1360 #[must_use]
1362 pub fn is_empty(&self) -> bool {
1363 self.len() == 0
1364 }
1365
1366 #[must_use]
1368 pub fn is_null(&self, i: usize) -> bool {
1369 match self {
1370 Self::UInt64(array) => array.is_null(i),
1371 Self::Int64(array) => array.is_null(i),
1372 Self::TimestampNanosecond(array) => array.is_null(i),
1373 }
1374 }
1375
1376 #[must_use]
1378 pub fn value(&self, i: usize) -> Option<u64> {
1379 match self {
1380 Self::UInt64(array) => Some(array.value(i)),
1381 Self::Int64(array) => u64::try_from(array.value(i)).ok(),
1382 Self::TimestampNanosecond(array) => u64::try_from(array.value(i)).ok(),
1383 }
1384 }
1385}
1386
1387#[derive(Debug)]
1389pub enum U32ColumnRef<'a> {
1390 UInt32(&'a UInt32Array),
1391 Int32(&'a Int32Array),
1392 Int64(&'a Int64Array),
1393}
1394
1395impl U32ColumnRef<'_> {
1396 #[must_use]
1398 pub fn try_from_array(array: &dyn Array) -> Option<U32ColumnRef<'_>> {
1399 if let Some(array) = array.as_any().downcast_ref::<UInt32Array>() {
1400 return Some(U32ColumnRef::UInt32(array));
1401 }
1402
1403 if let Some(array) = array.as_any().downcast_ref::<Int32Array>() {
1404 return Some(U32ColumnRef::Int32(array));
1405 }
1406
1407 array
1408 .as_any()
1409 .downcast_ref::<Int64Array>()
1410 .map(U32ColumnRef::Int64)
1411 }
1412}
1413
1414pub fn extract_column_binary<'a>(
1424 cols: &'a [ArrayRef],
1425 column_key: &'static str,
1426 column_index: usize,
1427) -> Result<BinaryColumnRef<'a>, EncodingError> {
1428 let column_values = cols
1429 .get(column_index)
1430 .ok_or(EncodingError::MissingColumn(column_key, column_index))?;
1431 let dt = column_values.data_type();
1432 if let Some(arr) = column_values.as_any().downcast_ref::<BinaryArray>() {
1433 Ok(BinaryColumnRef::Binary(arr))
1434 } else if let Some(arr) = column_values.as_any().downcast_ref::<BinaryViewArray>() {
1435 Ok(BinaryColumnRef::BinaryView(arr))
1436 } else {
1437 Err(EncodingError::InvalidColumnType(
1438 column_key,
1439 column_index,
1440 DataType::Binary,
1441 dt.clone(),
1442 ))
1443 }
1444}
1445
1446#[derive(Debug)]
1448pub enum BinaryColumnRef<'a> {
1449 Binary(&'a BinaryArray),
1450 BinaryView(&'a BinaryViewArray),
1451}
1452
1453impl BinaryColumnRef<'_> {
1454 #[inline]
1456 #[must_use]
1457 pub fn is_null(&self, i: usize) -> bool {
1458 match self {
1459 Self::Binary(arr) => arr.is_null(i),
1460 Self::BinaryView(arr) => arr.is_null(i),
1461 }
1462 }
1463
1464 #[inline]
1466 #[must_use]
1467 pub fn value(&self, i: usize) -> &[u8] {
1468 match self {
1469 Self::Binary(arr) => arr.value(i),
1470 Self::BinaryView(arr) => arr.value(i),
1471 }
1472 }
1473}
1474
1475pub fn extract_column<'a, T: Array + 'static>(
1483 cols: &'a [ArrayRef],
1484 column_key: &'static str,
1485 column_index: usize,
1486 expected_type: DataType,
1487) -> Result<&'a T, EncodingError> {
1488 let column_values = cols
1489 .get(column_index)
1490 .ok_or(EncodingError::MissingColumn(column_key, column_index))?;
1491
1492 let downcasted_values = column_values
1493 .as_any()
1494 .downcast_ref::<T>()
1495 .filter(|_| column_values.data_type() == &expected_type)
1496 .ok_or(EncodingError::InvalidColumnType(
1497 column_key,
1498 column_index,
1499 expected_type,
1500 column_values.data_type().clone(),
1501 ))?;
1502 Ok(downcasted_values)
1503}
1504
1505pub fn extract_column_by_name<'a, T: Array + 'static>(
1511 record_batch: &'a RecordBatch,
1512 column_key: &'static str,
1513 expected_type: DataType,
1514) -> Result<&'a T, EncodingError> {
1515 let column_index = record_batch.schema().index_of(column_key)?;
1516 extract_column::<T>(
1517 record_batch.columns(),
1518 column_key,
1519 column_index,
1520 expected_type,
1521 )
1522}
1523
1524pub fn extract_decimal_column<'a>(
1530 record_batch: &'a RecordBatch,
1531 column_key: &'static str,
1532) -> Result<&'a Decimal128Array, EncodingError> {
1533 let column_index = record_batch.schema().index_of(column_key)?;
1534 extract_column(
1535 record_batch.columns(),
1536 column_key,
1537 column_index,
1538 fixed_decimal_data_type(),
1539 )
1540}
1541
1542pub fn extract_optional_string_column_by_name<'a>(
1548 record_batch: &'a RecordBatch,
1549 column_key: &'static str,
1550) -> Result<Option<&'a StringArray>, EncodingError> {
1551 let Ok(column_index) = record_batch.schema().index_of(column_key) else {
1552 return Ok(None);
1553 };
1554 let column_values = record_batch
1555 .columns()
1556 .get(column_index)
1557 .ok_or(EncodingError::MissingColumn(column_key, column_index))?;
1558 let downcasted_values = column_values.as_any().downcast_ref::<StringArray>().ok_or(
1559 EncodingError::InvalidColumnType(
1560 column_key,
1561 column_index,
1562 DataType::Utf8,
1563 column_values.data_type().clone(),
1564 ),
1565 )?;
1566 Ok(Some(downcasted_values))
1567}
1568
1569#[must_use]
1571pub fn optional_ustr_value(values: Option<&StringArray>, row: usize) -> Option<Ustr> {
1572 values.and_then(|column| (!column.is_null(row)).then(|| Ustr::from(column.value(row))))
1573}
1574
1575pub fn book_deltas_to_arrow_record_batch_bytes(
1584 data: &[OrderBookDelta],
1585) -> Result<RecordBatch, EncodingError> {
1586 let Some(first) = data.first() else {
1587 return Err(EncodingError::EmptyData);
1588 };
1589
1590 let metadata = OrderBookDelta::chunk_metadata(data);
1591 let instrument_id = data
1592 .iter()
1593 .find(|delta| delta.action != BookAction::Clear)
1594 .unwrap_or(first)
1595 .instrument_id;
1596
1597 if let Some(index) = data.iter().position(|delta| {
1598 delta.instrument_id != instrument_id
1599 || (delta.action != BookAction::Clear && delta.metadata() != metadata)
1600 }) {
1601 return Err(EncodingError::MixedMetadata { index });
1602 }
1603
1604 OrderBookDelta::encode_batch(&metadata, data).map_err(EncodingError::ArrowError)
1605}
1606
1607pub fn book_depths_to_arrow_record_batch_bytes(
1616 data: &[OrderBookDepth],
1617) -> Result<RecordBatch, EncodingError> {
1618 if data.is_empty() {
1619 return Err(EncodingError::EmptyData);
1620 }
1621 let metadata = OrderBookDepth::chunk_metadata(data);
1622
1623 if let Some(index) = data
1624 .iter()
1625 .position(|depth| !depth.matches_chunk_metadata(&metadata))
1626 {
1627 return Err(EncodingError::MixedMetadata { index });
1628 }
1629
1630 OrderBookDepth::encode_batch(&metadata, data).map_err(EncodingError::ArrowError)
1631}
1632
1633pub fn quotes_to_arrow_record_batch_bytes(
1642 data: &[QuoteTick],
1643) -> Result<RecordBatch, EncodingError> {
1644 encode_batch_with_metadata(data)
1645}
1646
1647pub fn trades_to_arrow_record_batch_bytes(
1656 data: &[TradeTick],
1657) -> Result<RecordBatch, EncodingError> {
1658 encode_batch_with_metadata(data)
1659}
1660
1661pub fn bars_to_arrow_record_batch_bytes(data: &[Bar]) -> Result<RecordBatch, EncodingError> {
1670 encode_batch_with_metadata(data)
1671}
1672
1673pub fn mark_prices_to_arrow_record_batch_bytes(
1682 data: &[MarkPriceUpdate],
1683) -> Result<RecordBatch, EncodingError> {
1684 encode_batch_with_metadata(data)
1685}
1686
1687pub fn index_prices_to_arrow_record_batch_bytes(
1696 data: &[IndexPriceUpdate],
1697) -> Result<RecordBatch, EncodingError> {
1698 encode_batch_with_metadata(data)
1699}
1700
1701pub fn instrument_status_to_arrow_record_batch_bytes(
1710 data: &[InstrumentStatus],
1711) -> Result<RecordBatch, EncodingError> {
1712 encode_batch_with_metadata(data)
1713}
1714
1715pub fn option_greeks_to_arrow_record_batch_bytes(
1724 data: &[OptionGreeks],
1725) -> Result<RecordBatch, EncodingError> {
1726 encode_batch_with_metadata(data)
1727}
1728
1729pub fn instrument_closes_to_arrow_record_batch_bytes(
1738 data: &[InstrumentClose],
1739) -> Result<RecordBatch, EncodingError> {
1740 encode_batch_with_metadata(data)
1741}
1742
1743pub(crate) fn encode_batch_with_metadata<T>(data: &[T]) -> Result<RecordBatch, EncodingError>
1744where
1745 T: EncodeToRecordBatch,
1746{
1747 if data.is_empty() {
1748 return Err(EncodingError::EmptyData);
1749 }
1750
1751 let metadata = T::chunk_metadata(data);
1752 if let Some(index) = data.iter().position(|value| value.metadata() != metadata) {
1753 return Err(EncodingError::MixedMetadata { index });
1754 }
1755
1756 T::encode_batch(&metadata, data).map_err(EncodingError::ArrowError)
1757}
1758
1759#[cfg(test)]
1760mod tests {
1761 use nautilus_model::{
1762 data::{
1763 Bar, BarSpecification, BarType, BookOrder, DEPTH10_LEN, OrderBookDelta, OrderBookDepth,
1764 QuoteTick, order::NULL_ORDER,
1765 },
1766 enums::{
1767 AggregationSource, BarAggregation, BookAction, MarketStatusAction, OrderSide, PriceType,
1768 },
1769 identifiers::InstrumentId,
1770 types::{Price, Quantity},
1771 };
1772 use rstest::rstest;
1773
1774 use super::*;
1775
1776 #[cfg(feature = "high-precision")]
1777 #[rstest]
1778 fn test_money_rejects_currency_precision_above_catalog_scale() {
1779 use nautilus_model::enums::CurrencyType;
1780
1781 if nautilus_model::types::fixed::check_fixed_precision(18).is_err() {
1782 return;
1783 }
1784
1785 let currency = Currency::new("TST18", 18, 0, "Test token", CurrencyType::Crypto);
1786 let value = Money::from_raw(1_000_000_000_000_000_000, currency);
1787 let error = money_array([Some(value)]).unwrap_err();
1788
1789 assert_eq!(
1790 error.to_string(),
1791 "Invalid argument error: Money currency precision 18 exceeds catalog scale 16"
1792 );
1793 }
1794
1795 #[rstest]
1796 fn test_encode_rejects_defi_precision_metadata() {
1797 let mut metadata = QuoteTick::get_metadata(
1798 &InstrumentId::from("WETH-USDC.UNISWAP"),
1799 FIXED_DECIMAL_SCALE as u8,
1800 0,
1801 );
1802 metadata.insert(KEY_PRICE_PRECISION.to_string(), "17".to_string());
1803 let schema = Arc::new(QuoteTick::get_schema(Some(metadata)));
1804
1805 let err = record_batch_with_timestamps(schema, vec![]).unwrap_err();
1806
1807 assert_eq!(
1808 err.to_string(),
1809 "Invalid argument error: Metadata 'price_precision' is 17, maximum supported catalog scale is 16"
1810 );
1811 }
1812
1813 #[rstest]
1814 fn test_record_batch_with_timestamps_rejects_surplus_columns() {
1815 let schema = Arc::new(Schema::new(vec![Field::new(
1816 "ts_event",
1817 timestamp_data_type(),
1818 false,
1819 )]));
1820 let columns = vec![
1821 Arc::new(UInt64Array::from(vec![1])) as ArrayRef,
1822 Arc::new(UInt64Array::from(vec![2])) as ArrayRef,
1823 ];
1824
1825 let error = record_batch_with_timestamps(schema, columns).unwrap_err();
1826
1827 assert_eq!(
1828 error.to_string(),
1829 "Invalid argument error: number of columns(2) must match number of fields(1) in schema"
1830 );
1831 }
1832
1833 #[rstest]
1834 fn test_timestamp_arrays_preserve_utc_nanoseconds_and_nulls() {
1835 let values = [Some(1_788_652_800_123_456_789), None, Some(i64::MAX as u64)];
1836 let array = optional_timestamp_array(values).unwrap();
1837
1838 assert_eq!(
1839 timestamp_data_type(),
1840 DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
1841 );
1842 assert_eq!(array.data_type(), ×tamp_data_type());
1843 assert_eq!(
1844 array.iter().collect::<Vec<_>>(),
1845 vec![Some(1_788_652_800_123_456_789), None, Some(i64::MAX)]
1846 );
1847 assert_eq!(
1848 decode_timestamp(&array, "ts_event", 0).unwrap(),
1849 1_788_652_800_123_456_789
1850 );
1851 assert_eq!(
1852 decode_timestamp(&array, "ts_event", 2).unwrap(),
1853 i64::MAX as u64
1854 );
1855 assert!(
1856 decode_timestamp(&TimestampNanosecondArray::from(vec![-1]), "ts_event", 0).is_err()
1857 );
1858 }
1859
1860 #[rstest]
1861 fn test_timestamp_array_rejects_unsigned_overflow() {
1862 let error = timestamp_array([u64::MAX]).unwrap_err();
1863
1864 assert_eq!(
1865 error.to_string(),
1866 "Invalid argument error: Nanosecond timestamp 18446744073709551615 exceeds Arrow's signed timestamp range",
1867 );
1868 }
1869
1870 #[rstest]
1871 fn test_quotes_to_arrow_record_batch_rejects_mixed_instruments() {
1872 let first = QuoteTick::new(
1873 InstrumentId::from("AAPL.XNAS"),
1874 Price::from("100.01"),
1875 Price::from("100.02"),
1876 Quantity::from("10"),
1877 Quantity::from("11"),
1878 1.into(),
1879 1.into(),
1880 );
1881 let second = QuoteTick::new(
1882 InstrumentId::from("MSFT.XNAS"),
1883 Price::from("200.01"),
1884 Price::from("200.02"),
1885 Quantity::from("20"),
1886 Quantity::from("21"),
1887 2.into(),
1888 2.into(),
1889 );
1890
1891 let result = quotes_to_arrow_record_batch_bytes(&[first, second]);
1892
1893 assert!(matches!(
1894 result,
1895 Err(EncodingError::MixedMetadata { index: 1 })
1896 ));
1897 }
1898
1899 #[rstest]
1900 fn test_instrument_status_to_arrow_record_batch_rejects_mixed_instruments() {
1901 let status = |instrument_id: &str, ts: u64| {
1902 InstrumentStatus::new(
1903 InstrumentId::from(instrument_id),
1904 MarketStatusAction::Trading,
1905 ts.into(),
1906 ts.into(),
1907 None,
1908 None,
1909 None,
1910 None,
1911 None,
1912 )
1913 };
1914
1915 let result = instrument_status_to_arrow_record_batch_bytes(&[
1916 status("AAA.XNAS", 1),
1917 status("BBB.XNAS", 2),
1918 ]);
1919
1920 assert!(matches!(
1921 result,
1922 Err(EncodingError::MixedMetadata { index: 1 })
1923 ));
1924 }
1925
1926 #[rstest]
1927 fn test_option_greeks_to_arrow_record_batch_rejects_mixed_instruments() {
1928 let greeks = |instrument_id: &str| OptionGreeks {
1929 instrument_id: InstrumentId::from(instrument_id),
1930 ..OptionGreeks::default()
1931 };
1932
1933 let result = option_greeks_to_arrow_record_batch_bytes(&[
1934 greeks("BTC-20260529-100000-C.OKX"),
1935 greeks("BTC-20260529-110000-C.OKX"),
1936 ]);
1937
1938 assert!(matches!(
1939 result,
1940 Err(EncodingError::MixedMetadata { index: 1 })
1941 ));
1942 }
1943
1944 #[rstest]
1945 fn test_quotes_to_arrow_record_batch_rejects_mixed_precision() {
1946 let instrument_id = InstrumentId::from("AAPL.XNAS");
1947 let first = QuoteTick::new(
1948 instrument_id,
1949 Price::from("100.01"),
1950 Price::from("100.02"),
1951 Quantity::from("10.00"),
1952 Quantity::from("11.00"),
1953 1.into(),
1954 1.into(),
1955 );
1956 let second = QuoteTick::new(
1957 instrument_id,
1958 Price::from("100.010"),
1959 Price::from("100.020"),
1960 Quantity::from("10.000"),
1961 Quantity::from("11.000"),
1962 2.into(),
1963 2.into(),
1964 );
1965
1966 let result = quotes_to_arrow_record_batch_bytes(&[first, second]);
1967
1968 assert!(matches!(
1969 result,
1970 Err(EncodingError::MixedMetadata { index: 1 })
1971 ));
1972 }
1973
1974 #[rstest]
1975 fn test_bars_to_arrow_record_batch_rejects_mixed_bar_types() {
1976 let instrument_id = InstrumentId::from("AAPL.XNAS");
1977 let first_type = BarType::new(
1978 instrument_id,
1979 BarSpecification::new(1, BarAggregation::Minute, PriceType::Last),
1980 AggregationSource::Internal,
1981 );
1982 let second_type = BarType::new(
1983 instrument_id,
1984 BarSpecification::new(5, BarAggregation::Minute, PriceType::Last),
1985 AggregationSource::Internal,
1986 );
1987 let first = Bar::new(
1988 first_type,
1989 Price::from("100.01"),
1990 Price::from("100.02"),
1991 Price::from("100.00"),
1992 Price::from("100.01"),
1993 Quantity::from("10"),
1994 1.into(),
1995 1.into(),
1996 );
1997 let second = Bar::new(
1998 second_type,
1999 Price::from("100.01"),
2000 Price::from("100.02"),
2001 Price::from("100.00"),
2002 Price::from("100.01"),
2003 Quantity::from("11"),
2004 2.into(),
2005 2.into(),
2006 );
2007
2008 let result = bars_to_arrow_record_batch_bytes(&[first, second]);
2009
2010 assert!(matches!(
2011 result,
2012 Err(EncodingError::MixedMetadata { index: 1 })
2013 ));
2014 }
2015
2016 #[rstest]
2017 fn test_depths_to_arrow_record_batch_rejects_mixed_level_price_precision() {
2018 let instrument_id = InstrumentId::from("AUD/USD.SIM");
2019 let bid = BookOrder::new(
2020 OrderSide::Buy,
2021 Price::from("1.23"),
2022 Quantity::from("100.00"),
2023 1,
2024 );
2025 let ask = BookOrder::new(
2026 OrderSide::Sell,
2027 Price::from("1.24"),
2028 Quantity::from("100.00"),
2029 2,
2030 );
2031 let mut asks = [ask; DEPTH10_LEN];
2032 asks[1].price = Price::from("1.241");
2033 let depth = OrderBookDepth::new(
2034 instrument_id,
2035 [bid; DEPTH10_LEN],
2036 asks,
2037 [1; DEPTH10_LEN],
2038 [1; DEPTH10_LEN],
2039 0,
2040 1,
2041 1.into(),
2042 1.into(),
2043 );
2044
2045 let result = book_depths_to_arrow_record_batch_bytes(&[depth]);
2046
2047 assert!(matches!(
2048 result,
2049 Err(EncodingError::MixedMetadata { index: 0 })
2050 ));
2051 }
2052
2053 #[rstest]
2054 fn test_depths_to_arrow_record_batch_rejects_mixed_level_size_precision() {
2055 let instrument_id = InstrumentId::from("AUD/USD.SIM");
2056 let bid = BookOrder::new(
2057 OrderSide::Buy,
2058 Price::from("1.23"),
2059 Quantity::from("100.00"),
2060 1,
2061 );
2062 let ask = BookOrder::new(
2063 OrderSide::Sell,
2064 Price::from("1.24"),
2065 Quantity::from("100.00"),
2066 2,
2067 );
2068 let mut bids = [bid; DEPTH10_LEN];
2069 bids[1].size = Quantity::from("100.000");
2070 let depth = OrderBookDepth::new(
2071 instrument_id,
2072 bids,
2073 [ask; DEPTH10_LEN],
2074 [1; DEPTH10_LEN],
2075 [1; DEPTH10_LEN],
2076 0,
2077 1,
2078 1.into(),
2079 1.into(),
2080 );
2081
2082 let result = book_depths_to_arrow_record_batch_bytes(&[depth]);
2083
2084 assert!(matches!(
2085 result,
2086 Err(EncodingError::MixedMetadata { index: 0 })
2087 ));
2088 }
2089
2090 #[rstest]
2091 fn test_depths_to_arrow_record_batch_uses_first_defined_level_precision() {
2092 let instrument_id = InstrumentId::from("AUD/USD.SIM");
2093 let bid = BookOrder::new(
2094 OrderSide::Buy,
2095 Price::from("1.23"),
2096 Quantity::from("100.00"),
2097 1,
2098 );
2099 let ask = BookOrder::new(
2100 OrderSide::Sell,
2101 Price::from("1.24"),
2102 Quantity::from("100.00"),
2103 2,
2104 );
2105 let mut bids = [bid; DEPTH10_LEN];
2106 bids[0] = NULL_ORDER;
2107 let depth = OrderBookDepth::new(
2108 instrument_id,
2109 bids,
2110 [ask; DEPTH10_LEN],
2111 [0; DEPTH10_LEN],
2112 [1; DEPTH10_LEN],
2113 0,
2114 1,
2115 1.into(),
2116 1.into(),
2117 );
2118
2119 let result = book_depths_to_arrow_record_batch_bytes(&[depth]).unwrap();
2120
2121 assert_eq!(
2122 result.schema().metadata().get(KEY_PRICE_PRECISION).unwrap(),
2123 "2"
2124 );
2125 assert_eq!(
2126 result.schema().metadata().get(KEY_SIZE_PRECISION).unwrap(),
2127 "2"
2128 );
2129 }
2130
2131 #[rstest]
2132 fn test_deltas_to_arrow_record_batch_skips_leading_clears_for_precision() {
2133 let instrument_id = InstrumentId::from("AUD/USD.SIM");
2134 let first = OrderBookDelta::clear(instrument_id, 0, 1.into(), 1.into());
2135 let second = OrderBookDelta::clear(instrument_id, 1, 2.into(), 2.into());
2136 let third = OrderBookDelta::new(
2137 instrument_id,
2138 BookAction::Add,
2139 BookOrder::new(
2140 OrderSide::Buy,
2141 Price::from("1.23"),
2142 Quantity::from("100.000000"),
2143 1,
2144 ),
2145 0,
2146 2,
2147 3.into(),
2148 3.into(),
2149 );
2150 let expected = vec![first, second, third];
2151
2152 let batch = book_deltas_to_arrow_record_batch_bytes(&expected).unwrap();
2153 let metadata = batch.schema().metadata().clone();
2154 assert_eq!(
2155 metadata.get(KEY_PRICE_PRECISION).map(String::as_str),
2156 Some("2")
2157 );
2158 assert_eq!(
2159 metadata.get(KEY_SIZE_PRECISION).map(String::as_str),
2160 Some("6")
2161 );
2162
2163 let decoded = OrderBookDelta::decode_batch(&metadata, batch).unwrap();
2164
2165 assert_eq!(decoded, expected);
2166 assert_eq!(decoded[2].order.price.precision, 2);
2167 assert_eq!(decoded[2].order.size.precision, 6);
2168 }
2169
2170 #[rstest]
2171 fn test_deltas_to_arrow_record_batch_all_clear_roundtrip() {
2172 let instrument_id = InstrumentId::from("AUD/USD.SIM");
2173 let expected = vec![
2174 OrderBookDelta::clear(instrument_id, 0, 1.into(), 1.into()),
2175 OrderBookDelta::clear(instrument_id, 1, 2.into(), 2.into()),
2176 ];
2177
2178 let batch = book_deltas_to_arrow_record_batch_bytes(&expected).unwrap();
2179 let metadata = batch.schema().metadata().clone();
2180 let decoded = OrderBookDelta::decode_batch(&metadata, batch).unwrap();
2181
2182 assert_eq!(decoded, expected);
2183 }
2184
2185 #[rstest]
2186 fn test_deltas_to_arrow_record_batch_rejects_mixed_precision() {
2187 let instrument_id = InstrumentId::from("AUD/USD.SIM");
2188 let first = OrderBookDelta::new(
2189 instrument_id,
2190 BookAction::Add,
2191 BookOrder::new(
2192 OrderSide::Buy,
2193 Price::from("1.23"),
2194 Quantity::from("100.00"),
2195 1,
2196 ),
2197 0,
2198 1,
2199 1.into(),
2200 1.into(),
2201 );
2202 let second = OrderBookDelta::new(
2203 instrument_id,
2204 BookAction::Update,
2205 BookOrder::new(
2206 OrderSide::Buy,
2207 Price::from("1.234"),
2208 Quantity::from("100.000"),
2209 1,
2210 ),
2211 0,
2212 2,
2213 2.into(),
2214 2.into(),
2215 );
2216
2217 let result = book_deltas_to_arrow_record_batch_bytes(&[first, second]);
2218
2219 assert!(matches!(
2220 result,
2221 Err(EncodingError::MixedMetadata { index: 1 })
2222 ));
2223 }
2224
2225 #[rstest]
2226 fn test_deltas_to_arrow_record_batch_rejects_mixed_instruments() {
2227 let first = OrderBookDelta::clear(InstrumentId::from("AUD/USD.SIM"), 0, 1.into(), 1.into());
2228 let second = OrderBookDelta::new(
2229 InstrumentId::from("EUR/USD.SIM"),
2230 BookAction::Add,
2231 BookOrder::new(
2232 OrderSide::Buy,
2233 Price::from("1.23"),
2234 Quantity::from("100.00"),
2235 1,
2236 ),
2237 0,
2238 1,
2239 2.into(),
2240 2.into(),
2241 );
2242
2243 let result = book_deltas_to_arrow_record_batch_bytes(&[first, second]);
2244
2245 assert!(matches!(
2247 result,
2248 Err(EncodingError::MixedMetadata { index: 0 })
2249 ));
2250 }
2251}
2252
2253#[cfg(test)]
2254mod schema_invariant_tests {
2255 use arrow::{
2256 array::{Array, Decimal128Array},
2257 datatypes::{DataType, Field, Schema},
2258 };
2259 use nautilus_model::{
2260 data::{
2261 BookOrder, FundingRateUpdate, IndexPriceUpdate, InstrumentStatus, MarkPriceUpdate,
2262 OptionGreeks,
2263 bar::Bar,
2264 close::InstrumentClose,
2265 delta::OrderBookDelta,
2266 depth::OrderBookDepth,
2267 quote::QuoteTick,
2268 stubs::{stub_bar, stub_depth10},
2269 trade::TradeTick,
2270 },
2271 enums::{AggressorSide, BookAction, InstrumentCloseType, OrderSide},
2272 events::{
2273 AccountState, OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled,
2274 OrderDenied, OrderEmulated, OrderExpired, OrderFillVoided, OrderFilled,
2275 OrderInitialized, OrderModifyDenied, OrderModifyRejected, OrderPendingCancel,
2276 OrderPendingUpdate, OrderRejected, OrderReleased, OrderSnapshot, OrderSubmitted,
2277 OrderTriggered, OrderUpdated, PositionAdjusted, PositionChanged, PositionClosed,
2278 PositionOpened, PositionSnapshot,
2279 },
2280 identifiers::{InstrumentId, TradeId},
2281 instruments::{
2282 InstrumentAny, betting::BettingInstrument, binary_option::BinaryOption, cfd::Cfd,
2283 commodity::Commodity, crypto_future::CryptoFuture,
2284 crypto_futures_spread::CryptoFuturesSpread, crypto_option::CryptoOption,
2285 crypto_option_spread::CryptoOptionSpread, crypto_perpetual::CryptoPerpetual,
2286 currency_pair::CurrencyPair, equity::Equity, futures_contract::FuturesContract,
2287 futures_spread::FuturesSpread, index_instrument::IndexInstrument,
2288 option_contract::OptionContract, option_spread::OptionSpread,
2289 perpetual_contract::PerpetualContract, tokenized_asset::TokenizedAsset,
2290 },
2291 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
2292 types::{
2293 PRICE_ERROR, PRICE_UNDEF, Price, QUANTITY_UNDEF, Quantity,
2294 fixed::{FIXED_PRECISION, FIXED_PRECISION_STANDARD},
2295 },
2296 };
2297 use rstest::rstest;
2298 use rust_decimal::Decimal;
2299
2300 use super::{
2301 ArrowSchemaProvider, DecodeFromRecordBatch, EncodeToRecordBatch, FIXED_DECIMAL_PRECISION,
2302 FIXED_DECIMAL_SCALE, KEY_IDENTIFIER, QUANTITY_RAW_MAX, STANDARD_TO_DECIMAL_SCALE,
2303 decimal_to_arrow, decode_decimal, decode_decimal_price, decode_decimal_quantity,
2304 fixed_decimal_data_type, is_timestamp_field, price_decimal_array, quantity_decimal_array,
2305 timestamp_data_type,
2306 };
2307
2308 #[derive(Clone, Copy, Debug)]
2309 enum FixedFamily {
2310 Quote,
2311 Trade,
2312 Bar,
2313 Delta,
2314 Depth,
2315 MarkPrice,
2316 IndexPrice,
2317 Close,
2318 }
2319
2320 #[rstest]
2321 fn decimal_to_arrow_rejects_values_outside_decode_range() {
2322 let value = Decimal::from_i128_with_scale(8_000_000_000_000, 0);
2323
2324 let error = decimal_to_arrow(&value, "amount").unwrap_err();
2325
2326 assert!(error.to_string().contains("96-bit range"));
2327 }
2328
2329 #[rstest]
2330 fn decimal_to_arrow_normalizes_trailing_zero_scale() {
2331 let value = Decimal::from_i128_with_scale(10_000_000_000_000_000, 18);
2332 let encoded = decimal_to_arrow(&value, "amount").unwrap();
2333 let array = Decimal128Array::from(vec![encoded])
2334 .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
2335 .unwrap();
2336
2337 assert_eq!(
2338 decode_decimal(&array, "amount", 0).unwrap(),
2339 value.normalize()
2340 );
2341 }
2342
2343 macro_rules! collect_data_schemas {
2344 ($(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?) => {
2345 vec![
2346 $(
2347 (stringify!($type), <$type as ArrowSchemaProvider>::get_schema(None)),
2348 )+
2349 ]
2350 };
2351 }
2352
2353 macro_rules! assert_model_field_map {
2354 (InstrumentAny) => {};
2356 (InstrumentStatus) => {};
2358 (OptionGreeks) => {};
2360 (OrderBookDelta) => {
2361 assert_fields_match_schema(
2362 catalog_field_map(OrderBookDelta::get_fields()),
2363 &OrderBookDelta::get_schema(None),
2364 &["price", "size", KEY_IDENTIFIER],
2365 );
2366 };
2367 (OrderBookDepth) => {
2368 assert_fields_match_schema(
2369 catalog_field_map(OrderBookDepth::get_fields()),
2370 &OrderBookDepth::get_schema(None),
2371 &[KEY_IDENTIFIER],
2372 );
2373 };
2374 (QuoteTick) => {
2375 assert_fields_match_schema(
2376 catalog_field_map(QuoteTick::get_fields()),
2377 &QuoteTick::get_schema(None),
2378 &[KEY_IDENTIFIER],
2379 );
2380 };
2381 (TradeTick) => {
2382 assert_fields_match_schema(
2383 catalog_field_map(TradeTick::get_fields()),
2384 &TradeTick::get_schema(None),
2385 &[KEY_IDENTIFIER],
2386 );
2387 };
2388 (Bar) => {
2389 assert_fields_match_schema(
2390 catalog_field_map(Bar::get_fields()),
2391 &Bar::get_schema(None),
2392 &[KEY_IDENTIFIER],
2393 );
2394 };
2395 (MarkPriceUpdate) => {
2396 assert_fields_match_schema(
2397 catalog_field_map(MarkPriceUpdate::get_fields()),
2398 &MarkPriceUpdate::get_schema(None),
2399 &["value", KEY_IDENTIFIER],
2400 );
2401 };
2402 (IndexPriceUpdate) => {
2403 assert_fields_match_schema(
2404 catalog_field_map(IndexPriceUpdate::get_fields()),
2405 &IndexPriceUpdate::get_schema(None),
2406 &["value", KEY_IDENTIFIER],
2407 );
2408 };
2409 (FundingRateUpdate) => {
2410 assert_fields_match_schema(
2411 catalog_field_map(FundingRateUpdate::get_fields()),
2412 &FundingRateUpdate::get_schema(None),
2413 &["interval", "next_funding_ns", KEY_IDENTIFIER],
2414 );
2415 };
2416 (InstrumentClose) => {
2417 assert_fields_match_schema(
2418 catalog_field_map(InstrumentClose::get_fields()),
2419 &InstrumentClose::get_schema(None),
2420 &["close_price", KEY_IDENTIFIER],
2421 );
2422 };
2423 }
2424
2425 macro_rules! assert_registered_model_field_maps {
2426 ($(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?) => {
2427 $(assert_model_field_map!($type);)+
2428 };
2429 }
2430
2431 #[rstest]
2432 fn registered_write_schemas_have_no_opaque_byte_fields() {
2433 let mut schemas = nautilus_model::for_each_data_type!(collect_data_schemas);
2434 schemas.extend(instrument_schemas());
2435 schemas.extend(record_schemas());
2436
2437 for (name, schema) in schemas {
2438 assert_open_schema(name, &schema);
2439 }
2440 }
2441
2442 #[rstest]
2443 fn model_field_maps_match_encoder_schemas() {
2444 nautilus_model::for_each_data_type!(assert_registered_model_field_maps);
2445 }
2446
2447 #[rstest]
2448 fn fixed_point_storage_uses_uniform_scale() {
2449 let price = Price::from("1.23456789");
2450 let array = price_decimal_array([price.raw()], "price").unwrap();
2451
2452 assert_eq!(array.data_type(), &fixed_decimal_data_type());
2453 assert_eq!(
2454 decode_decimal_price(&array, price.precision, "price", 0).unwrap(),
2455 price,
2456 );
2457 }
2458
2459 #[rstest]
2460 #[case("price", false)]
2461 #[case("size", true)]
2462 fn decimal_decode_rejects_spurious_digits_with_row_context(
2463 #[case] field: &'static str,
2464 #[case] quantity: bool,
2465 ) {
2466 let array = Decimal128Array::from(vec![0, STANDARD_TO_DECIMAL_SCALE])
2467 .with_precision_and_scale(FIXED_DECIMAL_PRECISION, FIXED_DECIMAL_SCALE)
2468 .unwrap();
2469
2470 let error = if quantity {
2471 decode_decimal_quantity(&array, 0, field, 1).unwrap_err()
2472 } else {
2473 decode_decimal_price(&array, 0, field, 1).unwrap_err()
2474 };
2475
2476 let raw = if FIXED_PRECISION == FIXED_PRECISION_STANDARD {
2477 1
2478 } else {
2479 STANDARD_TO_DECIMAL_SCALE
2480 };
2481
2482 let scale = 10_i128.pow(u32::from(FIXED_PRECISION));
2483
2484 assert_eq!(
2485 error.to_string(),
2486 format!(
2487 "Error parsing `{field}`: row 1: Invalid fixed-point raw value {raw} for precision 0: \
2488 remainder {raw} when divided by scale {scale}. \
2489 Raw value should be a multiple of {scale}. \
2490 This indicates data corruption or incorrect precision/scaling upstream"
2491 )
2492 );
2493 }
2494
2495 #[rstest]
2496 fn undefined_fixed_point_values_encode_as_null() {
2497 let prices = price_decimal_array([PRICE_UNDEF], "price").unwrap();
2498 let quantities = quantity_decimal_array([QUANTITY_UNDEF], "quantity").unwrap();
2499
2500 assert!(prices.is_null(0));
2501 assert!(quantities.is_null(0));
2502 }
2503
2504 #[rstest]
2505 fn price_error_fails_with_field_and_value() {
2506 let error = price_decimal_array([PRICE_ERROR], "bid_price").unwrap_err();
2507
2508 assert_eq!(
2509 error.to_string(),
2510 format!(
2511 "Invalid argument error: Price field 'bid_price' contains PRICE_ERROR raw value {PRICE_ERROR}"
2512 ),
2513 );
2514 }
2515
2516 #[rstest]
2517 fn quantity_overflow_fails_with_field_and_value() {
2518 let raw = QUANTITY_RAW_MAX + 1;
2519 let error = quantity_decimal_array([raw], "bid_size").unwrap_err();
2520
2521 assert_eq!(
2522 error.to_string(),
2523 format!(
2524 "Invalid argument error: Quantity field 'bid_size' raw value {raw} exceeds QUANTITY_RAW_MAX={QUANTITY_RAW_MAX}"
2525 ),
2526 );
2527 }
2528
2529 #[cfg(not(feature = "high-precision"))]
2530 #[rstest]
2531 fn standard_precision_decode_rejects_nonzero_scale_remainder() {
2532 let array = Decimal128Array::from(vec![1_i128])
2533 .with_precision_and_scale(38, 16)
2534 .unwrap();
2535
2536 let error = decode_decimal_price(&array, 9, "bid_price", 0).unwrap_err();
2537
2538 assert_eq!(
2539 error.to_string(),
2540 "Error parsing `bid_price`: row 0: decimal value 1 has nonzero digits beyond build \
2541 precision 9",
2542 );
2543 }
2544
2545 #[cfg(feature = "high-precision")]
2546 #[rstest]
2547 fn high_precision_decimal_payload_is_bit_exact() {
2548 let raw = 1_234_567_890_123_456_i128;
2549 let array = price_decimal_array([raw], "price").unwrap();
2550
2551 assert_eq!(array.value(0), raw);
2552 assert_eq!(
2553 decode_decimal_price(&array, 16, "price", 0).unwrap().raw(),
2554 raw
2555 );
2556 }
2557
2558 #[rstest]
2559 #[case::quote(FixedFamily::Quote)]
2560 #[case::trade(FixedFamily::Trade)]
2561 #[case::bar(FixedFamily::Bar)]
2562 #[case::delta(FixedFamily::Delta)]
2563 #[case::depth(FixedFamily::Depth)]
2564 #[case::mark_price(FixedFamily::MarkPrice)]
2565 #[case::index_price(FixedFamily::IndexPrice)]
2566 #[case::close(FixedFamily::Close)]
2567 fn fixed_family_round_trips_multiple_precisions(#[case] family: FixedFamily) {
2568 round_trip_fixed_family(family, Price::from("1"), Quantity::from("2"));
2569 round_trip_fixed_family(family, Price::from("1.23456"), Quantity::from("2.34567"));
2570 #[cfg(feature = "high-precision")]
2571 round_trip_fixed_family(
2572 family,
2573 Price::from("1.2345678901234567"),
2574 Quantity::from("2.3456789012345678"),
2575 );
2576 }
2577
2578 #[rstest]
2579 #[case::quote(FixedFamily::Quote)]
2580 #[case::trade(FixedFamily::Trade)]
2581 #[case::bar(FixedFamily::Bar)]
2582 #[case::delta(FixedFamily::Delta)]
2583 #[case::mark_price(FixedFamily::MarkPrice)]
2584 #[case::index_price(FixedFamily::IndexPrice)]
2585 #[case::close(FixedFamily::Close)]
2586 fn fixed_family_sentinel_encoding(#[case] family: FixedFamily) {
2587 let instrument_id = InstrumentId::from("SENTINEL.TEST");
2588 let price = Price::from_raw(PRICE_UNDEF, 0);
2589 let quantity = Quantity::from_raw(QUANTITY_UNDEF, 0);
2590
2591 match family {
2592 FixedFamily::Quote => {
2593 let value = QuoteTick::new(
2594 instrument_id,
2595 price,
2596 Price::from("1"),
2597 quantity,
2598 Quantity::from("1"),
2599 1.into(),
2600 2.into(),
2601 );
2602 let metadata = QuoteTick::get_metadata(&instrument_id, 0, 0);
2603 let error = QuoteTick::encode_batch(&metadata, &[value]).unwrap_err();
2604 assert!(error.to_string().contains("bid_price"));
2605 assert!(error.to_string().contains("PRICE_UNDEF"));
2606 }
2607 FixedFamily::Trade => {
2608 let value = TradeTick {
2609 instrument_id,
2610 price,
2611 size: quantity,
2612 aggressor_side: AggressorSide::Buy,
2613 trade_id: TradeId::from("sentinel"),
2614 ts_event: 1.into(),
2615 ts_init: 2.into(),
2616 };
2617 let metadata = TradeTick::get_metadata(&instrument_id, 0, 0);
2618 let error = TradeTick::encode_batch(&metadata, &[value]).unwrap_err();
2619 assert!(error.to_string().contains("price"));
2620 assert!(error.to_string().contains("PRICE_UNDEF"));
2621 }
2622 FixedFamily::Bar => {
2623 let mut value = stub_bar();
2624 value.open = price;
2625 value.volume = quantity;
2626 let metadata = Bar::get_metadata(&value.bar_type, 0, 0);
2627 let error = Bar::encode_batch(&metadata, &[value]).unwrap_err();
2628 assert!(error.to_string().contains("open"));
2629 assert!(error.to_string().contains("PRICE_UNDEF"));
2630 }
2631 FixedFamily::Delta => {
2632 let value = OrderBookDelta {
2633 instrument_id,
2634 action: BookAction::Update,
2635 order: BookOrder {
2636 side: OrderSide::Buy.into(),
2637 price,
2638 size: quantity,
2639 order_id: 1,
2640 },
2641 flags: 0,
2642 sequence: 1,
2643 ts_event: 1.into(),
2644 ts_init: 2.into(),
2645 };
2646 let metadata = OrderBookDelta::get_metadata(&instrument_id, 0, 0);
2647 let batch = OrderBookDelta::encode_batch(&metadata, &[value]).unwrap();
2648 assert!(batch.column_by_name("price").unwrap().is_null(0));
2649 assert!(batch.column_by_name("size").unwrap().is_null(0));
2650 assert_eq!(
2651 OrderBookDelta::decode_batch(&metadata, batch).unwrap(),
2652 vec![value],
2653 );
2654 }
2655 FixedFamily::Depth => unreachable!("depth sides omit absent levels"),
2656 FixedFamily::MarkPrice => {
2657 let value = MarkPriceUpdate::new(instrument_id, price, 1.into(), 2.into());
2658 let metadata = MarkPriceUpdate::get_metadata(&instrument_id, 0);
2659 let batch = MarkPriceUpdate::encode_batch(&metadata, &[value]).unwrap();
2660 assert!(batch.column_by_name("value").unwrap().is_null(0));
2661 assert_eq!(
2662 MarkPriceUpdate::decode_batch(&metadata, batch).unwrap(),
2663 vec![value],
2664 );
2665 }
2666 FixedFamily::IndexPrice => {
2667 let value = IndexPriceUpdate::new(instrument_id, price, 1.into(), 2.into());
2668 let metadata = IndexPriceUpdate::get_metadata(&instrument_id, 0);
2669 let batch = IndexPriceUpdate::encode_batch(&metadata, &[value]).unwrap();
2670 assert!(batch.column_by_name("value").unwrap().is_null(0));
2671 assert_eq!(
2672 IndexPriceUpdate::decode_batch(&metadata, batch).unwrap(),
2673 vec![value],
2674 );
2675 }
2676 FixedFamily::Close => {
2677 let value = InstrumentClose::new(
2678 instrument_id,
2679 price,
2680 InstrumentCloseType::EndOfSession,
2681 1.into(),
2682 2.into(),
2683 );
2684 let metadata = InstrumentClose::get_metadata(&instrument_id, 0);
2685 let batch = InstrumentClose::encode_batch(&metadata, &[value]).unwrap();
2686 assert!(batch.column_by_name("close_price").unwrap().is_null(0));
2687 assert_eq!(
2688 InstrumentClose::decode_batch(&metadata, batch).unwrap(),
2689 vec![value],
2690 );
2691 }
2692 }
2693 }
2694
2695 fn round_trip_fixed_family(family: FixedFamily, price: Price, quantity: Quantity) {
2696 let instrument_id = InstrumentId::from("PRECISION.TEST");
2697
2698 match family {
2699 FixedFamily::Quote => {
2700 let value = QuoteTick::new(
2701 instrument_id,
2702 price,
2703 price,
2704 quantity,
2705 quantity,
2706 1.into(),
2707 2.into(),
2708 );
2709 let metadata =
2710 QuoteTick::get_metadata(&instrument_id, price.precision, quantity.precision);
2711 let batch = QuoteTick::encode_batch(&metadata, &[value]).unwrap();
2712 assert_eq!(
2713 QuoteTick::decode_batch(&metadata, batch).unwrap(),
2714 vec![value],
2715 );
2716 }
2717 FixedFamily::Trade => {
2718 let value = TradeTick::new(
2719 instrument_id,
2720 price,
2721 quantity,
2722 AggressorSide::Buy,
2723 TradeId::from("precision"),
2724 1.into(),
2725 2.into(),
2726 );
2727 let metadata =
2728 TradeTick::get_metadata(&instrument_id, price.precision, quantity.precision);
2729 let batch = TradeTick::encode_batch(&metadata, &[value]).unwrap();
2730 assert_eq!(
2731 TradeTick::decode_batch(&metadata, batch).unwrap(),
2732 vec![value],
2733 );
2734 }
2735 FixedFamily::Bar => {
2736 let mut value = stub_bar();
2737 value.open = price;
2738 value.high = price;
2739 value.low = price;
2740 value.close = price;
2741 value.volume = quantity;
2742 let metadata =
2743 Bar::get_metadata(&value.bar_type, price.precision, quantity.precision);
2744 let batch = Bar::encode_batch(&metadata, &[value]).unwrap();
2745 assert_eq!(Bar::decode_batch(&metadata, batch).unwrap(), vec![value]);
2746 }
2747 FixedFamily::Delta => {
2748 let value = OrderBookDelta::new(
2749 instrument_id,
2750 BookAction::Add,
2751 BookOrder {
2752 side: OrderSide::Buy.into(),
2753 price,
2754 size: quantity,
2755 order_id: 1,
2756 },
2757 0,
2758 1,
2759 1.into(),
2760 2.into(),
2761 );
2762 let metadata = OrderBookDelta::get_metadata(
2763 &instrument_id,
2764 price.precision,
2765 quantity.precision,
2766 );
2767 let batch = OrderBookDelta::encode_batch(&metadata, &[value]).unwrap();
2768 assert_eq!(
2769 OrderBookDelta::decode_batch(&metadata, batch).unwrap(),
2770 vec![value],
2771 );
2772 }
2773 FixedFamily::Depth => {
2774 let mut value = stub_depth10();
2775 for order in value.bids.iter_mut().chain(value.asks.iter_mut()) {
2776 order.price = price;
2777 order.size = quantity;
2778 }
2779 let metadata = OrderBookDepth::get_metadata(
2780 &value.instrument_id,
2781 price.precision,
2782 quantity.precision,
2783 );
2784 let batch = OrderBookDepth::encode_batch(&metadata, &[value.clone()]).unwrap();
2785 assert_eq!(
2786 OrderBookDepth::decode_batch(&metadata, batch).unwrap(),
2787 vec![value],
2788 );
2789 }
2790 FixedFamily::MarkPrice => {
2791 let value = MarkPriceUpdate::new(instrument_id, price, 1.into(), 2.into());
2792 let metadata = MarkPriceUpdate::get_metadata(&instrument_id, price.precision);
2793 let batch = MarkPriceUpdate::encode_batch(&metadata, &[value]).unwrap();
2794 assert_eq!(
2795 MarkPriceUpdate::decode_batch(&metadata, batch).unwrap(),
2796 vec![value],
2797 );
2798 }
2799 FixedFamily::IndexPrice => {
2800 let value = IndexPriceUpdate::new(instrument_id, price, 1.into(), 2.into());
2801 let metadata = IndexPriceUpdate::get_metadata(&instrument_id, price.precision);
2802 let batch = IndexPriceUpdate::encode_batch(&metadata, &[value]).unwrap();
2803 assert_eq!(
2804 IndexPriceUpdate::decode_batch(&metadata, batch).unwrap(),
2805 vec![value],
2806 );
2807 }
2808 FixedFamily::Close => {
2809 let value = InstrumentClose::new(
2810 instrument_id,
2811 price,
2812 InstrumentCloseType::EndOfSession,
2813 1.into(),
2814 2.into(),
2815 );
2816 let metadata = InstrumentClose::get_metadata(&instrument_id, price.precision);
2817 let batch = InstrumentClose::encode_batch(&metadata, &[value]).unwrap();
2818 assert_eq!(
2819 InstrumentClose::decode_batch(&metadata, batch).unwrap(),
2820 vec![value],
2821 );
2822 }
2823 }
2824 }
2825
2826 fn instrument_schemas() -> Vec<(&'static str, Schema)> {
2827 vec![
2829 schema::<BettingInstrument>(),
2830 schema::<BinaryOption>(),
2831 schema::<Cfd>(),
2832 schema::<Commodity>(),
2833 schema::<CryptoFuture>(),
2834 schema::<CryptoFuturesSpread>(),
2835 schema::<CryptoOption>(),
2836 schema::<CryptoOptionSpread>(),
2837 schema::<CryptoPerpetual>(),
2838 schema::<CurrencyPair>(),
2839 schema::<Equity>(),
2840 schema::<FuturesContract>(),
2841 schema::<FuturesSpread>(),
2842 schema::<IndexInstrument>(),
2843 schema::<OptionContract>(),
2844 schema::<OptionSpread>(),
2845 schema::<PerpetualContract>(),
2846 schema::<TokenizedAsset>(),
2847 ]
2848 }
2849
2850 fn record_schemas() -> Vec<(&'static str, Schema)> {
2851 vec![
2853 schema::<AccountState>(),
2854 schema::<OrderInitialized>(),
2855 schema::<OrderDenied>(),
2856 schema::<OrderEmulated>(),
2857 schema::<OrderSubmitted>(),
2858 schema::<OrderAccepted>(),
2859 schema::<OrderRejected>(),
2860 schema::<OrderPendingCancel>(),
2861 schema::<OrderCanceled>(),
2862 schema::<OrderCancelDenied>(),
2863 schema::<OrderCancelRejected>(),
2864 schema::<OrderExpired>(),
2865 schema::<OrderTriggered>(),
2866 schema::<OrderPendingUpdate>(),
2867 schema::<OrderReleased>(),
2868 schema::<OrderModifyDenied>(),
2869 schema::<OrderModifyRejected>(),
2870 schema::<OrderUpdated>(),
2871 schema::<OrderFilled>(),
2872 schema::<OrderFillVoided>(),
2873 schema::<PositionOpened>(),
2874 schema::<PositionChanged>(),
2875 schema::<PositionClosed>(),
2876 schema::<PositionAdjusted>(),
2877 schema::<OrderStatusReport>(),
2878 schema::<FillReport>(),
2879 schema::<PositionStatusReport>(),
2880 schema::<ExecutionMassStatus>(),
2881 schema::<OrderSnapshot>(),
2882 schema::<PositionSnapshot>(),
2883 ]
2884 }
2885
2886 fn schema<T: ArrowSchemaProvider>() -> (&'static str, Schema) {
2887 (std::any::type_name::<T>(), T::get_schema(None))
2888 }
2889
2890 fn catalog_field_map(
2891 fields: impl IntoIterator<Item = (String, String)>,
2892 ) -> Vec<(String, String)> {
2893 let mut fields = fields.into_iter().collect::<Vec<_>>();
2894 fields.push((KEY_IDENTIFIER.to_string(), "Utf8".to_string()));
2895 fields
2896 }
2897
2898 fn assert_fields_match_schema(
2899 expected: Vec<(String, String)>,
2900 schema: &Schema,
2901 nullable: &[&str],
2902 ) {
2903 let expected = expected
2904 .into_iter()
2905 .map(|(name, data_type)| {
2906 let is_nullable = nullable.contains(&name.as_str());
2907 (name, data_type, is_nullable)
2908 })
2909 .collect::<Vec<_>>();
2910 let actual: Vec<_> = schema
2911 .fields()
2912 .iter()
2913 .map(|field| {
2914 (
2915 field.name().clone(),
2916 arrow_type_name(field.data_type()),
2917 field.is_nullable(),
2918 )
2919 })
2920 .collect();
2921
2922 assert_eq!(actual, expected);
2923 }
2924
2925 fn arrow_type_name(data_type: &DataType) -> String {
2926 match data_type {
2927 DataType::List(field) => format!("List({})", arrow_type_name(field.data_type())),
2928 DataType::Struct(fields) => {
2929 let fields = fields
2930 .iter()
2931 .map(|field| {
2932 format!("{}: {}", field.name(), arrow_type_name(field.data_type()))
2933 })
2934 .collect::<Vec<_>>()
2935 .join(", ");
2936 format!("Struct({fields})")
2937 }
2938 _ => format!("{data_type:?}"),
2939 }
2940 }
2941
2942 fn assert_open_schema(name: &str, schema: &Schema) {
2943 for field in schema.fields() {
2944 assert_open_field(name, field);
2945 }
2946 }
2947
2948 fn assert_open_field(schema_name: &str, field: &Field) {
2949 if is_timestamp_field(field.name()) {
2950 assert_eq!(
2951 field.data_type(),
2952 ×tamp_data_type(),
2953 "write schema `{schema_name}` timestamp field `{}` is not a UTC nanosecond timestamp",
2954 field.name(),
2955 );
2956 }
2957
2958 match field.data_type() {
2959 DataType::Binary
2960 | DataType::LargeBinary
2961 | DataType::BinaryView
2962 | DataType::FixedSizeBinary(_) => {
2963 panic!(
2964 "write schema `{schema_name}` contains opaque byte field `{}`: {}",
2965 field.name(),
2966 field.data_type(),
2967 );
2968 }
2969 DataType::List(child)
2970 | DataType::LargeList(child)
2971 | DataType::ListView(child)
2972 | DataType::LargeListView(child)
2973 | DataType::FixedSizeList(child, _)
2974 | DataType::Map(child, _) => assert_open_field(schema_name, child),
2975 DataType::Struct(children) => {
2976 for child in children {
2977 assert_open_field(schema_name, child);
2978 }
2979 }
2980 _ => {}
2981 }
2982 }
2983}