1use std::{collections::HashMap, sync::Arc};
26
27use arrow::{
28 array::{
29 Array, FixedSizeListArray, GenericListArray, GenericListViewArray, LargeListArray,
30 LargeListViewArray, ListArray, ListViewArray, OffsetSizeTrait,
31 },
32 datatypes::{DataType as ArrowDataType, Schema},
33 record_batch::RecordBatch,
34};
35use nautilus_model::data::{
36 ArrowDecoder, ArrowEncoder, CustomData, CustomDataTrait, Data, DataType,
37 decode_custom_from_arrow, ensure_arrow_registered, ensure_custom_data_json_registered,
38 get_arrow_schema, validate_custom_arrow_schema,
39};
40
41use super::{ArrowSchemaProvider, DecodeDataFromRecordBatch, EncodeToRecordBatch};
42
43pub trait CustomDataSerialize: CustomDataTrait {
50 fn schema(&self) -> anyhow::Result<arrow::datatypes::Schema>;
55
56 fn encode_record_batch(
61 &self,
62 items: &[Arc<dyn CustomDataTrait>],
63 ) -> anyhow::Result<RecordBatch>;
64}
65
66pub fn ensure_custom_data_registered<T>()
75where
76 T: CustomDataTrait
77 + ArrowSchemaProvider
78 + EncodeToRecordBatch
79 + DecodeDataFromRecordBatch
80 + Clone
81 + Send
82 + Sync
83 + 'static,
84{
85 let type_name = T::type_name_static();
86
87 if let Some(schema) = get_arrow_schema(type_name) {
89 assert_custom_schema(type_name, &schema);
90 return;
91 }
92
93 let _ = ensure_custom_data_json_registered::<T>();
94
95 let schema = Arc::new(T::get_schema(None));
96 assert_custom_schema(type_name, &schema);
97
98 let encoder: ArrowEncoder = Box::new(move |items: &[Arc<dyn CustomDataTrait>]| {
99 let typed: Result<Vec<T>, _> = items
100 .iter()
101 .map(|b| {
102 b.as_any()
103 .downcast_ref::<T>()
104 .cloned()
105 .ok_or_else(|| anyhow::anyhow!("Expected {}", T::type_name_static()))
106 })
107 .collect();
108 let typed = typed?;
109
110 let metadata = if typed.is_empty() {
111 HashMap::new()
112 } else {
113 T::chunk_metadata(&typed)
114 };
115
116 T::encode_batch(&metadata, &typed).map_err(|e| anyhow::anyhow!("{e}"))
117 });
118
119 let decoder: ArrowDecoder = Box::new(|metadata, batch| {
120 T::decode_data_batch(metadata, batch).map_err(|e| anyhow::anyhow!("{e}"))
121 });
122
123 let _ = ensure_arrow_registered(type_name, schema, encoder, decoder);
124}
125
126fn assert_custom_schema(type_name: &str, schema: &Schema) {
127 validate_custom_arrow_schema(type_name, schema, true).unwrap();
128}
129
130#[cfg(test)]
131mod tests {
132 use std::sync::Arc;
133
134 use arrow::{
135 array::{Array, ArrayData, FixedSizeListArray, Int64Array, ListArray, ListViewArray},
136 buffer::{Buffer, NullBuffer},
137 datatypes::{DataType, Field, Schema, TimeUnit},
138 record_batch::RecordBatch,
139 };
140 use nautilus_model::{
141 data::{CustomDataTrait, Data, registry::encode_custom_to_arrow, stubs::StubRemoteData},
142 enums::AggressorSide as TestSide,
143 identifiers::InstrumentId as TestInstrumentId,
144 types::{Currency, Money as TestMoney, Price as TestPrice},
145 };
146 use rstest::rstest;
147 use rust_decimal_macros::dec;
148
149 use super::{
150 CustomDataSerialize, assert_custom_schema, ensure_custom_data_registered,
151 validate_required_list_child, validate_required_list_values,
152 };
153 use crate::{
154 arrow::{
155 ArrowSchemaProvider, DecodeDataFromRecordBatch, EncodeToRecordBatch, json_string_field,
156 },
157 arrow_custom_data_remote,
158 };
159
160 arrow_custom_data_remote!(nautilus_model::data::stubs::StubRemoteData);
162
163 #[rstest]
164 #[case::none(false)]
165 #[case::some(true)]
166 fn remote_custom_data_round_trip(#[case] optional_price: bool) {
167 let price = TestPrice::from("123.45");
168
169 let original = StubRemoteData {
170 instrument_id: TestInstrumentId::from("ES.GLBX"),
171 price,
172 optional_price: optional_price.then(|| TestPrice::from("67.890")),
173 side: TestSide::Sell,
174 params: serde_json::from_value(serde_json::json!({"scenario": "shocked"})).unwrap(),
175 balances: vec![
176 TestMoney::from_decimal(dec!(7.25), Currency::USD()).unwrap(),
177 TestMoney::from_decimal(dec!(-9.5), Currency::EUR()).unwrap(),
178 ],
179 curve: vec![0.25, 1.0, 3.5],
180 ts_event: 17.into(),
181 ts_init: 23.into(),
182 };
183
184 ensure_custom_data_registered::<StubRemoteData>();
185 let payload: Arc<dyn CustomDataTrait> = Arc::new(original.clone());
186 let payloads = [payload];
187 let encoded = original.encode_record_batch(&payloads).unwrap();
188 let registered = encode_custom_to_arrow("StubRemoteData", &payloads)
189 .unwrap()
190 .unwrap();
191 let metadata = original.metadata();
192 let batch = StubRemoteData::encode_batch(&metadata, &[&original]).unwrap();
193 let schema = batch.schema();
194 let expected_fields = vec![
195 Field::new("instrument_id", DataType::Utf8, false),
196 Field::new("price", DataType::Decimal128(38, 16), true),
197 Field::new("optional_price", DataType::Decimal128(38, 16), true),
198 Field::new(
199 "side",
200 DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
201 false,
202 ),
203 json_string_field("params", false),
204 json_string_field("balances", false),
205 Field::new(
206 "curve",
207 DataType::List(Arc::new(Field::new("item", DataType::Float64, true))),
208 false,
209 ),
210 Field::new(
211 "ts_event",
212 DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
213 false,
214 ),
215 Field::new(
216 "ts_init",
217 DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
218 false,
219 ),
220 ];
221 let decoded = StubRemoteData::decode_data_batch(schema.metadata(), batch).unwrap();
222
223 let [Data::Custom(custom)] = decoded.as_slice() else {
224 panic!("expected one native custom data value");
225 };
226
227 let restored = custom
228 .data
229 .as_any()
230 .downcast_ref::<StubRemoteData>()
231 .unwrap();
232
233 assert_eq!(encoded, registered);
234 assert_eq!(schema.fields(), Schema::new(expected_fields).fields());
235 assert_eq!(schema.metadata(), &metadata);
236 assert_eq!(StubRemoteData::get_schema(None).fields(), schema.fields());
237 assert_eq!(
238 schema.metadata().get("price_precision"),
239 Some(&"2".to_string())
240 );
241 assert_eq!(custom.data_type.type_name(), "StubRemoteData");
242 assert_eq!(restored, &original);
243 }
244
245 #[rstest]
246 fn custom_schema_allows_vec_u8_binary() {
247 let schema = Schema::new(vec![Field::new("payload", DataType::Binary, false)]);
248
249 assert_custom_schema("BinaryPayload", &schema);
250 }
251
252 #[rstest]
253 #[should_panic(
254 expected = "custom write schema `OpaquePayload` contains opaque byte field `item`: FixedSizeBinary(8)"
255 )]
256 fn custom_schema_rejects_nested_opaque_bytes() {
257 let child = Field::new("item", DataType::FixedSizeBinary(8), false);
258 let schema = Schema::new(vec![Field::new(
259 "values",
260 DataType::List(std::sync::Arc::new(child)),
261 false,
262 )]);
263
264 assert_custom_schema("OpaquePayload", &schema);
265 }
266
267 #[rstest]
268 fn required_list_rejects_null_child_value() {
269 let item = Field::new("item", DataType::Int64, false);
270 let values = Int64Array::from(vec![Some(1), None]);
271
272 let error = validate_required_list_child("values", &item, &values, std::iter::once((0, 2)))
273 .unwrap_err();
274
275 assert_eq!(
276 error.to_string(),
277 "Error parsing `custom_data`: field 'values': required list element 1 is null"
278 );
279 }
280
281 #[rstest]
282 fn required_fixed_size_list_allows_null_child_under_null_outer_row() {
283 let child = Arc::new(Field::new("item", DataType::Int64, false));
284 let values = Int64Array::from(vec![Some(1), Some(2), None, None, Some(5), Some(6)]);
285 let nulls = NullBuffer::from(vec![true, false, true]);
286 let list =
287 FixedSizeListArray::try_new(Arc::clone(&child), 2, Arc::new(values), Some(nulls))
288 .unwrap();
289 let schema = Schema::new(vec![Field::new(
290 "values",
291 DataType::FixedSizeList(child, 2),
292 true,
293 )]);
294 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list)]).unwrap();
295
296 validate_required_list_values(&batch).unwrap();
297 }
298
299 #[rstest]
300 #[allow(
301 unsafe_code,
302 reason = "storage can declare a non-nullable child while slots hold nulls; arrow's safe constructors reject building such data"
303 )]
304 fn required_list_allows_unreferenced_null_child_after_slice() {
305 let child = Arc::new(Field::new("item", DataType::Int64, false));
306 let values = Int64Array::from(vec![Some(1), Some(2), None]);
307 let data = unsafe {
308 ArrayData::builder(DataType::List(Arc::clone(&child)))
309 .len(3)
310 .add_buffer(Buffer::from_slice_ref([0i32, 1, 2, 3]))
311 .add_child_data(values.into_data())
312 .build_unchecked()
313 };
314 let list = ListArray::from(data).slice(0, 2);
315 let schema = Schema::new(vec![Field::new("values", DataType::List(child), false)]);
316 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list)]).unwrap();
317
318 validate_required_list_values(&batch).unwrap();
319 }
320
321 #[rstest]
322 #[allow(
323 unsafe_code,
324 reason = "storage can declare a non-nullable child while slots hold nulls; arrow's safe constructors reject building such data"
325 )]
326 fn required_list_rejects_null_child_referenced_by_valid_row() {
327 let child = Arc::new(Field::new("item", DataType::Int64, false));
328 let values = Int64Array::from(vec![Some(1), None]);
329 let data = unsafe {
330 ArrayData::builder(DataType::List(Arc::clone(&child)))
331 .len(2)
332 .add_buffer(Buffer::from_slice_ref([0i32, 1, 2]))
333 .add_child_data(values.into_data())
334 .build_unchecked()
335 };
336 let list = ListArray::from(data);
337 let schema = Schema::new(vec![Field::new("values", DataType::List(child), false)]);
338 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list)]).unwrap();
339
340 let error = validate_required_list_values(&batch).unwrap_err();
341
342 assert_eq!(
343 error.to_string(),
344 "Error parsing `custom_data`: field 'values': required list element 1 is null"
345 );
346 }
347
348 #[rstest]
349 fn required_list_view_rejects_referenced_null_child_value() {
350 let child = Arc::new(Field::new("item", DataType::Int64, false));
351 let values = Int64Array::from(vec![Some(1), None, Some(3)]);
352 let data = ArrayData::builder(DataType::ListView(Arc::clone(&child)))
353 .len(2)
354 .add_buffer(Buffer::from_slice_ref([0i32, 1]))
355 .add_buffer(Buffer::from_slice_ref([1i32, 2]))
356 .add_child_data(values.into_data())
357 .build()
358 .unwrap();
359 let list = ListViewArray::from(data);
360 let schema = Schema::new(vec![Field::new("values", DataType::ListView(child), false)]);
361 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list)]).unwrap();
362
363 let error = validate_required_list_values(&batch).unwrap_err();
364
365 assert_eq!(
366 error.to_string(),
367 "Error parsing `custom_data`: field 'values': required list element 1 is null"
368 );
369 }
370}
371
372#[derive(Debug)]
380pub struct CustomDataDecoder;
381
382impl ArrowSchemaProvider for CustomDataDecoder {
383 fn get_schema(
384 metadata: Option<std::collections::HashMap<String, String>>,
385 ) -> arrow::datatypes::Schema {
386 if let Some(metadata) = metadata
387 && let Some(type_name) = metadata.get("type_name")
388 && let Some(schema) = get_arrow_schema(type_name)
389 {
390 let schema = (*schema).clone();
391 let mut fields = schema.fields().iter().cloned().collect::<Vec<_>>();
392 if schema.field_with_name("data_type").is_err() {
393 fields.push(Arc::new(arrow::datatypes::Field::new(
394 "data_type",
395 arrow::datatypes::DataType::Utf8,
396 false,
397 )));
398 }
399 let mut merged_metadata = schema.metadata().clone();
400 merged_metadata.extend(metadata);
401 return arrow::datatypes::Schema::new_with_metadata(fields, merged_metadata);
402 }
403
404 arrow::datatypes::Schema::new(vec![arrow::datatypes::Field::new(
406 "dummy",
407 arrow::datatypes::DataType::Int64,
408 true,
409 )])
410 }
411}
412
413fn strip_data_type_column(
416 batch: &RecordBatch,
417) -> Result<(RecordBatch, Option<DataType>), super::EncodingError> {
418 use super::extract_column_string;
419
420 let Some(data_type_col_idx) = batch
421 .schema()
422 .fields()
423 .iter()
424 .position(|f| f.name() == "data_type")
425 else {
426 return Ok((batch.clone(), None));
427 };
428
429 if batch.num_rows() == 0 {
430 return Ok((batch.clone(), None));
431 }
432
433 let cols = batch.columns();
434 let data_type = if cols[data_type_col_idx].is_null(0) {
435 None
436 } else {
437 let string_col =
438 extract_column_string(cols, "data_type", data_type_col_idx).map_err(|e| {
439 super::EncodingError::ParseError("custom_data", format!("data_type column: {e}"))
440 })?;
441 let first_value = string_col.value(0);
442 Some(
443 DataType::from_persistence_json(first_value)
444 .map_err(|e| super::EncodingError::ParseError("custom_data", e.to_string()))?,
445 )
446 };
447
448 let new_fields: Vec<_> = batch
449 .schema()
450 .fields()
451 .iter()
452 .enumerate()
453 .filter(|(i, _)| *i != data_type_col_idx)
454 .map(|(_, f)| Arc::clone(f))
455 .collect();
456 let new_columns: Vec<Arc<dyn arrow::array::Array>> = batch
457 .columns()
458 .iter()
459 .enumerate()
460 .filter(|(i, _)| *i != data_type_col_idx)
461 .map(|(_, c)| Arc::clone(c))
462 .collect();
463 let new_schema =
464 arrow::datatypes::Schema::new_with_metadata(new_fields, batch.schema().metadata().clone());
465 let stripped_batch = RecordBatch::try_new(Arc::new(new_schema), new_columns)
466 .map_err(|e| super::EncodingError::ParseError("custom_data", e.to_string()))?;
467
468 Ok((stripped_batch, data_type))
469}
470
471impl CustomDataDecoder {
472 pub fn decode_custom_batch(
479 metadata: &std::collections::HashMap<String, String>,
480 record_batch: &RecordBatch,
481 ) -> Result<Vec<CustomData>, super::EncodingError> {
482 let type_name = metadata
483 .get("type_name")
484 .cloned()
485 .unwrap_or_else(|| "Unknown".to_string());
486
487 let (batch_to_decode, restored_data_type) = strip_data_type_column(record_batch)?;
488 validate_required_list_values(&batch_to_decode)?;
489
490 if batch_to_decode.num_rows() == 0 {
491 return Ok(Vec::new());
492 }
493
494 let data = match decode_custom_from_arrow(&type_name, metadata, batch_to_decode) {
495 Ok(Some(d)) => d,
496 Ok(None) => {
497 return Err(super::EncodingError::ParseError(
498 "custom_data",
499 format!(
500 "unknown custom data type '{type_name}'; only Rust-registered types are supported"
501 ),
502 ));
503 }
504 Err(e) => {
505 return Err(super::EncodingError::ParseError(
506 "custom_data",
507 format!("decode_custom_from_arrow: {e}"),
508 ));
509 }
510 };
511
512 data.into_iter()
513 .map(|d| match d {
514 Data::Custom(c) => Ok(match &restored_data_type {
515 Some(dt) => CustomData::new(Arc::clone(&c.data), dt.clone()),
516 None => c,
517 }),
518 _ => Err(super::EncodingError::ParseError(
519 "custom_data",
520 format!("registered decoder for '{type_name}' yielded a non-custom row"),
521 )),
522 })
523 .collect()
524 }
525}
526
527fn validate_required_list_values(batch: &RecordBatch) -> Result<(), super::EncodingError> {
528 let schema = batch.schema();
529 for (field, column) in schema.fields().iter().zip(batch.columns()) {
530 match field.data_type() {
531 ArrowDataType::List(child) => {
532 let list = downcast_list::<ListArray>(field.name(), column.as_ref(), "list")?;
533 validate_required_list_child(
534 field.name(),
535 child,
536 list.values().as_ref(),
537 list_ranges(list),
538 )?;
539 }
540 ArrowDataType::LargeList(child) => {
541 let list =
542 downcast_list::<LargeListArray>(field.name(), column.as_ref(), "large-list")?;
543 validate_required_list_child(
544 field.name(),
545 child,
546 list.values().as_ref(),
547 list_ranges(list),
548 )?;
549 }
550 ArrowDataType::ListView(child) => {
551 let list =
552 downcast_list::<ListViewArray>(field.name(), column.as_ref(), "list-view")?;
553 validate_required_list_child(
554 field.name(),
555 child,
556 list.values().as_ref(),
557 list_view_ranges(list),
558 )?;
559 }
560 ArrowDataType::LargeListView(child) => {
561 let list = downcast_list::<LargeListViewArray>(
562 field.name(),
563 column.as_ref(),
564 "large-list-view",
565 )?;
566 validate_required_list_child(
567 field.name(),
568 child,
569 list.values().as_ref(),
570 list_view_ranges(list),
571 )?;
572 }
573 ArrowDataType::FixedSizeList(child, _) => {
574 let list = downcast_list::<FixedSizeListArray>(
575 field.name(),
576 column.as_ref(),
577 "fixed-size-list",
578 )?;
579 validate_required_list_child(
580 field.name(),
581 child,
582 list.values().as_ref(),
583 fixed_size_list_ranges(list),
584 )?;
585 }
586 _ => {}
587 }
588 }
589 Ok(())
590}
591
592fn downcast_list<'a, T: Array + 'static>(
593 field_name: &str,
594 column: &'a dyn Array,
595 kind: &str,
596) -> Result<&'a T, super::EncodingError> {
597 column.as_any().downcast_ref::<T>().ok_or_else(|| {
598 super::EncodingError::ParseError(
599 "custom_data",
600 format!("field '{field_name}' is not a {kind} array"),
601 )
602 })
603}
604
605fn validate_required_list_child(
606 field_name: &str,
607 child: &arrow::datatypes::Field,
608 values: &dyn Array,
609 ranges: impl Iterator<Item = (usize, usize)>,
610) -> Result<(), super::EncodingError> {
611 if child.is_nullable() || values.null_count() == 0 {
612 return Ok(());
613 }
614
615 for (start, end) in ranges {
616 if let Some(index) = (start..end).find(|index| values.is_null(*index)) {
617 return Err(super::EncodingError::ParseError(
618 "custom_data",
619 format!("field '{field_name}': required list element {index} is null"),
620 ));
621 }
622 }
623 Ok(())
624}
625
626fn list_ranges<O: OffsetSizeTrait>(
627 list: &GenericListArray<O>,
628) -> impl Iterator<Item = (usize, usize)> + '_ {
629 let offsets = list.value_offsets();
630 (0..list.len())
631 .filter(|row| list.is_valid(*row))
632 .map(move |row| (offsets[row].as_usize(), offsets[row + 1].as_usize()))
633}
634
635fn list_view_ranges<O: OffsetSizeTrait>(
636 list: &GenericListViewArray<O>,
637) -> impl Iterator<Item = (usize, usize)> + '_ {
638 let offsets = list.value_offsets();
639 let sizes = list.value_sizes();
640 (0..list.len())
641 .filter(|row| list.is_valid(*row))
642 .map(move |row| {
643 let start = offsets[row].as_usize();
644 (start, start + sizes[row].as_usize())
645 })
646}
647
648fn fixed_size_list_ranges(list: &FixedSizeListArray) -> impl Iterator<Item = (usize, usize)> + '_ {
649 let length =
650 usize::try_from(list.value_length()).expect("fixed-size list length is non-negative");
651 (0..list.len())
652 .filter(|row| list.is_valid(*row))
653 .map(move |row| (row * length, (row + 1) * length))
654}
655
656impl DecodeDataFromRecordBatch for CustomDataDecoder {
657 fn decode_data_batch(
658 metadata: &std::collections::HashMap<String, String>,
659 record_batch: RecordBatch,
660 ) -> Result<Vec<Data>, super::EncodingError> {
661 Ok(Self::decode_custom_batch(metadata, &record_batch)?
662 .into_iter()
663 .map(Data::Custom)
664 .collect())
665 }
666}