1use std::{collections::HashMap, sync::Arc};
17
18use arrow::{
19 array::{
20 Array, Decimal128Array, Int32Array, TimestampNanosecondArray, UInt8Array, UInt16Array,
21 UInt32Array,
22 },
23 datatypes::{DataType, Field, Schema},
24 error::ArrowError,
25 record_batch::RecordBatch,
26};
27use databento::dbn;
28use nautilus_model::{
29 data::{Data, custom::CustomData},
30 identifiers::InstrumentId,
31 types::{PRICE_UNDEF, Price, QUANTITY_UNDEF, Quantity, fixed::FIXED_PRECISION},
32};
33use nautilus_serialization::arrow::{
34 ArrowSchemaProvider, DecodeDataFromRecordBatch, EncodeToRecordBatch, EncodingError,
35 KEY_TYPE_NAME, decode_decimal_price, decode_decimal_quantity, decode_timestamp,
36 enum_dictionary_array, enum_dictionary_data_type, extract_column, fixed_decimal_data_type,
37 optional_timestamp_array, price_decimal_array, quantity_decimal_array, timestamp_array,
38 timestamp_data_type,
39};
40
41use super::{EnumColumn, parse_metadata};
42use crate::{
43 enums::{DatabentoStatisticType, DatabentoStatisticUpdateAction},
44 types::DatabentoStatistics,
45};
46
47impl ArrowSchemaProvider for DatabentoStatistics {
48 fn get_schema(metadata: Option<HashMap<String, String>>) -> Schema {
49 let fields = vec![
50 Field::new("stat_type", enum_dictionary_data_type(), false),
51 Field::new("update_action", enum_dictionary_data_type(), false),
52 Field::new("price", fixed_decimal_data_type(), true),
53 Field::new("quantity", fixed_decimal_data_type(), true),
54 Field::new("channel_id", DataType::UInt16, false),
55 Field::new("stat_flags", DataType::UInt8, false),
56 Field::new("sequence", DataType::UInt32, false),
57 Field::new("ts_ref", timestamp_data_type(), true),
58 Field::new("ts_in_delta", DataType::Int32, false),
59 Field::new("ts_event", timestamp_data_type(), false),
60 Field::new("ts_recv", timestamp_data_type(), false),
61 Field::new("ts_init", timestamp_data_type(), false),
62 ];
63
64 match metadata {
65 Some(metadata) => Schema::new_with_metadata(fields, metadata),
66 None => Schema::new(fields),
67 }
68 }
69}
70
71impl EncodeToRecordBatch for DatabentoStatistics {
72 fn encode_batch<T>(
73 metadata: &HashMap<String, String>,
74 data: &[T],
75 ) -> Result<RecordBatch, ArrowError>
76 where
77 T: std::borrow::Borrow<Self>,
78 {
79 let (instrument_id, price_precision, size_precision) = parse_metadata(metadata)
80 .map_err(|e| ArrowError::InvalidArgumentError(e.to_string()))?;
81
82 let chunk_metadata = if data.is_empty() {
83 metadata.clone()
84 } else {
85 Self::chunk_metadata(data)
86 };
87
88 let mut channel_id_builder = UInt16Array::builder(data.len());
89 let mut stat_flags_builder = UInt8Array::builder(data.len());
90 let mut sequence_builder = UInt32Array::builder(data.len());
91 let mut ts_in_delta_builder = Int32Array::builder(data.len());
92
93 for (index, item) in data.iter().map(std::borrow::Borrow::borrow).enumerate() {
94 if item.instrument_id != instrument_id || !item.matches_chunk_metadata(&chunk_metadata)
95 {
96 return Err(ArrowError::InvalidArgumentError(
97 EncodingError::MixedMetadata { index }.to_string(),
98 ));
99 }
100
101 if let Some(price) = item.price.filter(|price| !price.is_undefined()) {
102 Price::from_raw_checked(price.raw(), price_precision).map_err(|e| {
103 ArrowError::InvalidArgumentError(format!("row {index}: price: {e}"))
104 })?;
105 }
106
107 if let Some(quantity) = item.quantity.filter(|quantity| !quantity.is_undefined()) {
108 Quantity::from_raw_checked(quantity.raw(), size_precision).map_err(|e| {
109 ArrowError::InvalidArgumentError(format!("row {index}: quantity: {e}"))
110 })?;
111 }
112
113 channel_id_builder.append_value(item.channel_id);
114 stat_flags_builder.append_value(item.stat_flags);
115 sequence_builder.append_value(item.sequence);
116 ts_in_delta_builder.append_value(item.ts_in_delta);
117 }
118
119 RecordBatch::try_new(
120 Self::get_schema(Some(metadata.clone())).into(),
121 vec![
122 Arc::new(enum_dictionary_array(
123 data.iter().map(|item| item.borrow().stat_type),
124 )?),
125 Arc::new(enum_dictionary_array(
126 data.iter().map(|item| item.borrow().update_action),
127 )?),
128 Arc::new(price_decimal_array(
129 data.iter()
130 .map(|item| item.borrow().price.map_or(PRICE_UNDEF, |value| value.raw())),
131 "price",
132 )?),
133 Arc::new(quantity_decimal_array(
134 data.iter().map(|item| {
135 item.borrow()
136 .quantity
137 .map_or(QUANTITY_UNDEF, |value| value.raw())
138 }),
139 "quantity",
140 )?),
141 Arc::new(channel_id_builder.finish()),
142 Arc::new(stat_flags_builder.finish()),
143 Arc::new(sequence_builder.finish()),
144 Arc::new(optional_timestamp_array(data.iter().map(|item| {
145 let value = item.borrow().ts_ref.as_u64();
146 (value != dbn::UNDEF_TIMESTAMP).then_some(value)
147 }))?),
148 Arc::new(ts_in_delta_builder.finish()),
149 Arc::new(timestamp_array(
150 data.iter().map(|item| item.borrow().ts_event.as_u64()),
151 )?),
152 Arc::new(timestamp_array(
153 data.iter().map(|item| item.borrow().ts_recv.as_u64()),
154 )?),
155 Arc::new(timestamp_array(
156 data.iter().map(|item| item.borrow().ts_init.as_u64()),
157 )?),
158 ],
159 )
160 }
161
162 fn metadata(&self) -> HashMap<String, String> {
163 statistics_metadata(
164 &self.instrument_id,
165 self.price.map_or(FIXED_PRECISION, |p| p.precision),
166 self.quantity.map_or(FIXED_PRECISION, |q| q.precision),
167 )
168 }
169
170 fn chunk_metadata<T>(chunk: &[T]) -> HashMap<String, String>
171 where
172 T: std::borrow::Borrow<Self>,
173 {
174 let first = chunk
175 .first()
176 .map(std::borrow::Borrow::borrow)
177 .expect("Chunk should have at least one element to encode");
178
179 let price_precision = chunk
180 .iter()
181 .map(std::borrow::Borrow::borrow)
182 .filter_map(|s| s.price)
183 .find(|price| !price.is_undefined())
184 .map_or(FIXED_PRECISION, |price| price.precision);
185 let size_precision = chunk
186 .iter()
187 .map(std::borrow::Borrow::borrow)
188 .filter_map(|s| s.quantity)
189 .find(|quantity| !quantity.is_undefined())
190 .map_or(FIXED_PRECISION, |quantity| quantity.precision);
191
192 statistics_metadata(&first.instrument_id, price_precision, size_precision)
193 }
194
195 fn matches_chunk_metadata(&self, metadata: &HashMap<String, String>) -> bool {
196 let Ok((instrument_id, price_precision, size_precision)) = parse_metadata(metadata) else {
197 return false;
198 };
199
200 self.instrument_id == instrument_id
201 && self
202 .price
203 .is_none_or(|price| price.is_undefined() || price.precision == price_precision)
204 && self.quantity.is_none_or(|quantity| {
205 quantity.is_undefined() || quantity.precision == size_precision
206 })
207 }
208}
209
210impl DecodeDataFromRecordBatch for DatabentoStatistics {
211 fn decode_data_batch(
212 metadata: &HashMap<String, String>,
213 record_batch: RecordBatch,
214 ) -> Result<Vec<Data>, EncodingError> {
215 let items = decode_statistics_batch(metadata, &record_batch)?;
216 Ok(items
217 .into_iter()
218 .map(|item| Data::Custom(CustomData::from_arc(Arc::new(item))))
219 .collect())
220 }
221}
222
223pub fn decode_statistics_batch(
229 metadata: &HashMap<String, String>,
230 record_batch: &RecordBatch,
231) -> Result<Vec<DatabentoStatistics>, EncodingError> {
232 let (instrument_id, price_precision, size_precision) = parse_metadata(metadata)?;
233 let cols = record_batch.columns();
234
235 let price_values =
236 extract_column::<Decimal128Array>(cols, "price", 2, fixed_decimal_data_type())?;
237 let quantity_values =
238 extract_column::<Decimal128Array>(cols, "quantity", 3, fixed_decimal_data_type())?;
239 let channel_id_values = extract_column::<UInt16Array>(cols, "channel_id", 4, DataType::UInt16)?;
240 let stat_flags_values = extract_column::<UInt8Array>(cols, "stat_flags", 5, DataType::UInt8)?;
241 let sequence_values = extract_column::<UInt32Array>(cols, "sequence", 6, DataType::UInt32)?;
242 let ts_ref_values =
243 extract_column::<TimestampNanosecondArray>(cols, "ts_ref", 7, timestamp_data_type())?;
244 let ts_in_delta_values = extract_column::<Int32Array>(cols, "ts_in_delta", 8, DataType::Int32)?;
245 let ts_event_values =
246 extract_column::<TimestampNanosecondArray>(cols, "ts_event", 9, timestamp_data_type())?;
247 let ts_recv_values =
248 extract_column::<TimestampNanosecondArray>(cols, "ts_recv", 10, timestamp_data_type())?;
249 let ts_init_values =
250 extract_column::<TimestampNanosecondArray>(cols, "ts_init", 11, timestamp_data_type())?;
251 let stat_type_column = EnumColumn::try_from_column(&cols[0], "stat_type", 0)?;
252 let update_action_column = EnumColumn::try_from_column(&cols[1], "update_action", 1)?;
253
254 (0..record_batch.num_rows())
255 .map(|row| {
256 let stat_type = stat_type_column.decode::<DatabentoStatisticType>(row)?;
257 let update_action =
258 update_action_column.decode::<DatabentoStatisticUpdateAction>(row)?;
259
260 let price = (!price_values.is_null(row))
261 .then(|| decode_decimal_price(price_values, price_precision, "price", row))
262 .transpose()?;
263 let quantity = (!quantity_values.is_null(row))
264 .then(|| decode_decimal_quantity(quantity_values, size_precision, "quantity", row))
265 .transpose()?;
266
267 Ok(DatabentoStatistics {
268 instrument_id,
269 stat_type,
270 update_action,
271 price,
272 quantity,
273 channel_id: channel_id_values.value(row),
274 stat_flags: stat_flags_values.value(row),
275 sequence: sequence_values.value(row),
276 ts_ref: if ts_ref_values.is_null(row) {
277 dbn::UNDEF_TIMESTAMP.into()
278 } else {
279 decode_timestamp(ts_ref_values, "ts_ref", row)?.into()
280 },
281 ts_in_delta: ts_in_delta_values.value(row),
282 ts_event: decode_timestamp(ts_event_values, "ts_event", row)?.into(),
283 ts_recv: decode_timestamp(ts_recv_values, "ts_recv", row)?.into(),
284 ts_init: decode_timestamp(ts_init_values, "ts_init", row)?.into(),
285 })
286 })
287 .collect()
288}
289
290fn statistics_metadata(
291 instrument_id: &InstrumentId,
292 price_precision: u8,
293 size_precision: u8,
294) -> HashMap<String, String> {
295 let mut metadata =
296 DatabentoStatistics::get_metadata(instrument_id, price_precision, size_precision);
297 metadata.insert(KEY_TYPE_NAME.to_string(), "DatabentoStatistics".to_string());
298 metadata
299}
300
301pub fn statistics_to_arrow_record_batch(
308 data: &[DatabentoStatistics],
309) -> Result<RecordBatch, EncodingError> {
310 if data.is_empty() {
311 return Err(EncodingError::EmptyData);
312 }
313
314 let metadata = DatabentoStatistics::chunk_metadata(data);
315 DatabentoStatistics::encode_batch(&metadata, data).map_err(EncodingError::ArrowError)
316}
317
318#[cfg(test)]
319mod tests {
320 use std::collections::HashMap;
321
322 use nautilus_model::{
323 data::{custom::CustomDataTrait, registry::encode_custom_to_arrow},
324 identifiers::InstrumentId,
325 types::{Price, Quantity},
326 };
327 use nautilus_serialization::arrow::{
328 ArrowSchemaProvider, EncodeToRecordBatch, KEY_INSTRUMENT_ID, KEY_PRICE_PRECISION,
329 KEY_SIZE_PRECISION, custom::ensure_custom_data_registered,
330 };
331 use rstest::rstest;
332
333 use super::*;
334
335 fn test_metadata() -> HashMap<String, String> {
336 HashMap::from([
337 (KEY_INSTRUMENT_ID.to_string(), "ESM4.GLBX".to_string()),
338 (KEY_PRICE_PRECISION.to_string(), "2".to_string()),
339 (KEY_SIZE_PRECISION.to_string(), "0".to_string()),
340 ])
341 }
342
343 fn test_statistics(instrument_id: InstrumentId) -> DatabentoStatistics {
344 DatabentoStatistics::new(
345 instrument_id,
346 DatabentoStatisticType::OpeningPrice,
347 DatabentoStatisticUpdateAction::Added,
348 Some(Price::from("5000.50")),
349 Some(Quantity::from("100")),
350 1,
351 0,
352 42,
353 1_000_000_000.into(),
354 500,
355 2_000_000_000.into(),
356 3_000_000_000.into(),
357 4_000_000_000.into(),
358 )
359 }
360
361 #[rstest]
362 fn test_get_schema() {
363 let schema = DatabentoStatistics::get_schema(None);
364 assert_eq!(schema.fields().len(), 12);
365 assert_eq!(schema.field(0).name(), "stat_type");
366 assert_eq!(schema.field(11).name(), "ts_init");
367 assert_eq!(schema.field(0).data_type(), &enum_dictionary_data_type());
368 assert_eq!(schema.field(2).data_type(), &fixed_decimal_data_type());
369 assert!(schema.field(2).is_nullable());
370 assert!(schema.field(7).is_nullable());
371 assert_eq!(schema.field(10).data_type(), ×tamp_data_type());
372 }
373
374 #[rstest]
375 fn test_encode_batch() {
376 let instrument_id = InstrumentId::from("ESM4.GLBX");
377 let metadata = test_metadata();
378 let data = vec![test_statistics(instrument_id)];
379 let batch = DatabentoStatistics::encode_batch(&metadata, &data).unwrap();
380
381 assert_eq!(batch.num_rows(), 1);
382 assert_eq!(batch.num_columns(), 12);
383 }
384
385 #[rstest]
386 fn test_encode_decode_round_trip() {
387 let instrument_id = InstrumentId::from("ESM4.GLBX");
388 let metadata = test_metadata();
389 let original = vec![test_statistics(instrument_id)];
390 let batch = DatabentoStatistics::encode_batch(&metadata, &original).unwrap();
391 let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
392
393 assert_eq!(decoded.len(), 1);
394 assert_eq!(decoded[0].instrument_id, instrument_id);
395 assert_eq!(decoded[0].stat_type, original[0].stat_type);
396 assert_eq!(decoded[0].update_action, original[0].update_action);
397 assert_eq!(decoded[0].price, original[0].price);
398 assert_eq!(decoded[0].quantity, original[0].quantity);
399 assert_eq!(decoded[0].channel_id, original[0].channel_id);
400 assert_eq!(decoded[0].stat_flags, original[0].stat_flags);
401 assert_eq!(decoded[0].sequence, original[0].sequence);
402 assert_eq!(decoded[0].ts_ref, original[0].ts_ref);
403 assert_eq!(decoded[0].ts_in_delta, original[0].ts_in_delta);
404 assert_eq!(decoded[0].ts_event, original[0].ts_event);
405 assert_eq!(decoded[0].ts_recv, original[0].ts_recv);
406 assert_eq!(decoded[0].ts_init, original[0].ts_init);
407 }
408
409 #[rstest]
410 fn test_decode_legacy_enum_columns() {
411 let instrument_id = InstrumentId::from("ESM4.GLBX");
412 let metadata = test_metadata();
413 let original = test_statistics(instrument_id);
414 let batch =
415 DatabentoStatistics::encode_batch(&metadata, std::slice::from_ref(&original)).unwrap();
416 let mut fields = batch.schema().fields().to_vec();
417 fields[0] = Arc::new(Field::new("stat_type", DataType::UInt8, false));
418 fields[1] = Arc::new(Field::new("update_action", DataType::UInt8, false));
419 let mut columns = batch.columns().to_vec();
420 columns[0] = Arc::new(UInt8Array::from(vec![original.stat_type as u8]));
421 columns[1] = Arc::new(UInt8Array::from(vec![original.update_action as u8]));
422 let legacy_batch = RecordBatch::try_new(
423 Arc::new(Schema::new_with_metadata(fields, metadata.clone())),
424 columns,
425 )
426 .unwrap();
427
428 let decoded = decode_statistics_batch(&metadata, &legacy_batch).unwrap();
429
430 assert_eq!(decoded, vec![original]);
431 }
432
433 #[rstest]
434 fn test_encode_decode_round_trip_with_none_values() {
435 let instrument_id = InstrumentId::from("ESM4.GLBX");
436 let metadata = test_metadata();
437 let stats = DatabentoStatistics::new(
438 instrument_id,
439 DatabentoStatisticType::ClearedVolume,
440 DatabentoStatisticUpdateAction::Added,
441 None,
442 None,
443 1,
444 0,
445 42,
446 dbn::UNDEF_TIMESTAMP.into(),
447 500,
448 2_000_000_000.into(),
449 3_000_000_000.into(),
450 4_000_000_000.into(),
451 );
452 let original = vec![stats];
453 let batch = DatabentoStatistics::encode_batch(&metadata, &original).unwrap();
454 let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
455
456 assert_eq!(decoded.len(), 1);
457 let ts_ref = batch
458 .column(7)
459 .as_any()
460 .downcast_ref::<TimestampNanosecondArray>()
461 .unwrap();
462 assert!(ts_ref.is_null(0));
463 assert_eq!(decoded[0].price, None);
464 assert_eq!(decoded[0].quantity, None);
465 assert_eq!(decoded[0].ts_ref.as_u64(), dbn::UNDEF_TIMESTAMP);
466 }
467
468 #[rstest]
469 fn test_chunk_metadata_uses_first_non_none_precision() {
470 let instrument_id = InstrumentId::from("ESM4.GLBX");
471 let none_stats = DatabentoStatistics::new(
472 instrument_id,
473 DatabentoStatisticType::ClearedVolume,
474 DatabentoStatisticUpdateAction::Added,
475 None,
476 None,
477 1,
478 0,
479 42,
480 1_000_000_000.into(),
481 500,
482 2_000_000_000.into(),
483 3_000_000_000.into(),
484 4_000_000_000.into(),
485 );
486 let some_stats = test_statistics(instrument_id);
487 let data = vec![none_stats, some_stats];
488
489 let batch = statistics_to_arrow_record_batch(&data).unwrap();
490 let metadata = batch.schema().metadata().clone();
491 let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
492
493 assert_eq!(decoded.len(), 2);
494 assert_eq!(decoded[0].price, None);
495 assert_eq!(decoded[0].quantity, None);
496 assert_eq!(decoded[1].price, data[1].price);
497 assert_eq!(decoded[1].quantity, data[1].quantity);
498 }
499
500 #[rstest]
501 fn test_encode_decode_multiple_rows() {
502 let instrument_id = InstrumentId::from("ESM4.GLBX");
503 let metadata = test_metadata();
504 let stats1 = test_statistics(instrument_id);
505 let stats2 = DatabentoStatistics::new(
506 instrument_id,
507 DatabentoStatisticType::ClearedVolume,
508 DatabentoStatisticUpdateAction::Added,
509 Some(Price::from("5100.25")),
510 None,
511 2,
512 1,
513 43,
514 2_000_000_000.into(),
515 600,
516 3_000_000_000.into(),
517 4_000_000_000.into(),
518 5_000_000_000.into(),
519 );
520 let stats3 = DatabentoStatistics::new(
521 instrument_id,
522 DatabentoStatisticType::OpeningPrice,
523 DatabentoStatisticUpdateAction::Added,
524 None,
525 Some(Quantity::from("200")),
526 3,
527 0,
528 44,
529 3_000_000_000.into(),
530 700,
531 4_000_000_000.into(),
532 5_000_000_000.into(),
533 6_000_000_000.into(),
534 );
535 let original = vec![stats1, stats2, stats3];
536
537 let batch = DatabentoStatistics::encode_batch(&metadata, &original).unwrap();
538 assert_eq!(batch.num_rows(), 3);
539
540 let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
541 assert_eq!(decoded.len(), 3);
542 for (orig, dec) in original.iter().zip(decoded.iter()) {
543 assert_eq!(dec.instrument_id, orig.instrument_id);
544 assert_eq!(dec.stat_type, orig.stat_type);
545 assert_eq!(dec.price, orig.price);
546 assert_eq!(dec.quantity, orig.quantity);
547 assert_eq!(dec.channel_id, orig.channel_id);
548 assert_eq!(dec.sequence, orig.sequence);
549 }
550 }
551
552 #[rstest]
553 fn test_statistics_to_arrow_record_batch_round_trip() {
554 let instrument_id = InstrumentId::from("ESM4.GLBX");
555 let original = vec![test_statistics(instrument_id)];
556 let batch = statistics_to_arrow_record_batch(&original).unwrap();
557 let metadata = batch.schema().metadata().clone();
558 let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
559
560 assert_eq!(decoded.len(), 1);
561 assert_eq!(decoded[0].price, original[0].price);
562 assert_eq!(decoded[0].quantity, original[0].quantity);
563 }
564
565 #[rstest]
566 #[case::price(false, false)]
567 #[case::price_reversed(false, true)]
568 #[case::quantity(true, false)]
569 #[case::quantity_reversed(true, true)]
570 fn test_automatic_encoding_rejects_mixed_precision(
571 #[case] quantity: bool,
572 #[case] reversed: bool,
573 ) {
574 let first = test_statistics(InstrumentId::from("ESM4.GLBX"));
575 let mut second = first.clone();
576 if quantity {
577 second.quantity = Some(Quantity::from("100.0"));
578 } else {
579 second.price = Some(Price::from("5000.500"));
580 }
581
582 let mut data = vec![first, second];
583 if reversed {
584 data.reverse();
585 }
586
587 ensure_custom_data_registered::<DatabentoStatistics>();
588 let custom: Vec<Arc<dyn CustomDataTrait>> = data
589 .iter()
590 .cloned()
591 .map(|item| Arc::new(item) as Arc<dyn CustomDataTrait>)
592 .collect();
593
594 let direct = statistics_to_arrow_record_batch(&data).unwrap_err();
595 let registered = encode_custom_to_arrow("DatabentoStatistics", &custom).unwrap_err();
596
597 let expected =
598 ArrowError::InvalidArgumentError(EncodingError::MixedMetadata { index: 1 }.to_string());
599 assert_eq!(direct.to_string(), format!("Arrow error: {expected}"));
600 assert_eq!(registered.to_string(), expected.to_string());
601 }
602
603 #[rstest]
604 #[case::price(false)]
605 #[case::quantity(true)]
606 fn test_explicit_encoding_rejects_insufficient_precision(#[case] quantity: bool) {
607 let mut item = test_statistics(InstrumentId::from("ESM4.GLBX"));
608
609 let (field, expected) = if quantity {
610 let value = Quantity::from("100.5");
611 item.quantity = Some(value);
612 (
613 "quantity",
614 Quantity::from_raw_checked(value.raw(), 0)
615 .unwrap_err()
616 .to_string(),
617 )
618 } else {
619 let value = Price::from("5000.501");
620 item.price = Some(value);
621 (
622 "price",
623 Price::from_raw_checked(value.raw(), 2)
624 .unwrap_err()
625 .to_string(),
626 )
627 };
628
629 let error = DatabentoStatistics::encode_batch(&test_metadata(), &[item]).unwrap_err();
630
631 assert_eq!(
632 error.to_string(),
633 format!("Invalid argument error: row 0: {field}: {expected}")
634 );
635 }
636
637 #[rstest]
638 fn test_encoding_rejects_mixed_instruments() {
639 let first = test_statistics(InstrumentId::from("ESM4.GLBX"));
640 let second = test_statistics(InstrumentId::from("ESU4.GLBX"));
641 let data = vec![first, second];
642 ensure_custom_data_registered::<DatabentoStatistics>();
643 let custom: Vec<Arc<dyn CustomDataTrait>> = data
644 .iter()
645 .cloned()
646 .map(|item| Arc::new(item) as Arc<dyn CustomDataTrait>)
647 .collect();
648
649 let direct = statistics_to_arrow_record_batch(&data).unwrap_err();
650 let registered = encode_custom_to_arrow("DatabentoStatistics", &custom).unwrap_err();
651 let explicit = DatabentoStatistics::encode_batch(&test_metadata(), &data).unwrap_err();
652
653 let expected =
654 ArrowError::InvalidArgumentError(EncodingError::MixedMetadata { index: 1 }.to_string());
655 assert_eq!(direct.to_string(), format!("Arrow error: {expected}"));
656 assert_eq!(registered.to_string(), expected.to_string());
657 assert_eq!(explicit.to_string(), expected.to_string());
658 }
659
660 #[rstest]
661 #[case::none(false)]
662 #[case::undefined(true)]
663 fn test_registered_encoding_uses_first_non_null_precision(#[case] undefined: bool) {
664 let item = test_statistics(InstrumentId::from("ESM4.GLBX"));
665 let mut empty = item.clone();
666 empty.price = undefined.then_some(Price::from_raw(PRICE_UNDEF, 0));
667 empty.quantity = undefined.then_some(Quantity::from_raw(QUANTITY_UNDEF, 0));
668 let original = vec![empty, item];
669 let custom: Vec<Arc<dyn CustomDataTrait>> = original
670 .iter()
671 .cloned()
672 .map(|item| Arc::new(item) as Arc<dyn CustomDataTrait>)
673 .collect();
674 ensure_custom_data_registered::<DatabentoStatistics>();
675
676 let batch = encode_custom_to_arrow("DatabentoStatistics", &custom)
677 .unwrap()
678 .unwrap();
679 let decoded = decode_statistics_batch(batch.schema().metadata(), &batch).unwrap();
680
681 let mut expected = original;
682 expected[0].price = None;
683 expected[0].quantity = None;
684 assert_eq!(decoded, expected);
685 assert_eq!(decoded[1].price.unwrap().precision, 2);
686 assert_eq!(decoded[1].quantity.unwrap().precision, 0);
687 }
688
689 #[rstest]
690 fn test_chunk_metadata_all_none_uses_fixed_precision() {
691 use nautilus_model::types::fixed::FIXED_PRECISION;
692
693 let instrument_id = InstrumentId::from("ESM4.GLBX");
694 let stats = DatabentoStatistics::new(
695 instrument_id,
696 DatabentoStatisticType::ClearedVolume,
697 DatabentoStatisticUpdateAction::Added,
698 None,
699 None,
700 1,
701 0,
702 42,
703 1_000_000_000.into(),
704 500,
705 2_000_000_000.into(),
706 3_000_000_000.into(),
707 4_000_000_000.into(),
708 );
709 let data = vec![stats];
710 let metadata = DatabentoStatistics::chunk_metadata(&data);
711
712 assert_eq!(
713 metadata.get(KEY_PRICE_PRECISION).unwrap(),
714 &FIXED_PRECISION.to_string(),
715 );
716 assert_eq!(
717 metadata.get(KEY_SIZE_PRECISION).unwrap(),
718 &FIXED_PRECISION.to_string(),
719 );
720 }
721
722 #[rstest]
723 fn test_all_none_metadata_decodes_real_prices_correctly() {
724 use nautilus_model::types::fixed::FIXED_PRECISION;
725
726 let instrument_id = InstrumentId::from("ESM4.GLBX");
727 let price = Price::from("5000.50");
728 let quantity = Quantity::from("100");
729 let stats = DatabentoStatistics::new(
730 instrument_id,
731 DatabentoStatisticType::OpeningPrice,
732 DatabentoStatisticUpdateAction::Added,
733 Some(price),
734 Some(quantity),
735 1,
736 0,
737 42,
738 1_000_000_000.into(),
739 500,
740 2_000_000_000.into(),
741 3_000_000_000.into(),
742 4_000_000_000.into(),
743 );
744
745 let metadata = HashMap::from([
747 (KEY_INSTRUMENT_ID.to_string(), "ESM4.GLBX".to_string()),
748 (KEY_PRICE_PRECISION.to_string(), FIXED_PRECISION.to_string()),
749 (KEY_SIZE_PRECISION.to_string(), FIXED_PRECISION.to_string()),
750 ]);
751
752 let batch = DatabentoStatistics::encode_batch(&metadata, &[stats]).unwrap();
753 let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
754
755 assert_eq!(decoded.len(), 1);
756 assert_eq!(decoded[0].price.unwrap().as_f64(), price.as_f64());
757 assert_eq!(decoded[0].quantity.unwrap().as_f64(), quantity.as_f64());
758 }
759
760 #[rstest]
761 fn test_get_schema_with_metadata() {
762 let metadata = test_metadata();
763 let schema = DatabentoStatistics::get_schema(Some(metadata.clone()));
764 assert_eq!(schema.metadata(), &metadata);
765 assert_eq!(schema.fields().len(), 12);
766 }
767
768 #[rstest]
769 fn test_decode_missing_metadata_returns_error() {
770 let instrument_id = InstrumentId::from("ESM4.GLBX");
771 let metadata = test_metadata();
772 let data = vec![test_statistics(instrument_id)];
773 let batch = DatabentoStatistics::encode_batch(&metadata, &data).unwrap();
774
775 let empty_metadata = HashMap::new();
776 let result = decode_statistics_batch(&empty_metadata, &batch);
777 assert!(result.is_err());
778 }
779
780 #[rstest]
781 fn test_statistics_to_arrow_record_batch_empty() {
782 let result = statistics_to_arrow_record_batch(&[]);
783 assert!(result.is_err());
784 }
785
786 #[rstest]
787 fn test_decode_data_batch_produces_custom_data() {
788 let instrument_id = InstrumentId::from("ESM4.GLBX");
789 let metadata = test_metadata();
790 let original = vec![test_statistics(instrument_id)];
791 let batch = DatabentoStatistics::encode_batch(&metadata, &original).unwrap();
792 let data_vec = DatabentoStatistics::decode_data_batch(&metadata, batch).unwrap();
793
794 assert_eq!(data_vec.len(), 1);
795 match &data_vec[0] {
796 Data::Custom(custom) => {
797 assert_eq!(custom.data.type_name(), "DatabentoStatistics");
798 let stats = custom
799 .data
800 .as_any()
801 .downcast_ref::<DatabentoStatistics>()
802 .unwrap();
803 assert_eq!(stats.instrument_id, instrument_id);
804 assert_eq!(stats.stat_type, original[0].stat_type);
805 assert_eq!(stats.price, original[0].price);
806 assert_eq!(stats.quantity, original[0].quantity);
807 assert_eq!(stats.ts_event, original[0].ts_event);
808 assert_eq!(stats.ts_init, original[0].ts_init);
809 }
810 other => panic!("Expected Data::Custom, was {other:?}"),
811 }
812 }
813
814 #[rstest]
815 fn test_decode_data_batch_multiple_rows() {
816 let instrument_id = InstrumentId::from("ESM4.GLBX");
817 let metadata = test_metadata();
818 let stats2 = DatabentoStatistics::new(
819 instrument_id,
820 DatabentoStatisticType::ClearedVolume,
821 DatabentoStatisticUpdateAction::Added,
822 None,
823 Some(Quantity::from("200")),
824 2,
825 1,
826 43,
827 2_000_000_000.into(),
828 600,
829 3_000_000_000.into(),
830 4_000_000_000.into(),
831 5_000_000_000.into(),
832 );
833 let original = vec![test_statistics(instrument_id), stats2];
834 let batch = DatabentoStatistics::encode_batch(&metadata, &original).unwrap();
835 let data_vec = DatabentoStatistics::decode_data_batch(&metadata, batch).unwrap();
836
837 assert_eq!(data_vec.len(), 2);
838 for (i, data) in data_vec.iter().enumerate() {
839 match data {
840 Data::Custom(custom) => {
841 let stats = custom
842 .data
843 .as_any()
844 .downcast_ref::<DatabentoStatistics>()
845 .unwrap();
846 assert_eq!(stats.instrument_id, original[i].instrument_id);
847 assert_eq!(stats.stat_type, original[i].stat_type);
848 assert_eq!(stats.price, original[i].price);
849 assert_eq!(stats.quantity, original[i].quantity);
850 }
851 other => panic!("Expected Data::Custom, was {other:?}"),
852 }
853 }
854 }
855
856 #[rstest]
857 fn test_ipc_stream_round_trip() {
858 use std::io::Cursor;
859
860 use arrow::ipc::{reader::StreamReader, writer::StreamWriter};
861
862 let instrument_id = InstrumentId::from("ESM4.GLBX");
863 let original = vec![
864 test_statistics(instrument_id),
865 DatabentoStatistics::new(
866 instrument_id,
867 DatabentoStatisticType::ClearedVolume,
868 DatabentoStatisticUpdateAction::Added,
869 None,
870 Some(Quantity::from("200")),
871 2,
872 1,
873 43,
874 2_000_000_000.into(),
875 600,
876 3_000_000_000.into(),
877 4_000_000_000.into(),
878 5_000_000_000.into(),
879 ),
880 ];
881 let batch = statistics_to_arrow_record_batch(&original).unwrap();
882
883 let mut cursor = Cursor::new(Vec::new());
884 {
885 let mut writer = StreamWriter::try_new(&mut cursor, &batch.schema()).unwrap();
886 writer.write(&batch).unwrap();
887 writer.finish().unwrap();
888 }
889
890 let buffer = cursor.into_inner();
891 let reader = StreamReader::try_new(Cursor::new(buffer), None).unwrap();
892 let mut decoded = Vec::new();
893
894 for batch_result in reader {
895 let batch = batch_result.unwrap();
896 let metadata = batch.schema().metadata().clone();
897 decoded.extend(decode_statistics_batch(&metadata, &batch).unwrap());
898 }
899
900 assert_eq!(decoded.len(), 2);
901 for (orig, dec) in original.iter().zip(decoded.iter()) {
902 assert_eq!(dec, orig);
903 }
904 }
905}