Skip to main content

nautilus_databento/arrow/
statistics.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{collections::HashMap, sync::Arc};
17
18use arrow::{
19    array::{
20        Array, Decimal128Array, Int32Array, TimestampNanosecondArray, UInt8Array, UInt16Array,
21        UInt32Array,
22    },
23    datatypes::{DataType, Field, Schema},
24    error::ArrowError,
25    record_batch::RecordBatch,
26};
27use databento::dbn;
28use nautilus_model::{
29    data::{Data, custom::CustomData},
30    identifiers::InstrumentId,
31    types::{PRICE_UNDEF, Price, QUANTITY_UNDEF, Quantity, fixed::FIXED_PRECISION},
32};
33use nautilus_serialization::arrow::{
34    ArrowSchemaProvider, DecodeDataFromRecordBatch, EncodeToRecordBatch, EncodingError,
35    KEY_TYPE_NAME, decode_decimal_price, decode_decimal_quantity, decode_timestamp,
36    enum_dictionary_array, enum_dictionary_data_type, extract_column, fixed_decimal_data_type,
37    optional_timestamp_array, price_decimal_array, quantity_decimal_array, timestamp_array,
38    timestamp_data_type,
39};
40
41use super::{EnumColumn, parse_metadata};
42use crate::{
43    enums::{DatabentoStatisticType, DatabentoStatisticUpdateAction},
44    types::DatabentoStatistics,
45};
46
47impl ArrowSchemaProvider for DatabentoStatistics {
48    fn get_schema(metadata: Option<HashMap<String, String>>) -> Schema {
49        let fields = vec![
50            Field::new("stat_type", enum_dictionary_data_type(), false),
51            Field::new("update_action", enum_dictionary_data_type(), false),
52            Field::new("price", fixed_decimal_data_type(), true),
53            Field::new("quantity", fixed_decimal_data_type(), true),
54            Field::new("channel_id", DataType::UInt16, false),
55            Field::new("stat_flags", DataType::UInt8, false),
56            Field::new("sequence", DataType::UInt32, false),
57            Field::new("ts_ref", timestamp_data_type(), true),
58            Field::new("ts_in_delta", DataType::Int32, false),
59            Field::new("ts_event", timestamp_data_type(), false),
60            Field::new("ts_recv", timestamp_data_type(), false),
61            Field::new("ts_init", timestamp_data_type(), false),
62        ];
63
64        match metadata {
65            Some(metadata) => Schema::new_with_metadata(fields, metadata),
66            None => Schema::new(fields),
67        }
68    }
69}
70
71impl EncodeToRecordBatch for DatabentoStatistics {
72    fn encode_batch<T>(
73        metadata: &HashMap<String, String>,
74        data: &[T],
75    ) -> Result<RecordBatch, ArrowError>
76    where
77        T: std::borrow::Borrow<Self>,
78    {
79        let (instrument_id, price_precision, size_precision) = parse_metadata(metadata)
80            .map_err(|e| ArrowError::InvalidArgumentError(e.to_string()))?;
81
82        let chunk_metadata = if data.is_empty() {
83            metadata.clone()
84        } else {
85            Self::chunk_metadata(data)
86        };
87
88        let mut channel_id_builder = UInt16Array::builder(data.len());
89        let mut stat_flags_builder = UInt8Array::builder(data.len());
90        let mut sequence_builder = UInt32Array::builder(data.len());
91        let mut ts_in_delta_builder = Int32Array::builder(data.len());
92
93        for (index, item) in data.iter().map(std::borrow::Borrow::borrow).enumerate() {
94            if item.instrument_id != instrument_id || !item.matches_chunk_metadata(&chunk_metadata)
95            {
96                return Err(ArrowError::InvalidArgumentError(
97                    EncodingError::MixedMetadata { index }.to_string(),
98                ));
99            }
100
101            if let Some(price) = item.price.filter(|price| !price.is_undefined()) {
102                Price::from_raw_checked(price.raw(), price_precision).map_err(|e| {
103                    ArrowError::InvalidArgumentError(format!("row {index}: price: {e}"))
104                })?;
105            }
106
107            if let Some(quantity) = item.quantity.filter(|quantity| !quantity.is_undefined()) {
108                Quantity::from_raw_checked(quantity.raw(), size_precision).map_err(|e| {
109                    ArrowError::InvalidArgumentError(format!("row {index}: quantity: {e}"))
110                })?;
111            }
112
113            channel_id_builder.append_value(item.channel_id);
114            stat_flags_builder.append_value(item.stat_flags);
115            sequence_builder.append_value(item.sequence);
116            ts_in_delta_builder.append_value(item.ts_in_delta);
117        }
118
119        RecordBatch::try_new(
120            Self::get_schema(Some(metadata.clone())).into(),
121            vec![
122                Arc::new(enum_dictionary_array(
123                    data.iter().map(|item| item.borrow().stat_type),
124                )?),
125                Arc::new(enum_dictionary_array(
126                    data.iter().map(|item| item.borrow().update_action),
127                )?),
128                Arc::new(price_decimal_array(
129                    data.iter()
130                        .map(|item| item.borrow().price.map_or(PRICE_UNDEF, |value| value.raw())),
131                    "price",
132                )?),
133                Arc::new(quantity_decimal_array(
134                    data.iter().map(|item| {
135                        item.borrow()
136                            .quantity
137                            .map_or(QUANTITY_UNDEF, |value| value.raw())
138                    }),
139                    "quantity",
140                )?),
141                Arc::new(channel_id_builder.finish()),
142                Arc::new(stat_flags_builder.finish()),
143                Arc::new(sequence_builder.finish()),
144                Arc::new(optional_timestamp_array(data.iter().map(|item| {
145                    let value = item.borrow().ts_ref.as_u64();
146                    (value != dbn::UNDEF_TIMESTAMP).then_some(value)
147                }))?),
148                Arc::new(ts_in_delta_builder.finish()),
149                Arc::new(timestamp_array(
150                    data.iter().map(|item| item.borrow().ts_event.as_u64()),
151                )?),
152                Arc::new(timestamp_array(
153                    data.iter().map(|item| item.borrow().ts_recv.as_u64()),
154                )?),
155                Arc::new(timestamp_array(
156                    data.iter().map(|item| item.borrow().ts_init.as_u64()),
157                )?),
158            ],
159        )
160    }
161
162    fn metadata(&self) -> HashMap<String, String> {
163        statistics_metadata(
164            &self.instrument_id,
165            self.price.map_or(FIXED_PRECISION, |p| p.precision),
166            self.quantity.map_or(FIXED_PRECISION, |q| q.precision),
167        )
168    }
169
170    fn chunk_metadata<T>(chunk: &[T]) -> HashMap<String, String>
171    where
172        T: std::borrow::Borrow<Self>,
173    {
174        let first = chunk
175            .first()
176            .map(std::borrow::Borrow::borrow)
177            .expect("Chunk should have at least one element to encode");
178
179        let price_precision = chunk
180            .iter()
181            .map(std::borrow::Borrow::borrow)
182            .filter_map(|s| s.price)
183            .find(|price| !price.is_undefined())
184            .map_or(FIXED_PRECISION, |price| price.precision);
185        let size_precision = chunk
186            .iter()
187            .map(std::borrow::Borrow::borrow)
188            .filter_map(|s| s.quantity)
189            .find(|quantity| !quantity.is_undefined())
190            .map_or(FIXED_PRECISION, |quantity| quantity.precision);
191
192        statistics_metadata(&first.instrument_id, price_precision, size_precision)
193    }
194
195    fn matches_chunk_metadata(&self, metadata: &HashMap<String, String>) -> bool {
196        let Ok((instrument_id, price_precision, size_precision)) = parse_metadata(metadata) else {
197            return false;
198        };
199
200        self.instrument_id == instrument_id
201            && self
202                .price
203                .is_none_or(|price| price.is_undefined() || price.precision == price_precision)
204            && self.quantity.is_none_or(|quantity| {
205                quantity.is_undefined() || quantity.precision == size_precision
206            })
207    }
208}
209
210impl DecodeDataFromRecordBatch for DatabentoStatistics {
211    fn decode_data_batch(
212        metadata: &HashMap<String, String>,
213        record_batch: RecordBatch,
214    ) -> Result<Vec<Data>, EncodingError> {
215        let items = decode_statistics_batch(metadata, &record_batch)?;
216        Ok(items
217            .into_iter()
218            .map(|item| Data::Custom(CustomData::from_arc(Arc::new(item))))
219            .collect())
220    }
221}
222
223/// Decodes a `RecordBatch` into a vector of [`DatabentoStatistics`].
224///
225/// # Errors
226///
227/// Returns an error if decoding fails.
228pub fn decode_statistics_batch(
229    metadata: &HashMap<String, String>,
230    record_batch: &RecordBatch,
231) -> Result<Vec<DatabentoStatistics>, EncodingError> {
232    let (instrument_id, price_precision, size_precision) = parse_metadata(metadata)?;
233    let cols = record_batch.columns();
234
235    let price_values =
236        extract_column::<Decimal128Array>(cols, "price", 2, fixed_decimal_data_type())?;
237    let quantity_values =
238        extract_column::<Decimal128Array>(cols, "quantity", 3, fixed_decimal_data_type())?;
239    let channel_id_values = extract_column::<UInt16Array>(cols, "channel_id", 4, DataType::UInt16)?;
240    let stat_flags_values = extract_column::<UInt8Array>(cols, "stat_flags", 5, DataType::UInt8)?;
241    let sequence_values = extract_column::<UInt32Array>(cols, "sequence", 6, DataType::UInt32)?;
242    let ts_ref_values =
243        extract_column::<TimestampNanosecondArray>(cols, "ts_ref", 7, timestamp_data_type())?;
244    let ts_in_delta_values = extract_column::<Int32Array>(cols, "ts_in_delta", 8, DataType::Int32)?;
245    let ts_event_values =
246        extract_column::<TimestampNanosecondArray>(cols, "ts_event", 9, timestamp_data_type())?;
247    let ts_recv_values =
248        extract_column::<TimestampNanosecondArray>(cols, "ts_recv", 10, timestamp_data_type())?;
249    let ts_init_values =
250        extract_column::<TimestampNanosecondArray>(cols, "ts_init", 11, timestamp_data_type())?;
251    let stat_type_column = EnumColumn::try_from_column(&cols[0], "stat_type", 0)?;
252    let update_action_column = EnumColumn::try_from_column(&cols[1], "update_action", 1)?;
253
254    (0..record_batch.num_rows())
255        .map(|row| {
256            let stat_type = stat_type_column.decode::<DatabentoStatisticType>(row)?;
257            let update_action =
258                update_action_column.decode::<DatabentoStatisticUpdateAction>(row)?;
259
260            let price = (!price_values.is_null(row))
261                .then(|| decode_decimal_price(price_values, price_precision, "price", row))
262                .transpose()?;
263            let quantity = (!quantity_values.is_null(row))
264                .then(|| decode_decimal_quantity(quantity_values, size_precision, "quantity", row))
265                .transpose()?;
266
267            Ok(DatabentoStatistics {
268                instrument_id,
269                stat_type,
270                update_action,
271                price,
272                quantity,
273                channel_id: channel_id_values.value(row),
274                stat_flags: stat_flags_values.value(row),
275                sequence: sequence_values.value(row),
276                ts_ref: if ts_ref_values.is_null(row) {
277                    dbn::UNDEF_TIMESTAMP.into()
278                } else {
279                    decode_timestamp(ts_ref_values, "ts_ref", row)?.into()
280                },
281                ts_in_delta: ts_in_delta_values.value(row),
282                ts_event: decode_timestamp(ts_event_values, "ts_event", row)?.into(),
283                ts_recv: decode_timestamp(ts_recv_values, "ts_recv", row)?.into(),
284                ts_init: decode_timestamp(ts_init_values, "ts_init", row)?.into(),
285            })
286        })
287        .collect()
288}
289
290fn statistics_metadata(
291    instrument_id: &InstrumentId,
292    price_precision: u8,
293    size_precision: u8,
294) -> HashMap<String, String> {
295    let mut metadata =
296        DatabentoStatistics::get_metadata(instrument_id, price_precision, size_precision);
297    metadata.insert(KEY_TYPE_NAME.to_string(), "DatabentoStatistics".to_string());
298    metadata
299}
300
301/// Encodes a vector of [`DatabentoStatistics`] into an Arrow `RecordBatch`.
302///
303/// # Errors
304///
305/// Returns an error if `data` is empty, instruments or non-null precisions differ, or encoding fails.
306// Guarded by empty check
307pub fn statistics_to_arrow_record_batch(
308    data: &[DatabentoStatistics],
309) -> Result<RecordBatch, EncodingError> {
310    if data.is_empty() {
311        return Err(EncodingError::EmptyData);
312    }
313
314    let metadata = DatabentoStatistics::chunk_metadata(data);
315    DatabentoStatistics::encode_batch(&metadata, data).map_err(EncodingError::ArrowError)
316}
317
318#[cfg(test)]
319mod tests {
320    use std::collections::HashMap;
321
322    use nautilus_model::{
323        data::{custom::CustomDataTrait, registry::encode_custom_to_arrow},
324        identifiers::InstrumentId,
325        types::{Price, Quantity},
326    };
327    use nautilus_serialization::arrow::{
328        ArrowSchemaProvider, EncodeToRecordBatch, KEY_INSTRUMENT_ID, KEY_PRICE_PRECISION,
329        KEY_SIZE_PRECISION, custom::ensure_custom_data_registered,
330    };
331    use rstest::rstest;
332
333    use super::*;
334
335    fn test_metadata() -> HashMap<String, String> {
336        HashMap::from([
337            (KEY_INSTRUMENT_ID.to_string(), "ESM4.GLBX".to_string()),
338            (KEY_PRICE_PRECISION.to_string(), "2".to_string()),
339            (KEY_SIZE_PRECISION.to_string(), "0".to_string()),
340        ])
341    }
342
343    fn test_statistics(instrument_id: InstrumentId) -> DatabentoStatistics {
344        DatabentoStatistics::new(
345            instrument_id,
346            DatabentoStatisticType::OpeningPrice,
347            DatabentoStatisticUpdateAction::Added,
348            Some(Price::from("5000.50")),
349            Some(Quantity::from("100")),
350            1,
351            0,
352            42,
353            1_000_000_000.into(),
354            500,
355            2_000_000_000.into(),
356            3_000_000_000.into(),
357            4_000_000_000.into(),
358        )
359    }
360
361    #[rstest]
362    fn test_get_schema() {
363        let schema = DatabentoStatistics::get_schema(None);
364        assert_eq!(schema.fields().len(), 12);
365        assert_eq!(schema.field(0).name(), "stat_type");
366        assert_eq!(schema.field(11).name(), "ts_init");
367        assert_eq!(schema.field(0).data_type(), &enum_dictionary_data_type());
368        assert_eq!(schema.field(2).data_type(), &fixed_decimal_data_type());
369        assert!(schema.field(2).is_nullable());
370        assert!(schema.field(7).is_nullable());
371        assert_eq!(schema.field(10).data_type(), &timestamp_data_type());
372    }
373
374    #[rstest]
375    fn test_encode_batch() {
376        let instrument_id = InstrumentId::from("ESM4.GLBX");
377        let metadata = test_metadata();
378        let data = vec![test_statistics(instrument_id)];
379        let batch = DatabentoStatistics::encode_batch(&metadata, &data).unwrap();
380
381        assert_eq!(batch.num_rows(), 1);
382        assert_eq!(batch.num_columns(), 12);
383    }
384
385    #[rstest]
386    fn test_encode_decode_round_trip() {
387        let instrument_id = InstrumentId::from("ESM4.GLBX");
388        let metadata = test_metadata();
389        let original = vec![test_statistics(instrument_id)];
390        let batch = DatabentoStatistics::encode_batch(&metadata, &original).unwrap();
391        let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
392
393        assert_eq!(decoded.len(), 1);
394        assert_eq!(decoded[0].instrument_id, instrument_id);
395        assert_eq!(decoded[0].stat_type, original[0].stat_type);
396        assert_eq!(decoded[0].update_action, original[0].update_action);
397        assert_eq!(decoded[0].price, original[0].price);
398        assert_eq!(decoded[0].quantity, original[0].quantity);
399        assert_eq!(decoded[0].channel_id, original[0].channel_id);
400        assert_eq!(decoded[0].stat_flags, original[0].stat_flags);
401        assert_eq!(decoded[0].sequence, original[0].sequence);
402        assert_eq!(decoded[0].ts_ref, original[0].ts_ref);
403        assert_eq!(decoded[0].ts_in_delta, original[0].ts_in_delta);
404        assert_eq!(decoded[0].ts_event, original[0].ts_event);
405        assert_eq!(decoded[0].ts_recv, original[0].ts_recv);
406        assert_eq!(decoded[0].ts_init, original[0].ts_init);
407    }
408
409    #[rstest]
410    fn test_decode_legacy_enum_columns() {
411        let instrument_id = InstrumentId::from("ESM4.GLBX");
412        let metadata = test_metadata();
413        let original = test_statistics(instrument_id);
414        let batch =
415            DatabentoStatistics::encode_batch(&metadata, std::slice::from_ref(&original)).unwrap();
416        let mut fields = batch.schema().fields().to_vec();
417        fields[0] = Arc::new(Field::new("stat_type", DataType::UInt8, false));
418        fields[1] = Arc::new(Field::new("update_action", DataType::UInt8, false));
419        let mut columns = batch.columns().to_vec();
420        columns[0] = Arc::new(UInt8Array::from(vec![original.stat_type as u8]));
421        columns[1] = Arc::new(UInt8Array::from(vec![original.update_action as u8]));
422        let legacy_batch = RecordBatch::try_new(
423            Arc::new(Schema::new_with_metadata(fields, metadata.clone())),
424            columns,
425        )
426        .unwrap();
427
428        let decoded = decode_statistics_batch(&metadata, &legacy_batch).unwrap();
429
430        assert_eq!(decoded, vec![original]);
431    }
432
433    #[rstest]
434    fn test_encode_decode_round_trip_with_none_values() {
435        let instrument_id = InstrumentId::from("ESM4.GLBX");
436        let metadata = test_metadata();
437        let stats = DatabentoStatistics::new(
438            instrument_id,
439            DatabentoStatisticType::ClearedVolume,
440            DatabentoStatisticUpdateAction::Added,
441            None,
442            None,
443            1,
444            0,
445            42,
446            dbn::UNDEF_TIMESTAMP.into(),
447            500,
448            2_000_000_000.into(),
449            3_000_000_000.into(),
450            4_000_000_000.into(),
451        );
452        let original = vec![stats];
453        let batch = DatabentoStatistics::encode_batch(&metadata, &original).unwrap();
454        let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
455
456        assert_eq!(decoded.len(), 1);
457        let ts_ref = batch
458            .column(7)
459            .as_any()
460            .downcast_ref::<TimestampNanosecondArray>()
461            .unwrap();
462        assert!(ts_ref.is_null(0));
463        assert_eq!(decoded[0].price, None);
464        assert_eq!(decoded[0].quantity, None);
465        assert_eq!(decoded[0].ts_ref.as_u64(), dbn::UNDEF_TIMESTAMP);
466    }
467
468    #[rstest]
469    fn test_chunk_metadata_uses_first_non_none_precision() {
470        let instrument_id = InstrumentId::from("ESM4.GLBX");
471        let none_stats = DatabentoStatistics::new(
472            instrument_id,
473            DatabentoStatisticType::ClearedVolume,
474            DatabentoStatisticUpdateAction::Added,
475            None,
476            None,
477            1,
478            0,
479            42,
480            1_000_000_000.into(),
481            500,
482            2_000_000_000.into(),
483            3_000_000_000.into(),
484            4_000_000_000.into(),
485        );
486        let some_stats = test_statistics(instrument_id);
487        let data = vec![none_stats, some_stats];
488
489        let batch = statistics_to_arrow_record_batch(&data).unwrap();
490        let metadata = batch.schema().metadata().clone();
491        let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
492
493        assert_eq!(decoded.len(), 2);
494        assert_eq!(decoded[0].price, None);
495        assert_eq!(decoded[0].quantity, None);
496        assert_eq!(decoded[1].price, data[1].price);
497        assert_eq!(decoded[1].quantity, data[1].quantity);
498    }
499
500    #[rstest]
501    fn test_encode_decode_multiple_rows() {
502        let instrument_id = InstrumentId::from("ESM4.GLBX");
503        let metadata = test_metadata();
504        let stats1 = test_statistics(instrument_id);
505        let stats2 = DatabentoStatistics::new(
506            instrument_id,
507            DatabentoStatisticType::ClearedVolume,
508            DatabentoStatisticUpdateAction::Added,
509            Some(Price::from("5100.25")),
510            None,
511            2,
512            1,
513            43,
514            2_000_000_000.into(),
515            600,
516            3_000_000_000.into(),
517            4_000_000_000.into(),
518            5_000_000_000.into(),
519        );
520        let stats3 = DatabentoStatistics::new(
521            instrument_id,
522            DatabentoStatisticType::OpeningPrice,
523            DatabentoStatisticUpdateAction::Added,
524            None,
525            Some(Quantity::from("200")),
526            3,
527            0,
528            44,
529            3_000_000_000.into(),
530            700,
531            4_000_000_000.into(),
532            5_000_000_000.into(),
533            6_000_000_000.into(),
534        );
535        let original = vec![stats1, stats2, stats3];
536
537        let batch = DatabentoStatistics::encode_batch(&metadata, &original).unwrap();
538        assert_eq!(batch.num_rows(), 3);
539
540        let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
541        assert_eq!(decoded.len(), 3);
542        for (orig, dec) in original.iter().zip(decoded.iter()) {
543            assert_eq!(dec.instrument_id, orig.instrument_id);
544            assert_eq!(dec.stat_type, orig.stat_type);
545            assert_eq!(dec.price, orig.price);
546            assert_eq!(dec.quantity, orig.quantity);
547            assert_eq!(dec.channel_id, orig.channel_id);
548            assert_eq!(dec.sequence, orig.sequence);
549        }
550    }
551
552    #[rstest]
553    fn test_statistics_to_arrow_record_batch_round_trip() {
554        let instrument_id = InstrumentId::from("ESM4.GLBX");
555        let original = vec![test_statistics(instrument_id)];
556        let batch = statistics_to_arrow_record_batch(&original).unwrap();
557        let metadata = batch.schema().metadata().clone();
558        let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
559
560        assert_eq!(decoded.len(), 1);
561        assert_eq!(decoded[0].price, original[0].price);
562        assert_eq!(decoded[0].quantity, original[0].quantity);
563    }
564
565    #[rstest]
566    #[case::price(false, false)]
567    #[case::price_reversed(false, true)]
568    #[case::quantity(true, false)]
569    #[case::quantity_reversed(true, true)]
570    fn test_automatic_encoding_rejects_mixed_precision(
571        #[case] quantity: bool,
572        #[case] reversed: bool,
573    ) {
574        let first = test_statistics(InstrumentId::from("ESM4.GLBX"));
575        let mut second = first.clone();
576        if quantity {
577            second.quantity = Some(Quantity::from("100.0"));
578        } else {
579            second.price = Some(Price::from("5000.500"));
580        }
581
582        let mut data = vec![first, second];
583        if reversed {
584            data.reverse();
585        }
586
587        ensure_custom_data_registered::<DatabentoStatistics>();
588        let custom: Vec<Arc<dyn CustomDataTrait>> = data
589            .iter()
590            .cloned()
591            .map(|item| Arc::new(item) as Arc<dyn CustomDataTrait>)
592            .collect();
593
594        let direct = statistics_to_arrow_record_batch(&data).unwrap_err();
595        let registered = encode_custom_to_arrow("DatabentoStatistics", &custom).unwrap_err();
596
597        let expected =
598            ArrowError::InvalidArgumentError(EncodingError::MixedMetadata { index: 1 }.to_string());
599        assert_eq!(direct.to_string(), format!("Arrow error: {expected}"));
600        assert_eq!(registered.to_string(), expected.to_string());
601    }
602
603    #[rstest]
604    #[case::price(false)]
605    #[case::quantity(true)]
606    fn test_explicit_encoding_rejects_insufficient_precision(#[case] quantity: bool) {
607        let mut item = test_statistics(InstrumentId::from("ESM4.GLBX"));
608
609        let (field, expected) = if quantity {
610            let value = Quantity::from("100.5");
611            item.quantity = Some(value);
612            (
613                "quantity",
614                Quantity::from_raw_checked(value.raw(), 0)
615                    .unwrap_err()
616                    .to_string(),
617            )
618        } else {
619            let value = Price::from("5000.501");
620            item.price = Some(value);
621            (
622                "price",
623                Price::from_raw_checked(value.raw(), 2)
624                    .unwrap_err()
625                    .to_string(),
626            )
627        };
628
629        let error = DatabentoStatistics::encode_batch(&test_metadata(), &[item]).unwrap_err();
630
631        assert_eq!(
632            error.to_string(),
633            format!("Invalid argument error: row 0: {field}: {expected}")
634        );
635    }
636
637    #[rstest]
638    fn test_encoding_rejects_mixed_instruments() {
639        let first = test_statistics(InstrumentId::from("ESM4.GLBX"));
640        let second = test_statistics(InstrumentId::from("ESU4.GLBX"));
641        let data = vec![first, second];
642        ensure_custom_data_registered::<DatabentoStatistics>();
643        let custom: Vec<Arc<dyn CustomDataTrait>> = data
644            .iter()
645            .cloned()
646            .map(|item| Arc::new(item) as Arc<dyn CustomDataTrait>)
647            .collect();
648
649        let direct = statistics_to_arrow_record_batch(&data).unwrap_err();
650        let registered = encode_custom_to_arrow("DatabentoStatistics", &custom).unwrap_err();
651        let explicit = DatabentoStatistics::encode_batch(&test_metadata(), &data).unwrap_err();
652
653        let expected =
654            ArrowError::InvalidArgumentError(EncodingError::MixedMetadata { index: 1 }.to_string());
655        assert_eq!(direct.to_string(), format!("Arrow error: {expected}"));
656        assert_eq!(registered.to_string(), expected.to_string());
657        assert_eq!(explicit.to_string(), expected.to_string());
658    }
659
660    #[rstest]
661    #[case::none(false)]
662    #[case::undefined(true)]
663    fn test_registered_encoding_uses_first_non_null_precision(#[case] undefined: bool) {
664        let item = test_statistics(InstrumentId::from("ESM4.GLBX"));
665        let mut empty = item.clone();
666        empty.price = undefined.then_some(Price::from_raw(PRICE_UNDEF, 0));
667        empty.quantity = undefined.then_some(Quantity::from_raw(QUANTITY_UNDEF, 0));
668        let original = vec![empty, item];
669        let custom: Vec<Arc<dyn CustomDataTrait>> = original
670            .iter()
671            .cloned()
672            .map(|item| Arc::new(item) as Arc<dyn CustomDataTrait>)
673            .collect();
674        ensure_custom_data_registered::<DatabentoStatistics>();
675
676        let batch = encode_custom_to_arrow("DatabentoStatistics", &custom)
677            .unwrap()
678            .unwrap();
679        let decoded = decode_statistics_batch(batch.schema().metadata(), &batch).unwrap();
680
681        let mut expected = original;
682        expected[0].price = None;
683        expected[0].quantity = None;
684        assert_eq!(decoded, expected);
685        assert_eq!(decoded[1].price.unwrap().precision, 2);
686        assert_eq!(decoded[1].quantity.unwrap().precision, 0);
687    }
688
689    #[rstest]
690    fn test_chunk_metadata_all_none_uses_fixed_precision() {
691        use nautilus_model::types::fixed::FIXED_PRECISION;
692
693        let instrument_id = InstrumentId::from("ESM4.GLBX");
694        let stats = DatabentoStatistics::new(
695            instrument_id,
696            DatabentoStatisticType::ClearedVolume,
697            DatabentoStatisticUpdateAction::Added,
698            None,
699            None,
700            1,
701            0,
702            42,
703            1_000_000_000.into(),
704            500,
705            2_000_000_000.into(),
706            3_000_000_000.into(),
707            4_000_000_000.into(),
708        );
709        let data = vec![stats];
710        let metadata = DatabentoStatistics::chunk_metadata(&data);
711
712        assert_eq!(
713            metadata.get(KEY_PRICE_PRECISION).unwrap(),
714            &FIXED_PRECISION.to_string(),
715        );
716        assert_eq!(
717            metadata.get(KEY_SIZE_PRECISION).unwrap(),
718            &FIXED_PRECISION.to_string(),
719        );
720    }
721
722    #[rstest]
723    fn test_all_none_metadata_decodes_real_prices_correctly() {
724        use nautilus_model::types::fixed::FIXED_PRECISION;
725
726        let instrument_id = InstrumentId::from("ESM4.GLBX");
727        let price = Price::from("5000.50");
728        let quantity = Quantity::from("100");
729        let stats = DatabentoStatistics::new(
730            instrument_id,
731            DatabentoStatisticType::OpeningPrice,
732            DatabentoStatisticUpdateAction::Added,
733            Some(price),
734            Some(quantity),
735            1,
736            0,
737            42,
738            1_000_000_000.into(),
739            500,
740            2_000_000_000.into(),
741            3_000_000_000.into(),
742            4_000_000_000.into(),
743        );
744
745        // Encode with FIXED_PRECISION metadata (as if from an all-None chunk)
746        let metadata = HashMap::from([
747            (KEY_INSTRUMENT_ID.to_string(), "ESM4.GLBX".to_string()),
748            (KEY_PRICE_PRECISION.to_string(), FIXED_PRECISION.to_string()),
749            (KEY_SIZE_PRECISION.to_string(), FIXED_PRECISION.to_string()),
750        ]);
751
752        let batch = DatabentoStatistics::encode_batch(&metadata, &[stats]).unwrap();
753        let decoded = decode_statistics_batch(&metadata, &batch).unwrap();
754
755        assert_eq!(decoded.len(), 1);
756        assert_eq!(decoded[0].price.unwrap().as_f64(), price.as_f64());
757        assert_eq!(decoded[0].quantity.unwrap().as_f64(), quantity.as_f64());
758    }
759
760    #[rstest]
761    fn test_get_schema_with_metadata() {
762        let metadata = test_metadata();
763        let schema = DatabentoStatistics::get_schema(Some(metadata.clone()));
764        assert_eq!(schema.metadata(), &metadata);
765        assert_eq!(schema.fields().len(), 12);
766    }
767
768    #[rstest]
769    fn test_decode_missing_metadata_returns_error() {
770        let instrument_id = InstrumentId::from("ESM4.GLBX");
771        let metadata = test_metadata();
772        let data = vec![test_statistics(instrument_id)];
773        let batch = DatabentoStatistics::encode_batch(&metadata, &data).unwrap();
774
775        let empty_metadata = HashMap::new();
776        let result = decode_statistics_batch(&empty_metadata, &batch);
777        assert!(result.is_err());
778    }
779
780    #[rstest]
781    fn test_statistics_to_arrow_record_batch_empty() {
782        let result = statistics_to_arrow_record_batch(&[]);
783        assert!(result.is_err());
784    }
785
786    #[rstest]
787    fn test_decode_data_batch_produces_custom_data() {
788        let instrument_id = InstrumentId::from("ESM4.GLBX");
789        let metadata = test_metadata();
790        let original = vec![test_statistics(instrument_id)];
791        let batch = DatabentoStatistics::encode_batch(&metadata, &original).unwrap();
792        let data_vec = DatabentoStatistics::decode_data_batch(&metadata, batch).unwrap();
793
794        assert_eq!(data_vec.len(), 1);
795        match &data_vec[0] {
796            Data::Custom(custom) => {
797                assert_eq!(custom.data.type_name(), "DatabentoStatistics");
798                let stats = custom
799                    .data
800                    .as_any()
801                    .downcast_ref::<DatabentoStatistics>()
802                    .unwrap();
803                assert_eq!(stats.instrument_id, instrument_id);
804                assert_eq!(stats.stat_type, original[0].stat_type);
805                assert_eq!(stats.price, original[0].price);
806                assert_eq!(stats.quantity, original[0].quantity);
807                assert_eq!(stats.ts_event, original[0].ts_event);
808                assert_eq!(stats.ts_init, original[0].ts_init);
809            }
810            other => panic!("Expected Data::Custom, was {other:?}"),
811        }
812    }
813
814    #[rstest]
815    fn test_decode_data_batch_multiple_rows() {
816        let instrument_id = InstrumentId::from("ESM4.GLBX");
817        let metadata = test_metadata();
818        let stats2 = DatabentoStatistics::new(
819            instrument_id,
820            DatabentoStatisticType::ClearedVolume,
821            DatabentoStatisticUpdateAction::Added,
822            None,
823            Some(Quantity::from("200")),
824            2,
825            1,
826            43,
827            2_000_000_000.into(),
828            600,
829            3_000_000_000.into(),
830            4_000_000_000.into(),
831            5_000_000_000.into(),
832        );
833        let original = vec![test_statistics(instrument_id), stats2];
834        let batch = DatabentoStatistics::encode_batch(&metadata, &original).unwrap();
835        let data_vec = DatabentoStatistics::decode_data_batch(&metadata, batch).unwrap();
836
837        assert_eq!(data_vec.len(), 2);
838        for (i, data) in data_vec.iter().enumerate() {
839            match data {
840                Data::Custom(custom) => {
841                    let stats = custom
842                        .data
843                        .as_any()
844                        .downcast_ref::<DatabentoStatistics>()
845                        .unwrap();
846                    assert_eq!(stats.instrument_id, original[i].instrument_id);
847                    assert_eq!(stats.stat_type, original[i].stat_type);
848                    assert_eq!(stats.price, original[i].price);
849                    assert_eq!(stats.quantity, original[i].quantity);
850                }
851                other => panic!("Expected Data::Custom, was {other:?}"),
852            }
853        }
854    }
855
856    #[rstest]
857    fn test_ipc_stream_round_trip() {
858        use std::io::Cursor;
859
860        use arrow::ipc::{reader::StreamReader, writer::StreamWriter};
861
862        let instrument_id = InstrumentId::from("ESM4.GLBX");
863        let original = vec![
864            test_statistics(instrument_id),
865            DatabentoStatistics::new(
866                instrument_id,
867                DatabentoStatisticType::ClearedVolume,
868                DatabentoStatisticUpdateAction::Added,
869                None,
870                Some(Quantity::from("200")),
871                2,
872                1,
873                43,
874                2_000_000_000.into(),
875                600,
876                3_000_000_000.into(),
877                4_000_000_000.into(),
878                5_000_000_000.into(),
879            ),
880        ];
881        let batch = statistics_to_arrow_record_batch(&original).unwrap();
882
883        let mut cursor = Cursor::new(Vec::new());
884        {
885            let mut writer = StreamWriter::try_new(&mut cursor, &batch.schema()).unwrap();
886            writer.write(&batch).unwrap();
887            writer.finish().unwrap();
888        }
889
890        let buffer = cursor.into_inner();
891        let reader = StreamReader::try_new(Cursor::new(buffer), None).unwrap();
892        let mut decoded = Vec::new();
893
894        for batch_result in reader {
895            let batch = batch_result.unwrap();
896            let metadata = batch.schema().metadata().clone();
897            decoded.extend(decode_statistics_batch(&metadata, &batch).unwrap());
898        }
899
900        assert_eq!(decoded.len(), 2);
901        for (orig, dec) in original.iter().zip(decoded.iter()) {
902            assert_eq!(dec, orig);
903        }
904    }
905}