1use std::{io::Cursor, sync::Arc};
17
18use arrow::{
19 datatypes::Schema,
20 ffi_stream::FFI_ArrowArrayStream,
21 ipc::{reader::StreamReader, writer::StreamWriter},
22 record_batch::{RecordBatch, RecordBatchIterator},
23};
24use nautilus_core::python::{to_pyruntime_err, to_pytype_err, to_pyvalue_err};
25use nautilus_model::{
26 data::{
27 Bar, FundingRateUpdate, IndexPriceUpdate, InstrumentStatus, MarkPriceUpdate, OptionGreeks,
28 OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick,
29 close::InstrumentClose,
30 greeks::{GreeksData, PortfolioGreeks},
31 },
32 python::data::{
33 pyobjects_to_bars, pyobjects_to_book_deltas, pyobjects_to_index_prices,
34 pyobjects_to_instrument_closes, pyobjects_to_instrument_statuses, pyobjects_to_mark_prices,
35 pyobjects_to_option_greeks, pyobjects_to_quotes, pyobjects_to_trades,
36 },
37};
38use pyo3::{
39 conversion::IntoPyObjectExt,
40 prelude::*,
41 types::{PyBytes, PyCapsule, PyType},
42};
43
44use crate::arrow::{
45 ArrowSchemaProvider, DecodeFromRecordBatch, DecodeTypedFromRecordBatch, EncodeToRecordBatch,
46 bars_to_arrow_record_batch_bytes, book_deltas_to_arrow_record_batch_bytes,
47 book_depths_to_arrow_record_batch_bytes, encode_batch_with_metadata,
48 index_prices_to_arrow_record_batch_bytes, instrument_closes_to_arrow_record_batch_bytes,
49 instrument_status_to_arrow_record_batch_bytes, mark_prices_to_arrow_record_batch_bytes,
50 option_greeks_to_arrow_record_batch_bytes, quotes_to_arrow_record_batch_bytes,
51 trades_to_arrow_record_batch_bytes,
52};
53
54pub fn arrow_record_batch_to_pybytes(py: Python, batch: &RecordBatch) -> PyResult<Py<PyBytes>> {
60 arrow_record_batches_to_pybytes(py, &batch.schema(), std::slice::from_ref(batch))
61}
62
63pub fn arrow_record_batches_to_pybytes(
69 py: Python,
70 schema: &Schema,
71 batches: &[RecordBatch],
72) -> PyResult<Py<PyBytes>> {
73 let mut cursor = Cursor::new(Vec::new());
74 {
75 let mut writer = StreamWriter::try_new(&mut cursor, schema).map_err(to_pyruntime_err)?;
76
77 for batch in batches {
78 writer.write(batch).map_err(to_pyruntime_err)?;
79 }
80
81 writer.finish().map_err(to_pyruntime_err)?;
82 }
83
84 let buffer = cursor.into_inner();
85 let pybytes = PyBytes::new(py, &buffer);
86
87 Ok(pybytes.into())
88}
89
90pub fn arrow_record_batches_to_pyarrow_stream(
96 py: Python<'_>,
97 schema: &Schema,
98 batches: Vec<RecordBatch>,
99) -> PyResult<Py<PyAny>> {
100 let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), Arc::new(schema.clone()));
101 let stream = FFI_ArrowArrayStream::new(Box::new(reader));
102
103 let capsule = PyCapsule::new_with_value(py, stream, c"arrow_array_stream")?;
105 Ok(capsule.into_any().unbind())
106}
107
108#[pyfunction]
114#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
115pub fn get_arrow_schema_map(py: Python<'_>, cls: &Bound<'_, PyType>) -> PyResult<Py<PyAny>> {
116 let cls_str: String = cls.getattr("__name__")?.extract()?;
117 let result_map = match cls_str.as_str() {
118 stringify!(OrderBookDelta) => OrderBookDelta::get_schema_map(),
119 stringify!(OrderBookDepth) => OrderBookDepth::get_schema_map(),
120 stringify!(QuoteTick) => QuoteTick::get_schema_map(),
121 stringify!(TradeTick) => TradeTick::get_schema_map(),
122 stringify!(Bar) => Bar::get_schema_map(),
123 stringify!(MarkPriceUpdate) => MarkPriceUpdate::get_schema_map(),
124 stringify!(IndexPriceUpdate) => IndexPriceUpdate::get_schema_map(),
125 stringify!(FundingRateUpdate) => FundingRateUpdate::get_schema_map(),
126 stringify!(InstrumentStatus) => InstrumentStatus::get_schema_map(),
127 stringify!(OptionGreeks) => OptionGreeks::get_schema_map(),
128 stringify!(InstrumentClose) => InstrumentClose::get_schema_map(),
129 stringify!(GreeksData) => GreeksData::get_schema_map(),
130 stringify!(PortfolioGreeks) => PortfolioGreeks::get_schema_map(),
131 _ => {
132 return Err(to_pytype_err(format!(
133 "Arrow schema for `{cls_str}` is not currently implemented in Rust."
134 )));
135 }
136 };
137
138 result_map.into_py_any(py)
139}
140
141#[pyfunction]
147#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
148pub fn get_arrow_schema_bytes(py: Python<'_>, cls: &Bound<'_, PyType>) -> PyResult<Py<PyBytes>> {
149 let cls_str: String = cls.getattr("__name__")?.extract()?;
150 let schema = match cls_str.as_str() {
151 stringify!(OrderBookDelta) => OrderBookDelta::get_schema(None),
152 stringify!(OrderBookDepth) => OrderBookDepth::get_schema(None),
153 stringify!(QuoteTick) => QuoteTick::get_schema(None),
154 stringify!(TradeTick) => TradeTick::get_schema(None),
155 stringify!(Bar) => Bar::get_schema(None),
156 stringify!(MarkPriceUpdate) => MarkPriceUpdate::get_schema(None),
157 stringify!(IndexPriceUpdate) => IndexPriceUpdate::get_schema(None),
158 stringify!(FundingRateUpdate) => FundingRateUpdate::get_schema(None),
159 stringify!(InstrumentStatus) => InstrumentStatus::get_schema(None),
160 stringify!(OptionGreeks) => OptionGreeks::get_schema(None),
161 stringify!(InstrumentClose) => InstrumentClose::get_schema(None),
162 stringify!(GreeksData) => GreeksData::get_schema(None),
163 stringify!(PortfolioGreeks) => PortfolioGreeks::get_schema(None),
164 _ => {
165 return Err(to_pytype_err(format!(
166 "Arrow schema for `{cls_str}` is not currently implemented in Rust."
167 )));
168 }
169 };
170
171 arrow_record_batches_to_pybytes(py, &schema, &[])
172}
173
174#[pyfunction]
183#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
184#[expect(clippy::missing_panics_doc)] pub fn pyobjects_to_arrow_record_batch_bytes(
186 py: Python,
187 data: Vec<Bound<'_, PyAny>>,
188) -> PyResult<Py<PyBytes>> {
189 if data.is_empty() {
190 return Err(to_pyvalue_err("Empty data"));
191 }
192
193 let data_type: String = data
194 .first()
195 .unwrap() .getattr("__class__")?
197 .getattr("__name__")?
198 .extract()?;
199
200 match data_type.as_str() {
201 stringify!(OrderBookDelta) => {
202 let deltas = pyobjects_to_book_deltas(data)?;
203 py_book_deltas_to_arrow_record_batch_bytes(py, deltas)
204 }
205 stringify!(OrderBookDepth) => {
206 let depth_snapshots: Vec<OrderBookDepth> = data
207 .into_iter()
208 .map(|obj| obj.extract::<OrderBookDepth>().map_err(Into::into))
209 .collect::<PyResult<Vec<OrderBookDepth>>>()?;
210 py_book_depths_to_arrow_record_batch_bytes(py, depth_snapshots)
211 }
212 stringify!(QuoteTick) => {
213 let quotes = pyobjects_to_quotes(data)?;
214 py_quotes_to_arrow_record_batch_bytes(py, quotes)
215 }
216 stringify!(TradeTick) => {
217 let trades = pyobjects_to_trades(data)?;
218 py_trades_to_arrow_record_batch_bytes(py, trades)
219 }
220 stringify!(Bar) => {
221 let bars = pyobjects_to_bars(data)?;
222 py_bars_to_arrow_record_batch_bytes(py, bars)
223 }
224 stringify!(MarkPriceUpdate) => {
225 let updates = pyobjects_to_mark_prices(data)?;
226 py_mark_prices_to_arrow_record_batch_bytes(py, updates)
227 }
228 stringify!(IndexPriceUpdate) => {
229 let index_prices = pyobjects_to_index_prices(data)?;
230 py_index_prices_to_arrow_record_batch_bytes(py, index_prices)
231 }
232 stringify!(InstrumentStatus) => {
233 let statuses = pyobjects_to_instrument_statuses(data)?;
234 py_instrument_status_to_arrow_record_batch_bytes(py, statuses)
235 }
236 stringify!(OptionGreeks) => {
237 let greeks = pyobjects_to_option_greeks(data)?;
238 py_option_greeks_to_arrow_record_batch_bytes(py, greeks)
239 }
240 stringify!(InstrumentClose) => {
241 let closes = pyobjects_to_instrument_closes(data)?;
242 py_instrument_closes_to_arrow_record_batch_bytes(py, closes)
243 }
244 stringify!(GreeksData) => encode_py_records::<GreeksData>(py, data),
245 stringify!(PortfolioGreeks) => encode_py_records::<PortfolioGreeks>(py, data),
246 _ => Err(to_pyvalue_err(format!(
247 "unsupported data type: {data_type}"
248 ))),
249 }
250}
251
252fn encode_py_records<T>(py: Python<'_>, data: Vec<Bound<'_, PyAny>>) -> PyResult<Py<PyBytes>>
253where
254 T: EncodeToRecordBatch + pyo3::PyClass + Clone,
255{
256 let values = data
257 .into_iter()
258 .map(|obj| {
259 obj.extract::<PyRef<'_, T>>()
260 .map(|value| (*value).clone())
261 .map_err(PyErr::from)
262 })
263 .collect::<PyResult<Vec<_>>>()?;
264
265 let batch = encode_batch_with_metadata(&values).map_err(to_pyvalue_err)?;
266 arrow_record_batch_to_pybytes(py, &batch)
267}
268
269#[pyfunction(name = "book_deltas_to_arrow_record_batch_bytes")]
278#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
279#[expect(clippy::needless_pass_by_value)]
280pub fn py_book_deltas_to_arrow_record_batch_bytes(
281 py: Python,
282 data: Vec<OrderBookDelta>,
283) -> PyResult<Py<PyBytes>> {
284 match book_deltas_to_arrow_record_batch_bytes(&data) {
285 Ok(batch) => arrow_record_batch_to_pybytes(py, &batch),
286 Err(e) => Err(to_pyvalue_err(e)),
287 }
288}
289
290#[pyfunction(name = "book_depths_to_arrow_record_batch_bytes")]
299#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
300#[expect(clippy::needless_pass_by_value)]
301pub fn py_book_depths_to_arrow_record_batch_bytes(
302 py: Python,
303 data: Vec<OrderBookDepth>,
304) -> PyResult<Py<PyBytes>> {
305 match book_depths_to_arrow_record_batch_bytes(&data) {
306 Ok(batch) => arrow_record_batch_to_pybytes(py, &batch),
307 Err(e) => Err(to_pyvalue_err(e)),
308 }
309}
310
311#[pyfunction(name = "quotes_to_arrow_record_batch_bytes")]
320#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
321#[expect(clippy::needless_pass_by_value)]
322pub fn py_quotes_to_arrow_record_batch_bytes(
323 py: Python,
324 data: Vec<QuoteTick>,
325) -> PyResult<Py<PyBytes>> {
326 match quotes_to_arrow_record_batch_bytes(&data) {
327 Ok(batch) => arrow_record_batch_to_pybytes(py, &batch),
328 Err(e) => Err(to_pyvalue_err(e)),
329 }
330}
331
332#[pyfunction(name = "trades_to_arrow_record_batch_bytes")]
341#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
342#[expect(clippy::needless_pass_by_value)]
343pub fn py_trades_to_arrow_record_batch_bytes(
344 py: Python,
345 data: Vec<TradeTick>,
346) -> PyResult<Py<PyBytes>> {
347 match trades_to_arrow_record_batch_bytes(&data) {
348 Ok(batch) => arrow_record_batch_to_pybytes(py, &batch),
349 Err(e) => Err(to_pyvalue_err(e)),
350 }
351}
352
353#[pyfunction(name = "bars_to_arrow_record_batch_bytes")]
362#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
363#[expect(clippy::needless_pass_by_value)]
364pub fn py_bars_to_arrow_record_batch_bytes(py: Python, data: Vec<Bar>) -> PyResult<Py<PyBytes>> {
365 match bars_to_arrow_record_batch_bytes(&data) {
366 Ok(batch) => arrow_record_batch_to_pybytes(py, &batch),
367 Err(e) => Err(to_pyvalue_err(e)),
368 }
369}
370
371#[pyfunction(name = "mark_prices_to_arrow_record_batch_bytes")]
380#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
381#[expect(clippy::needless_pass_by_value)]
382pub fn py_mark_prices_to_arrow_record_batch_bytes(
383 py: Python,
384 data: Vec<MarkPriceUpdate>,
385) -> PyResult<Py<PyBytes>> {
386 match mark_prices_to_arrow_record_batch_bytes(&data) {
387 Ok(batch) => arrow_record_batch_to_pybytes(py, &batch),
388 Err(e) => Err(to_pyvalue_err(e)),
389 }
390}
391
392#[pyfunction(name = "index_prices_to_arrow_record_batch_bytes")]
401#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
402#[expect(clippy::needless_pass_by_value)]
403pub fn py_index_prices_to_arrow_record_batch_bytes(
404 py: Python,
405 data: Vec<IndexPriceUpdate>,
406) -> PyResult<Py<PyBytes>> {
407 match index_prices_to_arrow_record_batch_bytes(&data) {
408 Ok(batch) => arrow_record_batch_to_pybytes(py, &batch),
409 Err(e) => Err(to_pyvalue_err(e)),
410 }
411}
412
413#[pyfunction(name = "instrument_status_to_arrow_record_batch_bytes")]
422#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
423#[expect(clippy::needless_pass_by_value)]
424pub fn py_instrument_status_to_arrow_record_batch_bytes(
425 py: Python,
426 data: Vec<InstrumentStatus>,
427) -> PyResult<Py<PyBytes>> {
428 match instrument_status_to_arrow_record_batch_bytes(&data) {
429 Ok(batch) => arrow_record_batch_to_pybytes(py, &batch),
430 Err(e) => Err(to_pyvalue_err(e)),
431 }
432}
433
434#[pyfunction(name = "option_greeks_to_arrow_record_batch_bytes")]
443#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
444#[expect(clippy::needless_pass_by_value)]
445pub fn py_option_greeks_to_arrow_record_batch_bytes(
446 py: Python,
447 data: Vec<OptionGreeks>,
448) -> PyResult<Py<PyBytes>> {
449 match option_greeks_to_arrow_record_batch_bytes(&data) {
450 Ok(batch) => arrow_record_batch_to_pybytes(py, &batch),
451 Err(e) => Err(to_pyvalue_err(e)),
452 }
453}
454
455#[pyfunction(name = "option_greeks_from_arrow_record_batch_bytes")]
461#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
462pub fn py_option_greeks_from_arrow_record_batch_bytes(
463 _py: Python,
464 data: Vec<u8>,
465) -> PyResult<Vec<OptionGreeks>> {
466 let cursor = Cursor::new(data);
467 let reader = StreamReader::try_new(cursor, None).map_err(to_pyruntime_err)?;
468
469 let mut results = Vec::new();
470
471 for batch_result in reader {
472 let batch = batch_result.map_err(to_pyruntime_err)?;
473 let metadata = batch.schema().metadata().clone();
474 let decoded = OptionGreeks::decode_batch(&metadata, batch).map_err(to_pyvalue_err)?;
475 results.extend(decoded);
476 }
477
478 Ok(results)
479}
480
481#[pyfunction(name = "instrument_status_from_arrow_record_batch_bytes")]
487#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
488pub fn py_instrument_status_from_arrow_record_batch_bytes(
489 _py: Python,
490 data: Vec<u8>,
491) -> PyResult<Vec<InstrumentStatus>> {
492 let cursor = Cursor::new(data);
493 let reader = StreamReader::try_new(cursor, None).map_err(to_pyruntime_err)?;
494
495 let mut results = Vec::new();
496
497 for batch_result in reader {
498 let batch = batch_result.map_err(to_pyruntime_err)?;
499 let metadata = batch.schema().metadata().clone();
500 let decoded =
501 InstrumentStatus::decode_typed_batch(&metadata, batch).map_err(to_pyvalue_err)?;
502 results.extend(decoded);
503 }
504
505 Ok(results)
506}
507
508#[pyfunction(name = "instrument_closes_to_arrow_record_batch_bytes")]
517#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")]
518#[expect(clippy::needless_pass_by_value)]
519pub fn py_instrument_closes_to_arrow_record_batch_bytes(
520 py: Python,
521 data: Vec<InstrumentClose>,
522) -> PyResult<Py<PyBytes>> {
523 match instrument_closes_to_arrow_record_batch_bytes(&data) {
524 Ok(batch) => arrow_record_batch_to_pybytes(py, &batch),
525 Err(e) => Err(to_pyvalue_err(e)),
526 }
527}
528
529#[cfg(test)]
530mod tests {
531 use std::collections::HashMap;
532
533 use nautilus_model::data::stubs::stub_instrument_status;
534 use pyo3::{
535 exceptions::{PyRuntimeError, PyTypeError, PyValueError},
536 types::PyString,
537 };
538 use rstest::rstest;
539
540 use super::*;
541 use crate::arrow::EncodeToRecordBatch;
542
543 #[rstest]
544 fn test_schema_bytes_and_map_match_rust_schema() {
545 Python::initialize();
546 Python::attach(|py| {
547 let cls = py.get_type::<InstrumentStatus>();
548 let bytes = get_arrow_schema_bytes(py, &cls).unwrap();
549 let mut reader = StreamReader::try_new(Cursor::new(bytes.as_bytes(py)), None).unwrap();
550 let map: HashMap<String, String> =
551 get_arrow_schema_map(py, &cls).unwrap().extract(py).unwrap();
552
553 assert_eq!(
554 reader.schema().as_ref(),
555 &InstrumentStatus::get_schema(None)
556 );
557 assert!(reader.next().is_none());
558 assert_eq!(map, InstrumentStatus::get_schema_map());
559 });
560 }
561
562 #[rstest]
563 fn test_schema_rejects_unsupported_class() {
564 Python::initialize();
565 Python::attach(|py| {
566 let cls = py.get_type::<PyString>();
567 let errors = [
568 get_arrow_schema_bytes(py, &cls).unwrap_err(),
569 get_arrow_schema_map(py, &cls).unwrap_err(),
570 ];
571
572 for error in errors {
573 assert!(error.is_instance_of::<PyTypeError>(py));
574 assert_eq!(
575 error.value(py).to_string(),
576 "Arrow schema for `str` is not currently implemented in Rust."
577 );
578 }
579 });
580 }
581
582 #[rstest]
583 fn test_status_ipc_preserves_multiple_batches() {
584 let first = stub_instrument_status();
585 let mut second = first;
586 second.ts_event = 31.into();
587 second.ts_init = 47.into();
588 second.reason = Some("venue halt".into());
589 second.is_trading = Some(false);
590 second.is_quoting = Some(true);
591 let rows = [first, second];
592 let batches = rows
593 .iter()
594 .map(|row| InstrumentStatus::encode_batch(&row.metadata(), &[*row]).unwrap())
595 .collect::<Vec<_>>();
596 Python::initialize();
597 Python::attach(|py| {
598 let bytes =
599 arrow_record_batches_to_pybytes(py, &batches[0].schema(), &batches).unwrap();
600 let decoded =
601 py_instrument_status_from_arrow_record_batch_bytes(py, bytes.as_bytes(py).to_vec())
602 .unwrap();
603 let single =
604 py_instrument_status_to_arrow_record_batch_bytes(py, rows.to_vec()).unwrap();
605 let single_decoded = py_instrument_status_from_arrow_record_batch_bytes(
606 py,
607 single.as_bytes(py).to_vec(),
608 )
609 .unwrap();
610
611 assert_eq!(decoded, rows);
612 assert_eq!(single_decoded, rows);
613 });
614 }
615
616 #[rstest]
617 fn test_ipc_decode_rejects_invalid_stream() {
618 Python::initialize();
619 Python::attach(|py| {
620 let errors = [
621 py_instrument_status_from_arrow_record_batch_bytes(py, vec![1, 2, 3]).unwrap_err(),
622 py_option_greeks_from_arrow_record_batch_bytes(py, vec![1, 2, 3]).unwrap_err(),
623 ];
624
625 for error in errors {
626 assert!(error.is_instance_of::<PyRuntimeError>(py));
627 assert_eq!(
628 error.value(py).to_string(),
629 "Ipc error: Expected schema message, found empty stream."
630 );
631 }
632 });
633 }
634
635 #[rstest]
636 fn test_python_encoders_reject_empty_data() {
637 Python::initialize();
638 Python::attach(|py| {
639 let errors = [
640 py_book_deltas_to_arrow_record_batch_bytes(py, vec![]).unwrap_err(),
641 py_book_depths_to_arrow_record_batch_bytes(py, vec![]).unwrap_err(),
642 py_quotes_to_arrow_record_batch_bytes(py, vec![]).unwrap_err(),
643 py_trades_to_arrow_record_batch_bytes(py, vec![]).unwrap_err(),
644 py_bars_to_arrow_record_batch_bytes(py, vec![]).unwrap_err(),
645 py_mark_prices_to_arrow_record_batch_bytes(py, vec![]).unwrap_err(),
646 py_index_prices_to_arrow_record_batch_bytes(py, vec![]).unwrap_err(),
647 py_instrument_status_to_arrow_record_batch_bytes(py, vec![]).unwrap_err(),
648 py_option_greeks_to_arrow_record_batch_bytes(py, vec![]).unwrap_err(),
649 py_instrument_closes_to_arrow_record_batch_bytes(py, vec![]).unwrap_err(),
650 ];
651
652 for error in errors {
653 assert!(error.is_instance_of::<PyValueError>(py));
654 assert_eq!(
655 error.value(py).to_string(),
656 crate::arrow::EncodingError::EmptyData.to_string()
657 );
658 }
659 });
660 }
661}