1#![expect(
17 clippy::missing_errors_doc,
18 reason = "Feather writer public methods forward encoding and file IO errors directly"
19)]
20
21use std::{
22 any::Any,
23 borrow::Cow,
24 cell::RefCell,
25 collections::{BTreeMap, HashMap, HashSet},
26 fmt::Debug,
27 fs::{self, File, OpenOptions, TryLockError},
28 io::{self, BufReader, BufWriter, Seek, Write},
29 path::{Path, PathBuf},
30 rc::Rc,
31 sync::{
32 Arc,
33 atomic::{AtomicU64, Ordering},
34 mpsc::Sender,
35 },
36};
37
38use ahash::AHashMap;
39use datafusion::arrow::{
40 array::{ArrayRef, DictionaryArray},
41 compute::concat_batches,
42 datatypes::{DataType, Field, Int32Type, Schema, SchemaRef},
43 error::ArrowError,
44 ipc::{reader::StreamReader, writer::StreamWriter},
45 record_batch::RecordBatch,
46};
47use jiff::{
48 SignedDuration,
49 civil::Time,
50 tz::{AmbiguousOffset, TimeZone},
51};
52use nautilus_common::{clock::Clock, live::LiveClock};
53use nautilus_core::{DurationNanos, UnixNanos, time::nanos_since_unix_epoch};
54use nautilus_model::{
55 data::{
56 Bar, CustomData, CustomDataTrait, Data, DataBatch, DataType as CustomDataType,
57 FundingRateUpdate, IndexPriceUpdate, InstrumentStatus, MarkPriceUpdate, NautilusDataType,
58 OptionGreeks, OrderBookDelta, OrderBookDeltas, OrderBookDepth, QuoteTick, TradeTick,
59 close::InstrumentClose,
60 encode_custom_to_arrow, get_arrow_schema,
61 greeks::{GreeksData, PortfolioGreeks},
62 },
63 events::{
64 AccountState, OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled,
65 OrderDenied, OrderEmulated, OrderEventAny, OrderExpired, OrderFillVoided, OrderFilled,
66 OrderInitialized, OrderModifyDenied, OrderModifyRejected, OrderPendingCancel,
67 OrderPendingUpdate, OrderRejected, OrderReleased, OrderSnapshot, OrderSubmitted,
68 OrderTriggered, OrderUpdated, PositionAdjusted, PositionChanged, PositionClosed,
69 PositionEvent, PositionOpened, PositionSnapshot,
70 },
71 instruments::{InstrumentAny, NautilusInstrumentType},
72 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
73};
74use nautilus_serialization::{
75 arrow::{
76 EncodeToRecordBatch, KEY_TYPE_NAME, catalog_identifier_from_metadata,
77 record_batch_with_identifier_column, schema_with_identifier_column,
78 },
79 ensure_custom_data_registered,
80};
81
82use crate::{
83 catalog::{
84 session::DEFAULT_DATA_BATCH_CHUNK_SIZE,
85 types::{
86 CatalogDataType, CatalogFamily, data_path_prefix, instrument_path_prefix,
87 record_path_prefix,
88 },
89 },
90 common::{
91 custom::{
92 augment_batch_with_data_type_column, schema_with_data_type_column,
93 validate_custom_catalog_schema,
94 },
95 metadata::record_batch_ts_init_range,
96 },
97 writer::{
98 filter::{WriterRecordFilter, catalog_family},
99 subscription::StreamingSinkSubscription,
100 traits::{StreamingDataSink, StreamingSink},
101 },
102};
103
104pub(crate) type FeatherWriteCommand =
105 Box<dyn FnOnce(&mut FeatherWriter) -> Result<(), Box<dyn std::error::Error>> + Send + 'static>;
106
107#[expect(
108 clippy::needless_pass_by_value,
109 reason = "map_err transfers ownership of the boxed writer error"
110)]
111pub(crate) fn feather_error(e: Box<dyn std::error::Error>) -> anyhow::Error {
112 anyhow::anyhow!("{e}")
113}
114
115macro_rules! define_builtin_data_batch_dispatch {
116 ($(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?) => {
117 fn write_builtin_data_batch(
118 writer: &mut FeatherWriter,
119 batch: &DataBatch,
120 ) -> Option<Result<(), Box<dyn std::error::Error>>> {
121 match batch {
122 $(
123 DataBatch::$batch(data) => Some(writer.write_batch(data.as_ref().to_vec())),
124 )+
125 _ => None,
126 }
127 }
128 };
129}
130
131nautilus_model::for_each_data_type!(define_builtin_data_batch_dispatch);
132
133pub(crate) const NAUTILUS_ARROW_METADATA_ID_COLUMN: &str = "nautilus_metadata_id";
134pub(crate) const NAUTILUS_ARROW_METADATA_JSON_COLUMN: &str = "nautilus_metadata_json";
135
136pub(crate) fn staged_metadata_data_type() -> DataType {
138 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
139}
140
141pub(crate) fn staged_metadata_array<'a>(values: impl IntoIterator<Item = &'a str>) -> ArrayRef {
142 Arc::new(values.into_iter().collect::<DictionaryArray<Int32Type>>())
143}
144
145#[derive(Clone, Debug, Eq, PartialEq)]
146struct StagedArrowMetadataRow {
147 metadata_id: String,
148 metadata_json: String,
149}
150
151#[derive(Debug, Clone, PartialEq, Eq, Hash)]
153pub struct FileWriterPath {
154 path: PathBuf,
155 data_type: CatalogDataType,
156}
157
158#[derive(Clone, Debug)]
164pub enum WriterClock {
165 Live,
167 Test(Arc<AtomicU64>),
169}
170
171impl WriterClock {
172 #[must_use]
174 pub fn timestamp_ns(&self) -> UnixNanos {
175 match self {
176 Self::Live => UnixNanos::from(nanos_since_unix_epoch()),
177 Self::Test(shared) => UnixNanos::from(shared.load(Ordering::Relaxed)),
178 }
179 }
180
181 #[must_use]
188 pub fn from_shared_clock(clock: &Rc<RefCell<dyn Clock>>) -> (Self, Option<Arc<AtomicU64>>) {
189 let borrowed = clock.borrow();
190 let any_ref: &dyn Any = &*borrowed;
191 if any_ref.downcast_ref::<LiveClock>().is_some() {
192 (Self::Live, None)
193 } else {
194 let shared = Arc::new(AtomicU64::new(borrowed.timestamp_ns().as_u64()));
195 (Self::Test(Arc::clone(&shared)), Some(shared))
196 }
197 }
198}
199
200fn record_batch_with_delta_staged_metadata(
201 batch: &RecordBatch,
202 metadata_rows: &[StagedArrowMetadataRow],
203) -> anyhow::Result<RecordBatch> {
204 anyhow::ensure!(
205 batch.num_rows() == metadata_rows.len(),
206 "Delta Feather staging metadata row count {} does not match batch row count {}",
207 metadata_rows.len(),
208 batch.num_rows()
209 );
210 anyhow::ensure!(
211 batch
212 .schema()
213 .index_of(NAUTILUS_ARROW_METADATA_ID_COLUMN)
214 .is_err(),
215 "Delta Feather staging schema already has {NAUTILUS_ARROW_METADATA_ID_COLUMN}"
216 );
217 anyhow::ensure!(
218 batch
219 .schema()
220 .index_of(NAUTILUS_ARROW_METADATA_JSON_COLUMN)
221 .is_err(),
222 "Delta Feather staging schema already has {NAUTILUS_ARROW_METADATA_JSON_COLUMN}"
223 );
224
225 let mut fields = batch
226 .schema()
227 .fields()
228 .iter()
229 .map(|field| {
230 Arc::new(Field::new(
231 field.name().clone(),
232 field.data_type().clone(),
233 field.is_nullable(),
234 ))
235 })
236 .collect::<Vec<_>>();
237
238 fields.push(Arc::new(Field::new(
239 NAUTILUS_ARROW_METADATA_ID_COLUMN,
240 staged_metadata_data_type(),
241 false,
242 )));
243 fields.push(Arc::new(Field::new(
244 NAUTILUS_ARROW_METADATA_JSON_COLUMN,
245 staged_metadata_data_type(),
246 false,
247 )));
248
249 let mut columns = batch.columns().to_vec();
250 columns.push(staged_metadata_array(
251 metadata_rows.iter().map(|row| row.metadata_id.as_str()),
252 ));
253 columns.push(staged_metadata_array(
254 metadata_rows.iter().map(|row| row.metadata_json.as_str()),
255 ));
256
257 Ok(RecordBatch::try_new(
258 Arc::new(Schema::new(fields)),
259 columns,
260 )?)
261}
262
263fn arrow_metadata_row(
264 metadata: &HashMap<String, String>,
265 fields: &arrow::datatypes::Fields,
266) -> anyhow::Result<StagedArrowMetadataRow> {
267 let schema_metadata = metadata.iter().collect::<BTreeMap<_, _>>();
268
269 let field_metadata = fields
270 .iter()
271 .filter(|field| !field.metadata().is_empty())
272 .map(|field| {
273 (
274 field.name().clone(),
275 field
276 .metadata()
277 .iter()
278 .map(|(key, value)| (key.clone(), value.clone()))
279 .collect::<BTreeMap<_, _>>(),
280 )
281 })
282 .collect::<BTreeMap<_, _>>();
283
284 let metadata_json = serde_json::to_string(&serde_json::json!({
285 "format_version": 1,
286 "schema_metadata": schema_metadata,
287 "field_metadata": field_metadata,
288 }))?;
289 let metadata_id = staged_metadata_id(&serde_json::to_string(&schema_metadata)?);
290
291 Ok(StagedArrowMetadataRow {
292 metadata_id,
293 metadata_json,
294 })
295}
296
297pub(crate) fn canonical_metadata_json(
302 metadata: &HashMap<String, String>,
303) -> anyhow::Result<String> {
304 Ok(serde_json::to_string(
305 &metadata
306 .iter()
307 .map(|(key, value)| (key.clone(), value.clone()))
308 .collect::<BTreeMap<_, _>>(),
309 )?)
310}
311
312pub(crate) fn staged_metadata_id(metadata_json: &str) -> String {
313 format!("blake3:{}", blake3::hash(metadata_json.as_bytes()).to_hex())
314}
315
316pub(crate) const FEATHER_EXTENSION: &str = "feather";
318
319pub(crate) const FEATHER_PARTIAL_EXTENSION: &str = "feather.partial";
321
322struct FeatherFile {
332 writer: StreamWriter<CountingWriter<BufWriter<File>>>,
333 partial_path: PathBuf,
334 path: PathBuf,
335 schema: SchemaRef,
336 max_size: Option<u64>,
337 buffer: Vec<RecordBatch>,
338 buffer_rows: usize,
339 buffer_bytes: u64,
340 last_ts_init: Option<u64>,
342 size_rotation_due: bool,
344}
345
346impl FeatherFile {
347 fn create(
348 path: &Path,
349 schema: &Schema,
350 rotation_config: &RotationConfig,
351 ) -> Result<Self, Box<dyn std::error::Error>> {
352 let partial_path = path.with_extension(FEATHER_PARTIAL_EXTENSION);
353 if let Some(parent) = path.parent() {
354 fs::create_dir_all(parent)?;
355 }
356
357 let file = OpenOptions::new()
358 .write(true)
359 .create_new(true)
360 .open(&partial_path)?;
361
362 file.lock().map_err(|e| {
364 format!(
365 "Failed to lock Feather file {}: {e}",
366 partial_path.display()
367 )
368 })?;
369
370 let writer = StreamWriter::try_new(CountingWriter::new(BufWriter::new(file)), schema)?;
371
372 let max_size = match rotation_config {
373 RotationConfig::Size { max_size } => Some(*max_size),
374 _ => None,
375 };
376
377 Ok(Self {
378 writer,
379 partial_path,
380 path: path.to_path_buf(),
381 schema: Arc::new(schema.clone()),
382 max_size,
383 buffer: Vec::new(),
384 buffer_rows: 0,
385 buffer_bytes: 0,
386 last_ts_init: None,
387 size_rotation_due: false,
388 })
389 }
390
391 fn write_record_batch(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
393 let batch = if batch.schema() == self.schema {
394 batch.clone()
395 } else {
396 RecordBatch::try_new(self.schema.clone(), batch.columns().to_vec())?
397 };
398
399 self.buffer_rows += batch.num_rows();
400 self.buffer_bytes += batch.get_array_memory_size() as u64;
401 self.buffer.push(batch);
402
403 if self.buffer_rows >= DEFAULT_DATA_BATCH_CHUNK_SIZE {
404 self.write_buffer()?;
405 }
406
407 let Some(max_size) = self.max_size else {
408 return Ok(());
409 };
410
411 if self.size() + self.buffer_bytes >= max_size {
413 self.write_buffer()?;
414 }
415
416 self.size_rotation_due = self.size() >= max_size;
417 Ok(())
418 }
419
420 fn write_buffer(&mut self) -> Result<(), ArrowError> {
421 if self.buffer.is_empty() {
422 return Ok(());
423 }
424
425 let batch = concat_batches(&self.schema, &self.buffer)?;
426 self.buffer.clear();
427 self.buffer_rows = 0;
428 self.buffer_bytes = 0;
429 self.writer.write(&batch)
430 }
431
432 fn size(&self) -> u64 {
433 self.writer.get_ref().bytes
434 }
435
436 fn flush(&mut self) -> Result<(), ArrowError> {
437 self.write_buffer()?;
438 self.writer.flush()
439 }
440
441 fn seal(mut self) -> Result<(), Box<dyn std::error::Error>> {
442 self.write_buffer()?;
443 self.writer.finish()?;
444 let file = self
445 .writer
446 .into_inner()?
447 .inner
448 .into_inner()
449 .map_err(io::IntoInnerError::into_error)?;
450 file.sync_all()?;
451
452 fs::rename(&self.partial_path, &self.path)?;
454 Ok(())
455 }
456}
457
458struct CountingWriter<W> {
460 inner: W,
461 bytes: u64,
462}
463
464impl<W> CountingWriter<W> {
465 const fn new(inner: W) -> Self {
466 Self { inner, bytes: 0 }
467 }
468}
469
470impl<W: Write> Write for CountingWriter<W> {
471 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
472 let written = self.inner.write(buf)?;
473 self.bytes += written as u64;
474 Ok(written)
475 }
476
477 fn flush(&mut self) -> io::Result<()> {
478 self.inner.flush()
479 }
480}
481
482#[derive(Debug, Clone)]
493pub enum RotationConfig {
494 Size {
496 max_size: u64,
498 },
499 Interval {
501 interval_ns: u64,
503 },
504 ScheduledDates {
506 interval_ns: u64,
508 rotation_time: UnixNanos,
510 rotation_timezone: TimeZone,
512 },
513 NoRotation,
515}
516
517pub struct FeatherWriter {
539 directory: PathBuf,
540 clock: WriterClock,
541 rotation_config: RotationConfig,
542 included_types: Option<HashSet<CatalogDataType>>,
543 record_filter: Option<WriterRecordFilter>,
544 writers: HashMap<FileWriterPath, FeatherFile>,
545 path_sequences: HashMap<CatalogDataType, (UnixNanos, u64)>,
546 sealed_tx: Option<Sender<PathBuf>>,
547 next_rotation_times: HashMap<FileWriterPath, UnixNanos>,
548 flush_interval_ms: u64,
549 last_flush_ns: UnixNanos,
550 pending_write_error: Option<String>,
551}
552
553impl FeatherWriter {
554 #[must_use]
556 pub fn new(
557 directory: PathBuf,
558 clock: WriterClock,
559 rotation_config: RotationConfig,
560 included_types: Option<HashSet<CatalogDataType>>,
561 flush_interval_ms: Option<u64>,
562 ) -> Self {
563 let flush_interval_ms = flush_interval_ms.unwrap_or(1000); let last_flush_ns = clock.timestamp_ns();
565
566 Self {
567 directory,
568 clock,
569 rotation_config,
570 included_types,
571 record_filter: None,
572 writers: HashMap::new(),
573 path_sequences: HashMap::new(),
574 sealed_tx: None,
575 next_rotation_times: HashMap::new(),
576 flush_interval_ms,
577 last_flush_ns,
578 pending_write_error: None,
579 }
580 }
581
582 #[must_use]
584 pub fn with_record_filter(mut self, record_filter: Option<WriterRecordFilter>) -> Self {
585 self.record_filter = record_filter;
586 self
587 }
588
589 pub(crate) fn with_sealed_sender(mut self, sealed_tx: Sender<PathBuf>) -> Self {
590 self.sealed_tx = Some(sealed_tx);
591 self
592 }
593
594 pub fn write<T>(&mut self, data: T) -> Result<(), Box<dyn std::error::Error>>
599 where
600 T: EncodeToRecordBatch + CatalogFamily + 'static,
601 {
602 let metadata = T::metadata(&data);
603 let identifier = catalog_identifier_from_metadata(&metadata);
604 let data_type = Self::staged_data_type::<T>(&metadata)?;
605
606 if !self.should_write_record(&data_type, identifier.as_deref()) {
607 return Ok(());
608 }
609
610 let path = self.get_writer_path(data_type);
611
612 if !self.writers.contains_key(&path) {
614 self.create_writer::<T>(path.clone(), &data)?;
615 }
616
617 let batch = T::encode_batch(&metadata, &[data])?;
619 let metadata_row = arrow_metadata_row(&metadata, batch.schema().fields())?;
620 let batch = record_batch_with_identifier_column(batch, identifier.as_deref())?;
621 let batch =
622 record_batch_with_delta_staged_metadata(&batch, std::slice::from_ref(&metadata_row))?;
623
624 self.stage_batch_writes(path, std::slice::from_ref(&batch))?;
626 self.flush_if_due()
627 }
628
629 pub fn write_batch<T>(&mut self, data: Vec<T>) -> Result<(), Box<dyn std::error::Error>>
642 where
643 T: EncodeToRecordBatch + CatalogFamily + 'static,
644 {
645 if data.is_empty() || !self.should_write_family(&T::catalog_family()) {
646 return Ok(());
647 }
648
649 let mut groups: AHashMap<Option<String>, (CatalogDataType, Vec<T>)> = AHashMap::new();
650
651 for item in data {
652 let metadata = T::metadata(&item);
653 let identifier = catalog_identifier_from_metadata(&metadata);
654 let data_type = Self::staged_data_type::<T>(&metadata)?;
655
656 if !self.should_write_record(&data_type, identifier.as_deref()) {
657 continue;
658 }
659
660 groups
661 .entry(identifier)
662 .or_insert_with(|| (data_type, Vec::new()))
663 .1
664 .push(item);
665 }
666
667 if groups.is_empty() {
668 return Ok(());
669 }
670
671 let mut staged: Vec<(FileWriterPath, Vec<RecordBatch>)> = Vec::new();
672
673 for (data_type, group) in groups.into_values() {
674 let path = self.get_writer_path(data_type);
675 let metadata = T::chunk_metadata(&group);
676
677 if !self.writers.contains_key(&path) {
678 self.create_writer_with_metadata::<T>(path.clone(), metadata.clone())?;
679 }
680
681 let identifier = catalog_identifier_from_metadata(&metadata);
682 let batch = T::encode_batch(&metadata, &group)?;
683 let metadata_rows = group
684 .iter()
685 .map(T::metadata)
686 .map(|metadata| arrow_metadata_row(&metadata, batch.schema().fields()))
687 .collect::<anyhow::Result<Vec<_>>>()?;
688 let batch = record_batch_with_identifier_column(batch, identifier.as_deref())?;
689 let batch = record_batch_with_delta_staged_metadata(&batch, &metadata_rows)?;
690
691 match staged.iter_mut().find(|(staged, _)| *staged == path) {
692 Some((_, batches)) => batches.push(batch),
693 None => staged.push((path, vec![batch])),
694 }
695 }
696
697 for (path, batches) in staged {
698 self.stage_batch_writes(path, &batches)?;
699 }
700
701 self.flush_if_due()
702 }
703
704 fn staged_data_type<T: CatalogFamily>(
706 metadata: &HashMap<String, String>,
707 ) -> Result<CatalogDataType, Box<dyn std::error::Error>> {
708 let family = T::catalog_family();
709 if family != CatalogDataType::Data(NautilusDataType::Instrument) {
710 return Ok(family);
711 }
712
713 let class = metadata
714 .get(KEY_TYPE_NAME)
715 .ok_or("Instrument metadata has no type_name")?
716 .parse::<NautilusInstrumentType>()?;
717 Ok(CatalogDataType::Instrument(class))
718 }
719
720 fn stage_batch_writes(
724 &mut self,
725 path: FileWriterPath,
726 batches: &[RecordBatch],
727 ) -> Result<(), Box<dyn std::error::Error>> {
728 if !self.writers.contains_key(&path) {
729 return Ok(());
730 }
731
732 let (first_ts, last_ts) = record_batch_ts_init_range(batches)?;
733 let path = self.rotated_for(path, first_ts)?;
734 let Some(file) = self.writers.get_mut(&path) else {
735 return Ok(());
736 };
737
738 for batch in batches {
739 if let Err(e) = file.write_record_batch(batch) {
740 self.abandon_file(&path);
741 return Err(e.into());
742 }
743 }
744
745 file.last_ts_init = Some(file.last_ts_init.map_or(last_ts, |last| last.max(last_ts)));
746 Ok(())
747 }
748
749 fn rotated_for(
759 &mut self,
760 path: FileWriterPath,
761 ts_init: u64,
762 ) -> Result<FileWriterPath, Box<dyn std::error::Error>> {
763 let scheduled = self.scheduled_rotation_due(&path, ts_init);
764 let Some(file) = self.writers.get(&path) else {
765 return Ok(path);
766 };
767
768 if !(scheduled || file.size_rotation_due)
769 || file.last_ts_init.is_some_and(|last| ts_init <= last)
770 {
771 return Ok(path);
772 }
773
774 let schema = file.schema.clone();
775 let data_type = path.data_type.clone();
776 self.seal_file(&path)?;
777
778 let path = self.reserve_writer_path(data_type);
779 let file = FeatherFile::create(&path.path, &schema, &self.rotation_config)?;
780 self.writers.insert(path.clone(), file);
781 self.scheduled_rotation_due(&path, ts_init);
783 Ok(path)
784 }
785
786 fn flush_is_due(&self) -> bool {
788 if self.flush_interval_ms == 0 {
789 return false; }
791
792 let now_ns = self.clock.timestamp_ns();
793 let elapsed_ms = now_ns.as_u64().saturating_sub(self.last_flush_ns.as_u64()) / 1_000_000;
794 elapsed_ms >= self.flush_interval_ms
795 }
796
797 fn flush_if_due(&mut self) -> Result<(), Box<dyn std::error::Error>> {
799 if self.flush_is_due() {
800 self.flush()?;
801 }
802
803 Ok(())
804 }
805
806 fn scheduled_rotation_due(&mut self, path: &FileWriterPath, ts_init: u64) -> bool {
810 if let Some(next) = self.next_rotation_times.get(path) {
811 return ts_init >= next.as_u64();
812 }
813
814 let next = match &self.rotation_config {
815 RotationConfig::Interval { interval_ns } => {
816 UnixNanos::from(ts_init) + DurationNanos::new(*interval_ns)
817 }
818 RotationConfig::ScheduledDates {
819 interval_ns,
820 rotation_time,
821 rotation_timezone,
822 } => Self::next_scheduled_rotation(
823 UnixNanos::from(ts_init),
824 *rotation_time,
825 rotation_timezone,
826 *interval_ns,
827 ),
828 _ => return false,
829 };
830 self.next_rotation_times.insert(path.clone(), next);
831 false
832 }
833
834 fn next_scheduled_rotation(
837 after: UnixNanos,
838 rotation_time: UnixNanos,
839 rotation_timezone: &TimeZone,
840 interval_ns: u64,
841 ) -> UnixNanos {
842 let now_utc = after.to_datetime_utc();
843 let now_local = rotation_timezone.to_datetime(now_utc);
844
845 let rotation_time_secs = u32::try_from(*rotation_time / 1_000_000_000).unwrap_or(0);
846 let rotation_time_nanos = i32::try_from(*rotation_time % 1_000_000_000).unwrap_or(0);
847
848 let rotation_time = if rotation_time_secs < 86_400 {
849 Time::new(
850 i8::try_from(rotation_time_secs / 3_600).unwrap_or(0),
851 i8::try_from(rotation_time_secs % 3_600 / 60).unwrap_or(0),
852 i8::try_from(rotation_time_secs % 60).unwrap_or(0),
853 rotation_time_nanos,
854 )
855 .unwrap_or(Time::MIN)
856 } else {
857 Time::MIN
858 };
859
860 let local_rotation = now_local.date().to_datetime(rotation_time);
861 let ambiguous = rotation_timezone.to_ambiguous_timestamp(local_rotation);
862
863 let mut next_rotation = match ambiguous.offset() {
864 AmbiguousOffset::Gap { .. } => now_utc,
865 _ => ambiguous.earlier().unwrap_or(now_utc),
866 };
867
868 if next_rotation <= now_utc {
869 while next_rotation <= now_utc {
872 next_rotation += SignedDuration::from_nanos_i128(i128::from(interval_ns));
873 }
874 }
875
876 UnixNanos::from(u64::try_from(next_rotation.as_nanosecond()).unwrap_or(0))
877 }
878
879 fn seal_file(&mut self, path: &FileWriterPath) -> Result<(), Box<dyn std::error::Error>> {
882 self.next_rotation_times.remove(path);
883
884 let Some(file) = self.writers.remove(path) else {
885 return Ok(());
886 };
887
888 let partial_path = file.partial_path.clone();
889 let result = file
890 .seal()
891 .inspect_err(|_| recover_released_feather_file(&partial_path));
892 self.notify_sealed(path);
893 result
894 }
895
896 fn abandon_file(&mut self, path: &FileWriterPath) {
899 self.next_rotation_times.remove(path);
900
901 let Some(file) = self.writers.remove(path) else {
902 return;
903 };
904
905 let partial_path = file.partial_path.clone();
906 drop(file);
907
908 log::error!(
909 "Abandoned Feather file {} after a failed write",
910 path.path.display()
911 );
912
913 recover_released_feather_file(&partial_path);
914 self.notify_sealed(path);
915 }
916
917 fn notify_sealed(&self, path: &FileWriterPath) {
918 if let Some(sealed_tx) = &self.sealed_tx
919 && path.path.exists()
920 {
921 let _ = sealed_tx.send(path.path.clone());
922 }
923 }
924
925 fn create_writer<T>(
927 &mut self,
928 path: FileWriterPath,
929 data: &T,
930 ) -> Result<(), Box<dyn std::error::Error>>
931 where
932 T: EncodeToRecordBatch + CatalogFamily + 'static,
933 {
934 self.create_writer_with_metadata::<T>(path, T::metadata(data))
935 }
936
937 fn create_writer_with_metadata<T>(
942 &mut self,
943 path: FileWriterPath,
944 metadata: HashMap<String, String>,
945 ) -> Result<(), Box<dyn std::error::Error>>
946 where
947 T: EncodeToRecordBatch + CatalogFamily + 'static,
948 {
949 let schema = Self::schema_with_delta_staging_columns(&schema_with_identifier_column(
950 &T::get_schema(Some(metadata)),
951 ));
952
953 let file = FeatherFile::create(&path.path, &schema, &self.rotation_config)?;
954 self.writers.insert(path, file);
955 Ok(())
956 }
957
958 fn create_custom_writer(
960 &mut self,
961 path: FileWriterPath,
962 type_name: &str,
963 ) -> Result<(), Box<dyn std::error::Error>> {
964 if self.writers.contains_key(&path) {
965 return Ok(());
966 }
967
968 let base_schema = get_arrow_schema(type_name).ok_or_else(|| {
969 format!("Custom data type \"{type_name}\" is not registered for Arrow encoding")
970 })?;
971
972 let schema = schema_with_data_type_column(base_schema.as_ref(), type_name);
973
974 let schema =
975 Self::schema_with_delta_staging_columns(&schema_with_identifier_column(&schema));
976
977 let file = FeatherFile::create(&path.path, &schema, &self.rotation_config)
978 .map_err(|e| format!("Failed to create Feather file for custom {type_name}: {e}"))?;
979 self.writers.insert(path, file);
980 Ok(())
981 }
982
983 pub(crate) fn encode_custom_to_batch(
985 custom: &CustomData,
986 ) -> Result<RecordBatch, Box<dyn std::error::Error>> {
987 let type_name = custom.data.type_name();
988 let data_type_json = custom
989 .data_type
990 .to_persistence_json()
991 .map_err(|e| format!("Failed to serialize data_type for persistence: {e}"))?;
992 let dt_meta = custom.data_type.metadata_string_map();
993 let items: [Arc<dyn CustomDataTrait>; 1] = [Arc::clone(&custom.data)];
994
995 let batch = encode_custom_to_arrow(type_name, &items)
996 .map_err(|e| format!("Failed to encode custom data: {e}"))?
997 .ok_or_else(|| {
998 format!("Custom data type \"{type_name}\" is not registered for Arrow")
999 })?;
1000
1001 let batch = augment_batch_with_data_type_column(
1002 &batch,
1003 &data_type_json,
1004 type_name,
1005 dt_meta.as_ref(),
1006 )
1007 .map_err(|e| e.to_string())?;
1008 Ok(batch)
1009 }
1010
1011 fn schema_with_delta_staging_columns(schema: &Schema) -> Schema {
1012 let mut fields = schema
1013 .fields()
1014 .iter()
1015 .map(|field| {
1016 Arc::new(Field::new(
1017 field.name().clone(),
1018 field.data_type().clone(),
1019 field.is_nullable(),
1020 ))
1021 })
1022 .collect::<Vec<_>>();
1023
1024 if schema.index_of(NAUTILUS_ARROW_METADATA_ID_COLUMN).is_err() {
1025 fields.push(Arc::new(Field::new(
1026 NAUTILUS_ARROW_METADATA_ID_COLUMN,
1027 staged_metadata_data_type(),
1028 false,
1029 )));
1030 }
1031
1032 if schema
1033 .index_of(NAUTILUS_ARROW_METADATA_JSON_COLUMN)
1034 .is_err()
1035 {
1036 fields.push(Arc::new(Field::new(
1037 NAUTILUS_ARROW_METADATA_JSON_COLUMN,
1038 staged_metadata_data_type(),
1039 false,
1040 )));
1041 }
1042
1043 Schema::new(fields)
1044 }
1045
1046 pub fn flush(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1051 let paths = self.writers.keys().cloned().collect::<Vec<_>>();
1052
1053 for path in paths {
1054 if let Some(file) = self.writers.get_mut(&path)
1055 && let Err(e) = file.flush()
1056 {
1057 self.abandon_file(&path);
1058 return Err(e.into());
1059 }
1060 }
1061
1062 self.last_flush_ns = self.clock.timestamp_ns();
1063
1064 if let Some(error) = self.pending_write_error.take() {
1065 return Err(error.into());
1066 }
1067
1068 Ok(())
1069 }
1070
1071 pub fn seal(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1075 let paths = self.writers.keys().cloned().collect::<Vec<_>>();
1076 let mut first_error = None;
1077
1078 for path in paths {
1079 if let Err(e) = self.seal_file(&path) {
1080 log::error!("Failed to seal Feather file {}: {e}", path.path.display());
1081 first_error.get_or_insert(e);
1082 }
1083 }
1084
1085 match first_error {
1086 Some(e) => Err(e),
1087 None => Ok(()),
1088 }
1089 }
1090
1091 pub fn close(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1095 self.seal()?;
1096
1097 if let Some(error) = self.pending_write_error.take() {
1098 return Err(error.into());
1099 }
1100
1101 Ok(())
1102 }
1103
1104 #[must_use]
1106 pub fn is_closed(&self) -> bool {
1107 self.writers.is_empty()
1108 }
1109
1110 #[must_use]
1112 pub fn get_current_file_info(&self) -> HashMap<CatalogDataType, (u64, String)> {
1113 self.writers
1114 .iter()
1115 .map(|(path, file)| {
1116 (
1117 path.data_type.clone(),
1118 (file.size(), path.path.display().to_string()),
1119 )
1120 })
1121 .collect()
1122 }
1123
1124 #[must_use]
1128 pub fn get_next_rotation_time(&self, data_type: &CatalogDataType) -> Option<UnixNanos> {
1129 self.next_rotation_times
1130 .iter()
1131 .find(|(path, _)| path.data_type == *data_type)
1132 .map(|(_, &next)| next)
1133 }
1134
1135 fn should_write_family(&self, family: &CatalogDataType) -> bool {
1137 self.included_types
1138 .as_ref()
1139 .is_none_or(|included| included.contains(family))
1140 && self
1141 .record_filter
1142 .as_ref()
1143 .is_none_or(|filter| filter.contains(family))
1144 }
1145
1146 fn should_write_record(&self, data_type: &CatalogDataType, identifier: Option<&str>) -> bool {
1147 self.included_types
1148 .as_ref()
1149 .is_none_or(|included| included.contains(&catalog_family(data_type)))
1150 && self
1151 .record_filter
1152 .as_ref()
1153 .is_none_or(|filter| filter.allows(data_type, identifier))
1154 }
1155
1156 fn reserve_writer_path(&mut self, data_type: CatalogDataType) -> FileWriterPath {
1157 let now = self.clock.timestamp_ns();
1158 let (timestamp, first_sequence) = self
1159 .path_sequences
1160 .get(&data_type)
1161 .copied()
1162 .filter(|(timestamp, _)| *timestamp >= now)
1163 .unwrap_or((now, 0));
1164
1165 for sequence in first_sequence.. {
1166 let path = self.build_writer_path(&data_type, timestamp, sequence);
1167
1168 if path.exists() || path.with_extension(FEATHER_PARTIAL_EXTENSION).exists() {
1170 continue;
1171 }
1172
1173 self.path_sequences
1174 .insert(data_type.clone(), (timestamp, sequence + 1));
1175 return FileWriterPath { path, data_type };
1176 }
1177
1178 unreachable!("unbounded writer path sequence exhausted")
1179 }
1180
1181 fn build_writer_path(
1182 &self,
1183 data_type: &CatalogDataType,
1184 timestamp: UnixNanos,
1185 sequence: u64,
1186 ) -> PathBuf {
1187 let (directory, file_stem) = match data_type {
1188 CatalogDataType::Data(NautilusDataType::Custom { type_name }) => (
1189 self.directory.join("data").join("custom").join(type_name),
1190 Cow::Borrowed(type_name.as_str()),
1191 ),
1192 CatalogDataType::Data(data_type) => {
1193 let prefix = data_path_prefix(data_type);
1194 (self.directory.join(prefix.as_ref()), prefix)
1195 }
1196 CatalogDataType::Record(record_type) => {
1197 let prefix = record_path_prefix(record_type);
1198 (self.directory.join(prefix.as_ref()), prefix)
1199 }
1200 CatalogDataType::Instrument(class) => {
1201 let prefix = data_path_prefix(&NautilusDataType::Instrument);
1202 (
1203 self.directory
1204 .join(prefix.as_ref())
1205 .join(instrument_path_prefix(class)),
1206 prefix,
1207 )
1208 }
1209 };
1210
1211 directory.join(Self::timestamped_feather_file_name(
1212 &file_stem, timestamp, sequence,
1213 ))
1214 }
1215
1216 fn timestamped_feather_file_name(stem: &str, timestamp: UnixNanos, sequence: u64) -> String {
1217 if sequence == 0 {
1218 format!("{stem}_{timestamp}.{FEATHER_EXTENSION}")
1219 } else {
1220 format!("{stem}_{timestamp}-{sequence}.{FEATHER_EXTENSION}")
1221 }
1222 }
1223
1224 fn get_writer_path_custom(
1226 &mut self,
1227 type_name: &str,
1228 ) -> Result<FileWriterPath, Box<dyn std::error::Error>> {
1229 let data_type = CatalogDataType::from(NautilusDataType::Custom {
1230 type_name: type_name.to_string(),
1231 });
1232
1233 if let Some(existing) = self.writers.keys().find(|path| path.data_type == data_type) {
1234 return Ok(existing.clone());
1235 }
1236
1237 if let Some(schema) = get_arrow_schema(type_name) {
1238 validate_custom_catalog_schema(type_name, &schema)?;
1239 }
1240
1241 Ok(self.reserve_writer_path(data_type))
1242 }
1243
1244 fn get_writer_path(&mut self, data_type: CatalogDataType) -> FileWriterPath {
1246 if let Some(existing) = self.writers.keys().find(|path| path.data_type == data_type) {
1247 return existing.clone();
1248 }
1249
1250 self.reserve_writer_path(data_type)
1251 }
1252
1253 pub fn write_data(&mut self, data: Data) -> Result<(), Box<dyn std::error::Error>> {
1258 match data {
1259 Data::Instrument(instrument) => self.write(*instrument),
1260 Data::Quote(quote) => self.write(quote),
1261 Data::Trade(trade) => self.write(trade),
1262 Data::Bar(bar) => self.write(bar),
1263 Data::BookDelta(delta) => self.write(delta),
1264 Data::BookDepth(depth) => self.write(*depth),
1265 Data::IndexPrice(price) => self.write(price),
1266 Data::MarkPrice(price) => self.write(price),
1267 Data::FundingRate(funding) => self.write(funding),
1268 Data::InstrumentStatus(status) => self.write(status),
1269 Data::OptionGreeks(greeks) => self.write(greeks),
1270 Data::InstrumentClose(close) => self.write(close),
1271 Data::Custom(custom) => self.write_custom_data(&custom),
1272 Data::BookDeltas(deltas_api) => {
1273 self.write_batch(deltas_api.deltas.clone())
1275 }
1276 #[cfg(feature = "defi")]
1277 Data::Defi(_) => Err("Unsupported DeFi data variant for feather writes".into()),
1278 #[cfg(not(feature = "defi"))]
1279 #[allow(
1280 unreachable_patterns,
1281 reason = "DeFi variants can exist without this crate's defi feature"
1282 )]
1283 _ => Err("Unsupported DeFi data variant for feather writes".into()),
1284 }
1285 }
1286
1287 #[expect(
1293 clippy::needless_pass_by_value,
1294 reason = "the public writer API accepts ownership of each submitted data batch"
1295 )]
1296 pub fn write_data_batch(&mut self, data: Vec<Data>) -> Result<(), Box<dyn std::error::Error>> {
1297 for batch in DataBatch::from_data_vec_grouped(&data)? {
1298 match &batch {
1299 DataBatch::Custom(data) => {
1300 for custom in data.as_ref() {
1301 self.write_custom_data(custom)?;
1302 }
1303 }
1304 batch => write_builtin_data_batch(self, batch)
1305 .expect("built-in data batch dispatch is exhaustive")?,
1306 }
1307 }
1308
1309 Ok(())
1310 }
1311
1312 pub fn write_any_message(
1314 &mut self,
1315 message: &dyn Any,
1316 ) -> Result<bool, Box<dyn std::error::Error>> {
1317 let Some(command) = Self::write_command(message) else {
1318 return Ok(false);
1319 };
1320
1321 if let Err(e) = command(self) {
1322 self.record_write_error(e.to_string());
1323 return Err(e);
1324 }
1325
1326 Ok(true)
1327 }
1328
1329 pub(crate) fn record_write_error(&mut self, error: String) {
1330 self.pending_write_error.get_or_insert(error);
1331 }
1332
1333 pub(crate) fn write_command(message: &dyn Any) -> Option<FeatherWriteCommand> {
1334 macro_rules! try_write {
1335 ($message:expr, $type:ty) => {
1336 if let Some(value) = $message.downcast_ref::<$type>() {
1337 let value = value.clone();
1338 return Some(Box::new(move |writer| writer.write(value)));
1339 }
1340 };
1341 }
1342
1343 try_write!(message, QuoteTick);
1344 try_write!(message, TradeTick);
1345 try_write!(message, Bar);
1346 try_write!(message, OrderBookDelta);
1347 try_write!(message, OrderBookDepth);
1348 try_write!(message, IndexPriceUpdate);
1349 try_write!(message, MarkPriceUpdate);
1350 try_write!(message, FundingRateUpdate);
1351 try_write!(message, InstrumentStatus);
1352 try_write!(message, OptionGreeks);
1353 try_write!(message, InstrumentClose);
1354 try_write!(message, InstrumentAny);
1355 try_write!(message, AccountState);
1356 try_write!(message, OrderInitialized);
1357 try_write!(message, OrderDenied);
1358 try_write!(message, OrderEmulated);
1359 try_write!(message, OrderSubmitted);
1360 try_write!(message, OrderAccepted);
1361 try_write!(message, OrderRejected);
1362 try_write!(message, OrderPendingCancel);
1363 try_write!(message, OrderCanceled);
1364 try_write!(message, OrderCancelDenied);
1365 try_write!(message, OrderCancelRejected);
1366 try_write!(message, OrderExpired);
1367 try_write!(message, OrderTriggered);
1368 try_write!(message, OrderPendingUpdate);
1369 try_write!(message, OrderReleased);
1370 try_write!(message, OrderModifyDenied);
1371 try_write!(message, OrderModifyRejected);
1372 try_write!(message, OrderUpdated);
1373 try_write!(message, OrderFilled);
1374 try_write!(message, OrderFillVoided);
1375 try_write!(message, PositionOpened);
1376 try_write!(message, PositionChanged);
1377 try_write!(message, PositionClosed);
1378 try_write!(message, PositionAdjusted);
1379 try_write!(message, OrderSnapshot);
1380 try_write!(message, PositionSnapshot);
1381 try_write!(message, OrderStatusReport);
1382 try_write!(message, FillReport);
1383 try_write!(message, PositionStatusReport);
1384 try_write!(message, ExecutionMassStatus);
1385
1386 if let Some(greeks) = message.downcast_ref::<GreeksData>() {
1387 ensure_custom_data_registered::<GreeksData>();
1388
1389 let data_type = CustomDataType::new(
1390 GreeksData::type_name_static(),
1391 None,
1392 Some(greeks.instrument_id.to_string()),
1393 );
1394
1395 let custom = CustomData::new(Arc::new(greeks.clone()), data_type);
1396 return Some(Box::new(move |writer| writer.write_custom_data(&custom)));
1397 }
1398
1399 if let Some(portfolio) = message.downcast_ref::<PortfolioGreeks>() {
1400 ensure_custom_data_registered::<PortfolioGreeks>();
1401 let custom = CustomData::from_arc(Arc::new(portfolio.clone()));
1402 return Some(Box::new(move |writer| writer.write_custom_data(&custom)));
1403 }
1404
1405 if let Some(deltas) = message.downcast_ref::<OrderBookDeltas>() {
1406 let deltas = deltas.deltas.clone();
1407 return Some(Box::new(move |writer| writer.write_batch(deltas)));
1408 }
1409
1410 if let Some(data) = message.downcast_ref::<Data>() {
1411 let data = data.clone();
1412 return Some(Box::new(move |writer| writer.write_data(data)));
1413 }
1414
1415 if let Some(custom) = message.downcast_ref::<CustomData>() {
1416 let custom = custom.clone();
1417 return Some(Box::new(move |writer| {
1418 writer.write_data(Data::Custom(custom))
1419 }));
1420 }
1421
1422 if let Some(event) = message.downcast_ref::<OrderEventAny>() {
1423 let event = event.clone();
1424 return Some(Box::new(move |writer| writer.write_order_event(&event)));
1425 }
1426
1427 if let Some(event) = message.downcast_ref::<PositionEvent>() {
1428 let event = event.clone();
1429 return Some(Box::new(move |writer| writer.write_position_event(&event)));
1430 }
1431
1432 None
1433 }
1434
1435 fn write_order_event(
1436 &mut self,
1437 event: &OrderEventAny,
1438 ) -> Result<(), Box<dyn std::error::Error>> {
1439 match event {
1440 OrderEventAny::Initialized(event) => self.write(event.clone()),
1441 OrderEventAny::Denied(event) => self.write(*event),
1442 OrderEventAny::Emulated(event) => self.write(*event),
1443 OrderEventAny::Released(event) => self.write(*event),
1444 OrderEventAny::Submitted(event) => self.write(*event),
1445 OrderEventAny::Accepted(event) => self.write(*event),
1446 OrderEventAny::Rejected(event) => self.write(*event),
1447 OrderEventAny::Canceled(event) => self.write(*event),
1448 OrderEventAny::Expired(event) => self.write(*event),
1449 OrderEventAny::Triggered(event) => self.write(*event),
1450 OrderEventAny::PendingUpdate(event) => self.write(*event),
1451 OrderEventAny::PendingCancel(event) => self.write(*event),
1452 OrderEventAny::ModifyDenied(event) => self.write(*event),
1453 OrderEventAny::ModifyRejected(event) => self.write(*event),
1454 OrderEventAny::CancelDenied(event) => self.write(*event),
1455 OrderEventAny::CancelRejected(event) => self.write(*event),
1456 OrderEventAny::Updated(event) => self.write(*event),
1457 OrderEventAny::Filled(event) => self.write(event.clone()),
1458 OrderEventAny::FillVoided(event) => self.write(event.clone()),
1459 }
1460 }
1461
1462 fn write_position_event(
1463 &mut self,
1464 event: &PositionEvent,
1465 ) -> Result<(), Box<dyn std::error::Error>> {
1466 match event {
1467 PositionEvent::PositionOpened(event) => self.write(event.clone()),
1468 PositionEvent::PositionChanged(event) => self.write(event.clone()),
1469 PositionEvent::PositionClosed(event) => self.write(event.clone()),
1470 PositionEvent::PositionAdjusted(event) => self.write(*event),
1471 }
1472 }
1473
1474 fn write_custom_data(&mut self, custom: &CustomData) -> Result<(), Box<dyn std::error::Error>> {
1476 let batch = Self::encode_custom_to_batch(custom)?;
1477 self.write_custom_batch(custom, batch)
1478 }
1479
1480 pub(crate) fn write_custom_batch(
1481 &mut self,
1482 custom: &CustomData,
1483 mut batch: RecordBatch,
1484 ) -> Result<(), Box<dyn std::error::Error>> {
1485 let type_name = custom.data.type_name();
1486 let identifier = custom.data_type.identifier().map(String::from);
1487
1488 if !self.should_write_custom(type_name, identifier.as_deref()) {
1489 return Ok(());
1490 }
1491
1492 let path = self.get_writer_path_custom(type_name)?;
1493 if !self.writers.contains_key(&path) {
1494 self.create_custom_writer(path.clone(), type_name)?;
1495 }
1496
1497 let metadata_row = arrow_metadata_row(batch.schema().metadata(), batch.schema().fields())?;
1498 batch = record_batch_with_identifier_column(batch, custom.data_type.identifier())?;
1499 batch =
1500 record_batch_with_delta_staged_metadata(&batch, std::slice::from_ref(&metadata_row))?;
1501
1502 self.stage_batch_writes(path, std::slice::from_ref(&batch))?;
1503 self.flush_if_due()
1504 }
1505
1506 pub(crate) fn should_write_custom(&self, type_name: &str, identifier: Option<&str>) -> bool {
1507 self.should_write_record(
1508 &CatalogDataType::from(NautilusDataType::Custom {
1509 type_name: type_name.to_string(),
1510 }),
1511 identifier,
1512 )
1513 }
1514
1515 pub fn write_instrument(
1519 &mut self,
1520 instrument: InstrumentAny,
1521 ) -> Result<(), Box<dyn std::error::Error>> {
1522 self.write(instrument)
1523 }
1524
1525 pub fn subscribe_to_message_bus(
1534 writer: Rc<RefCell<Self>>,
1535 ) -> Result<StreamingSinkSubscription, Box<dyn std::error::Error>> {
1536 let sink: StreamingDataSink = Box::new(writer);
1537 Ok(StreamingSinkSubscription::subscribe(
1538 Rc::new(RefCell::new(sink)),
1539 None,
1540 ))
1541 }
1542
1543 pub fn unsubscribe_from_message_bus(handler: &StreamingSinkSubscription) {
1545 handler.unsubscribe();
1546 }
1547}
1548
1549impl Debug for FeatherWriter {
1550 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1551 f.debug_struct(stringify!(FeatherWriter))
1552 .finish_non_exhaustive()
1553 }
1554}
1555
1556impl Drop for FeatherWriter {
1560 fn drop(&mut self) {
1561 if !self.writers.is_empty() {
1562 let _ = self.seal();
1563 }
1564 }
1565}
1566
1567impl StreamingSink for FeatherWriter {
1568 fn write_data(&mut self, data: Data) -> anyhow::Result<()> {
1569 Self::write_data(self, data).map_err(feather_error)
1570 }
1571
1572 fn write_batch(&mut self, data: Vec<Data>) -> anyhow::Result<()> {
1573 Self::write_data_batch(self, data).map_err(feather_error)
1574 }
1575
1576 fn write_any(&mut self, message: &dyn Any) -> anyhow::Result<bool> {
1577 self.write_any_message(message).map_err(feather_error)
1578 }
1579
1580 fn flush(&mut self) -> anyhow::Result<()> {
1581 Self::flush(self).map_err(feather_error)
1582 }
1583
1584 fn close(&mut self) -> anyhow::Result<()> {
1585 Self::close(self).map_err(feather_error)
1586 }
1587}
1588
1589impl StreamingSink for Rc<RefCell<FeatherWriter>> {
1590 fn write_data(&mut self, data: Data) -> anyhow::Result<()> {
1591 StreamingSink::write_data(&mut *self.borrow_mut(), data)
1592 }
1593
1594 fn write_batch(&mut self, data: Vec<Data>) -> anyhow::Result<()> {
1595 StreamingSink::write_batch(&mut *self.borrow_mut(), data)
1596 }
1597
1598 fn write_any(&mut self, message: &dyn Any) -> anyhow::Result<bool> {
1599 StreamingSink::write_any(&mut *self.borrow_mut(), message)
1600 }
1601
1602 fn flush(&mut self) -> anyhow::Result<()> {
1603 StreamingSink::flush(&mut *self.borrow_mut())
1604 }
1605
1606 fn close(&mut self) -> anyhow::Result<()> {
1607 StreamingSink::close(&mut *self.borrow_mut())
1608 }
1609}
1610
1611pub(crate) fn recover_partial_feather_files(directory: &Path) {
1620 let suffix = format!(".{FEATHER_PARTIAL_EXTENSION}");
1621 let mut directories = vec![directory.to_path_buf()];
1622
1623 while let Some(directory) = directories.pop() {
1624 let entries = match fs::read_dir(&directory) {
1625 Ok(entries) => entries,
1626 Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
1627 Err(e) => {
1628 log::warn!(
1629 "Failed to list {} for partial Feather files: {e}",
1630 directory.display()
1631 );
1632 continue;
1633 }
1634 };
1635
1636 for entry in entries.flatten() {
1637 let path = entry.path();
1638
1639 if entry.file_type().is_ok_and(|file_type| file_type.is_dir()) {
1640 directories.push(path);
1641 } else if path.to_string_lossy().ends_with(&suffix)
1642 && let Err(e) = recover_partial_feather_file(&path)
1643 {
1644 log::warn!("Failed to recover Feather file {}: {e}", path.display());
1645 }
1646 }
1647 }
1648}
1649
1650fn recover_released_feather_file(partial_path: &Path) {
1652 if let Err(e) = recover_partial_feather_file(partial_path) {
1653 log::error!(
1654 "Failed to recover Feather file {}: {e}",
1655 partial_path.display()
1656 );
1657 }
1658}
1659
1660fn recover_partial_feather_file(partial_path: &Path) -> anyhow::Result<()> {
1661 let metadata = match fs::symlink_metadata(partial_path) {
1662 Ok(metadata) => metadata,
1663 Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
1664 Err(e) => return Err(e.into()),
1665 };
1666
1667 anyhow::ensure!(
1668 metadata.is_file(),
1669 "Partial Feather path must be a regular file: {}",
1670 partial_path.display(),
1671 );
1672
1673 let file = match OpenOptions::new().read(true).write(true).open(partial_path) {
1674 Ok(file) => file,
1675 Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
1676 Err(e) => return Err(e.into()),
1677 };
1678
1679 match file.try_lock() {
1680 Ok(()) => {}
1681 Err(TryLockError::WouldBlock) => return Ok(()),
1682 Err(TryLockError::Error(e)) => return Err(e.into()),
1683 }
1684
1685 let len = file.metadata()?.len();
1686
1687 if !partial_path.try_exists()? || len == 0 {
1690 return Ok(());
1691 }
1692
1693 let mut batches = 0;
1694 let mut complete_len = 0;
1695 let mut tail_error = None;
1696
1697 match StreamReader::try_new(BufReader::new(&file), None) {
1698 Ok(mut reader) => loop {
1699 match reader.next() {
1700 Some(Ok(_)) => {
1701 batches += 1;
1702 complete_len = reader.get_mut().stream_position()?;
1703 }
1704 Some(Err(e)) => {
1705 tail_error = Some(e);
1706 break;
1707 }
1708 None => break,
1709 }
1710 },
1711 Err(e) => tail_error = Some(e),
1712 }
1713
1714 if let Some(e) = tail_error.take_if(|e| {
1717 !matches!(e, ArrowError::IoError(_, source) if source.kind() == io::ErrorKind::UnexpectedEof)
1718 }) {
1719 return Err(e.into());
1720 }
1721
1722 let cause = tail_error.map_or_else(String::new, |e| format!(": {e}"));
1723
1724 if batches == 0 {
1725 fs::remove_file(partial_path)?;
1726
1727 log::warn!(
1728 "Removed Feather file {} with no complete record batch{cause}",
1729 partial_path.display()
1730 );
1731 return Ok(());
1732 }
1733
1734 let dropped = len.saturating_sub(complete_len);
1735
1736 if dropped > 0 {
1739 file.set_len(complete_len)?;
1740 }
1741
1742 file.sync_all()?;
1743
1744 let sealed_path = partial_path.with_extension("");
1745 fs::rename(partial_path, &sealed_path)?;
1746
1747 let tail = if dropped == 0 {
1748 String::new()
1749 } else {
1750 format!(", dropped {dropped} trailing byte(s){cause}")
1751 };
1752
1753 log::warn!(
1754 "Recovered {batches} record batch(es) into Feather file {}{tail}",
1755 sealed_path.display()
1756 );
1757 Ok(())
1758}
1759
1760#[cfg(test)]
1761mod tests {
1762 use std::sync::{Arc, Mutex, atomic::Ordering};
1763
1764 use nautilus_common::{clock::VirtualClock, live::LiveClock};
1765 #[cfg(target_os = "linux")]
1766 use nautilus_model::data::HasTsInit;
1767 use nautilus_model::{
1768 data::{Data, NautilusRecordType, QuoteTick, TradeTick},
1769 enums::AggressorSide,
1770 identifiers::{InstrumentId, TradeId},
1771 types::{ERROR_PRICE, Price, Quantity},
1772 };
1773 use nautilus_serialization::arrow::{
1774 ArrowSchemaProvider, DecodeDataFromRecordBatch, EncodeToRecordBatch,
1775 };
1776 use rstest::rstest;
1777 use tempfile::TempDir;
1778
1779 use super::*;
1780 use crate::{
1781 common::datafusion::identifiers_from_record_batches,
1782 writer::materializer::restore_staged_record_batches,
1783 };
1784
1785 fn feather_files(directory: &Path, extension: &str) -> Vec<PathBuf> {
1786 let mut files = Vec::new();
1787 let mut stack = vec![directory.to_path_buf()];
1788
1789 while let Some(path) = stack.pop() {
1790 for entry in fs::read_dir(&path).unwrap().flatten() {
1791 let path = entry.path();
1792 if path.is_dir() {
1793 stack.push(path);
1794 } else if path.to_string_lossy().ends_with(&format!(".{extension}")) {
1795 files.push(path);
1796 }
1797 }
1798 }
1799
1800 files.sort();
1801 files
1802 }
1803
1804 fn read_feather_batches(path: &Path) -> Vec<RecordBatch> {
1806 StreamReader::try_new(File::open(path).unwrap(), None)
1807 .unwrap()
1808 .map(Result::unwrap)
1809 .flat_map(|batch| restore_staged_record_batches(batch).unwrap())
1810 .collect()
1811 }
1812
1813 fn read_quotes(path: &Path) -> Vec<Data> {
1814 read_feather_batches(path)
1815 .into_iter()
1816 .flat_map(|batch| {
1817 let metadata = batch.schema().metadata().clone();
1818 QuoteTick::decode_data_batch(&metadata, batch).unwrap()
1819 })
1820 .collect()
1821 }
1822
1823 fn quote_at(ts: u64) -> QuoteTick {
1824 quote_for("AUD/USD.SIM", ts)
1825 }
1826
1827 fn quote_for(instrument_id: &str, ts: u64) -> QuoteTick {
1828 QuoteTick::new(
1829 InstrumentId::from(instrument_id),
1830 Price::from("1.0"),
1831 Price::from("1.1"),
1832 Quantity::from("1000"),
1833 Quantity::from("1000"),
1834 UnixNanos::from(ts),
1835 UnixNanos::from(ts),
1836 )
1837 }
1838
1839 fn quotes_at(timestamps: &[u64]) -> Vec<Data> {
1840 timestamps
1841 .iter()
1842 .map(|&ts| Data::from(quote_at(ts)))
1843 .collect()
1844 }
1845
1846 fn write_partial_file(directory: &Path, timestamps: &[u64]) -> PathBuf {
1849 let mut writer = FeatherWriter::new(
1850 directory.to_path_buf(),
1851 WriterClock::Test(Arc::new(AtomicU64::new(0))),
1852 RotationConfig::NoRotation,
1853 None,
1854 Some(0),
1855 );
1856
1857 for &ts in timestamps {
1858 writer.write(quote_at(ts)).unwrap();
1859 writer.flush().unwrap();
1860 }
1861
1862 writer.close().unwrap();
1863
1864 let sealed = feather_files(directory, FEATHER_EXTENSION).remove(0);
1865 let partial = sealed.with_extension(FEATHER_PARTIAL_EXTENSION);
1866 fs::rename(&sealed, &partial).unwrap();
1867 partial
1868 }
1869
1870 const TORN_TAIL: &str = ": Io error: failed to fill whole buffer";
1871
1872 static RECOVERY_LOGS: RecoveryLogCapture = RecoveryLogCapture(Mutex::new(Vec::new()));
1873
1874 fn recover_with_warnings(directory: &Path) -> Vec<String> {
1876 let _ = log::set_logger(&RECOVERY_LOGS);
1877 log::set_max_level(log::LevelFilter::Warn);
1878
1879 recover_partial_feather_files(directory);
1880
1881 let directory = directory.display().to_string();
1882 RECOVERY_LOGS
1883 .0
1884 .lock()
1885 .unwrap()
1886 .iter()
1887 .filter(|(level, message)| *level == log::Level::Warn && message.contains(&directory))
1888 .map(|(_, message)| message.clone())
1889 .collect()
1890 }
1891
1892 struct RecoveryLogCapture(Mutex<Vec<(log::Level, String)>>);
1893
1894 impl log::Log for RecoveryLogCapture {
1895 fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
1896 metadata.target() == "nautilus_persistence::writer::feather"
1897 }
1898
1899 fn log(&self, record: &log::Record<'_>) {
1900 if self.enabled(record.metadata()) {
1901 self.0
1902 .lock()
1903 .unwrap()
1904 .push((record.level(), record.args().to_string()));
1905 }
1906 }
1907
1908 fn flush(&self) {}
1909 }
1910
1911 #[rstest]
1912 fn test_subscription_receives_typed_quotes_and_unsubscribes() {
1913 use nautilus_common::msgbus::{MStr, publish_quote};
1914
1915 let temp_dir = TempDir::new().unwrap();
1916
1917 let writer = Rc::new(RefCell::new(FeatherWriter::new(
1918 temp_dir.path().to_path_buf(),
1919 WriterClock::Test(Arc::new(AtomicU64::new(0))),
1920 RotationConfig::NoRotation,
1921 None,
1922 Some(0),
1923 )));
1924
1925 let quote = QuoteTick::new(
1926 InstrumentId::from("AUD/USD.SIM"),
1927 Price::from("1.0"),
1928 Price::from("1.1"),
1929 Quantity::from("2"),
1930 Quantity::from("3"),
1931 4.into(),
1932 5.into(),
1933 );
1934 let handler = FeatherWriter::subscribe_to_message_bus(writer.clone()).unwrap();
1935 publish_quote(MStr::topic("data.quotes.AUD/USD.SIM").unwrap(), "e);
1936 FeatherWriter::unsubscribe_from_message_bus(&handler);
1937 publish_quote(MStr::topic("data.quotes.AUD/USD.SIM").unwrap(), "e);
1938 writer.borrow_mut().close().unwrap();
1939
1940 let rows = feather_files(temp_dir.path(), FEATHER_EXTENSION)
1941 .iter()
1942 .flat_map(|path| read_feather_batches(path))
1943 .map(|batch| batch.num_rows())
1944 .sum::<usize>();
1945 assert_eq!(rows, 1);
1946 }
1947
1948 #[rstest]
1949 fn test_subscription_records_greeks_once_and_unsubscribes() {
1950 use nautilus_common::msgbus::{MStr, publish_any, publish_greeks};
1951 use nautilus_serialization::arrow::{DecodeDataFromRecordBatch, custom::CustomDataDecoder};
1952 let temp_dir = TempDir::new().unwrap();
1953
1954 let writer = Rc::new(RefCell::new(FeatherWriter::new(
1955 temp_dir.path().to_path_buf(),
1956 WriterClock::Test(Arc::new(AtomicU64::new(0))),
1957 RotationConfig::NoRotation,
1958 None,
1959 Some(0),
1960 )));
1961 let greeks = GreeksData::from_delta("AUD/USD.SIM".parse().unwrap(), 1.25, 8.0, 11.into());
1962 let portfolio = greeks.to_portfolio_greeks();
1963 let greeks_topic = MStr::topic("data.GreeksData.instrument_id=AUD/USD.SIM").unwrap();
1964 let portfolio_topic = MStr::topic("data.PortfolioGreeks").unwrap();
1965 let handler = FeatherWriter::subscribe_to_message_bus(writer.clone()).unwrap();
1966 publish_greeks(greeks_topic, &greeks);
1967 publish_any(portfolio_topic, &portfolio);
1968 FeatherWriter::unsubscribe_from_message_bus(&handler);
1969 publish_greeks(greeks_topic, &greeks);
1970 publish_any(portfolio_topic, &portfolio);
1971 writer.borrow_mut().close().unwrap();
1972 let mut counts = HashMap::new();
1973
1974 for batch in feather_files(&temp_dir.path().join("data/custom"), FEATHER_EXTENSION)
1975 .iter()
1976 .flat_map(|path| read_feather_batches(path))
1977 {
1978 let schema = batch.schema();
1979 let name = schema.metadata()["type_name"].clone();
1980 *counts.entry(name.clone()).or_insert(0) += batch.num_rows();
1981 let decoded = CustomDataDecoder::decode_data_batch(schema.metadata(), batch).unwrap();
1982 assert_eq!(decoded.len(), 1);
1983
1984 let Data::Custom(custom) = &decoded[0] else {
1985 panic!("expected custom data")
1986 };
1987
1988 match name.as_str() {
1989 "GreeksData" => {
1990 assert_eq!(
1991 custom.data_type,
1992 CustomDataType::new(
1993 "GreeksData",
1994 None,
1995 Some(greeks.instrument_id.to_string())
1996 )
1997 );
1998 assert_eq!(
1999 custom.data.as_any().downcast_ref::<GreeksData>(),
2000 Some(&greeks)
2001 );
2002 }
2003 "PortfolioGreeks" => {
2004 assert_eq!(
2005 custom.data_type,
2006 CustomDataType::new("PortfolioGreeks", None, None)
2007 );
2008 assert_eq!(
2009 custom.data.as_any().downcast_ref::<PortfolioGreeks>(),
2010 Some(&portfolio)
2011 );
2012 }
2013 _ => panic!("unexpected custom data type {name}"),
2014 }
2015 }
2016
2017 assert_eq!(
2018 counts,
2019 HashMap::from([
2020 ("GreeksData".to_string(), 1),
2021 ("PortfolioGreeks".to_string(), 1),
2022 ])
2023 );
2024 }
2025
2026 #[rstest]
2027 #[case(false)]
2028 #[case(true)]
2029 fn test_message_write_error_reaches_flush_or_close(#[case] close: bool) {
2030 let temp_dir = TempDir::new().unwrap();
2031
2032 let mut writer = FeatherWriter::new(
2033 temp_dir.path().to_path_buf(),
2034 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2035 RotationConfig::NoRotation,
2036 None,
2037 Some(0),
2038 );
2039
2040 let quote = QuoteTick::new(
2041 InstrumentId::from("AUD/USD.SIM"),
2042 ERROR_PRICE,
2043 ERROR_PRICE,
2044 Quantity::from("1"),
2045 Quantity::from("2"),
2046 3.into(),
2047 4.into(),
2048 );
2049 let write_error = writer.write_any_message("e).unwrap_err().to_string();
2050
2051 let error = if close {
2052 StreamingSink::close(&mut writer)
2053 } else {
2054 StreamingSink::flush(&mut writer)
2055 }
2056 .unwrap_err();
2057
2058 assert_eq!(error.to_string(), write_error);
2059 }
2060
2061 #[rstest]
2062 fn test_writer_manager_keys() {
2063 let temp_dir = TempDir::new().unwrap();
2064
2065 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2067 let timestamp = clock.timestamp_ns();
2068
2069 let mut manager = FeatherWriter::new(
2070 temp_dir.path().to_path_buf(),
2071 clock,
2072 RotationConfig::NoRotation,
2073 None,
2074 None, );
2076
2077 let instrument_id = "AAPL.AAPL";
2078
2079 let quote = QuoteTick::new(
2081 InstrumentId::from(instrument_id),
2082 Price::from("100.0"),
2083 Price::from("100.0"),
2084 Quantity::from("100.0"),
2085 Quantity::from("100.0"),
2086 UnixNanos::from(1_000_000_000_000_000_000),
2087 UnixNanos::from(1_000_000_000_000_000_000),
2088 );
2089
2090 let trade = TradeTick::new(
2091 InstrumentId::from(instrument_id),
2092 Price::from("100.0"),
2093 Quantity::from("100.0"),
2094 AggressorSide::Buy,
2095 TradeId::from("1"),
2096 UnixNanos::from(1_000_000_000_000_000_000),
2097 UnixNanos::from(1_000_000_000_000_000_000),
2098 );
2099
2100 manager.write(quote).unwrap();
2101 manager.write(trade).unwrap();
2102
2103 let path = manager.get_writer_path(QuoteTick::catalog_family());
2105 let expected_path = temp_dir
2106 .path()
2107 .join("quotes")
2108 .join(format!("quotes_{timestamp}.feather"));
2109 assert_eq!(path.path, expected_path);
2110 assert!(manager.writers.contains_key(&path));
2111 let writer = manager.writers.get(&path).unwrap();
2112 assert!(writer.size() > 0);
2113
2114 let path = manager.get_writer_path(TradeTick::catalog_family());
2115 let expected_path = temp_dir
2116 .path()
2117 .join("trades")
2118 .join(format!("trades_{timestamp}.feather"));
2119 assert_eq!(path.path, expected_path);
2120 assert!(manager.writers.contains_key(&path));
2121 let writer = manager.writers.get(&path).unwrap();
2122 assert!(writer.size() > 0);
2123 }
2124
2125 #[rstest]
2126 #[case::quotes(NautilusDataType::QuoteTick.into(), "quotes/quotes_0.feather")]
2127 #[case::record(NautilusRecordType::OrderFilled.into(), "order_filled/order_filled_0.feather")]
2128 #[case::instrument_class(
2129 CatalogDataType::Instrument(NautilusInstrumentType::CurrencyPair),
2130 "instruments/currency_pair/instruments_0.feather"
2131 )]
2132 #[case::custom(
2133 NautilusDataType::Custom { type_name: "Signal".to_string() }.into(),
2134 "data/custom/Signal/Signal_0.feather"
2135 )]
2136 fn writer_paths_hold_one_file_per_type(
2137 #[case] data_type: CatalogDataType,
2138 #[case] expected: &str,
2139 ) {
2140 let temp_dir = TempDir::new().unwrap();
2141
2142 let manager = FeatherWriter::new(
2143 temp_dir.path().to_path_buf(),
2144 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2145 RotationConfig::NoRotation,
2146 None,
2147 None,
2148 );
2149
2150 let path = manager.build_writer_path(&data_type, UnixNanos::default(), 0);
2151
2152 assert_eq!(path, temp_dir.path().join(expected));
2153 }
2154
2155 #[rstest]
2156 fn existing_feather_writer_implements_streaming_data_sink() {
2157 let temp_dir = TempDir::new().unwrap();
2158 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2159
2160 let mut writer = FeatherWriter::new(
2161 temp_dir.path().to_path_buf(),
2162 clock,
2163 RotationConfig::NoRotation,
2164 None,
2165 None,
2166 );
2167
2168 let quote = QuoteTick::new(
2169 InstrumentId::from("AUD/USD.SIM"),
2170 Price::from("1.0"),
2171 Price::from("1.1"),
2172 Quantity::from("1000"),
2173 Quantity::from("1000"),
2174 UnixNanos::from(1_000),
2175 UnixNanos::from(1_000),
2176 );
2177
2178 StreamingSink::write_data(&mut writer, Data::Quote(quote)).unwrap();
2179 StreamingSink::flush(&mut writer).unwrap();
2180 StreamingSink::close(&mut writer).unwrap();
2181
2182 let files = feather_files(&temp_dir.path().join("quotes"), FEATHER_EXTENSION);
2183 assert_eq!(files.len(), 1);
2184 let identifiers =
2185 identifiers_from_record_batches(&read_feather_batches(&files[0])).unwrap();
2186 assert_eq!(identifiers, vec!["AUD/USD.SIM".to_string()]);
2187 }
2188
2189 fn timestamps(path: &Path) -> Vec<u64> {
2191 read_quotes(path)
2192 .into_iter()
2193 .map(|data| match data {
2194 Data::Quote(quote) => quote.ts_init.as_u64(),
2195 other => panic!("not a quote: {other:?}"),
2196 })
2197 .collect()
2198 }
2199
2200 fn sealed_timestamps(directory: &Path) -> Vec<Vec<u64>> {
2202 let mut files: Vec<_> = feather_files(directory, FEATHER_EXTENSION)
2203 .iter()
2204 .map(|path| timestamps(path))
2205 .collect();
2206 files.sort();
2207 files
2208 }
2209
2210 #[rstest]
2211 fn scheduled_rotation_keeps_time_of_day_anchor_after_late_rotation() {
2212 let temp_dir = TempDir::new().unwrap();
2213 let quotes = NautilusDataType::QuoteTick.into();
2214
2215 let mut writer = FeatherWriter::new(
2216 temp_dir.path().to_path_buf(),
2217 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2218 RotationConfig::ScheduledDates {
2219 interval_ns: 86_400_000_000_000,
2220 rotation_time: UnixNanos::from(36_000_000_000_000u64),
2221 rotation_timezone: jiff::tz::TimeZone::UTC,
2222 },
2223 None,
2224 None,
2225 );
2226
2227 writer.write(quote_at(1_767_258_000_000_000_000)).unwrap();
2229 assert_eq!(
2230 writer.get_next_rotation_time("es),
2231 Some(UnixNanos::from(1_767_261_600_000_000_000u64)),
2232 );
2233
2234 writer.write(quote_at(1_767_263_400_000_000_000)).unwrap();
2236 assert_eq!(
2237 writer.get_next_rotation_time("es),
2238 Some(UnixNanos::from(1_767_348_000_000_000_000u64)),
2239 );
2240 assert_eq!(
2241 sealed_timestamps(temp_dir.path()),
2242 vec![vec![1_767_258_000_000_000_000]]
2243 );
2244 }
2245
2246 #[rstest]
2249 fn scheduled_rotation_opens_the_next_file_before_the_first_row_past_the_boundary() {
2250 let temp_dir = TempDir::new().unwrap();
2251
2252 let mut writer = FeatherWriter::new(
2253 temp_dir.path().to_path_buf(),
2254 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2255 RotationConfig::ScheduledDates {
2256 interval_ns: 3_600_000_000_000,
2257 rotation_time: UnixNanos::default(),
2258 rotation_timezone: jiff::tz::TimeZone::UTC,
2259 },
2260 None,
2261 None,
2262 );
2263
2264 let (before, after) = (1_794_073_635_000_000_000u64, 1_794_075_281_380_125_727u64);
2266 writer.write(quote_at(before)).unwrap();
2267 writer.write(quote_at(after)).unwrap();
2268 writer.write(quote_at(after)).unwrap();
2269 assert_eq!(sealed_timestamps(temp_dir.path()), vec![vec![before]]);
2270
2271 writer.close().unwrap();
2272 assert_eq!(
2273 sealed_timestamps(temp_dir.path()),
2274 vec![vec![before], vec![after, after]]
2275 );
2276 }
2277
2278 #[rstest]
2282 fn a_batch_shares_one_rotation_decision_whatever_its_group_order() {
2283 let temp_dir = TempDir::new().unwrap();
2284
2285 let mut writer = FeatherWriter::new(
2286 temp_dir.path().to_path_buf(),
2287 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2288 RotationConfig::Interval { interval_ns: 10 },
2289 None,
2290 None,
2291 );
2292
2293 writer.write(quote_at(1)).unwrap();
2294 writer
2295 .write_data_batch(vec![
2296 Data::from(quote_at(1)),
2297 Data::from(quote_for("EUR/USD.SIM", 11)),
2298 ])
2299 .unwrap();
2300 assert_eq!(sealed_timestamps(temp_dir.path()), Vec::<Vec<u64>>::new());
2301
2302 writer.write(quote_at(12)).unwrap();
2304 writer.close().unwrap();
2305
2306 let mut files = sealed_timestamps(temp_dir.path());
2308 for file in &mut files {
2309 file.sort_unstable();
2310 }
2311 assert_eq!(files, vec![vec![1, 1, 11], vec![12]]);
2312 }
2313
2314 #[rstest]
2315 fn size_rotation_waits_for_the_next_ts_init() {
2316 let temp_dir = TempDir::new().unwrap();
2317
2318 let mut writer = FeatherWriter::new(
2319 temp_dir.path().to_path_buf(),
2320 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2321 RotationConfig::Size { max_size: 1 },
2322 None,
2323 None,
2324 );
2325
2326 for ts in [1, 1, 2, 2, 3] {
2327 writer.write(quote_at(ts)).unwrap();
2328 }
2329 writer.close().unwrap();
2330
2331 assert_eq!(
2332 sealed_timestamps(temp_dir.path()),
2333 vec![vec![1, 1], vec![2, 2], vec![3]]
2334 );
2335 }
2336
2337 #[rstest]
2338 #[case::modify(true)]
2339 #[case::cancel(false)]
2340 fn test_command_denied_dynamic_round_trip(
2341 #[case] modify: bool,
2342 #[values(false, true)] envelope: bool,
2343 ) {
2344 use nautilus_core::UUID4;
2345 use nautilus_model::{
2346 events::order::spec::{OrderCancelDeniedSpec, OrderModifyDeniedSpec},
2347 identifiers::{AccountId, ClientOrderId, StrategyId, TraderId, VenueOrderId},
2348 };
2349 use nautilus_serialization::arrow::DecodeTypedFromRecordBatch;
2350 let cause = UUID4::new();
2351 let trader = TraderId::from("WRITER-017");
2352 let strategy = StrategyId::from("S-DENIAL-019");
2353 let instrument = InstrumentId::from("ETHUSDT-PERP.BINANCE");
2354 let client = ClientOrderId::from("O-DENIAL-023");
2355 let venue = VenueOrderId::from("V-DENIAL-029");
2356 let account = AccountId::from("BINANCE-031");
2357
2358 let event = if modify {
2359 OrderEventAny::ModifyDenied(
2360 OrderModifyDeniedSpec::builder()
2361 .trader_id(trader)
2362 .strategy_id(strategy)
2363 .instrument_id(instrument)
2364 .client_order_id(client)
2365 .venue_order_id(venue)
2366 .account_id(account)
2367 .reason("not sent to modify".into())
2368 .ts_event(37.into())
2369 .ts_init(41.into())
2370 .causation_id(cause)
2371 .build(),
2372 )
2373 } else {
2374 OrderEventAny::CancelDenied(
2375 OrderCancelDeniedSpec::builder()
2376 .trader_id(trader)
2377 .strategy_id(strategy)
2378 .instrument_id(instrument)
2379 .client_order_id(client)
2380 .venue_order_id(venue)
2381 .account_id(account)
2382 .reason("not sent to cancel".into())
2383 .ts_event(43.into())
2384 .ts_init(47.into())
2385 .causation_id(cause)
2386 .build(),
2387 )
2388 };
2389
2390 let temp = TempDir::new().unwrap();
2391 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2392
2393 let mut writer = FeatherWriter::new(
2394 temp.path().to_path_buf(),
2395 clock,
2396 RotationConfig::NoRotation,
2397 None,
2398 None,
2399 );
2400
2401 let written = if envelope {
2402 writer.write_any_message(&event).unwrap()
2403 } else {
2404 match &event {
2405 OrderEventAny::ModifyDenied(event) => writer.write_any_message(event).unwrap(),
2406 OrderEventAny::CancelDenied(event) => writer.write_any_message(event).unwrap(),
2407 _ => unreachable!(),
2408 }
2409 };
2410
2411 assert!(written);
2412 writer.close().unwrap();
2413 let files = feather_files(temp.path(), "feather");
2414 assert_eq!(files.len(), 1);
2415 let mut recovered = Vec::new();
2416
2417 for batch in read_feather_batches(&files[0]) {
2418 let metadata = batch.schema().metadata().clone();
2419 assert_eq!(
2420 metadata.get("type_name").unwrap(),
2421 if modify {
2422 "OrderModifyDenied"
2423 } else {
2424 "OrderCancelDenied"
2425 }
2426 );
2427
2428 if modify {
2429 recovered.extend(
2430 OrderModifyDenied::decode_typed_batch(&metadata, batch)
2431 .unwrap()
2432 .into_iter()
2433 .map(OrderEventAny::ModifyDenied),
2434 );
2435 } else {
2436 recovered.extend(
2437 OrderCancelDenied::decode_typed_batch(&metadata, batch)
2438 .unwrap()
2439 .into_iter()
2440 .map(OrderEventAny::CancelDenied),
2441 );
2442 }
2443 }
2444
2445 assert_eq!(recovered, vec![event]);
2446 }
2447
2448 #[rstest]
2449 fn test_file_writer_round_trip() {
2450 let instrument_id = "AAPL.AAPL";
2451
2452 let quote = QuoteTick::new(
2454 InstrumentId::from(instrument_id),
2455 Price::from("100.0"),
2456 Price::from("100.0"),
2457 Quantity::from("100.0"),
2458 Quantity::from("100.0"),
2459 UnixNanos::from(100),
2460 UnixNanos::from(100),
2461 );
2462 let metadata = QuoteTick::metadata("e);
2463 let schema = QuoteTick::get_schema(Some(metadata.clone()));
2464 let batch = QuoteTick::encode_batch(&QuoteTick::metadata("e), &[quote]).unwrap();
2465
2466 let temp_dir = TempDir::new().unwrap();
2467 let path = temp_dir.path().join("quotes.feather");
2468 let mut file = FeatherFile::create(&path, &schema, &RotationConfig::NoRotation).unwrap();
2469 file.write_record_batch(&batch).unwrap();
2470 file.seal().unwrap();
2471
2472 assert!(!path.with_extension(FEATHER_PARTIAL_EXTENSION).exists());
2473 let mut reader = StreamReader::try_new(File::open(&path).unwrap(), None).unwrap();
2474
2475 let read_metadata = reader.schema().metadata().clone();
2476 let mut expected_metadata = metadata.clone();
2477 expected_metadata.insert("type_name".to_string(), "QuoteTick".to_string());
2478 assert_eq!(read_metadata, expected_metadata);
2479
2480 let read_batch = reader.next().unwrap().unwrap();
2481 assert_eq!(read_batch.column(0), batch.column(0));
2482
2483 let decoded = QuoteTick::decode_data_batch(&metadata, batch).unwrap();
2484 assert_eq!(decoded[0], Data::from(quote));
2485 }
2486
2487 #[rstest]
2488 fn test_round_trip() {
2489 let temp_dir = TempDir::new_in(".").unwrap();
2491
2492 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2494
2495 let mut manager = FeatherWriter::new(
2496 temp_dir.path().to_path_buf(),
2497 clock,
2498 RotationConfig::NoRotation,
2499 None,
2500 None, );
2502
2503 let instrument_id = "AAPL.AAPL";
2504
2505 let quote = QuoteTick::new(
2507 InstrumentId::from(instrument_id),
2508 Price::from("100.0"),
2509 Price::from("100.0"),
2510 Quantity::from("100.0"),
2511 Quantity::from("100.0"),
2512 UnixNanos::from(100),
2513 UnixNanos::from(100),
2514 );
2515
2516 let trade = TradeTick::new(
2517 InstrumentId::from(instrument_id),
2518 Price::from("100.0"),
2519 Quantity::from("100.0"),
2520 AggressorSide::Buy,
2521 TradeId::from("1"),
2522 UnixNanos::from(100),
2523 UnixNanos::from(100),
2524 );
2525
2526 manager.write(quote).unwrap();
2527 manager.write(trade).unwrap();
2528
2529 let paths = manager.writers.keys().cloned().collect::<Vec<_>>();
2530 assert_eq!(paths.len(), 2);
2531
2532 manager.close().unwrap();
2533
2534 let mut recovered_quotes = Vec::new();
2535 let mut recovered_trades = Vec::new();
2536
2537 for path in paths {
2538 let path_str = path.path;
2539 for batch in read_feather_batches(&path_str) {
2540 let metadata = batch.schema().metadata().clone();
2541
2542 if path_str.to_str().unwrap().contains("quotes") {
2543 let decoded = QuoteTick::decode_data_batch(&metadata, batch).unwrap();
2544 recovered_quotes.extend(decoded);
2545 } else if path_str.to_str().unwrap().contains("trades") {
2546 let decoded = TradeTick::decode_data_batch(&metadata, batch).unwrap();
2547 recovered_trades.extend(decoded);
2548 }
2549 }
2550 }
2551
2552 assert_eq!(recovered_quotes.len(), 1, "Expected one QuoteTick record");
2554 assert_eq!(recovered_trades.len(), 1, "Expected one TradeTick record");
2555
2556 assert_eq!(recovered_quotes[0], Data::from(quote));
2558 assert_eq!(recovered_trades[0], Data::from(trade));
2559 }
2560
2561 #[rstest]
2562 fn test_write_data_enum() {
2563 let temp_dir = TempDir::new().unwrap();
2564 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2565
2566 let mut writer = FeatherWriter::new(
2567 temp_dir.path().to_path_buf(),
2568 clock,
2569 RotationConfig::NoRotation,
2570 None,
2571 None,
2572 );
2573
2574 let quote = QuoteTick::new(
2575 InstrumentId::from("AUD/USD.SIM"),
2576 Price::from("1.0"),
2577 Price::from("1.0"),
2578 Quantity::from("1000"),
2579 Quantity::from("1000"),
2580 UnixNanos::from(1000),
2581 UnixNanos::from(1000),
2582 );
2583
2584 writer.write_data(Data::Quote(quote)).unwrap();
2586 writer.flush().unwrap();
2587
2588 assert!(!writer.writers.is_empty() || temp_dir.path().read_dir().unwrap().count() > 0);
2590 }
2591
2592 #[rstest]
2593 fn test_write_data_all_types() {
2594 let temp_dir = TempDir::new().unwrap();
2595 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2596
2597 let mut writer = FeatherWriter::new(
2598 temp_dir.path().to_path_buf(),
2599 clock,
2600 RotationConfig::NoRotation,
2601 None,
2602 None,
2603 );
2604
2605 let instrument_id = InstrumentId::from("AUD/USD.SIM");
2606
2607 let quote = QuoteTick::new(
2609 instrument_id,
2610 Price::from("1.0"),
2611 Price::from("1.0"),
2612 Quantity::from("1000"),
2613 Quantity::from("1000"),
2614 UnixNanos::from(1000),
2615 UnixNanos::from(1000),
2616 );
2617 writer.write_data(Data::Quote(quote)).unwrap();
2618
2619 let trade = TradeTick::new(
2620 instrument_id,
2621 Price::from("1.0"),
2622 Quantity::from("1000"),
2623 AggressorSide::Buy,
2624 TradeId::from("1"),
2625 UnixNanos::from(2000),
2626 UnixNanos::from(2000),
2627 );
2628 writer.write_data(Data::Trade(trade)).unwrap();
2629
2630 let delta = OrderBookDelta::clear(
2631 instrument_id,
2632 0,
2633 UnixNanos::from(3000),
2634 UnixNanos::from(3000),
2635 );
2636 writer.write_data(Data::BookDelta(delta)).unwrap();
2637
2638 writer.flush().unwrap();
2639 }
2640
2641 #[rstest]
2642 fn test_auto_flush() {
2643 let temp_dir = TempDir::new().unwrap();
2644 let shared_time = Arc::new(AtomicU64::new(0));
2645 let clock = WriterClock::Test(Arc::clone(&shared_time));
2646
2647 let mut writer = FeatherWriter::new(
2648 temp_dir.path().to_path_buf(),
2649 clock,
2650 RotationConfig::NoRotation,
2651 None,
2652 Some(100), );
2654
2655 let quote = QuoteTick::new(
2656 InstrumentId::from("AUD/USD.SIM"),
2657 Price::from("1.0"),
2658 Price::from("1.0"),
2659 Quantity::from("1000"),
2660 Quantity::from("1000"),
2661 UnixNanos::from(1000),
2662 UnixNanos::from(1000),
2663 );
2664
2665 writer.write(quote).unwrap();
2667 assert_eq!(writer.writers.len(), 1);
2668 assert_eq!(writer.last_flush_ns, UnixNanos::from(0));
2669
2670 shared_time.store(200_000_000, Ordering::Relaxed);
2672
2673 let quote2 = QuoteTick::new(
2675 InstrumentId::from("AUD/USD.SIM"),
2676 Price::from("1.1"),
2677 Price::from("1.1"),
2678 Quantity::from("1000"),
2679 Quantity::from("1000"),
2680 UnixNanos::from(2000),
2681 UnixNanos::from(2000),
2682 );
2683 writer.write(quote2).unwrap();
2684
2685 let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
2688 let flushed_len = fs::metadata(&partial[0]).unwrap().len();
2689 let open_writers = writer.writers.len();
2690 let last_flush_ns = writer.last_flush_ns;
2691 let sealed_before_close = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2692 writer.close().unwrap();
2693
2694 let sealed = partial[0].with_extension("");
2695 let sealed_rows = read_feather_batches(&sealed)
2696 .iter()
2697 .map(RecordBatch::num_rows)
2698 .sum::<usize>();
2699 assert_eq!(open_writers, 1);
2700 assert_eq!(last_flush_ns, UnixNanos::from(200_000_000));
2701 assert_eq!(partial.len(), 1);
2702 assert!(sealed_before_close.is_empty());
2703 assert_eq!(fs::metadata(&sealed).unwrap().len(), flushed_len + 8);
2704 assert_eq!(sealed_rows, 2);
2705 }
2706
2707 #[rstest]
2708 fn flushes_append_to_one_file_per_type_until_close() {
2709 let temp_dir = TempDir::new().unwrap();
2710 let shared_time = Arc::new(AtomicU64::new(0));
2711
2712 let mut writer = FeatherWriter::new(
2713 temp_dir.path().to_path_buf(),
2714 WriterClock::Test(Arc::clone(&shared_time)),
2715 RotationConfig::NoRotation,
2716 None,
2717 Some(1_000),
2718 );
2719
2720 let quotes = (1..=10)
2721 .map(|minute| {
2722 QuoteTick::new(
2723 InstrumentId::from("AUD/USD.SIM"),
2724 Price::from("1.0"),
2725 Price::from("1.1"),
2726 Quantity::from("1000"),
2727 Quantity::from("1000"),
2728 UnixNanos::from(minute * 60_000_000_000),
2729 UnixNanos::from(minute * 60_000_000_000),
2730 )
2731 })
2732 .collect::<Vec<_>>();
2733
2734 for quote in "es {
2736 shared_time.store(quote.ts_init.as_u64(), Ordering::Relaxed);
2737 writer.write(*quote).unwrap();
2738 }
2739
2740 let partial_before_close = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
2741 let sealed_before_close = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2742 writer.close().unwrap();
2743 let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2744
2745 let recovered = read_feather_batches(&sealed[0])
2746 .into_iter()
2747 .flat_map(|batch| {
2748 let metadata = batch.schema().metadata().clone();
2749 QuoteTick::decode_data_batch(&metadata, batch).unwrap()
2750 })
2751 .collect::<Vec<_>>();
2752
2753 assert_eq!(partial_before_close.len(), 1);
2754 assert!(sealed_before_close.is_empty());
2755 assert_eq!(sealed.len(), 1);
2756 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
2757 assert_eq!(
2758 recovered,
2759 quotes.into_iter().map(Data::from).collect::<Vec<_>>()
2760 );
2761 }
2762
2763 #[rstest]
2764 #[case::below_row_cap(3, vec![3])]
2765 #[case::above_row_cap(
2766 DEFAULT_DATA_BATCH_CHUNK_SIZE + 1,
2767 vec![DEFAULT_DATA_BATCH_CHUNK_SIZE, 1]
2768 )]
2769 fn writes_between_flushes_append_one_batch_per_row_cap(
2770 #[case] count: usize,
2771 #[case] expected_batch_rows: Vec<usize>,
2772 ) {
2773 let temp_dir = TempDir::new().unwrap();
2774
2775 let mut writer = FeatherWriter::new(
2776 temp_dir.path().to_path_buf(),
2777 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2778 RotationConfig::NoRotation,
2779 None,
2780 Some(0),
2781 );
2782 let instrument_ids = [
2783 InstrumentId::from("AUD/USD.SIM"),
2784 InstrumentId::from("EUR/USD.SIM"),
2785 ];
2786
2787 for index in 0..count {
2788 writer
2789 .write(QuoteTick::new(
2790 instrument_ids[index % 2],
2791 Price::from("1.0"),
2792 Price::from("1.1"),
2793 Quantity::from("1000"),
2794 Quantity::from("1000"),
2795 UnixNanos::from(index as u64),
2796 UnixNanos::from(index as u64),
2797 ))
2798 .unwrap();
2799 }
2800
2801 writer.close().unwrap();
2802 let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2803 let batch_rows = StreamReader::try_new(File::open(&sealed[0]).unwrap(), None)
2804 .unwrap()
2805 .map(|batch| batch.unwrap().num_rows())
2806 .collect::<Vec<_>>();
2807
2808 assert_eq!(sealed.len(), 1);
2809 assert_eq!(batch_rows, expected_batch_rows);
2810 }
2811
2812 #[rstest]
2813 fn drop_seals_open_files() {
2814 let temp_dir = TempDir::new().unwrap();
2815
2816 let mut writer = FeatherWriter::new(
2817 temp_dir.path().to_path_buf(),
2818 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2819 RotationConfig::NoRotation,
2820 None,
2821 None,
2822 );
2823 writer
2824 .write(QuoteTick::new(
2825 InstrumentId::from("AUD/USD.SIM"),
2826 Price::from("1.0"),
2827 Price::from("1.1"),
2828 Quantity::from("1000"),
2829 Quantity::from("1000"),
2830 UnixNanos::from(1_000),
2831 UnixNanos::from(1_000),
2832 ))
2833 .unwrap();
2834
2835 drop(writer);
2836
2837 assert_eq!(feather_files(temp_dir.path(), FEATHER_EXTENSION).len(), 1);
2838 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
2839 }
2840
2841 #[rstest]
2842 fn reserved_paths_skip_files_left_in_the_directory() {
2843 let temp_dir = TempDir::new().unwrap();
2844
2845 let mut writer = FeatherWriter::new(
2846 temp_dir.path().to_path_buf(),
2847 WriterClock::Test(Arc::new(AtomicU64::new(5))),
2848 RotationConfig::NoRotation,
2849 None,
2850 None,
2851 );
2852 let sealed = temp_dir.path().join("trades").join("trades_5.feather");
2853 let partial = temp_dir
2854 .path()
2855 .join("trades")
2856 .join("trades_5-1.feather.partial");
2857 fs::create_dir_all(sealed.parent().unwrap()).unwrap();
2858 fs::write(&sealed, b"sealed").unwrap();
2859 fs::write(&partial, b"partial").unwrap();
2860
2861 let path = writer.reserve_writer_path(NautilusDataType::TradeTick.into());
2862
2863 assert_eq!(
2864 path.path,
2865 temp_dir.path().join("trades").join("trades_5-2.feather")
2866 );
2867 assert_eq!(fs::read(&sealed).unwrap(), b"sealed");
2868 assert_eq!(fs::read(&partial).unwrap(), b"partial");
2869 }
2870
2871 #[rstest]
2872 #[case::same_time(vec![5; 32])]
2873 #[case::clock_rewinds(vec![5, 3, 5])]
2874 #[case::clock_advances_then_rewinds(vec![5, 7, 5, 7])]
2875 fn path_reservations_are_bounded_after_files_are_deleted(#[case] timestamps: Vec<u64>) {
2876 let temp_dir = TempDir::new().unwrap();
2877 let time = Arc::new(AtomicU64::new(0));
2878 let mut writer = FeatherWriter::new(
2879 temp_dir.path().to_path_buf(),
2880 WriterClock::Test(Arc::clone(&time)),
2881 RotationConfig::NoRotation,
2882 None,
2883 None,
2884 );
2885 let mut paths = HashSet::new();
2886
2887 for timestamp in ×tamps {
2888 time.store(*timestamp, Ordering::Relaxed);
2889 writer.write(quote_at(1)).unwrap();
2890 writer.close().unwrap();
2891 let path = feather_files(temp_dir.path(), FEATHER_EXTENSION).remove(0);
2892 assert!(paths.insert(path.clone()));
2893 fs::remove_file(path).unwrap();
2894 }
2895
2896 assert_eq!(paths.len(), timestamps.len());
2897 assert_eq!(writer.path_sequences.len(), 1);
2898 }
2899
2900 #[rstest]
2901 fn test_close() {
2902 let temp_dir = TempDir::new().unwrap();
2903 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2904
2905 let mut writer = FeatherWriter::new(
2906 temp_dir.path().to_path_buf(),
2907 clock,
2908 RotationConfig::NoRotation,
2909 None,
2910 None,
2911 );
2912
2913 let quote = QuoteTick::new(
2914 InstrumentId::from("AUD/USD.SIM"),
2915 Price::from("1.0"),
2916 Price::from("1.0"),
2917 Quantity::from("1000"),
2918 Quantity::from("1000"),
2919 UnixNanos::from(1000),
2920 UnixNanos::from(1000),
2921 );
2922
2923 writer.write(quote).unwrap();
2924 assert!(!writer.writers.is_empty());
2925
2926 writer.close().unwrap();
2927 assert!(writer.writers.is_empty());
2928 }
2929
2930 #[rstest]
2931 fn test_write_data_orderbook_deltas() {
2932 let temp_dir = TempDir::new().unwrap();
2933 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2934
2935 let mut writer = FeatherWriter::new(
2936 temp_dir.path().to_path_buf(),
2937 clock,
2938 RotationConfig::NoRotation,
2939 None,
2940 None,
2941 );
2942
2943 let instrument_id = InstrumentId::from("AUD/USD.SIM");
2944 let delta1 = OrderBookDelta::clear(
2945 instrument_id,
2946 0,
2947 UnixNanos::from(1000),
2948 UnixNanos::from(1000),
2949 );
2950 let delta2 = OrderBookDelta::clear(
2951 instrument_id,
2952 0,
2953 UnixNanos::from(2000),
2954 UnixNanos::from(2000),
2955 );
2956
2957 let deltas = OrderBookDeltas::new(instrument_id, vec![delta1, delta2]);
2958 writer
2960 .write_data(Data::BookDeltas(Box::new(deltas)))
2961 .unwrap();
2962 writer.flush().unwrap();
2963 }
2964
2965 #[rstest]
2966 fn feather_writer_is_send() {
2967 fn assert_send<T: Send>() {}
2968 assert_send::<FeatherWriter>();
2969 assert_send::<WriterClock>();
2970 }
2971
2972 #[rstest]
2973 fn writer_clock_test_source_reads_shared_atomic() {
2974 let shared = Arc::new(AtomicU64::new(7));
2975 let clock = WriterClock::Test(Arc::clone(&shared));
2976 assert_eq!(clock.timestamp_ns(), UnixNanos::from(7));
2977
2978 shared.store(42, Ordering::Relaxed);
2979 assert_eq!(clock.timestamp_ns(), UnixNanos::from(42));
2980 }
2981
2982 #[rstest]
2983 fn writer_clock_from_shared_clock_wires_test_clock() {
2984 let test_clock = Rc::new(RefCell::new(VirtualClock::new()));
2985 test_clock
2986 .borrow_mut()
2987 .advance_time(UnixNanos::from(42), true);
2988 let clock: Rc<RefCell<dyn Clock>> = test_clock.clone();
2989
2990 let (writer_clock, shared) = WriterClock::from_shared_clock(&clock);
2991 let shared = shared.expect("non-live clocks must return a shared atomic");
2992
2993 assert_eq!(writer_clock.timestamp_ns(), UnixNanos::from(42));
2995
2996 test_clock
2999 .borrow_mut()
3000 .advance_time(UnixNanos::from(99), true);
3001 shared.store(clock.borrow().timestamp_ns().as_u64(), Ordering::Relaxed);
3002 assert_eq!(writer_clock.timestamp_ns(), UnixNanos::from(99));
3003 }
3004
3005 #[rstest]
3006 fn writer_clock_from_shared_clock_live_clock_is_live() {
3007 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(LiveClock::new(None)));
3008
3009 let (writer_clock, shared) = WriterClock::from_shared_clock(&clock);
3010
3011 assert!(matches!(writer_clock, WriterClock::Live));
3012 assert!(shared.is_none());
3013 }
3014
3015 #[rstest]
3016 #[cfg(unix)]
3017 fn failed_seal_of_a_removed_file_reports_the_error_and_publishes_nothing() {
3018 let temp_dir = TempDir::new().unwrap();
3019 let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
3020
3021 let mut writer = FeatherWriter::new(
3022 temp_dir.path().to_path_buf(),
3023 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3024 RotationConfig::NoRotation,
3025 None,
3026 None,
3027 )
3028 .with_sealed_sender(sealed_tx);
3029 writer
3030 .write(QuoteTick::new(
3031 InstrumentId::from("AUD/USD.SIM"),
3032 Price::from("1.0"),
3033 Price::from("1.1"),
3034 Quantity::from("1000"),
3035 Quantity::from("1000"),
3036 UnixNanos::from(1_000),
3037 UnixNanos::from(1_000),
3038 ))
3039 .unwrap();
3040
3041 let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
3043 fs::remove_file(&partial[0]).unwrap();
3044
3045 let error = writer.close().unwrap_err();
3046
3047 assert_eq!(error.to_string(), "No such file or directory (os error 2)");
3048 assert!(writer.is_closed());
3049 assert!(feather_files(temp_dir.path(), FEATHER_EXTENSION).is_empty());
3050 assert_eq!(
3051 sealed_rx.try_iter().collect::<Vec<_>>(),
3052 Vec::<PathBuf>::new()
3053 );
3054 }
3055
3056 #[rstest]
3057 #[cfg(target_os = "linux")]
3058 fn failed_seal_recovers_the_batches_that_reached_the_file() {
3059 let temp_dir = TempDir::new().unwrap();
3060 let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
3061
3062 let mut writer = FeatherWriter::new(
3063 temp_dir.path().to_path_buf(),
3064 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3065 RotationConfig::NoRotation,
3066 None,
3067 Some(0),
3068 )
3069 .with_sealed_sender(sealed_tx);
3070
3071 writer.write(quote_at(1)).unwrap();
3072 writer.flush().unwrap();
3073 writer.write(quote_at(2)).unwrap();
3074
3075 let (path, file) = writer.writers.iter_mut().next().unwrap();
3077 let sealed = path.path.clone();
3078 let full = OpenOptions::new().write(true).open("/dev/full").unwrap();
3079 file.writer =
3080 StreamWriter::try_new(CountingWriter::new(BufWriter::new(full)), &file.schema).unwrap();
3081
3082 let error = writer.close().unwrap_err();
3083
3084 assert_eq!(
3085 error.to_string(),
3086 "Io error: No space left on device (os error 28)"
3087 );
3088 assert!(writer.is_closed());
3089 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3090 assert_eq!(read_quotes(&sealed), quotes_at(&[1]));
3091 assert_eq!(sealed_rx.try_iter().collect::<Vec<_>>(), vec![sealed]);
3092 }
3093
3094 #[rstest]
3095 #[case::nothing_flushed(false, vec![3])]
3096 #[case::batch_flushed(true, vec![1, 3])]
3097 #[cfg(target_os = "linux")]
3098 fn failed_flush_recovers_the_abandoned_file_and_next_write_opens_a_new_one(
3099 #[case] flushed: bool,
3100 #[case] expected: Vec<u64>,
3101 ) {
3102 let temp_dir = TempDir::new().unwrap();
3103 let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
3104
3105 let mut writer = FeatherWriter::new(
3106 temp_dir.path().to_path_buf(),
3107 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3108 RotationConfig::NoRotation,
3109 None,
3110 Some(0),
3111 )
3112 .with_sealed_sender(sealed_tx);
3113
3114 writer.write(quote_at(1)).unwrap();
3115
3116 if flushed {
3117 writer.flush().unwrap();
3118 }
3119
3120 writer.write(quote_at(2)).unwrap();
3121
3122 let (abandoned, file) = writer.writers.iter_mut().next().unwrap();
3124 let abandoned = abandoned.clone();
3125 let full = OpenOptions::new().write(true).open("/dev/full").unwrap();
3126 file.writer =
3127 StreamWriter::try_new(CountingWriter::new(BufWriter::new(full)), &file.schema).unwrap();
3128
3129 let error = writer.flush().unwrap_err();
3130 writer.write(quote_at(3)).unwrap();
3131 writer.close().unwrap();
3132
3133 let mut recovered = feather_files(temp_dir.path(), FEATHER_EXTENSION)
3134 .iter()
3135 .flat_map(|path| read_quotes(path))
3136 .collect::<Vec<_>>();
3137 recovered.sort_by_key(HasTsInit::ts_init);
3138
3139 assert_eq!(
3142 error.to_string(),
3143 "Io error: No space left on device (os error 28)"
3144 );
3145 assert_eq!(abandoned.path.exists(), flushed);
3146 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3147 assert_eq!(recovered, quotes_at(&expected));
3148 let mut notified = sealed_rx.try_iter().collect::<Vec<_>>();
3149 notified.sort();
3150 assert_eq!(notified, feather_files(temp_dir.path(), FEATHER_EXTENSION));
3151 }
3152
3153 #[rstest]
3154 #[case::end_of_stream_marker(0, &[], vec![1, 2, 3], "")]
3155 #[case::no_end_of_stream_marker(8, &[], vec![1, 2, 3], "")]
3156 #[case::stray_bytes(8, &[0xFF, 0xFF], vec![1, 2, 3], "")]
3157 #[case::truncated_last_batch(9, &[], vec![1, 2], TORN_TAIL)]
3158 fn recovery_seals_complete_batches_and_drops_the_rest(
3159 #[case] cut: usize,
3160 #[case] stray: &[u8],
3161 #[case] expected: Vec<u64>,
3162 #[case] cause: &str,
3163 ) {
3164 let temp_dir = TempDir::new().unwrap();
3165 let directory = temp_dir.path().join("recover");
3166 let partial = write_partial_file(&directory, &[1, 2, 3]);
3167 let mut staged = fs::read(&partial).unwrap();
3168 staged.truncate(staged.len() - cut);
3169 staged.extend_from_slice(stray);
3170 fs::write(&partial, &staged).unwrap();
3171
3172 let reference = fs::read(write_partial_file(
3174 &temp_dir.path().join("reference"),
3175 &expected,
3176 ))
3177 .unwrap();
3178 let complete = &reference[..reference.len() - 8];
3179
3180 let warnings = recover_with_warnings(&directory);
3181
3182 let sealed = partial.with_extension("");
3183 let dropped = staged.len() - complete.len();
3184
3185 let tail = if dropped == 0 {
3186 String::new()
3187 } else {
3188 format!(", dropped {dropped} trailing byte(s){cause}")
3189 };
3190
3191 assert!(!partial.exists());
3192 assert_eq!(fs::read(&sealed).unwrap(), complete);
3193 assert_eq!(read_quotes(&sealed), quotes_at(&expected));
3194 assert_eq!(
3195 warnings,
3196 vec![format!(
3197 "Recovered {} record batch(es) into Feather file {}{tail}",
3198 expected.len(),
3199 sealed.display()
3200 )],
3201 );
3202 }
3203
3204 #[rstest]
3205 #[case::schema_only(0, "")]
3206 #[case::truncated_schema(1, TORN_TAIL)]
3207 fn recovery_removes_a_file_without_a_complete_batch(#[case] cut: u64, #[case] cause: &str) {
3208 let temp_dir = TempDir::new().unwrap();
3209 let partial = temp_dir.path().join("quotes_1.feather.partial");
3210 let file = File::create(&partial).unwrap();
3211 StreamWriter::try_new(&file, &QuoteTick::get_schema(None)).unwrap();
3212 file.set_len(file.metadata().unwrap().len() - cut).unwrap();
3213 drop(file);
3214
3215 let warnings = recover_with_warnings(temp_dir.path());
3216
3217 assert_eq!(fs::read_dir(temp_dir.path()).unwrap().count(), 0);
3218 assert_eq!(
3219 warnings,
3220 vec![format!(
3221 "Removed Feather file {} with no complete record batch{cause}",
3222 partial.display()
3223 )],
3224 );
3225 }
3226
3227 #[rstest]
3228 fn recovery_leaves_a_file_with_an_unreadable_tail() {
3229 let temp_dir = TempDir::new().unwrap();
3230 let partial = write_partial_file(temp_dir.path(), &[1, 2]);
3231
3232 let mut staged = fs::read(&partial).unwrap();
3234 staged.truncate(staged.len() - 8);
3235 staged.extend_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF, 0xF0, 0xFF, 0xFF, 0xFF]);
3236 fs::write(&partial, &staged).unwrap();
3237
3238 let warnings = recover_with_warnings(temp_dir.path());
3239
3240 assert_eq!(fs::read(&partial).unwrap(), staged);
3241 assert_eq!(
3242 warnings,
3243 vec![format!(
3244 "Failed to recover Feather file {}: Parser error: Invalid metadata length: -16",
3245 partial.display()
3246 )],
3247 );
3248 }
3249
3250 #[rstest]
3251 #[cfg(unix)]
3252 fn recovery_leaves_partial_file_symlinks_untouched() {
3253 let temp_dir = TempDir::new().unwrap();
3254 let outside = TempDir::new().unwrap();
3255 let partial = write_partial_file(outside.path(), &[1, 2]);
3256 let original = fs::read(&partial).unwrap();
3257 let link = temp_dir.path().join("quotes_0.feather.partial");
3258 std::os::unix::fs::symlink(&partial, &link).unwrap();
3259
3260 recover_partial_feather_files(temp_dir.path());
3261
3262 assert_eq!(fs::read(&partial).unwrap(), original);
3263 assert_eq!(fs::read_link(&link).unwrap(), partial);
3264 assert!(!link.with_extension("").exists());
3265 assert!(!partial.with_extension("").exists());
3266 }
3267
3268 #[rstest]
3269 #[cfg(unix)]
3270 fn recovery_continues_after_a_file_it_cannot_recover() {
3271 let temp_dir = TempDir::new().unwrap();
3272 let run = temp_dir.path().join("run");
3273 let partial = write_partial_file(&run, &[1]);
3274
3275 let unrecoverable = temp_dir.path().join("bad.feather.partial");
3276 std::os::unix::fs::symlink(&run, &unrecoverable).unwrap();
3277
3278 let warnings = recover_with_warnings(temp_dir.path());
3279
3280 let sealed = partial.with_extension("");
3281 assert_eq!(read_quotes(&sealed), quotes_at(&[1]));
3282 assert_eq!(
3283 warnings,
3284 vec![
3285 format!(
3286 "Failed to recover Feather file {}: Partial Feather path must be a regular file: {}",
3287 unrecoverable.display(),
3288 unrecoverable.display(),
3289 ),
3290 format!(
3291 "Recovered 1 record batch(es) into Feather file {}, dropped 8 trailing byte(s)",
3292 sealed.display()
3293 ),
3294 ],
3295 );
3296 }
3297
3298 #[rstest]
3299 fn recovery_leaves_open_and_empty_files() {
3300 let temp_dir = TempDir::new().unwrap();
3301
3302 let mut writer = FeatherWriter::new(
3303 temp_dir.path().join("open"),
3304 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3305 RotationConfig::NoRotation,
3306 None,
3307 Some(0),
3308 );
3309 writer.write(quote_at(1)).unwrap();
3310 writer.flush().unwrap();
3311
3312 let open = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).remove(0);
3314 let open_len = fs::metadata(&open).unwrap().len();
3315 let empty = temp_dir.path().join("empty.feather.partial");
3316 File::create(&empty).unwrap();
3317
3318 recover_partial_feather_files(temp_dir.path());
3319
3320 let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
3321 let open_len_after = fs::metadata(&open).unwrap().len();
3322 writer.close().unwrap();
3323
3324 assert_eq!(partial, vec![empty, open.clone()]);
3325 assert_eq!(open_len_after, open_len);
3326 assert_eq!(read_quotes(&open.with_extension("")), quotes_at(&[1]));
3327 }
3328
3329 #[rstest]
3330 fn size_rotation_in_mixed_identifier_batch_seals_the_shared_file_once() {
3331 let temp_dir = TempDir::new().unwrap();
3332
3333 let mut writer = FeatherWriter::new(
3334 temp_dir.path().to_path_buf(),
3335 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3336 RotationConfig::Size { max_size: 1 },
3337 None,
3338 None,
3339 );
3340
3341 let quotes = ["AUD/USD.SIM", "EUR/USD.SIM"]
3342 .map(|instrument_id| {
3343 QuoteTick::new(
3344 InstrumentId::from(instrument_id),
3345 Price::from("1.0"),
3346 Price::from("1.1"),
3347 Quantity::from("1000"),
3348 Quantity::from("1000"),
3349 UnixNanos::from(1),
3350 UnixNanos::from(1),
3351 )
3352 })
3353 .to_vec();
3354
3355 writer.write_batch(quotes).unwrap();
3358
3359 assert_eq!(writer.writers.len(), 1);
3360 assert!(feather_files(temp_dir.path(), FEATHER_EXTENSION).is_empty());
3361
3362 writer.close().unwrap();
3363
3364 let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
3365 assert_eq!(sealed.len(), 1);
3366 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3367 assert_eq!(
3368 identifiers_from_record_batches(&read_feather_batches(&sealed[0])).unwrap(),
3369 vec!["AUD/USD.SIM".to_string(), "EUR/USD.SIM".to_string()],
3370 );
3371 }
3372
3373 #[rstest]
3376 fn size_rotation_seals_each_full_file_before_the_next_later_row() {
3377 let temp_dir = TempDir::new().unwrap();
3378 let shared_clock = Arc::new(AtomicU64::new(0));
3379
3380 let mut writer = FeatherWriter::new(
3381 temp_dir.path().to_path_buf(),
3382 WriterClock::Test(Arc::clone(&shared_clock)),
3383 RotationConfig::Size { max_size: 1 },
3384 None,
3385 Some(1),
3386 );
3387
3388 for ts in [1_000, 2_000] {
3389 shared_clock.store(ts * 1_000_000, Ordering::Relaxed);
3390 writer
3391 .write(QuoteTick::new(
3392 InstrumentId::from("AUD/USD.SIM"),
3393 Price::from("1.0"),
3394 Price::from("1.0"),
3395 Quantity::from("1000"),
3396 Quantity::from("1000"),
3397 UnixNanos::from(ts),
3398 UnixNanos::from(ts),
3399 ))
3400 .unwrap();
3401 }
3402
3403 assert_eq!(sealed_timestamps(temp_dir.path()), vec![vec![1_000]]);
3404 assert_eq!(writer.writers.len(), 1);
3405
3406 writer.close().unwrap();
3407
3408 assert!(writer.writers.is_empty());
3409 assert_eq!(
3410 sealed_timestamps(temp_dir.path()),
3411 vec![vec![1_000], vec![2_000]]
3412 );
3413 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3414 }
3415
3416 #[rstest]
3417 #[case(
3418 "FeatherMissingTimestamp",
3419 Schema::empty(),
3420 "registered without an Arrow schema containing ts_init"
3421 )]
3422 #[case(
3423 "FeatherLegacyTimestamp",
3424 Schema::new(vec![Field::new("ts_init", DataType::UInt64, false)]),
3425 "registered with ts_init as UInt64",
3426 )]
3427 fn test_custom_writer_path_rejects_unqueryable_schema(
3428 #[case] type_name: &str,
3429 #[case] schema: Schema,
3430 #[case] expected_error: &str,
3431 ) {
3432 nautilus_model::data::registry::ensure_arrow_registered(
3433 type_name,
3434 Arc::new(schema),
3435 Box::new(|_| unreachable!("writer creation does not encode data")),
3436 Box::new(|_, _| unreachable!("writer creation does not decode data")),
3437 )
3438 .unwrap();
3439
3440 let temp_dir = TempDir::new().unwrap();
3441
3442 let mut writer = FeatherWriter::new(
3443 temp_dir.path().to_path_buf(),
3444 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3445 RotationConfig::NoRotation,
3446 None,
3447 None,
3448 );
3449
3450 for _ in 0..2 {
3451 let error = writer.get_writer_path_custom(type_name).unwrap_err();
3452
3453 assert!(error.to_string().contains(expected_error));
3454 assert!(writer.writers.is_empty());
3455 assert!(writer.path_sequences.is_empty());
3456 }
3457 }
3458
3459 #[rstest]
3460 #[cfg(feature = "python")]
3461 fn test_write_custom_data_round_trip() {
3462 use std::sync::Arc;
3463
3464 use nautilus_model::{
3465 data::{CustomData, Data, DataType},
3466 identifiers::InstrumentId,
3467 };
3468 use nautilus_serialization::{
3469 arrow::custom::CustomDataDecoder, ensure_custom_data_registered,
3470 };
3471
3472 use crate::test_data::RustTestCustomData;
3473
3474 ensure_custom_data_registered::<RustTestCustomData>();
3475
3476 let temp_dir = TempDir::new().unwrap();
3477 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
3478
3479 let mut writer = FeatherWriter::new(
3480 temp_dir.path().to_path_buf(),
3481 clock,
3482 RotationConfig::NoRotation,
3483 None,
3484 None,
3485 );
3486
3487 let instrument_id = InstrumentId::from("RUST.TEST");
3488 let data_type = DataType::new("RustTestCustomData", None, Some(instrument_id.to_string()));
3489
3490 let original = RustTestCustomData {
3491 instrument_id,
3492 value: 1.23,
3493 flag: true,
3494 ts_event: UnixNanos::from(1000),
3495 ts_init: UnixNanos::from(1000),
3496 };
3497
3498 let custom = CustomData::new(Arc::new(original.clone()), data_type);
3499
3500 writer
3501 .write_data(Data::Custom(custom))
3502 .expect("write_data CustomData");
3503 writer.close().expect("close");
3504
3505 let prefix = temp_dir
3506 .path()
3507 .join("data")
3508 .join("custom")
3509 .join("RustTestCustomData");
3510 let files = feather_files(&prefix, FEATHER_EXTENSION);
3511 let batch = read_feather_batches(&files[0]).remove(0);
3512 let metadata = batch.schema().metadata().clone();
3513 let decoded =
3514 CustomDataDecoder::decode_data_batch(&metadata, batch).expect("decode_data_batch");
3515 assert_eq!(decoded.len(), 1);
3516
3517 if let Data::Custom(decoded_custom) = &decoded[0] {
3518 assert_eq!(decoded_custom.data_type.type_name(), "RustTestCustomData");
3519 let rust: &RustTestCustomData = decoded_custom
3520 .data
3521 .as_any()
3522 .downcast_ref::<RustTestCustomData>()
3523 .expect("RustTestCustomData");
3524 assert_eq!(rust, &original);
3525 } else {
3526 panic!("Expected Data::Custom");
3527 }
3528 }
3529
3530 #[rstest]
3531 #[cfg(feature = "python")]
3532 fn test_write_custom_data_reuses_writer_until_rotation() {
3533 use nautilus_model::data::{CustomData, DataType};
3534 use nautilus_serialization::ensure_custom_data_registered;
3535
3536 use crate::test_data::RustTestCustomData;
3537
3538 ensure_custom_data_registered::<RustTestCustomData>();
3539 let temp_dir = TempDir::new().unwrap();
3540
3541 let mut writer = FeatherWriter::new(
3542 temp_dir.path().to_path_buf(),
3543 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3544 RotationConfig::NoRotation,
3545 None,
3546 None,
3547 );
3548 let instrument_id = InstrumentId::from("RUST.TEST");
3549 let data_type = DataType::new("RustTestCustomData", None, Some(instrument_id.to_string()));
3550
3551 for (ts, value) in [(1_000, 1_000.0), (2_000, 2_000.0)] {
3552 writer
3553 .write_data(Data::Custom(CustomData::new(
3554 Arc::new(RustTestCustomData {
3555 instrument_id,
3556 value,
3557 flag: true,
3558 ts_event: UnixNanos::from(ts),
3559 ts_init: UnixNanos::from(ts),
3560 }),
3561 data_type.clone(),
3562 )))
3563 .unwrap();
3564 }
3565
3566 assert_eq!(writer.get_current_file_info().len(), 1);
3567 writer.close().unwrap();
3568
3569 let prefix = temp_dir
3570 .path()
3571 .join("data")
3572 .join("custom")
3573 .join("RustTestCustomData");
3574 let files = feather_files(&prefix, FEATHER_EXTENSION);
3575 assert_eq!(files.len(), 1);
3576 let rows = read_feather_batches(&files[0])
3577 .iter()
3578 .map(RecordBatch::num_rows)
3579 .sum::<usize>();
3580 assert_eq!(rows, 2);
3581 }
3582}