1#![expect(
17 clippy::missing_errors_doc,
18 reason = "Feather writer public methods forward encoding and file IO errors directly"
19)]
20
21use std::{
22 any::Any,
23 borrow::Cow,
24 cell::RefCell,
25 collections::{BTreeMap, HashMap, HashSet},
26 fmt::Debug,
27 fs::{self, File, OpenOptions, TryLockError},
28 io::{self, BufReader, BufWriter, Seek, Write},
29 path::{Path, PathBuf},
30 rc::Rc,
31 sync::{
32 Arc,
33 atomic::{AtomicU64, Ordering},
34 mpsc::Sender,
35 },
36};
37
38use ahash::AHashMap;
39use datafusion::arrow::{
40 array::{ArrayRef, DictionaryArray},
41 compute::concat_batches,
42 datatypes::{DataType, Field, Int32Type, Schema, SchemaRef},
43 error::ArrowError,
44 ipc::{reader::StreamReader, writer::StreamWriter},
45 record_batch::RecordBatch,
46};
47use jiff::{
48 SignedDuration,
49 civil::Time,
50 tz::{AmbiguousOffset, TimeZone},
51};
52use nautilus_common::{clock::Clock, live::LiveClock};
53use nautilus_core::{DurationNanos, UnixNanos, time::nanos_since_unix_epoch};
54use nautilus_model::{
55 data::{
56 Bar, CustomData, CustomDataTrait, Data, DataBatch, DataType as CustomDataType,
57 FundingRateUpdate, IndexPriceUpdate, InstrumentStatus, MarkPriceUpdate, NautilusDataType,
58 OptionGreeks, OrderBookDelta, OrderBookDeltas, OrderBookDepth, QuoteTick, TradeTick,
59 close::InstrumentClose,
60 encode_custom_to_arrow, get_arrow_schema,
61 greeks::{GreeksData, PortfolioGreeks},
62 },
63 events::{
64 AccountState, OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled,
65 OrderDenied, OrderEmulated, OrderEventAny, OrderExpired, OrderFillVoided, OrderFilled,
66 OrderInitialized, OrderModifyDenied, OrderModifyRejected, OrderPendingCancel,
67 OrderPendingUpdate, OrderRejected, OrderReleased, OrderSnapshot, OrderSubmitted,
68 OrderTriggered, OrderUpdated, PositionAdjusted, PositionChanged, PositionClosed,
69 PositionEvent, PositionOpened, PositionSnapshot,
70 },
71 instruments::{InstrumentAny, NautilusInstrumentType},
72 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
73};
74use nautilus_serialization::{
75 arrow::{
76 EncodeToRecordBatch, KEY_TYPE_NAME, catalog_identifier_from_metadata,
77 record_batch_with_identifier_column, schema_with_identifier_column,
78 },
79 ensure_custom_data_registered,
80};
81
82use crate::{
83 catalog::{
84 session::DEFAULT_DATA_BATCH_CHUNK_SIZE,
85 types::{
86 CatalogDataType, CatalogFamily, data_path_prefix, instrument_path_prefix,
87 record_path_prefix,
88 },
89 },
90 common::custom::{
91 augment_batch_with_data_type_column, schema_with_data_type_column,
92 validate_custom_catalog_schema,
93 },
94 writer::{
95 filter::{WriterRecordFilter, catalog_family},
96 subscription::StreamingSinkSubscription,
97 traits::{StreamingDataSink, StreamingSink},
98 },
99};
100
101pub(crate) type FeatherWriteCommand =
102 Box<dyn FnOnce(&mut FeatherWriter) -> Result<(), Box<dyn std::error::Error>> + Send + 'static>;
103
104#[expect(
105 clippy::needless_pass_by_value,
106 reason = "map_err transfers ownership of the boxed writer error"
107)]
108pub(crate) fn feather_error(e: Box<dyn std::error::Error>) -> anyhow::Error {
109 anyhow::anyhow!("{e}")
110}
111
112macro_rules! define_builtin_data_batch_dispatch {
113 ($(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?) => {
114 fn write_builtin_data_batch(
115 writer: &mut FeatherWriter,
116 batch: &DataBatch,
117 ) -> Option<Result<(), Box<dyn std::error::Error>>> {
118 match batch {
119 $(
120 DataBatch::$batch(data) => Some(writer.write_batch(data.as_ref().to_vec())),
121 )+
122 _ => None,
123 }
124 }
125 };
126}
127
128nautilus_model::for_each_data_type!(define_builtin_data_batch_dispatch);
129
130pub(crate) const NAUTILUS_ARROW_METADATA_ID_COLUMN: &str = "nautilus_metadata_id";
131pub(crate) const NAUTILUS_ARROW_METADATA_JSON_COLUMN: &str = "nautilus_metadata_json";
132
133pub(crate) fn staged_metadata_data_type() -> DataType {
135 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
136}
137
138pub(crate) fn staged_metadata_array<'a>(values: impl IntoIterator<Item = &'a str>) -> ArrayRef {
139 Arc::new(values.into_iter().collect::<DictionaryArray<Int32Type>>())
140}
141
142#[derive(Clone, Debug, Eq, PartialEq)]
143struct StagedArrowMetadataRow {
144 metadata_id: String,
145 metadata_json: String,
146}
147
148#[derive(Debug, Clone, PartialEq, Eq, Hash)]
150pub struct FileWriterPath {
151 path: PathBuf,
152 data_type: CatalogDataType,
153}
154
155#[derive(Clone, Debug)]
161pub enum WriterClock {
162 Live,
164 Test(Arc<AtomicU64>),
166}
167
168impl WriterClock {
169 #[must_use]
171 pub fn timestamp_ns(&self) -> UnixNanos {
172 match self {
173 Self::Live => UnixNanos::from(nanos_since_unix_epoch()),
174 Self::Test(shared) => UnixNanos::from(shared.load(Ordering::Relaxed)),
175 }
176 }
177
178 #[must_use]
185 pub fn from_shared_clock(clock: &Rc<RefCell<dyn Clock>>) -> (Self, Option<Arc<AtomicU64>>) {
186 let borrowed = clock.borrow();
187 let any_ref: &dyn Any = &*borrowed;
188 if any_ref.downcast_ref::<LiveClock>().is_some() {
189 (Self::Live, None)
190 } else {
191 let shared = Arc::new(AtomicU64::new(borrowed.timestamp_ns().as_u64()));
192 (Self::Test(Arc::clone(&shared)), Some(shared))
193 }
194 }
195}
196
197fn record_batch_with_delta_staged_metadata(
198 batch: &RecordBatch,
199 metadata_rows: &[StagedArrowMetadataRow],
200) -> anyhow::Result<RecordBatch> {
201 anyhow::ensure!(
202 batch.num_rows() == metadata_rows.len(),
203 "Delta Feather staging metadata row count {} does not match batch row count {}",
204 metadata_rows.len(),
205 batch.num_rows()
206 );
207 anyhow::ensure!(
208 batch
209 .schema()
210 .index_of(NAUTILUS_ARROW_METADATA_ID_COLUMN)
211 .is_err(),
212 "Delta Feather staging schema already has {NAUTILUS_ARROW_METADATA_ID_COLUMN}"
213 );
214 anyhow::ensure!(
215 batch
216 .schema()
217 .index_of(NAUTILUS_ARROW_METADATA_JSON_COLUMN)
218 .is_err(),
219 "Delta Feather staging schema already has {NAUTILUS_ARROW_METADATA_JSON_COLUMN}"
220 );
221
222 let mut fields = batch
223 .schema()
224 .fields()
225 .iter()
226 .map(|field| {
227 Arc::new(Field::new(
228 field.name().clone(),
229 field.data_type().clone(),
230 field.is_nullable(),
231 ))
232 })
233 .collect::<Vec<_>>();
234
235 fields.push(Arc::new(Field::new(
236 NAUTILUS_ARROW_METADATA_ID_COLUMN,
237 staged_metadata_data_type(),
238 false,
239 )));
240 fields.push(Arc::new(Field::new(
241 NAUTILUS_ARROW_METADATA_JSON_COLUMN,
242 staged_metadata_data_type(),
243 false,
244 )));
245
246 let mut columns = batch.columns().to_vec();
247 columns.push(staged_metadata_array(
248 metadata_rows.iter().map(|row| row.metadata_id.as_str()),
249 ));
250 columns.push(staged_metadata_array(
251 metadata_rows.iter().map(|row| row.metadata_json.as_str()),
252 ));
253
254 Ok(RecordBatch::try_new(
255 Arc::new(Schema::new(fields)),
256 columns,
257 )?)
258}
259
260fn arrow_metadata_row(
261 metadata: &HashMap<String, String>,
262 fields: &arrow::datatypes::Fields,
263) -> anyhow::Result<StagedArrowMetadataRow> {
264 let schema_metadata = metadata
265 .iter()
266 .map(|(key, value)| (key.clone(), value.clone()))
267 .collect::<BTreeMap<_, _>>();
268
269 let field_metadata = fields
270 .iter()
271 .filter(|field| !field.metadata().is_empty())
272 .map(|field| {
273 (
274 field.name().clone(),
275 field
276 .metadata()
277 .iter()
278 .map(|(key, value)| (key.clone(), value.clone()))
279 .collect::<BTreeMap<_, _>>(),
280 )
281 })
282 .collect::<BTreeMap<_, _>>();
283
284 let metadata_json = serde_json::to_string(&serde_json::json!({
285 "format_version": 1,
286 "schema_metadata": schema_metadata,
287 "field_metadata": field_metadata,
288 }))?;
289 let metadata_id = staged_metadata_id(&canonical_metadata_json(metadata)?);
290
291 Ok(StagedArrowMetadataRow {
292 metadata_id,
293 metadata_json,
294 })
295}
296
297pub(crate) fn canonical_metadata_json(
302 metadata: &HashMap<String, String>,
303) -> anyhow::Result<String> {
304 Ok(serde_json::to_string(
305 &metadata
306 .iter()
307 .map(|(key, value)| (key.clone(), value.clone()))
308 .collect::<BTreeMap<_, _>>(),
309 )?)
310}
311
312pub(crate) fn staged_metadata_id(metadata_json: &str) -> String {
313 format!("blake3:{}", blake3::hash(metadata_json.as_bytes()).to_hex())
314}
315
316pub(crate) const FEATHER_EXTENSION: &str = "feather";
318
319pub(crate) const FEATHER_PARTIAL_EXTENSION: &str = "feather.partial";
321
322struct FeatherFile {
332 writer: StreamWriter<CountingWriter<BufWriter<File>>>,
333 partial_path: PathBuf,
334 path: PathBuf,
335 schema: SchemaRef,
336 max_size: Option<u64>,
337 buffer: Vec<RecordBatch>,
338 buffer_rows: usize,
339 buffer_bytes: u64,
340}
341
342impl FeatherFile {
343 fn create(
344 path: &Path,
345 schema: &Schema,
346 rotation_config: &RotationConfig,
347 ) -> Result<Self, Box<dyn std::error::Error>> {
348 let partial_path = path.with_extension(FEATHER_PARTIAL_EXTENSION);
349 if let Some(parent) = path.parent() {
350 fs::create_dir_all(parent)?;
351 }
352
353 let file = OpenOptions::new()
354 .write(true)
355 .create_new(true)
356 .open(&partial_path)?;
357
358 file.lock().map_err(|e| {
360 format!(
361 "Failed to lock Feather file {}: {e}",
362 partial_path.display()
363 )
364 })?;
365
366 let writer = StreamWriter::try_new(CountingWriter::new(BufWriter::new(file)), schema)?;
367
368 let max_size = match rotation_config {
369 RotationConfig::Size { max_size } => Some(*max_size),
370 _ => None,
371 };
372
373 Ok(Self {
374 writer,
375 partial_path,
376 path: path.to_path_buf(),
377 schema: Arc::new(schema.clone()),
378 max_size,
379 buffer: Vec::new(),
380 buffer_rows: 0,
381 buffer_bytes: 0,
382 })
383 }
384
385 fn write_record_batch(&mut self, batch: &RecordBatch) -> Result<bool, ArrowError> {
387 let batch = if batch.schema() == self.schema {
388 batch.clone()
389 } else {
390 RecordBatch::try_new(self.schema.clone(), batch.columns().to_vec())?
391 };
392
393 self.buffer_rows += batch.num_rows();
394 self.buffer_bytes += batch.get_array_memory_size() as u64;
395 self.buffer.push(batch);
396
397 if self.buffer_rows >= DEFAULT_DATA_BATCH_CHUNK_SIZE {
398 self.write_buffer()?;
399 }
400
401 let Some(max_size) = self.max_size else {
402 return Ok(false);
403 };
404
405 if self.size() + self.buffer_bytes >= max_size {
407 self.write_buffer()?;
408 }
409
410 Ok(self.size() >= max_size)
411 }
412
413 fn write_buffer(&mut self) -> Result<(), ArrowError> {
414 if self.buffer.is_empty() {
415 return Ok(());
416 }
417
418 let batch = concat_batches(&self.schema, &self.buffer)?;
419 self.buffer.clear();
420 self.buffer_rows = 0;
421 self.buffer_bytes = 0;
422 self.writer.write(&batch)
423 }
424
425 fn size(&self) -> u64 {
426 self.writer.get_ref().bytes
427 }
428
429 fn flush(&mut self) -> Result<(), ArrowError> {
430 self.write_buffer()?;
431 self.writer.flush()
432 }
433
434 fn seal(mut self) -> Result<(), Box<dyn std::error::Error>> {
435 self.write_buffer()?;
436 self.writer.finish()?;
437 let file = self
438 .writer
439 .into_inner()?
440 .inner
441 .into_inner()
442 .map_err(io::IntoInnerError::into_error)?;
443 file.sync_all()?;
444
445 fs::rename(&self.partial_path, &self.path)?;
447 Ok(())
448 }
449}
450
451struct CountingWriter<W> {
453 inner: W,
454 bytes: u64,
455}
456
457impl<W> CountingWriter<W> {
458 const fn new(inner: W) -> Self {
459 Self { inner, bytes: 0 }
460 }
461}
462
463impl<W: Write> Write for CountingWriter<W> {
464 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
465 let written = self.inner.write(buf)?;
466 self.bytes += written as u64;
467 Ok(written)
468 }
469
470 fn flush(&mut self) -> io::Result<()> {
471 self.inner.flush()
472 }
473}
474
475#[derive(Debug, Default)]
477struct PendingIo {
478 rotate_paths: Vec<FileWriterPath>,
479 flush_due: bool,
480}
481
482#[derive(Debug, Clone)]
484pub enum RotationConfig {
485 Size {
487 max_size: u64,
489 },
490 Interval {
492 interval_ns: u64,
494 },
495 ScheduledDates {
497 interval_ns: u64,
499 rotation_time: UnixNanos,
501 rotation_timezone: TimeZone,
503 },
504 NoRotation,
506}
507
508pub struct FeatherWriter {
530 directory: PathBuf,
531 clock: WriterClock,
532 rotation_config: RotationConfig,
533 included_types: Option<HashSet<CatalogDataType>>,
534 record_filter: Option<WriterRecordFilter>,
535 writers: HashMap<FileWriterPath, FeatherFile>,
536 path_sequences: HashMap<CatalogDataType, (UnixNanos, u64)>,
537 sealed_tx: Option<Sender<PathBuf>>,
538 next_rotation_times: HashMap<FileWriterPath, UnixNanos>,
539 flush_interval_ms: u64,
540 last_flush_ns: UnixNanos,
541 pending_write_error: Option<String>,
542}
543
544impl FeatherWriter {
545 #[must_use]
547 pub fn new(
548 directory: PathBuf,
549 clock: WriterClock,
550 rotation_config: RotationConfig,
551 included_types: Option<HashSet<CatalogDataType>>,
552 flush_interval_ms: Option<u64>,
553 ) -> Self {
554 let flush_interval_ms = flush_interval_ms.unwrap_or(1000); let last_flush_ns = clock.timestamp_ns();
556
557 Self {
558 directory,
559 clock,
560 rotation_config,
561 included_types,
562 record_filter: None,
563 writers: HashMap::new(),
564 path_sequences: HashMap::new(),
565 sealed_tx: None,
566 next_rotation_times: HashMap::new(),
567 flush_interval_ms,
568 last_flush_ns,
569 pending_write_error: None,
570 }
571 }
572
573 #[must_use]
575 pub fn with_record_filter(mut self, record_filter: Option<WriterRecordFilter>) -> Self {
576 self.record_filter = record_filter;
577 self
578 }
579
580 pub(crate) fn with_sealed_sender(mut self, sealed_tx: Sender<PathBuf>) -> Self {
581 self.sealed_tx = Some(sealed_tx);
582 self
583 }
584
585 pub fn write<T>(&mut self, data: T) -> Result<(), Box<dyn std::error::Error>>
590 where
591 T: EncodeToRecordBatch + CatalogFamily + 'static,
592 {
593 let metadata = T::metadata(&data);
594 let identifier = catalog_identifier_from_metadata(&metadata);
595 let data_type = Self::staged_data_type::<T>(&metadata)?;
596
597 if !self.should_write_record(&data_type, identifier.as_deref()) {
598 return Ok(());
599 }
600
601 let path = self.get_writer_path(data_type);
602
603 if !self.writers.contains_key(&path) {
605 self.create_writer::<T>(path.clone(), &data)?;
606 }
607
608 let batch = T::encode_batch(&metadata, &[data])?;
610 let metadata_row = arrow_metadata_row(&metadata, batch.schema().fields())?;
611 let batch = record_batch_with_identifier_column(batch, identifier.as_deref())?;
612 let batch =
613 record_batch_with_delta_staged_metadata(&batch, std::slice::from_ref(&metadata_row))?;
614
615 let mut pending = PendingIo::default();
617
618 self.stage_batch_write(path, &batch, &mut pending)?;
619 pending.flush_due = self.flush_is_due();
620
621 self.complete_pending_io(&pending)
622 }
623
624 pub fn write_batch<T>(&mut self, data: Vec<T>) -> Result<(), Box<dyn std::error::Error>>
634 where
635 T: EncodeToRecordBatch + CatalogFamily + 'static,
636 {
637 if data.is_empty() || !self.should_write_family(&T::catalog_family()) {
638 return Ok(());
639 }
640
641 let mut groups: AHashMap<Option<String>, (CatalogDataType, Vec<T>)> = AHashMap::new();
642
643 for item in data {
644 let metadata = T::metadata(&item);
645 let identifier = catalog_identifier_from_metadata(&metadata);
646 let data_type = Self::staged_data_type::<T>(&metadata)?;
647
648 if !self.should_write_record(&data_type, identifier.as_deref()) {
649 continue;
650 }
651
652 groups
653 .entry(identifier)
654 .or_insert_with(|| (data_type, Vec::new()))
655 .1
656 .push(item);
657 }
658
659 if groups.is_empty() {
660 return Ok(());
661 }
662
663 let mut pending = PendingIo::default();
664
665 for (data_type, group) in groups.into_values() {
666 let path = self.get_writer_path(data_type);
667 let metadata = T::chunk_metadata(&group);
668
669 if !self.writers.contains_key(&path) {
670 self.create_writer_with_metadata::<T>(path.clone(), metadata.clone())?;
671 }
672
673 let identifier = catalog_identifier_from_metadata(&metadata);
674 let batch = T::encode_batch(&metadata, &group)?;
675 let metadata_rows = group
676 .iter()
677 .map(T::metadata)
678 .map(|metadata| arrow_metadata_row(&metadata, batch.schema().fields()))
679 .collect::<anyhow::Result<Vec<_>>>()?;
680 let batch = record_batch_with_identifier_column(batch, identifier.as_deref())?;
681 let batch = record_batch_with_delta_staged_metadata(&batch, &metadata_rows)?;
682
683 self.stage_batch_write(path, &batch, &mut pending)?;
684 }
685
686 pending.flush_due = self.flush_is_due();
687
688 self.complete_pending_io(&pending)
689 }
690
691 fn staged_data_type<T: CatalogFamily>(
693 metadata: &HashMap<String, String>,
694 ) -> Result<CatalogDataType, Box<dyn std::error::Error>> {
695 let family = T::catalog_family();
696 if family != CatalogDataType::Data(NautilusDataType::Instrument) {
697 return Ok(family);
698 }
699
700 let class = metadata
701 .get(KEY_TYPE_NAME)
702 .ok_or("Instrument metadata has no type_name")?
703 .parse::<NautilusInstrumentType>()?;
704 Ok(CatalogDataType::Instrument(class))
705 }
706
707 fn stage_batch_write(
708 &mut self,
709 path: FileWriterPath,
710 batch: &RecordBatch,
711 pending: &mut PendingIo,
712 ) -> Result<(), Box<dyn std::error::Error>> {
713 let Some(file) = self.writers.get_mut(&path) else {
714 return Ok(());
715 };
716
717 let should_rotate = match file.write_record_batch(batch) {
718 Ok(should_rotate) => should_rotate,
719 Err(e) => {
720 self.abandon_file(&path);
721 return Err(e.into());
722 }
723 };
724
725 if should_rotate || self.check_scheduled_rotation(&path) {
726 pending.rotate_paths.push(path);
727 }
728
729 Ok(())
730 }
731
732 fn flush_is_due(&self) -> bool {
734 if self.flush_interval_ms == 0 {
735 return false; }
737
738 let now_ns = self.clock.timestamp_ns();
739 let elapsed_ms = now_ns.as_u64().saturating_sub(self.last_flush_ns.as_u64()) / 1_000_000;
740 elapsed_ms >= self.flush_interval_ms
741 }
742
743 fn complete_pending_io(
744 &mut self,
745 pending: &PendingIo,
746 ) -> Result<(), Box<dyn std::error::Error>> {
747 for path in &pending.rotate_paths {
748 self.seal_file(path)?;
749 }
750
751 if pending.flush_due {
752 self.flush()?;
753 }
754
755 Ok(())
756 }
757
758 fn check_scheduled_rotation(&mut self, path: &FileWriterPath) -> bool {
759 match &self.rotation_config {
760 RotationConfig::Interval { interval_ns } => {
761 let now = self.clock.timestamp_ns();
762 let next_rotation = self.next_rotation_times.get(path).copied();
763
764 match next_rotation {
765 None => {
766 self.next_rotation_times
767 .insert(path.clone(), now + DurationNanos::new(*interval_ns));
768 false
769 }
770 Some(next) if now >= next => {
771 self.next_rotation_times
772 .insert(path.clone(), now + DurationNanos::new(*interval_ns));
773 true
774 }
775 _ => false,
776 }
777 }
778 RotationConfig::ScheduledDates {
779 interval_ns,
780 rotation_time,
781 rotation_timezone,
782 } => {
783 let now = self.clock.timestamp_ns();
784 let next_rotation = self.next_rotation_times.get(path).copied();
785
786 match next_rotation {
787 None => {
788 let next = self.calculate_next_scheduled_rotation(
789 *rotation_time,
790 rotation_timezone,
791 *interval_ns,
792 );
793 self.next_rotation_times.insert(path.clone(), next);
794 false
795 }
796 Some(next) if now >= next => {
797 let next = self.calculate_next_scheduled_rotation(
798 *rotation_time,
799 rotation_timezone,
800 *interval_ns,
801 );
802 self.next_rotation_times.insert(path.clone(), next);
803 true
804 }
805 _ => false,
806 }
807 }
808 _ => false,
809 }
810 }
811
812 fn calculate_next_scheduled_rotation(
813 &self,
814 rotation_time: UnixNanos,
815 rotation_timezone: &TimeZone,
816 interval_ns: u64,
817 ) -> UnixNanos {
818 let now_utc = self.clock.timestamp_ns().to_datetime_utc();
819 let now_local = rotation_timezone.to_datetime(now_utc);
820
821 let rotation_time_secs = u32::try_from(*rotation_time / 1_000_000_000).unwrap_or(0);
822 let rotation_time_nanos = i32::try_from(*rotation_time % 1_000_000_000).unwrap_or(0);
823
824 let rotation_time = if rotation_time_secs < 86_400 {
825 Time::new(
826 i8::try_from(rotation_time_secs / 3_600).unwrap_or(0),
827 i8::try_from(rotation_time_secs % 3_600 / 60).unwrap_or(0),
828 i8::try_from(rotation_time_secs % 60).unwrap_or(0),
829 rotation_time_nanos,
830 )
831 .unwrap_or(Time::MIN)
832 } else {
833 Time::MIN
834 };
835
836 let local_rotation = now_local.date().to_datetime(rotation_time);
837 let ambiguous = rotation_timezone.to_ambiguous_timestamp(local_rotation);
838
839 let mut next_rotation = match ambiguous.offset() {
840 AmbiguousOffset::Gap { .. } => now_utc,
841 _ => ambiguous.earlier().unwrap_or(now_utc),
842 };
843
844 if next_rotation <= now_utc {
845 while next_rotation <= now_utc {
848 next_rotation += SignedDuration::from_nanos_i128(i128::from(interval_ns));
849 }
850 }
851
852 UnixNanos::from(u64::try_from(next_rotation.as_nanosecond()).unwrap_or(0))
853 }
854
855 fn seal_file(&mut self, path: &FileWriterPath) -> Result<(), Box<dyn std::error::Error>> {
858 self.next_rotation_times.remove(path);
859
860 let Some(file) = self.writers.remove(path) else {
861 return Ok(());
862 };
863
864 let partial_path = file.partial_path.clone();
865 let result = file
866 .seal()
867 .inspect_err(|_| recover_released_feather_file(&partial_path));
868 self.notify_sealed(path);
869 result
870 }
871
872 fn abandon_file(&mut self, path: &FileWriterPath) {
875 self.next_rotation_times.remove(path);
876
877 let Some(file) = self.writers.remove(path) else {
878 return;
879 };
880
881 let partial_path = file.partial_path.clone();
882 drop(file);
883
884 log::error!(
885 "Abandoned Feather file {} after a failed write",
886 path.path.display()
887 );
888
889 recover_released_feather_file(&partial_path);
890 self.notify_sealed(path);
891 }
892
893 fn notify_sealed(&self, path: &FileWriterPath) {
894 if let Some(sealed_tx) = &self.sealed_tx
895 && path.path.exists()
896 {
897 let _ = sealed_tx.send(path.path.clone());
898 }
899 }
900
901 fn create_writer<T>(
903 &mut self,
904 path: FileWriterPath,
905 data: &T,
906 ) -> Result<(), Box<dyn std::error::Error>>
907 where
908 T: EncodeToRecordBatch + CatalogFamily + 'static,
909 {
910 self.create_writer_with_metadata::<T>(path, T::metadata(data))
911 }
912
913 fn create_writer_with_metadata<T>(
918 &mut self,
919 path: FileWriterPath,
920 metadata: HashMap<String, String>,
921 ) -> Result<(), Box<dyn std::error::Error>>
922 where
923 T: EncodeToRecordBatch + CatalogFamily + 'static,
924 {
925 let schema = Self::schema_with_delta_staging_columns(&schema_with_identifier_column(
926 &T::get_schema(Some(metadata)),
927 ));
928
929 let file = FeatherFile::create(&path.path, &schema, &self.rotation_config)?;
930 self.writers.insert(path, file);
931 Ok(())
932 }
933
934 fn create_custom_writer(
936 &mut self,
937 path: FileWriterPath,
938 type_name: &str,
939 ) -> Result<(), Box<dyn std::error::Error>> {
940 if self.writers.contains_key(&path) {
941 return Ok(());
942 }
943
944 let base_schema = get_arrow_schema(type_name).ok_or_else(|| {
945 format!("Custom data type \"{type_name}\" is not registered for Arrow encoding")
946 })?;
947
948 let schema = schema_with_data_type_column(base_schema.as_ref(), type_name);
949
950 let schema =
951 Self::schema_with_delta_staging_columns(&schema_with_identifier_column(&schema));
952
953 let file = FeatherFile::create(&path.path, &schema, &self.rotation_config)
954 .map_err(|e| format!("Failed to create Feather file for custom {type_name}: {e}"))?;
955 self.writers.insert(path, file);
956 Ok(())
957 }
958
959 pub(crate) fn encode_custom_to_batch(
961 custom: &CustomData,
962 ) -> Result<RecordBatch, Box<dyn std::error::Error>> {
963 let type_name = custom.data.type_name();
964 let data_type_json = custom
965 .data_type
966 .to_persistence_json()
967 .map_err(|e| format!("Failed to serialize data_type for persistence: {e}"))?;
968 let dt_meta = custom.data_type.metadata_string_map();
969 let items: [Arc<dyn CustomDataTrait>; 1] = [Arc::clone(&custom.data)];
970
971 let batch = encode_custom_to_arrow(type_name, &items)
972 .map_err(|e| format!("Failed to encode custom data: {e}"))?
973 .ok_or_else(|| {
974 format!("Custom data type \"{type_name}\" is not registered for Arrow")
975 })?;
976
977 let batch = augment_batch_with_data_type_column(
978 &batch,
979 &data_type_json,
980 type_name,
981 dt_meta.as_ref(),
982 )
983 .map_err(|e| e.to_string())?;
984 Ok(batch)
985 }
986
987 fn schema_with_delta_staging_columns(schema: &Schema) -> Schema {
988 let mut fields = schema
989 .fields()
990 .iter()
991 .map(|field| {
992 Arc::new(Field::new(
993 field.name().clone(),
994 field.data_type().clone(),
995 field.is_nullable(),
996 ))
997 })
998 .collect::<Vec<_>>();
999
1000 if schema.index_of(NAUTILUS_ARROW_METADATA_ID_COLUMN).is_err() {
1001 fields.push(Arc::new(Field::new(
1002 NAUTILUS_ARROW_METADATA_ID_COLUMN,
1003 staged_metadata_data_type(),
1004 false,
1005 )));
1006 }
1007
1008 if schema
1009 .index_of(NAUTILUS_ARROW_METADATA_JSON_COLUMN)
1010 .is_err()
1011 {
1012 fields.push(Arc::new(Field::new(
1013 NAUTILUS_ARROW_METADATA_JSON_COLUMN,
1014 staged_metadata_data_type(),
1015 false,
1016 )));
1017 }
1018
1019 Schema::new(fields)
1020 }
1021
1022 pub fn flush(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1027 let paths = self.writers.keys().cloned().collect::<Vec<_>>();
1028
1029 for path in paths {
1030 if let Some(file) = self.writers.get_mut(&path)
1031 && let Err(e) = file.flush()
1032 {
1033 self.abandon_file(&path);
1034 return Err(e.into());
1035 }
1036 }
1037
1038 self.last_flush_ns = self.clock.timestamp_ns();
1039
1040 if let Some(error) = self.pending_write_error.take() {
1041 return Err(error.into());
1042 }
1043
1044 Ok(())
1045 }
1046
1047 pub fn seal(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1051 let paths = self.writers.keys().cloned().collect::<Vec<_>>();
1052 let mut first_error = None;
1053
1054 for path in paths {
1055 if let Err(e) = self.seal_file(&path) {
1056 log::error!("Failed to seal Feather file {}: {e}", path.path.display());
1057 first_error.get_or_insert(e);
1058 }
1059 }
1060
1061 match first_error {
1062 Some(e) => Err(e),
1063 None => Ok(()),
1064 }
1065 }
1066
1067 pub fn close(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1071 self.seal()?;
1072
1073 if let Some(error) = self.pending_write_error.take() {
1074 return Err(error.into());
1075 }
1076
1077 Ok(())
1078 }
1079
1080 #[must_use]
1082 pub fn is_closed(&self) -> bool {
1083 self.writers.is_empty()
1084 }
1085
1086 #[must_use]
1088 pub fn get_current_file_info(&self) -> HashMap<CatalogDataType, (u64, String)> {
1089 self.writers
1090 .iter()
1091 .map(|(path, file)| {
1092 (
1093 path.data_type.clone(),
1094 (file.size(), path.path.display().to_string()),
1095 )
1096 })
1097 .collect()
1098 }
1099
1100 #[must_use]
1104 pub fn get_next_rotation_time(&self, data_type: &CatalogDataType) -> Option<UnixNanos> {
1105 self.next_rotation_times
1106 .iter()
1107 .find(|(path, _)| path.data_type == *data_type)
1108 .map(|(_, &next)| next)
1109 }
1110
1111 fn should_write_family(&self, family: &CatalogDataType) -> bool {
1113 self.included_types
1114 .as_ref()
1115 .is_none_or(|included| included.contains(family))
1116 && self
1117 .record_filter
1118 .as_ref()
1119 .is_none_or(|filter| filter.contains(family))
1120 }
1121
1122 fn should_write_record(&self, data_type: &CatalogDataType, identifier: Option<&str>) -> bool {
1123 self.included_types
1124 .as_ref()
1125 .is_none_or(|included| included.contains(&catalog_family(data_type)))
1126 && self
1127 .record_filter
1128 .as_ref()
1129 .is_none_or(|filter| filter.allows(data_type, identifier))
1130 }
1131
1132 fn reserve_writer_path(&mut self, data_type: CatalogDataType) -> FileWriterPath {
1133 let now = self.clock.timestamp_ns();
1134 let (timestamp, first_sequence) = self
1135 .path_sequences
1136 .get(&data_type)
1137 .copied()
1138 .filter(|(timestamp, _)| *timestamp >= now)
1139 .unwrap_or((now, 0));
1140
1141 for sequence in first_sequence.. {
1142 let path = self.build_writer_path(&data_type, timestamp, sequence);
1143
1144 if path.exists() || path.with_extension(FEATHER_PARTIAL_EXTENSION).exists() {
1146 continue;
1147 }
1148
1149 self.path_sequences
1150 .insert(data_type.clone(), (timestamp, sequence + 1));
1151 return FileWriterPath { path, data_type };
1152 }
1153
1154 unreachable!("unbounded writer path sequence exhausted")
1155 }
1156
1157 fn build_writer_path(
1158 &self,
1159 data_type: &CatalogDataType,
1160 timestamp: UnixNanos,
1161 sequence: u64,
1162 ) -> PathBuf {
1163 let (directory, file_stem) = match data_type {
1164 CatalogDataType::Data(NautilusDataType::Custom { type_name }) => (
1165 self.directory.join("data").join("custom").join(type_name),
1166 Cow::Borrowed(type_name.as_str()),
1167 ),
1168 CatalogDataType::Data(data_type) => {
1169 let prefix = data_path_prefix(data_type);
1170 (self.directory.join(prefix.as_ref()), prefix)
1171 }
1172 CatalogDataType::Record(record_type) => {
1173 let prefix = record_path_prefix(record_type);
1174 (self.directory.join(prefix.as_ref()), prefix)
1175 }
1176 CatalogDataType::Instrument(class) => {
1177 let prefix = data_path_prefix(&NautilusDataType::Instrument);
1178 (
1179 self.directory
1180 .join(prefix.as_ref())
1181 .join(instrument_path_prefix(class)),
1182 prefix,
1183 )
1184 }
1185 };
1186
1187 directory.join(Self::timestamped_feather_file_name(
1188 &file_stem, timestamp, sequence,
1189 ))
1190 }
1191
1192 fn timestamped_feather_file_name(stem: &str, timestamp: UnixNanos, sequence: u64) -> String {
1193 if sequence == 0 {
1194 format!("{stem}_{timestamp}.{FEATHER_EXTENSION}")
1195 } else {
1196 format!("{stem}_{timestamp}-{sequence}.{FEATHER_EXTENSION}")
1197 }
1198 }
1199
1200 fn get_writer_path_custom(
1202 &mut self,
1203 type_name: &str,
1204 ) -> Result<FileWriterPath, Box<dyn std::error::Error>> {
1205 let data_type = CatalogDataType::from(NautilusDataType::Custom {
1206 type_name: type_name.to_string(),
1207 });
1208
1209 if let Some(existing) = self.writers.keys().find(|path| path.data_type == data_type) {
1210 return Ok(existing.clone());
1211 }
1212
1213 if let Some(schema) = get_arrow_schema(type_name) {
1214 validate_custom_catalog_schema(type_name, &schema)?;
1215 }
1216
1217 Ok(self.reserve_writer_path(data_type))
1218 }
1219
1220 fn get_writer_path(&mut self, data_type: CatalogDataType) -> FileWriterPath {
1222 if let Some(existing) = self.writers.keys().find(|path| path.data_type == data_type) {
1223 return existing.clone();
1224 }
1225
1226 self.reserve_writer_path(data_type)
1227 }
1228
1229 pub fn write_data(&mut self, data: Data) -> Result<(), Box<dyn std::error::Error>> {
1234 match data {
1235 Data::Instrument(instrument) => self.write(*instrument),
1236 Data::Quote(quote) => self.write(quote),
1237 Data::Trade(trade) => self.write(trade),
1238 Data::Bar(bar) => self.write(bar),
1239 Data::BookDelta(delta) => self.write(delta),
1240 Data::BookDepth(depth) => self.write(*depth),
1241 Data::IndexPrice(price) => self.write(price),
1242 Data::MarkPrice(price) => self.write(price),
1243 Data::FundingRate(funding) => self.write(funding),
1244 Data::InstrumentStatus(status) => self.write(status),
1245 Data::OptionGreeks(greeks) => self.write(greeks),
1246 Data::InstrumentClose(close) => self.write(close),
1247 Data::Custom(custom) => self.write_custom_data(&custom),
1248 Data::BookDeltas(deltas_api) => {
1249 self.write_batch(deltas_api.deltas.clone())
1251 }
1252 #[cfg(feature = "defi")]
1253 Data::Defi(_) => Err("Unsupported DeFi data variant for feather writes".into()),
1254 #[cfg(not(feature = "defi"))]
1255 #[allow(
1256 unreachable_patterns,
1257 reason = "DeFi variants can exist without this crate's defi feature"
1258 )]
1259 _ => Err("Unsupported DeFi data variant for feather writes".into()),
1260 }
1261 }
1262
1263 #[expect(
1269 clippy::needless_pass_by_value,
1270 reason = "the public writer API accepts ownership of each submitted data batch"
1271 )]
1272 pub fn write_data_batch(&mut self, data: Vec<Data>) -> Result<(), Box<dyn std::error::Error>> {
1273 for batch in DataBatch::from_data_vec_grouped(&data)? {
1274 match &batch {
1275 DataBatch::Custom(data) => {
1276 for custom in data.as_ref() {
1277 self.write_custom_data(custom)?;
1278 }
1279 }
1280 batch => write_builtin_data_batch(self, batch)
1281 .expect("built-in data batch dispatch is exhaustive")?,
1282 }
1283 }
1284
1285 Ok(())
1286 }
1287
1288 pub fn write_any_message(
1290 &mut self,
1291 message: &dyn Any,
1292 ) -> Result<bool, Box<dyn std::error::Error>> {
1293 let Some(command) = Self::write_command(message) else {
1294 return Ok(false);
1295 };
1296
1297 if let Err(e) = command(self) {
1298 self.record_write_error(e.to_string());
1299 return Err(e);
1300 }
1301
1302 Ok(true)
1303 }
1304
1305 pub(crate) fn record_write_error(&mut self, error: String) {
1306 self.pending_write_error.get_or_insert(error);
1307 }
1308
1309 pub(crate) fn write_command(message: &dyn Any) -> Option<FeatherWriteCommand> {
1310 macro_rules! try_write {
1311 ($message:expr, $type:ty) => {
1312 if let Some(value) = $message.downcast_ref::<$type>() {
1313 let value = value.clone();
1314 return Some(Box::new(move |writer| writer.write(value)));
1315 }
1316 };
1317 }
1318
1319 try_write!(message, QuoteTick);
1320 try_write!(message, TradeTick);
1321 try_write!(message, Bar);
1322 try_write!(message, OrderBookDelta);
1323 try_write!(message, OrderBookDepth);
1324 try_write!(message, IndexPriceUpdate);
1325 try_write!(message, MarkPriceUpdate);
1326 try_write!(message, FundingRateUpdate);
1327 try_write!(message, InstrumentStatus);
1328 try_write!(message, OptionGreeks);
1329 try_write!(message, InstrumentClose);
1330 try_write!(message, InstrumentAny);
1331 try_write!(message, AccountState);
1332 try_write!(message, OrderInitialized);
1333 try_write!(message, OrderDenied);
1334 try_write!(message, OrderEmulated);
1335 try_write!(message, OrderSubmitted);
1336 try_write!(message, OrderAccepted);
1337 try_write!(message, OrderRejected);
1338 try_write!(message, OrderPendingCancel);
1339 try_write!(message, OrderCanceled);
1340 try_write!(message, OrderCancelDenied);
1341 try_write!(message, OrderCancelRejected);
1342 try_write!(message, OrderExpired);
1343 try_write!(message, OrderTriggered);
1344 try_write!(message, OrderPendingUpdate);
1345 try_write!(message, OrderReleased);
1346 try_write!(message, OrderModifyDenied);
1347 try_write!(message, OrderModifyRejected);
1348 try_write!(message, OrderUpdated);
1349 try_write!(message, OrderFilled);
1350 try_write!(message, OrderFillVoided);
1351 try_write!(message, PositionOpened);
1352 try_write!(message, PositionChanged);
1353 try_write!(message, PositionClosed);
1354 try_write!(message, PositionAdjusted);
1355 try_write!(message, OrderSnapshot);
1356 try_write!(message, PositionSnapshot);
1357 try_write!(message, OrderStatusReport);
1358 try_write!(message, FillReport);
1359 try_write!(message, PositionStatusReport);
1360 try_write!(message, ExecutionMassStatus);
1361
1362 if let Some(greeks) = message.downcast_ref::<GreeksData>() {
1363 ensure_custom_data_registered::<GreeksData>();
1364
1365 let data_type = CustomDataType::new(
1366 GreeksData::type_name_static(),
1367 None,
1368 Some(greeks.instrument_id.to_string()),
1369 );
1370
1371 let custom = CustomData::new(Arc::new(greeks.clone()), data_type);
1372 return Some(Box::new(move |writer| writer.write_custom_data(&custom)));
1373 }
1374
1375 if let Some(portfolio) = message.downcast_ref::<PortfolioGreeks>() {
1376 ensure_custom_data_registered::<PortfolioGreeks>();
1377 let custom = CustomData::from_arc(Arc::new(portfolio.clone()));
1378 return Some(Box::new(move |writer| writer.write_custom_data(&custom)));
1379 }
1380
1381 if let Some(deltas) = message.downcast_ref::<OrderBookDeltas>() {
1382 let deltas = deltas.deltas.clone();
1383 return Some(Box::new(move |writer| writer.write_batch(deltas)));
1384 }
1385
1386 if let Some(data) = message.downcast_ref::<Data>() {
1387 let data = data.clone();
1388 return Some(Box::new(move |writer| writer.write_data(data)));
1389 }
1390
1391 if let Some(custom) = message.downcast_ref::<CustomData>() {
1392 let custom = custom.clone();
1393 return Some(Box::new(move |writer| {
1394 writer.write_data(Data::Custom(custom))
1395 }));
1396 }
1397
1398 if let Some(event) = message.downcast_ref::<OrderEventAny>() {
1399 let event = event.clone();
1400 return Some(Box::new(move |writer| writer.write_order_event(&event)));
1401 }
1402
1403 if let Some(event) = message.downcast_ref::<PositionEvent>() {
1404 let event = event.clone();
1405 return Some(Box::new(move |writer| writer.write_position_event(&event)));
1406 }
1407
1408 None
1409 }
1410
1411 fn write_order_event(
1412 &mut self,
1413 event: &OrderEventAny,
1414 ) -> Result<(), Box<dyn std::error::Error>> {
1415 match event {
1416 OrderEventAny::Initialized(event) => self.write(event.clone()),
1417 OrderEventAny::Denied(event) => self.write(*event),
1418 OrderEventAny::Emulated(event) => self.write(*event),
1419 OrderEventAny::Released(event) => self.write(*event),
1420 OrderEventAny::Submitted(event) => self.write(*event),
1421 OrderEventAny::Accepted(event) => self.write(*event),
1422 OrderEventAny::Rejected(event) => self.write(*event),
1423 OrderEventAny::Canceled(event) => self.write(*event),
1424 OrderEventAny::Expired(event) => self.write(*event),
1425 OrderEventAny::Triggered(event) => self.write(*event),
1426 OrderEventAny::PendingUpdate(event) => self.write(*event),
1427 OrderEventAny::PendingCancel(event) => self.write(*event),
1428 OrderEventAny::ModifyDenied(event) => self.write(*event),
1429 OrderEventAny::ModifyRejected(event) => self.write(*event),
1430 OrderEventAny::CancelDenied(event) => self.write(*event),
1431 OrderEventAny::CancelRejected(event) => self.write(*event),
1432 OrderEventAny::Updated(event) => self.write(*event),
1433 OrderEventAny::Filled(event) => self.write(event.clone()),
1434 OrderEventAny::FillVoided(event) => self.write(event.clone()),
1435 }
1436 }
1437
1438 fn write_position_event(
1439 &mut self,
1440 event: &PositionEvent,
1441 ) -> Result<(), Box<dyn std::error::Error>> {
1442 match event {
1443 PositionEvent::PositionOpened(event) => self.write(event.clone()),
1444 PositionEvent::PositionChanged(event) => self.write(event.clone()),
1445 PositionEvent::PositionClosed(event) => self.write(event.clone()),
1446 PositionEvent::PositionAdjusted(event) => self.write(*event),
1447 }
1448 }
1449
1450 fn write_custom_data(&mut self, custom: &CustomData) -> Result<(), Box<dyn std::error::Error>> {
1452 let batch = Self::encode_custom_to_batch(custom)?;
1453 self.write_custom_batch(custom, batch)
1454 }
1455
1456 pub(crate) fn write_custom_batch(
1457 &mut self,
1458 custom: &CustomData,
1459 mut batch: RecordBatch,
1460 ) -> Result<(), Box<dyn std::error::Error>> {
1461 let type_name = custom.data.type_name();
1462 let identifier = custom.data_type.identifier().map(String::from);
1463
1464 if !self.should_write_custom(type_name, identifier.as_deref()) {
1465 return Ok(());
1466 }
1467
1468 let path = self.get_writer_path_custom(type_name)?;
1469 if !self.writers.contains_key(&path) {
1470 self.create_custom_writer(path.clone(), type_name)?;
1471 }
1472
1473 let metadata_row = arrow_metadata_row(batch.schema().metadata(), batch.schema().fields())?;
1474 batch = record_batch_with_identifier_column(batch, custom.data_type.identifier())?;
1475 batch =
1476 record_batch_with_delta_staged_metadata(&batch, std::slice::from_ref(&metadata_row))?;
1477
1478 let mut pending = PendingIo::default();
1479
1480 self.stage_batch_write(path, &batch, &mut pending)?;
1481 pending.flush_due = self.flush_is_due();
1482
1483 self.complete_pending_io(&pending)
1484 }
1485
1486 pub(crate) fn should_write_custom(&self, type_name: &str, identifier: Option<&str>) -> bool {
1487 self.should_write_record(
1488 &CatalogDataType::from(NautilusDataType::Custom {
1489 type_name: type_name.to_string(),
1490 }),
1491 identifier,
1492 )
1493 }
1494
1495 pub fn write_instrument(
1499 &mut self,
1500 instrument: InstrumentAny,
1501 ) -> Result<(), Box<dyn std::error::Error>> {
1502 self.write(instrument)
1503 }
1504
1505 pub fn subscribe_to_message_bus(
1514 writer: Rc<RefCell<Self>>,
1515 ) -> Result<StreamingSinkSubscription, Box<dyn std::error::Error>> {
1516 let sink: StreamingDataSink = Box::new(writer);
1517 Ok(StreamingSinkSubscription::subscribe(
1518 Rc::new(RefCell::new(sink)),
1519 None,
1520 ))
1521 }
1522
1523 pub fn unsubscribe_from_message_bus(handler: &StreamingSinkSubscription) {
1525 handler.unsubscribe();
1526 }
1527}
1528
1529impl Debug for FeatherWriter {
1530 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1531 f.debug_struct(stringify!(FeatherWriter))
1532 .finish_non_exhaustive()
1533 }
1534}
1535
1536impl Drop for FeatherWriter {
1540 fn drop(&mut self) {
1541 if !self.writers.is_empty() {
1542 let _ = self.seal();
1543 }
1544 }
1545}
1546
1547impl StreamingSink for FeatherWriter {
1548 fn write_data(&mut self, data: Data) -> anyhow::Result<()> {
1549 Self::write_data(self, data).map_err(feather_error)
1550 }
1551
1552 fn write_batch(&mut self, data: Vec<Data>) -> anyhow::Result<()> {
1553 Self::write_data_batch(self, data).map_err(feather_error)
1554 }
1555
1556 fn write_any(&mut self, message: &dyn Any) -> anyhow::Result<bool> {
1557 self.write_any_message(message).map_err(feather_error)
1558 }
1559
1560 fn flush(&mut self) -> anyhow::Result<()> {
1561 Self::flush(self).map_err(feather_error)
1562 }
1563
1564 fn close(&mut self) -> anyhow::Result<()> {
1565 Self::close(self).map_err(feather_error)
1566 }
1567}
1568
1569impl StreamingSink for Rc<RefCell<FeatherWriter>> {
1570 fn write_data(&mut self, data: Data) -> anyhow::Result<()> {
1571 StreamingSink::write_data(&mut *self.borrow_mut(), data)
1572 }
1573
1574 fn write_batch(&mut self, data: Vec<Data>) -> anyhow::Result<()> {
1575 StreamingSink::write_batch(&mut *self.borrow_mut(), data)
1576 }
1577
1578 fn write_any(&mut self, message: &dyn Any) -> anyhow::Result<bool> {
1579 StreamingSink::write_any(&mut *self.borrow_mut(), message)
1580 }
1581
1582 fn flush(&mut self) -> anyhow::Result<()> {
1583 StreamingSink::flush(&mut *self.borrow_mut())
1584 }
1585
1586 fn close(&mut self) -> anyhow::Result<()> {
1587 StreamingSink::close(&mut *self.borrow_mut())
1588 }
1589}
1590
1591pub(crate) fn recover_partial_feather_files(directory: &Path) {
1600 let suffix = format!(".{FEATHER_PARTIAL_EXTENSION}");
1601 let mut directories = vec![directory.to_path_buf()];
1602
1603 while let Some(directory) = directories.pop() {
1604 let entries = match fs::read_dir(&directory) {
1605 Ok(entries) => entries,
1606 Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
1607 Err(e) => {
1608 log::warn!(
1609 "Failed to list {} for partial Feather files: {e}",
1610 directory.display()
1611 );
1612 continue;
1613 }
1614 };
1615
1616 for entry in entries.flatten() {
1617 let path = entry.path();
1618
1619 if entry.file_type().is_ok_and(|file_type| file_type.is_dir()) {
1620 directories.push(path);
1621 } else if path.to_string_lossy().ends_with(&suffix)
1622 && let Err(e) = recover_partial_feather_file(&path)
1623 {
1624 log::warn!("Failed to recover Feather file {}: {e}", path.display());
1625 }
1626 }
1627 }
1628}
1629
1630fn recover_released_feather_file(partial_path: &Path) {
1632 if let Err(e) = recover_partial_feather_file(partial_path) {
1633 log::error!(
1634 "Failed to recover Feather file {}: {e}",
1635 partial_path.display()
1636 );
1637 }
1638}
1639
1640fn recover_partial_feather_file(partial_path: &Path) -> anyhow::Result<()> {
1641 let metadata = match fs::symlink_metadata(partial_path) {
1642 Ok(metadata) => metadata,
1643 Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
1644 Err(e) => return Err(e.into()),
1645 };
1646
1647 anyhow::ensure!(
1648 metadata.is_file(),
1649 "Partial Feather path must be a regular file: {}",
1650 partial_path.display(),
1651 );
1652
1653 let file = match OpenOptions::new().read(true).write(true).open(partial_path) {
1654 Ok(file) => file,
1655 Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
1656 Err(e) => return Err(e.into()),
1657 };
1658
1659 match file.try_lock() {
1660 Ok(()) => {}
1661 Err(TryLockError::WouldBlock) => return Ok(()),
1662 Err(TryLockError::Error(e)) => return Err(e.into()),
1663 }
1664
1665 let len = file.metadata()?.len();
1666
1667 if !partial_path.try_exists()? || len == 0 {
1670 return Ok(());
1671 }
1672
1673 let mut batches = 0;
1674 let mut complete_len = 0;
1675 let mut tail_error = None;
1676
1677 match StreamReader::try_new(BufReader::new(&file), None) {
1678 Ok(mut reader) => loop {
1679 match reader.next() {
1680 Some(Ok(_)) => {
1681 batches += 1;
1682 complete_len = reader.get_mut().stream_position()?;
1683 }
1684 Some(Err(e)) => {
1685 tail_error = Some(e);
1686 break;
1687 }
1688 None => break,
1689 }
1690 },
1691 Err(e) => tail_error = Some(e),
1692 }
1693
1694 if let Some(e) = tail_error.take_if(|e| {
1697 !matches!(e, ArrowError::IoError(_, source) if source.kind() == io::ErrorKind::UnexpectedEof)
1698 }) {
1699 return Err(e.into());
1700 }
1701
1702 let cause = tail_error.map_or_else(String::new, |e| format!(": {e}"));
1703
1704 if batches == 0 {
1705 fs::remove_file(partial_path)?;
1706
1707 log::warn!(
1708 "Removed Feather file {} with no complete record batch{cause}",
1709 partial_path.display()
1710 );
1711 return Ok(());
1712 }
1713
1714 let dropped = len.saturating_sub(complete_len);
1715
1716 if dropped > 0 {
1719 file.set_len(complete_len)?;
1720 }
1721
1722 file.sync_all()?;
1723
1724 let sealed_path = partial_path.with_extension("");
1725 fs::rename(partial_path, &sealed_path)?;
1726
1727 let tail = if dropped == 0 {
1728 String::new()
1729 } else {
1730 format!(", dropped {dropped} trailing byte(s){cause}")
1731 };
1732
1733 log::warn!(
1734 "Recovered {batches} record batch(es) into Feather file {}{tail}",
1735 sealed_path.display()
1736 );
1737 Ok(())
1738}
1739
1740#[cfg(test)]
1741mod tests {
1742 use std::sync::{Arc, Mutex, atomic::Ordering};
1743
1744 use nautilus_common::{clock::VirtualClock, live::LiveClock};
1745 #[cfg(target_os = "linux")]
1746 use nautilus_model::data::HasTsInit;
1747 use nautilus_model::{
1748 data::{Data, NautilusRecordType, QuoteTick, TradeTick},
1749 enums::AggressorSide,
1750 identifiers::{InstrumentId, TradeId},
1751 types::{ERROR_PRICE, Price, Quantity},
1752 };
1753 use nautilus_serialization::arrow::{
1754 ArrowSchemaProvider, DecodeDataFromRecordBatch, EncodeToRecordBatch,
1755 };
1756 use rstest::rstest;
1757 use tempfile::TempDir;
1758
1759 use super::*;
1760 use crate::{
1761 common::datafusion::identifiers_from_record_batches,
1762 writer::materializer::restore_staged_record_batches,
1763 };
1764
1765 fn feather_files(directory: &Path, extension: &str) -> Vec<PathBuf> {
1766 let mut files = Vec::new();
1767 let mut stack = vec![directory.to_path_buf()];
1768
1769 while let Some(path) = stack.pop() {
1770 for entry in fs::read_dir(&path).unwrap().flatten() {
1771 let path = entry.path();
1772 if path.is_dir() {
1773 stack.push(path);
1774 } else if path.to_string_lossy().ends_with(&format!(".{extension}")) {
1775 files.push(path);
1776 }
1777 }
1778 }
1779
1780 files.sort();
1781 files
1782 }
1783
1784 fn read_feather_batches(path: &Path) -> Vec<RecordBatch> {
1786 StreamReader::try_new(File::open(path).unwrap(), None)
1787 .unwrap()
1788 .map(Result::unwrap)
1789 .flat_map(|batch| restore_staged_record_batches(batch).unwrap())
1790 .collect()
1791 }
1792
1793 fn read_quotes(path: &Path) -> Vec<Data> {
1794 read_feather_batches(path)
1795 .into_iter()
1796 .flat_map(|batch| {
1797 let metadata = batch.schema().metadata().clone();
1798 QuoteTick::decode_data_batch(&metadata, batch).unwrap()
1799 })
1800 .collect()
1801 }
1802
1803 fn quote_at(ts: u64) -> QuoteTick {
1804 QuoteTick::new(
1805 InstrumentId::from("AUD/USD.SIM"),
1806 Price::from("1.0"),
1807 Price::from("1.1"),
1808 Quantity::from("1000"),
1809 Quantity::from("1000"),
1810 UnixNanos::from(ts),
1811 UnixNanos::from(ts),
1812 )
1813 }
1814
1815 fn quotes_at(timestamps: &[u64]) -> Vec<Data> {
1816 timestamps
1817 .iter()
1818 .map(|&ts| Data::from(quote_at(ts)))
1819 .collect()
1820 }
1821
1822 fn write_partial_file(directory: &Path, timestamps: &[u64]) -> PathBuf {
1825 let mut writer = FeatherWriter::new(
1826 directory.to_path_buf(),
1827 WriterClock::Test(Arc::new(AtomicU64::new(0))),
1828 RotationConfig::NoRotation,
1829 None,
1830 Some(0),
1831 );
1832
1833 for &ts in timestamps {
1834 writer.write(quote_at(ts)).unwrap();
1835 writer.flush().unwrap();
1836 }
1837
1838 writer.close().unwrap();
1839
1840 let sealed = feather_files(directory, FEATHER_EXTENSION).remove(0);
1841 let partial = sealed.with_extension(FEATHER_PARTIAL_EXTENSION);
1842 fs::rename(&sealed, &partial).unwrap();
1843 partial
1844 }
1845
1846 const TORN_TAIL: &str = ": Io error: failed to fill whole buffer";
1847
1848 static RECOVERY_LOGS: RecoveryLogCapture = RecoveryLogCapture(Mutex::new(Vec::new()));
1849
1850 fn recover_with_warnings(directory: &Path) -> Vec<String> {
1852 let _ = log::set_logger(&RECOVERY_LOGS);
1853 log::set_max_level(log::LevelFilter::Warn);
1854
1855 recover_partial_feather_files(directory);
1856
1857 let directory = directory.display().to_string();
1858 RECOVERY_LOGS
1859 .0
1860 .lock()
1861 .unwrap()
1862 .iter()
1863 .filter(|(level, message)| *level == log::Level::Warn && message.contains(&directory))
1864 .map(|(_, message)| message.clone())
1865 .collect()
1866 }
1867
1868 struct RecoveryLogCapture(Mutex<Vec<(log::Level, String)>>);
1869
1870 impl log::Log for RecoveryLogCapture {
1871 fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
1872 metadata.target() == "nautilus_persistence::writer::feather"
1873 }
1874
1875 fn log(&self, record: &log::Record<'_>) {
1876 if self.enabled(record.metadata()) {
1877 self.0
1878 .lock()
1879 .unwrap()
1880 .push((record.level(), record.args().to_string()));
1881 }
1882 }
1883
1884 fn flush(&self) {}
1885 }
1886
1887 #[rstest]
1888 fn test_subscription_receives_typed_quotes_and_unsubscribes() {
1889 use nautilus_common::msgbus::{MStr, publish_quote};
1890
1891 let temp_dir = TempDir::new().unwrap();
1892
1893 let writer = Rc::new(RefCell::new(FeatherWriter::new(
1894 temp_dir.path().to_path_buf(),
1895 WriterClock::Test(Arc::new(AtomicU64::new(0))),
1896 RotationConfig::NoRotation,
1897 None,
1898 Some(0),
1899 )));
1900
1901 let quote = QuoteTick::new(
1902 InstrumentId::from("AUD/USD.SIM"),
1903 Price::from("1.0"),
1904 Price::from("1.1"),
1905 Quantity::from("2"),
1906 Quantity::from("3"),
1907 4.into(),
1908 5.into(),
1909 );
1910 let handler = FeatherWriter::subscribe_to_message_bus(writer.clone()).unwrap();
1911 publish_quote(MStr::topic("data.quotes.AUD/USD.SIM").unwrap(), "e);
1912 FeatherWriter::unsubscribe_from_message_bus(&handler);
1913 publish_quote(MStr::topic("data.quotes.AUD/USD.SIM").unwrap(), "e);
1914 writer.borrow_mut().close().unwrap();
1915
1916 let rows = feather_files(temp_dir.path(), FEATHER_EXTENSION)
1917 .iter()
1918 .flat_map(|path| read_feather_batches(path))
1919 .map(|batch| batch.num_rows())
1920 .sum::<usize>();
1921 assert_eq!(rows, 1);
1922 }
1923
1924 #[rstest]
1925 fn test_subscription_records_greeks_once_and_unsubscribes() {
1926 use nautilus_common::msgbus::{MStr, publish_any, publish_greeks};
1927 use nautilus_serialization::arrow::{DecodeDataFromRecordBatch, custom::CustomDataDecoder};
1928 let temp_dir = TempDir::new().unwrap();
1929
1930 let writer = Rc::new(RefCell::new(FeatherWriter::new(
1931 temp_dir.path().to_path_buf(),
1932 WriterClock::Test(Arc::new(AtomicU64::new(0))),
1933 RotationConfig::NoRotation,
1934 None,
1935 Some(0),
1936 )));
1937 let greeks = GreeksData::from_delta("AUD/USD.SIM".parse().unwrap(), 1.25, 8.0, 11.into());
1938 let portfolio = greeks.to_portfolio_greeks();
1939 let greeks_topic = MStr::topic("data.GreeksData.instrument_id=AUD/USD.SIM").unwrap();
1940 let portfolio_topic = MStr::topic("data.PortfolioGreeks").unwrap();
1941 let handler = FeatherWriter::subscribe_to_message_bus(writer.clone()).unwrap();
1942 publish_greeks(greeks_topic, &greeks);
1943 publish_any(portfolio_topic, &portfolio);
1944 FeatherWriter::unsubscribe_from_message_bus(&handler);
1945 publish_greeks(greeks_topic, &greeks);
1946 publish_any(portfolio_topic, &portfolio);
1947 writer.borrow_mut().close().unwrap();
1948 let mut counts = HashMap::new();
1949
1950 for batch in feather_files(&temp_dir.path().join("data/custom"), FEATHER_EXTENSION)
1951 .iter()
1952 .flat_map(|path| read_feather_batches(path))
1953 {
1954 let schema = batch.schema();
1955 let name = schema.metadata()["type_name"].clone();
1956 *counts.entry(name.clone()).or_insert(0) += batch.num_rows();
1957 let decoded = CustomDataDecoder::decode_data_batch(schema.metadata(), batch).unwrap();
1958 assert_eq!(decoded.len(), 1);
1959
1960 let Data::Custom(custom) = &decoded[0] else {
1961 panic!("expected custom data")
1962 };
1963
1964 match name.as_str() {
1965 "GreeksData" => {
1966 assert_eq!(
1967 custom.data_type,
1968 CustomDataType::new(
1969 "GreeksData",
1970 None,
1971 Some(greeks.instrument_id.to_string())
1972 )
1973 );
1974 assert_eq!(
1975 custom.data.as_any().downcast_ref::<GreeksData>(),
1976 Some(&greeks)
1977 );
1978 }
1979 "PortfolioGreeks" => {
1980 assert_eq!(
1981 custom.data_type,
1982 CustomDataType::new("PortfolioGreeks", None, None)
1983 );
1984 assert_eq!(
1985 custom.data.as_any().downcast_ref::<PortfolioGreeks>(),
1986 Some(&portfolio)
1987 );
1988 }
1989 _ => panic!("unexpected custom data type {name}"),
1990 }
1991 }
1992
1993 assert_eq!(
1994 counts,
1995 HashMap::from([
1996 ("GreeksData".to_string(), 1),
1997 ("PortfolioGreeks".to_string(), 1),
1998 ])
1999 );
2000 }
2001
2002 #[rstest]
2003 #[case(false)]
2004 #[case(true)]
2005 fn test_message_write_error_reaches_flush_or_close(#[case] close: bool) {
2006 let temp_dir = TempDir::new().unwrap();
2007
2008 let mut writer = FeatherWriter::new(
2009 temp_dir.path().to_path_buf(),
2010 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2011 RotationConfig::NoRotation,
2012 None,
2013 Some(0),
2014 );
2015
2016 let quote = QuoteTick::new(
2017 InstrumentId::from("AUD/USD.SIM"),
2018 ERROR_PRICE,
2019 ERROR_PRICE,
2020 Quantity::from("1"),
2021 Quantity::from("2"),
2022 3.into(),
2023 4.into(),
2024 );
2025 let write_error = writer.write_any_message("e).unwrap_err().to_string();
2026
2027 let error = if close {
2028 StreamingSink::close(&mut writer)
2029 } else {
2030 StreamingSink::flush(&mut writer)
2031 }
2032 .unwrap_err();
2033
2034 assert_eq!(error.to_string(), write_error);
2035 }
2036
2037 #[rstest]
2038 fn test_writer_manager_keys() {
2039 let temp_dir = TempDir::new().unwrap();
2040
2041 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2043 let timestamp = clock.timestamp_ns();
2044
2045 let mut manager = FeatherWriter::new(
2046 temp_dir.path().to_path_buf(),
2047 clock,
2048 RotationConfig::NoRotation,
2049 None,
2050 None, );
2052
2053 let instrument_id = "AAPL.AAPL";
2054
2055 let quote = QuoteTick::new(
2057 InstrumentId::from(instrument_id),
2058 Price::from("100.0"),
2059 Price::from("100.0"),
2060 Quantity::from("100.0"),
2061 Quantity::from("100.0"),
2062 UnixNanos::from(1_000_000_000_000_000_000),
2063 UnixNanos::from(1_000_000_000_000_000_000),
2064 );
2065
2066 let trade = TradeTick::new(
2067 InstrumentId::from(instrument_id),
2068 Price::from("100.0"),
2069 Quantity::from("100.0"),
2070 AggressorSide::Buy,
2071 TradeId::from("1"),
2072 UnixNanos::from(1_000_000_000_000_000_000),
2073 UnixNanos::from(1_000_000_000_000_000_000),
2074 );
2075
2076 manager.write(quote).unwrap();
2077 manager.write(trade).unwrap();
2078
2079 let path = manager.get_writer_path(QuoteTick::catalog_family());
2081 let expected_path = temp_dir
2082 .path()
2083 .join("quotes")
2084 .join(format!("quotes_{timestamp}.feather"));
2085 assert_eq!(path.path, expected_path);
2086 assert!(manager.writers.contains_key(&path));
2087 let writer = manager.writers.get(&path).unwrap();
2088 assert!(writer.size() > 0);
2089
2090 let path = manager.get_writer_path(TradeTick::catalog_family());
2091 let expected_path = temp_dir
2092 .path()
2093 .join("trades")
2094 .join(format!("trades_{timestamp}.feather"));
2095 assert_eq!(path.path, expected_path);
2096 assert!(manager.writers.contains_key(&path));
2097 let writer = manager.writers.get(&path).unwrap();
2098 assert!(writer.size() > 0);
2099 }
2100
2101 #[rstest]
2102 #[case::quotes(NautilusDataType::QuoteTick.into(), "quotes/quotes_0.feather")]
2103 #[case::record(NautilusRecordType::OrderFilled.into(), "order_filled/order_filled_0.feather")]
2104 #[case::instrument_class(
2105 CatalogDataType::Instrument(NautilusInstrumentType::CurrencyPair),
2106 "instruments/currency_pair/instruments_0.feather"
2107 )]
2108 #[case::custom(
2109 NautilusDataType::Custom { type_name: "Signal".to_string() }.into(),
2110 "data/custom/Signal/Signal_0.feather"
2111 )]
2112 fn writer_paths_hold_one_file_per_type(
2113 #[case] data_type: CatalogDataType,
2114 #[case] expected: &str,
2115 ) {
2116 let temp_dir = TempDir::new().unwrap();
2117
2118 let manager = FeatherWriter::new(
2119 temp_dir.path().to_path_buf(),
2120 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2121 RotationConfig::NoRotation,
2122 None,
2123 None,
2124 );
2125
2126 let path = manager.build_writer_path(&data_type, UnixNanos::default(), 0);
2127
2128 assert_eq!(path, temp_dir.path().join(expected));
2129 }
2130
2131 #[rstest]
2132 fn existing_feather_writer_implements_streaming_data_sink() {
2133 let temp_dir = TempDir::new().unwrap();
2134 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2135
2136 let mut writer = FeatherWriter::new(
2137 temp_dir.path().to_path_buf(),
2138 clock,
2139 RotationConfig::NoRotation,
2140 None,
2141 None,
2142 );
2143
2144 let quote = QuoteTick::new(
2145 InstrumentId::from("AUD/USD.SIM"),
2146 Price::from("1.0"),
2147 Price::from("1.1"),
2148 Quantity::from("1000"),
2149 Quantity::from("1000"),
2150 UnixNanos::from(1_000),
2151 UnixNanos::from(1_000),
2152 );
2153
2154 StreamingSink::write_data(&mut writer, Data::Quote(quote)).unwrap();
2155 StreamingSink::flush(&mut writer).unwrap();
2156 StreamingSink::close(&mut writer).unwrap();
2157
2158 let files = feather_files(&temp_dir.path().join("quotes"), FEATHER_EXTENSION);
2159 assert_eq!(files.len(), 1);
2160 let identifiers =
2161 identifiers_from_record_batches(&read_feather_batches(&files[0])).unwrap();
2162 assert_eq!(identifiers, vec!["AUD/USD.SIM".to_string()]);
2163 }
2164
2165 #[rstest]
2166 fn scheduled_rotation_keeps_time_of_day_anchor_after_late_rotation() {
2167 let temp_dir = TempDir::new().unwrap();
2168
2169 let now = Arc::new(AtomicU64::new(
2170 1_767_258_000_000_000_000, ));
2172
2173 let path = FileWriterPath {
2174 path: PathBuf::from("quotes.feather"),
2175 data_type: NautilusDataType::QuoteTick.into(),
2176 };
2177
2178 let mut writer = FeatherWriter::new(
2179 temp_dir.path().to_path_buf(),
2180 WriterClock::Test(Arc::clone(&now)),
2181 RotationConfig::ScheduledDates {
2182 interval_ns: 86_400_000_000_000,
2183 rotation_time: UnixNanos::from(36_000_000_000_000u64),
2184 rotation_timezone: jiff::tz::TimeZone::UTC,
2185 },
2186 None,
2187 None,
2188 );
2189
2190 assert!(!writer.check_scheduled_rotation(&path));
2191 assert_eq!(
2192 writer.next_rotation_times[&path],
2193 UnixNanos::from(1_767_261_600_000_000_000u64),
2194 );
2195
2196 now.store(1_767_263_400_000_000_000, Ordering::Relaxed); assert!(writer.check_scheduled_rotation(&path));
2198 assert_eq!(
2199 writer.next_rotation_times[&path],
2200 UnixNanos::from(1_767_348_000_000_000_000u64),
2201 );
2202 }
2203
2204 #[rstest]
2205 #[case::modify(true)]
2206 #[case::cancel(false)]
2207 fn test_command_denied_dynamic_round_trip(
2208 #[case] modify: bool,
2209 #[values(false, true)] envelope: bool,
2210 ) {
2211 use nautilus_core::UUID4;
2212 use nautilus_model::{
2213 events::order::spec::{OrderCancelDeniedSpec, OrderModifyDeniedSpec},
2214 identifiers::{AccountId, ClientOrderId, StrategyId, TraderId, VenueOrderId},
2215 };
2216 use nautilus_serialization::arrow::DecodeTypedFromRecordBatch;
2217 let cause = UUID4::new();
2218 let trader = TraderId::from("WRITER-017");
2219 let strategy = StrategyId::from("S-DENIAL-019");
2220 let instrument = InstrumentId::from("ETHUSDT-PERP.BINANCE");
2221 let client = ClientOrderId::from("O-DENIAL-023");
2222 let venue = VenueOrderId::from("V-DENIAL-029");
2223 let account = AccountId::from("BINANCE-031");
2224
2225 let event = if modify {
2226 OrderEventAny::ModifyDenied(
2227 OrderModifyDeniedSpec::builder()
2228 .trader_id(trader)
2229 .strategy_id(strategy)
2230 .instrument_id(instrument)
2231 .client_order_id(client)
2232 .venue_order_id(venue)
2233 .account_id(account)
2234 .reason("not sent to modify".into())
2235 .ts_event(37.into())
2236 .ts_init(41.into())
2237 .causation_id(cause)
2238 .build(),
2239 )
2240 } else {
2241 OrderEventAny::CancelDenied(
2242 OrderCancelDeniedSpec::builder()
2243 .trader_id(trader)
2244 .strategy_id(strategy)
2245 .instrument_id(instrument)
2246 .client_order_id(client)
2247 .venue_order_id(venue)
2248 .account_id(account)
2249 .reason("not sent to cancel".into())
2250 .ts_event(43.into())
2251 .ts_init(47.into())
2252 .causation_id(cause)
2253 .build(),
2254 )
2255 };
2256
2257 let temp = TempDir::new().unwrap();
2258 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2259
2260 let mut writer = FeatherWriter::new(
2261 temp.path().to_path_buf(),
2262 clock,
2263 RotationConfig::NoRotation,
2264 None,
2265 None,
2266 );
2267
2268 let written = if envelope {
2269 writer.write_any_message(&event).unwrap()
2270 } else {
2271 match &event {
2272 OrderEventAny::ModifyDenied(event) => writer.write_any_message(event).unwrap(),
2273 OrderEventAny::CancelDenied(event) => writer.write_any_message(event).unwrap(),
2274 _ => unreachable!(),
2275 }
2276 };
2277
2278 assert!(written);
2279 writer.close().unwrap();
2280 let files = feather_files(temp.path(), "feather");
2281 assert_eq!(files.len(), 1);
2282 let mut recovered = Vec::new();
2283
2284 for batch in read_feather_batches(&files[0]) {
2285 let metadata = batch.schema().metadata().clone();
2286 assert_eq!(
2287 metadata.get("type_name").unwrap(),
2288 if modify {
2289 "OrderModifyDenied"
2290 } else {
2291 "OrderCancelDenied"
2292 }
2293 );
2294
2295 if modify {
2296 recovered.extend(
2297 OrderModifyDenied::decode_typed_batch(&metadata, batch)
2298 .unwrap()
2299 .into_iter()
2300 .map(OrderEventAny::ModifyDenied),
2301 );
2302 } else {
2303 recovered.extend(
2304 OrderCancelDenied::decode_typed_batch(&metadata, batch)
2305 .unwrap()
2306 .into_iter()
2307 .map(OrderEventAny::CancelDenied),
2308 );
2309 }
2310 }
2311
2312 assert_eq!(recovered, vec![event]);
2313 }
2314
2315 #[rstest]
2316 fn test_file_writer_round_trip() {
2317 let instrument_id = "AAPL.AAPL";
2318
2319 let quote = QuoteTick::new(
2321 InstrumentId::from(instrument_id),
2322 Price::from("100.0"),
2323 Price::from("100.0"),
2324 Quantity::from("100.0"),
2325 Quantity::from("100.0"),
2326 UnixNanos::from(100),
2327 UnixNanos::from(100),
2328 );
2329 let metadata = QuoteTick::metadata("e);
2330 let schema = QuoteTick::get_schema(Some(metadata.clone()));
2331 let batch = QuoteTick::encode_batch(&QuoteTick::metadata("e), &[quote]).unwrap();
2332
2333 let temp_dir = TempDir::new().unwrap();
2334 let path = temp_dir.path().join("quotes.feather");
2335 let mut file = FeatherFile::create(&path, &schema, &RotationConfig::NoRotation).unwrap();
2336 file.write_record_batch(&batch).unwrap();
2337 file.seal().unwrap();
2338
2339 assert!(!path.with_extension(FEATHER_PARTIAL_EXTENSION).exists());
2340 let mut reader = StreamReader::try_new(File::open(&path).unwrap(), None).unwrap();
2341
2342 let read_metadata = reader.schema().metadata().clone();
2343 let mut expected_metadata = metadata.clone();
2344 expected_metadata.insert("type_name".to_string(), "QuoteTick".to_string());
2345 assert_eq!(read_metadata, expected_metadata);
2346
2347 let read_batch = reader.next().unwrap().unwrap();
2348 assert_eq!(read_batch.column(0), batch.column(0));
2349
2350 let decoded = QuoteTick::decode_data_batch(&metadata, batch).unwrap();
2351 assert_eq!(decoded[0], Data::from(quote));
2352 }
2353
2354 #[rstest]
2355 fn test_round_trip() {
2356 let temp_dir = TempDir::new_in(".").unwrap();
2358
2359 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2361
2362 let mut manager = FeatherWriter::new(
2363 temp_dir.path().to_path_buf(),
2364 clock,
2365 RotationConfig::NoRotation,
2366 None,
2367 None, );
2369
2370 let instrument_id = "AAPL.AAPL";
2371
2372 let quote = QuoteTick::new(
2374 InstrumentId::from(instrument_id),
2375 Price::from("100.0"),
2376 Price::from("100.0"),
2377 Quantity::from("100.0"),
2378 Quantity::from("100.0"),
2379 UnixNanos::from(100),
2380 UnixNanos::from(100),
2381 );
2382
2383 let trade = TradeTick::new(
2384 InstrumentId::from(instrument_id),
2385 Price::from("100.0"),
2386 Quantity::from("100.0"),
2387 AggressorSide::Buy,
2388 TradeId::from("1"),
2389 UnixNanos::from(100),
2390 UnixNanos::from(100),
2391 );
2392
2393 manager.write(quote).unwrap();
2394 manager.write(trade).unwrap();
2395
2396 let paths = manager.writers.keys().cloned().collect::<Vec<_>>();
2397 assert_eq!(paths.len(), 2);
2398
2399 manager.close().unwrap();
2400
2401 let mut recovered_quotes = Vec::new();
2402 let mut recovered_trades = Vec::new();
2403
2404 for path in paths {
2405 let path_str = path.path;
2406 for batch in read_feather_batches(&path_str) {
2407 let metadata = batch.schema().metadata().clone();
2408
2409 if path_str.to_str().unwrap().contains("quotes") {
2410 let decoded = QuoteTick::decode_data_batch(&metadata, batch).unwrap();
2411 recovered_quotes.extend(decoded);
2412 } else if path_str.to_str().unwrap().contains("trades") {
2413 let decoded = TradeTick::decode_data_batch(&metadata, batch).unwrap();
2414 recovered_trades.extend(decoded);
2415 }
2416 }
2417 }
2418
2419 assert_eq!(recovered_quotes.len(), 1, "Expected one QuoteTick record");
2421 assert_eq!(recovered_trades.len(), 1, "Expected one TradeTick record");
2422
2423 assert_eq!(recovered_quotes[0], Data::from(quote));
2425 assert_eq!(recovered_trades[0], Data::from(trade));
2426 }
2427
2428 #[rstest]
2429 fn test_write_data_enum() {
2430 let temp_dir = TempDir::new().unwrap();
2431 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2432
2433 let mut writer = FeatherWriter::new(
2434 temp_dir.path().to_path_buf(),
2435 clock,
2436 RotationConfig::NoRotation,
2437 None,
2438 None,
2439 );
2440
2441 let quote = QuoteTick::new(
2442 InstrumentId::from("AUD/USD.SIM"),
2443 Price::from("1.0"),
2444 Price::from("1.0"),
2445 Quantity::from("1000"),
2446 Quantity::from("1000"),
2447 UnixNanos::from(1000),
2448 UnixNanos::from(1000),
2449 );
2450
2451 writer.write_data(Data::Quote(quote)).unwrap();
2453 writer.flush().unwrap();
2454
2455 assert!(!writer.writers.is_empty() || temp_dir.path().read_dir().unwrap().count() > 0);
2457 }
2458
2459 #[rstest]
2460 fn test_write_data_all_types() {
2461 let temp_dir = TempDir::new().unwrap();
2462 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2463
2464 let mut writer = FeatherWriter::new(
2465 temp_dir.path().to_path_buf(),
2466 clock,
2467 RotationConfig::NoRotation,
2468 None,
2469 None,
2470 );
2471
2472 let instrument_id = InstrumentId::from("AUD/USD.SIM");
2473
2474 let quote = QuoteTick::new(
2476 instrument_id,
2477 Price::from("1.0"),
2478 Price::from("1.0"),
2479 Quantity::from("1000"),
2480 Quantity::from("1000"),
2481 UnixNanos::from(1000),
2482 UnixNanos::from(1000),
2483 );
2484 writer.write_data(Data::Quote(quote)).unwrap();
2485
2486 let trade = TradeTick::new(
2487 instrument_id,
2488 Price::from("1.0"),
2489 Quantity::from("1000"),
2490 AggressorSide::Buy,
2491 TradeId::from("1"),
2492 UnixNanos::from(2000),
2493 UnixNanos::from(2000),
2494 );
2495 writer.write_data(Data::Trade(trade)).unwrap();
2496
2497 let delta = OrderBookDelta::clear(
2498 instrument_id,
2499 0,
2500 UnixNanos::from(3000),
2501 UnixNanos::from(3000),
2502 );
2503 writer.write_data(Data::BookDelta(delta)).unwrap();
2504
2505 writer.flush().unwrap();
2506 }
2507
2508 #[rstest]
2509 fn test_auto_flush() {
2510 let temp_dir = TempDir::new().unwrap();
2511 let shared_time = Arc::new(AtomicU64::new(0));
2512 let clock = WriterClock::Test(Arc::clone(&shared_time));
2513
2514 let mut writer = FeatherWriter::new(
2515 temp_dir.path().to_path_buf(),
2516 clock,
2517 RotationConfig::NoRotation,
2518 None,
2519 Some(100), );
2521
2522 let quote = QuoteTick::new(
2523 InstrumentId::from("AUD/USD.SIM"),
2524 Price::from("1.0"),
2525 Price::from("1.0"),
2526 Quantity::from("1000"),
2527 Quantity::from("1000"),
2528 UnixNanos::from(1000),
2529 UnixNanos::from(1000),
2530 );
2531
2532 writer.write(quote).unwrap();
2534 assert_eq!(writer.writers.len(), 1);
2535 assert_eq!(writer.last_flush_ns, UnixNanos::from(0));
2536
2537 shared_time.store(200_000_000, Ordering::Relaxed);
2539
2540 let quote2 = QuoteTick::new(
2542 InstrumentId::from("AUD/USD.SIM"),
2543 Price::from("1.1"),
2544 Price::from("1.1"),
2545 Quantity::from("1000"),
2546 Quantity::from("1000"),
2547 UnixNanos::from(2000),
2548 UnixNanos::from(2000),
2549 );
2550 writer.write(quote2).unwrap();
2551
2552 let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
2555 let flushed_len = fs::metadata(&partial[0]).unwrap().len();
2556 let open_writers = writer.writers.len();
2557 let last_flush_ns = writer.last_flush_ns;
2558 let sealed_before_close = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2559 writer.close().unwrap();
2560
2561 let sealed = partial[0].with_extension("");
2562 let sealed_rows = read_feather_batches(&sealed)
2563 .iter()
2564 .map(RecordBatch::num_rows)
2565 .sum::<usize>();
2566 assert_eq!(open_writers, 1);
2567 assert_eq!(last_flush_ns, UnixNanos::from(200_000_000));
2568 assert_eq!(partial.len(), 1);
2569 assert!(sealed_before_close.is_empty());
2570 assert_eq!(fs::metadata(&sealed).unwrap().len(), flushed_len + 8);
2571 assert_eq!(sealed_rows, 2);
2572 }
2573
2574 #[rstest]
2575 fn flushes_append_to_one_file_per_type_until_close() {
2576 let temp_dir = TempDir::new().unwrap();
2577 let shared_time = Arc::new(AtomicU64::new(0));
2578
2579 let mut writer = FeatherWriter::new(
2580 temp_dir.path().to_path_buf(),
2581 WriterClock::Test(Arc::clone(&shared_time)),
2582 RotationConfig::NoRotation,
2583 None,
2584 Some(1_000),
2585 );
2586
2587 let quotes = (1..=10)
2588 .map(|minute| {
2589 QuoteTick::new(
2590 InstrumentId::from("AUD/USD.SIM"),
2591 Price::from("1.0"),
2592 Price::from("1.1"),
2593 Quantity::from("1000"),
2594 Quantity::from("1000"),
2595 UnixNanos::from(minute * 60_000_000_000),
2596 UnixNanos::from(minute * 60_000_000_000),
2597 )
2598 })
2599 .collect::<Vec<_>>();
2600
2601 for quote in "es {
2603 shared_time.store(quote.ts_init.as_u64(), Ordering::Relaxed);
2604 writer.write(*quote).unwrap();
2605 }
2606
2607 let partial_before_close = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
2608 let sealed_before_close = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2609 writer.close().unwrap();
2610 let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2611
2612 let recovered = read_feather_batches(&sealed[0])
2613 .into_iter()
2614 .flat_map(|batch| {
2615 let metadata = batch.schema().metadata().clone();
2616 QuoteTick::decode_data_batch(&metadata, batch).unwrap()
2617 })
2618 .collect::<Vec<_>>();
2619
2620 assert_eq!(partial_before_close.len(), 1);
2621 assert!(sealed_before_close.is_empty());
2622 assert_eq!(sealed.len(), 1);
2623 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
2624 assert_eq!(
2625 recovered,
2626 quotes.into_iter().map(Data::from).collect::<Vec<_>>()
2627 );
2628 }
2629
2630 #[rstest]
2631 #[case::below_row_cap(3, vec![3])]
2632 #[case::above_row_cap(
2633 DEFAULT_DATA_BATCH_CHUNK_SIZE + 1,
2634 vec![DEFAULT_DATA_BATCH_CHUNK_SIZE, 1]
2635 )]
2636 fn writes_between_flushes_append_one_batch_per_row_cap(
2637 #[case] count: usize,
2638 #[case] expected_batch_rows: Vec<usize>,
2639 ) {
2640 let temp_dir = TempDir::new().unwrap();
2641
2642 let mut writer = FeatherWriter::new(
2643 temp_dir.path().to_path_buf(),
2644 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2645 RotationConfig::NoRotation,
2646 None,
2647 Some(0),
2648 );
2649 let instrument_ids = [
2650 InstrumentId::from("AUD/USD.SIM"),
2651 InstrumentId::from("EUR/USD.SIM"),
2652 ];
2653
2654 for index in 0..count {
2655 writer
2656 .write(QuoteTick::new(
2657 instrument_ids[index % 2],
2658 Price::from("1.0"),
2659 Price::from("1.1"),
2660 Quantity::from("1000"),
2661 Quantity::from("1000"),
2662 UnixNanos::from(index as u64),
2663 UnixNanos::from(index as u64),
2664 ))
2665 .unwrap();
2666 }
2667
2668 writer.close().unwrap();
2669 let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2670 let batch_rows = StreamReader::try_new(File::open(&sealed[0]).unwrap(), None)
2671 .unwrap()
2672 .map(|batch| batch.unwrap().num_rows())
2673 .collect::<Vec<_>>();
2674
2675 assert_eq!(sealed.len(), 1);
2676 assert_eq!(batch_rows, expected_batch_rows);
2677 }
2678
2679 #[rstest]
2680 fn drop_seals_open_files() {
2681 let temp_dir = TempDir::new().unwrap();
2682
2683 let mut writer = FeatherWriter::new(
2684 temp_dir.path().to_path_buf(),
2685 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2686 RotationConfig::NoRotation,
2687 None,
2688 None,
2689 );
2690 writer
2691 .write(QuoteTick::new(
2692 InstrumentId::from("AUD/USD.SIM"),
2693 Price::from("1.0"),
2694 Price::from("1.1"),
2695 Quantity::from("1000"),
2696 Quantity::from("1000"),
2697 UnixNanos::from(1_000),
2698 UnixNanos::from(1_000),
2699 ))
2700 .unwrap();
2701
2702 drop(writer);
2703
2704 assert_eq!(feather_files(temp_dir.path(), FEATHER_EXTENSION).len(), 1);
2705 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
2706 }
2707
2708 #[rstest]
2709 fn reserved_paths_skip_files_left_in_the_directory() {
2710 let temp_dir = TempDir::new().unwrap();
2711
2712 let mut writer = FeatherWriter::new(
2713 temp_dir.path().to_path_buf(),
2714 WriterClock::Test(Arc::new(AtomicU64::new(5))),
2715 RotationConfig::NoRotation,
2716 None,
2717 None,
2718 );
2719 let sealed = temp_dir.path().join("trades").join("trades_5.feather");
2720 let partial = temp_dir
2721 .path()
2722 .join("trades")
2723 .join("trades_5-1.feather.partial");
2724 fs::create_dir_all(sealed.parent().unwrap()).unwrap();
2725 fs::write(&sealed, b"sealed").unwrap();
2726 fs::write(&partial, b"partial").unwrap();
2727
2728 let path = writer.reserve_writer_path(NautilusDataType::TradeTick.into());
2729
2730 assert_eq!(
2731 path.path,
2732 temp_dir.path().join("trades").join("trades_5-2.feather")
2733 );
2734 assert_eq!(fs::read(&sealed).unwrap(), b"sealed");
2735 assert_eq!(fs::read(&partial).unwrap(), b"partial");
2736 }
2737
2738 #[rstest]
2739 #[case::same_time(vec![5; 32])]
2740 #[case::clock_rewinds(vec![5, 3, 5])]
2741 #[case::clock_advances_then_rewinds(vec![5, 7, 5, 7])]
2742 fn path_reservations_are_bounded_after_files_are_deleted(#[case] timestamps: Vec<u64>) {
2743 let temp_dir = TempDir::new().unwrap();
2744 let time = Arc::new(AtomicU64::new(0));
2745 let mut writer = FeatherWriter::new(
2746 temp_dir.path().to_path_buf(),
2747 WriterClock::Test(Arc::clone(&time)),
2748 RotationConfig::NoRotation,
2749 None,
2750 None,
2751 );
2752 let mut paths = HashSet::new();
2753
2754 for timestamp in ×tamps {
2755 time.store(*timestamp, Ordering::Relaxed);
2756 writer.write(quote_at(1)).unwrap();
2757 writer.close().unwrap();
2758 let path = feather_files(temp_dir.path(), FEATHER_EXTENSION).remove(0);
2759 assert!(paths.insert(path.clone()));
2760 fs::remove_file(path).unwrap();
2761 }
2762
2763 assert_eq!(paths.len(), timestamps.len());
2764 assert_eq!(writer.path_sequences.len(), 1);
2765 }
2766
2767 #[rstest]
2768 fn test_close() {
2769 let temp_dir = TempDir::new().unwrap();
2770 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2771
2772 let mut writer = FeatherWriter::new(
2773 temp_dir.path().to_path_buf(),
2774 clock,
2775 RotationConfig::NoRotation,
2776 None,
2777 None,
2778 );
2779
2780 let quote = QuoteTick::new(
2781 InstrumentId::from("AUD/USD.SIM"),
2782 Price::from("1.0"),
2783 Price::from("1.0"),
2784 Quantity::from("1000"),
2785 Quantity::from("1000"),
2786 UnixNanos::from(1000),
2787 UnixNanos::from(1000),
2788 );
2789
2790 writer.write(quote).unwrap();
2791 assert!(!writer.writers.is_empty());
2792
2793 writer.close().unwrap();
2794 assert!(writer.writers.is_empty());
2795 }
2796
2797 #[rstest]
2798 fn test_write_data_orderbook_deltas() {
2799 let temp_dir = TempDir::new().unwrap();
2800 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2801
2802 let mut writer = FeatherWriter::new(
2803 temp_dir.path().to_path_buf(),
2804 clock,
2805 RotationConfig::NoRotation,
2806 None,
2807 None,
2808 );
2809
2810 let instrument_id = InstrumentId::from("AUD/USD.SIM");
2811 let delta1 = OrderBookDelta::clear(
2812 instrument_id,
2813 0,
2814 UnixNanos::from(1000),
2815 UnixNanos::from(1000),
2816 );
2817 let delta2 = OrderBookDelta::clear(
2818 instrument_id,
2819 0,
2820 UnixNanos::from(2000),
2821 UnixNanos::from(2000),
2822 );
2823
2824 let deltas = OrderBookDeltas::new(instrument_id, vec![delta1, delta2]);
2825 writer
2827 .write_data(Data::BookDeltas(Box::new(deltas)))
2828 .unwrap();
2829 writer.flush().unwrap();
2830 }
2831
2832 #[rstest]
2833 fn feather_writer_is_send() {
2834 fn assert_send<T: Send>() {}
2835 assert_send::<FeatherWriter>();
2836 assert_send::<WriterClock>();
2837 }
2838
2839 #[rstest]
2840 fn writer_clock_test_source_reads_shared_atomic() {
2841 let shared = Arc::new(AtomicU64::new(7));
2842 let clock = WriterClock::Test(Arc::clone(&shared));
2843 assert_eq!(clock.timestamp_ns(), UnixNanos::from(7));
2844
2845 shared.store(42, Ordering::Relaxed);
2846 assert_eq!(clock.timestamp_ns(), UnixNanos::from(42));
2847 }
2848
2849 #[rstest]
2850 fn writer_clock_from_shared_clock_wires_test_clock() {
2851 let test_clock = Rc::new(RefCell::new(VirtualClock::new()));
2852 test_clock
2853 .borrow_mut()
2854 .advance_time(UnixNanos::from(42), true);
2855 let clock: Rc<RefCell<dyn Clock>> = test_clock.clone();
2856
2857 let (writer_clock, shared) = WriterClock::from_shared_clock(&clock);
2858 let shared = shared.expect("non-live clocks must return a shared atomic");
2859
2860 assert_eq!(writer_clock.timestamp_ns(), UnixNanos::from(42));
2862
2863 test_clock
2866 .borrow_mut()
2867 .advance_time(UnixNanos::from(99), true);
2868 shared.store(clock.borrow().timestamp_ns().as_u64(), Ordering::Relaxed);
2869 assert_eq!(writer_clock.timestamp_ns(), UnixNanos::from(99));
2870 }
2871
2872 #[rstest]
2873 fn writer_clock_from_shared_clock_live_clock_is_live() {
2874 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(LiveClock::new(None)));
2875
2876 let (writer_clock, shared) = WriterClock::from_shared_clock(&clock);
2877
2878 assert!(matches!(writer_clock, WriterClock::Live));
2879 assert!(shared.is_none());
2880 }
2881
2882 #[rstest]
2883 #[cfg(unix)]
2884 fn failed_seal_of_a_removed_file_reports_the_error_and_publishes_nothing() {
2885 let temp_dir = TempDir::new().unwrap();
2886 let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
2887
2888 let mut writer = FeatherWriter::new(
2889 temp_dir.path().to_path_buf(),
2890 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2891 RotationConfig::NoRotation,
2892 None,
2893 None,
2894 )
2895 .with_sealed_sender(sealed_tx);
2896 writer
2897 .write(QuoteTick::new(
2898 InstrumentId::from("AUD/USD.SIM"),
2899 Price::from("1.0"),
2900 Price::from("1.1"),
2901 Quantity::from("1000"),
2902 Quantity::from("1000"),
2903 UnixNanos::from(1_000),
2904 UnixNanos::from(1_000),
2905 ))
2906 .unwrap();
2907
2908 let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
2910 fs::remove_file(&partial[0]).unwrap();
2911
2912 let error = writer.close().unwrap_err();
2913
2914 assert_eq!(error.to_string(), "No such file or directory (os error 2)");
2915 assert!(writer.is_closed());
2916 assert!(feather_files(temp_dir.path(), FEATHER_EXTENSION).is_empty());
2917 assert_eq!(
2918 sealed_rx.try_iter().collect::<Vec<_>>(),
2919 Vec::<PathBuf>::new()
2920 );
2921 }
2922
2923 #[rstest]
2924 #[cfg(target_os = "linux")]
2925 fn failed_seal_recovers_the_batches_that_reached_the_file() {
2926 let temp_dir = TempDir::new().unwrap();
2927 let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
2928
2929 let mut writer = FeatherWriter::new(
2930 temp_dir.path().to_path_buf(),
2931 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2932 RotationConfig::NoRotation,
2933 None,
2934 Some(0),
2935 )
2936 .with_sealed_sender(sealed_tx);
2937
2938 writer.write(quote_at(1)).unwrap();
2939 writer.flush().unwrap();
2940 writer.write(quote_at(2)).unwrap();
2941
2942 let (path, file) = writer.writers.iter_mut().next().unwrap();
2944 let sealed = path.path.clone();
2945 let full = OpenOptions::new().write(true).open("/dev/full").unwrap();
2946 file.writer =
2947 StreamWriter::try_new(CountingWriter::new(BufWriter::new(full)), &file.schema).unwrap();
2948
2949 let error = writer.close().unwrap_err();
2950
2951 assert_eq!(
2952 error.to_string(),
2953 "Io error: No space left on device (os error 28)"
2954 );
2955 assert!(writer.is_closed());
2956 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
2957 assert_eq!(read_quotes(&sealed), quotes_at(&[1]));
2958 assert_eq!(sealed_rx.try_iter().collect::<Vec<_>>(), vec![sealed]);
2959 }
2960
2961 #[rstest]
2962 #[case::nothing_flushed(false, vec![3])]
2963 #[case::batch_flushed(true, vec![1, 3])]
2964 #[cfg(target_os = "linux")]
2965 fn failed_flush_recovers_the_abandoned_file_and_next_write_opens_a_new_one(
2966 #[case] flushed: bool,
2967 #[case] expected: Vec<u64>,
2968 ) {
2969 let temp_dir = TempDir::new().unwrap();
2970 let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
2971
2972 let mut writer = FeatherWriter::new(
2973 temp_dir.path().to_path_buf(),
2974 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2975 RotationConfig::NoRotation,
2976 None,
2977 Some(0),
2978 )
2979 .with_sealed_sender(sealed_tx);
2980
2981 writer.write(quote_at(1)).unwrap();
2982
2983 if flushed {
2984 writer.flush().unwrap();
2985 }
2986
2987 writer.write(quote_at(2)).unwrap();
2988
2989 let (abandoned, file) = writer.writers.iter_mut().next().unwrap();
2991 let abandoned = abandoned.clone();
2992 let full = OpenOptions::new().write(true).open("/dev/full").unwrap();
2993 file.writer =
2994 StreamWriter::try_new(CountingWriter::new(BufWriter::new(full)), &file.schema).unwrap();
2995
2996 let error = writer.flush().unwrap_err();
2997 writer.write(quote_at(3)).unwrap();
2998 writer.close().unwrap();
2999
3000 let mut recovered = feather_files(temp_dir.path(), FEATHER_EXTENSION)
3001 .iter()
3002 .flat_map(|path| read_quotes(path))
3003 .collect::<Vec<_>>();
3004 recovered.sort_by_key(HasTsInit::ts_init);
3005
3006 assert_eq!(
3009 error.to_string(),
3010 "Io error: No space left on device (os error 28)"
3011 );
3012 assert_eq!(abandoned.path.exists(), flushed);
3013 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3014 assert_eq!(recovered, quotes_at(&expected));
3015 let mut notified = sealed_rx.try_iter().collect::<Vec<_>>();
3016 notified.sort();
3017 assert_eq!(notified, feather_files(temp_dir.path(), FEATHER_EXTENSION));
3018 }
3019
3020 #[rstest]
3021 #[case::end_of_stream_marker(0, &[], vec![1, 2, 3], "")]
3022 #[case::no_end_of_stream_marker(8, &[], vec![1, 2, 3], "")]
3023 #[case::stray_bytes(8, &[0xFF, 0xFF], vec![1, 2, 3], "")]
3024 #[case::truncated_last_batch(9, &[], vec![1, 2], TORN_TAIL)]
3025 fn recovery_seals_complete_batches_and_drops_the_rest(
3026 #[case] cut: usize,
3027 #[case] stray: &[u8],
3028 #[case] expected: Vec<u64>,
3029 #[case] cause: &str,
3030 ) {
3031 let temp_dir = TempDir::new().unwrap();
3032 let directory = temp_dir.path().join("recover");
3033 let partial = write_partial_file(&directory, &[1, 2, 3]);
3034 let mut staged = fs::read(&partial).unwrap();
3035 staged.truncate(staged.len() - cut);
3036 staged.extend_from_slice(stray);
3037 fs::write(&partial, &staged).unwrap();
3038
3039 let reference = fs::read(write_partial_file(
3041 &temp_dir.path().join("reference"),
3042 &expected,
3043 ))
3044 .unwrap();
3045 let complete = &reference[..reference.len() - 8];
3046
3047 let warnings = recover_with_warnings(&directory);
3048
3049 let sealed = partial.with_extension("");
3050 let dropped = staged.len() - complete.len();
3051
3052 let tail = if dropped == 0 {
3053 String::new()
3054 } else {
3055 format!(", dropped {dropped} trailing byte(s){cause}")
3056 };
3057
3058 assert!(!partial.exists());
3059 assert_eq!(fs::read(&sealed).unwrap(), complete);
3060 assert_eq!(read_quotes(&sealed), quotes_at(&expected));
3061 assert_eq!(
3062 warnings,
3063 vec![format!(
3064 "Recovered {} record batch(es) into Feather file {}{tail}",
3065 expected.len(),
3066 sealed.display()
3067 )],
3068 );
3069 }
3070
3071 #[rstest]
3072 #[case::schema_only(0, "")]
3073 #[case::truncated_schema(1, TORN_TAIL)]
3074 fn recovery_removes_a_file_without_a_complete_batch(#[case] cut: u64, #[case] cause: &str) {
3075 let temp_dir = TempDir::new().unwrap();
3076 let partial = temp_dir.path().join("quotes_1.feather.partial");
3077 let file = File::create(&partial).unwrap();
3078 StreamWriter::try_new(&file, &QuoteTick::get_schema(None)).unwrap();
3079 file.set_len(file.metadata().unwrap().len() - cut).unwrap();
3080 drop(file);
3081
3082 let warnings = recover_with_warnings(temp_dir.path());
3083
3084 assert_eq!(fs::read_dir(temp_dir.path()).unwrap().count(), 0);
3085 assert_eq!(
3086 warnings,
3087 vec![format!(
3088 "Removed Feather file {} with no complete record batch{cause}",
3089 partial.display()
3090 )],
3091 );
3092 }
3093
3094 #[rstest]
3095 fn recovery_leaves_a_file_with_an_unreadable_tail() {
3096 let temp_dir = TempDir::new().unwrap();
3097 let partial = write_partial_file(temp_dir.path(), &[1, 2]);
3098
3099 let mut staged = fs::read(&partial).unwrap();
3101 staged.truncate(staged.len() - 8);
3102 staged.extend_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF, 0xF0, 0xFF, 0xFF, 0xFF]);
3103 fs::write(&partial, &staged).unwrap();
3104
3105 let warnings = recover_with_warnings(temp_dir.path());
3106
3107 assert_eq!(fs::read(&partial).unwrap(), staged);
3108 assert_eq!(
3109 warnings,
3110 vec![format!(
3111 "Failed to recover Feather file {}: Parser error: Invalid metadata length: -16",
3112 partial.display()
3113 )],
3114 );
3115 }
3116
3117 #[rstest]
3118 #[cfg(unix)]
3119 fn recovery_leaves_partial_file_symlinks_untouched() {
3120 let temp_dir = TempDir::new().unwrap();
3121 let outside = TempDir::new().unwrap();
3122 let partial = write_partial_file(outside.path(), &[1, 2]);
3123 let original = fs::read(&partial).unwrap();
3124 let link = temp_dir.path().join("quotes_0.feather.partial");
3125 std::os::unix::fs::symlink(&partial, &link).unwrap();
3126
3127 recover_partial_feather_files(temp_dir.path());
3128
3129 assert_eq!(fs::read(&partial).unwrap(), original);
3130 assert_eq!(fs::read_link(&link).unwrap(), partial);
3131 assert!(!link.with_extension("").exists());
3132 assert!(!partial.with_extension("").exists());
3133 }
3134
3135 #[rstest]
3136 #[cfg(unix)]
3137 fn recovery_continues_after_a_file_it_cannot_recover() {
3138 let temp_dir = TempDir::new().unwrap();
3139 let run = temp_dir.path().join("run");
3140 let partial = write_partial_file(&run, &[1]);
3141
3142 let unrecoverable = temp_dir.path().join("bad.feather.partial");
3143 std::os::unix::fs::symlink(&run, &unrecoverable).unwrap();
3144
3145 let warnings = recover_with_warnings(temp_dir.path());
3146
3147 let sealed = partial.with_extension("");
3148 assert_eq!(read_quotes(&sealed), quotes_at(&[1]));
3149 assert_eq!(
3150 warnings,
3151 vec![
3152 format!(
3153 "Failed to recover Feather file {}: Partial Feather path must be a regular file: {}",
3154 unrecoverable.display(),
3155 unrecoverable.display(),
3156 ),
3157 format!(
3158 "Recovered 1 record batch(es) into Feather file {}, dropped 8 trailing byte(s)",
3159 sealed.display()
3160 ),
3161 ],
3162 );
3163 }
3164
3165 #[rstest]
3166 fn recovery_leaves_open_and_empty_files() {
3167 let temp_dir = TempDir::new().unwrap();
3168
3169 let mut writer = FeatherWriter::new(
3170 temp_dir.path().join("open"),
3171 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3172 RotationConfig::NoRotation,
3173 None,
3174 Some(0),
3175 );
3176 writer.write(quote_at(1)).unwrap();
3177 writer.flush().unwrap();
3178
3179 let open = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).remove(0);
3181 let open_len = fs::metadata(&open).unwrap().len();
3182 let empty = temp_dir.path().join("empty.feather.partial");
3183 File::create(&empty).unwrap();
3184
3185 recover_partial_feather_files(temp_dir.path());
3186
3187 let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
3188 let open_len_after = fs::metadata(&open).unwrap().len();
3189 writer.close().unwrap();
3190
3191 assert_eq!(partial, vec![empty, open.clone()]);
3192 assert_eq!(open_len_after, open_len);
3193 assert_eq!(read_quotes(&open.with_extension("")), quotes_at(&[1]));
3194 }
3195
3196 #[rstest]
3197 fn size_rotation_in_mixed_identifier_batch_seals_the_shared_file_once() {
3198 let temp_dir = TempDir::new().unwrap();
3199
3200 let mut writer = FeatherWriter::new(
3201 temp_dir.path().to_path_buf(),
3202 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3203 RotationConfig::Size { max_size: 1 },
3204 None,
3205 None,
3206 );
3207
3208 let quotes = ["AUD/USD.SIM", "EUR/USD.SIM"]
3209 .map(|instrument_id| {
3210 QuoteTick::new(
3211 InstrumentId::from(instrument_id),
3212 Price::from("1.0"),
3213 Price::from("1.1"),
3214 Quantity::from("1000"),
3215 Quantity::from("1000"),
3216 UnixNanos::from(1),
3217 UnixNanos::from(1),
3218 )
3219 })
3220 .to_vec();
3221
3222 writer.write_batch(quotes).unwrap();
3224
3225 let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
3226 let batches = read_feather_batches(&sealed[0]);
3227
3228 assert!(writer.writers.is_empty());
3229 assert_eq!(sealed.len(), 1);
3230 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3231 assert_eq!(
3232 identifiers_from_record_batches(&batches).unwrap(),
3233 vec!["AUD/USD.SIM".to_string(), "EUR/USD.SIM".to_string()],
3234 );
3235 }
3236
3237 #[rstest]
3238 fn size_rotation_seals_each_full_file_and_opens_the_next_lazily() {
3239 let temp_dir = TempDir::new().unwrap();
3240 let shared_clock = Arc::new(AtomicU64::new(0));
3241
3242 let mut writer = FeatherWriter::new(
3243 temp_dir.path().to_path_buf(),
3244 WriterClock::Test(Arc::clone(&shared_clock)),
3245 RotationConfig::Size { max_size: 1 },
3246 None,
3247 Some(1),
3248 );
3249
3250 for ts in [1_000, 2_000] {
3251 shared_clock.store(ts * 1_000_000, Ordering::Relaxed);
3252 writer
3253 .write(QuoteTick::new(
3254 InstrumentId::from("AUD/USD.SIM"),
3255 Price::from("1.0"),
3256 Price::from("1.0"),
3257 Quantity::from("1000"),
3258 Quantity::from("1000"),
3259 UnixNanos::from(ts),
3260 UnixNanos::from(ts),
3261 ))
3262 .unwrap();
3263 }
3264
3265 let rows = feather_files(temp_dir.path(), FEATHER_EXTENSION)
3266 .iter()
3267 .map(|path| {
3268 read_feather_batches(path)
3269 .iter()
3270 .map(RecordBatch::num_rows)
3271 .sum::<usize>()
3272 })
3273 .collect::<Vec<_>>();
3274
3275 assert!(writer.writers.is_empty());
3276 assert_eq!(rows, vec![1, 1]);
3277 assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3278 }
3279
3280 #[rstest]
3281 #[case(
3282 "FeatherMissingTimestamp",
3283 Schema::empty(),
3284 "registered without an Arrow schema containing ts_init"
3285 )]
3286 #[case(
3287 "FeatherLegacyTimestamp",
3288 Schema::new(vec![Field::new("ts_init", DataType::UInt64, false)]),
3289 "registered with ts_init as UInt64",
3290 )]
3291 fn test_custom_writer_path_rejects_unqueryable_schema(
3292 #[case] type_name: &str,
3293 #[case] schema: Schema,
3294 #[case] expected_error: &str,
3295 ) {
3296 nautilus_model::data::registry::ensure_arrow_registered(
3297 type_name,
3298 Arc::new(schema),
3299 Box::new(|_| unreachable!("writer creation does not encode data")),
3300 Box::new(|_, _| unreachable!("writer creation does not decode data")),
3301 )
3302 .unwrap();
3303
3304 let temp_dir = TempDir::new().unwrap();
3305
3306 let mut writer = FeatherWriter::new(
3307 temp_dir.path().to_path_buf(),
3308 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3309 RotationConfig::NoRotation,
3310 None,
3311 None,
3312 );
3313
3314 for _ in 0..2 {
3315 let error = writer.get_writer_path_custom(type_name).unwrap_err();
3316
3317 assert!(error.to_string().contains(expected_error));
3318 assert!(writer.writers.is_empty());
3319 assert!(writer.path_sequences.is_empty());
3320 }
3321 }
3322
3323 #[rstest]
3324 #[cfg(feature = "python")]
3325 fn test_write_custom_data_round_trip() {
3326 use std::sync::Arc;
3327
3328 use nautilus_model::{
3329 data::{CustomData, Data, DataType},
3330 identifiers::InstrumentId,
3331 };
3332 use nautilus_serialization::{
3333 arrow::custom::CustomDataDecoder, ensure_custom_data_registered,
3334 };
3335
3336 use crate::test_data::RustTestCustomData;
3337
3338 ensure_custom_data_registered::<RustTestCustomData>();
3339
3340 let temp_dir = TempDir::new().unwrap();
3341 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
3342
3343 let mut writer = FeatherWriter::new(
3344 temp_dir.path().to_path_buf(),
3345 clock,
3346 RotationConfig::NoRotation,
3347 None,
3348 None,
3349 );
3350
3351 let instrument_id = InstrumentId::from("RUST.TEST");
3352 let data_type = DataType::new("RustTestCustomData", None, Some(instrument_id.to_string()));
3353
3354 let original = RustTestCustomData {
3355 instrument_id,
3356 value: 1.23,
3357 flag: true,
3358 ts_event: UnixNanos::from(1000),
3359 ts_init: UnixNanos::from(1000),
3360 };
3361
3362 let custom = CustomData::new(Arc::new(original.clone()), data_type);
3363
3364 writer
3365 .write_data(Data::Custom(custom))
3366 .expect("write_data CustomData");
3367 writer.close().expect("close");
3368
3369 let prefix = temp_dir
3370 .path()
3371 .join("data")
3372 .join("custom")
3373 .join("RustTestCustomData");
3374 let files = feather_files(&prefix, FEATHER_EXTENSION);
3375 let batch = read_feather_batches(&files[0]).remove(0);
3376 let metadata = batch.schema().metadata().clone();
3377 let decoded =
3378 CustomDataDecoder::decode_data_batch(&metadata, batch).expect("decode_data_batch");
3379 assert_eq!(decoded.len(), 1);
3380
3381 if let Data::Custom(decoded_custom) = &decoded[0] {
3382 assert_eq!(decoded_custom.data_type.type_name(), "RustTestCustomData");
3383 let rust: &RustTestCustomData = decoded_custom
3384 .data
3385 .as_any()
3386 .downcast_ref::<RustTestCustomData>()
3387 .expect("RustTestCustomData");
3388 assert_eq!(rust, &original);
3389 } else {
3390 panic!("Expected Data::Custom");
3391 }
3392 }
3393
3394 #[rstest]
3395 #[cfg(feature = "python")]
3396 fn test_write_custom_data_reuses_writer_until_rotation() {
3397 use nautilus_model::data::{CustomData, DataType};
3398 use nautilus_serialization::ensure_custom_data_registered;
3399
3400 use crate::test_data::RustTestCustomData;
3401
3402 ensure_custom_data_registered::<RustTestCustomData>();
3403 let temp_dir = TempDir::new().unwrap();
3404
3405 let mut writer = FeatherWriter::new(
3406 temp_dir.path().to_path_buf(),
3407 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3408 RotationConfig::NoRotation,
3409 None,
3410 None,
3411 );
3412 let instrument_id = InstrumentId::from("RUST.TEST");
3413 let data_type = DataType::new("RustTestCustomData", None, Some(instrument_id.to_string()));
3414
3415 for (ts, value) in [(1_000, 1_000.0), (2_000, 2_000.0)] {
3416 writer
3417 .write_data(Data::Custom(CustomData::new(
3418 Arc::new(RustTestCustomData {
3419 instrument_id,
3420 value,
3421 flag: true,
3422 ts_event: UnixNanos::from(ts),
3423 ts_init: UnixNanos::from(ts),
3424 }),
3425 data_type.clone(),
3426 )))
3427 .unwrap();
3428 }
3429
3430 assert_eq!(writer.get_current_file_info().len(), 1);
3431 writer.close().unwrap();
3432
3433 let prefix = temp_dir
3434 .path()
3435 .join("data")
3436 .join("custom")
3437 .join("RustTestCustomData");
3438 let files = feather_files(&prefix, FEATHER_EXTENSION);
3439 assert_eq!(files.len(), 1);
3440 let rows = read_feather_batches(&files[0])
3441 .iter()
3442 .map(RecordBatch::num_rows)
3443 .sum::<usize>();
3444 assert_eq!(rows, 2);
3445 }
3446}