Skip to main content

nautilus_persistence/writer/
feather.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16#![expect(
17    clippy::missing_errors_doc,
18    reason = "Feather writer public methods forward encoding and file IO errors directly"
19)]
20
21use std::{
22    any::Any,
23    borrow::Cow,
24    cell::RefCell,
25    collections::{BTreeMap, HashMap, HashSet},
26    fmt::Debug,
27    fs::{self, File, OpenOptions, TryLockError},
28    io::{self, BufReader, BufWriter, Seek, Write},
29    path::{Path, PathBuf},
30    rc::Rc,
31    sync::{
32        Arc,
33        atomic::{AtomicU64, Ordering},
34        mpsc::Sender,
35    },
36};
37
38use ahash::AHashMap;
39use datafusion::arrow::{
40    array::{ArrayRef, DictionaryArray},
41    compute::concat_batches,
42    datatypes::{DataType, Field, Int32Type, Schema, SchemaRef},
43    error::ArrowError,
44    ipc::{reader::StreamReader, writer::StreamWriter},
45    record_batch::RecordBatch,
46};
47use jiff::{
48    SignedDuration,
49    civil::Time,
50    tz::{AmbiguousOffset, TimeZone},
51};
52use nautilus_common::{clock::Clock, live::LiveClock};
53use nautilus_core::{DurationNanos, UnixNanos, time::nanos_since_unix_epoch};
54use nautilus_model::{
55    data::{
56        Bar, CustomData, CustomDataTrait, Data, DataBatch, DataType as CustomDataType,
57        FundingRateUpdate, IndexPriceUpdate, InstrumentStatus, MarkPriceUpdate, NautilusDataType,
58        OptionGreeks, OrderBookDelta, OrderBookDeltas, OrderBookDepth, QuoteTick, TradeTick,
59        close::InstrumentClose,
60        encode_custom_to_arrow, get_arrow_schema,
61        greeks::{GreeksData, PortfolioGreeks},
62    },
63    events::{
64        AccountState, OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled,
65        OrderDenied, OrderEmulated, OrderEventAny, OrderExpired, OrderFillVoided, OrderFilled,
66        OrderInitialized, OrderModifyDenied, OrderModifyRejected, OrderPendingCancel,
67        OrderPendingUpdate, OrderRejected, OrderReleased, OrderSnapshot, OrderSubmitted,
68        OrderTriggered, OrderUpdated, PositionAdjusted, PositionChanged, PositionClosed,
69        PositionEvent, PositionOpened, PositionSnapshot,
70    },
71    instruments::{InstrumentAny, NautilusInstrumentType},
72    reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
73};
74use nautilus_serialization::{
75    arrow::{
76        EncodeToRecordBatch, KEY_TYPE_NAME, catalog_identifier_from_metadata,
77        record_batch_with_identifier_column, schema_with_identifier_column,
78    },
79    ensure_custom_data_registered,
80};
81
82use crate::{
83    catalog::{
84        session::DEFAULT_DATA_BATCH_CHUNK_SIZE,
85        types::{
86            CatalogDataType, CatalogFamily, data_path_prefix, instrument_path_prefix,
87            record_path_prefix,
88        },
89    },
90    common::{
91        custom::{
92            augment_batch_with_data_type_column, schema_with_data_type_column,
93            validate_custom_catalog_schema,
94        },
95        metadata::record_batch_ts_init_range,
96    },
97    writer::{
98        filter::{WriterRecordFilter, catalog_family},
99        subscription::StreamingSinkSubscription,
100        traits::{StreamingDataSink, StreamingSink},
101    },
102};
103
104pub(crate) type FeatherWriteCommand =
105    Box<dyn FnOnce(&mut FeatherWriter) -> Result<(), Box<dyn std::error::Error>> + Send + 'static>;
106
107#[expect(
108    clippy::needless_pass_by_value,
109    reason = "map_err transfers ownership of the boxed writer error"
110)]
111pub(crate) fn feather_error(e: Box<dyn std::error::Error>) -> anyhow::Error {
112    anyhow::anyhow!("{e}")
113}
114
115macro_rules! define_builtin_data_batch_dispatch {
116    ($(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?) => {
117        fn write_builtin_data_batch(
118            writer: &mut FeatherWriter,
119            batch: &DataBatch,
120        ) -> Option<Result<(), Box<dyn std::error::Error>>> {
121            match batch {
122                $(
123                    DataBatch::$batch(data) => Some(writer.write_batch(data.as_ref().to_vec())),
124                )+
125                _ => None,
126            }
127        }
128    };
129}
130
131nautilus_model::for_each_data_type!(define_builtin_data_batch_dispatch);
132
133pub(crate) const NAUTILUS_ARROW_METADATA_ID_COLUMN: &str = "nautilus_metadata_id";
134pub(crate) const NAUTILUS_ARROW_METADATA_JSON_COLUMN: &str = "nautilus_metadata_json";
135
136/// Arrow type of the staged metadata columns; rows of one flush share a few distinct values.
137pub(crate) fn staged_metadata_data_type() -> DataType {
138    DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
139}
140
141pub(crate) fn staged_metadata_array<'a>(values: impl IntoIterator<Item = &'a str>) -> ArrayRef {
142    Arc::new(values.into_iter().collect::<DictionaryArray<Int32Type>>())
143}
144
145#[derive(Clone, Debug, Eq, PartialEq)]
146struct StagedArrowMetadataRow {
147    metadata_id: String,
148    metadata_json: String,
149}
150
151/// An open file and the catalog type it stages; each instrument class stages its own file.
152#[derive(Debug, Clone, PartialEq, Eq, Hash)]
153pub struct FileWriterPath {
154    path: PathBuf,
155    data_type: CatalogDataType,
156}
157
158/// Minimal `Send` time source for streaming writers.
159///
160/// `Live` reads the wall clock directly; `Test` reads a shared atomic that the
161/// owner of the source clock (e.g. a PyO3 wrapper holding a `VirtualClock`)
162/// refreshes before forwarding writer calls.
163#[derive(Clone, Debug)]
164pub enum WriterClock {
165    /// Wall-clock time via [`nanos_since_unix_epoch`].
166    Live,
167    /// Externally driven time stored in a shared atomic (nanoseconds since the UNIX epoch).
168    Test(Arc<AtomicU64>),
169}
170
171impl WriterClock {
172    /// Returns the current timestamp in nanoseconds since the UNIX epoch.
173    #[must_use]
174    pub fn timestamp_ns(&self) -> UnixNanos {
175        match self {
176            Self::Live => UnixNanos::from(nanos_since_unix_epoch()),
177            Self::Test(shared) => UnixNanos::from(shared.load(Ordering::Relaxed)),
178        }
179    }
180
181    /// Builds a writer clock from a shared `dyn Clock`.
182    ///
183    /// Returns [`WriterClock::Live`] for a [`LiveClock`]. For any other clock
184    /// (e.g. `VirtualClock`) it returns a [`WriterClock::Test`] source seeded with
185    /// the clock's current time, plus the shared atomic the caller must refresh
186    /// from the source clock before each forwarded writer call.
187    #[must_use]
188    pub fn from_shared_clock(clock: &Rc<RefCell<dyn Clock>>) -> (Self, Option<Arc<AtomicU64>>) {
189        let borrowed = clock.borrow();
190        let any_ref: &dyn Any = &*borrowed;
191        if any_ref.downcast_ref::<LiveClock>().is_some() {
192            (Self::Live, None)
193        } else {
194            let shared = Arc::new(AtomicU64::new(borrowed.timestamp_ns().as_u64()));
195            (Self::Test(Arc::clone(&shared)), Some(shared))
196        }
197    }
198}
199
200fn record_batch_with_delta_staged_metadata(
201    batch: &RecordBatch,
202    metadata_rows: &[StagedArrowMetadataRow],
203) -> anyhow::Result<RecordBatch> {
204    anyhow::ensure!(
205        batch.num_rows() == metadata_rows.len(),
206        "Delta Feather staging metadata row count {} does not match batch row count {}",
207        metadata_rows.len(),
208        batch.num_rows()
209    );
210    anyhow::ensure!(
211        batch
212            .schema()
213            .index_of(NAUTILUS_ARROW_METADATA_ID_COLUMN)
214            .is_err(),
215        "Delta Feather staging schema already has {NAUTILUS_ARROW_METADATA_ID_COLUMN}"
216    );
217    anyhow::ensure!(
218        batch
219            .schema()
220            .index_of(NAUTILUS_ARROW_METADATA_JSON_COLUMN)
221            .is_err(),
222        "Delta Feather staging schema already has {NAUTILUS_ARROW_METADATA_JSON_COLUMN}"
223    );
224
225    let mut fields = batch
226        .schema()
227        .fields()
228        .iter()
229        .map(|field| {
230            Arc::new(Field::new(
231                field.name().clone(),
232                field.data_type().clone(),
233                field.is_nullable(),
234            ))
235        })
236        .collect::<Vec<_>>();
237
238    fields.push(Arc::new(Field::new(
239        NAUTILUS_ARROW_METADATA_ID_COLUMN,
240        staged_metadata_data_type(),
241        false,
242    )));
243    fields.push(Arc::new(Field::new(
244        NAUTILUS_ARROW_METADATA_JSON_COLUMN,
245        staged_metadata_data_type(),
246        false,
247    )));
248
249    let mut columns = batch.columns().to_vec();
250    columns.push(staged_metadata_array(
251        metadata_rows.iter().map(|row| row.metadata_id.as_str()),
252    ));
253    columns.push(staged_metadata_array(
254        metadata_rows.iter().map(|row| row.metadata_json.as_str()),
255    ));
256
257    Ok(RecordBatch::try_new(
258        Arc::new(Schema::new(fields)),
259        columns,
260    )?)
261}
262
263fn arrow_metadata_row(
264    metadata: &HashMap<String, String>,
265    fields: &arrow::datatypes::Fields,
266) -> anyhow::Result<StagedArrowMetadataRow> {
267    let schema_metadata = metadata.iter().collect::<BTreeMap<_, _>>();
268
269    let field_metadata = fields
270        .iter()
271        .filter(|field| !field.metadata().is_empty())
272        .map(|field| {
273            (
274                field.name().clone(),
275                field
276                    .metadata()
277                    .iter()
278                    .map(|(key, value)| (key.clone(), value.clone()))
279                    .collect::<BTreeMap<_, _>>(),
280            )
281        })
282        .collect::<BTreeMap<_, _>>();
283
284    let metadata_json = serde_json::to_string(&serde_json::json!({
285        "format_version": 1,
286        "schema_metadata": schema_metadata,
287        "field_metadata": field_metadata,
288    }))?;
289    let metadata_id = staged_metadata_id(&serde_json::to_string(&schema_metadata)?);
290
291    Ok(StagedArrowMetadataRow {
292        metadata_id,
293        metadata_json,
294    })
295}
296
297/// Canonical schema-level metadata JSON: keys serialized in sorted order.
298///
299/// The staged metadata id hashes this exact string so readers can verify the schema metadata
300/// independently of its storage location.
301pub(crate) fn canonical_metadata_json(
302    metadata: &HashMap<String, String>,
303) -> anyhow::Result<String> {
304    Ok(serde_json::to_string(
305        &metadata
306            .iter()
307            .map(|(key, value)| (key.clone(), value.clone()))
308            .collect::<BTreeMap<_, _>>(),
309    )?)
310}
311
312pub(crate) fn staged_metadata_id(metadata_json: &str) -> String {
313    format!("blake3:{}", blake3::hash(metadata_json.as_bytes()).to_hex())
314}
315
316/// File extension of a sealed Feather stream file.
317pub(crate) const FEATHER_EXTENSION: &str = "feather";
318
319/// File extension of a Feather stream file that is still being appended to.
320pub(crate) const FEATHER_PARTIAL_EXTENSION: &str = "feather.partial";
321
322/// An open local Feather file that appends Arrow IPC stream batches.
323///
324/// Batches go to `{name}.feather.partial`, which is renamed to `{name}.feather` when the file is
325/// sealed, so readers listing `.feather` files only see complete streams. The file stays
326/// exclusively locked until it is sealed, so [`recover_partial_feather_files`] can tell an open
327/// file from one its writer abandoned. Written rows are
328/// buffered and appended as one record batch on flush, on seal, when the buffer reaches
329/// [`DEFAULT_DATA_BATCH_CHUNK_SIZE`] rows, or when size rotation may be due, because each IPC
330/// batch carries a message header that outweighs a single row.
331struct FeatherFile {
332    writer: StreamWriter<CountingWriter<BufWriter<File>>>,
333    partial_path: PathBuf,
334    path: PathBuf,
335    schema: SchemaRef,
336    max_size: Option<u64>,
337    buffer: Vec<RecordBatch>,
338    buffer_rows: usize,
339    buffer_bytes: u64,
340    /// The largest `ts_init` written, which the next file's rows must exceed.
341    last_ts_init: Option<u64>,
342    /// The file reached `max_size`; the next row with a later `ts_init` opens a new file.
343    size_rotation_due: bool,
344}
345
346impl FeatherFile {
347    fn create(
348        path: &Path,
349        schema: &Schema,
350        rotation_config: &RotationConfig,
351    ) -> Result<Self, Box<dyn std::error::Error>> {
352        let partial_path = path.with_extension(FEATHER_PARTIAL_EXTENSION);
353        if let Some(parent) = path.parent() {
354            fs::create_dir_all(parent)?;
355        }
356
357        let file = OpenOptions::new()
358            .write(true)
359            .create_new(true)
360            .open(&partial_path)?;
361
362        // Recovery can briefly lock a new, empty file before its writer does, so wait it out
363        file.lock().map_err(|e| {
364            format!(
365                "Failed to lock Feather file {}: {e}",
366                partial_path.display()
367            )
368        })?;
369
370        let writer = StreamWriter::try_new(CountingWriter::new(BufWriter::new(file)), schema)?;
371
372        let max_size = match rotation_config {
373            RotationConfig::Size { max_size } => Some(*max_size),
374            _ => None,
375        };
376
377        Ok(Self {
378            writer,
379            partial_path,
380            path: path.to_path_buf(),
381            schema: Arc::new(schema.clone()),
382            max_size,
383            buffer: Vec::new(),
384            buffer_rows: 0,
385            buffer_bytes: 0,
386            last_ts_init: None,
387            size_rotation_due: false,
388        })
389    }
390
391    // Appends the batch and notes when size rotation is due
392    fn write_record_batch(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
393        let batch = if batch.schema() == self.schema {
394            batch.clone()
395        } else {
396            RecordBatch::try_new(self.schema.clone(), batch.columns().to_vec())?
397        };
398
399        self.buffer_rows += batch.num_rows();
400        self.buffer_bytes += batch.get_array_memory_size() as u64;
401        self.buffer.push(batch);
402
403        if self.buffer_rows >= DEFAULT_DATA_BATCH_CHUNK_SIZE {
404            self.write_buffer()?;
405        }
406
407        let Some(max_size) = self.max_size else {
408            return Ok(());
409        };
410
411        // The in-memory size only estimates the encoded size, so write before the exact check
412        if self.size() + self.buffer_bytes >= max_size {
413            self.write_buffer()?;
414        }
415
416        self.size_rotation_due = self.size() >= max_size;
417        Ok(())
418    }
419
420    fn write_buffer(&mut self) -> Result<(), ArrowError> {
421        if self.buffer.is_empty() {
422            return Ok(());
423        }
424
425        let batch = concat_batches(&self.schema, &self.buffer)?;
426        self.buffer.clear();
427        self.buffer_rows = 0;
428        self.buffer_bytes = 0;
429        self.writer.write(&batch)
430    }
431
432    fn size(&self) -> u64 {
433        self.writer.get_ref().bytes
434    }
435
436    fn flush(&mut self) -> Result<(), ArrowError> {
437        self.write_buffer()?;
438        self.writer.flush()
439    }
440
441    fn seal(mut self) -> Result<(), Box<dyn std::error::Error>> {
442        self.write_buffer()?;
443        self.writer.finish()?;
444        let file = self
445            .writer
446            .into_inner()?
447            .inner
448            .into_inner()
449            .map_err(io::IntoInnerError::into_error)?;
450        file.sync_all()?;
451
452        // Renaming before the file closes keeps it locked until it is sealed
453        fs::rename(&self.partial_path, &self.path)?;
454        Ok(())
455    }
456}
457
458/// Counts the bytes written through the inner writer.
459struct CountingWriter<W> {
460    inner: W,
461    bytes: u64,
462}
463
464impl<W> CountingWriter<W> {
465    const fn new(inner: W) -> Self {
466        Self { inner, bytes: 0 }
467    }
468}
469
470impl<W: Write> Write for CountingWriter<W> {
471    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
472        let written = self.inner.write(buf)?;
473        self.bytes += written as u64;
474        Ok(written)
475    }
476
477    fn flush(&mut self) -> io::Result<()> {
478        self.inner.flush()
479    }
480}
481
482/// Configuration for file rotation.
483///
484/// Time rotations are decided by the rows' `ts_init`, not the wall clock, and no rotation splits
485/// rows of one `ts_init` over two files (`FeatherWriter::rotated_for`): the catalog requires the
486/// promoted files' `ts_init` ranges to be disjoint.
487///
488/// That guarantee holds only while a staged type's `ts_init` does not decrease across its
489/// identifiers. A rotation compares a row with the open file alone, so a late row at or before a
490/// sealed file's last `ts_init` lands in a later file, which can then overlap the sealed file's
491/// range and fail promotion.
492#[derive(Debug, Clone)]
493pub enum RotationConfig {
494    /// Rotate based on file size, before the next row with a later `ts_init`.
495    Size {
496        /// Maximum buffer size in bytes before rotation.
497        max_size: u64,
498    },
499    /// Rotate before the first row whose `ts_init` is an interval past the file's first row.
500    Interval {
501        /// Interval in nanoseconds.
502        interval_ns: u64,
503    },
504    /// Rotate before the first row whose `ts_init` is at or after a scheduled time of day.
505    ScheduledDates {
506        /// Interval in nanoseconds.
507        interval_ns: u64,
508        /// Time of day for rotation (nanoseconds since midnight).
509        rotation_time: UnixNanos,
510        /// Timezone for rotation calculations.
511        rotation_timezone: TimeZone,
512    },
513    /// No automatic rotation.
514    NoRotation,
515}
516
517/// Streams encoded data into one local Feather file per data or record type, and per class for
518/// instruments.
519///
520/// Rows of every identifier share their type's file. Each row carries its catalog `identifier`
521/// and the Arrow metadata it was encoded with, such as its instrument's precision, so readers
522/// restore per-row metadata the file schema cannot hold.
523///
524/// The [`Self::write`] method accepts a data value of generic type `T`, encodes it using `T`'s
525/// metadata via `EncodeToRecordBatch`, routes it by `CatalogFamily`, and appends it to that key's
526/// open file.
527///
528/// # File Lifecycle
529///
530/// Open files are tracked by their final `.feather` paths, while writes go to `.feather.partial`
531/// files. Sealing completes the stream and renames the partial file to its final path. A file is
532/// sealed and replaced only on rotation, [`Self::seal`], or [`Self::close`].
533///
534/// # Flushing
535///
536/// [`Self::flush`] pushes buffered bytes to disk without starting a new file. `flush_interval_ms`
537/// controls automatic flushing in milliseconds; `0` disables automatic flushing.
538pub struct FeatherWriter {
539    directory: PathBuf,
540    clock: WriterClock,
541    rotation_config: RotationConfig,
542    included_types: Option<HashSet<CatalogDataType>>,
543    record_filter: Option<WriterRecordFilter>,
544    writers: HashMap<FileWriterPath, FeatherFile>,
545    path_sequences: HashMap<CatalogDataType, (UnixNanos, u64)>,
546    sealed_tx: Option<Sender<PathBuf>>,
547    next_rotation_times: HashMap<FileWriterPath, UnixNanos>,
548    flush_interval_ms: u64,
549    last_flush_ns: UnixNanos,
550    pending_write_error: Option<String>,
551}
552
553impl FeatherWriter {
554    /// Creates a new [`FeatherWriter`] instance.
555    #[must_use]
556    pub fn new(
557        directory: PathBuf,
558        clock: WriterClock,
559        rotation_config: RotationConfig,
560        included_types: Option<HashSet<CatalogDataType>>,
561        flush_interval_ms: Option<u64>,
562    ) -> Self {
563        let flush_interval_ms = flush_interval_ms.unwrap_or(1000); // Default 1 second
564        let last_flush_ns = clock.timestamp_ns();
565
566        Self {
567            directory,
568            clock,
569            rotation_config,
570            included_types,
571            record_filter: None,
572            writers: HashMap::new(),
573            path_sequences: HashMap::new(),
574            sealed_tx: None,
575            next_rotation_times: HashMap::new(),
576            flush_interval_ms,
577            last_flush_ns,
578            pending_write_error: None,
579        }
580    }
581
582    /// Sets typed record-family filter for subsequent writes.
583    #[must_use]
584    pub fn with_record_filter(mut self, record_filter: Option<WriterRecordFilter>) -> Self {
585        self.record_filter = record_filter;
586        self
587    }
588
589    pub(crate) fn with_sealed_sender(mut self, sealed_tx: Sender<PathBuf>) -> Self {
590        self.sealed_tx = Some(sealed_tx);
591        self
592    }
593
594    /// Writes a single data value.
595    ///
596    /// This is the user entry point. The data is encoded into a `RecordBatch` and appended to the
597    /// open file for its type, or for its class when the data is an instrument.
598    pub fn write<T>(&mut self, data: T) -> Result<(), Box<dyn std::error::Error>>
599    where
600        T: EncodeToRecordBatch + CatalogFamily + 'static,
601    {
602        let metadata = T::metadata(&data);
603        let identifier = catalog_identifier_from_metadata(&metadata);
604        let data_type = Self::staged_data_type::<T>(&metadata)?;
605
606        if !self.should_write_record(&data_type, identifier.as_deref()) {
607            return Ok(());
608        }
609
610        let path = self.get_writer_path(data_type);
611
612        // Create a new FileWriter if one does not exist.
613        if !self.writers.contains_key(&path) {
614            self.create_writer::<T>(path.clone(), &data)?;
615        }
616
617        // Encode the data into a RecordBatch using T's encoding logic.
618        let batch = T::encode_batch(&metadata, &[data])?;
619        let metadata_row = arrow_metadata_row(&metadata, batch.schema().fields())?;
620        let batch = record_batch_with_identifier_column(batch, identifier.as_deref())?;
621        let batch =
622            record_batch_with_delta_staged_metadata(&batch, std::slice::from_ref(&metadata_row))?;
623
624        // Write the RecordBatch to the appropriate FileWriter.
625        self.stage_batch_writes(path, std::slice::from_ref(&batch))?;
626        self.flush_if_due()
627    }
628
629    /// Writes a batch of data values as one or more `RecordBatch`es.
630    ///
631    /// Uses `T::chunk_metadata` to derive the file schema metadata. This protects
632    /// types like `OrderBookDelta` from having their file metadata poisoned by a
633    /// leading sentinel row (e.g. `BookAction::Clear`, which carries
634    /// `price_precision=0, size_precision=0`).
635    ///
636    /// Rows are grouped by identifier so each group encodes with its own metadata before it is
637    /// appended to the type's file. The groups of one file are staged together
638    /// (`stage_batch_writes`): the groups come out of their map in hash order, and a rotation
639    /// decided group by group could seal the file on a later group's row and put an earlier
640    /// group's row, equal to the sealed file's last `ts_init`, in the next file.
641    pub fn write_batch<T>(&mut self, data: Vec<T>) -> Result<(), Box<dyn std::error::Error>>
642    where
643        T: EncodeToRecordBatch + CatalogFamily + 'static,
644    {
645        if data.is_empty() || !self.should_write_family(&T::catalog_family()) {
646            return Ok(());
647        }
648
649        let mut groups: AHashMap<Option<String>, (CatalogDataType, Vec<T>)> = AHashMap::new();
650
651        for item in data {
652            let metadata = T::metadata(&item);
653            let identifier = catalog_identifier_from_metadata(&metadata);
654            let data_type = Self::staged_data_type::<T>(&metadata)?;
655
656            if !self.should_write_record(&data_type, identifier.as_deref()) {
657                continue;
658            }
659
660            groups
661                .entry(identifier)
662                .or_insert_with(|| (data_type, Vec::new()))
663                .1
664                .push(item);
665        }
666
667        if groups.is_empty() {
668            return Ok(());
669        }
670
671        let mut staged: Vec<(FileWriterPath, Vec<RecordBatch>)> = Vec::new();
672
673        for (data_type, group) in groups.into_values() {
674            let path = self.get_writer_path(data_type);
675            let metadata = T::chunk_metadata(&group);
676
677            if !self.writers.contains_key(&path) {
678                self.create_writer_with_metadata::<T>(path.clone(), metadata.clone())?;
679            }
680
681            let identifier = catalog_identifier_from_metadata(&metadata);
682            let batch = T::encode_batch(&metadata, &group)?;
683            let metadata_rows = group
684                .iter()
685                .map(T::metadata)
686                .map(|metadata| arrow_metadata_row(&metadata, batch.schema().fields()))
687                .collect::<anyhow::Result<Vec<_>>>()?;
688            let batch = record_batch_with_identifier_column(batch, identifier.as_deref())?;
689            let batch = record_batch_with_delta_staged_metadata(&batch, &metadata_rows)?;
690
691            match staged.iter_mut().find(|(staged, _)| *staged == path) {
692                Some((_, batches)) => batches.push(batch),
693                None => staged.push((path, vec![batch])),
694            }
695        }
696
697        for (path, batches) in staged {
698            self.stage_batch_writes(path, &batches)?;
699        }
700
701        self.flush_if_due()
702    }
703
704    // Instrument classes have different fields, so each class stages its own file
705    fn staged_data_type<T: CatalogFamily>(
706        metadata: &HashMap<String, String>,
707    ) -> Result<CatalogDataType, Box<dyn std::error::Error>> {
708        let family = T::catalog_family();
709        if family != CatalogDataType::Data(NautilusDataType::Instrument) {
710            return Ok(family);
711        }
712
713        let class = metadata
714            .get(KEY_TYPE_NAME)
715            .ok_or("Instrument metadata has no type_name")?
716            .parse::<NautilusInstrumentType>()?;
717        Ok(CatalogDataType::Instrument(class))
718    }
719
720    /// Appends `batches` to the open file of `path`, or to the file opened after rotating it when
721    /// their earliest `ts_init` is due one (`rotated_for`). The batches share that one decision,
722    /// so rows written together land in one file whatever their order.
723    fn stage_batch_writes(
724        &mut self,
725        path: FileWriterPath,
726        batches: &[RecordBatch],
727    ) -> Result<(), Box<dyn std::error::Error>> {
728        if !self.writers.contains_key(&path) {
729            return Ok(());
730        }
731
732        let (first_ts, last_ts) = record_batch_ts_init_range(batches)?;
733        let path = self.rotated_for(path, first_ts)?;
734        let Some(file) = self.writers.get_mut(&path) else {
735            return Ok(());
736        };
737
738        for batch in batches {
739            if let Err(e) = file.write_record_batch(batch) {
740                self.abandon_file(&path);
741                return Err(e.into());
742            }
743        }
744
745        file.last_ts_init = Some(file.last_ts_init.map_or(last_ts, |last| last.max(last_ts)));
746        Ok(())
747    }
748
749    /// The file a row of `path` at `ts_init` writes to: the open one, or the one opened after
750    /// sealing it when a rotation is due.
751    ///
752    /// The catalog promotes each sealed file to one Parquet file and requires the files' `ts_init`
753    /// ranges to be disjoint, so a rotation is decided by the row's own `ts_init` before the row is
754    /// written: the first row at or after a scheduled boundary opens the next file, and a row
755    /// sharing the `ts_init` of the file's last row stays in it whatever rotation is due. The
756    /// wall clock would put the first row after a boundary in the old file, and a second row with
757    /// the same `ts_init` (the prints of one fill) in the new one.
758    fn rotated_for(
759        &mut self,
760        path: FileWriterPath,
761        ts_init: u64,
762    ) -> Result<FileWriterPath, Box<dyn std::error::Error>> {
763        let scheduled = self.scheduled_rotation_due(&path, ts_init);
764        let Some(file) = self.writers.get(&path) else {
765            return Ok(path);
766        };
767
768        if !(scheduled || file.size_rotation_due)
769            || file.last_ts_init.is_some_and(|last| ts_init <= last)
770        {
771            return Ok(path);
772        }
773
774        let schema = file.schema.clone();
775        let data_type = path.data_type.clone();
776        self.seal_file(&path)?;
777
778        let path = self.reserve_writer_path(data_type);
779        let file = FeatherFile::create(&path.path, &schema, &self.rotation_config)?;
780        self.writers.insert(path.clone(), file);
781        // This row is the new file's first: it sets the file's next rotation
782        self.scheduled_rotation_due(&path, ts_init);
783        Ok(path)
784    }
785
786    /// Returns whether the auto-flush interval has elapsed since the last flush.
787    fn flush_is_due(&self) -> bool {
788        if self.flush_interval_ms == 0 {
789            return false; // Auto-flush disabled
790        }
791
792        let now_ns = self.clock.timestamp_ns();
793        let elapsed_ms = now_ns.as_u64().saturating_sub(self.last_flush_ns.as_u64()) / 1_000_000;
794        elapsed_ms >= self.flush_interval_ms
795    }
796
797    /// Flushes once the synchronous encode path completes, when the auto-flush interval elapsed.
798    fn flush_if_due(&mut self) -> Result<(), Box<dyn std::error::Error>> {
799        if self.flush_is_due() {
800            self.flush()?;
801        }
802
803        Ok(())
804    }
805
806    /// Whether a row at `ts_init` falls at or after the next interval or scheduled rotation of
807    /// `path`'s file. The file's first row sets that rotation from its own `ts_init`; sealing the
808    /// file forgets it, so a due rotation stays due until the file is sealed.
809    fn scheduled_rotation_due(&mut self, path: &FileWriterPath, ts_init: u64) -> bool {
810        if let Some(next) = self.next_rotation_times.get(path) {
811            return ts_init >= next.as_u64();
812        }
813
814        let next = match &self.rotation_config {
815            RotationConfig::Interval { interval_ns } => {
816                UnixNanos::from(ts_init) + DurationNanos::new(*interval_ns)
817            }
818            RotationConfig::ScheduledDates {
819                interval_ns,
820                rotation_time,
821                rotation_timezone,
822            } => Self::next_scheduled_rotation(
823                UnixNanos::from(ts_init),
824                *rotation_time,
825                rotation_timezone,
826                *interval_ns,
827            ),
828            _ => return false,
829        };
830        self.next_rotation_times.insert(path.clone(), next);
831        false
832    }
833
834    /// The first scheduled rotation after `after`: today's `rotation_time` in `rotation_timezone`,
835    /// advanced by `interval_ns` until it passes `after`.
836    fn next_scheduled_rotation(
837        after: UnixNanos,
838        rotation_time: UnixNanos,
839        rotation_timezone: &TimeZone,
840        interval_ns: u64,
841    ) -> UnixNanos {
842        let now_utc = after.to_datetime_utc();
843        let now_local = rotation_timezone.to_datetime(now_utc);
844
845        let rotation_time_secs = u32::try_from(*rotation_time / 1_000_000_000).unwrap_or(0);
846        let rotation_time_nanos = i32::try_from(*rotation_time % 1_000_000_000).unwrap_or(0);
847
848        let rotation_time = if rotation_time_secs < 86_400 {
849            Time::new(
850                i8::try_from(rotation_time_secs / 3_600).unwrap_or(0),
851                i8::try_from(rotation_time_secs % 3_600 / 60).unwrap_or(0),
852                i8::try_from(rotation_time_secs % 60).unwrap_or(0),
853                rotation_time_nanos,
854            )
855            .unwrap_or(Time::MIN)
856        } else {
857            Time::MIN
858        };
859
860        let local_rotation = now_local.date().to_datetime(rotation_time);
861        let ambiguous = rotation_timezone.to_ambiguous_timestamp(local_rotation);
862
863        let mut next_rotation = match ambiguous.offset() {
864            AmbiguousOffset::Gap { .. } => now_utc,
865            _ => ambiguous.earlier().unwrap_or(now_utc),
866        };
867
868        if next_rotation <= now_utc {
869            // If the time has already passed today, we would usually add the interval
870            // But let's align exactly with how Python does it:
871            while next_rotation <= now_utc {
872                next_rotation += SignedDuration::from_nanos_i128(i128::from(interval_ns));
873            }
874        }
875
876        UnixNanos::from(u64::try_from(next_rotation.as_nanosecond()).unwrap_or(0))
877    }
878
879    // Seals the open file at `path`; the next write for its key opens a new file. A failed seal
880    // leaves the stream unfinished, so the file is recovered to its complete batches instead.
881    fn seal_file(&mut self, path: &FileWriterPath) -> Result<(), Box<dyn std::error::Error>> {
882        self.next_rotation_times.remove(path);
883
884        let Some(file) = self.writers.remove(path) else {
885            return Ok(());
886        };
887
888        let partial_path = file.partial_path.clone();
889        let result = file
890            .seal()
891            .inspect_err(|_| recover_released_feather_file(&partial_path));
892        self.notify_sealed(path);
893        result
894    }
895
896    // A failed append can leave a partial IPC message, so the file is recovered to its complete
897    // batches rather than sealed as written; the next write for its key opens a new file.
898    fn abandon_file(&mut self, path: &FileWriterPath) {
899        self.next_rotation_times.remove(path);
900
901        let Some(file) = self.writers.remove(path) else {
902            return;
903        };
904
905        let partial_path = file.partial_path.clone();
906        drop(file);
907
908        log::error!(
909            "Abandoned Feather file {} after a failed write",
910            path.path.display()
911        );
912
913        recover_released_feather_file(&partial_path);
914        self.notify_sealed(path);
915    }
916
917    fn notify_sealed(&self, path: &FileWriterPath) {
918        if let Some(sealed_tx) = &self.sealed_tx
919            && path.path.exists()
920        {
921            let _ = sealed_tx.send(path.path.clone());
922        }
923    }
924
925    /// Creates (and inserts) a new `FileWriter` for type T.
926    fn create_writer<T>(
927        &mut self,
928        path: FileWriterPath,
929        data: &T,
930    ) -> Result<(), Box<dyn std::error::Error>>
931    where
932        T: EncodeToRecordBatch + CatalogFamily + 'static,
933    {
934        self.create_writer_with_metadata::<T>(path, T::metadata(data))
935    }
936
937    /// Creates (and inserts) a new `FileWriter` for type T with pre-computed metadata.
938    ///
939    /// Use this variant when the caller has selected metadata from a chunk
940    /// (e.g. via `T::chunk_metadata`) to avoid schema poisoning by sentinel rows.
941    fn create_writer_with_metadata<T>(
942        &mut self,
943        path: FileWriterPath,
944        metadata: HashMap<String, String>,
945    ) -> Result<(), Box<dyn std::error::Error>>
946    where
947        T: EncodeToRecordBatch + CatalogFamily + 'static,
948    {
949        let schema = Self::schema_with_delta_staging_columns(&schema_with_identifier_column(
950            &T::get_schema(Some(metadata)),
951        ));
952
953        let file = FeatherFile::create(&path.path, &schema, &self.rotation_config)?;
954        self.writers.insert(path, file);
955        Ok(())
956    }
957
958    /// Creates (and inserts) a new `FeatherBuffer` for custom data at the given path.
959    fn create_custom_writer(
960        &mut self,
961        path: FileWriterPath,
962        type_name: &str,
963    ) -> Result<(), Box<dyn std::error::Error>> {
964        if self.writers.contains_key(&path) {
965            return Ok(());
966        }
967
968        let base_schema = get_arrow_schema(type_name).ok_or_else(|| {
969            format!("Custom data type \"{type_name}\" is not registered for Arrow encoding")
970        })?;
971
972        let schema = schema_with_data_type_column(base_schema.as_ref(), type_name);
973
974        let schema =
975            Self::schema_with_delta_staging_columns(&schema_with_identifier_column(&schema));
976
977        let file = FeatherFile::create(&path.path, &schema, &self.rotation_config)
978            .map_err(|e| format!("Failed to create Feather file for custom {type_name}: {e}"))?;
979        self.writers.insert(path, file);
980        Ok(())
981    }
982
983    /// Encodes a single `CustomData` into a `RecordBatch` with `data_type` column (catalog-compatible).
984    pub(crate) fn encode_custom_to_batch(
985        custom: &CustomData,
986    ) -> Result<RecordBatch, Box<dyn std::error::Error>> {
987        let type_name = custom.data.type_name();
988        let data_type_json = custom
989            .data_type
990            .to_persistence_json()
991            .map_err(|e| format!("Failed to serialize data_type for persistence: {e}"))?;
992        let dt_meta = custom.data_type.metadata_string_map();
993        let items: [Arc<dyn CustomDataTrait>; 1] = [Arc::clone(&custom.data)];
994
995        let batch = encode_custom_to_arrow(type_name, &items)
996            .map_err(|e| format!("Failed to encode custom data: {e}"))?
997            .ok_or_else(|| {
998                format!("Custom data type \"{type_name}\" is not registered for Arrow")
999            })?;
1000
1001        let batch = augment_batch_with_data_type_column(
1002            &batch,
1003            &data_type_json,
1004            type_name,
1005            dt_meta.as_ref(),
1006        )
1007        .map_err(|e| e.to_string())?;
1008        Ok(batch)
1009    }
1010
1011    fn schema_with_delta_staging_columns(schema: &Schema) -> Schema {
1012        let mut fields = schema
1013            .fields()
1014            .iter()
1015            .map(|field| {
1016                Arc::new(Field::new(
1017                    field.name().clone(),
1018                    field.data_type().clone(),
1019                    field.is_nullable(),
1020                ))
1021            })
1022            .collect::<Vec<_>>();
1023
1024        if schema.index_of(NAUTILUS_ARROW_METADATA_ID_COLUMN).is_err() {
1025            fields.push(Arc::new(Field::new(
1026                NAUTILUS_ARROW_METADATA_ID_COLUMN,
1027                staged_metadata_data_type(),
1028                false,
1029            )));
1030        }
1031
1032        if schema
1033            .index_of(NAUTILUS_ARROW_METADATA_JSON_COLUMN)
1034            .is_err()
1035        {
1036            fields.push(Arc::new(Field::new(
1037                NAUTILUS_ARROW_METADATA_JSON_COLUMN,
1038                staged_metadata_data_type(),
1039                false,
1040            )));
1041        }
1042
1043        Schema::new(fields)
1044    }
1045
1046    /// Flushes buffered bytes of every open file to disk.
1047    ///
1048    /// This is called automatically based on `flush_interval_ms` if configured, but can also
1049    /// be called manually by the client. Files stay open, so flushing never starts a new file.
1050    pub fn flush(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1051        let paths = self.writers.keys().cloned().collect::<Vec<_>>();
1052
1053        for path in paths {
1054            if let Some(file) = self.writers.get_mut(&path)
1055                && let Err(e) = file.flush()
1056            {
1057                self.abandon_file(&path);
1058                return Err(e.into());
1059            }
1060        }
1061
1062        self.last_flush_ns = self.clock.timestamp_ns();
1063
1064        if let Some(error) = self.pending_write_error.take() {
1065            return Err(error.into());
1066        }
1067
1068        Ok(())
1069    }
1070
1071    /// Seals every open file so its complete stream is visible as a `.feather` file.
1072    ///
1073    /// Writes after sealing open new files, so this is the boundary promotion uses.
1074    pub fn seal(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1075        let paths = self.writers.keys().cloned().collect::<Vec<_>>();
1076        let mut first_error = None;
1077
1078        for path in paths {
1079            if let Err(e) = self.seal_file(&path) {
1080                log::error!("Failed to seal Feather file {}: {e}", path.path.display());
1081                first_error.get_or_insert(e);
1082            }
1083        }
1084
1085        match first_error {
1086            Some(e) => Err(e),
1087            None => Ok(()),
1088        }
1089    }
1090
1091    /// Seals all open files and reports any write error recorded since the last flush.
1092    ///
1093    /// Writes after closing open new files.
1094    pub fn close(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1095        self.seal()?;
1096
1097        if let Some(error) = self.pending_write_error.take() {
1098            return Err(error.into());
1099        }
1100
1101        Ok(())
1102    }
1103
1104    /// Returns whether the writer has been closed (all writers cleared).
1105    #[must_use]
1106    pub fn is_closed(&self) -> bool {
1107        self.writers.is_empty()
1108    }
1109
1110    /// Returns the current file size and path of each open file, keyed by the type it stages.
1111    #[must_use]
1112    pub fn get_current_file_info(&self) -> HashMap<CatalogDataType, (u64, String)> {
1113        self.writers
1114            .iter()
1115            .map(|(path, file)| {
1116                (
1117                    path.data_type.clone(),
1118                    (file.size(), path.path.display().to_string()),
1119                )
1120            })
1121            .collect()
1122    }
1123
1124    /// Returns the next rotation time of the file staging `data_type`, if set.
1125    ///
1126    /// Each instrument class stages its own file, so pass [`CatalogDataType::Instrument`].
1127    #[must_use]
1128    pub fn get_next_rotation_time(&self, data_type: &CatalogDataType) -> Option<UnixNanos> {
1129        self.next_rotation_times
1130            .iter()
1131            .find(|(path, _)| path.data_type == *data_type)
1132            .map(|(_, &next)| next)
1133    }
1134
1135    /// Determines whether a family can be written before row-level identifier checks.
1136    fn should_write_family(&self, family: &CatalogDataType) -> bool {
1137        self.included_types
1138            .as_ref()
1139            .is_none_or(|included| included.contains(family))
1140            && self
1141                .record_filter
1142                .as_ref()
1143                .is_none_or(|filter| filter.contains(family))
1144    }
1145
1146    fn should_write_record(&self, data_type: &CatalogDataType, identifier: Option<&str>) -> bool {
1147        self.included_types
1148            .as_ref()
1149            .is_none_or(|included| included.contains(&catalog_family(data_type)))
1150            && self
1151                .record_filter
1152                .as_ref()
1153                .is_none_or(|filter| filter.allows(data_type, identifier))
1154    }
1155
1156    fn reserve_writer_path(&mut self, data_type: CatalogDataType) -> FileWriterPath {
1157        let now = self.clock.timestamp_ns();
1158        let (timestamp, first_sequence) = self
1159            .path_sequences
1160            .get(&data_type)
1161            .copied()
1162            .filter(|(timestamp, _)| *timestamp >= now)
1163            .unwrap_or((now, 0));
1164
1165        for sequence in first_sequence.. {
1166            let path = self.build_writer_path(&data_type, timestamp, sequence);
1167
1168            // Files left by an earlier writer in the same directory keep their names
1169            if path.exists() || path.with_extension(FEATHER_PARTIAL_EXTENSION).exists() {
1170                continue;
1171            }
1172
1173            self.path_sequences
1174                .insert(data_type.clone(), (timestamp, sequence + 1));
1175            return FileWriterPath { path, data_type };
1176        }
1177
1178        unreachable!("unbounded writer path sequence exhausted")
1179    }
1180
1181    fn build_writer_path(
1182        &self,
1183        data_type: &CatalogDataType,
1184        timestamp: UnixNanos,
1185        sequence: u64,
1186    ) -> PathBuf {
1187        let (directory, file_stem) = match data_type {
1188            CatalogDataType::Data(NautilusDataType::Custom { type_name }) => (
1189                self.directory.join("data").join("custom").join(type_name),
1190                Cow::Borrowed(type_name.as_str()),
1191            ),
1192            CatalogDataType::Data(data_type) => {
1193                let prefix = data_path_prefix(data_type);
1194                (self.directory.join(prefix.as_ref()), prefix)
1195            }
1196            CatalogDataType::Record(record_type) => {
1197                let prefix = record_path_prefix(record_type);
1198                (self.directory.join(prefix.as_ref()), prefix)
1199            }
1200            CatalogDataType::Instrument(class) => {
1201                let prefix = data_path_prefix(&NautilusDataType::Instrument);
1202                (
1203                    self.directory
1204                        .join(prefix.as_ref())
1205                        .join(instrument_path_prefix(class)),
1206                    prefix,
1207                )
1208            }
1209        };
1210
1211        directory.join(Self::timestamped_feather_file_name(
1212            &file_stem, timestamp, sequence,
1213        ))
1214    }
1215
1216    fn timestamped_feather_file_name(stem: &str, timestamp: UnixNanos, sequence: u64) -> String {
1217        if sequence == 0 {
1218            format!("{stem}_{timestamp}.{FEATHER_EXTENSION}")
1219        } else {
1220            format!("{stem}_{timestamp}-{sequence}.{FEATHER_EXTENSION}")
1221        }
1222    }
1223
1224    /// Returns the open file of a custom type, or reserves `data/custom/{type_name}/` for it.
1225    fn get_writer_path_custom(
1226        &mut self,
1227        type_name: &str,
1228    ) -> Result<FileWriterPath, Box<dyn std::error::Error>> {
1229        let data_type = CatalogDataType::from(NautilusDataType::Custom {
1230            type_name: type_name.to_string(),
1231        });
1232
1233        if let Some(existing) = self.writers.keys().find(|path| path.data_type == data_type) {
1234            return Ok(existing.clone());
1235        }
1236
1237        if let Some(schema) = get_arrow_schema(type_name) {
1238            validate_custom_catalog_schema(type_name, &schema)?;
1239        }
1240
1241        Ok(self.reserve_writer_path(data_type))
1242    }
1243
1244    /// Returns the open file staging `data_type`, or reserves a new one.
1245    fn get_writer_path(&mut self, data_type: CatalogDataType) -> FileWriterPath {
1246        if let Some(existing) = self.writers.keys().find(|path| path.data_type == data_type) {
1247            return existing.clone();
1248        }
1249
1250        self.reserve_writer_path(data_type)
1251    }
1252
1253    /// Writes a Data enum value to the appropriate writer.
1254    ///
1255    /// This is a convenience method that routes the Data enum to the appropriate
1256    /// typed write method.
1257    pub fn write_data(&mut self, data: Data) -> Result<(), Box<dyn std::error::Error>> {
1258        match data {
1259            Data::Instrument(instrument) => self.write(*instrument),
1260            Data::Quote(quote) => self.write(quote),
1261            Data::Trade(trade) => self.write(trade),
1262            Data::Bar(bar) => self.write(bar),
1263            Data::BookDelta(delta) => self.write(delta),
1264            Data::BookDepth(depth) => self.write(*depth),
1265            Data::IndexPrice(price) => self.write(price),
1266            Data::MarkPrice(price) => self.write(price),
1267            Data::FundingRate(funding) => self.write(funding),
1268            Data::InstrumentStatus(status) => self.write(status),
1269            Data::OptionGreeks(greeks) => self.write(greeks),
1270            Data::InstrumentClose(close) => self.write(close),
1271            Data::Custom(custom) => self.write_custom_data(&custom),
1272            Data::BookDeltas(deltas_api) => {
1273                // Batch write so chunk_metadata can skip a leading BookAction::Clear sentinel
1274                self.write_batch(deltas_api.deltas.clone())
1275            }
1276            #[cfg(feature = "defi")]
1277            Data::Defi(_) => Err("Unsupported DeFi data variant for feather writes".into()),
1278            #[cfg(not(feature = "defi"))]
1279            #[allow(
1280                unreachable_patterns,
1281                reason = "DeFi variants can exist without this crate's defi feature"
1282            )]
1283            _ => Err("Unsupported DeFi data variant for feather writes".into()),
1284        }
1285    }
1286
1287    /// Writes mixed data as typed Arrow batches.
1288    ///
1289    /// # Panics
1290    ///
1291    /// Panics if the built-in data batch dispatch becomes non-exhaustive.
1292    #[expect(
1293        clippy::needless_pass_by_value,
1294        reason = "the public writer API accepts ownership of each submitted data batch"
1295    )]
1296    pub fn write_data_batch(&mut self, data: Vec<Data>) -> Result<(), Box<dyn std::error::Error>> {
1297        for batch in DataBatch::from_data_vec_grouped(&data)? {
1298            match &batch {
1299                DataBatch::Custom(data) => {
1300                    for custom in data.as_ref() {
1301                        self.write_custom_data(custom)?;
1302                    }
1303                }
1304                batch => write_builtin_data_batch(self, batch)
1305                    .expect("built-in data batch dispatch is exhaustive")?,
1306            }
1307        }
1308
1309        Ok(())
1310    }
1311
1312    /// Writes supported message bus value.
1313    pub fn write_any_message(
1314        &mut self,
1315        message: &dyn Any,
1316    ) -> Result<bool, Box<dyn std::error::Error>> {
1317        let Some(command) = Self::write_command(message) else {
1318            return Ok(false);
1319        };
1320
1321        if let Err(e) = command(self) {
1322            self.record_write_error(e.to_string());
1323            return Err(e);
1324        }
1325
1326        Ok(true)
1327    }
1328
1329    pub(crate) fn record_write_error(&mut self, error: String) {
1330        self.pending_write_error.get_or_insert(error);
1331    }
1332
1333    pub(crate) fn write_command(message: &dyn Any) -> Option<FeatherWriteCommand> {
1334        macro_rules! try_write {
1335            ($message:expr, $type:ty) => {
1336                if let Some(value) = $message.downcast_ref::<$type>() {
1337                    let value = value.clone();
1338                    return Some(Box::new(move |writer| writer.write(value)));
1339                }
1340            };
1341        }
1342
1343        try_write!(message, QuoteTick);
1344        try_write!(message, TradeTick);
1345        try_write!(message, Bar);
1346        try_write!(message, OrderBookDelta);
1347        try_write!(message, OrderBookDepth);
1348        try_write!(message, IndexPriceUpdate);
1349        try_write!(message, MarkPriceUpdate);
1350        try_write!(message, FundingRateUpdate);
1351        try_write!(message, InstrumentStatus);
1352        try_write!(message, OptionGreeks);
1353        try_write!(message, InstrumentClose);
1354        try_write!(message, InstrumentAny);
1355        try_write!(message, AccountState);
1356        try_write!(message, OrderInitialized);
1357        try_write!(message, OrderDenied);
1358        try_write!(message, OrderEmulated);
1359        try_write!(message, OrderSubmitted);
1360        try_write!(message, OrderAccepted);
1361        try_write!(message, OrderRejected);
1362        try_write!(message, OrderPendingCancel);
1363        try_write!(message, OrderCanceled);
1364        try_write!(message, OrderCancelDenied);
1365        try_write!(message, OrderCancelRejected);
1366        try_write!(message, OrderExpired);
1367        try_write!(message, OrderTriggered);
1368        try_write!(message, OrderPendingUpdate);
1369        try_write!(message, OrderReleased);
1370        try_write!(message, OrderModifyDenied);
1371        try_write!(message, OrderModifyRejected);
1372        try_write!(message, OrderUpdated);
1373        try_write!(message, OrderFilled);
1374        try_write!(message, OrderFillVoided);
1375        try_write!(message, PositionOpened);
1376        try_write!(message, PositionChanged);
1377        try_write!(message, PositionClosed);
1378        try_write!(message, PositionAdjusted);
1379        try_write!(message, OrderSnapshot);
1380        try_write!(message, PositionSnapshot);
1381        try_write!(message, OrderStatusReport);
1382        try_write!(message, FillReport);
1383        try_write!(message, PositionStatusReport);
1384        try_write!(message, ExecutionMassStatus);
1385
1386        if let Some(greeks) = message.downcast_ref::<GreeksData>() {
1387            ensure_custom_data_registered::<GreeksData>();
1388
1389            let data_type = CustomDataType::new(
1390                GreeksData::type_name_static(),
1391                None,
1392                Some(greeks.instrument_id.to_string()),
1393            );
1394
1395            let custom = CustomData::new(Arc::new(greeks.clone()), data_type);
1396            return Some(Box::new(move |writer| writer.write_custom_data(&custom)));
1397        }
1398
1399        if let Some(portfolio) = message.downcast_ref::<PortfolioGreeks>() {
1400            ensure_custom_data_registered::<PortfolioGreeks>();
1401            let custom = CustomData::from_arc(Arc::new(portfolio.clone()));
1402            return Some(Box::new(move |writer| writer.write_custom_data(&custom)));
1403        }
1404
1405        if let Some(deltas) = message.downcast_ref::<OrderBookDeltas>() {
1406            let deltas = deltas.deltas.clone();
1407            return Some(Box::new(move |writer| writer.write_batch(deltas)));
1408        }
1409
1410        if let Some(data) = message.downcast_ref::<Data>() {
1411            let data = data.clone();
1412            return Some(Box::new(move |writer| writer.write_data(data)));
1413        }
1414
1415        if let Some(custom) = message.downcast_ref::<CustomData>() {
1416            let custom = custom.clone();
1417            return Some(Box::new(move |writer| {
1418                writer.write_data(Data::Custom(custom))
1419            }));
1420        }
1421
1422        if let Some(event) = message.downcast_ref::<OrderEventAny>() {
1423            let event = event.clone();
1424            return Some(Box::new(move |writer| writer.write_order_event(&event)));
1425        }
1426
1427        if let Some(event) = message.downcast_ref::<PositionEvent>() {
1428            let event = event.clone();
1429            return Some(Box::new(move |writer| writer.write_position_event(&event)));
1430        }
1431
1432        None
1433    }
1434
1435    fn write_order_event(
1436        &mut self,
1437        event: &OrderEventAny,
1438    ) -> Result<(), Box<dyn std::error::Error>> {
1439        match event {
1440            OrderEventAny::Initialized(event) => self.write(event.clone()),
1441            OrderEventAny::Denied(event) => self.write(*event),
1442            OrderEventAny::Emulated(event) => self.write(*event),
1443            OrderEventAny::Released(event) => self.write(*event),
1444            OrderEventAny::Submitted(event) => self.write(*event),
1445            OrderEventAny::Accepted(event) => self.write(*event),
1446            OrderEventAny::Rejected(event) => self.write(*event),
1447            OrderEventAny::Canceled(event) => self.write(*event),
1448            OrderEventAny::Expired(event) => self.write(*event),
1449            OrderEventAny::Triggered(event) => self.write(*event),
1450            OrderEventAny::PendingUpdate(event) => self.write(*event),
1451            OrderEventAny::PendingCancel(event) => self.write(*event),
1452            OrderEventAny::ModifyDenied(event) => self.write(*event),
1453            OrderEventAny::ModifyRejected(event) => self.write(*event),
1454            OrderEventAny::CancelDenied(event) => self.write(*event),
1455            OrderEventAny::CancelRejected(event) => self.write(*event),
1456            OrderEventAny::Updated(event) => self.write(*event),
1457            OrderEventAny::Filled(event) => self.write(event.clone()),
1458            OrderEventAny::FillVoided(event) => self.write(event.clone()),
1459        }
1460    }
1461
1462    fn write_position_event(
1463        &mut self,
1464        event: &PositionEvent,
1465    ) -> Result<(), Box<dyn std::error::Error>> {
1466        match event {
1467            PositionEvent::PositionOpened(event) => self.write(event.clone()),
1468            PositionEvent::PositionChanged(event) => self.write(event.clone()),
1469            PositionEvent::PositionClosed(event) => self.write(event.clone()),
1470            PositionEvent::PositionAdjusted(event) => self.write(*event),
1471        }
1472    }
1473
1474    /// Writes a single custom data value (catalog layout: `data/custom/{type_name}[/{identifier}]`).
1475    fn write_custom_data(&mut self, custom: &CustomData) -> Result<(), Box<dyn std::error::Error>> {
1476        let batch = Self::encode_custom_to_batch(custom)?;
1477        self.write_custom_batch(custom, batch)
1478    }
1479
1480    pub(crate) fn write_custom_batch(
1481        &mut self,
1482        custom: &CustomData,
1483        mut batch: RecordBatch,
1484    ) -> Result<(), Box<dyn std::error::Error>> {
1485        let type_name = custom.data.type_name();
1486        let identifier = custom.data_type.identifier().map(String::from);
1487
1488        if !self.should_write_custom(type_name, identifier.as_deref()) {
1489            return Ok(());
1490        }
1491
1492        let path = self.get_writer_path_custom(type_name)?;
1493        if !self.writers.contains_key(&path) {
1494            self.create_custom_writer(path.clone(), type_name)?;
1495        }
1496
1497        let metadata_row = arrow_metadata_row(batch.schema().metadata(), batch.schema().fields())?;
1498        batch = record_batch_with_identifier_column(batch, custom.data_type.identifier())?;
1499        batch =
1500            record_batch_with_delta_staged_metadata(&batch, std::slice::from_ref(&metadata_row))?;
1501
1502        self.stage_batch_writes(path, std::slice::from_ref(&batch))?;
1503        self.flush_if_due()
1504    }
1505
1506    pub(crate) fn should_write_custom(&self, type_name: &str, identifier: Option<&str>) -> bool {
1507        self.should_write_record(
1508            &CatalogDataType::from(NautilusDataType::Custom {
1509                type_name: type_name.to_string(),
1510            }),
1511            identifier,
1512        )
1513    }
1514
1515    /// Writes an instrument to the appropriate writer.
1516    ///
1517    /// Each instrument class stages its own file, and every row carries its instrument ID.
1518    pub fn write_instrument(
1519        &mut self,
1520        instrument: InstrumentAny,
1521    ) -> Result<(), Box<dyn std::error::Error>> {
1522        self.write(instrument)
1523    }
1524
1525    /// Subscribes to all messages on the message bus (pattern "*").
1526    ///
1527    /// This will automatically write all supported data types that are published
1528    /// on the message bus to the feather files.
1529    ///
1530    /// The writer must be wrapped in `Rc<RefCell<>>` to be shareable with the message bus handler.
1531    ///
1532    /// Note: Writes are synchronous local file appends.
1533    pub fn subscribe_to_message_bus(
1534        writer: Rc<RefCell<Self>>,
1535    ) -> Result<StreamingSinkSubscription, Box<dyn std::error::Error>> {
1536        let sink: StreamingDataSink = Box::new(writer);
1537        Ok(StreamingSinkSubscription::subscribe(
1538            Rc::new(RefCell::new(sink)),
1539            None,
1540        ))
1541    }
1542
1543    /// Unsubscribes from message bus.
1544    pub fn unsubscribe_from_message_bus(handler: &StreamingSinkSubscription) {
1545        handler.unsubscribe();
1546    }
1547}
1548
1549impl Debug for FeatherWriter {
1550    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1551        f.debug_struct(stringify!(FeatherWriter))
1552            .finish_non_exhaustive()
1553    }
1554}
1555
1556/// Drop contract: seals open files so their streams stay readable and promotable.
1557///
1558/// Errors are logged, never raised. Call [`FeatherWriter::close`] for deterministic sealing.
1559impl Drop for FeatherWriter {
1560    fn drop(&mut self) {
1561        if !self.writers.is_empty() {
1562            let _ = self.seal();
1563        }
1564    }
1565}
1566
1567impl StreamingSink for FeatherWriter {
1568    fn write_data(&mut self, data: Data) -> anyhow::Result<()> {
1569        Self::write_data(self, data).map_err(feather_error)
1570    }
1571
1572    fn write_batch(&mut self, data: Vec<Data>) -> anyhow::Result<()> {
1573        Self::write_data_batch(self, data).map_err(feather_error)
1574    }
1575
1576    fn write_any(&mut self, message: &dyn Any) -> anyhow::Result<bool> {
1577        self.write_any_message(message).map_err(feather_error)
1578    }
1579
1580    fn flush(&mut self) -> anyhow::Result<()> {
1581        Self::flush(self).map_err(feather_error)
1582    }
1583
1584    fn close(&mut self) -> anyhow::Result<()> {
1585        Self::close(self).map_err(feather_error)
1586    }
1587}
1588
1589impl StreamingSink for Rc<RefCell<FeatherWriter>> {
1590    fn write_data(&mut self, data: Data) -> anyhow::Result<()> {
1591        StreamingSink::write_data(&mut *self.borrow_mut(), data)
1592    }
1593
1594    fn write_batch(&mut self, data: Vec<Data>) -> anyhow::Result<()> {
1595        StreamingSink::write_batch(&mut *self.borrow_mut(), data)
1596    }
1597
1598    fn write_any(&mut self, message: &dyn Any) -> anyhow::Result<bool> {
1599        StreamingSink::write_any(&mut *self.borrow_mut(), message)
1600    }
1601
1602    fn flush(&mut self) -> anyhow::Result<()> {
1603        StreamingSink::flush(&mut *self.borrow_mut())
1604    }
1605
1606    fn close(&mut self) -> anyhow::Result<()> {
1607        StreamingSink::close(&mut *self.borrow_mut())
1608    }
1609}
1610
1611/// Seals every abandoned `.feather.partial` file under `directory`.
1612///
1613/// A writer locks each file until it seals it, so a file that recovery can lock was left by a
1614/// writer that crashed, was killed, or released it after a failed write or seal. Recovery keeps the
1615/// complete record batches, drops any bytes after them with a warning, and renames the file to
1616/// `.feather`. It removes a file with no complete record batch and leaves an empty one, which a new
1617/// writer may not have locked yet. A read error other than a write cut short leaves the file for a
1618/// later pass. Failures are logged per file, so callers still read the sealed files.
1619pub(crate) fn recover_partial_feather_files(directory: &Path) {
1620    let suffix = format!(".{FEATHER_PARTIAL_EXTENSION}");
1621    let mut directories = vec![directory.to_path_buf()];
1622
1623    while let Some(directory) = directories.pop() {
1624        let entries = match fs::read_dir(&directory) {
1625            Ok(entries) => entries,
1626            Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
1627            Err(e) => {
1628                log::warn!(
1629                    "Failed to list {} for partial Feather files: {e}",
1630                    directory.display()
1631                );
1632                continue;
1633            }
1634        };
1635
1636        for entry in entries.flatten() {
1637            let path = entry.path();
1638
1639            if entry.file_type().is_ok_and(|file_type| file_type.is_dir()) {
1640                directories.push(path);
1641            } else if path.to_string_lossy().ends_with(&suffix)
1642                && let Err(e) = recover_partial_feather_file(&path)
1643            {
1644                log::warn!("Failed to recover Feather file {}: {e}", path.display());
1645            }
1646        }
1647    }
1648}
1649
1650// Recovers a file its writer released after a failure, so its complete batches stay promotable
1651fn recover_released_feather_file(partial_path: &Path) {
1652    if let Err(e) = recover_partial_feather_file(partial_path) {
1653        log::error!(
1654            "Failed to recover Feather file {}: {e}",
1655            partial_path.display()
1656        );
1657    }
1658}
1659
1660fn recover_partial_feather_file(partial_path: &Path) -> anyhow::Result<()> {
1661    let metadata = match fs::symlink_metadata(partial_path) {
1662        Ok(metadata) => metadata,
1663        Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
1664        Err(e) => return Err(e.into()),
1665    };
1666
1667    anyhow::ensure!(
1668        metadata.is_file(),
1669        "Partial Feather path must be a regular file: {}",
1670        partial_path.display(),
1671    );
1672
1673    let file = match OpenOptions::new().read(true).write(true).open(partial_path) {
1674        Ok(file) => file,
1675        Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
1676        Err(e) => return Err(e.into()),
1677    };
1678
1679    match file.try_lock() {
1680        Ok(()) => {}
1681        Err(TryLockError::WouldBlock) => return Ok(()),
1682        Err(TryLockError::Error(e)) => return Err(e.into()),
1683    }
1684
1685    let len = file.metadata()?.len();
1686
1687    // A writer renames its file before unlocking it, and a new file stays empty until its writer
1688    // locks it, so recovery leaves a missing or empty file alone
1689    if !partial_path.try_exists()? || len == 0 {
1690        return Ok(());
1691    }
1692
1693    let mut batches = 0;
1694    let mut complete_len = 0;
1695    let mut tail_error = None;
1696
1697    match StreamReader::try_new(BufReader::new(&file), None) {
1698        Ok(mut reader) => loop {
1699            match reader.next() {
1700                Some(Ok(_)) => {
1701                    batches += 1;
1702                    complete_len = reader.get_mut().stream_position()?;
1703                }
1704                Some(Err(e)) => {
1705                    tail_error = Some(e);
1706                    break;
1707                }
1708                None => break,
1709            }
1710        },
1711        Err(e) => tail_error = Some(e),
1712    }
1713
1714    // Only a write cut short ends the stream inside a message; any other read error could hide
1715    // complete batches after it, so the file waits for a later pass
1716    if let Some(e) = tail_error.take_if(|e| {
1717        !matches!(e, ArrowError::IoError(_, source) if source.kind() == io::ErrorKind::UnexpectedEof)
1718    }) {
1719        return Err(e.into());
1720    }
1721
1722    let cause = tail_error.map_or_else(String::new, |e| format!(": {e}"));
1723
1724    if batches == 0 {
1725        fs::remove_file(partial_path)?;
1726
1727        log::warn!(
1728            "Removed Feather file {} with no complete record batch{cause}",
1729            partial_path.display()
1730        );
1731        return Ok(());
1732    }
1733
1734    let dropped = len.saturating_sub(complete_len);
1735
1736    // Arrow C++ rejects trailing bytes that form no message, and EOF at a message boundary ends a
1737    // valid stream, so the file ends at its last complete batch
1738    if dropped > 0 {
1739        file.set_len(complete_len)?;
1740    }
1741
1742    file.sync_all()?;
1743
1744    let sealed_path = partial_path.with_extension("");
1745    fs::rename(partial_path, &sealed_path)?;
1746
1747    let tail = if dropped == 0 {
1748        String::new()
1749    } else {
1750        format!(", dropped {dropped} trailing byte(s){cause}")
1751    };
1752
1753    log::warn!(
1754        "Recovered {batches} record batch(es) into Feather file {}{tail}",
1755        sealed_path.display()
1756    );
1757    Ok(())
1758}
1759
1760#[cfg(test)]
1761mod tests {
1762    use std::sync::{Arc, Mutex, atomic::Ordering};
1763
1764    use nautilus_common::{clock::VirtualClock, live::LiveClock};
1765    #[cfg(target_os = "linux")]
1766    use nautilus_model::data::HasTsInit;
1767    use nautilus_model::{
1768        data::{Data, NautilusRecordType, QuoteTick, TradeTick},
1769        enums::AggressorSide,
1770        identifiers::{InstrumentId, TradeId},
1771        types::{ERROR_PRICE, Price, Quantity},
1772    };
1773    use nautilus_serialization::arrow::{
1774        ArrowSchemaProvider, DecodeDataFromRecordBatch, EncodeToRecordBatch,
1775    };
1776    use rstest::rstest;
1777    use tempfile::TempDir;
1778
1779    use super::*;
1780    use crate::{
1781        common::datafusion::identifiers_from_record_batches,
1782        writer::materializer::restore_staged_record_batches,
1783    };
1784
1785    fn feather_files(directory: &Path, extension: &str) -> Vec<PathBuf> {
1786        let mut files = Vec::new();
1787        let mut stack = vec![directory.to_path_buf()];
1788
1789        while let Some(path) = stack.pop() {
1790            for entry in fs::read_dir(&path).unwrap().flatten() {
1791                let path = entry.path();
1792                if path.is_dir() {
1793                    stack.push(path);
1794                } else if path.to_string_lossy().ends_with(&format!(".{extension}")) {
1795                    files.push(path);
1796                }
1797            }
1798        }
1799
1800        files.sort();
1801        files
1802    }
1803
1804    // Reads a type's file and splits it into batches that each carry their rows' metadata
1805    fn read_feather_batches(path: &Path) -> Vec<RecordBatch> {
1806        StreamReader::try_new(File::open(path).unwrap(), None)
1807            .unwrap()
1808            .map(Result::unwrap)
1809            .flat_map(|batch| restore_staged_record_batches(batch).unwrap())
1810            .collect()
1811    }
1812
1813    fn read_quotes(path: &Path) -> Vec<Data> {
1814        read_feather_batches(path)
1815            .into_iter()
1816            .flat_map(|batch| {
1817                let metadata = batch.schema().metadata().clone();
1818                QuoteTick::decode_data_batch(&metadata, batch).unwrap()
1819            })
1820            .collect()
1821    }
1822
1823    fn quote_at(ts: u64) -> QuoteTick {
1824        quote_for("AUD/USD.SIM", ts)
1825    }
1826
1827    fn quote_for(instrument_id: &str, ts: u64) -> QuoteTick {
1828        QuoteTick::new(
1829            InstrumentId::from(instrument_id),
1830            Price::from("1.0"),
1831            Price::from("1.1"),
1832            Quantity::from("1000"),
1833            Quantity::from("1000"),
1834            UnixNanos::from(ts),
1835            UnixNanos::from(ts),
1836        )
1837    }
1838
1839    fn quotes_at(timestamps: &[u64]) -> Vec<Data> {
1840        timestamps
1841            .iter()
1842            .map(|&ts| Data::from(quote_at(ts)))
1843            .collect()
1844    }
1845
1846    // Seals one flushed batch per quote, then reopens the stream as a partial file whose writer
1847    // exited before sealing it
1848    fn write_partial_file(directory: &Path, timestamps: &[u64]) -> PathBuf {
1849        let mut writer = FeatherWriter::new(
1850            directory.to_path_buf(),
1851            WriterClock::Test(Arc::new(AtomicU64::new(0))),
1852            RotationConfig::NoRotation,
1853            None,
1854            Some(0),
1855        );
1856
1857        for &ts in timestamps {
1858            writer.write(quote_at(ts)).unwrap();
1859            writer.flush().unwrap();
1860        }
1861
1862        writer.close().unwrap();
1863
1864        let sealed = feather_files(directory, FEATHER_EXTENSION).remove(0);
1865        let partial = sealed.with_extension(FEATHER_PARTIAL_EXTENSION);
1866        fs::rename(&sealed, &partial).unwrap();
1867        partial
1868    }
1869
1870    const TORN_TAIL: &str = ": Io error: failed to fill whole buffer";
1871
1872    static RECOVERY_LOGS: RecoveryLogCapture = RecoveryLogCapture(Mutex::new(Vec::new()));
1873
1874    // The logger is process-global, so only warnings naming `directory` belong to the caller
1875    fn recover_with_warnings(directory: &Path) -> Vec<String> {
1876        let _ = log::set_logger(&RECOVERY_LOGS);
1877        log::set_max_level(log::LevelFilter::Warn);
1878
1879        recover_partial_feather_files(directory);
1880
1881        let directory = directory.display().to_string();
1882        RECOVERY_LOGS
1883            .0
1884            .lock()
1885            .unwrap()
1886            .iter()
1887            .filter(|(level, message)| *level == log::Level::Warn && message.contains(&directory))
1888            .map(|(_, message)| message.clone())
1889            .collect()
1890    }
1891
1892    struct RecoveryLogCapture(Mutex<Vec<(log::Level, String)>>);
1893
1894    impl log::Log for RecoveryLogCapture {
1895        fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
1896            metadata.target() == "nautilus_persistence::writer::feather"
1897        }
1898
1899        fn log(&self, record: &log::Record<'_>) {
1900            if self.enabled(record.metadata()) {
1901                self.0
1902                    .lock()
1903                    .unwrap()
1904                    .push((record.level(), record.args().to_string()));
1905            }
1906        }
1907
1908        fn flush(&self) {}
1909    }
1910
1911    #[rstest]
1912    fn test_subscription_receives_typed_quotes_and_unsubscribes() {
1913        use nautilus_common::msgbus::{MStr, publish_quote};
1914
1915        let temp_dir = TempDir::new().unwrap();
1916
1917        let writer = Rc::new(RefCell::new(FeatherWriter::new(
1918            temp_dir.path().to_path_buf(),
1919            WriterClock::Test(Arc::new(AtomicU64::new(0))),
1920            RotationConfig::NoRotation,
1921            None,
1922            Some(0),
1923        )));
1924
1925        let quote = QuoteTick::new(
1926            InstrumentId::from("AUD/USD.SIM"),
1927            Price::from("1.0"),
1928            Price::from("1.1"),
1929            Quantity::from("2"),
1930            Quantity::from("3"),
1931            4.into(),
1932            5.into(),
1933        );
1934        let handler = FeatherWriter::subscribe_to_message_bus(writer.clone()).unwrap();
1935        publish_quote(MStr::topic("data.quotes.AUD/USD.SIM").unwrap(), &quote);
1936        FeatherWriter::unsubscribe_from_message_bus(&handler);
1937        publish_quote(MStr::topic("data.quotes.AUD/USD.SIM").unwrap(), &quote);
1938        writer.borrow_mut().close().unwrap();
1939
1940        let rows = feather_files(temp_dir.path(), FEATHER_EXTENSION)
1941            .iter()
1942            .flat_map(|path| read_feather_batches(path))
1943            .map(|batch| batch.num_rows())
1944            .sum::<usize>();
1945        assert_eq!(rows, 1);
1946    }
1947
1948    #[rstest]
1949    fn test_subscription_records_greeks_once_and_unsubscribes() {
1950        use nautilus_common::msgbus::{MStr, publish_any, publish_greeks};
1951        use nautilus_serialization::arrow::{DecodeDataFromRecordBatch, custom::CustomDataDecoder};
1952        let temp_dir = TempDir::new().unwrap();
1953
1954        let writer = Rc::new(RefCell::new(FeatherWriter::new(
1955            temp_dir.path().to_path_buf(),
1956            WriterClock::Test(Arc::new(AtomicU64::new(0))),
1957            RotationConfig::NoRotation,
1958            None,
1959            Some(0),
1960        )));
1961        let greeks = GreeksData::from_delta("AUD/USD.SIM".parse().unwrap(), 1.25, 8.0, 11.into());
1962        let portfolio = greeks.to_portfolio_greeks();
1963        let greeks_topic = MStr::topic("data.GreeksData.instrument_id=AUD/USD.SIM").unwrap();
1964        let portfolio_topic = MStr::topic("data.PortfolioGreeks").unwrap();
1965        let handler = FeatherWriter::subscribe_to_message_bus(writer.clone()).unwrap();
1966        publish_greeks(greeks_topic, &greeks);
1967        publish_any(portfolio_topic, &portfolio);
1968        FeatherWriter::unsubscribe_from_message_bus(&handler);
1969        publish_greeks(greeks_topic, &greeks);
1970        publish_any(portfolio_topic, &portfolio);
1971        writer.borrow_mut().close().unwrap();
1972        let mut counts = HashMap::new();
1973
1974        for batch in feather_files(&temp_dir.path().join("data/custom"), FEATHER_EXTENSION)
1975            .iter()
1976            .flat_map(|path| read_feather_batches(path))
1977        {
1978            let schema = batch.schema();
1979            let name = schema.metadata()["type_name"].clone();
1980            *counts.entry(name.clone()).or_insert(0) += batch.num_rows();
1981            let decoded = CustomDataDecoder::decode_data_batch(schema.metadata(), batch).unwrap();
1982            assert_eq!(decoded.len(), 1);
1983
1984            let Data::Custom(custom) = &decoded[0] else {
1985                panic!("expected custom data")
1986            };
1987
1988            match name.as_str() {
1989                "GreeksData" => {
1990                    assert_eq!(
1991                        custom.data_type,
1992                        CustomDataType::new(
1993                            "GreeksData",
1994                            None,
1995                            Some(greeks.instrument_id.to_string())
1996                        )
1997                    );
1998                    assert_eq!(
1999                        custom.data.as_any().downcast_ref::<GreeksData>(),
2000                        Some(&greeks)
2001                    );
2002                }
2003                "PortfolioGreeks" => {
2004                    assert_eq!(
2005                        custom.data_type,
2006                        CustomDataType::new("PortfolioGreeks", None, None)
2007                    );
2008                    assert_eq!(
2009                        custom.data.as_any().downcast_ref::<PortfolioGreeks>(),
2010                        Some(&portfolio)
2011                    );
2012                }
2013                _ => panic!("unexpected custom data type {name}"),
2014            }
2015        }
2016
2017        assert_eq!(
2018            counts,
2019            HashMap::from([
2020                ("GreeksData".to_string(), 1),
2021                ("PortfolioGreeks".to_string(), 1),
2022            ])
2023        );
2024    }
2025
2026    #[rstest]
2027    #[case(false)]
2028    #[case(true)]
2029    fn test_message_write_error_reaches_flush_or_close(#[case] close: bool) {
2030        let temp_dir = TempDir::new().unwrap();
2031
2032        let mut writer = FeatherWriter::new(
2033            temp_dir.path().to_path_buf(),
2034            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2035            RotationConfig::NoRotation,
2036            None,
2037            Some(0),
2038        );
2039
2040        let quote = QuoteTick::new(
2041            InstrumentId::from("AUD/USD.SIM"),
2042            ERROR_PRICE,
2043            ERROR_PRICE,
2044            Quantity::from("1"),
2045            Quantity::from("2"),
2046            3.into(),
2047            4.into(),
2048        );
2049        let write_error = writer.write_any_message(&quote).unwrap_err().to_string();
2050
2051        let error = if close {
2052            StreamingSink::close(&mut writer)
2053        } else {
2054            StreamingSink::flush(&mut writer)
2055        }
2056        .unwrap_err();
2057
2058        assert_eq!(error.to_string(), write_error);
2059    }
2060
2061    #[rstest]
2062    fn test_writer_manager_keys() {
2063        let temp_dir = TempDir::new().unwrap();
2064
2065        // Create a test time source
2066        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2067        let timestamp = clock.timestamp_ns();
2068
2069        let mut manager = FeatherWriter::new(
2070            temp_dir.path().to_path_buf(),
2071            clock,
2072            RotationConfig::NoRotation,
2073            None,
2074            None, // flush_interval_ms
2075        );
2076
2077        let instrument_id = "AAPL.AAPL";
2078
2079        // Write a dummy value
2080        let quote = QuoteTick::new(
2081            InstrumentId::from(instrument_id),
2082            Price::from("100.0"),
2083            Price::from("100.0"),
2084            Quantity::from("100.0"),
2085            Quantity::from("100.0"),
2086            UnixNanos::from(1_000_000_000_000_000_000),
2087            UnixNanos::from(1_000_000_000_000_000_000),
2088        );
2089
2090        let trade = TradeTick::new(
2091            InstrumentId::from(instrument_id),
2092            Price::from("100.0"),
2093            Quantity::from("100.0"),
2094            AggressorSide::Buy,
2095            TradeId::from("1"),
2096            UnixNanos::from(1_000_000_000_000_000_000),
2097            UnixNanos::from(1_000_000_000_000_000_000),
2098        );
2099
2100        manager.write(quote).unwrap();
2101        manager.write(trade).unwrap();
2102
2103        // Check keys and paths for quotes and trades
2104        let path = manager.get_writer_path(QuoteTick::catalog_family());
2105        let expected_path = temp_dir
2106            .path()
2107            .join("quotes")
2108            .join(format!("quotes_{timestamp}.feather"));
2109        assert_eq!(path.path, expected_path);
2110        assert!(manager.writers.contains_key(&path));
2111        let writer = manager.writers.get(&path).unwrap();
2112        assert!(writer.size() > 0);
2113
2114        let path = manager.get_writer_path(TradeTick::catalog_family());
2115        let expected_path = temp_dir
2116            .path()
2117            .join("trades")
2118            .join(format!("trades_{timestamp}.feather"));
2119        assert_eq!(path.path, expected_path);
2120        assert!(manager.writers.contains_key(&path));
2121        let writer = manager.writers.get(&path).unwrap();
2122        assert!(writer.size() > 0);
2123    }
2124
2125    #[rstest]
2126    #[case::quotes(NautilusDataType::QuoteTick.into(), "quotes/quotes_0.feather")]
2127    #[case::record(NautilusRecordType::OrderFilled.into(), "order_filled/order_filled_0.feather")]
2128    #[case::instrument_class(
2129        CatalogDataType::Instrument(NautilusInstrumentType::CurrencyPair),
2130        "instruments/currency_pair/instruments_0.feather"
2131    )]
2132    #[case::custom(
2133        NautilusDataType::Custom { type_name: "Signal".to_string() }.into(),
2134        "data/custom/Signal/Signal_0.feather"
2135    )]
2136    fn writer_paths_hold_one_file_per_type(
2137        #[case] data_type: CatalogDataType,
2138        #[case] expected: &str,
2139    ) {
2140        let temp_dir = TempDir::new().unwrap();
2141
2142        let manager = FeatherWriter::new(
2143            temp_dir.path().to_path_buf(),
2144            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2145            RotationConfig::NoRotation,
2146            None,
2147            None,
2148        );
2149
2150        let path = manager.build_writer_path(&data_type, UnixNanos::default(), 0);
2151
2152        assert_eq!(path, temp_dir.path().join(expected));
2153    }
2154
2155    #[rstest]
2156    fn existing_feather_writer_implements_streaming_data_sink() {
2157        let temp_dir = TempDir::new().unwrap();
2158        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2159
2160        let mut writer = FeatherWriter::new(
2161            temp_dir.path().to_path_buf(),
2162            clock,
2163            RotationConfig::NoRotation,
2164            None,
2165            None,
2166        );
2167
2168        let quote = QuoteTick::new(
2169            InstrumentId::from("AUD/USD.SIM"),
2170            Price::from("1.0"),
2171            Price::from("1.1"),
2172            Quantity::from("1000"),
2173            Quantity::from("1000"),
2174            UnixNanos::from(1_000),
2175            UnixNanos::from(1_000),
2176        );
2177
2178        StreamingSink::write_data(&mut writer, Data::Quote(quote)).unwrap();
2179        StreamingSink::flush(&mut writer).unwrap();
2180        StreamingSink::close(&mut writer).unwrap();
2181
2182        let files = feather_files(&temp_dir.path().join("quotes"), FEATHER_EXTENSION);
2183        assert_eq!(files.len(), 1);
2184        let identifiers =
2185            identifiers_from_record_batches(&read_feather_batches(&files[0])).unwrap();
2186        assert_eq!(identifiers, vec!["AUD/USD.SIM".to_string()]);
2187    }
2188
2189    // The `ts_init` of every quote in a sealed file, in file order
2190    fn timestamps(path: &Path) -> Vec<u64> {
2191        read_quotes(path)
2192            .into_iter()
2193            .map(|data| match data {
2194                Data::Quote(quote) => quote.ts_init.as_u64(),
2195                other => panic!("not a quote: {other:?}"),
2196            })
2197            .collect()
2198    }
2199
2200    // Every sealed file's timestamps, ordered by content
2201    fn sealed_timestamps(directory: &Path) -> Vec<Vec<u64>> {
2202        let mut files: Vec<_> = feather_files(directory, FEATHER_EXTENSION)
2203            .iter()
2204            .map(|path| timestamps(path))
2205            .collect();
2206        files.sort();
2207        files
2208    }
2209
2210    #[rstest]
2211    fn scheduled_rotation_keeps_time_of_day_anchor_after_late_rotation() {
2212        let temp_dir = TempDir::new().unwrap();
2213        let quotes = NautilusDataType::QuoteTick.into();
2214
2215        let mut writer = FeatherWriter::new(
2216            temp_dir.path().to_path_buf(),
2217            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2218            RotationConfig::ScheduledDates {
2219                interval_ns: 86_400_000_000_000,
2220                rotation_time: UnixNanos::from(36_000_000_000_000u64),
2221                rotation_timezone: jiff::tz::TimeZone::UTC,
2222            },
2223            None,
2224            None,
2225        );
2226
2227        // 2026-01-01 09:00:00 UTC: the file rotates at 10:00
2228        writer.write(quote_at(1_767_258_000_000_000_000)).unwrap();
2229        assert_eq!(
2230            writer.get_next_rotation_time(&quotes),
2231            Some(UnixNanos::from(1_767_261_600_000_000_000u64)),
2232        );
2233
2234        // The first row after it, at 10:30, opens the next file, which rotates at 10:00 tomorrow
2235        writer.write(quote_at(1_767_263_400_000_000_000)).unwrap();
2236        assert_eq!(
2237            writer.get_next_rotation_time(&quotes),
2238            Some(UnixNanos::from(1_767_348_000_000_000_000u64)),
2239        );
2240        assert_eq!(
2241            sealed_timestamps(temp_dir.path()),
2242            vec![vec![1_767_258_000_000_000_000]]
2243        );
2244    }
2245
2246    // The prints of one fill share a `ts_init`; a scheduled rotation between them left the catalog
2247    // two files with touching ranges, which its promotion refuses
2248    #[rstest]
2249    fn scheduled_rotation_opens_the_next_file_before_the_first_row_past_the_boundary() {
2250        let temp_dir = TempDir::new().unwrap();
2251
2252        let mut writer = FeatherWriter::new(
2253            temp_dir.path().to_path_buf(),
2254            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2255            RotationConfig::ScheduledDates {
2256                interval_ns: 3_600_000_000_000,
2257                rotation_time: UnixNanos::default(),
2258                rotation_timezone: jiff::tz::TimeZone::UTC,
2259            },
2260            None,
2261            None,
2262        );
2263
2264        // 2026-11-07 17:47:15 UTC, then two rows at 18:14:41.380125727 UTC
2265        let (before, after) = (1_794_073_635_000_000_000u64, 1_794_075_281_380_125_727u64);
2266        writer.write(quote_at(before)).unwrap();
2267        writer.write(quote_at(after)).unwrap();
2268        writer.write(quote_at(after)).unwrap();
2269        assert_eq!(sealed_timestamps(temp_dir.path()), vec![vec![before]]);
2270
2271        writer.close().unwrap();
2272        assert_eq!(
2273            sealed_timestamps(temp_dir.path()),
2274            vec![vec![before], vec![after, after]]
2275        );
2276    }
2277
2278    // `write_batch` stages its identifier groups in hash order. Rotating group by group could
2279    // seal the file on the later row of one group and then put the other group's row, sharing the
2280    // sealed file's last `ts_init`, in the next file: the rows of one call share one file
2281    #[rstest]
2282    fn a_batch_shares_one_rotation_decision_whatever_its_group_order() {
2283        let temp_dir = TempDir::new().unwrap();
2284
2285        let mut writer = FeatherWriter::new(
2286            temp_dir.path().to_path_buf(),
2287            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2288            RotationConfig::Interval { interval_ns: 10 },
2289            None,
2290            None,
2291        );
2292
2293        writer.write(quote_at(1)).unwrap();
2294        writer
2295            .write_data_batch(vec![
2296                Data::from(quote_at(1)),
2297                Data::from(quote_for("EUR/USD.SIM", 11)),
2298            ])
2299            .unwrap();
2300        assert_eq!(sealed_timestamps(temp_dir.path()), Vec::<Vec<u64>>::new());
2301
2302        // The rotation the batch's row at 11 made due opens the next file at the next later row
2303        writer.write(quote_at(12)).unwrap();
2304        writer.close().unwrap();
2305
2306        // The groups' order within the file follows their map
2307        let mut files = sealed_timestamps(temp_dir.path());
2308        for file in &mut files {
2309            file.sort_unstable();
2310        }
2311        assert_eq!(files, vec![vec![1, 1, 11], vec![12]]);
2312    }
2313
2314    #[rstest]
2315    fn size_rotation_waits_for_the_next_ts_init() {
2316        let temp_dir = TempDir::new().unwrap();
2317
2318        let mut writer = FeatherWriter::new(
2319            temp_dir.path().to_path_buf(),
2320            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2321            RotationConfig::Size { max_size: 1 },
2322            None,
2323            None,
2324        );
2325
2326        for ts in [1, 1, 2, 2, 3] {
2327            writer.write(quote_at(ts)).unwrap();
2328        }
2329        writer.close().unwrap();
2330
2331        assert_eq!(
2332            sealed_timestamps(temp_dir.path()),
2333            vec![vec![1, 1], vec![2, 2], vec![3]]
2334        );
2335    }
2336
2337    #[rstest]
2338    #[case::modify(true)]
2339    #[case::cancel(false)]
2340    fn test_command_denied_dynamic_round_trip(
2341        #[case] modify: bool,
2342        #[values(false, true)] envelope: bool,
2343    ) {
2344        use nautilus_core::UUID4;
2345        use nautilus_model::{
2346            events::order::spec::{OrderCancelDeniedSpec, OrderModifyDeniedSpec},
2347            identifiers::{AccountId, ClientOrderId, StrategyId, TraderId, VenueOrderId},
2348        };
2349        use nautilus_serialization::arrow::DecodeTypedFromRecordBatch;
2350        let cause = UUID4::new();
2351        let trader = TraderId::from("WRITER-017");
2352        let strategy = StrategyId::from("S-DENIAL-019");
2353        let instrument = InstrumentId::from("ETHUSDT-PERP.BINANCE");
2354        let client = ClientOrderId::from("O-DENIAL-023");
2355        let venue = VenueOrderId::from("V-DENIAL-029");
2356        let account = AccountId::from("BINANCE-031");
2357
2358        let event = if modify {
2359            OrderEventAny::ModifyDenied(
2360                OrderModifyDeniedSpec::builder()
2361                    .trader_id(trader)
2362                    .strategy_id(strategy)
2363                    .instrument_id(instrument)
2364                    .client_order_id(client)
2365                    .venue_order_id(venue)
2366                    .account_id(account)
2367                    .reason("not sent to modify".into())
2368                    .ts_event(37.into())
2369                    .ts_init(41.into())
2370                    .causation_id(cause)
2371                    .build(),
2372            )
2373        } else {
2374            OrderEventAny::CancelDenied(
2375                OrderCancelDeniedSpec::builder()
2376                    .trader_id(trader)
2377                    .strategy_id(strategy)
2378                    .instrument_id(instrument)
2379                    .client_order_id(client)
2380                    .venue_order_id(venue)
2381                    .account_id(account)
2382                    .reason("not sent to cancel".into())
2383                    .ts_event(43.into())
2384                    .ts_init(47.into())
2385                    .causation_id(cause)
2386                    .build(),
2387            )
2388        };
2389
2390        let temp = TempDir::new().unwrap();
2391        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2392
2393        let mut writer = FeatherWriter::new(
2394            temp.path().to_path_buf(),
2395            clock,
2396            RotationConfig::NoRotation,
2397            None,
2398            None,
2399        );
2400
2401        let written = if envelope {
2402            writer.write_any_message(&event).unwrap()
2403        } else {
2404            match &event {
2405                OrderEventAny::ModifyDenied(event) => writer.write_any_message(event).unwrap(),
2406                OrderEventAny::CancelDenied(event) => writer.write_any_message(event).unwrap(),
2407                _ => unreachable!(),
2408            }
2409        };
2410
2411        assert!(written);
2412        writer.close().unwrap();
2413        let files = feather_files(temp.path(), "feather");
2414        assert_eq!(files.len(), 1);
2415        let mut recovered = Vec::new();
2416
2417        for batch in read_feather_batches(&files[0]) {
2418            let metadata = batch.schema().metadata().clone();
2419            assert_eq!(
2420                metadata.get("type_name").unwrap(),
2421                if modify {
2422                    "OrderModifyDenied"
2423                } else {
2424                    "OrderCancelDenied"
2425                }
2426            );
2427
2428            if modify {
2429                recovered.extend(
2430                    OrderModifyDenied::decode_typed_batch(&metadata, batch)
2431                        .unwrap()
2432                        .into_iter()
2433                        .map(OrderEventAny::ModifyDenied),
2434                );
2435            } else {
2436                recovered.extend(
2437                    OrderCancelDenied::decode_typed_batch(&metadata, batch)
2438                        .unwrap()
2439                        .into_iter()
2440                        .map(OrderEventAny::CancelDenied),
2441                );
2442            }
2443        }
2444
2445        assert_eq!(recovered, vec![event]);
2446    }
2447
2448    #[rstest]
2449    fn test_file_writer_round_trip() {
2450        let instrument_id = "AAPL.AAPL";
2451
2452        // Write a dummy value.
2453        let quote = QuoteTick::new(
2454            InstrumentId::from(instrument_id),
2455            Price::from("100.0"),
2456            Price::from("100.0"),
2457            Quantity::from("100.0"),
2458            Quantity::from("100.0"),
2459            UnixNanos::from(100),
2460            UnixNanos::from(100),
2461        );
2462        let metadata = QuoteTick::metadata(&quote);
2463        let schema = QuoteTick::get_schema(Some(metadata.clone()));
2464        let batch = QuoteTick::encode_batch(&QuoteTick::metadata(&quote), &[quote]).unwrap();
2465
2466        let temp_dir = TempDir::new().unwrap();
2467        let path = temp_dir.path().join("quotes.feather");
2468        let mut file = FeatherFile::create(&path, &schema, &RotationConfig::NoRotation).unwrap();
2469        file.write_record_batch(&batch).unwrap();
2470        file.seal().unwrap();
2471
2472        assert!(!path.with_extension(FEATHER_PARTIAL_EXTENSION).exists());
2473        let mut reader = StreamReader::try_new(File::open(&path).unwrap(), None).unwrap();
2474
2475        let read_metadata = reader.schema().metadata().clone();
2476        let mut expected_metadata = metadata.clone();
2477        expected_metadata.insert("type_name".to_string(), "QuoteTick".to_string());
2478        assert_eq!(read_metadata, expected_metadata);
2479
2480        let read_batch = reader.next().unwrap().unwrap();
2481        assert_eq!(read_batch.column(0), batch.column(0));
2482
2483        let decoded = QuoteTick::decode_data_batch(&metadata, batch).unwrap();
2484        assert_eq!(decoded[0], Data::from(quote));
2485    }
2486
2487    #[rstest]
2488    fn test_round_trip() {
2489        // Create a temporary directory for base path
2490        let temp_dir = TempDir::new_in(".").unwrap();
2491
2492        // Create a test time source
2493        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2494
2495        let mut manager = FeatherWriter::new(
2496            temp_dir.path().to_path_buf(),
2497            clock,
2498            RotationConfig::NoRotation,
2499            None,
2500            None, // flush_interval_ms
2501        );
2502
2503        let instrument_id = "AAPL.AAPL";
2504
2505        // Write a dummy value.
2506        let quote = QuoteTick::new(
2507            InstrumentId::from(instrument_id),
2508            Price::from("100.0"),
2509            Price::from("100.0"),
2510            Quantity::from("100.0"),
2511            Quantity::from("100.0"),
2512            UnixNanos::from(100),
2513            UnixNanos::from(100),
2514        );
2515
2516        let trade = TradeTick::new(
2517            InstrumentId::from(instrument_id),
2518            Price::from("100.0"),
2519            Quantity::from("100.0"),
2520            AggressorSide::Buy,
2521            TradeId::from("1"),
2522            UnixNanos::from(100),
2523            UnixNanos::from(100),
2524        );
2525
2526        manager.write(quote).unwrap();
2527        manager.write(trade).unwrap();
2528
2529        let paths = manager.writers.keys().cloned().collect::<Vec<_>>();
2530        assert_eq!(paths.len(), 2);
2531
2532        manager.close().unwrap();
2533
2534        let mut recovered_quotes = Vec::new();
2535        let mut recovered_trades = Vec::new();
2536
2537        for path in paths {
2538            let path_str = path.path;
2539            for batch in read_feather_batches(&path_str) {
2540                let metadata = batch.schema().metadata().clone();
2541
2542                if path_str.to_str().unwrap().contains("quotes") {
2543                    let decoded = QuoteTick::decode_data_batch(&metadata, batch).unwrap();
2544                    recovered_quotes.extend(decoded);
2545                } else if path_str.to_str().unwrap().contains("trades") {
2546                    let decoded = TradeTick::decode_data_batch(&metadata, batch).unwrap();
2547                    recovered_trades.extend(decoded);
2548                }
2549            }
2550        }
2551
2552        // Assert that the recovered data matches the written data
2553        assert_eq!(recovered_quotes.len(), 1, "Expected one QuoteTick record");
2554        assert_eq!(recovered_trades.len(), 1, "Expected one TradeTick record");
2555
2556        // Check key fields to ensure the data round-tripped correctly
2557        assert_eq!(recovered_quotes[0], Data::from(quote));
2558        assert_eq!(recovered_trades[0], Data::from(trade));
2559    }
2560
2561    #[rstest]
2562    fn test_write_data_enum() {
2563        let temp_dir = TempDir::new().unwrap();
2564        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2565
2566        let mut writer = FeatherWriter::new(
2567            temp_dir.path().to_path_buf(),
2568            clock,
2569            RotationConfig::NoRotation,
2570            None,
2571            None,
2572        );
2573
2574        let quote = QuoteTick::new(
2575            InstrumentId::from("AUD/USD.SIM"),
2576            Price::from("1.0"),
2577            Price::from("1.0"),
2578            Quantity::from("1000"),
2579            Quantity::from("1000"),
2580            UnixNanos::from(1000),
2581            UnixNanos::from(1000),
2582        );
2583
2584        // Test writing via write_data
2585        writer.write_data(Data::Quote(quote)).unwrap();
2586        writer.flush().unwrap();
2587
2588        // Verify file was created
2589        assert!(!writer.writers.is_empty() || temp_dir.path().read_dir().unwrap().count() > 0);
2590    }
2591
2592    #[rstest]
2593    fn test_write_data_all_types() {
2594        let temp_dir = TempDir::new().unwrap();
2595        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2596
2597        let mut writer = FeatherWriter::new(
2598            temp_dir.path().to_path_buf(),
2599            clock,
2600            RotationConfig::NoRotation,
2601            None,
2602            None,
2603        );
2604
2605        let instrument_id = InstrumentId::from("AUD/USD.SIM");
2606
2607        // Test all data types
2608        let quote = QuoteTick::new(
2609            instrument_id,
2610            Price::from("1.0"),
2611            Price::from("1.0"),
2612            Quantity::from("1000"),
2613            Quantity::from("1000"),
2614            UnixNanos::from(1000),
2615            UnixNanos::from(1000),
2616        );
2617        writer.write_data(Data::Quote(quote)).unwrap();
2618
2619        let trade = TradeTick::new(
2620            instrument_id,
2621            Price::from("1.0"),
2622            Quantity::from("1000"),
2623            AggressorSide::Buy,
2624            TradeId::from("1"),
2625            UnixNanos::from(2000),
2626            UnixNanos::from(2000),
2627        );
2628        writer.write_data(Data::Trade(trade)).unwrap();
2629
2630        let delta = OrderBookDelta::clear(
2631            instrument_id,
2632            0,
2633            UnixNanos::from(3000),
2634            UnixNanos::from(3000),
2635        );
2636        writer.write_data(Data::BookDelta(delta)).unwrap();
2637
2638        writer.flush().unwrap();
2639    }
2640
2641    #[rstest]
2642    fn test_auto_flush() {
2643        let temp_dir = TempDir::new().unwrap();
2644        let shared_time = Arc::new(AtomicU64::new(0));
2645        let clock = WriterClock::Test(Arc::clone(&shared_time));
2646
2647        let mut writer = FeatherWriter::new(
2648            temp_dir.path().to_path_buf(),
2649            clock,
2650            RotationConfig::NoRotation,
2651            None,
2652            Some(100), // 100ms flush interval
2653        );
2654
2655        let quote = QuoteTick::new(
2656            InstrumentId::from("AUD/USD.SIM"),
2657            Price::from("1.0"),
2658            Price::from("1.0"),
2659            Quantity::from("1000"),
2660            Quantity::from("1000"),
2661            UnixNanos::from(1000),
2662            UnixNanos::from(1000),
2663        );
2664
2665        // Write first quote; interval not elapsed, so nothing is flushed yet
2666        writer.write(quote).unwrap();
2667        assert_eq!(writer.writers.len(), 1);
2668        assert_eq!(writer.last_flush_ns, UnixNanos::from(0));
2669
2670        // Advance the shared time source past the 100ms flush interval
2671        shared_time.store(200_000_000, Ordering::Relaxed);
2672
2673        // Second write hits the flush boundary: both quotes reach the same open file
2674        let quote2 = QuoteTick::new(
2675            InstrumentId::from("AUD/USD.SIM"),
2676            Price::from("1.1"),
2677            Price::from("1.1"),
2678            Quantity::from("1000"),
2679            Quantity::from("1000"),
2680            UnixNanos::from(2000),
2681            UnixNanos::from(2000),
2682        );
2683        writer.write(quote2).unwrap();
2684
2685        // Windows locks block reads through other handles, so the open file is measured instead:
2686        // if closing adds only the 8-byte end-of-stream marker, both quotes were already on disk
2687        let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
2688        let flushed_len = fs::metadata(&partial[0]).unwrap().len();
2689        let open_writers = writer.writers.len();
2690        let last_flush_ns = writer.last_flush_ns;
2691        let sealed_before_close = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2692        writer.close().unwrap();
2693
2694        let sealed = partial[0].with_extension("");
2695        let sealed_rows = read_feather_batches(&sealed)
2696            .iter()
2697            .map(RecordBatch::num_rows)
2698            .sum::<usize>();
2699        assert_eq!(open_writers, 1);
2700        assert_eq!(last_flush_ns, UnixNanos::from(200_000_000));
2701        assert_eq!(partial.len(), 1);
2702        assert!(sealed_before_close.is_empty());
2703        assert_eq!(fs::metadata(&sealed).unwrap().len(), flushed_len + 8);
2704        assert_eq!(sealed_rows, 2);
2705    }
2706
2707    #[rstest]
2708    fn flushes_append_to_one_file_per_type_until_close() {
2709        let temp_dir = TempDir::new().unwrap();
2710        let shared_time = Arc::new(AtomicU64::new(0));
2711
2712        let mut writer = FeatherWriter::new(
2713            temp_dir.path().to_path_buf(),
2714            WriterClock::Test(Arc::clone(&shared_time)),
2715            RotationConfig::NoRotation,
2716            None,
2717            Some(1_000),
2718        );
2719
2720        let quotes = (1..=10)
2721            .map(|minute| {
2722                QuoteTick::new(
2723                    InstrumentId::from("AUD/USD.SIM"),
2724                    Price::from("1.0"),
2725                    Price::from("1.1"),
2726                    Quantity::from("1000"),
2727                    Quantity::from("1000"),
2728                    UnixNanos::from(minute * 60_000_000_000),
2729                    UnixNanos::from(minute * 60_000_000_000),
2730                )
2731            })
2732            .collect::<Vec<_>>();
2733
2734        // Each write crosses a flush interval, as a 1-minute bar does in a backtest
2735        for quote in &quotes {
2736            shared_time.store(quote.ts_init.as_u64(), Ordering::Relaxed);
2737            writer.write(*quote).unwrap();
2738        }
2739
2740        let partial_before_close = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
2741        let sealed_before_close = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2742        writer.close().unwrap();
2743        let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2744
2745        let recovered = read_feather_batches(&sealed[0])
2746            .into_iter()
2747            .flat_map(|batch| {
2748                let metadata = batch.schema().metadata().clone();
2749                QuoteTick::decode_data_batch(&metadata, batch).unwrap()
2750            })
2751            .collect::<Vec<_>>();
2752
2753        assert_eq!(partial_before_close.len(), 1);
2754        assert!(sealed_before_close.is_empty());
2755        assert_eq!(sealed.len(), 1);
2756        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
2757        assert_eq!(
2758            recovered,
2759            quotes.into_iter().map(Data::from).collect::<Vec<_>>()
2760        );
2761    }
2762
2763    #[rstest]
2764    #[case::below_row_cap(3, vec![3])]
2765    #[case::above_row_cap(
2766        DEFAULT_DATA_BATCH_CHUNK_SIZE + 1,
2767        vec![DEFAULT_DATA_BATCH_CHUNK_SIZE, 1]
2768    )]
2769    fn writes_between_flushes_append_one_batch_per_row_cap(
2770        #[case] count: usize,
2771        #[case] expected_batch_rows: Vec<usize>,
2772    ) {
2773        let temp_dir = TempDir::new().unwrap();
2774
2775        let mut writer = FeatherWriter::new(
2776            temp_dir.path().to_path_buf(),
2777            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2778            RotationConfig::NoRotation,
2779            None,
2780            Some(0),
2781        );
2782        let instrument_ids = [
2783            InstrumentId::from("AUD/USD.SIM"),
2784            InstrumentId::from("EUR/USD.SIM"),
2785        ];
2786
2787        for index in 0..count {
2788            writer
2789                .write(QuoteTick::new(
2790                    instrument_ids[index % 2],
2791                    Price::from("1.0"),
2792                    Price::from("1.1"),
2793                    Quantity::from("1000"),
2794                    Quantity::from("1000"),
2795                    UnixNanos::from(index as u64),
2796                    UnixNanos::from(index as u64),
2797                ))
2798                .unwrap();
2799        }
2800
2801        writer.close().unwrap();
2802        let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2803        let batch_rows = StreamReader::try_new(File::open(&sealed[0]).unwrap(), None)
2804            .unwrap()
2805            .map(|batch| batch.unwrap().num_rows())
2806            .collect::<Vec<_>>();
2807
2808        assert_eq!(sealed.len(), 1);
2809        assert_eq!(batch_rows, expected_batch_rows);
2810    }
2811
2812    #[rstest]
2813    fn drop_seals_open_files() {
2814        let temp_dir = TempDir::new().unwrap();
2815
2816        let mut writer = FeatherWriter::new(
2817            temp_dir.path().to_path_buf(),
2818            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2819            RotationConfig::NoRotation,
2820            None,
2821            None,
2822        );
2823        writer
2824            .write(QuoteTick::new(
2825                InstrumentId::from("AUD/USD.SIM"),
2826                Price::from("1.0"),
2827                Price::from("1.1"),
2828                Quantity::from("1000"),
2829                Quantity::from("1000"),
2830                UnixNanos::from(1_000),
2831                UnixNanos::from(1_000),
2832            ))
2833            .unwrap();
2834
2835        drop(writer);
2836
2837        assert_eq!(feather_files(temp_dir.path(), FEATHER_EXTENSION).len(), 1);
2838        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
2839    }
2840
2841    #[rstest]
2842    fn reserved_paths_skip_files_left_in_the_directory() {
2843        let temp_dir = TempDir::new().unwrap();
2844
2845        let mut writer = FeatherWriter::new(
2846            temp_dir.path().to_path_buf(),
2847            WriterClock::Test(Arc::new(AtomicU64::new(5))),
2848            RotationConfig::NoRotation,
2849            None,
2850            None,
2851        );
2852        let sealed = temp_dir.path().join("trades").join("trades_5.feather");
2853        let partial = temp_dir
2854            .path()
2855            .join("trades")
2856            .join("trades_5-1.feather.partial");
2857        fs::create_dir_all(sealed.parent().unwrap()).unwrap();
2858        fs::write(&sealed, b"sealed").unwrap();
2859        fs::write(&partial, b"partial").unwrap();
2860
2861        let path = writer.reserve_writer_path(NautilusDataType::TradeTick.into());
2862
2863        assert_eq!(
2864            path.path,
2865            temp_dir.path().join("trades").join("trades_5-2.feather")
2866        );
2867        assert_eq!(fs::read(&sealed).unwrap(), b"sealed");
2868        assert_eq!(fs::read(&partial).unwrap(), b"partial");
2869    }
2870
2871    #[rstest]
2872    #[case::same_time(vec![5; 32])]
2873    #[case::clock_rewinds(vec![5, 3, 5])]
2874    #[case::clock_advances_then_rewinds(vec![5, 7, 5, 7])]
2875    fn path_reservations_are_bounded_after_files_are_deleted(#[case] timestamps: Vec<u64>) {
2876        let temp_dir = TempDir::new().unwrap();
2877        let time = Arc::new(AtomicU64::new(0));
2878        let mut writer = FeatherWriter::new(
2879            temp_dir.path().to_path_buf(),
2880            WriterClock::Test(Arc::clone(&time)),
2881            RotationConfig::NoRotation,
2882            None,
2883            None,
2884        );
2885        let mut paths = HashSet::new();
2886
2887        for timestamp in &timestamps {
2888            time.store(*timestamp, Ordering::Relaxed);
2889            writer.write(quote_at(1)).unwrap();
2890            writer.close().unwrap();
2891            let path = feather_files(temp_dir.path(), FEATHER_EXTENSION).remove(0);
2892            assert!(paths.insert(path.clone()));
2893            fs::remove_file(path).unwrap();
2894        }
2895
2896        assert_eq!(paths.len(), timestamps.len());
2897        assert_eq!(writer.path_sequences.len(), 1);
2898    }
2899
2900    #[rstest]
2901    fn test_close() {
2902        let temp_dir = TempDir::new().unwrap();
2903        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2904
2905        let mut writer = FeatherWriter::new(
2906            temp_dir.path().to_path_buf(),
2907            clock,
2908            RotationConfig::NoRotation,
2909            None,
2910            None,
2911        );
2912
2913        let quote = QuoteTick::new(
2914            InstrumentId::from("AUD/USD.SIM"),
2915            Price::from("1.0"),
2916            Price::from("1.0"),
2917            Quantity::from("1000"),
2918            Quantity::from("1000"),
2919            UnixNanos::from(1000),
2920            UnixNanos::from(1000),
2921        );
2922
2923        writer.write(quote).unwrap();
2924        assert!(!writer.writers.is_empty());
2925
2926        writer.close().unwrap();
2927        assert!(writer.writers.is_empty());
2928    }
2929
2930    #[rstest]
2931    fn test_write_data_orderbook_deltas() {
2932        let temp_dir = TempDir::new().unwrap();
2933        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2934
2935        let mut writer = FeatherWriter::new(
2936            temp_dir.path().to_path_buf(),
2937            clock,
2938            RotationConfig::NoRotation,
2939            None,
2940            None,
2941        );
2942
2943        let instrument_id = InstrumentId::from("AUD/USD.SIM");
2944        let delta1 = OrderBookDelta::clear(
2945            instrument_id,
2946            0,
2947            UnixNanos::from(1000),
2948            UnixNanos::from(1000),
2949        );
2950        let delta2 = OrderBookDelta::clear(
2951            instrument_id,
2952            0,
2953            UnixNanos::from(2000),
2954            UnixNanos::from(2000),
2955        );
2956
2957        let deltas = OrderBookDeltas::new(instrument_id, vec![delta1, delta2]);
2958        // Test writing OrderBookDeltas via write_data
2959        writer
2960            .write_data(Data::BookDeltas(Box::new(deltas)))
2961            .unwrap();
2962        writer.flush().unwrap();
2963    }
2964
2965    #[rstest]
2966    fn feather_writer_is_send() {
2967        fn assert_send<T: Send>() {}
2968        assert_send::<FeatherWriter>();
2969        assert_send::<WriterClock>();
2970    }
2971
2972    #[rstest]
2973    fn writer_clock_test_source_reads_shared_atomic() {
2974        let shared = Arc::new(AtomicU64::new(7));
2975        let clock = WriterClock::Test(Arc::clone(&shared));
2976        assert_eq!(clock.timestamp_ns(), UnixNanos::from(7));
2977
2978        shared.store(42, Ordering::Relaxed);
2979        assert_eq!(clock.timestamp_ns(), UnixNanos::from(42));
2980    }
2981
2982    #[rstest]
2983    fn writer_clock_from_shared_clock_wires_test_clock() {
2984        let test_clock = Rc::new(RefCell::new(VirtualClock::new()));
2985        test_clock
2986            .borrow_mut()
2987            .advance_time(UnixNanos::from(42), true);
2988        let clock: Rc<RefCell<dyn Clock>> = test_clock.clone();
2989
2990        let (writer_clock, shared) = WriterClock::from_shared_clock(&clock);
2991        let shared = shared.expect("non-live clocks must return a shared atomic");
2992
2993        // Seeded with the source clock's current time
2994        assert_eq!(writer_clock.timestamp_ns(), UnixNanos::from(42));
2995
2996        // Bridge refresh: advancing the source clock and storing into the atomic
2997        // is what the PyO3 wrapper does before each forwarded call
2998        test_clock
2999            .borrow_mut()
3000            .advance_time(UnixNanos::from(99), true);
3001        shared.store(clock.borrow().timestamp_ns().as_u64(), Ordering::Relaxed);
3002        assert_eq!(writer_clock.timestamp_ns(), UnixNanos::from(99));
3003    }
3004
3005    #[rstest]
3006    fn writer_clock_from_shared_clock_live_clock_is_live() {
3007        let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(LiveClock::new(None)));
3008
3009        let (writer_clock, shared) = WriterClock::from_shared_clock(&clock);
3010
3011        assert!(matches!(writer_clock, WriterClock::Live));
3012        assert!(shared.is_none());
3013    }
3014
3015    #[rstest]
3016    #[cfg(unix)]
3017    fn failed_seal_of_a_removed_file_reports_the_error_and_publishes_nothing() {
3018        let temp_dir = TempDir::new().unwrap();
3019        let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
3020
3021        let mut writer = FeatherWriter::new(
3022            temp_dir.path().to_path_buf(),
3023            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3024            RotationConfig::NoRotation,
3025            None,
3026            None,
3027        )
3028        .with_sealed_sender(sealed_tx);
3029        writer
3030            .write(QuoteTick::new(
3031                InstrumentId::from("AUD/USD.SIM"),
3032                Price::from("1.0"),
3033                Price::from("1.1"),
3034                Quantity::from("1000"),
3035                Quantity::from("1000"),
3036                UnixNanos::from(1_000),
3037                UnixNanos::from(1_000),
3038            ))
3039            .unwrap();
3040
3041        // Removing the open partial file makes the seal's rename fail after the stream finishes
3042        let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
3043        fs::remove_file(&partial[0]).unwrap();
3044
3045        let error = writer.close().unwrap_err();
3046
3047        assert_eq!(error.to_string(), "No such file or directory (os error 2)");
3048        assert!(writer.is_closed());
3049        assert!(feather_files(temp_dir.path(), FEATHER_EXTENSION).is_empty());
3050        assert_eq!(
3051            sealed_rx.try_iter().collect::<Vec<_>>(),
3052            Vec::<PathBuf>::new()
3053        );
3054    }
3055
3056    #[rstest]
3057    #[cfg(target_os = "linux")]
3058    fn failed_seal_recovers_the_batches_that_reached_the_file() {
3059        let temp_dir = TempDir::new().unwrap();
3060        let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
3061
3062        let mut writer = FeatherWriter::new(
3063            temp_dir.path().to_path_buf(),
3064            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3065            RotationConfig::NoRotation,
3066            None,
3067            Some(0),
3068        )
3069        .with_sealed_sender(sealed_tx);
3070
3071        writer.write(quote_at(1)).unwrap();
3072        writer.flush().unwrap();
3073        writer.write(quote_at(2)).unwrap();
3074
3075        // Route the open stream to a device that reports a full disk, so sealing it fails
3076        let (path, file) = writer.writers.iter_mut().next().unwrap();
3077        let sealed = path.path.clone();
3078        let full = OpenOptions::new().write(true).open("/dev/full").unwrap();
3079        file.writer =
3080            StreamWriter::try_new(CountingWriter::new(BufWriter::new(full)), &file.schema).unwrap();
3081
3082        let error = writer.close().unwrap_err();
3083
3084        assert_eq!(
3085            error.to_string(),
3086            "Io error: No space left on device (os error 28)"
3087        );
3088        assert!(writer.is_closed());
3089        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3090        assert_eq!(read_quotes(&sealed), quotes_at(&[1]));
3091        assert_eq!(sealed_rx.try_iter().collect::<Vec<_>>(), vec![sealed]);
3092    }
3093
3094    #[rstest]
3095    #[case::nothing_flushed(false, vec![3])]
3096    #[case::batch_flushed(true, vec![1, 3])]
3097    #[cfg(target_os = "linux")]
3098    fn failed_flush_recovers_the_abandoned_file_and_next_write_opens_a_new_one(
3099        #[case] flushed: bool,
3100        #[case] expected: Vec<u64>,
3101    ) {
3102        let temp_dir = TempDir::new().unwrap();
3103        let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
3104
3105        let mut writer = FeatherWriter::new(
3106            temp_dir.path().to_path_buf(),
3107            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3108            RotationConfig::NoRotation,
3109            None,
3110            Some(0),
3111        )
3112        .with_sealed_sender(sealed_tx);
3113
3114        writer.write(quote_at(1)).unwrap();
3115
3116        if flushed {
3117            writer.flush().unwrap();
3118        }
3119
3120        writer.write(quote_at(2)).unwrap();
3121
3122        // Route the open stream to a device that reports a full disk on every write
3123        let (abandoned, file) = writer.writers.iter_mut().next().unwrap();
3124        let abandoned = abandoned.clone();
3125        let full = OpenOptions::new().write(true).open("/dev/full").unwrap();
3126        file.writer =
3127            StreamWriter::try_new(CountingWriter::new(BufWriter::new(full)), &file.schema).unwrap();
3128
3129        let error = writer.flush().unwrap_err();
3130        writer.write(quote_at(3)).unwrap();
3131        writer.close().unwrap();
3132
3133        let mut recovered = feather_files(temp_dir.path(), FEATHER_EXTENSION)
3134            .iter()
3135            .flat_map(|path| read_quotes(path))
3136            .collect::<Vec<_>>();
3137        recovered.sort_by_key(HasTsInit::ts_init);
3138
3139        // Quote 2 was buffered when the flush failed, and the full device also fails the writer's
3140        // drop-time flush, so it reaches no file
3141        assert_eq!(
3142            error.to_string(),
3143            "Io error: No space left on device (os error 28)"
3144        );
3145        assert_eq!(abandoned.path.exists(), flushed);
3146        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3147        assert_eq!(recovered, quotes_at(&expected));
3148        let mut notified = sealed_rx.try_iter().collect::<Vec<_>>();
3149        notified.sort();
3150        assert_eq!(notified, feather_files(temp_dir.path(), FEATHER_EXTENSION));
3151    }
3152
3153    #[rstest]
3154    #[case::end_of_stream_marker(0, &[], vec![1, 2, 3], "")]
3155    #[case::no_end_of_stream_marker(8, &[], vec![1, 2, 3], "")]
3156    #[case::stray_bytes(8, &[0xFF, 0xFF], vec![1, 2, 3], "")]
3157    #[case::truncated_last_batch(9, &[], vec![1, 2], TORN_TAIL)]
3158    fn recovery_seals_complete_batches_and_drops_the_rest(
3159        #[case] cut: usize,
3160        #[case] stray: &[u8],
3161        #[case] expected: Vec<u64>,
3162        #[case] cause: &str,
3163    ) {
3164        let temp_dir = TempDir::new().unwrap();
3165        let directory = temp_dir.path().join("recover");
3166        let partial = write_partial_file(&directory, &[1, 2, 3]);
3167        let mut staged = fs::read(&partial).unwrap();
3168        staged.truncate(staged.len() - cut);
3169        staged.extend_from_slice(stray);
3170        fs::write(&partial, &staged).unwrap();
3171
3172        // The expected batches alone form this prefix, which ends before the end-of-stream marker
3173        let reference = fs::read(write_partial_file(
3174            &temp_dir.path().join("reference"),
3175            &expected,
3176        ))
3177        .unwrap();
3178        let complete = &reference[..reference.len() - 8];
3179
3180        let warnings = recover_with_warnings(&directory);
3181
3182        let sealed = partial.with_extension("");
3183        let dropped = staged.len() - complete.len();
3184
3185        let tail = if dropped == 0 {
3186            String::new()
3187        } else {
3188            format!(", dropped {dropped} trailing byte(s){cause}")
3189        };
3190
3191        assert!(!partial.exists());
3192        assert_eq!(fs::read(&sealed).unwrap(), complete);
3193        assert_eq!(read_quotes(&sealed), quotes_at(&expected));
3194        assert_eq!(
3195            warnings,
3196            vec![format!(
3197                "Recovered {} record batch(es) into Feather file {}{tail}",
3198                expected.len(),
3199                sealed.display()
3200            )],
3201        );
3202    }
3203
3204    #[rstest]
3205    #[case::schema_only(0, "")]
3206    #[case::truncated_schema(1, TORN_TAIL)]
3207    fn recovery_removes_a_file_without_a_complete_batch(#[case] cut: u64, #[case] cause: &str) {
3208        let temp_dir = TempDir::new().unwrap();
3209        let partial = temp_dir.path().join("quotes_1.feather.partial");
3210        let file = File::create(&partial).unwrap();
3211        StreamWriter::try_new(&file, &QuoteTick::get_schema(None)).unwrap();
3212        file.set_len(file.metadata().unwrap().len() - cut).unwrap();
3213        drop(file);
3214
3215        let warnings = recover_with_warnings(temp_dir.path());
3216
3217        assert_eq!(fs::read_dir(temp_dir.path()).unwrap().count(), 0);
3218        assert_eq!(
3219            warnings,
3220            vec![format!(
3221                "Removed Feather file {} with no complete record batch{cause}",
3222                partial.display()
3223            )],
3224        );
3225    }
3226
3227    #[rstest]
3228    fn recovery_leaves_a_file_with_an_unreadable_tail() {
3229        let temp_dir = TempDir::new().unwrap();
3230        let partial = write_partial_file(temp_dir.path(), &[1, 2]);
3231
3232        // A continuation marker followed by a negative metadata length is corrupt, not cut short
3233        let mut staged = fs::read(&partial).unwrap();
3234        staged.truncate(staged.len() - 8);
3235        staged.extend_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF, 0xF0, 0xFF, 0xFF, 0xFF]);
3236        fs::write(&partial, &staged).unwrap();
3237
3238        let warnings = recover_with_warnings(temp_dir.path());
3239
3240        assert_eq!(fs::read(&partial).unwrap(), staged);
3241        assert_eq!(
3242            warnings,
3243            vec![format!(
3244                "Failed to recover Feather file {}: Parser error: Invalid metadata length: -16",
3245                partial.display()
3246            )],
3247        );
3248    }
3249
3250    #[rstest]
3251    #[cfg(unix)]
3252    fn recovery_leaves_partial_file_symlinks_untouched() {
3253        let temp_dir = TempDir::new().unwrap();
3254        let outside = TempDir::new().unwrap();
3255        let partial = write_partial_file(outside.path(), &[1, 2]);
3256        let original = fs::read(&partial).unwrap();
3257        let link = temp_dir.path().join("quotes_0.feather.partial");
3258        std::os::unix::fs::symlink(&partial, &link).unwrap();
3259
3260        recover_partial_feather_files(temp_dir.path());
3261
3262        assert_eq!(fs::read(&partial).unwrap(), original);
3263        assert_eq!(fs::read_link(&link).unwrap(), partial);
3264        assert!(!link.with_extension("").exists());
3265        assert!(!partial.with_extension("").exists());
3266    }
3267
3268    #[rstest]
3269    #[cfg(unix)]
3270    fn recovery_continues_after_a_file_it_cannot_recover() {
3271        let temp_dir = TempDir::new().unwrap();
3272        let run = temp_dir.path().join("run");
3273        let partial = write_partial_file(&run, &[1]);
3274
3275        let unrecoverable = temp_dir.path().join("bad.feather.partial");
3276        std::os::unix::fs::symlink(&run, &unrecoverable).unwrap();
3277
3278        let warnings = recover_with_warnings(temp_dir.path());
3279
3280        let sealed = partial.with_extension("");
3281        assert_eq!(read_quotes(&sealed), quotes_at(&[1]));
3282        assert_eq!(
3283            warnings,
3284            vec![
3285                format!(
3286                    "Failed to recover Feather file {}: Partial Feather path must be a regular file: {}",
3287                    unrecoverable.display(),
3288                    unrecoverable.display(),
3289                ),
3290                format!(
3291                    "Recovered 1 record batch(es) into Feather file {}, dropped 8 trailing byte(s)",
3292                    sealed.display()
3293                ),
3294            ],
3295        );
3296    }
3297
3298    #[rstest]
3299    fn recovery_leaves_open_and_empty_files() {
3300        let temp_dir = TempDir::new().unwrap();
3301
3302        let mut writer = FeatherWriter::new(
3303            temp_dir.path().join("open"),
3304            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3305            RotationConfig::NoRotation,
3306            None,
3307            Some(0),
3308        );
3309        writer.write(quote_at(1)).unwrap();
3310        writer.flush().unwrap();
3311
3312        // Windows locks block reads through other handles, so the open file is compared by length
3313        let open = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).remove(0);
3314        let open_len = fs::metadata(&open).unwrap().len();
3315        let empty = temp_dir.path().join("empty.feather.partial");
3316        File::create(&empty).unwrap();
3317
3318        recover_partial_feather_files(temp_dir.path());
3319
3320        let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
3321        let open_len_after = fs::metadata(&open).unwrap().len();
3322        writer.close().unwrap();
3323
3324        assert_eq!(partial, vec![empty, open.clone()]);
3325        assert_eq!(open_len_after, open_len);
3326        assert_eq!(read_quotes(&open.with_extension("")), quotes_at(&[1]));
3327    }
3328
3329    #[rstest]
3330    fn size_rotation_in_mixed_identifier_batch_seals_the_shared_file_once() {
3331        let temp_dir = TempDir::new().unwrap();
3332
3333        let mut writer = FeatherWriter::new(
3334            temp_dir.path().to_path_buf(),
3335            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3336            RotationConfig::Size { max_size: 1 },
3337            None,
3338            None,
3339        );
3340
3341        let quotes = ["AUD/USD.SIM", "EUR/USD.SIM"]
3342            .map(|instrument_id| {
3343                QuoteTick::new(
3344                    InstrumentId::from(instrument_id),
3345                    Price::from("1.0"),
3346                    Price::from("1.1"),
3347                    Quantity::from("1000"),
3348                    Quantity::from("1000"),
3349                    UnixNanos::from(1),
3350                    UnixNanos::from(1),
3351                )
3352            })
3353            .to_vec();
3354
3355        // The first identifier group fills the shared quotes file; the second shares its
3356        // `ts_init`, so it joins the full file instead of opening the next
3357        writer.write_batch(quotes).unwrap();
3358
3359        assert_eq!(writer.writers.len(), 1);
3360        assert!(feather_files(temp_dir.path(), FEATHER_EXTENSION).is_empty());
3361
3362        writer.close().unwrap();
3363
3364        let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
3365        assert_eq!(sealed.len(), 1);
3366        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3367        assert_eq!(
3368            identifiers_from_record_batches(&read_feather_batches(&sealed[0])).unwrap(),
3369            vec!["AUD/USD.SIM".to_string(), "EUR/USD.SIM".to_string()],
3370        );
3371    }
3372
3373    // A full file is sealed when the next row with a later `ts_init` arrives, which opens the
3374    // next file; the last full file waits for close
3375    #[rstest]
3376    fn size_rotation_seals_each_full_file_before_the_next_later_row() {
3377        let temp_dir = TempDir::new().unwrap();
3378        let shared_clock = Arc::new(AtomicU64::new(0));
3379
3380        let mut writer = FeatherWriter::new(
3381            temp_dir.path().to_path_buf(),
3382            WriterClock::Test(Arc::clone(&shared_clock)),
3383            RotationConfig::Size { max_size: 1 },
3384            None,
3385            Some(1),
3386        );
3387
3388        for ts in [1_000, 2_000] {
3389            shared_clock.store(ts * 1_000_000, Ordering::Relaxed);
3390            writer
3391                .write(QuoteTick::new(
3392                    InstrumentId::from("AUD/USD.SIM"),
3393                    Price::from("1.0"),
3394                    Price::from("1.0"),
3395                    Quantity::from("1000"),
3396                    Quantity::from("1000"),
3397                    UnixNanos::from(ts),
3398                    UnixNanos::from(ts),
3399                ))
3400                .unwrap();
3401        }
3402
3403        assert_eq!(sealed_timestamps(temp_dir.path()), vec![vec![1_000]]);
3404        assert_eq!(writer.writers.len(), 1);
3405
3406        writer.close().unwrap();
3407
3408        assert!(writer.writers.is_empty());
3409        assert_eq!(
3410            sealed_timestamps(temp_dir.path()),
3411            vec![vec![1_000], vec![2_000]]
3412        );
3413        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3414    }
3415
3416    #[rstest]
3417    #[case(
3418        "FeatherMissingTimestamp",
3419        Schema::empty(),
3420        "registered without an Arrow schema containing ts_init"
3421    )]
3422    #[case(
3423        "FeatherLegacyTimestamp",
3424        Schema::new(vec![Field::new("ts_init", DataType::UInt64, false)]),
3425        "registered with ts_init as UInt64",
3426    )]
3427    fn test_custom_writer_path_rejects_unqueryable_schema(
3428        #[case] type_name: &str,
3429        #[case] schema: Schema,
3430        #[case] expected_error: &str,
3431    ) {
3432        nautilus_model::data::registry::ensure_arrow_registered(
3433            type_name,
3434            Arc::new(schema),
3435            Box::new(|_| unreachable!("writer creation does not encode data")),
3436            Box::new(|_, _| unreachable!("writer creation does not decode data")),
3437        )
3438        .unwrap();
3439
3440        let temp_dir = TempDir::new().unwrap();
3441
3442        let mut writer = FeatherWriter::new(
3443            temp_dir.path().to_path_buf(),
3444            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3445            RotationConfig::NoRotation,
3446            None,
3447            None,
3448        );
3449
3450        for _ in 0..2 {
3451            let error = writer.get_writer_path_custom(type_name).unwrap_err();
3452
3453            assert!(error.to_string().contains(expected_error));
3454            assert!(writer.writers.is_empty());
3455            assert!(writer.path_sequences.is_empty());
3456        }
3457    }
3458
3459    #[rstest]
3460    #[cfg(feature = "python")]
3461    fn test_write_custom_data_round_trip() {
3462        use std::sync::Arc;
3463
3464        use nautilus_model::{
3465            data::{CustomData, Data, DataType},
3466            identifiers::InstrumentId,
3467        };
3468        use nautilus_serialization::{
3469            arrow::custom::CustomDataDecoder, ensure_custom_data_registered,
3470        };
3471
3472        use crate::test_data::RustTestCustomData;
3473
3474        ensure_custom_data_registered::<RustTestCustomData>();
3475
3476        let temp_dir = TempDir::new().unwrap();
3477        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
3478
3479        let mut writer = FeatherWriter::new(
3480            temp_dir.path().to_path_buf(),
3481            clock,
3482            RotationConfig::NoRotation,
3483            None,
3484            None,
3485        );
3486
3487        let instrument_id = InstrumentId::from("RUST.TEST");
3488        let data_type = DataType::new("RustTestCustomData", None, Some(instrument_id.to_string()));
3489
3490        let original = RustTestCustomData {
3491            instrument_id,
3492            value: 1.23,
3493            flag: true,
3494            ts_event: UnixNanos::from(1000),
3495            ts_init: UnixNanos::from(1000),
3496        };
3497
3498        let custom = CustomData::new(Arc::new(original.clone()), data_type);
3499
3500        writer
3501            .write_data(Data::Custom(custom))
3502            .expect("write_data CustomData");
3503        writer.close().expect("close");
3504
3505        let prefix = temp_dir
3506            .path()
3507            .join("data")
3508            .join("custom")
3509            .join("RustTestCustomData");
3510        let files = feather_files(&prefix, FEATHER_EXTENSION);
3511        let batch = read_feather_batches(&files[0]).remove(0);
3512        let metadata = batch.schema().metadata().clone();
3513        let decoded =
3514            CustomDataDecoder::decode_data_batch(&metadata, batch).expect("decode_data_batch");
3515        assert_eq!(decoded.len(), 1);
3516
3517        if let Data::Custom(decoded_custom) = &decoded[0] {
3518            assert_eq!(decoded_custom.data_type.type_name(), "RustTestCustomData");
3519            let rust: &RustTestCustomData = decoded_custom
3520                .data
3521                .as_any()
3522                .downcast_ref::<RustTestCustomData>()
3523                .expect("RustTestCustomData");
3524            assert_eq!(rust, &original);
3525        } else {
3526            panic!("Expected Data::Custom");
3527        }
3528    }
3529
3530    #[rstest]
3531    #[cfg(feature = "python")]
3532    fn test_write_custom_data_reuses_writer_until_rotation() {
3533        use nautilus_model::data::{CustomData, DataType};
3534        use nautilus_serialization::ensure_custom_data_registered;
3535
3536        use crate::test_data::RustTestCustomData;
3537
3538        ensure_custom_data_registered::<RustTestCustomData>();
3539        let temp_dir = TempDir::new().unwrap();
3540
3541        let mut writer = FeatherWriter::new(
3542            temp_dir.path().to_path_buf(),
3543            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3544            RotationConfig::NoRotation,
3545            None,
3546            None,
3547        );
3548        let instrument_id = InstrumentId::from("RUST.TEST");
3549        let data_type = DataType::new("RustTestCustomData", None, Some(instrument_id.to_string()));
3550
3551        for (ts, value) in [(1_000, 1_000.0), (2_000, 2_000.0)] {
3552            writer
3553                .write_data(Data::Custom(CustomData::new(
3554                    Arc::new(RustTestCustomData {
3555                        instrument_id,
3556                        value,
3557                        flag: true,
3558                        ts_event: UnixNanos::from(ts),
3559                        ts_init: UnixNanos::from(ts),
3560                    }),
3561                    data_type.clone(),
3562                )))
3563                .unwrap();
3564        }
3565
3566        assert_eq!(writer.get_current_file_info().len(), 1);
3567        writer.close().unwrap();
3568
3569        let prefix = temp_dir
3570            .path()
3571            .join("data")
3572            .join("custom")
3573            .join("RustTestCustomData");
3574        let files = feather_files(&prefix, FEATHER_EXTENSION);
3575        assert_eq!(files.len(), 1);
3576        let rows = read_feather_batches(&files[0])
3577            .iter()
3578            .map(RecordBatch::num_rows)
3579            .sum::<usize>();
3580        assert_eq!(rows, 2);
3581    }
3582}