Skip to main content

nautilus_persistence/writer/
feather.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16#![expect(
17    clippy::missing_errors_doc,
18    reason = "Feather writer public methods forward encoding and file IO errors directly"
19)]
20
21use std::{
22    any::Any,
23    borrow::Cow,
24    cell::RefCell,
25    collections::{BTreeMap, HashMap, HashSet},
26    fmt::Debug,
27    fs::{self, File, OpenOptions, TryLockError},
28    io::{self, BufReader, BufWriter, Seek, Write},
29    path::{Path, PathBuf},
30    rc::Rc,
31    sync::{
32        Arc,
33        atomic::{AtomicU64, Ordering},
34        mpsc::Sender,
35    },
36};
37
38use ahash::AHashMap;
39use datafusion::arrow::{
40    array::{ArrayRef, DictionaryArray},
41    compute::concat_batches,
42    datatypes::{DataType, Field, Int32Type, Schema, SchemaRef},
43    error::ArrowError,
44    ipc::{reader::StreamReader, writer::StreamWriter},
45    record_batch::RecordBatch,
46};
47use jiff::{
48    SignedDuration,
49    civil::Time,
50    tz::{AmbiguousOffset, TimeZone},
51};
52use nautilus_common::{clock::Clock, live::LiveClock};
53use nautilus_core::{DurationNanos, UnixNanos, time::nanos_since_unix_epoch};
54use nautilus_model::{
55    data::{
56        Bar, CustomData, CustomDataTrait, Data, DataBatch, DataType as CustomDataType,
57        FundingRateUpdate, IndexPriceUpdate, InstrumentStatus, MarkPriceUpdate, NautilusDataType,
58        OptionGreeks, OrderBookDelta, OrderBookDeltas, OrderBookDepth, QuoteTick, TradeTick,
59        close::InstrumentClose,
60        encode_custom_to_arrow, get_arrow_schema,
61        greeks::{GreeksData, PortfolioGreeks},
62    },
63    events::{
64        AccountState, OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled,
65        OrderDenied, OrderEmulated, OrderEventAny, OrderExpired, OrderFillVoided, OrderFilled,
66        OrderInitialized, OrderModifyDenied, OrderModifyRejected, OrderPendingCancel,
67        OrderPendingUpdate, OrderRejected, OrderReleased, OrderSnapshot, OrderSubmitted,
68        OrderTriggered, OrderUpdated, PositionAdjusted, PositionChanged, PositionClosed,
69        PositionEvent, PositionOpened, PositionSnapshot,
70    },
71    instruments::{InstrumentAny, NautilusInstrumentType},
72    reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
73};
74use nautilus_serialization::{
75    arrow::{
76        EncodeToRecordBatch, KEY_TYPE_NAME, catalog_identifier_from_metadata,
77        record_batch_with_identifier_column, schema_with_identifier_column,
78    },
79    ensure_custom_data_registered,
80};
81
82use crate::{
83    catalog::{
84        session::DEFAULT_DATA_BATCH_CHUNK_SIZE,
85        types::{
86            CatalogDataType, CatalogFamily, data_path_prefix, instrument_path_prefix,
87            record_path_prefix,
88        },
89    },
90    common::custom::{
91        augment_batch_with_data_type_column, schema_with_data_type_column,
92        validate_custom_catalog_schema,
93    },
94    writer::{
95        filter::{WriterRecordFilter, catalog_family},
96        subscription::StreamingSinkSubscription,
97        traits::{StreamingDataSink, StreamingSink},
98    },
99};
100
101pub(crate) type FeatherWriteCommand =
102    Box<dyn FnOnce(&mut FeatherWriter) -> Result<(), Box<dyn std::error::Error>> + Send + 'static>;
103
104#[expect(
105    clippy::needless_pass_by_value,
106    reason = "map_err transfers ownership of the boxed writer error"
107)]
108pub(crate) fn feather_error(e: Box<dyn std::error::Error>) -> anyhow::Error {
109    anyhow::anyhow!("{e}")
110}
111
112macro_rules! define_builtin_data_batch_dispatch {
113    ($(($variant:ident, $type:ident, $data:ident, $batch:ident, $prefix:literal)),+ $(,)?) => {
114        fn write_builtin_data_batch(
115            writer: &mut FeatherWriter,
116            batch: &DataBatch,
117        ) -> Option<Result<(), Box<dyn std::error::Error>>> {
118            match batch {
119                $(
120                    DataBatch::$batch(data) => Some(writer.write_batch(data.as_ref().to_vec())),
121                )+
122                _ => None,
123            }
124        }
125    };
126}
127
128nautilus_model::for_each_data_type!(define_builtin_data_batch_dispatch);
129
130pub(crate) const NAUTILUS_ARROW_METADATA_ID_COLUMN: &str = "nautilus_metadata_id";
131pub(crate) const NAUTILUS_ARROW_METADATA_JSON_COLUMN: &str = "nautilus_metadata_json";
132
133/// Arrow type of the staged metadata columns; rows of one flush share a few distinct values.
134pub(crate) fn staged_metadata_data_type() -> DataType {
135    DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
136}
137
138pub(crate) fn staged_metadata_array<'a>(values: impl IntoIterator<Item = &'a str>) -> ArrayRef {
139    Arc::new(values.into_iter().collect::<DictionaryArray<Int32Type>>())
140}
141
142#[derive(Clone, Debug, Eq, PartialEq)]
143struct StagedArrowMetadataRow {
144    metadata_id: String,
145    metadata_json: String,
146}
147
148/// An open file and the catalog type it stages; each instrument class stages its own file.
149#[derive(Debug, Clone, PartialEq, Eq, Hash)]
150pub struct FileWriterPath {
151    path: PathBuf,
152    data_type: CatalogDataType,
153}
154
155/// Minimal `Send` time source for streaming writers.
156///
157/// `Live` reads the wall clock directly; `Test` reads a shared atomic that the
158/// owner of the source clock (e.g. a PyO3 wrapper holding a `VirtualClock`)
159/// refreshes before forwarding writer calls.
160#[derive(Clone, Debug)]
161pub enum WriterClock {
162    /// Wall-clock time via [`nanos_since_unix_epoch`].
163    Live,
164    /// Externally driven time stored in a shared atomic (nanoseconds since the UNIX epoch).
165    Test(Arc<AtomicU64>),
166}
167
168impl WriterClock {
169    /// Returns the current timestamp in nanoseconds since the UNIX epoch.
170    #[must_use]
171    pub fn timestamp_ns(&self) -> UnixNanos {
172        match self {
173            Self::Live => UnixNanos::from(nanos_since_unix_epoch()),
174            Self::Test(shared) => UnixNanos::from(shared.load(Ordering::Relaxed)),
175        }
176    }
177
178    /// Builds a writer clock from a shared `dyn Clock`.
179    ///
180    /// Returns [`WriterClock::Live`] for a [`LiveClock`]. For any other clock
181    /// (e.g. `VirtualClock`) it returns a [`WriterClock::Test`] source seeded with
182    /// the clock's current time, plus the shared atomic the caller must refresh
183    /// from the source clock before each forwarded writer call.
184    #[must_use]
185    pub fn from_shared_clock(clock: &Rc<RefCell<dyn Clock>>) -> (Self, Option<Arc<AtomicU64>>) {
186        let borrowed = clock.borrow();
187        let any_ref: &dyn Any = &*borrowed;
188        if any_ref.downcast_ref::<LiveClock>().is_some() {
189            (Self::Live, None)
190        } else {
191            let shared = Arc::new(AtomicU64::new(borrowed.timestamp_ns().as_u64()));
192            (Self::Test(Arc::clone(&shared)), Some(shared))
193        }
194    }
195}
196
197fn record_batch_with_delta_staged_metadata(
198    batch: &RecordBatch,
199    metadata_rows: &[StagedArrowMetadataRow],
200) -> anyhow::Result<RecordBatch> {
201    anyhow::ensure!(
202        batch.num_rows() == metadata_rows.len(),
203        "Delta Feather staging metadata row count {} does not match batch row count {}",
204        metadata_rows.len(),
205        batch.num_rows()
206    );
207    anyhow::ensure!(
208        batch
209            .schema()
210            .index_of(NAUTILUS_ARROW_METADATA_ID_COLUMN)
211            .is_err(),
212        "Delta Feather staging schema already has {NAUTILUS_ARROW_METADATA_ID_COLUMN}"
213    );
214    anyhow::ensure!(
215        batch
216            .schema()
217            .index_of(NAUTILUS_ARROW_METADATA_JSON_COLUMN)
218            .is_err(),
219        "Delta Feather staging schema already has {NAUTILUS_ARROW_METADATA_JSON_COLUMN}"
220    );
221
222    let mut fields = batch
223        .schema()
224        .fields()
225        .iter()
226        .map(|field| {
227            Arc::new(Field::new(
228                field.name().clone(),
229                field.data_type().clone(),
230                field.is_nullable(),
231            ))
232        })
233        .collect::<Vec<_>>();
234
235    fields.push(Arc::new(Field::new(
236        NAUTILUS_ARROW_METADATA_ID_COLUMN,
237        staged_metadata_data_type(),
238        false,
239    )));
240    fields.push(Arc::new(Field::new(
241        NAUTILUS_ARROW_METADATA_JSON_COLUMN,
242        staged_metadata_data_type(),
243        false,
244    )));
245
246    let mut columns = batch.columns().to_vec();
247    columns.push(staged_metadata_array(
248        metadata_rows.iter().map(|row| row.metadata_id.as_str()),
249    ));
250    columns.push(staged_metadata_array(
251        metadata_rows.iter().map(|row| row.metadata_json.as_str()),
252    ));
253
254    Ok(RecordBatch::try_new(
255        Arc::new(Schema::new(fields)),
256        columns,
257    )?)
258}
259
260fn arrow_metadata_row(
261    metadata: &HashMap<String, String>,
262    fields: &arrow::datatypes::Fields,
263) -> anyhow::Result<StagedArrowMetadataRow> {
264    let schema_metadata = metadata
265        .iter()
266        .map(|(key, value)| (key.clone(), value.clone()))
267        .collect::<BTreeMap<_, _>>();
268
269    let field_metadata = fields
270        .iter()
271        .filter(|field| !field.metadata().is_empty())
272        .map(|field| {
273            (
274                field.name().clone(),
275                field
276                    .metadata()
277                    .iter()
278                    .map(|(key, value)| (key.clone(), value.clone()))
279                    .collect::<BTreeMap<_, _>>(),
280            )
281        })
282        .collect::<BTreeMap<_, _>>();
283
284    let metadata_json = serde_json::to_string(&serde_json::json!({
285        "format_version": 1,
286        "schema_metadata": schema_metadata,
287        "field_metadata": field_metadata,
288    }))?;
289    let metadata_id = staged_metadata_id(&canonical_metadata_json(metadata)?);
290
291    Ok(StagedArrowMetadataRow {
292        metadata_id,
293        metadata_json,
294    })
295}
296
297/// Canonical schema-level metadata JSON: keys serialized in sorted order.
298///
299/// The staged metadata id hashes this exact string so readers can verify the schema metadata
300/// independently of its storage location.
301pub(crate) fn canonical_metadata_json(
302    metadata: &HashMap<String, String>,
303) -> anyhow::Result<String> {
304    Ok(serde_json::to_string(
305        &metadata
306            .iter()
307            .map(|(key, value)| (key.clone(), value.clone()))
308            .collect::<BTreeMap<_, _>>(),
309    )?)
310}
311
312pub(crate) fn staged_metadata_id(metadata_json: &str) -> String {
313    format!("blake3:{}", blake3::hash(metadata_json.as_bytes()).to_hex())
314}
315
316/// File extension of a sealed Feather stream file.
317pub(crate) const FEATHER_EXTENSION: &str = "feather";
318
319/// File extension of a Feather stream file that is still being appended to.
320pub(crate) const FEATHER_PARTIAL_EXTENSION: &str = "feather.partial";
321
322/// An open local Feather file that appends Arrow IPC stream batches.
323///
324/// Batches go to `{name}.feather.partial`, which is renamed to `{name}.feather` when the file is
325/// sealed, so readers listing `.feather` files only see complete streams. The file stays
326/// exclusively locked until it is sealed, so [`recover_partial_feather_files`] can tell an open
327/// file from one its writer abandoned. Written rows are
328/// buffered and appended as one record batch on flush, on seal, when the buffer reaches
329/// [`DEFAULT_DATA_BATCH_CHUNK_SIZE`] rows, or when size rotation may be due, because each IPC
330/// batch carries a message header that outweighs a single row.
331struct FeatherFile {
332    writer: StreamWriter<CountingWriter<BufWriter<File>>>,
333    partial_path: PathBuf,
334    path: PathBuf,
335    schema: SchemaRef,
336    max_size: Option<u64>,
337    buffer: Vec<RecordBatch>,
338    buffer_rows: usize,
339    buffer_bytes: u64,
340}
341
342impl FeatherFile {
343    fn create(
344        path: &Path,
345        schema: &Schema,
346        rotation_config: &RotationConfig,
347    ) -> Result<Self, Box<dyn std::error::Error>> {
348        let partial_path = path.with_extension(FEATHER_PARTIAL_EXTENSION);
349        if let Some(parent) = path.parent() {
350            fs::create_dir_all(parent)?;
351        }
352
353        let file = OpenOptions::new()
354            .write(true)
355            .create_new(true)
356            .open(&partial_path)?;
357
358        // Recovery can briefly lock a new, empty file before its writer does, so wait it out
359        file.lock().map_err(|e| {
360            format!(
361                "Failed to lock Feather file {}: {e}",
362                partial_path.display()
363            )
364        })?;
365
366        let writer = StreamWriter::try_new(CountingWriter::new(BufWriter::new(file)), schema)?;
367
368        let max_size = match rotation_config {
369            RotationConfig::Size { max_size } => Some(*max_size),
370            _ => None,
371        };
372
373        Ok(Self {
374            writer,
375            partial_path,
376            path: path.to_path_buf(),
377            schema: Arc::new(schema.clone()),
378            max_size,
379            buffer: Vec::new(),
380            buffer_rows: 0,
381            buffer_bytes: 0,
382        })
383    }
384
385    // Returns true when size rotation is due
386    fn write_record_batch(&mut self, batch: &RecordBatch) -> Result<bool, ArrowError> {
387        let batch = if batch.schema() == self.schema {
388            batch.clone()
389        } else {
390            RecordBatch::try_new(self.schema.clone(), batch.columns().to_vec())?
391        };
392
393        self.buffer_rows += batch.num_rows();
394        self.buffer_bytes += batch.get_array_memory_size() as u64;
395        self.buffer.push(batch);
396
397        if self.buffer_rows >= DEFAULT_DATA_BATCH_CHUNK_SIZE {
398            self.write_buffer()?;
399        }
400
401        let Some(max_size) = self.max_size else {
402            return Ok(false);
403        };
404
405        // The in-memory size only estimates the encoded size, so write before the exact check
406        if self.size() + self.buffer_bytes >= max_size {
407            self.write_buffer()?;
408        }
409
410        Ok(self.size() >= max_size)
411    }
412
413    fn write_buffer(&mut self) -> Result<(), ArrowError> {
414        if self.buffer.is_empty() {
415            return Ok(());
416        }
417
418        let batch = concat_batches(&self.schema, &self.buffer)?;
419        self.buffer.clear();
420        self.buffer_rows = 0;
421        self.buffer_bytes = 0;
422        self.writer.write(&batch)
423    }
424
425    fn size(&self) -> u64 {
426        self.writer.get_ref().bytes
427    }
428
429    fn flush(&mut self) -> Result<(), ArrowError> {
430        self.write_buffer()?;
431        self.writer.flush()
432    }
433
434    fn seal(mut self) -> Result<(), Box<dyn std::error::Error>> {
435        self.write_buffer()?;
436        self.writer.finish()?;
437        let file = self
438            .writer
439            .into_inner()?
440            .inner
441            .into_inner()
442            .map_err(io::IntoInnerError::into_error)?;
443        file.sync_all()?;
444
445        // Renaming before the file closes keeps it locked until it is sealed
446        fs::rename(&self.partial_path, &self.path)?;
447        Ok(())
448    }
449}
450
451/// Counts the bytes written through the inner writer.
452struct CountingWriter<W> {
453    inner: W,
454    bytes: u64,
455}
456
457impl<W> CountingWriter<W> {
458    const fn new(inner: W) -> Self {
459        Self { inner, bytes: 0 }
460    }
461}
462
463impl<W: Write> Write for CountingWriter<W> {
464    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
465        let written = self.inner.write(buf)?;
466        self.bytes += written as u64;
467        Ok(written)
468    }
469
470    fn flush(&mut self) -> io::Result<()> {
471        self.inner.flush()
472    }
473}
474
475/// Rotations and flushes collected by the synchronous encode path and run once it completes.
476#[derive(Debug, Default)]
477struct PendingIo {
478    rotate_paths: Vec<FileWriterPath>,
479    flush_due: bool,
480}
481
482/// Configuration for file rotation.
483#[derive(Debug, Clone)]
484pub enum RotationConfig {
485    /// Rotate based on file size.
486    Size {
487        /// Maximum buffer size in bytes before rotation.
488        max_size: u64,
489    },
490    /// Rotate based on a time interval.
491    Interval {
492        /// Interval in nanoseconds.
493        interval_ns: u64,
494    },
495    /// Rotate based on scheduled dates.
496    ScheduledDates {
497        /// Interval in nanoseconds.
498        interval_ns: u64,
499        /// Time of day for rotation (nanoseconds since midnight).
500        rotation_time: UnixNanos,
501        /// Timezone for rotation calculations.
502        rotation_timezone: TimeZone,
503    },
504    /// No automatic rotation.
505    NoRotation,
506}
507
508/// Streams encoded data into one local Feather file per data or record type, and per class for
509/// instruments.
510///
511/// Rows of every identifier share their type's file. Each row carries its catalog `identifier`
512/// and the Arrow metadata it was encoded with, such as its instrument's precision, so readers
513/// restore per-row metadata the file schema cannot hold.
514///
515/// The [`Self::write`] method accepts a data value of generic type `T`, encodes it using `T`'s
516/// metadata via `EncodeToRecordBatch`, routes it by `CatalogFamily`, and appends it to that key's
517/// open file.
518///
519/// # File Lifecycle
520///
521/// Open files are tracked by their final `.feather` paths, while writes go to `.feather.partial`
522/// files. Sealing completes the stream and renames the partial file to its final path. A file is
523/// sealed and replaced only on rotation, [`Self::seal`], or [`Self::close`].
524///
525/// # Flushing
526///
527/// [`Self::flush`] pushes buffered bytes to disk without starting a new file. `flush_interval_ms`
528/// controls automatic flushing in milliseconds; `0` disables automatic flushing.
529pub struct FeatherWriter {
530    directory: PathBuf,
531    clock: WriterClock,
532    rotation_config: RotationConfig,
533    included_types: Option<HashSet<CatalogDataType>>,
534    record_filter: Option<WriterRecordFilter>,
535    writers: HashMap<FileWriterPath, FeatherFile>,
536    path_sequences: HashMap<CatalogDataType, (UnixNanos, u64)>,
537    sealed_tx: Option<Sender<PathBuf>>,
538    next_rotation_times: HashMap<FileWriterPath, UnixNanos>,
539    flush_interval_ms: u64,
540    last_flush_ns: UnixNanos,
541    pending_write_error: Option<String>,
542}
543
544impl FeatherWriter {
545    /// Creates a new [`FeatherWriter`] instance.
546    #[must_use]
547    pub fn new(
548        directory: PathBuf,
549        clock: WriterClock,
550        rotation_config: RotationConfig,
551        included_types: Option<HashSet<CatalogDataType>>,
552        flush_interval_ms: Option<u64>,
553    ) -> Self {
554        let flush_interval_ms = flush_interval_ms.unwrap_or(1000); // Default 1 second
555        let last_flush_ns = clock.timestamp_ns();
556
557        Self {
558            directory,
559            clock,
560            rotation_config,
561            included_types,
562            record_filter: None,
563            writers: HashMap::new(),
564            path_sequences: HashMap::new(),
565            sealed_tx: None,
566            next_rotation_times: HashMap::new(),
567            flush_interval_ms,
568            last_flush_ns,
569            pending_write_error: None,
570        }
571    }
572
573    /// Sets typed record-family filter for subsequent writes.
574    #[must_use]
575    pub fn with_record_filter(mut self, record_filter: Option<WriterRecordFilter>) -> Self {
576        self.record_filter = record_filter;
577        self
578    }
579
580    pub(crate) fn with_sealed_sender(mut self, sealed_tx: Sender<PathBuf>) -> Self {
581        self.sealed_tx = Some(sealed_tx);
582        self
583    }
584
585    /// Writes a single data value.
586    ///
587    /// This is the user entry point. The data is encoded into a `RecordBatch` and appended to the
588    /// open file for its type, or for its class when the data is an instrument.
589    pub fn write<T>(&mut self, data: T) -> Result<(), Box<dyn std::error::Error>>
590    where
591        T: EncodeToRecordBatch + CatalogFamily + 'static,
592    {
593        let metadata = T::metadata(&data);
594        let identifier = catalog_identifier_from_metadata(&metadata);
595        let data_type = Self::staged_data_type::<T>(&metadata)?;
596
597        if !self.should_write_record(&data_type, identifier.as_deref()) {
598            return Ok(());
599        }
600
601        let path = self.get_writer_path(data_type);
602
603        // Create a new FileWriter if one does not exist.
604        if !self.writers.contains_key(&path) {
605            self.create_writer::<T>(path.clone(), &data)?;
606        }
607
608        // Encode the data into a RecordBatch using T's encoding logic.
609        let batch = T::encode_batch(&metadata, &[data])?;
610        let metadata_row = arrow_metadata_row(&metadata, batch.schema().fields())?;
611        let batch = record_batch_with_identifier_column(batch, identifier.as_deref())?;
612        let batch =
613            record_batch_with_delta_staged_metadata(&batch, std::slice::from_ref(&metadata_row))?;
614
615        // Write the RecordBatch to the appropriate FileWriter.
616        let mut pending = PendingIo::default();
617
618        self.stage_batch_write(path, &batch, &mut pending)?;
619        pending.flush_due = self.flush_is_due();
620
621        self.complete_pending_io(&pending)
622    }
623
624    /// Writes a batch of data values as one or more `RecordBatch`es.
625    ///
626    /// Uses `T::chunk_metadata` to derive the file schema metadata. This protects
627    /// types like `OrderBookDelta` from having their file metadata poisoned by a
628    /// leading sentinel row (e.g. `BookAction::Clear`, which carries
629    /// `price_precision=0, size_precision=0`).
630    ///
631    /// Rows are grouped by identifier so each group encodes with its own metadata before it is
632    /// appended to the type's file.
633    pub fn write_batch<T>(&mut self, data: Vec<T>) -> Result<(), Box<dyn std::error::Error>>
634    where
635        T: EncodeToRecordBatch + CatalogFamily + 'static,
636    {
637        if data.is_empty() || !self.should_write_family(&T::catalog_family()) {
638            return Ok(());
639        }
640
641        let mut groups: AHashMap<Option<String>, (CatalogDataType, Vec<T>)> = AHashMap::new();
642
643        for item in data {
644            let metadata = T::metadata(&item);
645            let identifier = catalog_identifier_from_metadata(&metadata);
646            let data_type = Self::staged_data_type::<T>(&metadata)?;
647
648            if !self.should_write_record(&data_type, identifier.as_deref()) {
649                continue;
650            }
651
652            groups
653                .entry(identifier)
654                .or_insert_with(|| (data_type, Vec::new()))
655                .1
656                .push(item);
657        }
658
659        if groups.is_empty() {
660            return Ok(());
661        }
662
663        let mut pending = PendingIo::default();
664
665        for (data_type, group) in groups.into_values() {
666            let path = self.get_writer_path(data_type);
667            let metadata = T::chunk_metadata(&group);
668
669            if !self.writers.contains_key(&path) {
670                self.create_writer_with_metadata::<T>(path.clone(), metadata.clone())?;
671            }
672
673            let identifier = catalog_identifier_from_metadata(&metadata);
674            let batch = T::encode_batch(&metadata, &group)?;
675            let metadata_rows = group
676                .iter()
677                .map(T::metadata)
678                .map(|metadata| arrow_metadata_row(&metadata, batch.schema().fields()))
679                .collect::<anyhow::Result<Vec<_>>>()?;
680            let batch = record_batch_with_identifier_column(batch, identifier.as_deref())?;
681            let batch = record_batch_with_delta_staged_metadata(&batch, &metadata_rows)?;
682
683            self.stage_batch_write(path, &batch, &mut pending)?;
684        }
685
686        pending.flush_due = self.flush_is_due();
687
688        self.complete_pending_io(&pending)
689    }
690
691    // Instrument classes have different fields, so each class stages its own file
692    fn staged_data_type<T: CatalogFamily>(
693        metadata: &HashMap<String, String>,
694    ) -> Result<CatalogDataType, Box<dyn std::error::Error>> {
695        let family = T::catalog_family();
696        if family != CatalogDataType::Data(NautilusDataType::Instrument) {
697            return Ok(family);
698        }
699
700        let class = metadata
701            .get(KEY_TYPE_NAME)
702            .ok_or("Instrument metadata has no type_name")?
703            .parse::<NautilusInstrumentType>()?;
704        Ok(CatalogDataType::Instrument(class))
705    }
706
707    fn stage_batch_write(
708        &mut self,
709        path: FileWriterPath,
710        batch: &RecordBatch,
711        pending: &mut PendingIo,
712    ) -> Result<(), Box<dyn std::error::Error>> {
713        let Some(file) = self.writers.get_mut(&path) else {
714            return Ok(());
715        };
716
717        let should_rotate = match file.write_record_batch(batch) {
718            Ok(should_rotate) => should_rotate,
719            Err(e) => {
720                self.abandon_file(&path);
721                return Err(e.into());
722            }
723        };
724
725        if should_rotate || self.check_scheduled_rotation(&path) {
726            pending.rotate_paths.push(path);
727        }
728
729        Ok(())
730    }
731
732    /// Returns whether the auto-flush interval has elapsed since the last flush.
733    fn flush_is_due(&self) -> bool {
734        if self.flush_interval_ms == 0 {
735            return false; // Auto-flush disabled
736        }
737
738        let now_ns = self.clock.timestamp_ns();
739        let elapsed_ms = now_ns.as_u64().saturating_sub(self.last_flush_ns.as_u64()) / 1_000_000;
740        elapsed_ms >= self.flush_interval_ms
741    }
742
743    fn complete_pending_io(
744        &mut self,
745        pending: &PendingIo,
746    ) -> Result<(), Box<dyn std::error::Error>> {
747        for path in &pending.rotate_paths {
748            self.seal_file(path)?;
749        }
750
751        if pending.flush_due {
752            self.flush()?;
753        }
754
755        Ok(())
756    }
757
758    fn check_scheduled_rotation(&mut self, path: &FileWriterPath) -> bool {
759        match &self.rotation_config {
760            RotationConfig::Interval { interval_ns } => {
761                let now = self.clock.timestamp_ns();
762                let next_rotation = self.next_rotation_times.get(path).copied();
763
764                match next_rotation {
765                    None => {
766                        self.next_rotation_times
767                            .insert(path.clone(), now + DurationNanos::new(*interval_ns));
768                        false
769                    }
770                    Some(next) if now >= next => {
771                        self.next_rotation_times
772                            .insert(path.clone(), now + DurationNanos::new(*interval_ns));
773                        true
774                    }
775                    _ => false,
776                }
777            }
778            RotationConfig::ScheduledDates {
779                interval_ns,
780                rotation_time,
781                rotation_timezone,
782            } => {
783                let now = self.clock.timestamp_ns();
784                let next_rotation = self.next_rotation_times.get(path).copied();
785
786                match next_rotation {
787                    None => {
788                        let next = self.calculate_next_scheduled_rotation(
789                            *rotation_time,
790                            rotation_timezone,
791                            *interval_ns,
792                        );
793                        self.next_rotation_times.insert(path.clone(), next);
794                        false
795                    }
796                    Some(next) if now >= next => {
797                        let next = self.calculate_next_scheduled_rotation(
798                            *rotation_time,
799                            rotation_timezone,
800                            *interval_ns,
801                        );
802                        self.next_rotation_times.insert(path.clone(), next);
803                        true
804                    }
805                    _ => false,
806                }
807            }
808            _ => false,
809        }
810    }
811
812    fn calculate_next_scheduled_rotation(
813        &self,
814        rotation_time: UnixNanos,
815        rotation_timezone: &TimeZone,
816        interval_ns: u64,
817    ) -> UnixNanos {
818        let now_utc = self.clock.timestamp_ns().to_datetime_utc();
819        let now_local = rotation_timezone.to_datetime(now_utc);
820
821        let rotation_time_secs = u32::try_from(*rotation_time / 1_000_000_000).unwrap_or(0);
822        let rotation_time_nanos = i32::try_from(*rotation_time % 1_000_000_000).unwrap_or(0);
823
824        let rotation_time = if rotation_time_secs < 86_400 {
825            Time::new(
826                i8::try_from(rotation_time_secs / 3_600).unwrap_or(0),
827                i8::try_from(rotation_time_secs % 3_600 / 60).unwrap_or(0),
828                i8::try_from(rotation_time_secs % 60).unwrap_or(0),
829                rotation_time_nanos,
830            )
831            .unwrap_or(Time::MIN)
832        } else {
833            Time::MIN
834        };
835
836        let local_rotation = now_local.date().to_datetime(rotation_time);
837        let ambiguous = rotation_timezone.to_ambiguous_timestamp(local_rotation);
838
839        let mut next_rotation = match ambiguous.offset() {
840            AmbiguousOffset::Gap { .. } => now_utc,
841            _ => ambiguous.earlier().unwrap_or(now_utc),
842        };
843
844        if next_rotation <= now_utc {
845            // If the time has already passed today, we would usually add the interval
846            // But let's align exactly with how Python does it:
847            while next_rotation <= now_utc {
848                next_rotation += SignedDuration::from_nanos_i128(i128::from(interval_ns));
849            }
850        }
851
852        UnixNanos::from(u64::try_from(next_rotation.as_nanosecond()).unwrap_or(0))
853    }
854
855    // Seals the open file at `path`; the next write for its key opens a new file. A failed seal
856    // leaves the stream unfinished, so the file is recovered to its complete batches instead.
857    fn seal_file(&mut self, path: &FileWriterPath) -> Result<(), Box<dyn std::error::Error>> {
858        self.next_rotation_times.remove(path);
859
860        let Some(file) = self.writers.remove(path) else {
861            return Ok(());
862        };
863
864        let partial_path = file.partial_path.clone();
865        let result = file
866            .seal()
867            .inspect_err(|_| recover_released_feather_file(&partial_path));
868        self.notify_sealed(path);
869        result
870    }
871
872    // A failed append can leave a partial IPC message, so the file is recovered to its complete
873    // batches rather than sealed as written; the next write for its key opens a new file.
874    fn abandon_file(&mut self, path: &FileWriterPath) {
875        self.next_rotation_times.remove(path);
876
877        let Some(file) = self.writers.remove(path) else {
878            return;
879        };
880
881        let partial_path = file.partial_path.clone();
882        drop(file);
883
884        log::error!(
885            "Abandoned Feather file {} after a failed write",
886            path.path.display()
887        );
888
889        recover_released_feather_file(&partial_path);
890        self.notify_sealed(path);
891    }
892
893    fn notify_sealed(&self, path: &FileWriterPath) {
894        if let Some(sealed_tx) = &self.sealed_tx
895            && path.path.exists()
896        {
897            let _ = sealed_tx.send(path.path.clone());
898        }
899    }
900
901    /// Creates (and inserts) a new `FileWriter` for type T.
902    fn create_writer<T>(
903        &mut self,
904        path: FileWriterPath,
905        data: &T,
906    ) -> Result<(), Box<dyn std::error::Error>>
907    where
908        T: EncodeToRecordBatch + CatalogFamily + 'static,
909    {
910        self.create_writer_with_metadata::<T>(path, T::metadata(data))
911    }
912
913    /// Creates (and inserts) a new `FileWriter` for type T with pre-computed metadata.
914    ///
915    /// Use this variant when the caller has selected metadata from a chunk
916    /// (e.g. via `T::chunk_metadata`) to avoid schema poisoning by sentinel rows.
917    fn create_writer_with_metadata<T>(
918        &mut self,
919        path: FileWriterPath,
920        metadata: HashMap<String, String>,
921    ) -> Result<(), Box<dyn std::error::Error>>
922    where
923        T: EncodeToRecordBatch + CatalogFamily + 'static,
924    {
925        let schema = Self::schema_with_delta_staging_columns(&schema_with_identifier_column(
926            &T::get_schema(Some(metadata)),
927        ));
928
929        let file = FeatherFile::create(&path.path, &schema, &self.rotation_config)?;
930        self.writers.insert(path, file);
931        Ok(())
932    }
933
934    /// Creates (and inserts) a new `FeatherBuffer` for custom data at the given path.
935    fn create_custom_writer(
936        &mut self,
937        path: FileWriterPath,
938        type_name: &str,
939    ) -> Result<(), Box<dyn std::error::Error>> {
940        if self.writers.contains_key(&path) {
941            return Ok(());
942        }
943
944        let base_schema = get_arrow_schema(type_name).ok_or_else(|| {
945            format!("Custom data type \"{type_name}\" is not registered for Arrow encoding")
946        })?;
947
948        let schema = schema_with_data_type_column(base_schema.as_ref(), type_name);
949
950        let schema =
951            Self::schema_with_delta_staging_columns(&schema_with_identifier_column(&schema));
952
953        let file = FeatherFile::create(&path.path, &schema, &self.rotation_config)
954            .map_err(|e| format!("Failed to create Feather file for custom {type_name}: {e}"))?;
955        self.writers.insert(path, file);
956        Ok(())
957    }
958
959    /// Encodes a single `CustomData` into a `RecordBatch` with `data_type` column (catalog-compatible).
960    pub(crate) fn encode_custom_to_batch(
961        custom: &CustomData,
962    ) -> Result<RecordBatch, Box<dyn std::error::Error>> {
963        let type_name = custom.data.type_name();
964        let data_type_json = custom
965            .data_type
966            .to_persistence_json()
967            .map_err(|e| format!("Failed to serialize data_type for persistence: {e}"))?;
968        let dt_meta = custom.data_type.metadata_string_map();
969        let items: [Arc<dyn CustomDataTrait>; 1] = [Arc::clone(&custom.data)];
970
971        let batch = encode_custom_to_arrow(type_name, &items)
972            .map_err(|e| format!("Failed to encode custom data: {e}"))?
973            .ok_or_else(|| {
974                format!("Custom data type \"{type_name}\" is not registered for Arrow")
975            })?;
976
977        let batch = augment_batch_with_data_type_column(
978            &batch,
979            &data_type_json,
980            type_name,
981            dt_meta.as_ref(),
982        )
983        .map_err(|e| e.to_string())?;
984        Ok(batch)
985    }
986
987    fn schema_with_delta_staging_columns(schema: &Schema) -> Schema {
988        let mut fields = schema
989            .fields()
990            .iter()
991            .map(|field| {
992                Arc::new(Field::new(
993                    field.name().clone(),
994                    field.data_type().clone(),
995                    field.is_nullable(),
996                ))
997            })
998            .collect::<Vec<_>>();
999
1000        if schema.index_of(NAUTILUS_ARROW_METADATA_ID_COLUMN).is_err() {
1001            fields.push(Arc::new(Field::new(
1002                NAUTILUS_ARROW_METADATA_ID_COLUMN,
1003                staged_metadata_data_type(),
1004                false,
1005            )));
1006        }
1007
1008        if schema
1009            .index_of(NAUTILUS_ARROW_METADATA_JSON_COLUMN)
1010            .is_err()
1011        {
1012            fields.push(Arc::new(Field::new(
1013                NAUTILUS_ARROW_METADATA_JSON_COLUMN,
1014                staged_metadata_data_type(),
1015                false,
1016            )));
1017        }
1018
1019        Schema::new(fields)
1020    }
1021
1022    /// Flushes buffered bytes of every open file to disk.
1023    ///
1024    /// This is called automatically based on `flush_interval_ms` if configured, but can also
1025    /// be called manually by the client. Files stay open, so flushing never starts a new file.
1026    pub fn flush(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1027        let paths = self.writers.keys().cloned().collect::<Vec<_>>();
1028
1029        for path in paths {
1030            if let Some(file) = self.writers.get_mut(&path)
1031                && let Err(e) = file.flush()
1032            {
1033                self.abandon_file(&path);
1034                return Err(e.into());
1035            }
1036        }
1037
1038        self.last_flush_ns = self.clock.timestamp_ns();
1039
1040        if let Some(error) = self.pending_write_error.take() {
1041            return Err(error.into());
1042        }
1043
1044        Ok(())
1045    }
1046
1047    /// Seals every open file so its complete stream is visible as a `.feather` file.
1048    ///
1049    /// Writes after sealing open new files, so this is the boundary promotion uses.
1050    pub fn seal(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1051        let paths = self.writers.keys().cloned().collect::<Vec<_>>();
1052        let mut first_error = None;
1053
1054        for path in paths {
1055            if let Err(e) = self.seal_file(&path) {
1056                log::error!("Failed to seal Feather file {}: {e}", path.path.display());
1057                first_error.get_or_insert(e);
1058            }
1059        }
1060
1061        match first_error {
1062            Some(e) => Err(e),
1063            None => Ok(()),
1064        }
1065    }
1066
1067    /// Seals all open files and reports any write error recorded since the last flush.
1068    ///
1069    /// Writes after closing open new files.
1070    pub fn close(&mut self) -> Result<(), Box<dyn std::error::Error>> {
1071        self.seal()?;
1072
1073        if let Some(error) = self.pending_write_error.take() {
1074            return Err(error.into());
1075        }
1076
1077        Ok(())
1078    }
1079
1080    /// Returns whether the writer has been closed (all writers cleared).
1081    #[must_use]
1082    pub fn is_closed(&self) -> bool {
1083        self.writers.is_empty()
1084    }
1085
1086    /// Returns the current file size and path of each open file, keyed by the type it stages.
1087    #[must_use]
1088    pub fn get_current_file_info(&self) -> HashMap<CatalogDataType, (u64, String)> {
1089        self.writers
1090            .iter()
1091            .map(|(path, file)| {
1092                (
1093                    path.data_type.clone(),
1094                    (file.size(), path.path.display().to_string()),
1095                )
1096            })
1097            .collect()
1098    }
1099
1100    /// Returns the next rotation time of the file staging `data_type`, if set.
1101    ///
1102    /// Each instrument class stages its own file, so pass [`CatalogDataType::Instrument`].
1103    #[must_use]
1104    pub fn get_next_rotation_time(&self, data_type: &CatalogDataType) -> Option<UnixNanos> {
1105        self.next_rotation_times
1106            .iter()
1107            .find(|(path, _)| path.data_type == *data_type)
1108            .map(|(_, &next)| next)
1109    }
1110
1111    /// Determines whether a family can be written before row-level identifier checks.
1112    fn should_write_family(&self, family: &CatalogDataType) -> bool {
1113        self.included_types
1114            .as_ref()
1115            .is_none_or(|included| included.contains(family))
1116            && self
1117                .record_filter
1118                .as_ref()
1119                .is_none_or(|filter| filter.contains(family))
1120    }
1121
1122    fn should_write_record(&self, data_type: &CatalogDataType, identifier: Option<&str>) -> bool {
1123        self.included_types
1124            .as_ref()
1125            .is_none_or(|included| included.contains(&catalog_family(data_type)))
1126            && self
1127                .record_filter
1128                .as_ref()
1129                .is_none_or(|filter| filter.allows(data_type, identifier))
1130    }
1131
1132    fn reserve_writer_path(&mut self, data_type: CatalogDataType) -> FileWriterPath {
1133        let now = self.clock.timestamp_ns();
1134        let (timestamp, first_sequence) = self
1135            .path_sequences
1136            .get(&data_type)
1137            .copied()
1138            .filter(|(timestamp, _)| *timestamp >= now)
1139            .unwrap_or((now, 0));
1140
1141        for sequence in first_sequence.. {
1142            let path = self.build_writer_path(&data_type, timestamp, sequence);
1143
1144            // Files left by an earlier writer in the same directory keep their names
1145            if path.exists() || path.with_extension(FEATHER_PARTIAL_EXTENSION).exists() {
1146                continue;
1147            }
1148
1149            self.path_sequences
1150                .insert(data_type.clone(), (timestamp, sequence + 1));
1151            return FileWriterPath { path, data_type };
1152        }
1153
1154        unreachable!("unbounded writer path sequence exhausted")
1155    }
1156
1157    fn build_writer_path(
1158        &self,
1159        data_type: &CatalogDataType,
1160        timestamp: UnixNanos,
1161        sequence: u64,
1162    ) -> PathBuf {
1163        let (directory, file_stem) = match data_type {
1164            CatalogDataType::Data(NautilusDataType::Custom { type_name }) => (
1165                self.directory.join("data").join("custom").join(type_name),
1166                Cow::Borrowed(type_name.as_str()),
1167            ),
1168            CatalogDataType::Data(data_type) => {
1169                let prefix = data_path_prefix(data_type);
1170                (self.directory.join(prefix.as_ref()), prefix)
1171            }
1172            CatalogDataType::Record(record_type) => {
1173                let prefix = record_path_prefix(record_type);
1174                (self.directory.join(prefix.as_ref()), prefix)
1175            }
1176            CatalogDataType::Instrument(class) => {
1177                let prefix = data_path_prefix(&NautilusDataType::Instrument);
1178                (
1179                    self.directory
1180                        .join(prefix.as_ref())
1181                        .join(instrument_path_prefix(class)),
1182                    prefix,
1183                )
1184            }
1185        };
1186
1187        directory.join(Self::timestamped_feather_file_name(
1188            &file_stem, timestamp, sequence,
1189        ))
1190    }
1191
1192    fn timestamped_feather_file_name(stem: &str, timestamp: UnixNanos, sequence: u64) -> String {
1193        if sequence == 0 {
1194            format!("{stem}_{timestamp}.{FEATHER_EXTENSION}")
1195        } else {
1196            format!("{stem}_{timestamp}-{sequence}.{FEATHER_EXTENSION}")
1197        }
1198    }
1199
1200    /// Returns the open file of a custom type, or reserves `data/custom/{type_name}/` for it.
1201    fn get_writer_path_custom(
1202        &mut self,
1203        type_name: &str,
1204    ) -> Result<FileWriterPath, Box<dyn std::error::Error>> {
1205        let data_type = CatalogDataType::from(NautilusDataType::Custom {
1206            type_name: type_name.to_string(),
1207        });
1208
1209        if let Some(existing) = self.writers.keys().find(|path| path.data_type == data_type) {
1210            return Ok(existing.clone());
1211        }
1212
1213        if let Some(schema) = get_arrow_schema(type_name) {
1214            validate_custom_catalog_schema(type_name, &schema)?;
1215        }
1216
1217        Ok(self.reserve_writer_path(data_type))
1218    }
1219
1220    /// Returns the open file staging `data_type`, or reserves a new one.
1221    fn get_writer_path(&mut self, data_type: CatalogDataType) -> FileWriterPath {
1222        if let Some(existing) = self.writers.keys().find(|path| path.data_type == data_type) {
1223            return existing.clone();
1224        }
1225
1226        self.reserve_writer_path(data_type)
1227    }
1228
1229    /// Writes a Data enum value to the appropriate writer.
1230    ///
1231    /// This is a convenience method that routes the Data enum to the appropriate
1232    /// typed write method.
1233    pub fn write_data(&mut self, data: Data) -> Result<(), Box<dyn std::error::Error>> {
1234        match data {
1235            Data::Instrument(instrument) => self.write(*instrument),
1236            Data::Quote(quote) => self.write(quote),
1237            Data::Trade(trade) => self.write(trade),
1238            Data::Bar(bar) => self.write(bar),
1239            Data::BookDelta(delta) => self.write(delta),
1240            Data::BookDepth(depth) => self.write(*depth),
1241            Data::IndexPrice(price) => self.write(price),
1242            Data::MarkPrice(price) => self.write(price),
1243            Data::FundingRate(funding) => self.write(funding),
1244            Data::InstrumentStatus(status) => self.write(status),
1245            Data::OptionGreeks(greeks) => self.write(greeks),
1246            Data::InstrumentClose(close) => self.write(close),
1247            Data::Custom(custom) => self.write_custom_data(&custom),
1248            Data::BookDeltas(deltas_api) => {
1249                // Batch write so chunk_metadata can skip a leading BookAction::Clear sentinel
1250                self.write_batch(deltas_api.deltas.clone())
1251            }
1252            #[cfg(feature = "defi")]
1253            Data::Defi(_) => Err("Unsupported DeFi data variant for feather writes".into()),
1254            #[cfg(not(feature = "defi"))]
1255            #[allow(
1256                unreachable_patterns,
1257                reason = "DeFi variants can exist without this crate's defi feature"
1258            )]
1259            _ => Err("Unsupported DeFi data variant for feather writes".into()),
1260        }
1261    }
1262
1263    /// Writes mixed data as typed Arrow batches.
1264    ///
1265    /// # Panics
1266    ///
1267    /// Panics if the built-in data batch dispatch becomes non-exhaustive.
1268    #[expect(
1269        clippy::needless_pass_by_value,
1270        reason = "the public writer API accepts ownership of each submitted data batch"
1271    )]
1272    pub fn write_data_batch(&mut self, data: Vec<Data>) -> Result<(), Box<dyn std::error::Error>> {
1273        for batch in DataBatch::from_data_vec_grouped(&data)? {
1274            match &batch {
1275                DataBatch::Custom(data) => {
1276                    for custom in data.as_ref() {
1277                        self.write_custom_data(custom)?;
1278                    }
1279                }
1280                batch => write_builtin_data_batch(self, batch)
1281                    .expect("built-in data batch dispatch is exhaustive")?,
1282            }
1283        }
1284
1285        Ok(())
1286    }
1287
1288    /// Writes supported message bus value.
1289    pub fn write_any_message(
1290        &mut self,
1291        message: &dyn Any,
1292    ) -> Result<bool, Box<dyn std::error::Error>> {
1293        let Some(command) = Self::write_command(message) else {
1294            return Ok(false);
1295        };
1296
1297        if let Err(e) = command(self) {
1298            self.record_write_error(e.to_string());
1299            return Err(e);
1300        }
1301
1302        Ok(true)
1303    }
1304
1305    pub(crate) fn record_write_error(&mut self, error: String) {
1306        self.pending_write_error.get_or_insert(error);
1307    }
1308
1309    pub(crate) fn write_command(message: &dyn Any) -> Option<FeatherWriteCommand> {
1310        macro_rules! try_write {
1311            ($message:expr, $type:ty) => {
1312                if let Some(value) = $message.downcast_ref::<$type>() {
1313                    let value = value.clone();
1314                    return Some(Box::new(move |writer| writer.write(value)));
1315                }
1316            };
1317        }
1318
1319        try_write!(message, QuoteTick);
1320        try_write!(message, TradeTick);
1321        try_write!(message, Bar);
1322        try_write!(message, OrderBookDelta);
1323        try_write!(message, OrderBookDepth);
1324        try_write!(message, IndexPriceUpdate);
1325        try_write!(message, MarkPriceUpdate);
1326        try_write!(message, FundingRateUpdate);
1327        try_write!(message, InstrumentStatus);
1328        try_write!(message, OptionGreeks);
1329        try_write!(message, InstrumentClose);
1330        try_write!(message, InstrumentAny);
1331        try_write!(message, AccountState);
1332        try_write!(message, OrderInitialized);
1333        try_write!(message, OrderDenied);
1334        try_write!(message, OrderEmulated);
1335        try_write!(message, OrderSubmitted);
1336        try_write!(message, OrderAccepted);
1337        try_write!(message, OrderRejected);
1338        try_write!(message, OrderPendingCancel);
1339        try_write!(message, OrderCanceled);
1340        try_write!(message, OrderCancelDenied);
1341        try_write!(message, OrderCancelRejected);
1342        try_write!(message, OrderExpired);
1343        try_write!(message, OrderTriggered);
1344        try_write!(message, OrderPendingUpdate);
1345        try_write!(message, OrderReleased);
1346        try_write!(message, OrderModifyDenied);
1347        try_write!(message, OrderModifyRejected);
1348        try_write!(message, OrderUpdated);
1349        try_write!(message, OrderFilled);
1350        try_write!(message, OrderFillVoided);
1351        try_write!(message, PositionOpened);
1352        try_write!(message, PositionChanged);
1353        try_write!(message, PositionClosed);
1354        try_write!(message, PositionAdjusted);
1355        try_write!(message, OrderSnapshot);
1356        try_write!(message, PositionSnapshot);
1357        try_write!(message, OrderStatusReport);
1358        try_write!(message, FillReport);
1359        try_write!(message, PositionStatusReport);
1360        try_write!(message, ExecutionMassStatus);
1361
1362        if let Some(greeks) = message.downcast_ref::<GreeksData>() {
1363            ensure_custom_data_registered::<GreeksData>();
1364
1365            let data_type = CustomDataType::new(
1366                GreeksData::type_name_static(),
1367                None,
1368                Some(greeks.instrument_id.to_string()),
1369            );
1370
1371            let custom = CustomData::new(Arc::new(greeks.clone()), data_type);
1372            return Some(Box::new(move |writer| writer.write_custom_data(&custom)));
1373        }
1374
1375        if let Some(portfolio) = message.downcast_ref::<PortfolioGreeks>() {
1376            ensure_custom_data_registered::<PortfolioGreeks>();
1377            let custom = CustomData::from_arc(Arc::new(portfolio.clone()));
1378            return Some(Box::new(move |writer| writer.write_custom_data(&custom)));
1379        }
1380
1381        if let Some(deltas) = message.downcast_ref::<OrderBookDeltas>() {
1382            let deltas = deltas.deltas.clone();
1383            return Some(Box::new(move |writer| writer.write_batch(deltas)));
1384        }
1385
1386        if let Some(data) = message.downcast_ref::<Data>() {
1387            let data = data.clone();
1388            return Some(Box::new(move |writer| writer.write_data(data)));
1389        }
1390
1391        if let Some(custom) = message.downcast_ref::<CustomData>() {
1392            let custom = custom.clone();
1393            return Some(Box::new(move |writer| {
1394                writer.write_data(Data::Custom(custom))
1395            }));
1396        }
1397
1398        if let Some(event) = message.downcast_ref::<OrderEventAny>() {
1399            let event = event.clone();
1400            return Some(Box::new(move |writer| writer.write_order_event(&event)));
1401        }
1402
1403        if let Some(event) = message.downcast_ref::<PositionEvent>() {
1404            let event = event.clone();
1405            return Some(Box::new(move |writer| writer.write_position_event(&event)));
1406        }
1407
1408        None
1409    }
1410
1411    fn write_order_event(
1412        &mut self,
1413        event: &OrderEventAny,
1414    ) -> Result<(), Box<dyn std::error::Error>> {
1415        match event {
1416            OrderEventAny::Initialized(event) => self.write(event.clone()),
1417            OrderEventAny::Denied(event) => self.write(*event),
1418            OrderEventAny::Emulated(event) => self.write(*event),
1419            OrderEventAny::Released(event) => self.write(*event),
1420            OrderEventAny::Submitted(event) => self.write(*event),
1421            OrderEventAny::Accepted(event) => self.write(*event),
1422            OrderEventAny::Rejected(event) => self.write(*event),
1423            OrderEventAny::Canceled(event) => self.write(*event),
1424            OrderEventAny::Expired(event) => self.write(*event),
1425            OrderEventAny::Triggered(event) => self.write(*event),
1426            OrderEventAny::PendingUpdate(event) => self.write(*event),
1427            OrderEventAny::PendingCancel(event) => self.write(*event),
1428            OrderEventAny::ModifyDenied(event) => self.write(*event),
1429            OrderEventAny::ModifyRejected(event) => self.write(*event),
1430            OrderEventAny::CancelDenied(event) => self.write(*event),
1431            OrderEventAny::CancelRejected(event) => self.write(*event),
1432            OrderEventAny::Updated(event) => self.write(*event),
1433            OrderEventAny::Filled(event) => self.write(event.clone()),
1434            OrderEventAny::FillVoided(event) => self.write(event.clone()),
1435        }
1436    }
1437
1438    fn write_position_event(
1439        &mut self,
1440        event: &PositionEvent,
1441    ) -> Result<(), Box<dyn std::error::Error>> {
1442        match event {
1443            PositionEvent::PositionOpened(event) => self.write(event.clone()),
1444            PositionEvent::PositionChanged(event) => self.write(event.clone()),
1445            PositionEvent::PositionClosed(event) => self.write(event.clone()),
1446            PositionEvent::PositionAdjusted(event) => self.write(*event),
1447        }
1448    }
1449
1450    /// Writes a single custom data value (catalog layout: `data/custom/{type_name}[/{identifier}]`).
1451    fn write_custom_data(&mut self, custom: &CustomData) -> Result<(), Box<dyn std::error::Error>> {
1452        let batch = Self::encode_custom_to_batch(custom)?;
1453        self.write_custom_batch(custom, batch)
1454    }
1455
1456    pub(crate) fn write_custom_batch(
1457        &mut self,
1458        custom: &CustomData,
1459        mut batch: RecordBatch,
1460    ) -> Result<(), Box<dyn std::error::Error>> {
1461        let type_name = custom.data.type_name();
1462        let identifier = custom.data_type.identifier().map(String::from);
1463
1464        if !self.should_write_custom(type_name, identifier.as_deref()) {
1465            return Ok(());
1466        }
1467
1468        let path = self.get_writer_path_custom(type_name)?;
1469        if !self.writers.contains_key(&path) {
1470            self.create_custom_writer(path.clone(), type_name)?;
1471        }
1472
1473        let metadata_row = arrow_metadata_row(batch.schema().metadata(), batch.schema().fields())?;
1474        batch = record_batch_with_identifier_column(batch, custom.data_type.identifier())?;
1475        batch =
1476            record_batch_with_delta_staged_metadata(&batch, std::slice::from_ref(&metadata_row))?;
1477
1478        let mut pending = PendingIo::default();
1479
1480        self.stage_batch_write(path, &batch, &mut pending)?;
1481        pending.flush_due = self.flush_is_due();
1482
1483        self.complete_pending_io(&pending)
1484    }
1485
1486    pub(crate) fn should_write_custom(&self, type_name: &str, identifier: Option<&str>) -> bool {
1487        self.should_write_record(
1488            &CatalogDataType::from(NautilusDataType::Custom {
1489                type_name: type_name.to_string(),
1490            }),
1491            identifier,
1492        )
1493    }
1494
1495    /// Writes an instrument to the appropriate writer.
1496    ///
1497    /// Each instrument class stages its own file, and every row carries its instrument ID.
1498    pub fn write_instrument(
1499        &mut self,
1500        instrument: InstrumentAny,
1501    ) -> Result<(), Box<dyn std::error::Error>> {
1502        self.write(instrument)
1503    }
1504
1505    /// Subscribes to all messages on the message bus (pattern "*").
1506    ///
1507    /// This will automatically write all supported data types that are published
1508    /// on the message bus to the feather files.
1509    ///
1510    /// The writer must be wrapped in `Rc<RefCell<>>` to be shareable with the message bus handler.
1511    ///
1512    /// Note: Writes are synchronous local file appends.
1513    pub fn subscribe_to_message_bus(
1514        writer: Rc<RefCell<Self>>,
1515    ) -> Result<StreamingSinkSubscription, Box<dyn std::error::Error>> {
1516        let sink: StreamingDataSink = Box::new(writer);
1517        Ok(StreamingSinkSubscription::subscribe(
1518            Rc::new(RefCell::new(sink)),
1519            None,
1520        ))
1521    }
1522
1523    /// Unsubscribes from message bus.
1524    pub fn unsubscribe_from_message_bus(handler: &StreamingSinkSubscription) {
1525        handler.unsubscribe();
1526    }
1527}
1528
1529impl Debug for FeatherWriter {
1530    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1531        f.debug_struct(stringify!(FeatherWriter))
1532            .finish_non_exhaustive()
1533    }
1534}
1535
1536/// Drop contract: seals open files so their streams stay readable and promotable.
1537///
1538/// Errors are logged, never raised. Call [`FeatherWriter::close`] for deterministic sealing.
1539impl Drop for FeatherWriter {
1540    fn drop(&mut self) {
1541        if !self.writers.is_empty() {
1542            let _ = self.seal();
1543        }
1544    }
1545}
1546
1547impl StreamingSink for FeatherWriter {
1548    fn write_data(&mut self, data: Data) -> anyhow::Result<()> {
1549        Self::write_data(self, data).map_err(feather_error)
1550    }
1551
1552    fn write_batch(&mut self, data: Vec<Data>) -> anyhow::Result<()> {
1553        Self::write_data_batch(self, data).map_err(feather_error)
1554    }
1555
1556    fn write_any(&mut self, message: &dyn Any) -> anyhow::Result<bool> {
1557        self.write_any_message(message).map_err(feather_error)
1558    }
1559
1560    fn flush(&mut self) -> anyhow::Result<()> {
1561        Self::flush(self).map_err(feather_error)
1562    }
1563
1564    fn close(&mut self) -> anyhow::Result<()> {
1565        Self::close(self).map_err(feather_error)
1566    }
1567}
1568
1569impl StreamingSink for Rc<RefCell<FeatherWriter>> {
1570    fn write_data(&mut self, data: Data) -> anyhow::Result<()> {
1571        StreamingSink::write_data(&mut *self.borrow_mut(), data)
1572    }
1573
1574    fn write_batch(&mut self, data: Vec<Data>) -> anyhow::Result<()> {
1575        StreamingSink::write_batch(&mut *self.borrow_mut(), data)
1576    }
1577
1578    fn write_any(&mut self, message: &dyn Any) -> anyhow::Result<bool> {
1579        StreamingSink::write_any(&mut *self.borrow_mut(), message)
1580    }
1581
1582    fn flush(&mut self) -> anyhow::Result<()> {
1583        StreamingSink::flush(&mut *self.borrow_mut())
1584    }
1585
1586    fn close(&mut self) -> anyhow::Result<()> {
1587        StreamingSink::close(&mut *self.borrow_mut())
1588    }
1589}
1590
1591/// Seals every abandoned `.feather.partial` file under `directory`.
1592///
1593/// A writer locks each file until it seals it, so a file that recovery can lock was left by a
1594/// writer that crashed, was killed, or released it after a failed write or seal. Recovery keeps the
1595/// complete record batches, drops any bytes after them with a warning, and renames the file to
1596/// `.feather`. It removes a file with no complete record batch and leaves an empty one, which a new
1597/// writer may not have locked yet. A read error other than a write cut short leaves the file for a
1598/// later pass. Failures are logged per file, so callers still read the sealed files.
1599pub(crate) fn recover_partial_feather_files(directory: &Path) {
1600    let suffix = format!(".{FEATHER_PARTIAL_EXTENSION}");
1601    let mut directories = vec![directory.to_path_buf()];
1602
1603    while let Some(directory) = directories.pop() {
1604        let entries = match fs::read_dir(&directory) {
1605            Ok(entries) => entries,
1606            Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
1607            Err(e) => {
1608                log::warn!(
1609                    "Failed to list {} for partial Feather files: {e}",
1610                    directory.display()
1611                );
1612                continue;
1613            }
1614        };
1615
1616        for entry in entries.flatten() {
1617            let path = entry.path();
1618
1619            if entry.file_type().is_ok_and(|file_type| file_type.is_dir()) {
1620                directories.push(path);
1621            } else if path.to_string_lossy().ends_with(&suffix)
1622                && let Err(e) = recover_partial_feather_file(&path)
1623            {
1624                log::warn!("Failed to recover Feather file {}: {e}", path.display());
1625            }
1626        }
1627    }
1628}
1629
1630// Recovers a file its writer released after a failure, so its complete batches stay promotable
1631fn recover_released_feather_file(partial_path: &Path) {
1632    if let Err(e) = recover_partial_feather_file(partial_path) {
1633        log::error!(
1634            "Failed to recover Feather file {}: {e}",
1635            partial_path.display()
1636        );
1637    }
1638}
1639
1640fn recover_partial_feather_file(partial_path: &Path) -> anyhow::Result<()> {
1641    let metadata = match fs::symlink_metadata(partial_path) {
1642        Ok(metadata) => metadata,
1643        Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
1644        Err(e) => return Err(e.into()),
1645    };
1646
1647    anyhow::ensure!(
1648        metadata.is_file(),
1649        "Partial Feather path must be a regular file: {}",
1650        partial_path.display(),
1651    );
1652
1653    let file = match OpenOptions::new().read(true).write(true).open(partial_path) {
1654        Ok(file) => file,
1655        Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
1656        Err(e) => return Err(e.into()),
1657    };
1658
1659    match file.try_lock() {
1660        Ok(()) => {}
1661        Err(TryLockError::WouldBlock) => return Ok(()),
1662        Err(TryLockError::Error(e)) => return Err(e.into()),
1663    }
1664
1665    let len = file.metadata()?.len();
1666
1667    // A writer renames its file before unlocking it, and a new file stays empty until its writer
1668    // locks it, so recovery leaves a missing or empty file alone
1669    if !partial_path.try_exists()? || len == 0 {
1670        return Ok(());
1671    }
1672
1673    let mut batches = 0;
1674    let mut complete_len = 0;
1675    let mut tail_error = None;
1676
1677    match StreamReader::try_new(BufReader::new(&file), None) {
1678        Ok(mut reader) => loop {
1679            match reader.next() {
1680                Some(Ok(_)) => {
1681                    batches += 1;
1682                    complete_len = reader.get_mut().stream_position()?;
1683                }
1684                Some(Err(e)) => {
1685                    tail_error = Some(e);
1686                    break;
1687                }
1688                None => break,
1689            }
1690        },
1691        Err(e) => tail_error = Some(e),
1692    }
1693
1694    // Only a write cut short ends the stream inside a message; any other read error could hide
1695    // complete batches after it, so the file waits for a later pass
1696    if let Some(e) = tail_error.take_if(|e| {
1697        !matches!(e, ArrowError::IoError(_, source) if source.kind() == io::ErrorKind::UnexpectedEof)
1698    }) {
1699        return Err(e.into());
1700    }
1701
1702    let cause = tail_error.map_or_else(String::new, |e| format!(": {e}"));
1703
1704    if batches == 0 {
1705        fs::remove_file(partial_path)?;
1706
1707        log::warn!(
1708            "Removed Feather file {} with no complete record batch{cause}",
1709            partial_path.display()
1710        );
1711        return Ok(());
1712    }
1713
1714    let dropped = len.saturating_sub(complete_len);
1715
1716    // Arrow C++ rejects trailing bytes that form no message, and EOF at a message boundary ends a
1717    // valid stream, so the file ends at its last complete batch
1718    if dropped > 0 {
1719        file.set_len(complete_len)?;
1720    }
1721
1722    file.sync_all()?;
1723
1724    let sealed_path = partial_path.with_extension("");
1725    fs::rename(partial_path, &sealed_path)?;
1726
1727    let tail = if dropped == 0 {
1728        String::new()
1729    } else {
1730        format!(", dropped {dropped} trailing byte(s){cause}")
1731    };
1732
1733    log::warn!(
1734        "Recovered {batches} record batch(es) into Feather file {}{tail}",
1735        sealed_path.display()
1736    );
1737    Ok(())
1738}
1739
1740#[cfg(test)]
1741mod tests {
1742    use std::sync::{Arc, Mutex, atomic::Ordering};
1743
1744    use nautilus_common::{clock::VirtualClock, live::LiveClock};
1745    #[cfg(target_os = "linux")]
1746    use nautilus_model::data::HasTsInit;
1747    use nautilus_model::{
1748        data::{Data, NautilusRecordType, QuoteTick, TradeTick},
1749        enums::AggressorSide,
1750        identifiers::{InstrumentId, TradeId},
1751        types::{ERROR_PRICE, Price, Quantity},
1752    };
1753    use nautilus_serialization::arrow::{
1754        ArrowSchemaProvider, DecodeDataFromRecordBatch, EncodeToRecordBatch,
1755    };
1756    use rstest::rstest;
1757    use tempfile::TempDir;
1758
1759    use super::*;
1760    use crate::{
1761        common::datafusion::identifiers_from_record_batches,
1762        writer::materializer::restore_staged_record_batches,
1763    };
1764
1765    fn feather_files(directory: &Path, extension: &str) -> Vec<PathBuf> {
1766        let mut files = Vec::new();
1767        let mut stack = vec![directory.to_path_buf()];
1768
1769        while let Some(path) = stack.pop() {
1770            for entry in fs::read_dir(&path).unwrap().flatten() {
1771                let path = entry.path();
1772                if path.is_dir() {
1773                    stack.push(path);
1774                } else if path.to_string_lossy().ends_with(&format!(".{extension}")) {
1775                    files.push(path);
1776                }
1777            }
1778        }
1779
1780        files.sort();
1781        files
1782    }
1783
1784    // Reads a type's file and splits it into batches that each carry their rows' metadata
1785    fn read_feather_batches(path: &Path) -> Vec<RecordBatch> {
1786        StreamReader::try_new(File::open(path).unwrap(), None)
1787            .unwrap()
1788            .map(Result::unwrap)
1789            .flat_map(|batch| restore_staged_record_batches(batch).unwrap())
1790            .collect()
1791    }
1792
1793    fn read_quotes(path: &Path) -> Vec<Data> {
1794        read_feather_batches(path)
1795            .into_iter()
1796            .flat_map(|batch| {
1797                let metadata = batch.schema().metadata().clone();
1798                QuoteTick::decode_data_batch(&metadata, batch).unwrap()
1799            })
1800            .collect()
1801    }
1802
1803    fn quote_at(ts: u64) -> QuoteTick {
1804        QuoteTick::new(
1805            InstrumentId::from("AUD/USD.SIM"),
1806            Price::from("1.0"),
1807            Price::from("1.1"),
1808            Quantity::from("1000"),
1809            Quantity::from("1000"),
1810            UnixNanos::from(ts),
1811            UnixNanos::from(ts),
1812        )
1813    }
1814
1815    fn quotes_at(timestamps: &[u64]) -> Vec<Data> {
1816        timestamps
1817            .iter()
1818            .map(|&ts| Data::from(quote_at(ts)))
1819            .collect()
1820    }
1821
1822    // Seals one flushed batch per quote, then reopens the stream as a partial file whose writer
1823    // exited before sealing it
1824    fn write_partial_file(directory: &Path, timestamps: &[u64]) -> PathBuf {
1825        let mut writer = FeatherWriter::new(
1826            directory.to_path_buf(),
1827            WriterClock::Test(Arc::new(AtomicU64::new(0))),
1828            RotationConfig::NoRotation,
1829            None,
1830            Some(0),
1831        );
1832
1833        for &ts in timestamps {
1834            writer.write(quote_at(ts)).unwrap();
1835            writer.flush().unwrap();
1836        }
1837
1838        writer.close().unwrap();
1839
1840        let sealed = feather_files(directory, FEATHER_EXTENSION).remove(0);
1841        let partial = sealed.with_extension(FEATHER_PARTIAL_EXTENSION);
1842        fs::rename(&sealed, &partial).unwrap();
1843        partial
1844    }
1845
1846    const TORN_TAIL: &str = ": Io error: failed to fill whole buffer";
1847
1848    static RECOVERY_LOGS: RecoveryLogCapture = RecoveryLogCapture(Mutex::new(Vec::new()));
1849
1850    // The logger is process-global, so only warnings naming `directory` belong to the caller
1851    fn recover_with_warnings(directory: &Path) -> Vec<String> {
1852        let _ = log::set_logger(&RECOVERY_LOGS);
1853        log::set_max_level(log::LevelFilter::Warn);
1854
1855        recover_partial_feather_files(directory);
1856
1857        let directory = directory.display().to_string();
1858        RECOVERY_LOGS
1859            .0
1860            .lock()
1861            .unwrap()
1862            .iter()
1863            .filter(|(level, message)| *level == log::Level::Warn && message.contains(&directory))
1864            .map(|(_, message)| message.clone())
1865            .collect()
1866    }
1867
1868    struct RecoveryLogCapture(Mutex<Vec<(log::Level, String)>>);
1869
1870    impl log::Log for RecoveryLogCapture {
1871        fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
1872            metadata.target() == "nautilus_persistence::writer::feather"
1873        }
1874
1875        fn log(&self, record: &log::Record<'_>) {
1876            if self.enabled(record.metadata()) {
1877                self.0
1878                    .lock()
1879                    .unwrap()
1880                    .push((record.level(), record.args().to_string()));
1881            }
1882        }
1883
1884        fn flush(&self) {}
1885    }
1886
1887    #[rstest]
1888    fn test_subscription_receives_typed_quotes_and_unsubscribes() {
1889        use nautilus_common::msgbus::{MStr, publish_quote};
1890
1891        let temp_dir = TempDir::new().unwrap();
1892
1893        let writer = Rc::new(RefCell::new(FeatherWriter::new(
1894            temp_dir.path().to_path_buf(),
1895            WriterClock::Test(Arc::new(AtomicU64::new(0))),
1896            RotationConfig::NoRotation,
1897            None,
1898            Some(0),
1899        )));
1900
1901        let quote = QuoteTick::new(
1902            InstrumentId::from("AUD/USD.SIM"),
1903            Price::from("1.0"),
1904            Price::from("1.1"),
1905            Quantity::from("2"),
1906            Quantity::from("3"),
1907            4.into(),
1908            5.into(),
1909        );
1910        let handler = FeatherWriter::subscribe_to_message_bus(writer.clone()).unwrap();
1911        publish_quote(MStr::topic("data.quotes.AUD/USD.SIM").unwrap(), &quote);
1912        FeatherWriter::unsubscribe_from_message_bus(&handler);
1913        publish_quote(MStr::topic("data.quotes.AUD/USD.SIM").unwrap(), &quote);
1914        writer.borrow_mut().close().unwrap();
1915
1916        let rows = feather_files(temp_dir.path(), FEATHER_EXTENSION)
1917            .iter()
1918            .flat_map(|path| read_feather_batches(path))
1919            .map(|batch| batch.num_rows())
1920            .sum::<usize>();
1921        assert_eq!(rows, 1);
1922    }
1923
1924    #[rstest]
1925    fn test_subscription_records_greeks_once_and_unsubscribes() {
1926        use nautilus_common::msgbus::{MStr, publish_any, publish_greeks};
1927        use nautilus_serialization::arrow::{DecodeDataFromRecordBatch, custom::CustomDataDecoder};
1928        let temp_dir = TempDir::new().unwrap();
1929
1930        let writer = Rc::new(RefCell::new(FeatherWriter::new(
1931            temp_dir.path().to_path_buf(),
1932            WriterClock::Test(Arc::new(AtomicU64::new(0))),
1933            RotationConfig::NoRotation,
1934            None,
1935            Some(0),
1936        )));
1937        let greeks = GreeksData::from_delta("AUD/USD.SIM".parse().unwrap(), 1.25, 8.0, 11.into());
1938        let portfolio = greeks.to_portfolio_greeks();
1939        let greeks_topic = MStr::topic("data.GreeksData.instrument_id=AUD/USD.SIM").unwrap();
1940        let portfolio_topic = MStr::topic("data.PortfolioGreeks").unwrap();
1941        let handler = FeatherWriter::subscribe_to_message_bus(writer.clone()).unwrap();
1942        publish_greeks(greeks_topic, &greeks);
1943        publish_any(portfolio_topic, &portfolio);
1944        FeatherWriter::unsubscribe_from_message_bus(&handler);
1945        publish_greeks(greeks_topic, &greeks);
1946        publish_any(portfolio_topic, &portfolio);
1947        writer.borrow_mut().close().unwrap();
1948        let mut counts = HashMap::new();
1949
1950        for batch in feather_files(&temp_dir.path().join("data/custom"), FEATHER_EXTENSION)
1951            .iter()
1952            .flat_map(|path| read_feather_batches(path))
1953        {
1954            let schema = batch.schema();
1955            let name = schema.metadata()["type_name"].clone();
1956            *counts.entry(name.clone()).or_insert(0) += batch.num_rows();
1957            let decoded = CustomDataDecoder::decode_data_batch(schema.metadata(), batch).unwrap();
1958            assert_eq!(decoded.len(), 1);
1959
1960            let Data::Custom(custom) = &decoded[0] else {
1961                panic!("expected custom data")
1962            };
1963
1964            match name.as_str() {
1965                "GreeksData" => {
1966                    assert_eq!(
1967                        custom.data_type,
1968                        CustomDataType::new(
1969                            "GreeksData",
1970                            None,
1971                            Some(greeks.instrument_id.to_string())
1972                        )
1973                    );
1974                    assert_eq!(
1975                        custom.data.as_any().downcast_ref::<GreeksData>(),
1976                        Some(&greeks)
1977                    );
1978                }
1979                "PortfolioGreeks" => {
1980                    assert_eq!(
1981                        custom.data_type,
1982                        CustomDataType::new("PortfolioGreeks", None, None)
1983                    );
1984                    assert_eq!(
1985                        custom.data.as_any().downcast_ref::<PortfolioGreeks>(),
1986                        Some(&portfolio)
1987                    );
1988                }
1989                _ => panic!("unexpected custom data type {name}"),
1990            }
1991        }
1992
1993        assert_eq!(
1994            counts,
1995            HashMap::from([
1996                ("GreeksData".to_string(), 1),
1997                ("PortfolioGreeks".to_string(), 1),
1998            ])
1999        );
2000    }
2001
2002    #[rstest]
2003    #[case(false)]
2004    #[case(true)]
2005    fn test_message_write_error_reaches_flush_or_close(#[case] close: bool) {
2006        let temp_dir = TempDir::new().unwrap();
2007
2008        let mut writer = FeatherWriter::new(
2009            temp_dir.path().to_path_buf(),
2010            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2011            RotationConfig::NoRotation,
2012            None,
2013            Some(0),
2014        );
2015
2016        let quote = QuoteTick::new(
2017            InstrumentId::from("AUD/USD.SIM"),
2018            ERROR_PRICE,
2019            ERROR_PRICE,
2020            Quantity::from("1"),
2021            Quantity::from("2"),
2022            3.into(),
2023            4.into(),
2024        );
2025        let write_error = writer.write_any_message(&quote).unwrap_err().to_string();
2026
2027        let error = if close {
2028            StreamingSink::close(&mut writer)
2029        } else {
2030            StreamingSink::flush(&mut writer)
2031        }
2032        .unwrap_err();
2033
2034        assert_eq!(error.to_string(), write_error);
2035    }
2036
2037    #[rstest]
2038    fn test_writer_manager_keys() {
2039        let temp_dir = TempDir::new().unwrap();
2040
2041        // Create a test time source
2042        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2043        let timestamp = clock.timestamp_ns();
2044
2045        let mut manager = FeatherWriter::new(
2046            temp_dir.path().to_path_buf(),
2047            clock,
2048            RotationConfig::NoRotation,
2049            None,
2050            None, // flush_interval_ms
2051        );
2052
2053        let instrument_id = "AAPL.AAPL";
2054
2055        // Write a dummy value
2056        let quote = QuoteTick::new(
2057            InstrumentId::from(instrument_id),
2058            Price::from("100.0"),
2059            Price::from("100.0"),
2060            Quantity::from("100.0"),
2061            Quantity::from("100.0"),
2062            UnixNanos::from(1_000_000_000_000_000_000),
2063            UnixNanos::from(1_000_000_000_000_000_000),
2064        );
2065
2066        let trade = TradeTick::new(
2067            InstrumentId::from(instrument_id),
2068            Price::from("100.0"),
2069            Quantity::from("100.0"),
2070            AggressorSide::Buy,
2071            TradeId::from("1"),
2072            UnixNanos::from(1_000_000_000_000_000_000),
2073            UnixNanos::from(1_000_000_000_000_000_000),
2074        );
2075
2076        manager.write(quote).unwrap();
2077        manager.write(trade).unwrap();
2078
2079        // Check keys and paths for quotes and trades
2080        let path = manager.get_writer_path(QuoteTick::catalog_family());
2081        let expected_path = temp_dir
2082            .path()
2083            .join("quotes")
2084            .join(format!("quotes_{timestamp}.feather"));
2085        assert_eq!(path.path, expected_path);
2086        assert!(manager.writers.contains_key(&path));
2087        let writer = manager.writers.get(&path).unwrap();
2088        assert!(writer.size() > 0);
2089
2090        let path = manager.get_writer_path(TradeTick::catalog_family());
2091        let expected_path = temp_dir
2092            .path()
2093            .join("trades")
2094            .join(format!("trades_{timestamp}.feather"));
2095        assert_eq!(path.path, expected_path);
2096        assert!(manager.writers.contains_key(&path));
2097        let writer = manager.writers.get(&path).unwrap();
2098        assert!(writer.size() > 0);
2099    }
2100
2101    #[rstest]
2102    #[case::quotes(NautilusDataType::QuoteTick.into(), "quotes/quotes_0.feather")]
2103    #[case::record(NautilusRecordType::OrderFilled.into(), "order_filled/order_filled_0.feather")]
2104    #[case::instrument_class(
2105        CatalogDataType::Instrument(NautilusInstrumentType::CurrencyPair),
2106        "instruments/currency_pair/instruments_0.feather"
2107    )]
2108    #[case::custom(
2109        NautilusDataType::Custom { type_name: "Signal".to_string() }.into(),
2110        "data/custom/Signal/Signal_0.feather"
2111    )]
2112    fn writer_paths_hold_one_file_per_type(
2113        #[case] data_type: CatalogDataType,
2114        #[case] expected: &str,
2115    ) {
2116        let temp_dir = TempDir::new().unwrap();
2117
2118        let manager = FeatherWriter::new(
2119            temp_dir.path().to_path_buf(),
2120            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2121            RotationConfig::NoRotation,
2122            None,
2123            None,
2124        );
2125
2126        let path = manager.build_writer_path(&data_type, UnixNanos::default(), 0);
2127
2128        assert_eq!(path, temp_dir.path().join(expected));
2129    }
2130
2131    #[rstest]
2132    fn existing_feather_writer_implements_streaming_data_sink() {
2133        let temp_dir = TempDir::new().unwrap();
2134        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2135
2136        let mut writer = FeatherWriter::new(
2137            temp_dir.path().to_path_buf(),
2138            clock,
2139            RotationConfig::NoRotation,
2140            None,
2141            None,
2142        );
2143
2144        let quote = QuoteTick::new(
2145            InstrumentId::from("AUD/USD.SIM"),
2146            Price::from("1.0"),
2147            Price::from("1.1"),
2148            Quantity::from("1000"),
2149            Quantity::from("1000"),
2150            UnixNanos::from(1_000),
2151            UnixNanos::from(1_000),
2152        );
2153
2154        StreamingSink::write_data(&mut writer, Data::Quote(quote)).unwrap();
2155        StreamingSink::flush(&mut writer).unwrap();
2156        StreamingSink::close(&mut writer).unwrap();
2157
2158        let files = feather_files(&temp_dir.path().join("quotes"), FEATHER_EXTENSION);
2159        assert_eq!(files.len(), 1);
2160        let identifiers =
2161            identifiers_from_record_batches(&read_feather_batches(&files[0])).unwrap();
2162        assert_eq!(identifiers, vec!["AUD/USD.SIM".to_string()]);
2163    }
2164
2165    #[rstest]
2166    fn scheduled_rotation_keeps_time_of_day_anchor_after_late_rotation() {
2167        let temp_dir = TempDir::new().unwrap();
2168
2169        let now = Arc::new(AtomicU64::new(
2170            1_767_258_000_000_000_000, // 2026-01-01 09:00:00 UTC
2171        ));
2172
2173        let path = FileWriterPath {
2174            path: PathBuf::from("quotes.feather"),
2175            data_type: NautilusDataType::QuoteTick.into(),
2176        };
2177
2178        let mut writer = FeatherWriter::new(
2179            temp_dir.path().to_path_buf(),
2180            WriterClock::Test(Arc::clone(&now)),
2181            RotationConfig::ScheduledDates {
2182                interval_ns: 86_400_000_000_000,
2183                rotation_time: UnixNanos::from(36_000_000_000_000u64),
2184                rotation_timezone: jiff::tz::TimeZone::UTC,
2185            },
2186            None,
2187            None,
2188        );
2189
2190        assert!(!writer.check_scheduled_rotation(&path));
2191        assert_eq!(
2192            writer.next_rotation_times[&path],
2193            UnixNanos::from(1_767_261_600_000_000_000u64),
2194        );
2195
2196        now.store(1_767_263_400_000_000_000, Ordering::Relaxed); // 10:30 UTC
2197        assert!(writer.check_scheduled_rotation(&path));
2198        assert_eq!(
2199            writer.next_rotation_times[&path],
2200            UnixNanos::from(1_767_348_000_000_000_000u64),
2201        );
2202    }
2203
2204    #[rstest]
2205    #[case::modify(true)]
2206    #[case::cancel(false)]
2207    fn test_command_denied_dynamic_round_trip(
2208        #[case] modify: bool,
2209        #[values(false, true)] envelope: bool,
2210    ) {
2211        use nautilus_core::UUID4;
2212        use nautilus_model::{
2213            events::order::spec::{OrderCancelDeniedSpec, OrderModifyDeniedSpec},
2214            identifiers::{AccountId, ClientOrderId, StrategyId, TraderId, VenueOrderId},
2215        };
2216        use nautilus_serialization::arrow::DecodeTypedFromRecordBatch;
2217        let cause = UUID4::new();
2218        let trader = TraderId::from("WRITER-017");
2219        let strategy = StrategyId::from("S-DENIAL-019");
2220        let instrument = InstrumentId::from("ETHUSDT-PERP.BINANCE");
2221        let client = ClientOrderId::from("O-DENIAL-023");
2222        let venue = VenueOrderId::from("V-DENIAL-029");
2223        let account = AccountId::from("BINANCE-031");
2224
2225        let event = if modify {
2226            OrderEventAny::ModifyDenied(
2227                OrderModifyDeniedSpec::builder()
2228                    .trader_id(trader)
2229                    .strategy_id(strategy)
2230                    .instrument_id(instrument)
2231                    .client_order_id(client)
2232                    .venue_order_id(venue)
2233                    .account_id(account)
2234                    .reason("not sent to modify".into())
2235                    .ts_event(37.into())
2236                    .ts_init(41.into())
2237                    .causation_id(cause)
2238                    .build(),
2239            )
2240        } else {
2241            OrderEventAny::CancelDenied(
2242                OrderCancelDeniedSpec::builder()
2243                    .trader_id(trader)
2244                    .strategy_id(strategy)
2245                    .instrument_id(instrument)
2246                    .client_order_id(client)
2247                    .venue_order_id(venue)
2248                    .account_id(account)
2249                    .reason("not sent to cancel".into())
2250                    .ts_event(43.into())
2251                    .ts_init(47.into())
2252                    .causation_id(cause)
2253                    .build(),
2254            )
2255        };
2256
2257        let temp = TempDir::new().unwrap();
2258        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2259
2260        let mut writer = FeatherWriter::new(
2261            temp.path().to_path_buf(),
2262            clock,
2263            RotationConfig::NoRotation,
2264            None,
2265            None,
2266        );
2267
2268        let written = if envelope {
2269            writer.write_any_message(&event).unwrap()
2270        } else {
2271            match &event {
2272                OrderEventAny::ModifyDenied(event) => writer.write_any_message(event).unwrap(),
2273                OrderEventAny::CancelDenied(event) => writer.write_any_message(event).unwrap(),
2274                _ => unreachable!(),
2275            }
2276        };
2277
2278        assert!(written);
2279        writer.close().unwrap();
2280        let files = feather_files(temp.path(), "feather");
2281        assert_eq!(files.len(), 1);
2282        let mut recovered = Vec::new();
2283
2284        for batch in read_feather_batches(&files[0]) {
2285            let metadata = batch.schema().metadata().clone();
2286            assert_eq!(
2287                metadata.get("type_name").unwrap(),
2288                if modify {
2289                    "OrderModifyDenied"
2290                } else {
2291                    "OrderCancelDenied"
2292                }
2293            );
2294
2295            if modify {
2296                recovered.extend(
2297                    OrderModifyDenied::decode_typed_batch(&metadata, batch)
2298                        .unwrap()
2299                        .into_iter()
2300                        .map(OrderEventAny::ModifyDenied),
2301                );
2302            } else {
2303                recovered.extend(
2304                    OrderCancelDenied::decode_typed_batch(&metadata, batch)
2305                        .unwrap()
2306                        .into_iter()
2307                        .map(OrderEventAny::CancelDenied),
2308                );
2309            }
2310        }
2311
2312        assert_eq!(recovered, vec![event]);
2313    }
2314
2315    #[rstest]
2316    fn test_file_writer_round_trip() {
2317        let instrument_id = "AAPL.AAPL";
2318
2319        // Write a dummy value.
2320        let quote = QuoteTick::new(
2321            InstrumentId::from(instrument_id),
2322            Price::from("100.0"),
2323            Price::from("100.0"),
2324            Quantity::from("100.0"),
2325            Quantity::from("100.0"),
2326            UnixNanos::from(100),
2327            UnixNanos::from(100),
2328        );
2329        let metadata = QuoteTick::metadata(&quote);
2330        let schema = QuoteTick::get_schema(Some(metadata.clone()));
2331        let batch = QuoteTick::encode_batch(&QuoteTick::metadata(&quote), &[quote]).unwrap();
2332
2333        let temp_dir = TempDir::new().unwrap();
2334        let path = temp_dir.path().join("quotes.feather");
2335        let mut file = FeatherFile::create(&path, &schema, &RotationConfig::NoRotation).unwrap();
2336        file.write_record_batch(&batch).unwrap();
2337        file.seal().unwrap();
2338
2339        assert!(!path.with_extension(FEATHER_PARTIAL_EXTENSION).exists());
2340        let mut reader = StreamReader::try_new(File::open(&path).unwrap(), None).unwrap();
2341
2342        let read_metadata = reader.schema().metadata().clone();
2343        let mut expected_metadata = metadata.clone();
2344        expected_metadata.insert("type_name".to_string(), "QuoteTick".to_string());
2345        assert_eq!(read_metadata, expected_metadata);
2346
2347        let read_batch = reader.next().unwrap().unwrap();
2348        assert_eq!(read_batch.column(0), batch.column(0));
2349
2350        let decoded = QuoteTick::decode_data_batch(&metadata, batch).unwrap();
2351        assert_eq!(decoded[0], Data::from(quote));
2352    }
2353
2354    #[rstest]
2355    fn test_round_trip() {
2356        // Create a temporary directory for base path
2357        let temp_dir = TempDir::new_in(".").unwrap();
2358
2359        // Create a test time source
2360        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2361
2362        let mut manager = FeatherWriter::new(
2363            temp_dir.path().to_path_buf(),
2364            clock,
2365            RotationConfig::NoRotation,
2366            None,
2367            None, // flush_interval_ms
2368        );
2369
2370        let instrument_id = "AAPL.AAPL";
2371
2372        // Write a dummy value.
2373        let quote = QuoteTick::new(
2374            InstrumentId::from(instrument_id),
2375            Price::from("100.0"),
2376            Price::from("100.0"),
2377            Quantity::from("100.0"),
2378            Quantity::from("100.0"),
2379            UnixNanos::from(100),
2380            UnixNanos::from(100),
2381        );
2382
2383        let trade = TradeTick::new(
2384            InstrumentId::from(instrument_id),
2385            Price::from("100.0"),
2386            Quantity::from("100.0"),
2387            AggressorSide::Buy,
2388            TradeId::from("1"),
2389            UnixNanos::from(100),
2390            UnixNanos::from(100),
2391        );
2392
2393        manager.write(quote).unwrap();
2394        manager.write(trade).unwrap();
2395
2396        let paths = manager.writers.keys().cloned().collect::<Vec<_>>();
2397        assert_eq!(paths.len(), 2);
2398
2399        manager.close().unwrap();
2400
2401        let mut recovered_quotes = Vec::new();
2402        let mut recovered_trades = Vec::new();
2403
2404        for path in paths {
2405            let path_str = path.path;
2406            for batch in read_feather_batches(&path_str) {
2407                let metadata = batch.schema().metadata().clone();
2408
2409                if path_str.to_str().unwrap().contains("quotes") {
2410                    let decoded = QuoteTick::decode_data_batch(&metadata, batch).unwrap();
2411                    recovered_quotes.extend(decoded);
2412                } else if path_str.to_str().unwrap().contains("trades") {
2413                    let decoded = TradeTick::decode_data_batch(&metadata, batch).unwrap();
2414                    recovered_trades.extend(decoded);
2415                }
2416            }
2417        }
2418
2419        // Assert that the recovered data matches the written data
2420        assert_eq!(recovered_quotes.len(), 1, "Expected one QuoteTick record");
2421        assert_eq!(recovered_trades.len(), 1, "Expected one TradeTick record");
2422
2423        // Check key fields to ensure the data round-tripped correctly
2424        assert_eq!(recovered_quotes[0], Data::from(quote));
2425        assert_eq!(recovered_trades[0], Data::from(trade));
2426    }
2427
2428    #[rstest]
2429    fn test_write_data_enum() {
2430        let temp_dir = TempDir::new().unwrap();
2431        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2432
2433        let mut writer = FeatherWriter::new(
2434            temp_dir.path().to_path_buf(),
2435            clock,
2436            RotationConfig::NoRotation,
2437            None,
2438            None,
2439        );
2440
2441        let quote = QuoteTick::new(
2442            InstrumentId::from("AUD/USD.SIM"),
2443            Price::from("1.0"),
2444            Price::from("1.0"),
2445            Quantity::from("1000"),
2446            Quantity::from("1000"),
2447            UnixNanos::from(1000),
2448            UnixNanos::from(1000),
2449        );
2450
2451        // Test writing via write_data
2452        writer.write_data(Data::Quote(quote)).unwrap();
2453        writer.flush().unwrap();
2454
2455        // Verify file was created
2456        assert!(!writer.writers.is_empty() || temp_dir.path().read_dir().unwrap().count() > 0);
2457    }
2458
2459    #[rstest]
2460    fn test_write_data_all_types() {
2461        let temp_dir = TempDir::new().unwrap();
2462        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2463
2464        let mut writer = FeatherWriter::new(
2465            temp_dir.path().to_path_buf(),
2466            clock,
2467            RotationConfig::NoRotation,
2468            None,
2469            None,
2470        );
2471
2472        let instrument_id = InstrumentId::from("AUD/USD.SIM");
2473
2474        // Test all data types
2475        let quote = QuoteTick::new(
2476            instrument_id,
2477            Price::from("1.0"),
2478            Price::from("1.0"),
2479            Quantity::from("1000"),
2480            Quantity::from("1000"),
2481            UnixNanos::from(1000),
2482            UnixNanos::from(1000),
2483        );
2484        writer.write_data(Data::Quote(quote)).unwrap();
2485
2486        let trade = TradeTick::new(
2487            instrument_id,
2488            Price::from("1.0"),
2489            Quantity::from("1000"),
2490            AggressorSide::Buy,
2491            TradeId::from("1"),
2492            UnixNanos::from(2000),
2493            UnixNanos::from(2000),
2494        );
2495        writer.write_data(Data::Trade(trade)).unwrap();
2496
2497        let delta = OrderBookDelta::clear(
2498            instrument_id,
2499            0,
2500            UnixNanos::from(3000),
2501            UnixNanos::from(3000),
2502        );
2503        writer.write_data(Data::BookDelta(delta)).unwrap();
2504
2505        writer.flush().unwrap();
2506    }
2507
2508    #[rstest]
2509    fn test_auto_flush() {
2510        let temp_dir = TempDir::new().unwrap();
2511        let shared_time = Arc::new(AtomicU64::new(0));
2512        let clock = WriterClock::Test(Arc::clone(&shared_time));
2513
2514        let mut writer = FeatherWriter::new(
2515            temp_dir.path().to_path_buf(),
2516            clock,
2517            RotationConfig::NoRotation,
2518            None,
2519            Some(100), // 100ms flush interval
2520        );
2521
2522        let quote = QuoteTick::new(
2523            InstrumentId::from("AUD/USD.SIM"),
2524            Price::from("1.0"),
2525            Price::from("1.0"),
2526            Quantity::from("1000"),
2527            Quantity::from("1000"),
2528            UnixNanos::from(1000),
2529            UnixNanos::from(1000),
2530        );
2531
2532        // Write first quote; interval not elapsed, so nothing is flushed yet
2533        writer.write(quote).unwrap();
2534        assert_eq!(writer.writers.len(), 1);
2535        assert_eq!(writer.last_flush_ns, UnixNanos::from(0));
2536
2537        // Advance the shared time source past the 100ms flush interval
2538        shared_time.store(200_000_000, Ordering::Relaxed);
2539
2540        // Second write hits the flush boundary: both quotes reach the same open file
2541        let quote2 = QuoteTick::new(
2542            InstrumentId::from("AUD/USD.SIM"),
2543            Price::from("1.1"),
2544            Price::from("1.1"),
2545            Quantity::from("1000"),
2546            Quantity::from("1000"),
2547            UnixNanos::from(2000),
2548            UnixNanos::from(2000),
2549        );
2550        writer.write(quote2).unwrap();
2551
2552        // Windows locks block reads through other handles, so the open file is measured instead:
2553        // if closing adds only the 8-byte end-of-stream marker, both quotes were already on disk
2554        let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
2555        let flushed_len = fs::metadata(&partial[0]).unwrap().len();
2556        let open_writers = writer.writers.len();
2557        let last_flush_ns = writer.last_flush_ns;
2558        let sealed_before_close = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2559        writer.close().unwrap();
2560
2561        let sealed = partial[0].with_extension("");
2562        let sealed_rows = read_feather_batches(&sealed)
2563            .iter()
2564            .map(RecordBatch::num_rows)
2565            .sum::<usize>();
2566        assert_eq!(open_writers, 1);
2567        assert_eq!(last_flush_ns, UnixNanos::from(200_000_000));
2568        assert_eq!(partial.len(), 1);
2569        assert!(sealed_before_close.is_empty());
2570        assert_eq!(fs::metadata(&sealed).unwrap().len(), flushed_len + 8);
2571        assert_eq!(sealed_rows, 2);
2572    }
2573
2574    #[rstest]
2575    fn flushes_append_to_one_file_per_type_until_close() {
2576        let temp_dir = TempDir::new().unwrap();
2577        let shared_time = Arc::new(AtomicU64::new(0));
2578
2579        let mut writer = FeatherWriter::new(
2580            temp_dir.path().to_path_buf(),
2581            WriterClock::Test(Arc::clone(&shared_time)),
2582            RotationConfig::NoRotation,
2583            None,
2584            Some(1_000),
2585        );
2586
2587        let quotes = (1..=10)
2588            .map(|minute| {
2589                QuoteTick::new(
2590                    InstrumentId::from("AUD/USD.SIM"),
2591                    Price::from("1.0"),
2592                    Price::from("1.1"),
2593                    Quantity::from("1000"),
2594                    Quantity::from("1000"),
2595                    UnixNanos::from(minute * 60_000_000_000),
2596                    UnixNanos::from(minute * 60_000_000_000),
2597                )
2598            })
2599            .collect::<Vec<_>>();
2600
2601        // Each write crosses a flush interval, as a 1-minute bar does in a backtest
2602        for quote in &quotes {
2603            shared_time.store(quote.ts_init.as_u64(), Ordering::Relaxed);
2604            writer.write(*quote).unwrap();
2605        }
2606
2607        let partial_before_close = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
2608        let sealed_before_close = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2609        writer.close().unwrap();
2610        let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2611
2612        let recovered = read_feather_batches(&sealed[0])
2613            .into_iter()
2614            .flat_map(|batch| {
2615                let metadata = batch.schema().metadata().clone();
2616                QuoteTick::decode_data_batch(&metadata, batch).unwrap()
2617            })
2618            .collect::<Vec<_>>();
2619
2620        assert_eq!(partial_before_close.len(), 1);
2621        assert!(sealed_before_close.is_empty());
2622        assert_eq!(sealed.len(), 1);
2623        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
2624        assert_eq!(
2625            recovered,
2626            quotes.into_iter().map(Data::from).collect::<Vec<_>>()
2627        );
2628    }
2629
2630    #[rstest]
2631    #[case::below_row_cap(3, vec![3])]
2632    #[case::above_row_cap(
2633        DEFAULT_DATA_BATCH_CHUNK_SIZE + 1,
2634        vec![DEFAULT_DATA_BATCH_CHUNK_SIZE, 1]
2635    )]
2636    fn writes_between_flushes_append_one_batch_per_row_cap(
2637        #[case] count: usize,
2638        #[case] expected_batch_rows: Vec<usize>,
2639    ) {
2640        let temp_dir = TempDir::new().unwrap();
2641
2642        let mut writer = FeatherWriter::new(
2643            temp_dir.path().to_path_buf(),
2644            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2645            RotationConfig::NoRotation,
2646            None,
2647            Some(0),
2648        );
2649        let instrument_ids = [
2650            InstrumentId::from("AUD/USD.SIM"),
2651            InstrumentId::from("EUR/USD.SIM"),
2652        ];
2653
2654        for index in 0..count {
2655            writer
2656                .write(QuoteTick::new(
2657                    instrument_ids[index % 2],
2658                    Price::from("1.0"),
2659                    Price::from("1.1"),
2660                    Quantity::from("1000"),
2661                    Quantity::from("1000"),
2662                    UnixNanos::from(index as u64),
2663                    UnixNanos::from(index as u64),
2664                ))
2665                .unwrap();
2666        }
2667
2668        writer.close().unwrap();
2669        let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
2670        let batch_rows = StreamReader::try_new(File::open(&sealed[0]).unwrap(), None)
2671            .unwrap()
2672            .map(|batch| batch.unwrap().num_rows())
2673            .collect::<Vec<_>>();
2674
2675        assert_eq!(sealed.len(), 1);
2676        assert_eq!(batch_rows, expected_batch_rows);
2677    }
2678
2679    #[rstest]
2680    fn drop_seals_open_files() {
2681        let temp_dir = TempDir::new().unwrap();
2682
2683        let mut writer = FeatherWriter::new(
2684            temp_dir.path().to_path_buf(),
2685            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2686            RotationConfig::NoRotation,
2687            None,
2688            None,
2689        );
2690        writer
2691            .write(QuoteTick::new(
2692                InstrumentId::from("AUD/USD.SIM"),
2693                Price::from("1.0"),
2694                Price::from("1.1"),
2695                Quantity::from("1000"),
2696                Quantity::from("1000"),
2697                UnixNanos::from(1_000),
2698                UnixNanos::from(1_000),
2699            ))
2700            .unwrap();
2701
2702        drop(writer);
2703
2704        assert_eq!(feather_files(temp_dir.path(), FEATHER_EXTENSION).len(), 1);
2705        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
2706    }
2707
2708    #[rstest]
2709    fn reserved_paths_skip_files_left_in_the_directory() {
2710        let temp_dir = TempDir::new().unwrap();
2711
2712        let mut writer = FeatherWriter::new(
2713            temp_dir.path().to_path_buf(),
2714            WriterClock::Test(Arc::new(AtomicU64::new(5))),
2715            RotationConfig::NoRotation,
2716            None,
2717            None,
2718        );
2719        let sealed = temp_dir.path().join("trades").join("trades_5.feather");
2720        let partial = temp_dir
2721            .path()
2722            .join("trades")
2723            .join("trades_5-1.feather.partial");
2724        fs::create_dir_all(sealed.parent().unwrap()).unwrap();
2725        fs::write(&sealed, b"sealed").unwrap();
2726        fs::write(&partial, b"partial").unwrap();
2727
2728        let path = writer.reserve_writer_path(NautilusDataType::TradeTick.into());
2729
2730        assert_eq!(
2731            path.path,
2732            temp_dir.path().join("trades").join("trades_5-2.feather")
2733        );
2734        assert_eq!(fs::read(&sealed).unwrap(), b"sealed");
2735        assert_eq!(fs::read(&partial).unwrap(), b"partial");
2736    }
2737
2738    #[rstest]
2739    #[case::same_time(vec![5; 32])]
2740    #[case::clock_rewinds(vec![5, 3, 5])]
2741    #[case::clock_advances_then_rewinds(vec![5, 7, 5, 7])]
2742    fn path_reservations_are_bounded_after_files_are_deleted(#[case] timestamps: Vec<u64>) {
2743        let temp_dir = TempDir::new().unwrap();
2744        let time = Arc::new(AtomicU64::new(0));
2745        let mut writer = FeatherWriter::new(
2746            temp_dir.path().to_path_buf(),
2747            WriterClock::Test(Arc::clone(&time)),
2748            RotationConfig::NoRotation,
2749            None,
2750            None,
2751        );
2752        let mut paths = HashSet::new();
2753
2754        for timestamp in &timestamps {
2755            time.store(*timestamp, Ordering::Relaxed);
2756            writer.write(quote_at(1)).unwrap();
2757            writer.close().unwrap();
2758            let path = feather_files(temp_dir.path(), FEATHER_EXTENSION).remove(0);
2759            assert!(paths.insert(path.clone()));
2760            fs::remove_file(path).unwrap();
2761        }
2762
2763        assert_eq!(paths.len(), timestamps.len());
2764        assert_eq!(writer.path_sequences.len(), 1);
2765    }
2766
2767    #[rstest]
2768    fn test_close() {
2769        let temp_dir = TempDir::new().unwrap();
2770        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2771
2772        let mut writer = FeatherWriter::new(
2773            temp_dir.path().to_path_buf(),
2774            clock,
2775            RotationConfig::NoRotation,
2776            None,
2777            None,
2778        );
2779
2780        let quote = QuoteTick::new(
2781            InstrumentId::from("AUD/USD.SIM"),
2782            Price::from("1.0"),
2783            Price::from("1.0"),
2784            Quantity::from("1000"),
2785            Quantity::from("1000"),
2786            UnixNanos::from(1000),
2787            UnixNanos::from(1000),
2788        );
2789
2790        writer.write(quote).unwrap();
2791        assert!(!writer.writers.is_empty());
2792
2793        writer.close().unwrap();
2794        assert!(writer.writers.is_empty());
2795    }
2796
2797    #[rstest]
2798    fn test_write_data_orderbook_deltas() {
2799        let temp_dir = TempDir::new().unwrap();
2800        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2801
2802        let mut writer = FeatherWriter::new(
2803            temp_dir.path().to_path_buf(),
2804            clock,
2805            RotationConfig::NoRotation,
2806            None,
2807            None,
2808        );
2809
2810        let instrument_id = InstrumentId::from("AUD/USD.SIM");
2811        let delta1 = OrderBookDelta::clear(
2812            instrument_id,
2813            0,
2814            UnixNanos::from(1000),
2815            UnixNanos::from(1000),
2816        );
2817        let delta2 = OrderBookDelta::clear(
2818            instrument_id,
2819            0,
2820            UnixNanos::from(2000),
2821            UnixNanos::from(2000),
2822        );
2823
2824        let deltas = OrderBookDeltas::new(instrument_id, vec![delta1, delta2]);
2825        // Test writing OrderBookDeltas via write_data
2826        writer
2827            .write_data(Data::BookDeltas(Box::new(deltas)))
2828            .unwrap();
2829        writer.flush().unwrap();
2830    }
2831
2832    #[rstest]
2833    fn feather_writer_is_send() {
2834        fn assert_send<T: Send>() {}
2835        assert_send::<FeatherWriter>();
2836        assert_send::<WriterClock>();
2837    }
2838
2839    #[rstest]
2840    fn writer_clock_test_source_reads_shared_atomic() {
2841        let shared = Arc::new(AtomicU64::new(7));
2842        let clock = WriterClock::Test(Arc::clone(&shared));
2843        assert_eq!(clock.timestamp_ns(), UnixNanos::from(7));
2844
2845        shared.store(42, Ordering::Relaxed);
2846        assert_eq!(clock.timestamp_ns(), UnixNanos::from(42));
2847    }
2848
2849    #[rstest]
2850    fn writer_clock_from_shared_clock_wires_test_clock() {
2851        let test_clock = Rc::new(RefCell::new(VirtualClock::new()));
2852        test_clock
2853            .borrow_mut()
2854            .advance_time(UnixNanos::from(42), true);
2855        let clock: Rc<RefCell<dyn Clock>> = test_clock.clone();
2856
2857        let (writer_clock, shared) = WriterClock::from_shared_clock(&clock);
2858        let shared = shared.expect("non-live clocks must return a shared atomic");
2859
2860        // Seeded with the source clock's current time
2861        assert_eq!(writer_clock.timestamp_ns(), UnixNanos::from(42));
2862
2863        // Bridge refresh: advancing the source clock and storing into the atomic
2864        // is what the PyO3 wrapper does before each forwarded call
2865        test_clock
2866            .borrow_mut()
2867            .advance_time(UnixNanos::from(99), true);
2868        shared.store(clock.borrow().timestamp_ns().as_u64(), Ordering::Relaxed);
2869        assert_eq!(writer_clock.timestamp_ns(), UnixNanos::from(99));
2870    }
2871
2872    #[rstest]
2873    fn writer_clock_from_shared_clock_live_clock_is_live() {
2874        let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(LiveClock::new(None)));
2875
2876        let (writer_clock, shared) = WriterClock::from_shared_clock(&clock);
2877
2878        assert!(matches!(writer_clock, WriterClock::Live));
2879        assert!(shared.is_none());
2880    }
2881
2882    #[rstest]
2883    #[cfg(unix)]
2884    fn failed_seal_of_a_removed_file_reports_the_error_and_publishes_nothing() {
2885        let temp_dir = TempDir::new().unwrap();
2886        let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
2887
2888        let mut writer = FeatherWriter::new(
2889            temp_dir.path().to_path_buf(),
2890            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2891            RotationConfig::NoRotation,
2892            None,
2893            None,
2894        )
2895        .with_sealed_sender(sealed_tx);
2896        writer
2897            .write(QuoteTick::new(
2898                InstrumentId::from("AUD/USD.SIM"),
2899                Price::from("1.0"),
2900                Price::from("1.1"),
2901                Quantity::from("1000"),
2902                Quantity::from("1000"),
2903                UnixNanos::from(1_000),
2904                UnixNanos::from(1_000),
2905            ))
2906            .unwrap();
2907
2908        // Removing the open partial file makes the seal's rename fail after the stream finishes
2909        let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
2910        fs::remove_file(&partial[0]).unwrap();
2911
2912        let error = writer.close().unwrap_err();
2913
2914        assert_eq!(error.to_string(), "No such file or directory (os error 2)");
2915        assert!(writer.is_closed());
2916        assert!(feather_files(temp_dir.path(), FEATHER_EXTENSION).is_empty());
2917        assert_eq!(
2918            sealed_rx.try_iter().collect::<Vec<_>>(),
2919            Vec::<PathBuf>::new()
2920        );
2921    }
2922
2923    #[rstest]
2924    #[cfg(target_os = "linux")]
2925    fn failed_seal_recovers_the_batches_that_reached_the_file() {
2926        let temp_dir = TempDir::new().unwrap();
2927        let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
2928
2929        let mut writer = FeatherWriter::new(
2930            temp_dir.path().to_path_buf(),
2931            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2932            RotationConfig::NoRotation,
2933            None,
2934            Some(0),
2935        )
2936        .with_sealed_sender(sealed_tx);
2937
2938        writer.write(quote_at(1)).unwrap();
2939        writer.flush().unwrap();
2940        writer.write(quote_at(2)).unwrap();
2941
2942        // Route the open stream to a device that reports a full disk, so sealing it fails
2943        let (path, file) = writer.writers.iter_mut().next().unwrap();
2944        let sealed = path.path.clone();
2945        let full = OpenOptions::new().write(true).open("/dev/full").unwrap();
2946        file.writer =
2947            StreamWriter::try_new(CountingWriter::new(BufWriter::new(full)), &file.schema).unwrap();
2948
2949        let error = writer.close().unwrap_err();
2950
2951        assert_eq!(
2952            error.to_string(),
2953            "Io error: No space left on device (os error 28)"
2954        );
2955        assert!(writer.is_closed());
2956        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
2957        assert_eq!(read_quotes(&sealed), quotes_at(&[1]));
2958        assert_eq!(sealed_rx.try_iter().collect::<Vec<_>>(), vec![sealed]);
2959    }
2960
2961    #[rstest]
2962    #[case::nothing_flushed(false, vec![3])]
2963    #[case::batch_flushed(true, vec![1, 3])]
2964    #[cfg(target_os = "linux")]
2965    fn failed_flush_recovers_the_abandoned_file_and_next_write_opens_a_new_one(
2966        #[case] flushed: bool,
2967        #[case] expected: Vec<u64>,
2968    ) {
2969        let temp_dir = TempDir::new().unwrap();
2970        let (sealed_tx, sealed_rx) = std::sync::mpsc::channel();
2971
2972        let mut writer = FeatherWriter::new(
2973            temp_dir.path().to_path_buf(),
2974            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2975            RotationConfig::NoRotation,
2976            None,
2977            Some(0),
2978        )
2979        .with_sealed_sender(sealed_tx);
2980
2981        writer.write(quote_at(1)).unwrap();
2982
2983        if flushed {
2984            writer.flush().unwrap();
2985        }
2986
2987        writer.write(quote_at(2)).unwrap();
2988
2989        // Route the open stream to a device that reports a full disk on every write
2990        let (abandoned, file) = writer.writers.iter_mut().next().unwrap();
2991        let abandoned = abandoned.clone();
2992        let full = OpenOptions::new().write(true).open("/dev/full").unwrap();
2993        file.writer =
2994            StreamWriter::try_new(CountingWriter::new(BufWriter::new(full)), &file.schema).unwrap();
2995
2996        let error = writer.flush().unwrap_err();
2997        writer.write(quote_at(3)).unwrap();
2998        writer.close().unwrap();
2999
3000        let mut recovered = feather_files(temp_dir.path(), FEATHER_EXTENSION)
3001            .iter()
3002            .flat_map(|path| read_quotes(path))
3003            .collect::<Vec<_>>();
3004        recovered.sort_by_key(HasTsInit::ts_init);
3005
3006        // Quote 2 was buffered when the flush failed, and the full device also fails the writer's
3007        // drop-time flush, so it reaches no file
3008        assert_eq!(
3009            error.to_string(),
3010            "Io error: No space left on device (os error 28)"
3011        );
3012        assert_eq!(abandoned.path.exists(), flushed);
3013        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3014        assert_eq!(recovered, quotes_at(&expected));
3015        let mut notified = sealed_rx.try_iter().collect::<Vec<_>>();
3016        notified.sort();
3017        assert_eq!(notified, feather_files(temp_dir.path(), FEATHER_EXTENSION));
3018    }
3019
3020    #[rstest]
3021    #[case::end_of_stream_marker(0, &[], vec![1, 2, 3], "")]
3022    #[case::no_end_of_stream_marker(8, &[], vec![1, 2, 3], "")]
3023    #[case::stray_bytes(8, &[0xFF, 0xFF], vec![1, 2, 3], "")]
3024    #[case::truncated_last_batch(9, &[], vec![1, 2], TORN_TAIL)]
3025    fn recovery_seals_complete_batches_and_drops_the_rest(
3026        #[case] cut: usize,
3027        #[case] stray: &[u8],
3028        #[case] expected: Vec<u64>,
3029        #[case] cause: &str,
3030    ) {
3031        let temp_dir = TempDir::new().unwrap();
3032        let directory = temp_dir.path().join("recover");
3033        let partial = write_partial_file(&directory, &[1, 2, 3]);
3034        let mut staged = fs::read(&partial).unwrap();
3035        staged.truncate(staged.len() - cut);
3036        staged.extend_from_slice(stray);
3037        fs::write(&partial, &staged).unwrap();
3038
3039        // The expected batches alone form this prefix, which ends before the end-of-stream marker
3040        let reference = fs::read(write_partial_file(
3041            &temp_dir.path().join("reference"),
3042            &expected,
3043        ))
3044        .unwrap();
3045        let complete = &reference[..reference.len() - 8];
3046
3047        let warnings = recover_with_warnings(&directory);
3048
3049        let sealed = partial.with_extension("");
3050        let dropped = staged.len() - complete.len();
3051
3052        let tail = if dropped == 0 {
3053            String::new()
3054        } else {
3055            format!(", dropped {dropped} trailing byte(s){cause}")
3056        };
3057
3058        assert!(!partial.exists());
3059        assert_eq!(fs::read(&sealed).unwrap(), complete);
3060        assert_eq!(read_quotes(&sealed), quotes_at(&expected));
3061        assert_eq!(
3062            warnings,
3063            vec![format!(
3064                "Recovered {} record batch(es) into Feather file {}{tail}",
3065                expected.len(),
3066                sealed.display()
3067            )],
3068        );
3069    }
3070
3071    #[rstest]
3072    #[case::schema_only(0, "")]
3073    #[case::truncated_schema(1, TORN_TAIL)]
3074    fn recovery_removes_a_file_without_a_complete_batch(#[case] cut: u64, #[case] cause: &str) {
3075        let temp_dir = TempDir::new().unwrap();
3076        let partial = temp_dir.path().join("quotes_1.feather.partial");
3077        let file = File::create(&partial).unwrap();
3078        StreamWriter::try_new(&file, &QuoteTick::get_schema(None)).unwrap();
3079        file.set_len(file.metadata().unwrap().len() - cut).unwrap();
3080        drop(file);
3081
3082        let warnings = recover_with_warnings(temp_dir.path());
3083
3084        assert_eq!(fs::read_dir(temp_dir.path()).unwrap().count(), 0);
3085        assert_eq!(
3086            warnings,
3087            vec![format!(
3088                "Removed Feather file {} with no complete record batch{cause}",
3089                partial.display()
3090            )],
3091        );
3092    }
3093
3094    #[rstest]
3095    fn recovery_leaves_a_file_with_an_unreadable_tail() {
3096        let temp_dir = TempDir::new().unwrap();
3097        let partial = write_partial_file(temp_dir.path(), &[1, 2]);
3098
3099        // A continuation marker followed by a negative metadata length is corrupt, not cut short
3100        let mut staged = fs::read(&partial).unwrap();
3101        staged.truncate(staged.len() - 8);
3102        staged.extend_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF, 0xF0, 0xFF, 0xFF, 0xFF]);
3103        fs::write(&partial, &staged).unwrap();
3104
3105        let warnings = recover_with_warnings(temp_dir.path());
3106
3107        assert_eq!(fs::read(&partial).unwrap(), staged);
3108        assert_eq!(
3109            warnings,
3110            vec![format!(
3111                "Failed to recover Feather file {}: Parser error: Invalid metadata length: -16",
3112                partial.display()
3113            )],
3114        );
3115    }
3116
3117    #[rstest]
3118    #[cfg(unix)]
3119    fn recovery_leaves_partial_file_symlinks_untouched() {
3120        let temp_dir = TempDir::new().unwrap();
3121        let outside = TempDir::new().unwrap();
3122        let partial = write_partial_file(outside.path(), &[1, 2]);
3123        let original = fs::read(&partial).unwrap();
3124        let link = temp_dir.path().join("quotes_0.feather.partial");
3125        std::os::unix::fs::symlink(&partial, &link).unwrap();
3126
3127        recover_partial_feather_files(temp_dir.path());
3128
3129        assert_eq!(fs::read(&partial).unwrap(), original);
3130        assert_eq!(fs::read_link(&link).unwrap(), partial);
3131        assert!(!link.with_extension("").exists());
3132        assert!(!partial.with_extension("").exists());
3133    }
3134
3135    #[rstest]
3136    #[cfg(unix)]
3137    fn recovery_continues_after_a_file_it_cannot_recover() {
3138        let temp_dir = TempDir::new().unwrap();
3139        let run = temp_dir.path().join("run");
3140        let partial = write_partial_file(&run, &[1]);
3141
3142        let unrecoverable = temp_dir.path().join("bad.feather.partial");
3143        std::os::unix::fs::symlink(&run, &unrecoverable).unwrap();
3144
3145        let warnings = recover_with_warnings(temp_dir.path());
3146
3147        let sealed = partial.with_extension("");
3148        assert_eq!(read_quotes(&sealed), quotes_at(&[1]));
3149        assert_eq!(
3150            warnings,
3151            vec![
3152                format!(
3153                    "Failed to recover Feather file {}: Partial Feather path must be a regular file: {}",
3154                    unrecoverable.display(),
3155                    unrecoverable.display(),
3156                ),
3157                format!(
3158                    "Recovered 1 record batch(es) into Feather file {}, dropped 8 trailing byte(s)",
3159                    sealed.display()
3160                ),
3161            ],
3162        );
3163    }
3164
3165    #[rstest]
3166    fn recovery_leaves_open_and_empty_files() {
3167        let temp_dir = TempDir::new().unwrap();
3168
3169        let mut writer = FeatherWriter::new(
3170            temp_dir.path().join("open"),
3171            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3172            RotationConfig::NoRotation,
3173            None,
3174            Some(0),
3175        );
3176        writer.write(quote_at(1)).unwrap();
3177        writer.flush().unwrap();
3178
3179        // Windows locks block reads through other handles, so the open file is compared by length
3180        let open = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).remove(0);
3181        let open_len = fs::metadata(&open).unwrap().len();
3182        let empty = temp_dir.path().join("empty.feather.partial");
3183        File::create(&empty).unwrap();
3184
3185        recover_partial_feather_files(temp_dir.path());
3186
3187        let partial = feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION);
3188        let open_len_after = fs::metadata(&open).unwrap().len();
3189        writer.close().unwrap();
3190
3191        assert_eq!(partial, vec![empty, open.clone()]);
3192        assert_eq!(open_len_after, open_len);
3193        assert_eq!(read_quotes(&open.with_extension("")), quotes_at(&[1]));
3194    }
3195
3196    #[rstest]
3197    fn size_rotation_in_mixed_identifier_batch_seals_the_shared_file_once() {
3198        let temp_dir = TempDir::new().unwrap();
3199
3200        let mut writer = FeatherWriter::new(
3201            temp_dir.path().to_path_buf(),
3202            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3203            RotationConfig::Size { max_size: 1 },
3204            None,
3205            None,
3206        );
3207
3208        let quotes = ["AUD/USD.SIM", "EUR/USD.SIM"]
3209            .map(|instrument_id| {
3210                QuoteTick::new(
3211                    InstrumentId::from(instrument_id),
3212                    Price::from("1.0"),
3213                    Price::from("1.1"),
3214                    Quantity::from("1000"),
3215                    Quantity::from("1000"),
3216                    UnixNanos::from(1),
3217                    UnixNanos::from(1),
3218                )
3219            })
3220            .to_vec();
3221
3222        // Both identifier groups fill the shared quotes file, so the batch queues its seal twice
3223        writer.write_batch(quotes).unwrap();
3224
3225        let sealed = feather_files(temp_dir.path(), FEATHER_EXTENSION);
3226        let batches = read_feather_batches(&sealed[0]);
3227
3228        assert!(writer.writers.is_empty());
3229        assert_eq!(sealed.len(), 1);
3230        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3231        assert_eq!(
3232            identifiers_from_record_batches(&batches).unwrap(),
3233            vec!["AUD/USD.SIM".to_string(), "EUR/USD.SIM".to_string()],
3234        );
3235    }
3236
3237    #[rstest]
3238    fn size_rotation_seals_each_full_file_and_opens_the_next_lazily() {
3239        let temp_dir = TempDir::new().unwrap();
3240        let shared_clock = Arc::new(AtomicU64::new(0));
3241
3242        let mut writer = FeatherWriter::new(
3243            temp_dir.path().to_path_buf(),
3244            WriterClock::Test(Arc::clone(&shared_clock)),
3245            RotationConfig::Size { max_size: 1 },
3246            None,
3247            Some(1),
3248        );
3249
3250        for ts in [1_000, 2_000] {
3251            shared_clock.store(ts * 1_000_000, Ordering::Relaxed);
3252            writer
3253                .write(QuoteTick::new(
3254                    InstrumentId::from("AUD/USD.SIM"),
3255                    Price::from("1.0"),
3256                    Price::from("1.0"),
3257                    Quantity::from("1000"),
3258                    Quantity::from("1000"),
3259                    UnixNanos::from(ts),
3260                    UnixNanos::from(ts),
3261                ))
3262                .unwrap();
3263        }
3264
3265        let rows = feather_files(temp_dir.path(), FEATHER_EXTENSION)
3266            .iter()
3267            .map(|path| {
3268                read_feather_batches(path)
3269                    .iter()
3270                    .map(RecordBatch::num_rows)
3271                    .sum::<usize>()
3272            })
3273            .collect::<Vec<_>>();
3274
3275        assert!(writer.writers.is_empty());
3276        assert_eq!(rows, vec![1, 1]);
3277        assert!(feather_files(temp_dir.path(), FEATHER_PARTIAL_EXTENSION).is_empty());
3278    }
3279
3280    #[rstest]
3281    #[case(
3282        "FeatherMissingTimestamp",
3283        Schema::empty(),
3284        "registered without an Arrow schema containing ts_init"
3285    )]
3286    #[case(
3287        "FeatherLegacyTimestamp",
3288        Schema::new(vec![Field::new("ts_init", DataType::UInt64, false)]),
3289        "registered with ts_init as UInt64",
3290    )]
3291    fn test_custom_writer_path_rejects_unqueryable_schema(
3292        #[case] type_name: &str,
3293        #[case] schema: Schema,
3294        #[case] expected_error: &str,
3295    ) {
3296        nautilus_model::data::registry::ensure_arrow_registered(
3297            type_name,
3298            Arc::new(schema),
3299            Box::new(|_| unreachable!("writer creation does not encode data")),
3300            Box::new(|_, _| unreachable!("writer creation does not decode data")),
3301        )
3302        .unwrap();
3303
3304        let temp_dir = TempDir::new().unwrap();
3305
3306        let mut writer = FeatherWriter::new(
3307            temp_dir.path().to_path_buf(),
3308            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3309            RotationConfig::NoRotation,
3310            None,
3311            None,
3312        );
3313
3314        for _ in 0..2 {
3315            let error = writer.get_writer_path_custom(type_name).unwrap_err();
3316
3317            assert!(error.to_string().contains(expected_error));
3318            assert!(writer.writers.is_empty());
3319            assert!(writer.path_sequences.is_empty());
3320        }
3321    }
3322
3323    #[rstest]
3324    #[cfg(feature = "python")]
3325    fn test_write_custom_data_round_trip() {
3326        use std::sync::Arc;
3327
3328        use nautilus_model::{
3329            data::{CustomData, Data, DataType},
3330            identifiers::InstrumentId,
3331        };
3332        use nautilus_serialization::{
3333            arrow::custom::CustomDataDecoder, ensure_custom_data_registered,
3334        };
3335
3336        use crate::test_data::RustTestCustomData;
3337
3338        ensure_custom_data_registered::<RustTestCustomData>();
3339
3340        let temp_dir = TempDir::new().unwrap();
3341        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
3342
3343        let mut writer = FeatherWriter::new(
3344            temp_dir.path().to_path_buf(),
3345            clock,
3346            RotationConfig::NoRotation,
3347            None,
3348            None,
3349        );
3350
3351        let instrument_id = InstrumentId::from("RUST.TEST");
3352        let data_type = DataType::new("RustTestCustomData", None, Some(instrument_id.to_string()));
3353
3354        let original = RustTestCustomData {
3355            instrument_id,
3356            value: 1.23,
3357            flag: true,
3358            ts_event: UnixNanos::from(1000),
3359            ts_init: UnixNanos::from(1000),
3360        };
3361
3362        let custom = CustomData::new(Arc::new(original.clone()), data_type);
3363
3364        writer
3365            .write_data(Data::Custom(custom))
3366            .expect("write_data CustomData");
3367        writer.close().expect("close");
3368
3369        let prefix = temp_dir
3370            .path()
3371            .join("data")
3372            .join("custom")
3373            .join("RustTestCustomData");
3374        let files = feather_files(&prefix, FEATHER_EXTENSION);
3375        let batch = read_feather_batches(&files[0]).remove(0);
3376        let metadata = batch.schema().metadata().clone();
3377        let decoded =
3378            CustomDataDecoder::decode_data_batch(&metadata, batch).expect("decode_data_batch");
3379        assert_eq!(decoded.len(), 1);
3380
3381        if let Data::Custom(decoded_custom) = &decoded[0] {
3382            assert_eq!(decoded_custom.data_type.type_name(), "RustTestCustomData");
3383            let rust: &RustTestCustomData = decoded_custom
3384                .data
3385                .as_any()
3386                .downcast_ref::<RustTestCustomData>()
3387                .expect("RustTestCustomData");
3388            assert_eq!(rust, &original);
3389        } else {
3390            panic!("Expected Data::Custom");
3391        }
3392    }
3393
3394    #[rstest]
3395    #[cfg(feature = "python")]
3396    fn test_write_custom_data_reuses_writer_until_rotation() {
3397        use nautilus_model::data::{CustomData, DataType};
3398        use nautilus_serialization::ensure_custom_data_registered;
3399
3400        use crate::test_data::RustTestCustomData;
3401
3402        ensure_custom_data_registered::<RustTestCustomData>();
3403        let temp_dir = TempDir::new().unwrap();
3404
3405        let mut writer = FeatherWriter::new(
3406            temp_dir.path().to_path_buf(),
3407            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3408            RotationConfig::NoRotation,
3409            None,
3410            None,
3411        );
3412        let instrument_id = InstrumentId::from("RUST.TEST");
3413        let data_type = DataType::new("RustTestCustomData", None, Some(instrument_id.to_string()));
3414
3415        for (ts, value) in [(1_000, 1_000.0), (2_000, 2_000.0)] {
3416            writer
3417                .write_data(Data::Custom(CustomData::new(
3418                    Arc::new(RustTestCustomData {
3419                        instrument_id,
3420                        value,
3421                        flag: true,
3422                        ts_event: UnixNanos::from(ts),
3423                        ts_init: UnixNanos::from(ts),
3424                    }),
3425                    data_type.clone(),
3426                )))
3427                .unwrap();
3428        }
3429
3430        assert_eq!(writer.get_current_file_info().len(), 1);
3431        writer.close().unwrap();
3432
3433        let prefix = temp_dir
3434            .path()
3435            .join("data")
3436            .join("custom")
3437            .join("RustTestCustomData");
3438        let files = feather_files(&prefix, FEATHER_EXTENSION);
3439        assert_eq!(files.len(), 1);
3440        let rows = read_feather_batches(&files[0])
3441            .iter()
3442            .map(RecordBatch::num_rows)
3443            .sum::<usize>();
3444        assert_eq!(rows, 2);
3445    }
3446}