Skip to main content

nautilus_persistence/python/backend/
feather.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Python bindings for the Rust `FeatherWriter` as `StreamingFeatherWriter`.
17
18use std::{
19    cell::RefCell,
20    collections::{HashMap, HashSet},
21    rc::Rc,
22    sync::{
23        Arc,
24        atomic::{AtomicU64, Ordering},
25    },
26};
27
28use nautilus_common::{
29    clock::Clock,
30    enums::Environment,
31    live::block_on_nautilus_with,
32    python::{cache::PyCache, clock::PyClock},
33};
34use nautilus_core::python::to_pyvalue_err;
35use nautilus_model::{
36    data::{
37        Bar, CustomData, Data, FundingRateUpdate, IndexPriceUpdate, InstrumentStatus,
38        MarkPriceUpdate, OptionGreeks, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick,
39        close::InstrumentClose,
40    },
41    events::{
42        AccountState, OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled,
43        OrderDenied, OrderEmulated, OrderExpired, OrderFillVoided, OrderFilled, OrderInitialized,
44        OrderModifyDenied, OrderModifyRejected, OrderPendingCancel, OrderPendingUpdate,
45        OrderRejected, OrderReleased, OrderSnapshot, OrderSubmitted, OrderTriggered, OrderUpdated,
46        PositionAdjusted, PositionChanged, PositionClosed, PositionOpened, PositionSnapshot,
47    },
48    python::instruments::pyobject_to_instrument_any,
49    reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
50};
51use pyo3::{exceptions::PyIOError, prelude::*};
52
53use crate::{
54    common::{
55        paths::{environment_from_directory, local_writer_directory, normalize_path_separators},
56        storage::{StorageBackend, create_storage_backend_from_path},
57    },
58    config::RotationConfig,
59    python::{
60        backend::{PyCatalogDataType, catalog_filter_family_from_py, writer_record_filter_from_py},
61        config::PyRotationConfig,
62    },
63    writer::{
64        factory::{WriterConnectConfig, replace_existing_writer_data},
65        feather::{FeatherWriter, WriterClock, recover_partial_feather_files},
66        run::RunStatus,
67        subscription::StreamingSinkSubscription,
68    },
69};
70
71/// Source clock plus the shared atomic the writer reads time from.
72type ClockBridge = (Rc<RefCell<dyn Clock>>, Arc<AtomicU64>);
73
74/// Python binding for the Rust `FeatherWriter`.
75///
76/// This provides a streaming writer of Nautilus objects into feather files with rotation
77/// capabilities, matching the interface of Python's `StreamingFeatherWriter`.
78#[pyclass(
79    name = "StreamingFeatherWriter",
80    module = "nautilus_trader.persistence",
81    unsendable
82)]
83#[pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.persistence")]
84pub struct PyStreamingFeatherWriter {
85    writer: Rc<RefCell<FeatherWriter>>,
86    handler: Option<StreamingSinkSubscription>,
87    run_manifest: Option<(StorageBackend, Environment, String)>,
88    run_manifest_has_data: RefCell<bool>,
89    /// Present when constructed with a non-live clock: the source clock plus the
90    /// shared atomic the core writer reads, refreshed before each forwarded call.
91    /// Note: writes arriving via the message bus subscription do not refresh the
92    /// bridge; live usage pairs the subscription with a `LiveClock`.
93    clock_bridge: Option<ClockBridge>,
94}
95
96#[pymethods]
97#[pyo3_stub_gen::derive::gen_stub_pymethods]
98impl PyStreamingFeatherWriter {
99    /// Creates a new `StreamingFeatherWriter` instance.
100    ///
101    /// # Parameters
102    ///
103    /// - `path`: The local directory to append the stream files to.
104    /// - `cache`: The cache for query info (`PyCache`).
105    /// - `clock`: The clock to use for time-related operations (`PyClock`).
106    /// - `include_types`: Optional data or record types to include, as `NautilusDataType` or
107    ///   `NautilusRecordType` values.
108    /// - `rotation_config`: File rotation policy (default: no rotation).
109    /// - `flush_interval_ms`: Interval in milliseconds for flushing open files to disk (default:
110    ///   1000). Set to 0 to disable auto-flush.
111    /// - `replace`: If existing files at the given path should be replaced (default: False).
112    #[new]
113    #[pyo3(signature = (
114        path,
115        cache,
116        clock,
117        include_types=None,
118        record_types=None,
119        record_filters=None,
120        rotation_config=None,
121        flush_interval_ms=None,
122        replace=false
123    ))]
124    #[expect(
125        clippy::too_many_arguments,
126        clippy::needless_pass_by_value,
127        reason = "PyO3 constructor mirrors the writer configuration fields"
128    )]
129    pub fn py_new(
130        path: String,
131        cache: PyCache,
132        clock: PyClock,
133        include_types: Option<Vec<Bound<'_, PyAny>>>,
134        record_types: Option<&Bound<'_, PyAny>>,
135        record_filters: Option<&Bound<'_, PyAny>>,
136        rotation_config: Option<PyRotationConfig>,
137        flush_interval_ms: Option<u64>,
138        replace: bool,
139    ) -> PyResult<Self> {
140        let directory =
141            local_writer_directory(&path).map_err(|e| PyIOError::new_err(e.to_string()))?;
142        let rotation_config = rotation_config
143            .map_or(RotationConfig::NoRotation, Into::into)
144            .to_writer_rotation_config()
145            .map_err(to_pyvalue_err)?;
146
147        if replace {
148            replace_existing_writer_data(&WriterConnectConfig::new(path.clone(), None)).map_err(
149                |e| PyIOError::new_err(format!("Failed to replace existing files: {e}")),
150            )?;
151        }
152
153        recover_partial_feather_files(&directory);
154
155        let storage = create_storage_backend_from_path(&path, None)
156            .map_err(|e| PyIOError::new_err(format!("Failed to create storage backend: {e}")))?;
157
158        let run_manifest = if let Some((environment, instance_id)) =
159            run_environment_and_instance_id_from_path(&path)
160        {
161            let manifest_storage = storage.clone();
162            let manifest_instance_id = instance_id.clone();
163            block_on_nautilus_with(move || async move {
164                manifest_storage
165                    .write_current_run_manifest(
166                        environment,
167                        &manifest_instance_id,
168                        RunStatus::InProgress,
169                        true,
170                    )
171                    .await
172            })
173            .map_err(|e| PyIOError::new_err(format!("Failed to write run manifest: {e}")))?;
174
175            Some((storage, environment, instance_id))
176        } else {
177            None
178        };
179
180        let type_filter = include_types
181            .map(|types| {
182                types
183                    .iter()
184                    .map(catalog_filter_family_from_py)
185                    .collect::<PyResult<HashSet<_>>>()
186            })
187            .transpose()?;
188
189        let record_filter = writer_record_filter_from_py(record_types, record_filters)?;
190
191        // Extract Clock from Python wrapper and translate it into the core
192        // writer's Send time source (live clocks read the wall clock directly;
193        // test clocks are bridged through a shared atomic)
194        let clock_rc = clock.clock_rc();
195        let (writer_clock, shared_time) = WriterClock::from_shared_clock(&clock_rc);
196        // Note: Cache parameter is kept for API compatibility with Python StreamingFeatherWriter
197        // but is not directly used by FeatherWriter
198        let _cache = cache;
199
200        // Create FeatherWriter
201        let writer = FeatherWriter::new(
202            directory,
203            writer_clock,
204            rotation_config,
205            type_filter,
206            flush_interval_ms, // Auto-flush interval in milliseconds
207        )
208        .with_record_filter(record_filter);
209
210        Ok(Self {
211            writer: Rc::new(RefCell::new(writer)),
212            handler: None,
213            run_manifest,
214            run_manifest_has_data: RefCell::new(false),
215            clock_bridge: shared_time.map(|shared| (clock_rc, shared)),
216        })
217    }
218
219    /// Subscribes to all messages on the message bus (pattern "*").
220    ///
221    /// This matches the behavior of Python's `StreamingFeatherWriter` when subscribed
222    /// via `trader.subscribe("*", writer.write)`.
223    pub fn subscribe(&mut self) -> PyResult<()> {
224        if self.handler.is_some() {
225            // Already subscribed
226            return Ok(());
227        }
228
229        let handler = FeatherWriter::subscribe_to_message_bus(self.writer.clone())
230            .map_err(|e| PyIOError::new_err(format!("Failed to subscribe to message bus: {e}")))?;
231
232        self.handler = Some(handler);
233        Ok(())
234    }
235
236    /// Unsubscribes from the message bus.
237    pub fn unsubscribe(&mut self) -> PyResult<()> {
238        if let Some(handler) = self.handler.take() {
239            FeatherWriter::unsubscribe_from_message_bus(&handler);
240        }
241
242        Ok(())
243    }
244
245    /// Writes a data object to the stream.
246    ///
247    /// # Parameters
248    ///
249    /// - `data`: The data object to write (must be a Nautilus data type from pyo3).
250    #[expect(
251        clippy::needless_pass_by_value,
252        reason = "PyO3 writer binding must downcast supported data variants inline"
253    )]
254    pub fn write(&self, py: Python, data: Py<PyAny>) -> PyResult<()> {
255        self.refresh_writer_clock();
256
257        macro_rules! try_write {
258            ($type:ty, $name:literal) => {
259                if let Ok(value) = data.extract::<$type>(py) {
260                    let result =
261                        self.writer.borrow_mut().write(value).map_err(|e| {
262                            PyIOError::new_err(format!("Failed to write {}: {e}", $name))
263                        });
264                    return self.finish_write_result(result);
265                }
266            };
267        }
268
269        macro_rules! try_write_data {
270            ($data:expr, $name:literal) => {{
271                let result = self
272                    .writer
273                    .borrow_mut()
274                    .write_data($data)
275                    .map_err(|e| PyIOError::new_err(format!("Failed to write {}: {e}", $name)));
276                return self.finish_write_result(result);
277            }};
278        }
279
280        // Try to convert from common pyo3 data types
281        if let Ok(quote) = data.extract::<QuoteTick>(py) {
282            try_write_data!(Data::Quote(quote), "QuoteTick");
283        }
284
285        if let Ok(trade) = data.extract::<TradeTick>(py) {
286            try_write_data!(Data::Trade(trade), "TradeTick");
287        }
288
289        if let Ok(bar) = data.extract::<Bar>(py) {
290            try_write_data!(Data::Bar(bar), "Bar");
291        }
292
293        if let Ok(delta) = data.extract::<OrderBookDelta>(py) {
294            try_write_data!(Data::BookDelta(delta), "OrderBookDelta");
295        }
296
297        if let Ok(depth) = data.extract::<OrderBookDepth>(py) {
298            try_write_data!(Data::BookDepth(Box::new(depth)), "OrderBookDepth");
299        }
300
301        if let Ok(price) = data.extract::<IndexPriceUpdate>(py) {
302            try_write_data!(Data::IndexPrice(price), "IndexPriceUpdate");
303        }
304
305        if let Ok(price) = data.extract::<MarkPriceUpdate>(py) {
306            try_write_data!(Data::MarkPrice(price), "MarkPriceUpdate");
307        }
308
309        if let Ok(greeks) = data.extract::<OptionGreeks>(py) {
310            try_write_data!(Data::OptionGreeks(greeks), "OptionGreeks");
311        }
312
313        if let Ok(close) = data.extract::<InstrumentClose>(py) {
314            try_write_data!(Data::InstrumentClose(close), "InstrumentClose");
315        }
316
317        if let Ok(custom) = data.extract::<CustomData>(py) {
318            try_write_data!(Data::Custom(custom), "CustomData");
319        }
320
321        try_write!(FundingRateUpdate, "FundingRateUpdate");
322        try_write!(InstrumentStatus, "InstrumentStatus");
323        try_write!(AccountState, "AccountState");
324        try_write!(OrderInitialized, "OrderInitialized");
325        try_write!(OrderDenied, "OrderDenied");
326        try_write!(OrderEmulated, "OrderEmulated");
327        try_write!(OrderSubmitted, "OrderSubmitted");
328        try_write!(OrderAccepted, "OrderAccepted");
329        try_write!(OrderRejected, "OrderRejected");
330        try_write!(OrderPendingCancel, "OrderPendingCancel");
331        try_write!(OrderCanceled, "OrderCanceled");
332        try_write!(OrderCancelDenied, "OrderCancelDenied");
333        try_write!(OrderCancelRejected, "OrderCancelRejected");
334        try_write!(OrderExpired, "OrderExpired");
335        try_write!(OrderTriggered, "OrderTriggered");
336        try_write!(OrderPendingUpdate, "OrderPendingUpdate");
337        try_write!(OrderReleased, "OrderReleased");
338        try_write!(OrderModifyDenied, "OrderModifyDenied");
339        try_write!(OrderModifyRejected, "OrderModifyRejected");
340        try_write!(OrderUpdated, "OrderUpdated");
341        try_write!(OrderFilled, "OrderFilled");
342        try_write!(OrderFillVoided, "OrderFillVoided");
343        try_write!(PositionOpened, "PositionOpened");
344        try_write!(PositionChanged, "PositionChanged");
345        try_write!(PositionClosed, "PositionClosed");
346        try_write!(PositionAdjusted, "PositionAdjusted");
347        try_write!(OrderSnapshot, "OrderSnapshot");
348        try_write!(PositionSnapshot, "PositionSnapshot");
349        try_write!(OrderStatusReport, "OrderStatusReport");
350        try_write!(FillReport, "FillReport");
351        try_write!(PositionStatusReport, "PositionStatusReport");
352        try_write!(ExecutionMassStatus, "ExecutionMassStatus");
353
354        // Try instrument types (uses type_str attribute for dispatch)
355        if let Ok(instrument) = pyobject_to_instrument_any(py, data.clone_ref(py)) {
356            let result = self
357                .writer
358                .borrow_mut()
359                .write_instrument(instrument)
360                .map_err(|e| PyIOError::new_err(format!("Failed to write instrument: {e}")));
361            return self.finish_write_result(result);
362        }
363
364        Err(PyIOError::new_err(
365            "Unsupported data type for feather writer",
366        ))
367    }
368
369    /// Flushes buffered bytes of every open file to disk.
370    ///
371    /// This is called automatically based on `flush_interval_ms` if configured, but can also
372    /// be called manually by the client. Files stay open, so flushing never starts a new file.
373    pub fn flush(&self) -> PyResult<()> {
374        self.refresh_writer_clock();
375        self.writer
376            .borrow_mut()
377            .flush()
378            .map_err(|e| PyIOError::new_err(format!("Failed to flush: {e}")))
379    }
380
381    /// Seals all open files so each complete stream is visible as a `.feather` file.
382    ///
383    /// Writes after closing open new files.
384    pub fn close(&self) -> PyResult<()> {
385        self.refresh_writer_clock();
386        self.writer
387            .borrow_mut()
388            .close()
389            .map_err(|e| PyIOError::new_err(format!("Failed to close: {e}")))?;
390
391        self.write_run_manifest(
392            RunStatus::Completed,
393            !*self.run_manifest_has_data.borrow(),
394            "complete",
395        )
396    }
397
398    /// Returns whether the writer has been closed (no active writers).
399    #[getter]
400    #[must_use]
401    pub fn is_closed(&self) -> bool {
402        self.writer.borrow().is_closed()
403    }
404
405    /// Returns the current file size and path of each open file, keyed by the type it stages.
406    #[must_use]
407    pub fn get_current_file_info(&self) -> HashMap<PyCatalogDataType, (u64, String)> {
408        self.writer
409            .borrow()
410            .get_current_file_info()
411            .into_iter()
412            .map(|(data_type, info)| (PyCatalogDataType::new(data_type), info))
413            .collect()
414    }
415
416    /// Returns the next rotation time of a type's file, or None if not set.
417    ///
418    /// Pass a `NautilusInstrumentType` for the file of one instrument class.
419    #[must_use]
420    pub fn get_next_rotation_time(&self, data_type: PyCatalogDataType) -> Option<u64> {
421        self.writer
422            .borrow()
423            .get_next_rotation_time(&data_type.into_inner())
424            .map(|ns| ns.as_u64())
425    }
426}
427
428impl PyStreamingFeatherWriter {
429    // Pushes the source clock's current time into the core writer's shared
430    // atomic so test clocks drive flush/rotation cadence correctly.
431    fn refresh_writer_clock(&self) {
432        if let Some((clock, shared)) = &self.clock_bridge {
433            shared.store(clock.borrow().timestamp_ns().as_u64(), Ordering::Relaxed);
434        }
435    }
436
437    fn finish_write_result(&self, result: PyResult<()>) -> PyResult<()> {
438        result?;
439        self.mark_run_non_empty()
440    }
441
442    fn mark_run_non_empty(&self) -> PyResult<()> {
443        if *self.run_manifest_has_data.borrow() {
444            return Ok(());
445        }
446
447        self.write_run_manifest(RunStatus::InProgress, false, "update")?;
448        *self.run_manifest_has_data.borrow_mut() = true;
449        Ok(())
450    }
451
452    fn write_run_manifest(&self, status: RunStatus, empty: bool, operation: &str) -> PyResult<()> {
453        let Some((storage, environment, instance_id)) = &self.run_manifest else {
454            return Ok(());
455        };
456
457        let storage = storage.clone();
458        let environment = *environment;
459        let instance_id = instance_id.clone();
460        block_on_nautilus_with(move || async move {
461            storage
462                .write_current_run_manifest(environment, &instance_id, status, empty)
463                .await
464        })
465        .map_err(|e| PyIOError::new_err(format!("Failed to {operation} run manifest: {e}")))
466    }
467}
468
469fn run_environment_and_instance_id_from_path(path: &str) -> Option<(Environment, String)> {
470    let normalized = normalize_path_separators(path);
471    let parsed_url = url::Url::parse(&normalized).ok();
472
473    let path = parsed_url.as_ref().map_or(normalized.as_str(), |url| {
474        url.path().trim_start_matches('/')
475    });
476
477    let components: Vec<&str> = path
478        .trim_matches('/')
479        .split('/')
480        .filter(|component| !component.is_empty())
481        .collect();
482    let instance_id = components.last()?;
483    let environment =
484        environment_from_directory(components.get(components.len().checked_sub(2)?)?)?;
485
486    Some((environment, (*instance_id).to_string()))
487}
488
489#[cfg(test)]
490mod tests {
491    use nautilus_common::enums::Environment;
492    use rstest::rstest;
493
494    use super::run_environment_and_instance_id_from_path;
495
496    #[rstest]
497    #[case(
498        r"C:\Users\Administrator\AppData\Local\Temp\pytest-0\backtest\run-greeks",
499        Environment::Backtest,
500        "run-greeks"
501    )]
502    #[case("C:/catalog/backtest/run-1", Environment::Backtest, "run-1")]
503    #[case(r"\\server\share\live\run-2", Environment::Live, "run-2")]
504    #[case("/tmp/catalog/sandbox/run-3", Environment::Sandbox, "run-3")]
505    #[case("file:///C:/catalog/backtest/run-1", Environment::Backtest, "run-1")]
506    fn run_environment_and_instance_id_handles_platform_paths(
507        #[case] path: &str,
508        #[case] environment: Environment,
509        #[case] instance_id: &str,
510    ) {
511        assert_eq!(
512            run_environment_and_instance_id_from_path(path),
513            Some((environment, instance_id.to_string())),
514        );
515    }
516
517    #[rstest]
518    fn run_environment_and_instance_id_rejects_non_run_paths() {
519        assert_eq!(
520            run_environment_and_instance_id_from_path(r"C:\catalog\data\quotes"),
521            None
522        );
523        assert_eq!(
524            run_environment_and_instance_id_from_path("/tmp/catalog"),
525            None
526        );
527    }
528}