1use std::{
19 cell::RefCell,
20 collections::{HashMap, HashSet},
21 rc::Rc,
22 sync::{
23 Arc,
24 atomic::{AtomicU64, Ordering},
25 },
26};
27
28use nautilus_common::{
29 clock::Clock,
30 enums::Environment,
31 live::block_on_nautilus_with,
32 python::{cache::PyCache, clock::PyClock},
33};
34use nautilus_core::python::to_pyvalue_err;
35use nautilus_model::{
36 data::{
37 Bar, CustomData, Data, FundingRateUpdate, IndexPriceUpdate, InstrumentStatus,
38 MarkPriceUpdate, OptionGreeks, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick,
39 close::InstrumentClose,
40 },
41 events::{
42 AccountState, OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled,
43 OrderDenied, OrderEmulated, OrderExpired, OrderFillVoided, OrderFilled, OrderInitialized,
44 OrderModifyDenied, OrderModifyRejected, OrderPendingCancel, OrderPendingUpdate,
45 OrderRejected, OrderReleased, OrderSnapshot, OrderSubmitted, OrderTriggered, OrderUpdated,
46 PositionAdjusted, PositionChanged, PositionClosed, PositionOpened, PositionSnapshot,
47 },
48 python::instruments::pyobject_to_instrument_any,
49 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
50};
51use pyo3::{exceptions::PyIOError, prelude::*};
52
53use crate::{
54 common::{
55 paths::{environment_from_directory, local_writer_directory, normalize_path_separators},
56 storage::{StorageBackend, create_storage_backend_from_path},
57 },
58 config::RotationConfig,
59 python::{
60 backend::{PyCatalogDataType, catalog_filter_family_from_py, writer_record_filter_from_py},
61 config::PyRotationConfig,
62 },
63 writer::{
64 factory::{WriterConnectConfig, replace_existing_writer_data},
65 feather::{FeatherWriter, WriterClock, recover_partial_feather_files},
66 run::RunStatus,
67 subscription::StreamingSinkSubscription,
68 },
69};
70
71type ClockBridge = (Rc<RefCell<dyn Clock>>, Arc<AtomicU64>);
73
74#[pyclass(
79 name = "StreamingFeatherWriter",
80 module = "nautilus_trader.persistence",
81 unsendable
82)]
83#[pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.persistence")]
84pub struct PyStreamingFeatherWriter {
85 writer: Rc<RefCell<FeatherWriter>>,
86 handler: Option<StreamingSinkSubscription>,
87 run_manifest: Option<(StorageBackend, Environment, String)>,
88 run_manifest_has_data: RefCell<bool>,
89 clock_bridge: Option<ClockBridge>,
94}
95
96#[pymethods]
97#[pyo3_stub_gen::derive::gen_stub_pymethods]
98impl PyStreamingFeatherWriter {
99 #[new]
113 #[pyo3(signature = (
114 path,
115 cache,
116 clock,
117 include_types=None,
118 record_types=None,
119 record_filters=None,
120 rotation_config=None,
121 flush_interval_ms=None,
122 replace=false
123 ))]
124 #[expect(
125 clippy::too_many_arguments,
126 clippy::needless_pass_by_value,
127 reason = "PyO3 constructor mirrors the writer configuration fields"
128 )]
129 pub fn py_new(
130 path: String,
131 cache: PyCache,
132 clock: PyClock,
133 include_types: Option<Vec<Bound<'_, PyAny>>>,
134 record_types: Option<&Bound<'_, PyAny>>,
135 record_filters: Option<&Bound<'_, PyAny>>,
136 rotation_config: Option<PyRotationConfig>,
137 flush_interval_ms: Option<u64>,
138 replace: bool,
139 ) -> PyResult<Self> {
140 let directory =
141 local_writer_directory(&path).map_err(|e| PyIOError::new_err(e.to_string()))?;
142 let rotation_config = rotation_config
143 .map_or(RotationConfig::NoRotation, Into::into)
144 .to_writer_rotation_config()
145 .map_err(to_pyvalue_err)?;
146
147 if replace {
148 replace_existing_writer_data(&WriterConnectConfig::new(path.clone(), None)).map_err(
149 |e| PyIOError::new_err(format!("Failed to replace existing files: {e}")),
150 )?;
151 }
152
153 recover_partial_feather_files(&directory);
154
155 let storage = create_storage_backend_from_path(&path, None)
156 .map_err(|e| PyIOError::new_err(format!("Failed to create storage backend: {e}")))?;
157
158 let run_manifest = if let Some((environment, instance_id)) =
159 run_environment_and_instance_id_from_path(&path)
160 {
161 let manifest_storage = storage.clone();
162 let manifest_instance_id = instance_id.clone();
163 block_on_nautilus_with(move || async move {
164 manifest_storage
165 .write_current_run_manifest(
166 environment,
167 &manifest_instance_id,
168 RunStatus::InProgress,
169 true,
170 )
171 .await
172 })
173 .map_err(|e| PyIOError::new_err(format!("Failed to write run manifest: {e}")))?;
174
175 Some((storage, environment, instance_id))
176 } else {
177 None
178 };
179
180 let type_filter = include_types
181 .map(|types| {
182 types
183 .iter()
184 .map(catalog_filter_family_from_py)
185 .collect::<PyResult<HashSet<_>>>()
186 })
187 .transpose()?;
188
189 let record_filter = writer_record_filter_from_py(record_types, record_filters)?;
190
191 let clock_rc = clock.clock_rc();
195 let (writer_clock, shared_time) = WriterClock::from_shared_clock(&clock_rc);
196 let _cache = cache;
199
200 let writer = FeatherWriter::new(
202 directory,
203 writer_clock,
204 rotation_config,
205 type_filter,
206 flush_interval_ms, )
208 .with_record_filter(record_filter);
209
210 Ok(Self {
211 writer: Rc::new(RefCell::new(writer)),
212 handler: None,
213 run_manifest,
214 run_manifest_has_data: RefCell::new(false),
215 clock_bridge: shared_time.map(|shared| (clock_rc, shared)),
216 })
217 }
218
219 pub fn subscribe(&mut self) -> PyResult<()> {
224 if self.handler.is_some() {
225 return Ok(());
227 }
228
229 let handler = FeatherWriter::subscribe_to_message_bus(self.writer.clone())
230 .map_err(|e| PyIOError::new_err(format!("Failed to subscribe to message bus: {e}")))?;
231
232 self.handler = Some(handler);
233 Ok(())
234 }
235
236 pub fn unsubscribe(&mut self) -> PyResult<()> {
238 if let Some(handler) = self.handler.take() {
239 FeatherWriter::unsubscribe_from_message_bus(&handler);
240 }
241
242 Ok(())
243 }
244
245 #[expect(
251 clippy::needless_pass_by_value,
252 reason = "PyO3 writer binding must downcast supported data variants inline"
253 )]
254 pub fn write(&self, py: Python, data: Py<PyAny>) -> PyResult<()> {
255 self.refresh_writer_clock();
256
257 macro_rules! try_write {
258 ($type:ty, $name:literal) => {
259 if let Ok(value) = data.extract::<$type>(py) {
260 let result =
261 self.writer.borrow_mut().write(value).map_err(|e| {
262 PyIOError::new_err(format!("Failed to write {}: {e}", $name))
263 });
264 return self.finish_write_result(result);
265 }
266 };
267 }
268
269 macro_rules! try_write_data {
270 ($data:expr, $name:literal) => {{
271 let result = self
272 .writer
273 .borrow_mut()
274 .write_data($data)
275 .map_err(|e| PyIOError::new_err(format!("Failed to write {}: {e}", $name)));
276 return self.finish_write_result(result);
277 }};
278 }
279
280 if let Ok(quote) = data.extract::<QuoteTick>(py) {
282 try_write_data!(Data::Quote(quote), "QuoteTick");
283 }
284
285 if let Ok(trade) = data.extract::<TradeTick>(py) {
286 try_write_data!(Data::Trade(trade), "TradeTick");
287 }
288
289 if let Ok(bar) = data.extract::<Bar>(py) {
290 try_write_data!(Data::Bar(bar), "Bar");
291 }
292
293 if let Ok(delta) = data.extract::<OrderBookDelta>(py) {
294 try_write_data!(Data::BookDelta(delta), "OrderBookDelta");
295 }
296
297 if let Ok(depth) = data.extract::<OrderBookDepth>(py) {
298 try_write_data!(Data::BookDepth(Box::new(depth)), "OrderBookDepth");
299 }
300
301 if let Ok(price) = data.extract::<IndexPriceUpdate>(py) {
302 try_write_data!(Data::IndexPrice(price), "IndexPriceUpdate");
303 }
304
305 if let Ok(price) = data.extract::<MarkPriceUpdate>(py) {
306 try_write_data!(Data::MarkPrice(price), "MarkPriceUpdate");
307 }
308
309 if let Ok(greeks) = data.extract::<OptionGreeks>(py) {
310 try_write_data!(Data::OptionGreeks(greeks), "OptionGreeks");
311 }
312
313 if let Ok(close) = data.extract::<InstrumentClose>(py) {
314 try_write_data!(Data::InstrumentClose(close), "InstrumentClose");
315 }
316
317 if let Ok(custom) = data.extract::<CustomData>(py) {
318 try_write_data!(Data::Custom(custom), "CustomData");
319 }
320
321 try_write!(FundingRateUpdate, "FundingRateUpdate");
322 try_write!(InstrumentStatus, "InstrumentStatus");
323 try_write!(AccountState, "AccountState");
324 try_write!(OrderInitialized, "OrderInitialized");
325 try_write!(OrderDenied, "OrderDenied");
326 try_write!(OrderEmulated, "OrderEmulated");
327 try_write!(OrderSubmitted, "OrderSubmitted");
328 try_write!(OrderAccepted, "OrderAccepted");
329 try_write!(OrderRejected, "OrderRejected");
330 try_write!(OrderPendingCancel, "OrderPendingCancel");
331 try_write!(OrderCanceled, "OrderCanceled");
332 try_write!(OrderCancelDenied, "OrderCancelDenied");
333 try_write!(OrderCancelRejected, "OrderCancelRejected");
334 try_write!(OrderExpired, "OrderExpired");
335 try_write!(OrderTriggered, "OrderTriggered");
336 try_write!(OrderPendingUpdate, "OrderPendingUpdate");
337 try_write!(OrderReleased, "OrderReleased");
338 try_write!(OrderModifyDenied, "OrderModifyDenied");
339 try_write!(OrderModifyRejected, "OrderModifyRejected");
340 try_write!(OrderUpdated, "OrderUpdated");
341 try_write!(OrderFilled, "OrderFilled");
342 try_write!(OrderFillVoided, "OrderFillVoided");
343 try_write!(PositionOpened, "PositionOpened");
344 try_write!(PositionChanged, "PositionChanged");
345 try_write!(PositionClosed, "PositionClosed");
346 try_write!(PositionAdjusted, "PositionAdjusted");
347 try_write!(OrderSnapshot, "OrderSnapshot");
348 try_write!(PositionSnapshot, "PositionSnapshot");
349 try_write!(OrderStatusReport, "OrderStatusReport");
350 try_write!(FillReport, "FillReport");
351 try_write!(PositionStatusReport, "PositionStatusReport");
352 try_write!(ExecutionMassStatus, "ExecutionMassStatus");
353
354 if let Ok(instrument) = pyobject_to_instrument_any(py, data.clone_ref(py)) {
356 let result = self
357 .writer
358 .borrow_mut()
359 .write_instrument(instrument)
360 .map_err(|e| PyIOError::new_err(format!("Failed to write instrument: {e}")));
361 return self.finish_write_result(result);
362 }
363
364 Err(PyIOError::new_err(
365 "Unsupported data type for feather writer",
366 ))
367 }
368
369 pub fn flush(&self) -> PyResult<()> {
374 self.refresh_writer_clock();
375 self.writer
376 .borrow_mut()
377 .flush()
378 .map_err(|e| PyIOError::new_err(format!("Failed to flush: {e}")))
379 }
380
381 pub fn close(&self) -> PyResult<()> {
385 self.refresh_writer_clock();
386 self.writer
387 .borrow_mut()
388 .close()
389 .map_err(|e| PyIOError::new_err(format!("Failed to close: {e}")))?;
390
391 self.write_run_manifest(
392 RunStatus::Completed,
393 !*self.run_manifest_has_data.borrow(),
394 "complete",
395 )
396 }
397
398 #[getter]
400 #[must_use]
401 pub fn is_closed(&self) -> bool {
402 self.writer.borrow().is_closed()
403 }
404
405 #[must_use]
407 pub fn get_current_file_info(&self) -> HashMap<PyCatalogDataType, (u64, String)> {
408 self.writer
409 .borrow()
410 .get_current_file_info()
411 .into_iter()
412 .map(|(data_type, info)| (PyCatalogDataType::new(data_type), info))
413 .collect()
414 }
415
416 #[must_use]
420 pub fn get_next_rotation_time(&self, data_type: PyCatalogDataType) -> Option<u64> {
421 self.writer
422 .borrow()
423 .get_next_rotation_time(&data_type.into_inner())
424 .map(|ns| ns.as_u64())
425 }
426}
427
428impl PyStreamingFeatherWriter {
429 fn refresh_writer_clock(&self) {
432 if let Some((clock, shared)) = &self.clock_bridge {
433 shared.store(clock.borrow().timestamp_ns().as_u64(), Ordering::Relaxed);
434 }
435 }
436
437 fn finish_write_result(&self, result: PyResult<()>) -> PyResult<()> {
438 result?;
439 self.mark_run_non_empty()
440 }
441
442 fn mark_run_non_empty(&self) -> PyResult<()> {
443 if *self.run_manifest_has_data.borrow() {
444 return Ok(());
445 }
446
447 self.write_run_manifest(RunStatus::InProgress, false, "update")?;
448 *self.run_manifest_has_data.borrow_mut() = true;
449 Ok(())
450 }
451
452 fn write_run_manifest(&self, status: RunStatus, empty: bool, operation: &str) -> PyResult<()> {
453 let Some((storage, environment, instance_id)) = &self.run_manifest else {
454 return Ok(());
455 };
456
457 let storage = storage.clone();
458 let environment = *environment;
459 let instance_id = instance_id.clone();
460 block_on_nautilus_with(move || async move {
461 storage
462 .write_current_run_manifest(environment, &instance_id, status, empty)
463 .await
464 })
465 .map_err(|e| PyIOError::new_err(format!("Failed to {operation} run manifest: {e}")))
466 }
467}
468
469fn run_environment_and_instance_id_from_path(path: &str) -> Option<(Environment, String)> {
470 let normalized = normalize_path_separators(path);
471 let parsed_url = normalized
472 .split_once("://")
473 .filter(|(scheme, _)| *scheme != "file")
474 .and_then(|_| url::Url::parse(&normalized).ok());
475 let path = parsed_url.as_ref().map_or_else(
476 || normalized.strip_prefix("file://").unwrap_or(&normalized),
477 |url| url.path().trim_start_matches('/'),
478 );
479
480 let components: Vec<&str> = path
481 .trim_matches('/')
482 .split('/')
483 .filter(|component| !component.is_empty())
484 .collect();
485 let instance_id = components.last()?;
486 let environment =
487 environment_from_directory(components.get(components.len().checked_sub(2)?)?)?;
488
489 Some((environment, (*instance_id).to_string()))
490}
491
492#[cfg(test)]
493mod tests {
494 use nautilus_common::enums::Environment;
495 use rstest::rstest;
496
497 use super::run_environment_and_instance_id_from_path;
498
499 #[rstest]
500 #[case(
501 r"C:\Users\Administrator\AppData\Local\Temp\pytest-0\backtest\run-greeks",
502 Environment::Backtest,
503 "run-greeks"
504 )]
505 #[case("C:/catalog/backtest/run-1", Environment::Backtest, "run-1")]
506 #[case(r"\\server\share\live\run-2", Environment::Live, "run-2")]
507 #[case("/tmp/catalog/sandbox/run-3", Environment::Sandbox, "run-3")]
508 #[case("file:///C:/catalog/backtest/run-1", Environment::Backtest, "run-1")]
509 #[case(r"C:\C# work\backtest\run-1", Environment::Backtest, "run-1")]
510 #[case("file:///C:/C# work/backtest/run-1", Environment::Backtest, "run-1")]
511 #[case("/tmp/a?b/live/run-2", Environment::Live, "run-2")]
512 #[case("file:///tmp/a?b/live/run-2", Environment::Live, "run-2")]
513 #[case(
514 "file:///tmp/a%20b/backtest/run%20id",
515 Environment::Backtest,
516 "run%20id"
517 )]
518 #[case::file_uri_space("file:///tmp/catalog/backtest/run 1", Environment::Backtest, "run 1")]
519 #[case::windows_drive_space(r"C:\catalog\backtest\run 1", Environment::Backtest, "run 1")]
520 #[case("s3://bucket/catalog/sandbox/run-3", Environment::Sandbox, "run-3")]
521 fn run_environment_and_instance_id_handles_platform_paths(
522 #[case] path: &str,
523 #[case] environment: Environment,
524 #[case] instance_id: &str,
525 ) {
526 assert_eq!(
527 run_environment_and_instance_id_from_path(path),
528 Some((environment, instance_id.to_string())),
529 );
530 }
531
532 #[rstest]
533 fn run_environment_and_instance_id_rejects_non_run_paths() {
534 assert_eq!(
535 run_environment_and_instance_id_from_path(r"C:\catalog\data\quotes"),
536 None
537 );
538 assert_eq!(
539 run_environment_and_instance_id_from_path("/tmp/catalog"),
540 None
541 );
542 }
543}