1use std::{
13 fmt::Debug,
14 sync::{
15 Arc,
16 atomic::{AtomicBool, Ordering},
17 },
18};
19
20use nautilus_common::live::block_on_nautilus_with;
21use object_store::{
22 Error as ObjectStoreError, ObjectStoreExt, PutMode, PutOptions, path::Path as ObjectPath,
23};
24use serde::{Deserialize, Serialize};
25
26use super::catalog::ParquetDataCatalog;
27use crate::{
28 catalog::factory::CatalogConnectConfig,
29 common::{
30 conversion::FeatherConversionSummary,
31 datafusion::identifiers_from_record_batches,
32 paths::{
33 catalog_data_type_from_session_feather_path, create_local_directory,
34 identifier_from_session_feather_path,
35 },
36 storage::create_storage_backend_from_path,
37 },
38 writer::{
39 factory::{PARQUET_WRITER_FACTORY_NAME, WriterConnectConfig, WriterFactoryRegistry},
40 feather::WriterClock,
41 materializer::read_feather_record_batches_with_identity,
42 promotion::{
43 PromotionBackend, PromotionResult, PromotionSession, PromotionWork,
44 StagedPromotionBackend,
45 },
46 run::{FeatherSessionSource, RunStatus},
47 staged::{StagedFeatherWriter, StagedWriter},
48 traits::StreamingDataSink,
49 },
50};
51
52pub(crate) fn register_factory(registry: &mut WriterFactoryRegistry) {
53 registry.insert(
54 PARQUET_WRITER_FACTORY_NAME.to_string(),
55 Arc::new(parquet_writer_factory),
56 );
57}
58
59fn parquet_writer_factory(
60 config: &WriterConnectConfig,
61 clock: WriterClock,
62) -> anyhow::Result<StreamingDataSink> {
63 Ok(Box::new(ParquetWriter::new(config, clock)?))
64}
65
66#[expect(
67 clippy::struct_excessive_bools,
68 reason = "the booleans are independent writer policy and lifecycle flags"
69)]
70struct ParquetWriter {
71 core: StagedFeatherWriter<ParquetPromotionBackend>,
72 session: PromotionSession,
73 source: FeatherSessionSource,
74 catalog: CatalogConnectConfig,
75 interval_ms: Option<u64>,
76 promote_on_close: bool,
77 delete_feather_after_commit: bool,
78 use_ts_event_for_ts_init: bool,
79 legacy_manifest_missing: Arc<AtomicBool>,
80 has_data: bool,
81 run_status: RunStatus,
82}
83
84impl Debug for ParquetWriter {
85 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86 f.debug_struct(stringify!(ParquetWriter))
87 .field("staging_uri", &self.core.storage.original_uri)
88 .field("catalog_uri", &self.catalog.uri)
89 .field("interval_ms", &self.interval_ms)
90 .finish_non_exhaustive()
91 }
92}
93
94impl ParquetWriter {
95 fn new(config: &WriterConnectConfig, clock: WriterClock) -> anyhow::Result<Self> {
96 let catalog = config
97 .required_catalog(PARQUET_WRITER_FACTORY_NAME)?
98 .clone();
99 let storage = create_storage_backend_from_path(&config.uri, None)?;
100 let session = StagedFeatherWriter::<ParquetPromotionBackend>::required_session(
101 &storage.original_uri,
102 "Parquet writer URI",
103 )?;
104 let source_storage = create_storage_backend_from_path(&session.root_uri, None)?;
105
106 let source = FeatherSessionSource::new(
107 source_storage,
108 session.environment,
109 session.instance_id.clone(),
110 );
111
112 create_local_directory(&catalog.uri)?;
115
116 let legacy_manifest_missing = Arc::new(AtomicBool::new(false));
117
118 ParquetPromotionBackend::new(&catalog, Arc::clone(&legacy_manifest_missing))?;
119
120 let mut core = StagedFeatherWriter::new(
121 storage.clone(),
122 &source,
123 clock,
124 config.rotation_config.clone(),
125 None,
126 config.flush_interval_ms,
127 config.record_filter.clone(),
128 )?;
129
130 block_on_nautilus_with(|| {
131 storage.write_current_run_manifest(
132 session.environment,
133 &session.instance_id,
134 RunStatus::InProgress,
135 true,
136 )
137 })?;
138
139 let timer_catalog = catalog.clone();
140 core.warn_if_orphan_feather_present("Parquet");
141 let timer_legacy_manifest_missing = Arc::clone(&legacy_manifest_missing);
142 core.start_promotion_timer(
143 config.promotion_interval_ms,
144 source.clone(),
145 "parquet-promotion",
146 move || {
147 ParquetPromotionBackend::new(
148 &timer_catalog,
149 Arc::clone(&timer_legacy_manifest_missing),
150 )
151 },
152 config.use_ts_event_for_ts_init,
153 config.delete_feather_after_promotion,
154 )?;
155
156 Ok(Self {
157 core,
158 catalog,
159 session,
160 source,
161 interval_ms: config.promotion_interval_ms,
162 promote_on_close: config.promote_on_close,
163 delete_feather_after_commit: config.delete_feather_after_promotion,
164 use_ts_event_for_ts_init: config.use_ts_event_for_ts_init,
165 legacy_manifest_missing,
166 has_data: false,
167 run_status: RunStatus::InProgress,
168 })
169 }
170
171 fn record_status(&mut self, status: RunStatus) -> anyhow::Result<()> {
172 if self.run_status == status {
173 return Ok(());
174 }
175
176 block_on_nautilus_with(|| {
177 self.core.storage.write_current_run_manifest(
178 self.session.environment,
179 &self.session.instance_id,
180 status,
181 !self.has_data,
182 )
183 })?;
184
185 self.run_status = status;
186 Ok(())
187 }
188
189 fn mark_non_empty(&mut self) -> anyhow::Result<()> {
190 if self.has_data {
191 return Ok(());
192 }
193
194 block_on_nautilus_with(|| {
195 self.core.storage.write_current_run_manifest(
196 self.session.environment,
197 &self.session.instance_id,
198 RunStatus::InProgress,
199 false,
200 )
201 })?;
202
203 self.has_data = true;
204 Ok(())
205 }
206
207 fn interval_ns(&self) -> Option<u64> {
208 self.interval_ms
209 .filter(|interval_ms| *interval_ms > 0)
210 .map(|interval_ms| interval_ms.saturating_mul(1_000_000))
211 }
212
213 fn prepare_promotion(&self) -> anyhow::Result<Option<PromotionWork<ParquetPromotionBackend>>> {
214 let backend =
215 ParquetPromotionBackend::new(&self.catalog, Arc::clone(&self.legacy_manifest_missing))?;
216 self.core.prepare_promotion(
217 backend,
218 self.source.clone(),
219 self.use_ts_event_for_ts_init,
220 self.delete_feather_after_commit,
221 )
222 }
223
224 fn finalize(
225 &mut self,
226 result: PromotionResult,
227 ) -> anyhow::Result<Vec<FeatherConversionSummary>> {
228 if result.converted.is_err() {
229 if result.run_state_recorded {
230 self.run_status = RunStatus::Failed;
231 } else if let Err(e) = self.record_status(RunStatus::Failed) {
232 log::warn!("Failed to record Parquet run Failed state: {e}");
233 }
234 } else if result.run_state_recorded {
235 self.run_status = RunStatus::Promoted;
236 }
237
238 self.core.finalize_promotion(result)
239 }
240
241 fn drain_completed(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
242 let mut converted = Vec::new();
243 for result in self.core.promotion_driver.drain_completed()? {
244 converted.extend(self.finalize(result)?);
245 }
246
247 Ok(converted)
248 }
249
250 fn wait(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
251 let mut converted = Vec::new();
252 for result in self.core.promotion_driver.wait()? {
253 converted.extend(self.finalize(result)?);
254 }
255
256 Ok(converted)
257 }
258
259 fn promote_now(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
260 let mut converted = self.wait()?;
261 if let Some(work) = self.prepare_promotion()? {
262 converted.extend(self.finalize(work.execute())?);
263 }
264
265 Ok(converted)
266 }
267}
268
269impl StagedWriter for ParquetWriter {
270 type Backend = ParquetPromotionBackend;
271
272 fn staged(&mut self) -> &mut StagedFeatherWriter<Self::Backend> {
273 &mut self.core
274 }
275
276 fn mark_non_empty(&mut self) -> anyhow::Result<()> {
277 Self::mark_non_empty(self)
278 }
279
280 fn maybe_promote(&mut self) -> anyhow::Result<()> {
281 self.drain_completed()?;
282
283 let Some(interval_ns) = self.interval_ns() else {
284 return Ok(());
285 };
286
287 if self
288 .core
289 .promotion_driver
290 .is_due(self.core.clock.timestamp_ns(), interval_ns)
291 && let Some(work) = self.prepare_promotion()?
292 {
293 self.core
294 .promotion_driver
295 .submit(work, "parquet-promotion")?;
296 self.core
297 .promotion_driver
298 .mark_committed_at(self.core.clock.timestamp_ns());
299 }
300
301 Ok(())
302 }
303
304 fn wait_for_promotions(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
305 self.wait()
306 }
307
308 fn promote_on_flush(&self) -> bool {
309 self.interval_ns().is_some_and(|interval_ns| {
310 self.core
311 .promotion_driver
312 .is_due(self.core.clock.timestamp_ns(), interval_ns)
313 })
314 }
315
316 fn promote_on_close(&self) -> bool {
317 self.promote_on_close
318 }
319
320 fn promote_now(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
321 Self::promote_now(self)
322 }
323
324 fn record_completed(&mut self) {
325 if self.run_status == RunStatus::InProgress
326 && let Err(e) = self.record_status(RunStatus::Completed)
327 {
328 log::warn!("Failed to complete Parquet run manifest: {e}");
329 }
330 }
331}
332
333impl Drop for ParquetWriter {
334 fn drop(&mut self) {
335 self.core.stop_promotion_timer();
336
337 let promotion_failed = if let Err(e) = self.wait() {
338 log::warn!("ParquetWriter dropped with pending promotion error: {e}");
339 true
340 } else {
341 false
342 };
343
344 if self.run_status == RunStatus::InProgress {
345 let status = if promotion_failed || !self.core.is_closed().unwrap_or(false) {
346 RunStatus::Failed
347 } else {
348 RunStatus::Completed
349 };
350
351 if let Err(e) = self.record_status(status) {
352 log::warn!(
353 "Failed to record Parquet run {} state during drop: {e}",
354 status.as_str()
355 );
356 }
357 }
358 }
359}
360
361const PROMOTION_MANIFEST: &str = "_nautilus_promotions.json";
362const PROMOTION_MARKERS: &str = "_nautilus_promotions";
363
364struct ParquetPromotionBackend {
365 catalog: ParquetDataCatalog,
366 legacy_manifest_missing: Arc<AtomicBool>,
367}
368
369impl ParquetPromotionBackend {
370 fn new(
371 catalog: &CatalogConnectConfig,
372 legacy_manifest_missing: Arc<AtomicBool>,
373 ) -> anyhow::Result<Self> {
374 Ok(Self {
375 catalog: super::open_catalog(catalog)?,
376 legacy_manifest_missing,
377 })
378 }
379
380 fn manifest_path(&self) -> ObjectPath {
381 self.catalog_path(PROMOTION_MANIFEST)
382 }
383
384 fn manifest(&self) -> anyhow::Result<ParquetPromotionManifest> {
385 if self.legacy_manifest_missing.load(Ordering::Relaxed) {
386 return Ok(ParquetPromotionManifest::default());
387 }
388
389 let path = self.manifest_path();
390 block_on_nautilus_with(|| async {
391 let result = match self.catalog.object_store.get(&path).await {
392 Ok(result) => result,
393 Err(ObjectStoreError::NotFound { .. }) => {
394 self.legacy_manifest_missing.store(true, Ordering::Relaxed);
395 return Ok(ParquetPromotionManifest::default());
396 }
397 Err(e) => return Err(e.into()),
398 };
399
400 Ok(serde_json::from_slice(&result.bytes().await?)?)
401 })
402 }
403
404 fn marker_path(&self, identity: &str) -> ObjectPath {
405 let digest = blake3::hash(identity.as_bytes()).to_hex();
406 self.catalog_path(&format!("{PROMOTION_MARKERS}/{digest}.json"))
407 }
408
409 fn catalog_path(&self, path: &str) -> ObjectPath {
410 let base = self.catalog.base_path.trim_matches('/');
411 if base.is_empty() {
412 ObjectPath::from(path)
413 } else {
414 ObjectPath::from(format!("{base}/{path}"))
415 }
416 }
417
418 fn identity_recorded(&self, identity: &str) -> anyhow::Result<bool> {
419 let path = self.marker_path(identity);
420
421 let marker_exists = block_on_nautilus_with(|| async {
422 match self.catalog.object_store.head(&path).await {
423 Ok(_) => Ok::<bool, anyhow::Error>(true),
424 Err(ObjectStoreError::NotFound { .. }) => Ok(false),
425 Err(e) => Err(anyhow::Error::from(e)),
426 }
427 })?;
428
429 Ok(marker_exists
430 || self
431 .manifest()?
432 .identities
433 .iter()
434 .any(|item| item == identity))
435 }
436
437 fn record_identity(&self, identity: &str) -> anyhow::Result<()> {
438 let bytes = serde_json::to_vec(identity)?;
439 let path = self.marker_path(identity);
440 block_on_nautilus_with(|| async {
441 match self
442 .catalog
443 .object_store
444 .put_opts(
445 &path,
446 bytes.into(),
447 PutOptions {
448 mode: PutMode::Create,
449 ..Default::default()
450 },
451 )
452 .await
453 {
454 Ok(_) | Err(ObjectStoreError::AlreadyExists { .. }) => Ok(()),
455 Err(e) => Err(e.into()),
456 }
457 })
458 }
459}
460
461impl PromotionBackend for ParquetPromotionBackend {
462 type Source = FeatherSessionSource;
463
464 const NAME: &'static str = "Parquet";
465
466 fn convert_file(
467 &mut self,
468 source: &Self::Source,
469 file: &str,
470 use_ts_event_for_ts_init: bool,
471 record_promoted: bool,
472 ) -> anyhow::Result<Option<FeatherConversionSummary>> {
473 let object_path = ObjectPath::parse(file)?;
474
475 let read = block_on_nautilus_with(|| {
476 read_feather_record_batches_with_identity(
477 source.storage.object_store.clone(),
478 &object_path,
479 )
480 })?;
481
482 let identifiers = identifiers_from_record_batches(&read.batches)
483 .ok()
484 .filter(|identifiers| !identifiers.is_empty());
485 let identity = feather_replay_identity(
486 &source.storage.original_uri,
487 file,
488 &read.content_hash,
489 identifiers.as_deref(),
490 );
491
492 if self.identity_recorded(&identity)? {
493 let identifier = match identifiers {
494 Some(mut identifiers) if identifiers.len() == 1 => identifiers.pop(),
495 Some(_) => None,
496 None => identifier_from_session_feather_path(
497 file,
498 source.environment,
499 &source.instance_id,
500 ),
501 };
502
503 return Ok(Some(FeatherConversionSummary {
504 data_type: catalog_data_type_from_session_feather_path(
505 file,
506 source.environment,
507 &source.instance_id,
508 )?,
509 identifier,
510 feather_path: file.to_string(),
511 native_version: None,
512 unmatched_identifiers: None,
513 }));
514 }
515
516 let summary = self.catalog.promote_feather_file(
517 source,
518 file,
519 read.batches,
520 use_ts_event_for_ts_init,
521 &identity,
522 )?;
523
524 if summary.is_some() && record_promoted {
525 self.record_identity(&identity)?;
526 }
527
528 Ok(summary)
529 }
530
531 fn delete_file(&mut self, source: &Self::Source, file: &str) -> anyhow::Result<()> {
532 let object_path = ObjectPath::parse(file)?;
533 block_on_nautilus_with(|| async {
534 source.storage.object_store.delete(&object_path).await?;
535 Ok::<(), anyhow::Error>(())
536 })
537 }
538}
539
540impl StagedPromotionBackend for ParquetPromotionBackend {
541 const REQUIRES_SESSION: bool = true;
542
543 fn record_run_state(
544 &mut self,
545 source: &FeatherSessionSource,
546 _staging_uri: &str,
547 status: RunStatus,
548 empty: bool,
549 _error: Option<&str>,
550 ) -> anyhow::Result<()> {
551 block_on_nautilus_with(|| {
552 source.storage.write_run_manifest(
553 source.environment,
554 &source.instance_id,
555 status,
556 empty,
557 )
558 })
559 }
560}
561
562#[derive(Debug, Default, Deserialize, Serialize)]
563struct ParquetPromotionManifest {
564 identities: Vec<String>,
565}
566
567fn feather_replay_identity(
568 source_uri: &str,
569 source_path: &str,
570 content_hash: &str,
571 identifiers: Option<&[String]>,
572) -> String {
573 let mut identifiers = identifiers.map(<[String]>::to_vec);
574 if let Some(identifiers) = identifiers.as_mut() {
575 identifiers.sort();
576 identifiers.dedup();
577 }
578
579 let identity = serde_json::json!({
580 "source_uri": source_uri,
581 "source_path": source_path,
582 "content_hash": content_hash,
583 "identifiers": identifiers,
584 });
585 format!(
586 "nautilus-feather:{}",
587 blake3::hash(identity.to_string().as_bytes()).to_hex(),
588 )
589}
590
591#[cfg(test)]
592mod tests {
593 use std::{
594 fs,
595 path::{Path, PathBuf},
596 sync::atomic::AtomicU64,
597 };
598
599 use arrow::datatypes::SchemaRef;
600 use nautilus_common::enums::Environment;
601 use nautilus_core::{Params, UnixNanos};
602 use nautilus_model::{
603 data::{
604 Bar, BarType, BookOrder, Data, DataBatch, NautilusDataType, NautilusRecordType,
605 OrderBookDelta, QuoteTick,
606 },
607 enums::{BookAction, OrderSide, RecordFlag},
608 identifiers::InstrumentId,
609 types::{ERROR_PRICE, Price, Quantity},
610 };
611 use nautilus_serialization::arrow::{DecodeTypedFromRecordBatch, EncodeToRecordBatch};
612 use object_store::memory::InMemory;
613 use parquet::{
614 basic::Compression,
615 file::{
616 metadata::RowGroupMetaData,
617 reader::{FileReader, SerializedFileReader},
618 },
619 };
620 use rstest::rstest;
621 use tempfile::TempDir;
622
623 use super::*;
624 use crate::{
625 backend::parquet::io::read_parquet_from_object_store,
626 catalog::traits::{CatalogQuery, CatalogReader},
627 common::{paths::normalize_path_to_uri, storage::RUN_MANIFEST_FILENAME},
628 config::{CatalogCompression, DataCatalogConfig},
629 test_data::RustTestHashMapCustomData,
630 writer::{
631 feather::{FEATHER_PARTIAL_EXTENSION, FeatherWriter, RotationConfig},
632 promotion::{list_session_feather_files, tests::assert_scheduled_count},
633 },
634 };
635
636 #[rstest]
637 fn parquet_default_close_promotes_and_honors_source_retention(
638 #[values(false, true)] delete_source: bool,
639 ) {
640 let directory = TempDir::new().unwrap();
641 let staging = directory.path().join("backtest").join("run-1");
642 let mut config =
643 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
644 config.delete_feather_after_promotion = delete_source;
645 let mut sink =
646 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
647 .unwrap();
648 let quote = sample_quote();
649 let mut second = quote;
650 second.instrument_id = InstrumentId::from("BTC/USD.SIM");
651 second.bid_price = Price::from("100.1234");
652 second.ask_price = Price::from("100.5678");
653 second.ts_init = UnixNanos::from(24);
654 let mut catalog = ParquetDataCatalog::from_uri(
655 directory.path().to_str().unwrap(),
656 None,
657 None,
658 None,
659 None,
660 )
661 .unwrap();
662 sink.write_data(Data::Quote(quote)).unwrap();
663 sink.write_data(Data::Quote(second)).unwrap();
664 sink.flush().unwrap();
665 let before = catalog
666 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
667 .unwrap();
668 sink.close().unwrap();
669 let after = catalog
670 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
671 .unwrap();
672 let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
673 let staged = block_on_nautilus_with(|| storage.list_files("", None))
674 .unwrap()
675 .into_iter()
676 .filter(|path| path.ends_with(".feather"))
677 .count();
678 assert!(before.is_empty());
679
680 let DataBatch::Quote(rows) = after else {
681 panic!("expected quotes")
682 };
683
684 assert_eq!(rows.as_ref(), &[quote, second]);
685 assert_eq!(staged, usize::from(!delete_source));
686 }
687
688 #[rstest]
689 fn parquet_writer_recovers_partial_files_at_startup() {
690 let directory = TempDir::new().unwrap();
691 let staging = directory.path().join("backtest").join("run-recover");
692 let quote = sample_quote();
693
694 let mut crashed = FeatherWriter::new(
696 staging.clone(),
697 WriterClock::Test(Arc::new(AtomicU64::new(0))),
698 RotationConfig::NoRotation,
699 None,
700 None,
701 );
702 crashed.write(quote).unwrap();
703 crashed.close().unwrap();
704 let sealed = staging.join("quotes").join("quotes_0.feather");
705 std::fs::rename(&sealed, sealed.with_extension(FEATHER_PARTIAL_EXTENSION)).unwrap();
706
707 let config =
708 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
709 let mut sink =
710 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
711 .unwrap();
712 sink.close().unwrap();
713
714 let mut catalog = ParquetDataCatalog::from_uri(
715 directory.path().to_str().unwrap(),
716 None,
717 None,
718 None,
719 None,
720 )
721 .unwrap();
722 let rows = catalog
723 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
724 .unwrap();
725
726 let DataBatch::Quote(rows) = rows else {
727 panic!("expected quotes")
728 };
729
730 assert_eq!(rows.as_ref(), &[quote]);
731 }
732
733 #[rstest]
734 fn parquet_close_promotes_per_identifier_file_with_non_ascii_identifier(
735 #[values(false, true)] delete_source: bool,
736 ) {
737 let directory = TempDir::new().unwrap();
738 let staging = directory.path().join("backtest").join("run-1");
739 let (quote, staged_file) = stage_per_identifier_quote(&staging);
740 let mut config =
741 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
742 config.delete_feather_after_promotion = delete_source;
743 let mut sink =
744 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
745 .unwrap();
746
747 sink.close().unwrap();
748
749 let mut catalog = ParquetDataCatalog::from_uri(
750 directory.path().to_str().unwrap(),
751 None,
752 None,
753 None,
754 None,
755 )
756 .unwrap();
757
758 let DataBatch::Quote(rows) = catalog
759 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
760 .unwrap()
761 else {
762 panic!("expected quotes")
763 };
764
765 assert_eq!(rows.as_ref(), &[quote]);
766 assert_eq!(staged_file.exists(), !delete_source);
767 }
768
769 #[rstest]
770 fn parquet_manual_conversion_reads_per_identifier_file_with_non_ascii_identifier() {
771 let directory = TempDir::new().unwrap();
772 let staging = directory.path().join("backtest").join("run-1");
773 let (quote, _) = stage_per_identifier_quote(&staging);
774 let mut catalog = ParquetDataCatalog::from_uri(
775 directory.path().to_str().unwrap(),
776 None,
777 None,
778 None,
779 None,
780 )
781 .unwrap();
782
783 catalog
784 .convert_stream_to_data(
785 "run-1",
786 &NautilusDataType::QuoteTick.into(),
787 Environment::Backtest,
788 None,
789 false,
790 )
791 .unwrap();
792
793 let DataBatch::Quote(rows) = catalog
794 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
795 .unwrap()
796 else {
797 panic!("expected quotes")
798 };
799
800 assert_eq!(rows.as_ref(), &[quote]);
801 }
802
803 #[rstest]
804 fn parquet_promotion_writes_non_ascii_identifier_to_object_store() {
805 let directory = TempDir::new().unwrap();
806 let staging = directory.path().join("backtest").join("run-1");
807 let (quote, staged_file) = stage_per_identifier_quote(&staging);
808
809 let source = FeatherSessionSource::new(
810 create_storage_backend_from_path(directory.path().to_str().unwrap(), None).unwrap(),
811 Environment::Backtest,
812 "run-1",
813 );
814 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
815 catalog.base_path = "catalog".to_string();
816 catalog.original_uri = "s3://test-bucket/catalog".to_string();
817 catalog.object_store = Arc::new(InMemory::new());
818
819 let mut backend = ParquetPromotionBackend {
820 catalog,
821 legacy_manifest_missing: Arc::default(),
822 };
823
824 let files =
825 list_session_feather_files(&source.storage, Environment::Backtest, "run-1").unwrap();
826
827 for file in &files {
828 backend.convert_file(&source, file, false, true).unwrap();
829 backend.delete_file(&source, file).unwrap();
830 }
831
832 let DataBatch::Quote(rows) = backend
833 .catalog
834 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
835 .unwrap()
836 else {
837 panic!("expected quotes")
838 };
839
840 assert_eq!(files.len(), 1);
841 assert_eq!(rows.as_ref(), &[quote]);
842 assert!(!staged_file.exists());
843 }
844
845 #[rstest]
846 #[case("backtest")]
847 #[case("sandbox")]
848 #[case("live")]
849 fn parquet_close_promotes_into_missing_local_catalog(#[case] environment: &str) {
850 let directory = TempDir::new().unwrap();
851 let staging = directory
852 .path()
853 .join("stream")
854 .join(environment)
855 .join("run-1");
856 let catalog_path = directory.path().join("charts").join("catalog");
857
858 let config = WriterConnectConfig::new(
859 staging.to_string_lossy(),
860 Some(CatalogConnectConfig::new(
861 catalog_path.to_string_lossy(),
862 None,
863 )),
864 );
865 let mut sink =
866 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
867 .unwrap();
868 let quote = sample_quote();
869
870 sink.write_data(Data::Quote(quote)).unwrap();
871 sink.close().unwrap();
872
873 let mut catalog =
874 ParquetDataCatalog::from_uri(catalog_path.to_str().unwrap(), None, None, None, None)
875 .unwrap();
876
877 let DataBatch::Quote(rows) = catalog
878 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
879 .unwrap()
880 else {
881 panic!("expected quotes")
882 };
883
884 let manifest: serde_json::Value =
885 serde_json::from_slice(&std::fs::read(staging.join(RUN_MANIFEST_FILENAME)).unwrap())
886 .unwrap();
887
888 assert_eq!(rows.as_ref(), &[quote]);
889 assert_eq!(manifest["kind"], environment);
890 assert_eq!(manifest["status"], "promoted");
891 assert!(!catalog_path.join(environment).exists());
892 }
893
894 #[rstest]
895 #[case::non_ascii("run-é")]
896 #[case::space("run 1")]
897 #[case::fragment("run#1")]
898 fn parquet_writer_rejects_run_id_that_paths_encode(#[case] run_id: &str) {
899 let directory = TempDir::new().unwrap();
900 let staging = directory.path().join("backtest").join(run_id);
901 let config =
902 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
903
904 let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
905 .unwrap_err();
906
907 let staging_uri = normalize_path_to_uri(&staging.to_string_lossy()).unwrap();
908 assert_eq!(
909 error.to_string(),
910 format!(
911 "Parquet writer URI '{staging_uri}' has a run ID that object-store paths \
912 percent-encode; use a run ID without spaces, non-ASCII, or reserved characters",
913 )
914 );
915 }
916
917 #[rstest]
918 #[case::windows_separators(r"file:///C:\catalog\backtest\run-1", "file:///C:/catalog")]
919 #[case::windows_trailing_separator(r"file:///C:\catalog\backtest\run-1\", "file:///C:/catalog")]
920 #[case::trailing_separator("file:///tmp/catalog/backtest/run-1/", "file:///tmp/catalog")]
921 #[case::parent_components(
922 "file:///tmp/unused/../catalog/backtest/run-1",
923 "file:///tmp/catalog"
924 )]
925 fn parquet_writer_session_accepts_separator_forms(#[case] uri: &str, #[case] root_uri: &str) {
926 let session = StagedFeatherWriter::<ParquetPromotionBackend>::required_session(
927 uri,
928 "Parquet writer URI",
929 )
930 .unwrap();
931
932 assert_eq!(
933 session,
934 PromotionSession {
935 root_uri: root_uri.to_string(),
936 environment: Environment::Backtest,
937 instance_id: "run-1".to_string(),
938 }
939 );
940 }
941
942 #[rstest]
943 fn parquet_writer_requires_a_catalog() {
944 let directory = TempDir::new().unwrap();
945 let staging = directory.path().join("backtest").join("run-1");
946 let config = WriterConnectConfig::new(staging.to_string_lossy(), None);
947
948 let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
949 .unwrap_err();
950
951 assert_eq!(
952 error.to_string(),
953 "Parquet writer requires a promotion catalog"
954 );
955 assert!(!staging.exists());
956 }
957
958 #[cfg(feature = "cloud")]
961 #[rstest]
962 fn parquet_writer_stages_locally_for_a_remote_catalog() {
963 let directory = TempDir::new().unwrap();
964 let staging = directory.path().join("backtest").join("run-1");
965
966 let config = WriterConnectConfig::new(
967 staging.to_string_lossy(),
968 Some(CatalogConnectConfig::new("s3://bucket/catalog", None)),
969 );
970
971 let mut sink =
972 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
973 .unwrap();
974 sink.close().unwrap();
975 drop(sink);
976
977 let manifest: serde_json::Value =
978 serde_json::from_slice(&std::fs::read(staging.join(RUN_MANIFEST_FILENAME)).unwrap())
979 .unwrap();
980 assert_eq!(manifest["kind"], "backtest");
981 assert_eq!(manifest["instance_id"], "run-1");
982 assert_eq!(manifest["status"], "completed");
983 assert_eq!(manifest["empty"], true);
984 }
985
986 #[rstest]
987 fn parquet_promotion_writes_the_direct_write_schema() {
988 let directory = TempDir::new().unwrap();
989
990 let config = WriterConnectConfig::new(
991 directory
992 .path()
993 .join("backtest/run-schema")
994 .to_string_lossy(),
995 Some(local_catalog(&directory)),
996 );
997 let mut sink =
998 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
999 .unwrap();
1000 let quote = sample_quote();
1001 sink.write_data(Data::Quote(quote)).unwrap();
1002 sink.close().unwrap();
1003
1004 let promoted = ParquetDataCatalog::from_uri(
1005 directory.path().to_str().unwrap(),
1006 None,
1007 None,
1008 None,
1009 None,
1010 )
1011 .unwrap();
1012 let direct_directory = TempDir::new().unwrap();
1013 let direct = ParquetDataCatalog::from_uri(
1014 direct_directory.path().to_str().unwrap(),
1015 None,
1016 None,
1017 None,
1018 None,
1019 )
1020 .unwrap();
1021 direct.write_to_parquet(&[quote], None, None, None).unwrap();
1022
1023 assert_eq!(
1024 quote_file_schema(&promoted).fields(),
1025 quote_file_schema(&direct).fields(),
1026 );
1027 }
1028
1029 #[rstest]
1030 fn parquet_promotion_honors_catalog_settings() {
1031 let directory = TempDir::new().unwrap();
1032 let staging = directory.path().join("backtest").join("run-1");
1033 let catalog = DataCatalogConfig::builder()
1034 .path(directory.path().to_string_lossy().to_string())
1035 .batch_size(7)
1036 .compression(CatalogCompression::Uncompressed)
1037 .max_row_group_size(1)
1038 .build()
1039 .unwrap()
1040 .connect_config();
1041 let backend =
1042 ParquetPromotionBackend::new(&catalog, Arc::new(AtomicBool::new(false))).unwrap();
1043
1044 let config = WriterConnectConfig::new(staging.to_string_lossy(), Some(catalog));
1045 let mut sink =
1046 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1047 .unwrap();
1048 let quote = sample_quote();
1049 sink.write_data(Data::Quote(quote)).unwrap();
1050 sink.write_data(Data::Quote(QuoteTick {
1051 ts_event: UnixNanos::from(29),
1052 ts_init: UnixNanos::from(31),
1053 ..quote
1054 }))
1055 .unwrap();
1056
1057 sink.close().unwrap();
1058
1059 let files = ParquetDataCatalog::new(directory.path(), None, None, None, None)
1060 .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
1061 .unwrap();
1062 let reader = SerializedFileReader::new(
1063 std::fs::File::open(directory.path().join(&files[0])).unwrap(),
1064 )
1065 .unwrap();
1066 let metadata = reader.metadata();
1067
1068 assert_eq!(backend.catalog.batch_size, 7);
1069 assert_eq!(files.len(), 1);
1070 assert_eq!(metadata.num_row_groups(), 2);
1071 assert!(
1072 metadata
1073 .row_groups()
1074 .iter()
1075 .flat_map(RowGroupMetaData::columns)
1076 .all(|column| column.compression() == Compression::UNCOMPRESSED)
1077 );
1078 }
1079
1080 #[rstest]
1081 fn parquet_writer_rejects_unknown_catalog_param() {
1082 let directory = TempDir::new().unwrap();
1083 let staging = directory.path().join("backtest").join("run-1");
1084 let mut catalog = local_catalog(&directory);
1085 let params = Params::new([("compression".to_string(), serde_json::json!(0))].into());
1086 catalog.params = Some(params);
1087
1088 let config = WriterConnectConfig::new(staging.to_string_lossy(), Some(catalog));
1089 let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1090 .unwrap_err();
1091
1092 assert_eq!(
1093 error.to_string(),
1094 "Unknown Parquet catalog param 'compression': this catalog takes no params"
1095 );
1096 }
1097
1098 fn quote_file_schema(catalog: &ParquetDataCatalog) -> SchemaRef {
1099 let files = catalog
1100 .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
1101 .unwrap();
1102 assert_eq!(files.len(), 1);
1103
1104 let path = ObjectPath::from(files[0].as_str());
1105
1106 let (_, schema) = block_on_nautilus_with(|| {
1107 read_parquet_from_object_store(catalog.object_store.clone(), &path)
1108 })
1109 .unwrap();
1110
1111 schema
1112 }
1113
1114 #[rstest]
1115 #[case(None)]
1116 #[case(Some(0))]
1117 #[case(Some(1))]
1118 fn parquet_interval_uses_test_clock_and_close_can_leave_staged_data(
1119 #[case] interval: Option<u64>,
1120 ) {
1121 let directory = TempDir::new().unwrap();
1122 let staging = directory.path().join("backtest").join("run-2");
1123 let mut config =
1124 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1125 config.promotion_interval_ms = interval;
1126 config.promote_on_close = false;
1127 let clock = Arc::new(AtomicU64::new(0));
1128 let mut sink =
1129 parquet_writer_factory(&config, WriterClock::Test(Arc::clone(&clock))).unwrap();
1130 let quote = sample_quote();
1131 sink.write_data(Data::Quote(quote)).unwrap();
1132 clock.store(1_000_000, Ordering::Relaxed);
1133 sink.flush().unwrap();
1134 sink.close().unwrap();
1135 let mut catalog = ParquetDataCatalog::from_uri(
1136 directory.path().to_str().unwrap(),
1137 None,
1138 None,
1139 None,
1140 None,
1141 )
1142 .unwrap();
1143 let before_manual = catalog
1144 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
1145 .unwrap();
1146
1147 if interval != Some(1) {
1148 catalog
1149 .convert_stream_to_data(
1150 "run-2",
1151 &NautilusDataType::QuoteTick.into(),
1152 Environment::Backtest,
1153 None,
1154 false,
1155 )
1156 .unwrap();
1157 }
1158
1159 let after = catalog
1160 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
1161 .unwrap();
1162 assert_eq!(before_manual.len(), usize::from(interval == Some(1)));
1163
1164 let DataBatch::Quote(rows) = after else {
1165 panic!("expected quotes")
1166 };
1167
1168 assert_eq!(rows.as_ref(), &[quote]);
1169 }
1170
1171 #[rstest]
1172 fn parquet_streaming_funding_round_trip() {
1173 use nautilus_model::data::FundingRateUpdate;
1174 let directory = TempDir::new().unwrap();
1175
1176 let config = WriterConnectConfig::new(
1177 directory
1178 .path()
1179 .join("backtest/run-funding")
1180 .to_string_lossy(),
1181 Some(local_catalog(&directory)),
1182 );
1183 let mut sink =
1184 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1185 .unwrap();
1186
1187 let funding = FundingRateUpdate::new(
1188 InstrumentId::from("AUD/USD.SIM"),
1189 "0.0012".parse().unwrap(),
1190 Some(480),
1191 Some(789.into()),
1192 123.into(),
1193 456.into(),
1194 );
1195 assert!(sink.write_any(&funding).unwrap());
1196 sink.close().unwrap();
1197 let mut catalog = ParquetDataCatalog::from_uri(
1198 directory.path().to_str().unwrap(),
1199 None,
1200 None,
1201 None,
1202 None,
1203 )
1204 .unwrap();
1205 let rows = catalog
1206 .query_batch(&CatalogQuery::new(NautilusDataType::FundingRateUpdate))
1207 .unwrap();
1208
1209 let DataBatch::FundingRate(rows) = rows else {
1210 panic!("expected funding rates")
1211 };
1212
1213 assert_eq!(rows.as_ref(), &[funding]);
1214 }
1215
1216 #[rstest]
1217 fn parquet_streaming_write_error_reaches_flush() {
1218 let directory = TempDir::new().unwrap();
1219
1220 let config = WriterConnectConfig::new(
1221 directory
1222 .path()
1223 .join("backtest/run-error")
1224 .to_string_lossy(),
1225 Some(local_catalog(&directory)),
1226 );
1227 let mut sink =
1228 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1229 .unwrap();
1230 let mut quote = sample_quote();
1231 quote.bid_price = ERROR_PRICE;
1232 let write_error = sink.write_any("e).unwrap_err().to_string();
1233 let flush_error = sink.flush().unwrap_err().to_string();
1234 assert_eq!(flush_error, write_error);
1235 }
1236
1237 #[rstest]
1238 fn parquet_streaming_instrument_promotes() {
1239 use nautilus_model::instruments::{InstrumentAny, stubs::audusd_sim};
1240 let directory = TempDir::new().unwrap();
1241
1242 let config = WriterConnectConfig::new(
1243 directory
1244 .path()
1245 .join("backtest/run-instrument")
1246 .to_string_lossy(),
1247 Some(local_catalog(&directory)),
1248 );
1249 let mut sink =
1250 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1251 .unwrap();
1252 let instrument = InstrumentAny::CurrencyPair(audusd_sim());
1253 assert!(sink.write_any(&instrument).unwrap());
1254 sink.close().unwrap();
1255 let mut catalog = ParquetDataCatalog::from_uri(
1256 directory.path().to_str().unwrap(),
1257 None,
1258 None,
1259 None,
1260 None,
1261 )
1262 .unwrap();
1263 let rows = catalog
1264 .query_batch(&CatalogQuery::new(NautilusDataType::Instrument))
1265 .unwrap();
1266
1267 let DataBatch::Instrument(rows) = rows else {
1268 panic!("expected instruments")
1269 };
1270
1271 assert_eq!(rows.as_ref(), &[instrument]);
1272 }
1273
1274 #[rstest]
1275 fn parquet_promotion_preserves_every_record_family(#[values(false, true)] automatic: bool) {
1276 use nautilus_core::UUID4;
1277 use nautilus_model::{
1278 enums::{
1279 LiquiditySide, OrderSide, OrderStatus, OrderType, PositionAdjustmentType,
1280 PositionSide, TimeInForce,
1281 },
1282 events::{
1283 OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled, OrderDenied,
1284 OrderEmulated, OrderExpired, OrderInitialized, OrderModifyDenied,
1285 OrderModifyRejected, OrderPendingCancel, OrderPendingUpdate, OrderRejected,
1286 OrderReleased, OrderSnapshot, OrderSubmitted, OrderTriggered, OrderUpdated,
1287 PositionAdjusted, PositionChanged, PositionClosed, PositionOpened,
1288 PositionSnapshot,
1289 account::stubs::cash_account_state,
1290 order::spec::{OrderFillVoidedSpec, OrderFilledSpec},
1291 },
1292 identifiers::{
1293 AccountId, ClientId, ClientOrderId, PositionId, TradeId, Venue, VenueOrderId,
1294 },
1295 instruments::{InstrumentAny, stubs::audusd_sim},
1296 orders::OrderTestBuilder,
1297 position::Position,
1298 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
1299 stubs::TestDefault,
1300 types::Money,
1301 };
1302 use strum::IntoEnumIterator;
1303 let mut families = ahash::AHashSet::new();
1304 macro_rules! promote {
1305 ($record:expr) => {
1306 families.insert(assert_record_promotion($record, automatic));
1307 };
1308 }
1309
1310 promote!(cash_account_state());
1311 promote!(OrderInitialized::test_default());
1312 promote!(OrderDenied::test_default());
1313 promote!(OrderEmulated::test_default());
1314 promote!(OrderSubmitted::test_default());
1315 promote!(OrderAccepted::test_default());
1316 promote!(OrderRejected::test_default());
1317 promote!(OrderPendingCancel::test_default());
1318 promote!(OrderCanceled::test_default());
1319 let mut cancel_denied = OrderCancelDenied::test_default();
1320 cancel_denied.causation_id = Some(UUID4::new());
1321 promote!(cancel_denied);
1322 promote!(OrderCancelRejected::test_default());
1323 promote!(OrderExpired::test_default());
1324 promote!(OrderTriggered::test_default());
1325 promote!(OrderPendingUpdate::test_default());
1326 promote!(OrderReleased::test_default());
1327 let mut modify_denied = OrderModifyDenied::test_default();
1328 modify_denied.causation_id = Some(UUID4::new());
1329 promote!(modify_denied);
1330 promote!(OrderModifyRejected::test_default());
1331 promote!(OrderUpdated::test_default());
1332 promote!(OrderFillVoidedSpec::builder().is_reopened(true).build());
1333 let id = InstrumentId::from("AUD/USD.SIM");
1334 let fill = OrderFilledSpec::builder()
1335 .instrument_id(id)
1336 .position_id(PositionId::from("P-009"))
1337 .last_px(Price::from("1.23456"))
1338 .last_qty(Quantity::from("1234"))
1339 .ts_event(11.into())
1340 .ts_init(13.into())
1341 .build();
1342 promote!(fill.clone());
1343 let mut position = Position::new(&InstrumentAny::CurrencyPair(audusd_sim()), fill.clone());
1344 promote!(PositionOpened::create(
1345 &position,
1346 &fill,
1347 UUID4::default(),
1348 17.into()
1349 ));
1350 promote!(PositionChanged::create(
1351 &position,
1352 &fill,
1353 UUID4::default(),
1354 19.into()
1355 ));
1356 promote!(PositionAdjusted::new(
1357 fill.trader_id,
1358 fill.strategy_id,
1359 id,
1360 position.id,
1361 fill.account_id,
1362 PositionAdjustmentType::Funding,
1363 None,
1364 Some(Money::from("1.23 USD")),
1365 Some("funding".into()),
1366 UUID4::default(),
1367 23.into(),
1368 29.into()
1369 ));
1370 promote!(PositionSnapshot::from(
1371 &position,
1372 Some(Money::from("4.56 USD"))
1373 ));
1374 let closing = OrderFilledSpec::builder()
1375 .instrument_id(id)
1376 .position_id(position.id)
1377 .order_side(OrderSide::Sell)
1378 .last_px(Price::from("1.34567"))
1379 .last_qty(fill.last_qty)
1380 .trade_id(TradeId::from("T-011"))
1381 .ts_event(31.into())
1382 .ts_init(37.into())
1383 .build();
1384 position.apply(&closing);
1385 promote!(PositionClosed::create(
1386 &position,
1387 &closing,
1388 UUID4::default(),
1389 41.into()
1390 ));
1391 promote!(OrderSnapshot::from(
1392 OrderTestBuilder::new(OrderType::Limit)
1393 .instrument_id(id)
1394 .price(Price::from("1.45678"))
1395 .quantity(Quantity::from("5678"))
1396 .build()
1397 ));
1398
1399 let order = OrderStatusReport::new(
1400 AccountId::from("SIM-007"),
1401 id,
1402 Some(ClientOrderId::from("O-013")),
1403 VenueOrderId::from("V-017"),
1404 Some(OrderSide::Buy),
1405 OrderType::Limit,
1406 TimeInForce::Gtc,
1407 OrderStatus::PartiallyFilled,
1408 Quantity::from("123"),
1409 Quantity::from("45"),
1410 43.into(),
1411 47.into(),
1412 53.into(),
1413 None,
1414 );
1415
1416 let report_fill = FillReport::new(
1417 AccountId::from("SIM-007"),
1418 id,
1419 VenueOrderId::from("V-017"),
1420 TradeId::from("T-019"),
1421 OrderSide::Buy,
1422 Quantity::from("45"),
1423 Price::from("1.56789"),
1424 Money::from("2.34 USD"),
1425 LiquiditySide::Maker,
1426 Some(ClientOrderId::from("O-013")),
1427 Some(PositionId::from("P-023")),
1428 59.into(),
1429 61.into(),
1430 None,
1431 );
1432
1433 let report_position = PositionStatusReport::new(
1434 AccountId::from("SIM-007"),
1435 id,
1436 PositionSide::Long,
1437 Quantity::from("789"),
1438 67.into(),
1439 71.into(),
1440 None,
1441 Some(PositionId::from("P-023")),
1442 Some(rust_decimal::Decimal::new(16789, 4)),
1443 );
1444
1445 let mut mass = ExecutionMassStatus::new(
1446 ClientId::from("CLIENT-029"),
1447 AccountId::from("SIM-007"),
1448 Venue::from("SIM"),
1449 73.into(),
1450 None,
1451 );
1452 mass.add_order_reports(vec![order.clone()]);
1453 mass.add_fill_reports(vec![report_fill.clone()]);
1454 mass.add_position_reports(vec![report_position.clone()]);
1455 promote!(order);
1456 promote!(report_fill);
1457 promote!(report_position);
1458 promote!(mass);
1459
1460 let expected = NautilusRecordType::iter()
1461 .filter(|family| {
1462 #[cfg(feature = "defi")]
1463 if *family == NautilusRecordType::Defi {
1464 return false;
1465 }
1466
1467 let _ = family;
1468 true
1469 })
1470 .collect::<ahash::AHashSet<_>>();
1471
1472 assert_eq!(families, expected);
1473 }
1474
1475 #[rstest]
1476 fn parquet_promotion_preserves_calculated_greeks_as_custom_data(
1477 #[values(false, true)] automatic: bool,
1478 #[values(false, true)] portfolio: bool,
1479 ) {
1480 use nautilus_model::data::{
1481 CustomData, DataType,
1482 greeks::{GreeksData, OptionGreekValues, PortfolioGreeks},
1483 };
1484
1485 let greeks = GreeksData::new(
1486 "ESZ26 C7000.XCME".parse().unwrap(),
1487 false,
1488 7000.5,
1489 20_261_218,
1490 68,
1491 0.186,
1492 50.0,
1493 -3.0,
1494 6950.25,
1495 0.0425,
1496 0.0125,
1497 0.175,
1498 12.5,
1499 98.75,
1500 OptionGreekValues {
1501 delta: 0.41,
1502 gamma: 0.0012,
1503 vega: 11.2,
1504 theta: -2.3,
1505 rho: 1.7,
1506 },
1507 0.62,
1508 11.into(),
1509 13.into(),
1510 );
1511
1512 let mut second = greeks.clone();
1513 second.ts_init = 113.into();
1514 second.instrument_id = "ESZ26 P6900.XCME".parse().unwrap();
1515
1516 let greeks = vec![greeks, second]
1517 .into_iter()
1518 .map(|value| {
1519 let data_type =
1520 DataType::new("GreeksData", None, Some(value.instrument_id.to_string()));
1521 CustomData::new(Arc::new(value), data_type)
1522 })
1523 .collect::<Vec<_>>();
1524
1525 let portfolio_value = PortfolioGreeks {
1526 pnl: -125.5,
1527 price: 4310.25,
1528 greeks: OptionGreekValues {
1529 delta: 0.83,
1530 gamma: 0.004,
1531 vega: 45.5,
1532 theta: -12.25,
1533 rho: 3.75,
1534 },
1535 ts_event: 19.into(),
1536 ts_init: 17.into(),
1537 };
1538
1539 let mut second = portfolio_value.clone();
1540 second.ts_init = 117.into();
1541 let portfolios = vec![portfolio_value, second]
1542 .into_iter()
1543 .map(|value| CustomData::from_arc(Arc::new(value)))
1544 .collect::<Vec<_>>();
1545
1546 let expected = if portfolio { portfolios } else { greeks };
1547
1548 let directory = TempDir::new().unwrap();
1549 let staging = directory.path().join("backtest/run-greeks-custom");
1550 let mut config =
1551 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1552 config.promote_on_close = automatic;
1553 let mut sink =
1554 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1555 .unwrap();
1556
1557 for custom in &expected {
1558 assert!(sink.write_any(custom.data.as_any()).unwrap());
1559 }
1560
1561 sink.close().unwrap();
1562
1563 let data_type = NautilusDataType::Custom {
1564 type_name: expected[0].data_type.type_name().to_string(),
1565 };
1566
1567 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1568 if !automatic {
1569 catalog
1570 .convert_stream_to_data(
1571 "run-greeks-custom",
1572 &data_type.clone().into(),
1573 Environment::Backtest,
1574 None,
1575 false,
1576 )
1577 .unwrap();
1578 }
1579
1580 let DataBatch::Custom(actual) = catalog.query_batch(&CatalogQuery::new(data_type)).unwrap()
1581 else {
1582 panic!("expected custom data")
1583 };
1584
1585 assert_eq!(actual.as_ref(), expected.as_slice());
1586 }
1587
1588 fn assert_record_promotion<T>(record: T, automatic: bool) -> NautilusRecordType
1589 where
1590 T: std::any::Any
1591 + serde::Serialize
1592 + serde::de::DeserializeOwned
1593 + EncodeToRecordBatch
1594 + DecodeTypedFromRecordBatch,
1595 {
1596 let directory = TempDir::new().unwrap();
1597 let staging = directory.path().join("backtest/run-record");
1598 let mut config =
1599 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1600 config.promote_on_close = automatic;
1601 let mut sink =
1602 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1603 .unwrap();
1604 let mut second = serde_json::to_value(&record).unwrap();
1605 second["ts_init"] = serde_json::json!(second["ts_init"].as_u64().unwrap() + 101);
1606 let second: T = serde_json::from_value(second).unwrap();
1607 assert!(sink.write_any(&record).unwrap());
1608 assert!(sink.write_any(&second).unwrap());
1609 sink.close().unwrap();
1610 let record_type = T::get_schema(None).metadata()["type_name"]
1611 .parse::<NautilusRecordType>()
1612 .unwrap();
1613 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1614 if !automatic {
1615 catalog
1616 .convert_stream_to_data(
1617 "run-record",
1618 &record_type.into(),
1619 Environment::Backtest,
1620 None,
1621 false,
1622 )
1623 .unwrap();
1624 }
1625
1626 let batches = catalog
1627 .query_record_batches(&record_type.into(), None, None, None, None, true)
1628 .unwrap();
1629
1630 let actual = batches
1631 .into_iter()
1632 .flat_map(|batch| {
1633 let schema = batch.schema();
1634 T::decode_typed_batch(schema.metadata(), batch).unwrap()
1635 })
1636 .map(|record| serde_json::to_value(record).unwrap())
1637 .collect::<Vec<_>>();
1638
1639 assert_eq!(
1640 actual,
1641 vec![
1642 serde_json::to_value(record).unwrap(),
1643 serde_json::to_value(second).unwrap()
1644 ]
1645 );
1646 record_type
1647 }
1648
1649 #[rstest]
1650 fn parquet_streaming_voided_fill_promotes() {
1651 use nautilus_model::events::{OrderFillVoided, order::spec::OrderFillVoidedSpec};
1652 use nautilus_serialization::arrow::DecodeTypedFromRecordBatch;
1653 let directory = TempDir::new().unwrap();
1654
1655 let config = WriterConnectConfig::new(
1656 directory.path().join("backtest/run-void").to_string_lossy(),
1657 Some(local_catalog(&directory)),
1658 );
1659 let mut sink =
1660 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1661 .unwrap();
1662 let event = OrderFillVoidedSpec::builder().is_reopened(true).build();
1663 assert!(sink.write_any(&event).unwrap());
1664 sink.close().unwrap();
1665 let mut catalog = ParquetDataCatalog::from_uri(
1666 directory.path().to_str().unwrap(),
1667 None,
1668 None,
1669 None,
1670 None,
1671 )
1672 .unwrap();
1673 let batches = catalog
1674 .query_record_batches(
1675 &NautilusRecordType::OrderFillVoided.into(),
1676 None,
1677 None,
1678 None,
1679 None,
1680 true,
1681 )
1682 .unwrap();
1683 let mut rows = Vec::new();
1684 for batch in batches {
1685 rows.extend(
1686 OrderFillVoided::decode_typed_batch(batch.schema().metadata(), batch.clone())
1687 .unwrap(),
1688 );
1689 }
1690
1691 assert_eq!(rows, vec![event]);
1692 }
1693
1694 #[rstest]
1695 fn parquet_promotion_preserves_clear_source_order(
1696 #[values(false, true)] automatic: bool,
1697 #[values(false, true)] batched: bool,
1698 #[values(false, true)] undefined_price: bool,
1699 ) {
1700 use nautilus_model::{
1701 data::{BookOrder, OrderBookDelta},
1702 enums::{BookAction, OrderSide, RecordFlag},
1703 types::PRICE_UNDEF,
1704 };
1705 let directory = TempDir::new().unwrap();
1706 let staging = directory.path().join("backtest/run-clear");
1707 let mut config =
1708 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1709 config.promote_on_close = automatic;
1710 let mut sink =
1711 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1712 .unwrap();
1713 let id = InstrumentId::from("AUD/USD.SIM");
1714 let ts = UnixNanos::from(1_700_000_000_000_000_123);
1715 let mut clear = OrderBookDelta::clear(id, 7, ts, ts);
1716 if undefined_price {
1717 clear.order.price = Price::from_raw(PRICE_UNDEF, 0);
1718 }
1719
1720 let add = OrderBookDelta::new(
1721 id,
1722 BookAction::Add,
1723 BookOrder::new(
1724 OrderSide::Buy,
1725 Price::from("1.23456"),
1726 Quantity::from("7.890"),
1727 19,
1728 ),
1729 RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8,
1730 7,
1731 ts,
1732 ts,
1733 );
1734 let mut second_add = add;
1735 second_add.order.order_id = 29;
1736 second_add.order.price = Price::from("1.34567");
1737 second_add.order.size = Quantity::from("8.901");
1738 let rows = vec![clear, add, clear, second_add];
1739 if batched {
1740 sink.write_batch(rows.iter().copied().map(Data::BookDelta).collect())
1741 .unwrap();
1742 } else {
1743 for row in &rows {
1744 sink.write_data(Data::BookDelta(*row)).unwrap();
1745 }
1746 }
1747
1748 sink.close().unwrap();
1749 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1750 let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
1751 let staged = block_on_nautilus_with(|| storage.list_files("", Some(".feather"))).unwrap();
1752 let mut originals = Vec::new();
1753
1754 for path in &staged {
1755 let bytes = block_on_nautilus_with(|| async {
1756 storage
1757 .object_store
1758 .get(&object_store::path::Path::from(path.as_str()))
1759 .await
1760 .unwrap()
1761 .bytes()
1762 .await
1763 .unwrap()
1764 });
1765
1766 originals.push(bytes);
1767 }
1768
1769 if !automatic {
1770 catalog
1771 .convert_stream_to_data(
1772 "run-clear",
1773 &NautilusDataType::OrderBookDelta.into(),
1774 Environment::Backtest,
1775 None,
1776 false,
1777 )
1778 .unwrap();
1779 }
1780
1781 let actual = catalog
1782 .query::<OrderBookDelta>(None, None, None, None, None, true)
1783 .unwrap();
1784 assert_eq!(actual.len(), rows.len());
1785
1786 for (actual, expected) in actual.iter().zip(&rows) {
1787 assert_eq!(actual.instrument_id, expected.instrument_id);
1788 assert_eq!(actual.action, expected.action);
1789 assert_eq!(actual.order.side, expected.order.side);
1790 assert_eq!(actual.order.price.raw(), expected.order.price.raw());
1791
1792 let price_precision = if expected.order.price.is_undefined() {
1793 0
1794 } else {
1795 5
1796 };
1797
1798 assert_eq!(actual.order.price.precision, price_precision);
1799 assert_eq!(
1800 actual.order.size.as_decimal(),
1801 expected.order.size.as_decimal()
1802 );
1803 assert_eq!(actual.order.size.precision, 3);
1804 assert_eq!(actual.order.order_id, expected.order.order_id);
1805 assert_eq!(actual.flags, expected.flags);
1806 assert_eq!(actual.sequence, expected.sequence);
1807 assert_eq!(actual.ts_event, expected.ts_event);
1808 assert_eq!(actual.ts_init, expected.ts_init);
1809 }
1810
1811 for (path, original) in staged.iter().zip(originals) {
1812 let bytes = block_on_nautilus_with(|| async {
1813 storage
1814 .object_store
1815 .get(&object_store::path::Path::from(path.as_str()))
1816 .await
1817 .unwrap()
1818 .bytes()
1819 .await
1820 .unwrap()
1821 });
1822
1823 assert_eq!(bytes, original);
1824 }
1825
1826 assert_eq!(staged.len(), 1);
1827 }
1828
1829 #[rstest]
1830 #[case::manual(false, false)]
1831 #[case::automatic_retained(true, false)]
1832 #[case::automatic_delete(true, true)]
1833 fn parquet_failed_clear_promotion_preserves_source(
1834 #[case] automatic: bool,
1835 #[case] delete_source: bool,
1836 #[values(false, true)] batched: bool,
1837 ) {
1838 use nautilus_model::{
1839 data::{BookOrder, OrderBookDelta},
1840 enums::{BookAction, OrderSide, RecordFlag},
1841 };
1842 let directory = TempDir::new().unwrap();
1843 let staging = directory.path().join("backtest/run-invalid-clear");
1844 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
1845
1846 let mut writer = FeatherWriter::new(
1847 staging.clone(),
1848 clock.clone(),
1849 RotationConfig::NoRotation,
1850 None,
1851 None,
1852 );
1853 let id = InstrumentId::from("AUD/USD.SIM");
1854 let ts = UnixNanos::from(23);
1855 let mut clear = OrderBookDelta::clear(id, 7, ts, ts);
1856 clear.order.price = Price::from("1");
1857
1858 let add = OrderBookDelta::new(
1859 id,
1860 BookAction::Add,
1861 BookOrder::new(
1862 OrderSide::Buy,
1863 Price::from("1.23456"),
1864 Quantity::from("7.890"),
1865 19,
1866 ),
1867 RecordFlag::F_LAST as u8,
1868 7,
1869 ts,
1870 ts,
1871 );
1872
1873 if batched {
1874 writer.write_batch(vec![clear, add]).unwrap();
1875 } else {
1876 writer.write(clear).unwrap();
1877 writer.write(add).unwrap();
1878 }
1879
1880 writer.close().unwrap();
1881 let source = staging.join("order_book_deltas/order_book_deltas_0.feather");
1882 let original = fs::read(&source).unwrap();
1883 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1884
1885 let failure = if automatic {
1886 let mut config = WriterConnectConfig::new(
1887 staging.to_string_lossy(),
1888 Some(local_catalog(&directory)),
1889 );
1890 config.delete_feather_after_promotion = delete_source;
1891 let mut sink = parquet_writer_factory(&config, clock).unwrap();
1892 sink.close().unwrap_err()
1893 } else {
1894 catalog
1895 .convert_stream_to_data(
1896 "run-invalid-clear",
1897 &NautilusDataType::OrderBookDelta.into(),
1898 Environment::Backtest,
1899 None,
1900 false,
1901 )
1902 .unwrap_err()
1903 };
1904
1905 assert!(
1906 format!("{failure:#}").contains(
1907 "cannot relabel precision metadata for a promotion group that contains decimal values"
1908 ),
1909 "{failure:#}"
1910 );
1911 assert_eq!(fs::read(source).unwrap(), original);
1912 assert_eq!(
1913 catalog
1914 .query_files(&NautilusDataType::OrderBookDelta.into(), None, None, None)
1915 .unwrap(),
1916 Vec::<String>::new()
1917 );
1918 }
1919
1920 #[rstest]
1921 #[case(true)]
1922 #[case(false)]
1923 fn parquet_promotion_unifies_empty_and_populated_depth(#[case] automatic: bool) {
1924 use nautilus_model::{
1925 data::{BookOrder, OrderBookDepth},
1926 enums::OrderSide,
1927 };
1928 let directory = TempDir::new().unwrap();
1929 let staging = directory.path().join("backtest/run-depth-ties");
1930 let mut config =
1931 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1932 config.delete_feather_after_promotion = true;
1933 config.promote_on_close = automatic;
1934 let mut sink =
1935 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1936 .unwrap();
1937 let id = InstrumentId::from("AUD/USD.SIM");
1938 let empty =
1939 OrderBookDepth::new(id, vec![], vec![], vec![], vec![], 1, 2, 3.into(), 4.into());
1940 let order = BookOrder::new(
1941 OrderSide::Buy,
1942 Price::from("1.23"),
1943 Quantity::from("4.5"),
1944 6,
1945 );
1946 let populated = OrderBookDepth::new(
1947 id,
1948 vec![order],
1949 vec![],
1950 vec![7],
1951 vec![],
1952 8,
1953 9,
1954 3.into(),
1955 4.into(),
1956 );
1957 sink.write_any(&empty).unwrap();
1958 sink.write_any(&populated).unwrap();
1959 sink.close().unwrap();
1960
1961 let mut catalog = ParquetDataCatalog::from_uri(
1962 directory.path().to_str().unwrap(),
1963 None,
1964 None,
1965 None,
1966 None,
1967 )
1968 .unwrap();
1969
1970 if !automatic {
1971 catalog
1972 .convert_stream_to_data(
1973 "run-depth-ties",
1974 &NautilusDataType::OrderBookDepth.into(),
1975 Environment::Backtest,
1976 None,
1977 false,
1978 )
1979 .unwrap();
1980 }
1981
1982 let batch = catalog
1983 .query_batch(&CatalogQuery::new(NautilusDataType::OrderBookDepth))
1984 .unwrap();
1985 let DataBatch::BookDepth(rows) = batch else {
1986 panic!("expected order book depth");
1987 };
1988 let mut rows = rows.as_ref().to_vec();
1989 rows.sort_by_key(|depth| depth.sequence);
1990 assert_eq!(rows, vec![empty, populated]);
1991
1992 let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
1993 let staged = block_on_nautilus_with(|| storage.list_files("", Some(".feather"))).unwrap();
1994 assert_eq!(staged.len(), usize::from(!automatic));
1995 }
1996
1997 #[rstest]
1998 fn parquet_promotion_preserves_clear_snapshots(
1999 #[values(false, true)] automatic: bool,
2000 #[values(false, true)] batch_write: bool,
2001 ) {
2002 let directory = TempDir::new().unwrap();
2003 let staging = directory.path().join("backtest").join("run-clear");
2004 let mut config =
2005 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2006 config.promote_on_close = automatic;
2007 let mut sink =
2008 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2009 .unwrap();
2010 let id = InstrumentId::from("AUD/USD.SIM");
2011 let mut deltas = Vec::new();
2012 for (sequence, timestamp, price, size) in [
2013 (41, 100, "1.23", "4.500"),
2014 (42, 200, "1.24", "6.700"),
2015 (43, 300, "1.2345", "8.90000"),
2016 ] {
2017 deltas.push(OrderBookDelta::clear(
2018 id,
2019 sequence,
2020 (timestamp - 1).into(),
2021 timestamp.into(),
2022 ));
2023
2024 for (side, order_id, flags) in [
2025 (OrderSide::Buy, 71, RecordFlag::F_SNAPSHOT as u8),
2026 (
2027 OrderSide::Sell,
2028 72,
2029 RecordFlag::F_SNAPSHOT as u8 | RecordFlag::F_LAST as u8,
2030 ),
2031 ] {
2032 deltas.push(OrderBookDelta::new(
2033 id,
2034 BookAction::Add,
2035 BookOrder::new(side, Price::from(price), Quantity::from(size), order_id),
2036 flags,
2037 sequence,
2038 (timestamp - 1).into(),
2039 timestamp.into(),
2040 ));
2041 }
2042 }
2043
2044 deltas.push(OrderBookDelta::clear(id, 44, 399.into(), 400.into()));
2045 if batch_write {
2046 sink.write_batch(deltas.iter().copied().map(Data::from).collect())
2047 .unwrap();
2048 } else {
2049 for delta in &deltas {
2050 sink.write_data(Data::BookDelta(*delta)).unwrap();
2051 }
2052 }
2053
2054 sink.close().unwrap();
2055 let mut catalog = ParquetDataCatalog::from_uri(
2056 directory.path().to_str().unwrap(),
2057 None,
2058 None,
2059 None,
2060 None,
2061 )
2062 .unwrap();
2063
2064 if !automatic {
2065 catalog
2066 .convert_stream_to_data(
2067 "run-clear",
2068 &NautilusDataType::OrderBookDelta.into(),
2069 Environment::Backtest,
2070 None,
2071 false,
2072 )
2073 .unwrap();
2074 }
2075
2076 let batch = catalog
2077 .query_batch(&CatalogQuery::new(NautilusDataType::OrderBookDelta))
2078 .unwrap();
2079
2080 let DataBatch::BookDelta(rows) = batch else {
2081 panic!("expected order book deltas");
2082 };
2083
2084 assert_eq!(rows.len(), deltas.len());
2085
2086 for (actual, expected) in rows.as_ref().iter().zip(&deltas) {
2087 let (price_precision, size_precision) = if expected.sequence < 43 {
2088 (2, 3)
2089 } else {
2090 (4, 5)
2091 };
2092
2093 assert_eq!(actual.instrument_id, expected.instrument_id);
2094 assert_eq!(actual.action, expected.action);
2095 assert_eq!(actual.order.side, expected.order.side);
2096 assert_eq!(actual.order.price.raw(), expected.order.price.raw());
2097 assert_eq!(actual.order.price.precision, price_precision);
2098 assert_eq!(actual.order.size.raw(), expected.order.size.raw());
2099 assert_eq!(actual.order.size.precision, size_precision);
2100 assert_eq!(actual.order.order_id, expected.order.order_id);
2101 assert_eq!(actual.flags, expected.flags);
2102 assert_eq!(actual.sequence, expected.sequence);
2103 assert_eq!(actual.ts_event, expected.ts_event);
2104 assert_eq!(actual.ts_init, expected.ts_init);
2105 }
2106 }
2107
2108 #[rstest]
2109 #[case::real_add(BookAction::Add, Some(OrderSide::Buy), "1", "2", 0)]
2110 #[case::clear_side(BookAction::Clear, Some(OrderSide::Buy), "0", "0", 0)]
2111 #[case::clear_price(BookAction::Clear, None, "1", "0", 0)]
2112 #[case::clear_size(BookAction::Clear, None, "0", "1", 0)]
2113 #[case::clear_order_id(BookAction::Clear, None, "0", "0", 1)]
2114 fn parquet_promotion_rejects_noncanonical_zero_precision_orders(
2115 #[case] action: BookAction,
2116 #[case] side: Option<OrderSide>,
2117 #[case] price: &str,
2118 #[case] size: &str,
2119 #[case] order_id: u64,
2120 #[values(false, true)] automatic: bool,
2121 ) {
2122 let directory = TempDir::new().unwrap();
2123 let staging = directory.path().join("backtest").join("run-clear-invalid");
2124 let mut config =
2125 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2126 config.promote_on_close = automatic;
2127 let mut sink =
2128 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2129 .unwrap();
2130 let id = InstrumentId::from("AUD/USD.SIM");
2131 let deltas = vec![
2132 OrderBookDelta::clear(id, 41, 99.into(), 100.into()),
2133 OrderBookDelta::new(
2134 id,
2135 action,
2136 BookOrder {
2137 side,
2138 price: Price::from(price),
2139 size: Quantity::from(size),
2140 order_id,
2141 },
2142 0,
2143 41,
2144 99.into(),
2145 100.into(),
2146 ),
2147 OrderBookDelta::new(
2148 id,
2149 BookAction::Add,
2150 BookOrder::new(
2151 OrderSide::Sell,
2152 Price::from("1.23"),
2153 Quantity::from("4.500"),
2154 72,
2155 ),
2156 RecordFlag::F_LAST as u8,
2157 41,
2158 99.into(),
2159 100.into(),
2160 ),
2161 ];
2162 sink.write_batch(deltas.into_iter().map(Data::from).collect())
2163 .unwrap();
2164 let mut catalog = ParquetDataCatalog::from_uri(
2165 directory.path().to_str().unwrap(),
2166 None,
2167 None,
2168 None,
2169 None,
2170 )
2171 .unwrap();
2172
2173 let error = if automatic {
2174 sink.close().unwrap_err()
2175 } else {
2176 sink.close().unwrap();
2177 catalog
2178 .convert_stream_to_data(
2179 "run-clear-invalid",
2180 &NautilusDataType::OrderBookDelta.into(),
2181 Environment::Backtest,
2182 None,
2183 false,
2184 )
2185 .unwrap_err()
2186 };
2187
2188 assert!(error.to_string().contains(
2189 "cannot relabel precision metadata for a promotion group that contains decimal values"
2190 ));
2191 assert!(
2192 catalog
2193 .query_files(&NautilusDataType::OrderBookDelta.into(), None, None, None)
2194 .unwrap()
2195 .is_empty()
2196 );
2197 }
2198
2199 #[rstest]
2200 fn parquet_manual_conversion_promotes_custom_data() {
2201 use nautilus_model::data::{CustomData, DataType};
2202 use nautilus_serialization::ensure_custom_data_registered;
2203
2204 ensure_custom_data_registered::<RustTestHashMapCustomData>();
2205
2206 let directory = TempDir::new().unwrap();
2207 let staging = directory.path().join("backtest").join("run-custom");
2208 let mut config =
2209 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2210 config.promote_on_close = false;
2211 let mut sink =
2212 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2213 .unwrap();
2214
2215 let data_type = DataType::new("RustTestHashMapCustomData", None, None);
2216 let records = [
2217 sample_custom_data("first", "AUD/USD.SIM", "1.23456", 11),
2218 sample_custom_data("second", "BTCUSDT.BINANCE", "65432.10", 21),
2219 ];
2220
2221 for record in &records {
2222 sink.write_data(Data::Custom(CustomData::new(
2223 Arc::new(record.clone()),
2224 data_type.clone(),
2225 )))
2226 .unwrap();
2227 }
2228
2229 sink.close().unwrap();
2230
2231 let mut catalog = ParquetDataCatalog::from_uri(
2232 directory.path().to_str().unwrap(),
2233 None,
2234 None,
2235 None,
2236 None,
2237 )
2238 .unwrap();
2239
2240 for _ in 0..2 {
2241 catalog
2242 .convert_stream_to_data(
2243 "run-custom",
2244 &NautilusDataType::Custom {
2245 type_name: "RustTestHashMapCustomData".to_string(),
2246 }
2247 .into(),
2248 Environment::Backtest,
2249 None,
2250 false,
2251 )
2252 .unwrap();
2253 }
2254
2255 let expected: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2256 .iter()
2257 .map(|record| (None, record.clone()))
2258 .collect();
2259 assert_eq!(query_custom_records(&mut catalog, None), expected);
2260 }
2261
2262 #[rstest]
2263 fn parquet_manual_conversion_filters_custom_data_identifiers() {
2264 use nautilus_model::data::{CustomData, DataType};
2265 use nautilus_serialization::ensure_custom_data_registered;
2266
2267 ensure_custom_data_registered::<RustTestHashMapCustomData>();
2268
2269 let directory = TempDir::new().unwrap();
2270 let staging = directory.path().join("backtest").join("run-custom-ids");
2271 let mut config =
2272 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2273 config.promote_on_close = false;
2274 let mut sink =
2275 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2276 .unwrap();
2277
2278 let audusd = "AUD/USD.SIM";
2279 let btcusdt = "BTCUSDT.BINANCE";
2280 let records = [
2281 (audusd, sample_custom_data("first", audusd, "1.23456", 11)),
2282 (audusd, sample_custom_data("second", audusd, "1.23457", 21)),
2283 (
2284 btcusdt,
2285 sample_custom_data("third", btcusdt, "65432.10", 31),
2286 ),
2287 ];
2288
2289 for (identifier, record) in &records {
2290 let data_type = DataType::new(
2291 "RustTestHashMapCustomData",
2292 None,
2293 Some((*identifier).to_string()),
2294 );
2295 sink.write_data(Data::Custom(CustomData::new(
2296 Arc::new(record.clone()),
2297 data_type,
2298 )))
2299 .unwrap();
2300 }
2301
2302 sink.close().unwrap();
2303
2304 let mut catalog = ParquetDataCatalog::from_uri(
2305 directory.path().to_str().unwrap(),
2306 None,
2307 None,
2308 None,
2309 None,
2310 )
2311 .unwrap();
2312 catalog
2313 .convert_stream_to_data(
2314 "run-custom-ids",
2315 &NautilusDataType::Custom {
2316 type_name: "RustTestHashMapCustomData".to_string(),
2317 }
2318 .into(),
2319 Environment::Backtest,
2320 Some(&[audusd.to_string()]),
2321 false,
2322 )
2323 .unwrap();
2324
2325 let expected_audusd: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2326 .iter()
2327 .filter(|(identifier, _)| *identifier == audusd)
2328 .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2329 .collect();
2330 assert_eq!(query_custom_records(&mut catalog, None), expected_audusd);
2331
2332 assert_eq!(
2335 query_custom_records(&mut catalog, Some(&["AUDUSD.SIM".to_string()])),
2336 expected_audusd,
2337 );
2338
2339 catalog
2340 .convert_stream_to_data(
2341 "run-custom-ids",
2342 &NautilusDataType::Custom {
2343 type_name: "RustTestHashMapCustomData".to_string(),
2344 }
2345 .into(),
2346 Environment::Backtest,
2347 None,
2348 false,
2349 )
2350 .unwrap();
2351
2352 let expected_all: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2353 .iter()
2354 .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2355 .collect();
2356 assert_eq!(query_custom_records(&mut catalog, None), expected_all);
2357 let expected_btcusdt: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2358 .iter()
2359 .filter(|(identifier, _)| *identifier == btcusdt)
2360 .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2361 .collect();
2362 assert_eq!(
2363 query_custom_records(&mut catalog, Some(&[btcusdt.to_string()])),
2364 expected_btcusdt,
2365 );
2366 }
2367
2368 fn sample_custom_data(
2369 name: &str,
2370 instrument_id: &str,
2371 price: &str,
2372 ts: u64,
2373 ) -> RustTestHashMapCustomData {
2374 RustTestHashMapCustomData {
2375 name: name.to_string(),
2376 prices: [(instrument_id.to_string(), Price::from(price))]
2377 .into_iter()
2378 .collect(),
2379 ts_event: UnixNanos::from(ts - 1),
2380 ts_init: UnixNanos::from(ts),
2381 }
2382 }
2383
2384 fn query_custom_records(
2385 catalog: &mut ParquetDataCatalog,
2386 identifiers: Option<&[String]>,
2387 ) -> Vec<(Option<String>, RustTestHashMapCustomData)> {
2388 let loaded = catalog
2389 .query_custom_data_dynamic(
2390 "RustTestHashMapCustomData",
2391 identifiers,
2392 None,
2393 None,
2394 None,
2395 None,
2396 true,
2397 )
2398 .unwrap();
2399 let mut decoded = Vec::new();
2400
2401 for data in &loaded {
2402 let Data::Custom(custom) = data else {
2403 panic!("expected custom data, found {data:?}")
2404 };
2405
2406 assert_eq!(custom.data_type.type_name(), "RustTestHashMapCustomData");
2407 decoded.push((
2408 custom.data_type.identifier().map(ToString::to_string),
2409 custom
2410 .data
2411 .as_any()
2412 .downcast_ref::<RustTestHashMapCustomData>()
2413 .expect("expected RustTestHashMapCustomData")
2414 .clone(),
2415 ));
2416 }
2417
2418 decoded.sort_by_key(|(_, record)| record.ts_init);
2419 decoded
2420 }
2421
2422 fn local_catalog(directory: &TempDir) -> CatalogConnectConfig {
2423 CatalogConnectConfig::new(directory.path().to_string_lossy(), None)
2424 }
2425
2426 fn stage_per_identifier_quote(staging: &Path) -> (QuoteTick, PathBuf) {
2428 let mut quote = sample_quote();
2429 quote.instrument_id = InstrumentId::from("CAFÉ.SIM");
2430
2431 let mut writer = FeatherWriter::new(
2432 staging.to_path_buf(),
2433 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2434 RotationConfig::NoRotation,
2435 None,
2436 None,
2437 );
2438 writer.write(quote).unwrap();
2439 writer.close().unwrap();
2440
2441 let identifier_directory = staging.join("quotes").join("CAFÉ.SIM");
2442 let staged_file = identifier_directory.join("quotes_0.feather");
2443 fs::create_dir(&identifier_directory).unwrap();
2444 fs::rename(
2445 staging.join("quotes").join("quotes_0.feather"),
2446 &staged_file,
2447 )
2448 .unwrap();
2449
2450 (quote, staged_file)
2451 }
2452
2453 fn sample_quote() -> QuoteTick {
2454 QuoteTick::new(
2455 InstrumentId::from("AUD/USD.SIM"),
2456 Price::from("0.65"),
2457 Price::from("0.67"),
2458 Quantity::from("11"),
2459 Quantity::from("17"),
2460 UnixNanos::from(19),
2461 UnixNanos::from(23),
2462 )
2463 }
2464
2465 #[rstest]
2466 #[case::ascii("AUD/USD.SIM-1-MINUTE-LAST-EXTERNAL")]
2467 #[case::non_ascii("CAFÉ.SIM-1-MINUTE-LAST-EXTERNAL")]
2468 fn bar_replay_across_promotions_is_idempotent(
2469 #[case] bar_type: &str,
2470 #[values(false, true)] restart: bool,
2471 #[values(false, true)] replay_only: bool,
2472 #[values(false, true)] use_ts_event: bool,
2473 ) {
2474 let directory = TempDir::new().unwrap();
2475
2476 let mut config = WriterConnectConfig::new(
2477 directory.path().join("live/run-bars").to_string_lossy(),
2478 Some(local_catalog(&directory)),
2479 );
2480 config.use_ts_event_for_ts_init = use_ts_event;
2481 config.delete_feather_after_promotion = true;
2482 config.promote_on_close = false;
2483 let clock = Arc::new(AtomicU64::new(1));
2484 let mut writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2485
2486 let first = Bar {
2487 bar_type: BarType::from(bar_type),
2488 ..sample_bar()
2489 };
2490
2491 writer.core.write_data(Data::Bar(first)).unwrap();
2492 assert_eq!(writer.promote_now().unwrap().len(), 1);
2493
2494 if restart {
2495 drop(writer);
2496 writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2497 }
2498
2499 clock.store(2, Ordering::Relaxed);
2500
2501 let replay = Bar {
2502 ts_init: if use_ts_event {
2503 UnixNanos::from(63_000_000_000)
2504 } else {
2505 first.ts_init
2506 },
2507 ..first
2508 };
2509
2510 let next = Bar {
2511 ts_event: UnixNanos::from(120_000_000_000),
2512 ts_init: UnixNanos::from(121_000_000_000),
2513 ..first
2514 };
2515
2516 writer.core.write_data(Data::Bar(replay)).unwrap();
2517 if !replay_only {
2518 writer.core.write_data(Data::Bar(next)).unwrap();
2519 }
2520
2521 let promoted = writer.promote_now().unwrap();
2522 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2523
2524 let DataBatch::Bar(rows) = catalog
2525 .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2526 .unwrap()
2527 else {
2528 panic!("expected bars")
2529 };
2530
2531 let mut expected = vec![Bar {
2532 ts_init: if use_ts_event {
2533 first.ts_event
2534 } else {
2535 first.ts_init
2536 },
2537 ..first
2538 }];
2539
2540 if !replay_only {
2541 expected.push(Bar {
2542 ts_init: if use_ts_event {
2543 next.ts_event
2544 } else {
2545 next.ts_init
2546 },
2547 ..next
2548 });
2549 }
2550
2551 assert_eq!(promoted.len(), 1);
2552 assert_eq!(rows.as_ref(), expected.as_slice());
2553 assert_eq!(
2554 catalog
2555 .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2556 .unwrap()
2557 .len(),
2558 expected.len()
2559 );
2560 assert!(
2561 list_session_feather_files(&writer.source.storage, Environment::Live, "run-bars")
2562 .unwrap()
2563 .is_empty()
2564 );
2565 }
2566
2567 #[rstest]
2568 #[case::open("open")]
2569 #[case::high("high")]
2570 #[case::low("low")]
2571 #[case::close("close")]
2572 #[case::volume("volume")]
2573 #[case::price_precision("price_precision")]
2574 #[case::size_precision("size_precision")]
2575 #[case::event_timestamp("ts_event")]
2576 #[case::init_timestamp("ts_init")]
2577 fn bar_replay_rejects_changed_values(#[case] field: &str) {
2578 let directory = TempDir::new().unwrap();
2579
2580 let mut config = WriterConnectConfig::new(
2581 directory.path().join("live/run-conflict").to_string_lossy(),
2582 Some(local_catalog(&directory)),
2583 );
2584 config.use_ts_event_for_ts_init = !matches!(field, "ts_event" | "ts_init");
2585 config.delete_feather_after_promotion = true;
2586 config.promote_on_close = false;
2587 let clock = Arc::new(AtomicU64::new(1));
2588 let mut writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2589 let first = sample_bar();
2590 writer.core.write_data(Data::Bar(first)).unwrap();
2591 writer.promote_now().unwrap();
2592 clock.store(2, Ordering::Relaxed);
2593 let mut correction = first;
2594 match field {
2595 "open" => correction.open = Price::from("1.02"),
2596 "high" => correction.high = Price::from("1.06"),
2597 "low" => correction.low = Price::from("0.98"),
2598 "close" => correction.close = Price::from("1.04"),
2599 "volume" => correction.volume = Quantity::from("18.000"),
2600 "price_precision" => {
2601 correction.open = Price::from("1.010");
2602 correction.high = Price::from("1.050");
2603 correction.low = Price::from("0.990");
2604 correction.close = Price::from("1.030");
2605 }
2606 "size_precision" => correction.volume = Quantity::from("17.0000"),
2607 "ts_event" => correction.ts_event = UnixNanos::from(59_000_000_000),
2608 "ts_init" => correction.ts_init = UnixNanos::from(60_500_000_000),
2609 _ => unreachable!(),
2610 }
2611
2612 let next = Bar {
2613 ts_event: UnixNanos::from(120_000_000_000),
2614 ts_init: UnixNanos::from(121_000_000_000),
2615 ..first
2616 };
2617
2618 writer.core.write_data(Data::Bar(correction)).unwrap();
2619 writer.core.write_data(Data::Bar(next)).unwrap();
2620 let work = writer.prepare_promotion().unwrap().unwrap();
2621 let files = work.files().to_vec();
2622 let error = writer.finalize(work.execute()).unwrap_err();
2623 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2624
2625 let DataBatch::Bar(rows) = catalog
2626 .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2627 .unwrap()
2628 else {
2629 panic!("expected bars")
2630 };
2631
2632 let expected = if config.use_ts_event_for_ts_init {
2633 Bar {
2634 ts_init: first.ts_event,
2635 ..first
2636 }
2637 } else {
2638 first
2639 };
2640
2641 assert!(error.to_string().contains("non-disjoint intervals"));
2642 assert_eq!(rows.as_ref(), &[expected]);
2643 assert_eq!(
2644 catalog
2645 .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2646 .unwrap()
2647 .len(),
2648 1
2649 );
2650 assert_eq!(files.len(), 1);
2651 assert!(directory.path().join(&files[0]).exists());
2652 }
2653
2654 #[rstest]
2655 fn bar_replay_manual_conversion_is_idempotent() {
2656 let directory = TempDir::new().unwrap();
2657 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2658 let first = sample_bar();
2659
2660 let next = Bar {
2661 ts_event: UnixNanos::from(120_000_000_000),
2662 ts_init: UnixNanos::from(121_000_000_000),
2663 ..first
2664 };
2665
2666 for run in ["first", "restart"] {
2667 let mut writer = FeatherWriter::new(
2668 directory.path().join("backtest").join(run),
2669 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2670 RotationConfig::NoRotation,
2671 None,
2672 None,
2673 );
2674 writer.write(first).unwrap();
2675 if run == "restart" {
2676 writer.write(next).unwrap();
2677 }
2678
2679 writer.close().unwrap();
2680 catalog
2681 .convert_stream_to_data(
2682 run,
2683 &NautilusDataType::Bar.into(),
2684 Environment::Backtest,
2685 None,
2686 true,
2687 )
2688 .unwrap();
2689 }
2690
2691 let DataBatch::Bar(rows) = catalog
2692 .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2693 .unwrap()
2694 else {
2695 panic!("expected bars")
2696 };
2697
2698 assert_eq!(
2699 rows.as_ref(),
2700 &[
2701 Bar {
2702 ts_init: first.ts_event,
2703 ..first
2704 },
2705 Bar {
2706 ts_init: next.ts_event,
2707 ..next
2708 },
2709 ]
2710 );
2711 assert_eq!(
2712 catalog
2713 .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2714 .unwrap()
2715 .len(),
2716 2
2717 );
2718 }
2719
2720 fn sample_bar() -> Bar {
2721 Bar::new(
2722 BarType::from("AUD/USD.SIM-1-MINUTE-LAST-EXTERNAL"),
2723 Price::from("1.01"),
2724 Price::from("1.05"),
2725 Price::from("0.99"),
2726 Price::from("1.03"),
2727 Quantity::from("17.000"),
2728 UnixNanos::from(60_000_000_000),
2729 UnixNanos::from(61_000_000_000),
2730 )
2731 }
2732
2733 #[rstest]
2734 fn kept_promotions_release_scheduled_paths() {
2735 let directory = TempDir::new().unwrap();
2736 let config = WriterConnectConfig::new(
2737 directory.path().join("live/run-bounded").to_string_lossy(),
2738 Some(local_catalog(&directory)),
2739 );
2740 let mut writer = ParquetWriter::new(&config, WriterClock::Live).unwrap();
2741
2742 for timestamp in 1..=32 {
2743 writer
2744 .core
2745 .write_data(Data::Quote(QuoteTick {
2746 ts_event: UnixNanos::from(timestamp),
2747 ts_init: UnixNanos::from(timestamp),
2748 ..sample_quote()
2749 }))
2750 .unwrap();
2751 let work = writer.prepare_promotion().unwrap().unwrap();
2752 assert_eq!(work.files().len(), 1);
2753 writer.finalize(work.execute()).unwrap();
2754 assert_scheduled_count(&writer.core.promotion_driver.schedule(), 0);
2755 assert!(writer.prepare_promotion().unwrap().is_none());
2756 }
2757
2758 assert_eq!(
2759 list_session_feather_files(&writer.source.storage, Environment::Live, "run-bounded")
2760 .unwrap()
2761 .len(),
2762 32
2763 );
2764 }
2765
2766 #[rstest]
2767 #[case::timer_seal(RotationConfig::NoRotation, false, 1)]
2768 #[case::close(RotationConfig::NoRotation, true, 1)]
2769 #[case::rotation(RotationConfig::Size { max_size: 1 }, false, 2)]
2770 fn sealed_paths_promote_once_and_do_not_resubmit_in_flight(
2771 #[case] rotation: RotationConfig,
2772 #[case] close: bool,
2773 #[case] expected_files: usize,
2774 ) {
2775 let directory = TempDir::new().unwrap();
2776 let mut config = WriterConnectConfig::new(
2777 directory.path().join("live/run-sealing").to_string_lossy(),
2778 Some(local_catalog(&directory)),
2779 );
2780 config.rotation_config = rotation;
2781 let mut writer = ParquetWriter::new(&config, WriterClock::Live).unwrap();
2782 let first = sample_quote();
2783 let second = QuoteTick {
2784 ts_event: UnixNanos::from(29),
2785 ts_init: UnixNanos::from(31),
2786 ..first
2787 };
2788 writer.core.write_data(Data::Quote(first)).unwrap();
2789 writer.core.write_data(Data::Quote(second)).unwrap();
2790
2791 if close {
2792 writer.core.close().unwrap();
2793 }
2794
2795 let work = writer.prepare_promotion().unwrap().unwrap();
2796 let files = work.files().to_vec();
2797 assert_eq!(files.len(), expected_files);
2798 assert!(writer.prepare_promotion().unwrap().is_none());
2799 assert_scheduled_count(&writer.core.promotion_driver.schedule(), expected_files);
2800 let result = work.execute();
2801 assert_scheduled_count(&writer.core.promotion_driver.schedule(), 0);
2802 assert_eq!(result.committed_paths, files);
2803 assert_eq!(writer.finalize(result).unwrap().len(), expected_files);
2804 assert!(writer.prepare_promotion().unwrap().is_none());
2805
2806 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2807 let DataBatch::Quote(rows) = catalog
2808 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2809 .unwrap()
2810 else {
2811 panic!("expected quotes")
2812 };
2813 assert_eq!(rows.as_ref(), &[first, second]);
2814 }
2815
2816 #[rstest]
2817 #[case::space("my data")]
2818 #[case::non_ascii("Données")]
2819 #[case::fragment("data#1")]
2820 #[case::literal_percent("data%20")]
2821 fn writer_root_path_promotes_and_replays(
2822 #[case] root_name: &str,
2823 #[values(false, true)] file_uri: bool,
2824 ) {
2825 let directory = TempDir::new().unwrap();
2826 let root = directory.path().join(root_name);
2827 let staging = root.join("live/run-root");
2828 let staging_uri = if file_uri {
2829 normalize_path_to_uri(&staging.to_string_lossy()).unwrap()
2830 } else {
2831 staging.to_string_lossy().into_owned()
2832 };
2833 let config = WriterConnectConfig::new(staging_uri, Some(local_catalog(&directory)));
2834 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2835 let quote = sample_quote();
2836 let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
2837 writer.core.write_data(Data::Quote(quote)).unwrap();
2838 let work = writer.prepare_promotion().unwrap().unwrap();
2839 assert_eq!(work.files(), &["live/run-root/quotes/quotes_0.feather"]);
2840 let promoted = writer.finalize(work.execute()).unwrap();
2841 assert_eq!(promoted.len(), 1);
2842 assert!(writer.prepare_promotion().unwrap().is_none());
2843 drop(writer);
2844
2845 let mut restarted = ParquetWriter::new(&config, clock).unwrap();
2846 let work = restarted.prepare_promotion().unwrap().unwrap();
2847 assert_eq!(work.files(), &["live/run-root/quotes/quotes_0.feather"]);
2848 assert_eq!(restarted.finalize(work.execute()).unwrap(), promoted);
2849 assert!(restarted.prepare_promotion().unwrap().is_none());
2850
2851 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2852 let DataBatch::Quote(rows) = catalog
2853 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2854 .unwrap()
2855 else {
2856 panic!("expected quotes")
2857 };
2858 assert_eq!(rows.as_ref(), &[quote]);
2859 }
2860
2861 #[rstest]
2862 fn parent_relative_writer_path_promotes_and_replays() {
2863 let directory = TempDir::new_in(".").unwrap();
2864 let current = std::env::current_dir().unwrap();
2865 let staging = PathBuf::from("..")
2866 .join(current.file_name().unwrap())
2867 .join(directory.path().file_name().unwrap())
2868 .join("live/run-relative");
2869 let config =
2870 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2871 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2872 let quote = sample_quote();
2873 let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
2874 writer.core.write_data(Data::Quote(quote)).unwrap();
2875 let work = writer.prepare_promotion().unwrap().unwrap();
2876 assert_eq!(work.files(), &["live/run-relative/quotes/quotes_0.feather"]);
2877 let promoted = writer.finalize(work.execute()).unwrap();
2878 assert_eq!(promoted.len(), 1);
2879 assert!(writer.prepare_promotion().unwrap().is_none());
2880 drop(writer);
2881
2882 let mut restarted = ParquetWriter::new(&config, clock).unwrap();
2883 let work = restarted.prepare_promotion().unwrap().unwrap();
2884 assert_eq!(work.files(), &["live/run-relative/quotes/quotes_0.feather"]);
2885 assert_eq!(restarted.finalize(work.execute()).unwrap(), promoted);
2886 assert!(restarted.prepare_promotion().unwrap().is_none());
2887
2888 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2889 let DataBatch::Quote(rows) = catalog
2890 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2891 .unwrap()
2892 else {
2893 panic!("expected quotes")
2894 };
2895 assert_eq!(rows.as_ref(), &[quote]);
2896 }
2897
2898 #[rstest]
2899 #[case::sealed(false)]
2900 #[case::partial(true)]
2901 fn restart_discovers_crashed_files_and_deduplicates_promotions(
2902 #[case] partial: bool,
2903 #[values(false, true)] delete_source: bool,
2904 #[values(false, true)] restage: bool,
2905 ) {
2906 let directory = TempDir::new().unwrap();
2907 let staging = directory.path().join("backtest/run-restart");
2908 let quote = sample_quote();
2909 let mut crashed = FeatherWriter::new(
2910 staging.clone(),
2911 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2912 RotationConfig::NoRotation,
2913 None,
2914 None,
2915 );
2916 crashed.write(quote).unwrap();
2917 crashed.close().unwrap();
2918 let sealed = staging.join("quotes/quotes_0.feather");
2919
2920 if partial {
2921 fs::rename(&sealed, sealed.with_extension(FEATHER_PARTIAL_EXTENSION)).unwrap();
2922 }
2923
2924 drop(crashed);
2925 let mut config =
2926 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2927 config.delete_feather_after_promotion = restage;
2928 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2929 let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
2930 let source_bytes = fs::read(&sealed).unwrap();
2931 let work = writer.prepare_promotion().unwrap().unwrap();
2932 assert_eq!(
2933 work.files(),
2934 &["backtest/run-restart/quotes/quotes_0.feather".to_string()]
2935 );
2936 let promoted = writer.finalize(work.execute()).unwrap();
2937 assert_eq!(promoted.len(), 1);
2938 assert!(writer.prepare_promotion().unwrap().is_none());
2939 drop(writer);
2940
2941 if restage {
2942 assert!(!sealed.exists());
2943 fs::write(&sealed, source_bytes).unwrap();
2944 }
2945
2946 config.delete_feather_after_promotion = delete_source;
2947 let mut restarted = ParquetWriter::new(&config, clock).unwrap();
2948 let work = restarted.prepare_promotion().unwrap().unwrap();
2949 assert_eq!(work.files().len(), 1);
2950 let files = work.files().to_vec();
2951 let result = work.execute();
2952 assert_eq!(result.committed_paths, files);
2953 assert_eq!(
2954 result.deleted_paths,
2955 if delete_source { files } else { Vec::new() }
2956 );
2957 assert_eq!(restarted.finalize(result).unwrap(), promoted);
2958 assert!(restarted.prepare_promotion().unwrap().is_none());
2959 assert_scheduled_count(&restarted.core.promotion_driver.schedule(), 0);
2960 assert_eq!(sealed.exists(), !delete_source);
2961
2962 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2963 let DataBatch::Quote(rows) = catalog
2964 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2965 .unwrap()
2966 else {
2967 panic!("expected quotes")
2968 };
2969 assert_eq!(rows.as_ref(), &[quote]);
2970 }
2971
2972 #[rstest]
2973 #[case::nested_base("/prefix/catalog/", "prefix/catalog/")]
2974 #[case::root_base("", "")]
2975 fn promotion_paths_live_under_the_catalog_base_path(
2976 #[case] base_path: &str,
2977 #[case] expected_prefix: &str,
2978 ) {
2979 let directory = TempDir::new().unwrap();
2980 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2981 catalog.base_path = base_path.to_string();
2982
2983 let backend = ParquetPromotionBackend {
2984 catalog,
2985 legacy_manifest_missing: Arc::default(),
2986 };
2987
2988 assert_eq!(
2989 backend.manifest_path(),
2990 ObjectPath::from(format!("{expected_prefix}_nautilus_promotions.json")),
2991 );
2992 assert_eq!(
2993 backend.marker_path("replay-id"),
2994 ObjectPath::from(format!(
2995 "{expected_prefix}_nautilus_promotions/\
2996 cd3112001080bc2d11985ffe2b8b90b324d336d82c17ddb66127d3a05f08c69c.json"
2997 )),
2998 );
2999 }
3000
3001 #[rstest]
3002 fn feather_replay_identity_is_stable_and_ignores_identifier_order() {
3003 let identity = |identifiers: Option<&[String]>| {
3004 feather_replay_identity(
3005 "file:///catalog/backtest/run-1",
3006 "quotes/AUDUSD.SIM/part-0.feather",
3007 "content-hash",
3008 identifiers,
3009 )
3010 };
3011
3012 let unordered = ["B".to_string(), "A".to_string(), "A".to_string()];
3013 let ordered = ["A".to_string(), "B".to_string()];
3014
3015 let expected =
3016 "nautilus-feather:10a9435c28f7536f26653c3fc808571be89bfe769e06c7307cb4743e273a23fd";
3017
3018 assert_eq!(identity(Some(&unordered)), expected);
3019 assert_eq!(identity(Some(&ordered)), expected);
3020 assert_ne!(identity(None), expected);
3021 }
3022}