1use std::{
13 fmt::Debug,
14 sync::{
15 Arc,
16 atomic::{AtomicBool, Ordering},
17 },
18};
19
20use nautilus_common::live::block_on_nautilus_with;
21use object_store::{
22 Error as ObjectStoreError, ObjectStoreExt, PutMode, PutOptions, path::Path as ObjectPath,
23};
24use serde::{Deserialize, Serialize};
25
26use super::catalog::ParquetDataCatalog;
27use crate::{
28 catalog::factory::CatalogConnectConfig,
29 common::{
30 conversion::FeatherConversionSummary,
31 datafusion::identifiers_from_record_batches,
32 paths::{
33 catalog_data_type_from_session_feather_path, create_local_directory,
34 identifier_from_session_feather_path,
35 },
36 storage::create_storage_backend_from_path,
37 },
38 writer::{
39 factory::{PARQUET_WRITER_FACTORY_NAME, WriterConnectConfig, WriterFactoryRegistry},
40 feather::WriterClock,
41 materializer::read_feather_record_batches_with_identity,
42 promotion::{
43 PromotionBackend, PromotionResult, PromotionSession, PromotionWork,
44 StagedPromotionBackend,
45 },
46 run::{FeatherSessionSource, RunStatus},
47 staged::{StagedFeatherWriter, StagedWriter},
48 traits::StreamingDataSink,
49 },
50};
51
52pub(crate) fn register_factory(registry: &mut WriterFactoryRegistry) {
53 registry.insert(
54 PARQUET_WRITER_FACTORY_NAME.to_string(),
55 Arc::new(parquet_writer_factory),
56 );
57}
58
59fn parquet_writer_factory(
60 config: &WriterConnectConfig,
61 clock: WriterClock,
62) -> anyhow::Result<StreamingDataSink> {
63 Ok(Box::new(ParquetWriter::new(config, clock)?))
64}
65
66#[expect(
67 clippy::struct_excessive_bools,
68 reason = "the booleans are independent writer policy and lifecycle flags"
69)]
70struct ParquetWriter {
71 core: StagedFeatherWriter<ParquetPromotionBackend>,
72 session: PromotionSession,
73 source: FeatherSessionSource,
74 catalog: CatalogConnectConfig,
75 interval_ms: Option<u64>,
76 promote_on_close: bool,
77 delete_feather_after_commit: bool,
78 use_ts_event_for_ts_init: bool,
79 legacy_manifest_missing: Arc<AtomicBool>,
80 has_data: bool,
81 run_status: RunStatus,
82}
83
84impl Debug for ParquetWriter {
85 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86 f.debug_struct(stringify!(ParquetWriter))
87 .field("staging_uri", &self.core.storage.original_uri)
88 .field("catalog_uri", &self.catalog.uri)
89 .field("interval_ms", &self.interval_ms)
90 .finish_non_exhaustive()
91 }
92}
93
94impl ParquetWriter {
95 fn new(config: &WriterConnectConfig, clock: WriterClock) -> anyhow::Result<Self> {
96 let catalog = config
97 .required_catalog(PARQUET_WRITER_FACTORY_NAME)?
98 .clone();
99 let storage = create_storage_backend_from_path(&config.uri, None)?;
100 let session = StagedFeatherWriter::<ParquetPromotionBackend>::required_session(
101 &storage.original_uri,
102 "Parquet writer URI",
103 )?;
104 let source_storage = create_storage_backend_from_path(&session.root_uri, None)?;
105
106 let source = FeatherSessionSource::new(
107 source_storage,
108 session.environment,
109 session.instance_id.clone(),
110 );
111
112 create_local_directory(&catalog.uri)?;
115
116 let legacy_manifest_missing = Arc::new(AtomicBool::new(false));
117
118 ParquetPromotionBackend::new(&catalog, Arc::clone(&legacy_manifest_missing))?;
119
120 let mut core = StagedFeatherWriter::new(
121 storage.clone(),
122 &source,
123 clock,
124 config.rotation_config.clone(),
125 None,
126 config.flush_interval_ms,
127 config.record_filter.clone(),
128 )?;
129
130 block_on_nautilus_with(|| {
131 storage.write_current_run_manifest(
132 session.environment,
133 &session.instance_id,
134 RunStatus::InProgress,
135 true,
136 )
137 })?;
138
139 let timer_catalog = catalog.clone();
140 core.warn_if_orphan_feather_present("Parquet");
141 let timer_legacy_manifest_missing = Arc::clone(&legacy_manifest_missing);
142 core.start_promotion_timer(
143 config.promotion_interval_ms,
144 source.clone(),
145 "parquet-promotion",
146 move || {
147 ParquetPromotionBackend::new(
148 &timer_catalog,
149 Arc::clone(&timer_legacy_manifest_missing),
150 )
151 },
152 config.use_ts_event_for_ts_init,
153 config.delete_feather_after_promotion,
154 )?;
155
156 Ok(Self {
157 core,
158 catalog,
159 session,
160 source,
161 interval_ms: config.promotion_interval_ms,
162 promote_on_close: config.promote_on_close,
163 delete_feather_after_commit: config.delete_feather_after_promotion,
164 use_ts_event_for_ts_init: config.use_ts_event_for_ts_init,
165 legacy_manifest_missing,
166 has_data: false,
167 run_status: RunStatus::InProgress,
168 })
169 }
170
171 fn record_status(&mut self, status: RunStatus) -> anyhow::Result<()> {
172 if self.run_status == status {
173 return Ok(());
174 }
175
176 block_on_nautilus_with(|| {
177 self.core.storage.write_current_run_manifest(
178 self.session.environment,
179 &self.session.instance_id,
180 status,
181 !self.has_data,
182 )
183 })?;
184
185 self.run_status = status;
186 Ok(())
187 }
188
189 fn mark_non_empty(&mut self) -> anyhow::Result<()> {
190 if self.has_data {
191 return Ok(());
192 }
193
194 block_on_nautilus_with(|| {
195 self.core.storage.write_current_run_manifest(
196 self.session.environment,
197 &self.session.instance_id,
198 RunStatus::InProgress,
199 false,
200 )
201 })?;
202
203 self.has_data = true;
204 Ok(())
205 }
206
207 fn interval_ns(&self) -> Option<u64> {
208 self.interval_ms
209 .filter(|interval_ms| *interval_ms > 0)
210 .map(|interval_ms| interval_ms.saturating_mul(1_000_000))
211 }
212
213 fn prepare_promotion(&self) -> anyhow::Result<Option<PromotionWork<ParquetPromotionBackend>>> {
214 self.core.prepare_promotion(
215 || {
216 ParquetPromotionBackend::new(
217 &self.catalog,
218 Arc::clone(&self.legacy_manifest_missing),
219 )
220 },
221 self.source.clone(),
222 self.use_ts_event_for_ts_init,
223 self.delete_feather_after_commit,
224 )
225 }
226
227 fn finalize(
228 &mut self,
229 result: PromotionResult,
230 ) -> anyhow::Result<Vec<FeatherConversionSummary>> {
231 if result.converted.is_err() {
232 if result.run_state_recorded {
233 self.run_status = RunStatus::Failed;
234 } else if let Err(e) = self.record_status(RunStatus::Failed) {
235 log::warn!("Failed to record Parquet run Failed state: {e}");
236 }
237 } else if result.run_state_recorded {
238 self.run_status = RunStatus::Promoted;
239 }
240
241 self.core.finalize_promotion(result)
242 }
243
244 fn drain_completed(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
245 let mut converted = Vec::new();
246 for result in self.core.promotion_driver.drain_completed()? {
247 converted.extend(self.finalize(result)?);
248 }
249
250 Ok(converted)
251 }
252
253 fn wait(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
254 let mut converted = Vec::new();
255 for result in self.core.promotion_driver.wait()? {
256 converted.extend(self.finalize(result)?);
257 }
258
259 Ok(converted)
260 }
261
262 fn promote_now(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
263 let mut converted = self.wait()?;
264 if let Some(work) = self.prepare_promotion()? {
265 converted.extend(self.finalize(work.execute())?);
266 }
267
268 Ok(converted)
269 }
270}
271
272impl StagedWriter for ParquetWriter {
273 type Backend = ParquetPromotionBackend;
274
275 fn staged(&mut self) -> &mut StagedFeatherWriter<Self::Backend> {
276 &mut self.core
277 }
278
279 fn mark_non_empty(&mut self) -> anyhow::Result<()> {
280 Self::mark_non_empty(self)
281 }
282
283 fn maybe_promote(&mut self) -> anyhow::Result<()> {
284 self.drain_completed()?;
285
286 let Some(interval_ns) = self.interval_ns() else {
287 return Ok(());
288 };
289
290 let now = self.core.clock.timestamp_ns();
291 if !self.core.promotion_driver.is_due(now, interval_ns) {
292 return Ok(());
293 }
294
295 self.core.promotion_driver.mark_committed_at(now);
298
299 if let Some(work) = self.prepare_promotion()? {
300 self.core
301 .promotion_driver
302 .submit(work, "parquet-promotion")?;
303 }
304
305 Ok(())
306 }
307
308 fn wait_for_promotions(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
309 self.wait()
310 }
311
312 fn promote_on_flush(&self) -> bool {
313 self.interval_ns().is_some_and(|interval_ns| {
314 self.core
315 .promotion_driver
316 .is_due(self.core.clock.timestamp_ns(), interval_ns)
317 })
318 }
319
320 fn promote_on_close(&self) -> bool {
321 self.promote_on_close
322 }
323
324 fn promote_now(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
325 Self::promote_now(self)
326 }
327
328 fn record_completed(&mut self) {
329 if self.run_status == RunStatus::InProgress
330 && let Err(e) = self.record_status(RunStatus::Completed)
331 {
332 log::warn!("Failed to complete Parquet run manifest: {e}");
333 }
334 }
335}
336
337impl Drop for ParquetWriter {
338 fn drop(&mut self) {
339 self.core.stop_promotion_timer();
340
341 let promotion_failed = if let Err(e) = self.wait() {
342 log::warn!("ParquetWriter dropped with pending promotion error: {e}");
343 true
344 } else {
345 false
346 };
347
348 if self.run_status == RunStatus::InProgress {
349 let status = if promotion_failed || !self.core.is_closed().unwrap_or(false) {
350 RunStatus::Failed
351 } else {
352 RunStatus::Completed
353 };
354
355 if let Err(e) = self.record_status(status) {
356 log::warn!(
357 "Failed to record Parquet run {} state during drop: {e}",
358 status.as_str()
359 );
360 }
361 }
362 }
363}
364
365const PROMOTION_MANIFEST: &str = "_nautilus_promotions.json";
366const PROMOTION_MARKERS: &str = "_nautilus_promotions";
367
368struct ParquetPromotionBackend {
369 catalog: ParquetDataCatalog,
370 legacy_manifest_missing: Arc<AtomicBool>,
371}
372
373impl ParquetPromotionBackend {
374 fn new(
375 catalog: &CatalogConnectConfig,
376 legacy_manifest_missing: Arc<AtomicBool>,
377 ) -> anyhow::Result<Self> {
378 Ok(Self {
379 catalog: super::open_catalog(catalog)?,
380 legacy_manifest_missing,
381 })
382 }
383
384 fn manifest_path(&self) -> ObjectPath {
385 self.catalog_path(PROMOTION_MANIFEST)
386 }
387
388 fn manifest(&self) -> anyhow::Result<ParquetPromotionManifest> {
389 if self.legacy_manifest_missing.load(Ordering::Relaxed) {
390 return Ok(ParquetPromotionManifest::default());
391 }
392
393 let path = self.manifest_path();
394 block_on_nautilus_with(|| async {
395 let result = match self.catalog.object_store.get(&path).await {
396 Ok(result) => result,
397 Err(ObjectStoreError::NotFound { .. }) => {
398 self.legacy_manifest_missing.store(true, Ordering::Relaxed);
399 return Ok(ParquetPromotionManifest::default());
400 }
401 Err(e) => return Err(e.into()),
402 };
403
404 Ok(serde_json::from_slice(&result.bytes().await?)?)
405 })
406 }
407
408 fn marker_path(&self, identity: &str) -> ObjectPath {
409 let digest = blake3::hash(identity.as_bytes()).to_hex();
410 self.catalog_path(&format!("{PROMOTION_MARKERS}/{digest}.json"))
411 }
412
413 fn catalog_path(&self, path: &str) -> ObjectPath {
414 let base = self.catalog.base_path.trim_matches('/');
415 if base.is_empty() {
416 ObjectPath::from(path)
417 } else {
418 ObjectPath::from(format!("{base}/{path}"))
419 }
420 }
421
422 fn identity_recorded(&self, identity: &str) -> anyhow::Result<bool> {
423 let path = self.marker_path(identity);
424
425 let marker_exists = block_on_nautilus_with(|| async {
426 match self.catalog.object_store.head(&path).await {
427 Ok(_) => Ok::<bool, anyhow::Error>(true),
428 Err(ObjectStoreError::NotFound { .. }) => Ok(false),
429 Err(e) => Err(anyhow::Error::from(e)),
430 }
431 })?;
432
433 Ok(marker_exists
434 || self
435 .manifest()?
436 .identities
437 .iter()
438 .any(|item| item == identity))
439 }
440
441 fn record_identity(&self, identity: &str) -> anyhow::Result<()> {
442 let bytes = serde_json::to_vec(identity)?;
443 let path = self.marker_path(identity);
444 block_on_nautilus_with(|| async {
445 match self
446 .catalog
447 .object_store
448 .put_opts(
449 &path,
450 bytes.into(),
451 PutOptions {
452 mode: PutMode::Create,
453 ..Default::default()
454 },
455 )
456 .await
457 {
458 Ok(_) | Err(ObjectStoreError::AlreadyExists { .. }) => Ok(()),
459 Err(e) => Err(e.into()),
460 }
461 })
462 }
463}
464
465impl PromotionBackend for ParquetPromotionBackend {
466 type Source = FeatherSessionSource;
467
468 const NAME: &'static str = "Parquet";
469
470 fn convert_file(
471 &mut self,
472 source: &Self::Source,
473 file: &str,
474 use_ts_event_for_ts_init: bool,
475 record_promoted: bool,
476 ) -> anyhow::Result<Option<FeatherConversionSummary>> {
477 let object_path = ObjectPath::parse(file)?;
478
479 let read = block_on_nautilus_with(|| {
480 read_feather_record_batches_with_identity(
481 source.storage.object_store.clone(),
482 &object_path,
483 )
484 })?;
485
486 let identifiers = identifiers_from_record_batches(&read.batches)
487 .ok()
488 .filter(|identifiers| !identifiers.is_empty());
489 let identity = feather_replay_identity(
490 &source.storage.original_uri,
491 file,
492 &read.content_hash,
493 identifiers.as_deref(),
494 );
495
496 if self.identity_recorded(&identity)? {
497 let identifier = match identifiers {
498 Some(mut identifiers) if identifiers.len() == 1 => identifiers.pop(),
499 Some(_) => None,
500 None => identifier_from_session_feather_path(
501 file,
502 source.environment,
503 &source.instance_id,
504 ),
505 };
506
507 return Ok(Some(FeatherConversionSummary {
508 data_type: catalog_data_type_from_session_feather_path(
509 file,
510 source.environment,
511 &source.instance_id,
512 )?,
513 identifier,
514 feather_path: file.to_string(),
515 native_version: None,
516 unmatched_identifiers: None,
517 }));
518 }
519
520 let summary = self.catalog.promote_feather_file(
521 source,
522 file,
523 read.batches,
524 use_ts_event_for_ts_init,
525 &identity,
526 )?;
527
528 if summary.is_some() && record_promoted {
529 self.record_identity(&identity)?;
530 }
531
532 Ok(summary)
533 }
534
535 fn delete_file(&mut self, source: &Self::Source, file: &str) -> anyhow::Result<()> {
536 let object_path = ObjectPath::parse(file)?;
537 block_on_nautilus_with(|| async {
538 source.storage.object_store.delete(&object_path).await?;
539 Ok::<(), anyhow::Error>(())
540 })
541 }
542}
543
544impl StagedPromotionBackend for ParquetPromotionBackend {
545 const REQUIRES_SESSION: bool = true;
546
547 fn record_run_state(
548 &mut self,
549 source: &FeatherSessionSource,
550 _staging_uri: &str,
551 status: RunStatus,
552 empty: bool,
553 _error: Option<&str>,
554 ) -> anyhow::Result<()> {
555 block_on_nautilus_with(|| {
556 source.storage.write_run_manifest(
557 source.environment,
558 &source.instance_id,
559 status,
560 empty,
561 )
562 })
563 }
564}
565
566#[derive(Debug, Default, Deserialize, Serialize)]
567struct ParquetPromotionManifest {
568 identities: Vec<String>,
569}
570
571fn feather_replay_identity(
572 source_uri: &str,
573 source_path: &str,
574 content_hash: &str,
575 identifiers: Option<&[String]>,
576) -> String {
577 let mut identifiers = identifiers.map(<[String]>::to_vec);
578 if let Some(identifiers) = identifiers.as_mut() {
579 identifiers.sort();
580 identifiers.dedup();
581 }
582
583 let identity = serde_json::json!({
584 "source_uri": source_uri,
585 "source_path": source_path,
586 "content_hash": content_hash,
587 "identifiers": identifiers,
588 });
589 format!(
590 "nautilus-feather:{}",
591 blake3::hash(identity.to_string().as_bytes()).to_hex(),
592 )
593}
594
595#[cfg(test)]
596mod tests {
597 use std::{
598 fs,
599 path::{Path, PathBuf},
600 sync::atomic::AtomicU64,
601 };
602
603 use arrow::datatypes::SchemaRef;
604 use nautilus_common::enums::Environment;
605 use nautilus_core::{Params, UnixNanos};
606 use nautilus_model::{
607 data::{
608 Bar, BarType, BookOrder, Data, DataBatch, NautilusDataType, NautilusRecordType,
609 OrderBookDelta, QuoteTick,
610 },
611 enums::{BookAction, OrderSide, RecordFlag},
612 identifiers::InstrumentId,
613 types::{ERROR_PRICE, Price, Quantity},
614 };
615 use nautilus_serialization::arrow::{DecodeTypedFromRecordBatch, EncodeToRecordBatch};
616 use object_store::memory::InMemory;
617 use parquet::{
618 basic::Compression,
619 file::{
620 metadata::RowGroupMetaData,
621 reader::{FileReader, SerializedFileReader},
622 },
623 };
624 use rstest::rstest;
625 use tempfile::TempDir;
626
627 use super::*;
628 use crate::{
629 backend::parquet::io::read_parquet_from_object_store,
630 catalog::traits::{CatalogQuery, CatalogReader},
631 common::{paths::normalize_path_to_uri, storage::RUN_MANIFEST_FILENAME},
632 config::{CatalogCompression, DataCatalogConfig},
633 test_data::RustTestHashMapCustomData,
634 writer::{
635 feather::{FEATHER_PARTIAL_EXTENSION, FeatherWriter, RotationConfig},
636 promotion::{list_session_feather_files, tests::assert_scheduled_count},
637 traits::StreamingSink,
638 },
639 };
640
641 #[rstest]
642 fn parquet_default_close_promotes_and_honors_source_retention(
643 #[values(false, true)] delete_source: bool,
644 ) {
645 let directory = TempDir::new().unwrap();
646 let staging = directory.path().join("backtest").join("run-1");
647 let mut config =
648 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
649 config.delete_feather_after_promotion = delete_source;
650 let mut sink =
651 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
652 .unwrap();
653 let quote = sample_quote();
654 let mut second = quote;
655 second.instrument_id = InstrumentId::from("BTC/USD.SIM");
656 second.bid_price = Price::from("100.1234");
657 second.ask_price = Price::from("100.5678");
658 second.ts_init = UnixNanos::from(24);
659 let mut catalog = ParquetDataCatalog::from_uri(
660 directory.path().to_str().unwrap(),
661 None,
662 None,
663 None,
664 None,
665 )
666 .unwrap();
667 sink.write_data(Data::Quote(quote)).unwrap();
668 sink.write_data(Data::Quote(second)).unwrap();
669 sink.flush().unwrap();
670 let before = catalog
671 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
672 .unwrap();
673 sink.close().unwrap();
674 let after = catalog
675 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
676 .unwrap();
677 let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
678 let staged = block_on_nautilus_with(|| storage.list_files("", None))
679 .unwrap()
680 .into_iter()
681 .filter(|path| path.ends_with(".feather"))
682 .count();
683 assert!(before.is_empty());
684
685 let DataBatch::Quote(rows) = after else {
686 panic!("expected quotes")
687 };
688
689 assert_eq!(rows.as_ref(), &[quote, second]);
690 assert_eq!(staged, usize::from(!delete_source));
691 }
692
693 #[rstest]
694 fn parquet_writer_recovers_partial_files_at_startup() {
695 let directory = TempDir::new().unwrap();
696 let staging = directory.path().join("backtest").join("run-recover");
697 let quote = sample_quote();
698
699 let mut crashed = FeatherWriter::new(
701 staging.clone(),
702 WriterClock::Test(Arc::new(AtomicU64::new(0))),
703 RotationConfig::NoRotation,
704 None,
705 None,
706 );
707 crashed.write(quote).unwrap();
708 crashed.close().unwrap();
709 let sealed = staging.join("quotes").join("quotes_0.feather");
710 std::fs::rename(&sealed, sealed.with_extension(FEATHER_PARTIAL_EXTENSION)).unwrap();
711
712 let config =
713 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
714 let mut sink =
715 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
716 .unwrap();
717 sink.close().unwrap();
718
719 let mut catalog = ParquetDataCatalog::from_uri(
720 directory.path().to_str().unwrap(),
721 None,
722 None,
723 None,
724 None,
725 )
726 .unwrap();
727 let rows = catalog
728 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
729 .unwrap();
730
731 let DataBatch::Quote(rows) = rows else {
732 panic!("expected quotes")
733 };
734
735 assert_eq!(rows.as_ref(), &[quote]);
736 }
737
738 #[rstest]
739 fn parquet_close_promotes_per_identifier_file_with_non_ascii_identifier(
740 #[values(false, true)] delete_source: bool,
741 ) {
742 let directory = TempDir::new().unwrap();
743 let staging = directory.path().join("backtest").join("run-1");
744 let (quote, staged_file) = stage_per_identifier_quote(&staging);
745 let mut config =
746 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
747 config.delete_feather_after_promotion = delete_source;
748 let mut sink =
749 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
750 .unwrap();
751
752 sink.close().unwrap();
753
754 let mut catalog = ParquetDataCatalog::from_uri(
755 directory.path().to_str().unwrap(),
756 None,
757 None,
758 None,
759 None,
760 )
761 .unwrap();
762
763 let DataBatch::Quote(rows) = catalog
764 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
765 .unwrap()
766 else {
767 panic!("expected quotes")
768 };
769
770 assert_eq!(rows.as_ref(), &[quote]);
771 assert_eq!(staged_file.exists(), !delete_source);
772 }
773
774 #[rstest]
775 fn parquet_manual_conversion_reads_per_identifier_file_with_non_ascii_identifier() {
776 let directory = TempDir::new().unwrap();
777 let staging = directory.path().join("backtest").join("run-1");
778 let (quote, _) = stage_per_identifier_quote(&staging);
779 let mut catalog = ParquetDataCatalog::from_uri(
780 directory.path().to_str().unwrap(),
781 None,
782 None,
783 None,
784 None,
785 )
786 .unwrap();
787
788 catalog
789 .convert_stream_to_data(
790 "run-1",
791 &NautilusDataType::QuoteTick.into(),
792 Environment::Backtest,
793 None,
794 false,
795 )
796 .unwrap();
797
798 let DataBatch::Quote(rows) = catalog
799 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
800 .unwrap()
801 else {
802 panic!("expected quotes")
803 };
804
805 assert_eq!(rows.as_ref(), &[quote]);
806 }
807
808 #[rstest]
809 fn parquet_promotion_writes_non_ascii_identifier_to_object_store() {
810 let directory = TempDir::new().unwrap();
811 let staging = directory.path().join("backtest").join("run-1");
812 let (quote, staged_file) = stage_per_identifier_quote(&staging);
813
814 let source = FeatherSessionSource::new(
815 create_storage_backend_from_path(directory.path().to_str().unwrap(), None).unwrap(),
816 Environment::Backtest,
817 "run-1",
818 );
819 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
820 catalog.base_path = "catalog".to_string();
821 catalog.original_uri = "s3://test-bucket/catalog".to_string();
822 catalog.object_store = Arc::new(InMemory::new());
823
824 let mut backend = ParquetPromotionBackend {
825 catalog,
826 legacy_manifest_missing: Arc::default(),
827 };
828
829 let files =
830 list_session_feather_files(&source.storage, Environment::Backtest, "run-1").unwrap();
831
832 for file in &files {
833 backend.convert_file(&source, file, false, true).unwrap();
834 backend.delete_file(&source, file).unwrap();
835 }
836
837 let DataBatch::Quote(rows) = backend
838 .catalog
839 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
840 .unwrap()
841 else {
842 panic!("expected quotes")
843 };
844
845 assert_eq!(files.len(), 1);
846 assert_eq!(rows.as_ref(), &[quote]);
847 assert!(!staged_file.exists());
848 }
849
850 #[rstest]
851 #[case("backtest")]
852 #[case("sandbox")]
853 #[case("live")]
854 fn parquet_close_promotes_into_missing_local_catalog(#[case] environment: &str) {
855 let directory = TempDir::new().unwrap();
856 let staging = directory
857 .path()
858 .join("stream")
859 .join(environment)
860 .join("run-1");
861 let catalog_path = directory.path().join("charts").join("catalog");
862
863 let config = WriterConnectConfig::new(
864 staging.to_string_lossy(),
865 Some(CatalogConnectConfig::new(
866 catalog_path.to_string_lossy(),
867 None,
868 )),
869 );
870 let mut sink =
871 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
872 .unwrap();
873 let quote = sample_quote();
874
875 sink.write_data(Data::Quote(quote)).unwrap();
876 sink.close().unwrap();
877
878 let mut catalog =
879 ParquetDataCatalog::from_uri(catalog_path.to_str().unwrap(), None, None, None, None)
880 .unwrap();
881
882 let DataBatch::Quote(rows) = catalog
883 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
884 .unwrap()
885 else {
886 panic!("expected quotes")
887 };
888
889 let manifest: serde_json::Value =
890 serde_json::from_slice(&std::fs::read(staging.join(RUN_MANIFEST_FILENAME)).unwrap())
891 .unwrap();
892
893 assert_eq!(rows.as_ref(), &[quote]);
894 assert_eq!(manifest["kind"], environment);
895 assert_eq!(manifest["status"], "promoted");
896 assert!(!catalog_path.join(environment).exists());
897 }
898
899 #[rstest]
900 #[case::non_ascii("run-é")]
901 #[case::space("run 1")]
902 #[case::fragment("run#1")]
903 fn parquet_writer_rejects_run_id_that_paths_encode(#[case] run_id: &str) {
904 let directory = TempDir::new().unwrap();
905 let staging = directory.path().join("backtest").join(run_id);
906 let config =
907 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
908
909 let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
910 .unwrap_err();
911
912 let staging_uri = normalize_path_to_uri(&staging.to_string_lossy()).unwrap();
913 assert_eq!(
914 error.to_string(),
915 format!(
916 "Parquet writer URI '{staging_uri}' has a run ID that object-store paths \
917 percent-encode; use a run ID without spaces, non-ASCII, or reserved characters",
918 )
919 );
920 }
921
922 #[rstest]
923 #[case::windows_separators(r"file:///C:\catalog\backtest\run-1", "file:///C:/catalog")]
924 #[case::windows_trailing_separator(r"file:///C:\catalog\backtest\run-1\", "file:///C:/catalog")]
925 #[case::trailing_separator("file:///tmp/catalog/backtest/run-1/", "file:///tmp/catalog")]
926 #[case::parent_components(
927 "file:///tmp/unused/../catalog/backtest/run-1",
928 "file:///tmp/catalog"
929 )]
930 fn parquet_writer_session_accepts_separator_forms(#[case] uri: &str, #[case] root_uri: &str) {
931 let session = StagedFeatherWriter::<ParquetPromotionBackend>::required_session(
932 uri,
933 "Parquet writer URI",
934 )
935 .unwrap();
936
937 assert_eq!(
938 session,
939 PromotionSession {
940 root_uri: root_uri.to_string(),
941 environment: Environment::Backtest,
942 instance_id: "run-1".to_string(),
943 }
944 );
945 }
946
947 #[rstest]
948 fn parquet_writer_requires_a_catalog() {
949 let directory = TempDir::new().unwrap();
950 let staging = directory.path().join("backtest").join("run-1");
951 let config = WriterConnectConfig::new(staging.to_string_lossy(), None);
952
953 let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
954 .unwrap_err();
955
956 assert_eq!(
957 error.to_string(),
958 "Parquet writer requires a promotion catalog"
959 );
960 assert!(!staging.exists());
961 }
962
963 #[cfg(feature = "cloud")]
966 #[rstest]
967 fn parquet_writer_stages_locally_for_a_remote_catalog() {
968 let directory = TempDir::new().unwrap();
969 let staging = directory.path().join("backtest").join("run-1");
970
971 let config = WriterConnectConfig::new(
972 staging.to_string_lossy(),
973 Some(CatalogConnectConfig::new("s3://bucket/catalog", None)),
974 );
975
976 let mut sink =
977 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
978 .unwrap();
979 sink.close().unwrap();
980 drop(sink);
981
982 let manifest: serde_json::Value =
983 serde_json::from_slice(&std::fs::read(staging.join(RUN_MANIFEST_FILENAME)).unwrap())
984 .unwrap();
985 assert_eq!(manifest["kind"], "backtest");
986 assert_eq!(manifest["instance_id"], "run-1");
987 assert_eq!(manifest["status"], "completed");
988 assert_eq!(manifest["empty"], true);
989 }
990
991 #[rstest]
992 fn parquet_promotion_writes_the_direct_write_schema() {
993 let directory = TempDir::new().unwrap();
994
995 let config = WriterConnectConfig::new(
996 directory
997 .path()
998 .join("backtest/run-schema")
999 .to_string_lossy(),
1000 Some(local_catalog(&directory)),
1001 );
1002 let mut sink =
1003 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1004 .unwrap();
1005 let quote = sample_quote();
1006 sink.write_data(Data::Quote(quote)).unwrap();
1007 sink.close().unwrap();
1008
1009 let promoted = ParquetDataCatalog::from_uri(
1010 directory.path().to_str().unwrap(),
1011 None,
1012 None,
1013 None,
1014 None,
1015 )
1016 .unwrap();
1017 let direct_directory = TempDir::new().unwrap();
1018 let direct = ParquetDataCatalog::from_uri(
1019 direct_directory.path().to_str().unwrap(),
1020 None,
1021 None,
1022 None,
1023 None,
1024 )
1025 .unwrap();
1026 direct.write_to_parquet(&[quote], None, None, None).unwrap();
1027
1028 assert_eq!(
1029 quote_file_schema(&promoted).fields(),
1030 quote_file_schema(&direct).fields(),
1031 );
1032 }
1033
1034 #[rstest]
1035 fn parquet_promotion_honors_catalog_settings() {
1036 let directory = TempDir::new().unwrap();
1037 let staging = directory.path().join("backtest").join("run-1");
1038 let catalog = DataCatalogConfig::builder()
1039 .path(directory.path().to_string_lossy().to_string())
1040 .batch_size(7)
1041 .compression(CatalogCompression::Uncompressed)
1042 .max_row_group_size(1)
1043 .build()
1044 .unwrap()
1045 .connect_config();
1046 let backend =
1047 ParquetPromotionBackend::new(&catalog, Arc::new(AtomicBool::new(false))).unwrap();
1048
1049 let config = WriterConnectConfig::new(staging.to_string_lossy(), Some(catalog));
1050 let mut sink =
1051 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1052 .unwrap();
1053 let quote = sample_quote();
1054 sink.write_data(Data::Quote(quote)).unwrap();
1055 sink.write_data(Data::Quote(QuoteTick {
1056 ts_event: UnixNanos::from(29),
1057 ts_init: UnixNanos::from(31),
1058 ..quote
1059 }))
1060 .unwrap();
1061
1062 sink.close().unwrap();
1063
1064 let files = ParquetDataCatalog::new(directory.path(), None, None, None, None)
1065 .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
1066 .unwrap();
1067 let reader = SerializedFileReader::new(
1068 std::fs::File::open(directory.path().join(&files[0])).unwrap(),
1069 )
1070 .unwrap();
1071 let metadata = reader.metadata();
1072
1073 assert_eq!(backend.catalog.batch_size, 7);
1074 assert_eq!(files.len(), 1);
1075 assert_eq!(metadata.num_row_groups(), 2);
1076 assert!(
1077 metadata
1078 .row_groups()
1079 .iter()
1080 .flat_map(RowGroupMetaData::columns)
1081 .all(|column| column.compression() == Compression::UNCOMPRESSED)
1082 );
1083 }
1084
1085 #[rstest]
1086 fn parquet_writer_rejects_unknown_catalog_param() {
1087 let directory = TempDir::new().unwrap();
1088 let staging = directory.path().join("backtest").join("run-1");
1089 let mut catalog = local_catalog(&directory);
1090 let params = Params::new([("compression".to_string(), serde_json::json!(0))].into());
1091 catalog.params = Some(params);
1092
1093 let config = WriterConnectConfig::new(staging.to_string_lossy(), Some(catalog));
1094 let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1095 .unwrap_err();
1096
1097 assert_eq!(
1098 error.to_string(),
1099 "Unknown Parquet catalog param 'compression': this catalog takes no params"
1100 );
1101 }
1102
1103 fn quote_file_schema(catalog: &ParquetDataCatalog) -> SchemaRef {
1104 let files = catalog
1105 .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
1106 .unwrap();
1107 assert_eq!(files.len(), 1);
1108
1109 let path = ObjectPath::from(files[0].as_str());
1110
1111 let (_, schema) = block_on_nautilus_with(|| {
1112 read_parquet_from_object_store(catalog.object_store.clone(), &path)
1113 })
1114 .unwrap();
1115
1116 schema
1117 }
1118
1119 #[rstest]
1120 #[case(None)]
1121 #[case(Some(0))]
1122 #[case(Some(1))]
1123 fn parquet_interval_uses_test_clock_and_close_can_leave_staged_data(
1124 #[case] interval: Option<u64>,
1125 ) {
1126 let directory = TempDir::new().unwrap();
1127 let staging = directory.path().join("backtest").join("run-2");
1128 let mut config =
1129 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1130 config.promotion_interval_ms = interval;
1131 config.promote_on_close = false;
1132 let clock = Arc::new(AtomicU64::new(0));
1133 let mut sink =
1134 parquet_writer_factory(&config, WriterClock::Test(Arc::clone(&clock))).unwrap();
1135 let quote = sample_quote();
1136 sink.write_data(Data::Quote(quote)).unwrap();
1137 clock.store(1_000_000, Ordering::Relaxed);
1138 sink.flush().unwrap();
1139 sink.close().unwrap();
1140 let mut catalog = ParquetDataCatalog::from_uri(
1141 directory.path().to_str().unwrap(),
1142 None,
1143 None,
1144 None,
1145 None,
1146 )
1147 .unwrap();
1148 let before_manual = catalog
1149 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
1150 .unwrap();
1151
1152 if interval != Some(1) {
1153 catalog
1154 .convert_stream_to_data(
1155 "run-2",
1156 &NautilusDataType::QuoteTick.into(),
1157 Environment::Backtest,
1158 None,
1159 false,
1160 )
1161 .unwrap();
1162 }
1163
1164 let after = catalog
1165 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
1166 .unwrap();
1167 assert_eq!(before_manual.len(), usize::from(interval == Some(1)));
1168
1169 let DataBatch::Quote(rows) = after else {
1170 panic!("expected quotes")
1171 };
1172
1173 assert_eq!(rows.as_ref(), &[quote]);
1174 }
1175
1176 #[rstest]
1177 fn due_promotion_without_staged_files_skips_catalog_and_waits_for_next_interval() {
1178 let directory = TempDir::new().unwrap();
1179 let staging = directory.path().join("backtest").join("run-idle");
1180 let mut config =
1181 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1182 config.promotion_interval_ms = Some(1);
1183 let clock = Arc::new(AtomicU64::new(0));
1184 let mut writer =
1185 ParquetWriter::new(&config, WriterClock::Test(Arc::clone(&clock))).unwrap();
1186
1187 writer.catalog.params = Some(Params::new(
1189 [("compression".to_string(), serde_json::json!(0))].into(),
1190 ));
1191
1192 clock.store(1_000_000, Ordering::Relaxed);
1193 writer.maybe_promote().unwrap();
1194
1195 writer.catalog.params = None;
1197 writer.core.write_data(Data::Quote(sample_quote())).unwrap();
1198 writer.maybe_promote().unwrap();
1199 assert!(writer.wait().unwrap().is_empty());
1200
1201 clock.store(2_000_000, Ordering::Relaxed);
1202 writer.maybe_promote().unwrap();
1203 assert_eq!(writer.wait().unwrap().len(), 1);
1204 }
1205
1206 #[rstest]
1207 fn failed_catalog_open_seals_once_per_interval_and_keeps_files_retryable() {
1208 let directory = TempDir::new().unwrap();
1209 let staging = directory.path().join("backtest").join("run-failing");
1210 let mut config =
1211 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1212 config.promotion_interval_ms = Some(1);
1213 let clock = Arc::new(AtomicU64::new(0));
1214 let mut writer =
1215 ParquetWriter::new(&config, WriterClock::Test(Arc::clone(&clock))).unwrap();
1216
1217 let quotes = (1..=5)
1218 .map(|timestamp| QuoteTick {
1219 ts_event: UnixNanos::from(timestamp),
1220 ts_init: UnixNanos::from(timestamp),
1221 ..sample_quote()
1222 })
1223 .collect::<Vec<_>>();
1224
1225 let sealed_count = |writer: &ParquetWriter| {
1226 list_session_feather_files(&writer.source.storage, Environment::Backtest, "run-failing")
1227 .unwrap()
1228 .len()
1229 };
1230
1231 writer.catalog.params = Some(Params::new(
1233 [("compression".to_string(), serde_json::json!(0))].into(),
1234 ));
1235
1236 clock.store(1_000_000, Ordering::Relaxed);
1237 let first_error = writer.write_data(Data::Quote(quotes[0])).unwrap_err();
1238 writer.write_data(Data::Quote(quotes[1])).unwrap();
1239 writer.write_data(Data::Quote(quotes[2])).unwrap();
1240 let sealed_first_interval = sealed_count(&writer);
1241
1242 clock.store(2_000_000, Ordering::Relaxed);
1243 let second_error = writer.write_data(Data::Quote(quotes[3])).unwrap_err();
1244 writer.write_data(Data::Quote(quotes[4])).unwrap();
1245 let sealed_second_interval = sealed_count(&writer);
1246
1247 writer.catalog.params = None;
1248 clock.store(3_000_000, Ordering::Relaxed);
1249 writer.maybe_promote().unwrap();
1250 writer.wait().unwrap();
1251
1252 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1253
1254 let DataBatch::Quote(rows) = catalog
1255 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
1256 .unwrap()
1257 else {
1258 panic!("expected quotes")
1259 };
1260
1261 let open_error =
1262 "Unknown Parquet catalog param 'compression': this catalog takes no params";
1263 assert_eq!(first_error.to_string(), open_error);
1264 assert_eq!(second_error.to_string(), open_error);
1265 assert_eq!(sealed_first_interval, 1);
1266 assert_eq!(sealed_second_interval, 2);
1267 assert_eq!(rows.as_ref(), "es[..]);
1268 }
1269
1270 #[rstest]
1271 fn parquet_streaming_funding_round_trip() {
1272 use nautilus_model::data::FundingRateUpdate;
1273 let directory = TempDir::new().unwrap();
1274
1275 let config = WriterConnectConfig::new(
1276 directory
1277 .path()
1278 .join("backtest/run-funding")
1279 .to_string_lossy(),
1280 Some(local_catalog(&directory)),
1281 );
1282 let mut sink =
1283 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1284 .unwrap();
1285
1286 let funding = FundingRateUpdate::new(
1287 InstrumentId::from("AUD/USD.SIM"),
1288 "0.0012".parse().unwrap(),
1289 Some(480),
1290 Some(789.into()),
1291 123.into(),
1292 456.into(),
1293 );
1294 assert!(sink.write_any(&funding).unwrap());
1295 sink.close().unwrap();
1296 let mut catalog = ParquetDataCatalog::from_uri(
1297 directory.path().to_str().unwrap(),
1298 None,
1299 None,
1300 None,
1301 None,
1302 )
1303 .unwrap();
1304 let rows = catalog
1305 .query_batch(&CatalogQuery::new(NautilusDataType::FundingRateUpdate))
1306 .unwrap();
1307
1308 let DataBatch::FundingRate(rows) = rows else {
1309 panic!("expected funding rates")
1310 };
1311
1312 assert_eq!(rows.as_ref(), &[funding]);
1313 }
1314
1315 #[rstest]
1316 fn parquet_streaming_write_error_reaches_flush() {
1317 let directory = TempDir::new().unwrap();
1318
1319 let config = WriterConnectConfig::new(
1320 directory
1321 .path()
1322 .join("backtest/run-error")
1323 .to_string_lossy(),
1324 Some(local_catalog(&directory)),
1325 );
1326 let mut sink =
1327 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1328 .unwrap();
1329 let mut quote = sample_quote();
1330 quote.bid_price = ERROR_PRICE;
1331 let write_error = sink.write_any("e).unwrap_err().to_string();
1332 let flush_error = sink.flush().unwrap_err().to_string();
1333 assert_eq!(flush_error, write_error);
1334 }
1335
1336 #[rstest]
1337 fn parquet_streaming_instrument_promotes() {
1338 use nautilus_model::instruments::{InstrumentAny, stubs::audusd_sim};
1339 let directory = TempDir::new().unwrap();
1340
1341 let config = WriterConnectConfig::new(
1342 directory
1343 .path()
1344 .join("backtest/run-instrument")
1345 .to_string_lossy(),
1346 Some(local_catalog(&directory)),
1347 );
1348 let mut sink =
1349 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1350 .unwrap();
1351 let instrument = InstrumentAny::CurrencyPair(audusd_sim());
1352 assert!(sink.write_any(&instrument).unwrap());
1353 sink.close().unwrap();
1354 let mut catalog = ParquetDataCatalog::from_uri(
1355 directory.path().to_str().unwrap(),
1356 None,
1357 None,
1358 None,
1359 None,
1360 )
1361 .unwrap();
1362 let rows = catalog
1363 .query_batch(&CatalogQuery::new(NautilusDataType::Instrument))
1364 .unwrap();
1365
1366 let DataBatch::Instrument(rows) = rows else {
1367 panic!("expected instruments")
1368 };
1369
1370 assert_eq!(rows.as_ref(), &[instrument]);
1371 }
1372
1373 #[rstest]
1374 fn parquet_promotion_preserves_every_record_family(#[values(false, true)] automatic: bool) {
1375 use nautilus_core::UUID4;
1376 use nautilus_model::{
1377 enums::{
1378 LiquiditySide, OrderSide, OrderStatus, OrderType, PositionAdjustmentType,
1379 PositionSide, TimeInForce,
1380 },
1381 events::{
1382 OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled, OrderDenied,
1383 OrderEmulated, OrderExpired, OrderInitialized, OrderModifyDenied,
1384 OrderModifyRejected, OrderPendingCancel, OrderPendingUpdate, OrderRejected,
1385 OrderReleased, OrderSnapshot, OrderSubmitted, OrderTriggered, OrderUpdated,
1386 PositionAdjusted, PositionChanged, PositionClosed, PositionOpened,
1387 PositionSnapshot,
1388 account::stubs::cash_account_state,
1389 order::spec::{OrderFillVoidedSpec, OrderFilledSpec},
1390 },
1391 identifiers::{
1392 AccountId, ClientId, ClientOrderId, PositionId, TradeId, Venue, VenueOrderId,
1393 },
1394 instruments::{InstrumentAny, stubs::audusd_sim},
1395 orders::OrderTestBuilder,
1396 position::Position,
1397 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
1398 stubs::TestDefault,
1399 types::Money,
1400 };
1401 use strum::IntoEnumIterator;
1402 let mut families = ahash::AHashSet::new();
1403 macro_rules! promote {
1404 ($record:expr) => {
1405 families.insert(assert_record_promotion($record, automatic));
1406 };
1407 }
1408
1409 promote!(cash_account_state());
1410 promote!(OrderInitialized::test_default());
1411 promote!(OrderDenied::test_default());
1412 promote!(OrderEmulated::test_default());
1413 promote!(OrderSubmitted::test_default());
1414 promote!(OrderAccepted::test_default());
1415 promote!(OrderRejected::test_default());
1416 promote!(OrderPendingCancel::test_default());
1417 promote!(OrderCanceled::test_default());
1418 let mut cancel_denied = OrderCancelDenied::test_default();
1419 cancel_denied.causation_id = Some(UUID4::new());
1420 promote!(cancel_denied);
1421 promote!(OrderCancelRejected::test_default());
1422 promote!(OrderExpired::test_default());
1423 promote!(OrderTriggered::test_default());
1424 promote!(OrderPendingUpdate::test_default());
1425 promote!(OrderReleased::test_default());
1426 let mut modify_denied = OrderModifyDenied::test_default();
1427 modify_denied.causation_id = Some(UUID4::new());
1428 promote!(modify_denied);
1429 promote!(OrderModifyRejected::test_default());
1430 promote!(OrderUpdated::test_default());
1431 promote!(OrderFillVoidedSpec::builder().is_reopened(true).build());
1432 let id = InstrumentId::from("AUD/USD.SIM");
1433 let fill = OrderFilledSpec::builder()
1434 .instrument_id(id)
1435 .position_id(PositionId::from("P-009"))
1436 .last_px(Price::from("1.23456"))
1437 .last_qty(Quantity::from("1234"))
1438 .ts_event(11.into())
1439 .ts_init(13.into())
1440 .build();
1441 promote!(fill.clone());
1442 let mut position = Position::new(&InstrumentAny::CurrencyPair(audusd_sim()), fill.clone());
1443 promote!(PositionOpened::create(
1444 &position,
1445 &fill,
1446 UUID4::default(),
1447 17.into()
1448 ));
1449 promote!(PositionChanged::create(
1450 &position,
1451 &fill,
1452 UUID4::default(),
1453 19.into()
1454 ));
1455 promote!(PositionAdjusted::new(
1456 fill.trader_id,
1457 fill.strategy_id,
1458 id,
1459 position.id,
1460 fill.account_id,
1461 PositionAdjustmentType::Funding,
1462 None,
1463 Some(Money::from("1.23 USD")),
1464 Some("funding".into()),
1465 UUID4::default(),
1466 23.into(),
1467 29.into()
1468 ));
1469 promote!(PositionSnapshot::from(
1470 &position,
1471 Some(Money::from("4.56 USD"))
1472 ));
1473 let closing = OrderFilledSpec::builder()
1474 .instrument_id(id)
1475 .position_id(position.id)
1476 .order_side(OrderSide::Sell)
1477 .last_px(Price::from("1.34567"))
1478 .last_qty(fill.last_qty)
1479 .trade_id(TradeId::from("T-011"))
1480 .ts_event(31.into())
1481 .ts_init(37.into())
1482 .build();
1483 position.apply(&closing);
1484 promote!(PositionClosed::create(
1485 &position,
1486 &closing,
1487 UUID4::default(),
1488 41.into()
1489 ));
1490 promote!(OrderSnapshot::from(
1491 OrderTestBuilder::new(OrderType::Limit)
1492 .instrument_id(id)
1493 .price(Price::from("1.45678"))
1494 .quantity(Quantity::from("5678"))
1495 .build()
1496 ));
1497
1498 let order = OrderStatusReport::new(
1499 AccountId::from("SIM-007"),
1500 id,
1501 Some(ClientOrderId::from("O-013")),
1502 VenueOrderId::from("V-017"),
1503 Some(OrderSide::Buy),
1504 OrderType::Limit,
1505 TimeInForce::Gtc,
1506 OrderStatus::PartiallyFilled,
1507 Quantity::from("123"),
1508 Quantity::from("45"),
1509 43.into(),
1510 47.into(),
1511 53.into(),
1512 None,
1513 );
1514
1515 let report_fill = FillReport::new(
1516 AccountId::from("SIM-007"),
1517 id,
1518 VenueOrderId::from("V-017"),
1519 TradeId::from("T-019"),
1520 OrderSide::Buy,
1521 Quantity::from("45"),
1522 Price::from("1.56789"),
1523 Money::from("2.34 USD"),
1524 LiquiditySide::Maker,
1525 Some(ClientOrderId::from("O-013")),
1526 Some(PositionId::from("P-023")),
1527 59.into(),
1528 61.into(),
1529 None,
1530 );
1531
1532 let report_position = PositionStatusReport::new(
1533 AccountId::from("SIM-007"),
1534 id,
1535 PositionSide::Long,
1536 Quantity::from("789"),
1537 67.into(),
1538 71.into(),
1539 None,
1540 Some(PositionId::from("P-023")),
1541 Some(rust_decimal::Decimal::new(16789, 4)),
1542 );
1543
1544 let mut mass = ExecutionMassStatus::new(
1545 ClientId::from("CLIENT-029"),
1546 AccountId::from("SIM-007"),
1547 Venue::from("SIM"),
1548 73.into(),
1549 None,
1550 );
1551 mass.add_order_reports(vec![order.clone()]);
1552 mass.add_fill_reports(vec![report_fill.clone()]);
1553 mass.add_position_reports(vec![report_position.clone()]);
1554 promote!(order);
1555 promote!(report_fill);
1556 promote!(report_position);
1557 promote!(mass);
1558
1559 let expected = NautilusRecordType::iter()
1560 .filter(|family| {
1561 #[cfg(feature = "defi")]
1562 if *family == NautilusRecordType::Defi {
1563 return false;
1564 }
1565
1566 let _ = family;
1567 true
1568 })
1569 .collect::<ahash::AHashSet<_>>();
1570
1571 assert_eq!(families, expected);
1572 }
1573
1574 #[rstest]
1575 fn parquet_promotion_preserves_calculated_greeks_as_custom_data(
1576 #[values(false, true)] automatic: bool,
1577 #[values(false, true)] portfolio: bool,
1578 ) {
1579 use nautilus_model::data::{
1580 CustomData, DataType,
1581 greeks::{GreeksData, OptionGreekValues, PortfolioGreeks},
1582 };
1583
1584 let greeks = GreeksData::new(
1585 "ESZ26 C7000.XCME".parse().unwrap(),
1586 false,
1587 7000.5,
1588 20_261_218,
1589 68,
1590 0.186,
1591 50.0,
1592 -3.0,
1593 6950.25,
1594 0.0425,
1595 0.0125,
1596 0.175,
1597 12.5,
1598 98.75,
1599 OptionGreekValues {
1600 delta: 0.41,
1601 gamma: 0.0012,
1602 vega: 11.2,
1603 theta: -2.3,
1604 rho: 1.7,
1605 },
1606 0.62,
1607 11.into(),
1608 13.into(),
1609 );
1610
1611 let mut second = greeks.clone();
1612 second.ts_init = 113.into();
1613 second.instrument_id = "ESZ26 P6900.XCME".parse().unwrap();
1614
1615 let greeks = vec![greeks, second]
1616 .into_iter()
1617 .map(|value| {
1618 let data_type =
1619 DataType::new("GreeksData", None, Some(value.instrument_id.to_string()));
1620 CustomData::new(Arc::new(value), data_type)
1621 })
1622 .collect::<Vec<_>>();
1623
1624 let portfolio_value = PortfolioGreeks {
1625 pnl: -125.5,
1626 price: 4310.25,
1627 greeks: OptionGreekValues {
1628 delta: 0.83,
1629 gamma: 0.004,
1630 vega: 45.5,
1631 theta: -12.25,
1632 rho: 3.75,
1633 },
1634 ts_event: 19.into(),
1635 ts_init: 17.into(),
1636 };
1637
1638 let mut second = portfolio_value.clone();
1639 second.ts_init = 117.into();
1640 let portfolios = vec![portfolio_value, second]
1641 .into_iter()
1642 .map(|value| CustomData::from_arc(Arc::new(value)))
1643 .collect::<Vec<_>>();
1644
1645 let expected = if portfolio { portfolios } else { greeks };
1646
1647 let directory = TempDir::new().unwrap();
1648 let staging = directory.path().join("backtest/run-greeks-custom");
1649 let mut config =
1650 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1651 config.promote_on_close = automatic;
1652 let mut sink =
1653 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1654 .unwrap();
1655
1656 for custom in &expected {
1657 assert!(sink.write_any(custom.data.as_any()).unwrap());
1658 }
1659
1660 sink.close().unwrap();
1661
1662 let data_type = NautilusDataType::Custom {
1663 type_name: expected[0].data_type.type_name().to_string(),
1664 };
1665
1666 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1667 if !automatic {
1668 catalog
1669 .convert_stream_to_data(
1670 "run-greeks-custom",
1671 &data_type.clone().into(),
1672 Environment::Backtest,
1673 None,
1674 false,
1675 )
1676 .unwrap();
1677 }
1678
1679 let DataBatch::Custom(actual) = catalog.query_batch(&CatalogQuery::new(data_type)).unwrap()
1680 else {
1681 panic!("expected custom data")
1682 };
1683
1684 assert_eq!(actual.as_ref(), expected.as_slice());
1685 }
1686
1687 fn assert_record_promotion<T>(record: T, automatic: bool) -> NautilusRecordType
1688 where
1689 T: std::any::Any
1690 + serde::Serialize
1691 + serde::de::DeserializeOwned
1692 + EncodeToRecordBatch
1693 + DecodeTypedFromRecordBatch,
1694 {
1695 let directory = TempDir::new().unwrap();
1696 let staging = directory.path().join("backtest/run-record");
1697 let mut config =
1698 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1699 config.promote_on_close = automatic;
1700 let mut sink =
1701 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1702 .unwrap();
1703 let mut second = serde_json::to_value(&record).unwrap();
1704 second["ts_init"] = serde_json::json!(second["ts_init"].as_u64().unwrap() + 101);
1705 let second: T = serde_json::from_value(second).unwrap();
1706 assert!(sink.write_any(&record).unwrap());
1707 assert!(sink.write_any(&second).unwrap());
1708 sink.close().unwrap();
1709 let record_type = T::get_schema(None).metadata()["type_name"]
1710 .parse::<NautilusRecordType>()
1711 .unwrap();
1712 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1713 if !automatic {
1714 catalog
1715 .convert_stream_to_data(
1716 "run-record",
1717 &record_type.into(),
1718 Environment::Backtest,
1719 None,
1720 false,
1721 )
1722 .unwrap();
1723 }
1724
1725 let batches = catalog
1726 .query_record_batches(&record_type.into(), None, None, None, None, true)
1727 .unwrap();
1728
1729 let actual = batches
1730 .into_iter()
1731 .flat_map(|batch| {
1732 let schema = batch.schema();
1733 T::decode_typed_batch(schema.metadata(), batch).unwrap()
1734 })
1735 .map(|record| serde_json::to_value(record).unwrap())
1736 .collect::<Vec<_>>();
1737
1738 assert_eq!(
1739 actual,
1740 vec![
1741 serde_json::to_value(record).unwrap(),
1742 serde_json::to_value(second).unwrap()
1743 ]
1744 );
1745 record_type
1746 }
1747
1748 #[rstest]
1749 fn parquet_streaming_voided_fill_promotes() {
1750 use nautilus_model::events::{OrderFillVoided, order::spec::OrderFillVoidedSpec};
1751 use nautilus_serialization::arrow::DecodeTypedFromRecordBatch;
1752 let directory = TempDir::new().unwrap();
1753
1754 let config = WriterConnectConfig::new(
1755 directory.path().join("backtest/run-void").to_string_lossy(),
1756 Some(local_catalog(&directory)),
1757 );
1758 let mut sink =
1759 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1760 .unwrap();
1761 let event = OrderFillVoidedSpec::builder().is_reopened(true).build();
1762 assert!(sink.write_any(&event).unwrap());
1763 sink.close().unwrap();
1764 let mut catalog = ParquetDataCatalog::from_uri(
1765 directory.path().to_str().unwrap(),
1766 None,
1767 None,
1768 None,
1769 None,
1770 )
1771 .unwrap();
1772 let batches = catalog
1773 .query_record_batches(
1774 &NautilusRecordType::OrderFillVoided.into(),
1775 None,
1776 None,
1777 None,
1778 None,
1779 true,
1780 )
1781 .unwrap();
1782 let mut rows = Vec::new();
1783 for batch in batches {
1784 rows.extend(
1785 OrderFillVoided::decode_typed_batch(batch.schema().metadata(), batch).unwrap(),
1786 );
1787 }
1788
1789 assert_eq!(rows, vec![event]);
1790 }
1791
1792 #[rstest]
1793 fn parquet_promotion_preserves_clear_source_order(
1794 #[values(false, true)] automatic: bool,
1795 #[values(false, true)] batched: bool,
1796 #[values(false, true)] undefined_price: bool,
1797 ) {
1798 use nautilus_model::{
1799 data::{BookOrder, OrderBookDelta},
1800 enums::{BookAction, OrderSide, RecordFlag},
1801 types::PRICE_UNDEF,
1802 };
1803 let directory = TempDir::new().unwrap();
1804 let staging = directory.path().join("backtest/run-clear");
1805 let mut config =
1806 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1807 config.promote_on_close = automatic;
1808 let mut sink =
1809 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1810 .unwrap();
1811 let id = InstrumentId::from("AUD/USD.SIM");
1812 let ts = UnixNanos::from(1_700_000_000_000_000_123);
1813 let mut clear = OrderBookDelta::clear(id, 7, ts, ts);
1814 if undefined_price {
1815 clear.order.price = Price::from_raw(PRICE_UNDEF, 0);
1816 }
1817
1818 let add = OrderBookDelta::new(
1819 id,
1820 BookAction::Add,
1821 BookOrder::new(
1822 OrderSide::Buy,
1823 Price::from("1.23456"),
1824 Quantity::from("7.890"),
1825 19,
1826 ),
1827 RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8,
1828 7,
1829 ts,
1830 ts,
1831 );
1832 let mut second_add = add;
1833 second_add.order.order_id = 29;
1834 second_add.order.price = Price::from("1.34567");
1835 second_add.order.size = Quantity::from("8.901");
1836 let rows = vec![clear, add, clear, second_add];
1837 if batched {
1838 sink.write_batch(rows.iter().copied().map(Data::BookDelta).collect())
1839 .unwrap();
1840 } else {
1841 for row in &rows {
1842 sink.write_data(Data::BookDelta(*row)).unwrap();
1843 }
1844 }
1845
1846 sink.close().unwrap();
1847 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1848 let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
1849 let staged = block_on_nautilus_with(|| storage.list_files("", Some(".feather"))).unwrap();
1850 let mut originals = Vec::new();
1851
1852 for path in &staged {
1853 let bytes = block_on_nautilus_with(|| async {
1854 storage
1855 .object_store
1856 .get(&object_store::path::Path::from(path.as_str()))
1857 .await
1858 .unwrap()
1859 .bytes()
1860 .await
1861 .unwrap()
1862 });
1863
1864 originals.push(bytes);
1865 }
1866
1867 if !automatic {
1868 catalog
1869 .convert_stream_to_data(
1870 "run-clear",
1871 &NautilusDataType::OrderBookDelta.into(),
1872 Environment::Backtest,
1873 None,
1874 false,
1875 )
1876 .unwrap();
1877 }
1878
1879 let actual = catalog
1880 .query::<OrderBookDelta>(None, None, None, None, None, true)
1881 .unwrap();
1882 assert_eq!(actual.len(), rows.len());
1883
1884 for (actual, expected) in actual.iter().zip(&rows) {
1885 assert_eq!(actual.instrument_id, expected.instrument_id);
1886 assert_eq!(actual.action, expected.action);
1887 assert_eq!(actual.order.side, expected.order.side);
1888 assert_eq!(actual.order.price.raw(), expected.order.price.raw());
1889
1890 let price_precision = if expected.order.price.is_undefined() {
1891 0
1892 } else {
1893 5
1894 };
1895
1896 assert_eq!(actual.order.price.precision, price_precision);
1897 assert_eq!(
1898 actual.order.size.as_decimal(),
1899 expected.order.size.as_decimal()
1900 );
1901 assert_eq!(actual.order.size.precision, 3);
1902 assert_eq!(actual.order.order_id, expected.order.order_id);
1903 assert_eq!(actual.flags, expected.flags);
1904 assert_eq!(actual.sequence, expected.sequence);
1905 assert_eq!(actual.ts_event, expected.ts_event);
1906 assert_eq!(actual.ts_init, expected.ts_init);
1907 }
1908
1909 for (path, original) in staged.iter().zip(originals) {
1910 let bytes = block_on_nautilus_with(|| async {
1911 storage
1912 .object_store
1913 .get(&object_store::path::Path::from(path.as_str()))
1914 .await
1915 .unwrap()
1916 .bytes()
1917 .await
1918 .unwrap()
1919 });
1920
1921 assert_eq!(bytes, original);
1922 }
1923
1924 assert_eq!(staged.len(), 1);
1925 }
1926
1927 #[rstest]
1928 #[case::manual(false, false)]
1929 #[case::automatic_retained(true, false)]
1930 #[case::automatic_delete(true, true)]
1931 fn parquet_failed_clear_promotion_preserves_source(
1932 #[case] automatic: bool,
1933 #[case] delete_source: bool,
1934 #[values(false, true)] batched: bool,
1935 ) {
1936 use nautilus_model::{
1937 data::{BookOrder, OrderBookDelta},
1938 enums::{BookAction, OrderSide, RecordFlag},
1939 };
1940 let directory = TempDir::new().unwrap();
1941 let staging = directory.path().join("backtest/run-invalid-clear");
1942 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
1943
1944 let mut writer = FeatherWriter::new(
1945 staging.clone(),
1946 clock.clone(),
1947 RotationConfig::NoRotation,
1948 None,
1949 None,
1950 );
1951 let id = InstrumentId::from("AUD/USD.SIM");
1952 let ts = UnixNanos::from(23);
1953 let mut clear = OrderBookDelta::clear(id, 7, ts, ts);
1954 clear.order.price = Price::from("1");
1955
1956 let add = OrderBookDelta::new(
1957 id,
1958 BookAction::Add,
1959 BookOrder::new(
1960 OrderSide::Buy,
1961 Price::from("1.23456"),
1962 Quantity::from("7.890"),
1963 19,
1964 ),
1965 RecordFlag::F_LAST as u8,
1966 7,
1967 ts,
1968 ts,
1969 );
1970
1971 if batched {
1972 writer.write_batch(vec![clear, add]).unwrap();
1973 } else {
1974 writer.write(clear).unwrap();
1975 writer.write(add).unwrap();
1976 }
1977
1978 writer.close().unwrap();
1979 let source = staging.join("order_book_deltas/order_book_deltas_0.feather");
1980 let original = fs::read(&source).unwrap();
1981 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1982
1983 let failure = if automatic {
1984 let mut config = WriterConnectConfig::new(
1985 staging.to_string_lossy(),
1986 Some(local_catalog(&directory)),
1987 );
1988 config.delete_feather_after_promotion = delete_source;
1989 let mut sink = parquet_writer_factory(&config, clock).unwrap();
1990 sink.close().unwrap_err()
1991 } else {
1992 catalog
1993 .convert_stream_to_data(
1994 "run-invalid-clear",
1995 &NautilusDataType::OrderBookDelta.into(),
1996 Environment::Backtest,
1997 None,
1998 false,
1999 )
2000 .unwrap_err()
2001 };
2002
2003 assert!(
2004 format!("{failure:#}").contains(
2005 "cannot relabel precision metadata for a promotion group that contains decimal values"
2006 ),
2007 "{failure:#}"
2008 );
2009 assert_eq!(fs::read(source).unwrap(), original);
2010 assert_eq!(
2011 catalog
2012 .query_files(&NautilusDataType::OrderBookDelta.into(), None, None, None)
2013 .unwrap(),
2014 Vec::<String>::new()
2015 );
2016 }
2017
2018 #[rstest]
2019 #[case(true)]
2020 #[case(false)]
2021 fn parquet_promotion_unifies_empty_and_populated_depth(#[case] automatic: bool) {
2022 use nautilus_model::{
2023 data::{BookOrder, OrderBookDepth},
2024 enums::OrderSide,
2025 };
2026 let directory = TempDir::new().unwrap();
2027 let staging = directory.path().join("backtest/run-depth-ties");
2028 let mut config =
2029 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2030 config.delete_feather_after_promotion = true;
2031 config.promote_on_close = automatic;
2032 let mut sink =
2033 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2034 .unwrap();
2035 let id = InstrumentId::from("AUD/USD.SIM");
2036 let empty =
2037 OrderBookDepth::new(id, vec![], vec![], vec![], vec![], 1, 2, 3.into(), 4.into());
2038 let order = BookOrder::new(
2039 OrderSide::Buy,
2040 Price::from("1.23"),
2041 Quantity::from("4.5"),
2042 6,
2043 );
2044 let populated = OrderBookDepth::new(
2045 id,
2046 vec![order],
2047 vec![],
2048 vec![7],
2049 vec![],
2050 8,
2051 9,
2052 3.into(),
2053 4.into(),
2054 );
2055 sink.write_any(&empty).unwrap();
2056 sink.write_any(&populated).unwrap();
2057 sink.close().unwrap();
2058
2059 let mut catalog = ParquetDataCatalog::from_uri(
2060 directory.path().to_str().unwrap(),
2061 None,
2062 None,
2063 None,
2064 None,
2065 )
2066 .unwrap();
2067
2068 if !automatic {
2069 catalog
2070 .convert_stream_to_data(
2071 "run-depth-ties",
2072 &NautilusDataType::OrderBookDepth.into(),
2073 Environment::Backtest,
2074 None,
2075 false,
2076 )
2077 .unwrap();
2078 }
2079
2080 let batch = catalog
2081 .query_batch(&CatalogQuery::new(NautilusDataType::OrderBookDepth))
2082 .unwrap();
2083 let DataBatch::BookDepth(rows) = batch else {
2084 panic!("expected order book depth");
2085 };
2086 let mut rows = rows.as_ref().to_vec();
2087 rows.sort_by_key(|depth| depth.sequence);
2088 assert_eq!(rows, vec![empty, populated]);
2089
2090 let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
2091 let staged = block_on_nautilus_with(|| storage.list_files("", Some(".feather"))).unwrap();
2092 assert_eq!(staged.len(), usize::from(!automatic));
2093 }
2094
2095 #[rstest]
2096 fn parquet_promotion_preserves_clear_snapshots(
2097 #[values(false, true)] automatic: bool,
2098 #[values(false, true)] batch_write: bool,
2099 ) {
2100 let directory = TempDir::new().unwrap();
2101 let staging = directory.path().join("backtest").join("run-clear");
2102 let mut config =
2103 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2104 config.promote_on_close = automatic;
2105 let mut sink =
2106 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2107 .unwrap();
2108 let id = InstrumentId::from("AUD/USD.SIM");
2109 let mut deltas = Vec::new();
2110 for (sequence, timestamp, price, size) in [
2111 (41, 100, "1.23", "4.500"),
2112 (42, 200, "1.24", "6.700"),
2113 (43, 300, "1.2345", "8.90000"),
2114 ] {
2115 deltas.push(OrderBookDelta::clear(
2116 id,
2117 sequence,
2118 (timestamp - 1).into(),
2119 timestamp.into(),
2120 ));
2121
2122 for (side, order_id, flags) in [
2123 (OrderSide::Buy, 71, RecordFlag::F_SNAPSHOT as u8),
2124 (
2125 OrderSide::Sell,
2126 72,
2127 RecordFlag::F_SNAPSHOT as u8 | RecordFlag::F_LAST as u8,
2128 ),
2129 ] {
2130 deltas.push(OrderBookDelta::new(
2131 id,
2132 BookAction::Add,
2133 BookOrder::new(side, Price::from(price), Quantity::from(size), order_id),
2134 flags,
2135 sequence,
2136 (timestamp - 1).into(),
2137 timestamp.into(),
2138 ));
2139 }
2140 }
2141
2142 deltas.push(OrderBookDelta::clear(id, 44, 399.into(), 400.into()));
2143 if batch_write {
2144 sink.write_batch(deltas.iter().copied().map(Data::from).collect())
2145 .unwrap();
2146 } else {
2147 for delta in &deltas {
2148 sink.write_data(Data::BookDelta(*delta)).unwrap();
2149 }
2150 }
2151
2152 sink.close().unwrap();
2153 let mut catalog = ParquetDataCatalog::from_uri(
2154 directory.path().to_str().unwrap(),
2155 None,
2156 None,
2157 None,
2158 None,
2159 )
2160 .unwrap();
2161
2162 if !automatic {
2163 catalog
2164 .convert_stream_to_data(
2165 "run-clear",
2166 &NautilusDataType::OrderBookDelta.into(),
2167 Environment::Backtest,
2168 None,
2169 false,
2170 )
2171 .unwrap();
2172 }
2173
2174 let batch = catalog
2175 .query_batch(&CatalogQuery::new(NautilusDataType::OrderBookDelta))
2176 .unwrap();
2177
2178 let DataBatch::BookDelta(rows) = batch else {
2179 panic!("expected order book deltas");
2180 };
2181
2182 assert_eq!(rows.len(), deltas.len());
2183
2184 for (actual, expected) in rows.as_ref().iter().zip(&deltas) {
2185 let (price_precision, size_precision) = if expected.sequence < 43 {
2186 (2, 3)
2187 } else {
2188 (4, 5)
2189 };
2190
2191 assert_eq!(actual.instrument_id, expected.instrument_id);
2192 assert_eq!(actual.action, expected.action);
2193 assert_eq!(actual.order.side, expected.order.side);
2194 assert_eq!(actual.order.price.raw(), expected.order.price.raw());
2195 assert_eq!(actual.order.price.precision, price_precision);
2196 assert_eq!(actual.order.size.raw(), expected.order.size.raw());
2197 assert_eq!(actual.order.size.precision, size_precision);
2198 assert_eq!(actual.order.order_id, expected.order.order_id);
2199 assert_eq!(actual.flags, expected.flags);
2200 assert_eq!(actual.sequence, expected.sequence);
2201 assert_eq!(actual.ts_event, expected.ts_event);
2202 assert_eq!(actual.ts_init, expected.ts_init);
2203 }
2204 }
2205
2206 #[rstest]
2207 #[case::real_add(BookAction::Add, Some(OrderSide::Buy), "1", "2", 0)]
2208 #[case::clear_side(BookAction::Clear, Some(OrderSide::Buy), "0", "0", 0)]
2209 #[case::clear_price(BookAction::Clear, None, "1", "0", 0)]
2210 #[case::clear_size(BookAction::Clear, None, "0", "1", 0)]
2211 #[case::clear_order_id(BookAction::Clear, None, "0", "0", 1)]
2212 fn parquet_promotion_rejects_noncanonical_zero_precision_orders(
2213 #[case] action: BookAction,
2214 #[case] side: Option<OrderSide>,
2215 #[case] price: &str,
2216 #[case] size: &str,
2217 #[case] order_id: u64,
2218 #[values(false, true)] automatic: bool,
2219 ) {
2220 let directory = TempDir::new().unwrap();
2221 let staging = directory.path().join("backtest").join("run-clear-invalid");
2222 let mut config =
2223 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2224 config.promote_on_close = automatic;
2225 let mut sink =
2226 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2227 .unwrap();
2228 let id = InstrumentId::from("AUD/USD.SIM");
2229 let deltas = vec![
2230 OrderBookDelta::clear(id, 41, 99.into(), 100.into()),
2231 OrderBookDelta::new(
2232 id,
2233 action,
2234 BookOrder {
2235 side,
2236 price: Price::from(price),
2237 size: Quantity::from(size),
2238 order_id,
2239 },
2240 0,
2241 41,
2242 99.into(),
2243 100.into(),
2244 ),
2245 OrderBookDelta::new(
2246 id,
2247 BookAction::Add,
2248 BookOrder::new(
2249 OrderSide::Sell,
2250 Price::from("1.23"),
2251 Quantity::from("4.500"),
2252 72,
2253 ),
2254 RecordFlag::F_LAST as u8,
2255 41,
2256 99.into(),
2257 100.into(),
2258 ),
2259 ];
2260 sink.write_batch(deltas.into_iter().map(Data::from).collect())
2261 .unwrap();
2262 let mut catalog = ParquetDataCatalog::from_uri(
2263 directory.path().to_str().unwrap(),
2264 None,
2265 None,
2266 None,
2267 None,
2268 )
2269 .unwrap();
2270
2271 let error = if automatic {
2272 sink.close().unwrap_err()
2273 } else {
2274 sink.close().unwrap();
2275 catalog
2276 .convert_stream_to_data(
2277 "run-clear-invalid",
2278 &NautilusDataType::OrderBookDelta.into(),
2279 Environment::Backtest,
2280 None,
2281 false,
2282 )
2283 .unwrap_err()
2284 };
2285
2286 assert!(error.to_string().contains(
2287 "cannot relabel precision metadata for a promotion group that contains decimal values"
2288 ));
2289 assert!(
2290 catalog
2291 .query_files(&NautilusDataType::OrderBookDelta.into(), None, None, None)
2292 .unwrap()
2293 .is_empty()
2294 );
2295 }
2296
2297 #[rstest]
2298 fn parquet_manual_conversion_promotes_custom_data() {
2299 use nautilus_model::data::{CustomData, DataType};
2300 use nautilus_serialization::ensure_custom_data_registered;
2301
2302 ensure_custom_data_registered::<RustTestHashMapCustomData>();
2303
2304 let directory = TempDir::new().unwrap();
2305 let staging = directory.path().join("backtest").join("run-custom");
2306 let mut config =
2307 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2308 config.promote_on_close = false;
2309 let mut sink =
2310 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2311 .unwrap();
2312
2313 let data_type = DataType::new("RustTestHashMapCustomData", None, None);
2314 let records = [
2315 sample_custom_data("first", "AUD/USD.SIM", "1.23456", 11),
2316 sample_custom_data("second", "BTCUSDT.BINANCE", "65432.10", 21),
2317 ];
2318
2319 for record in &records {
2320 sink.write_data(Data::Custom(CustomData::new(
2321 Arc::new(record.clone()),
2322 data_type.clone(),
2323 )))
2324 .unwrap();
2325 }
2326
2327 sink.close().unwrap();
2328
2329 let mut catalog = ParquetDataCatalog::from_uri(
2330 directory.path().to_str().unwrap(),
2331 None,
2332 None,
2333 None,
2334 None,
2335 )
2336 .unwrap();
2337
2338 for _ in 0..2 {
2339 catalog
2340 .convert_stream_to_data(
2341 "run-custom",
2342 &NautilusDataType::Custom {
2343 type_name: "RustTestHashMapCustomData".to_string(),
2344 }
2345 .into(),
2346 Environment::Backtest,
2347 None,
2348 false,
2349 )
2350 .unwrap();
2351 }
2352
2353 let expected: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2354 .iter()
2355 .map(|record| (None, record.clone()))
2356 .collect();
2357 assert_eq!(query_custom_records(&mut catalog, None), expected);
2358 }
2359
2360 #[rstest]
2361 fn parquet_manual_conversion_filters_custom_data_identifiers() {
2362 use nautilus_model::data::{CustomData, DataType};
2363 use nautilus_serialization::ensure_custom_data_registered;
2364
2365 ensure_custom_data_registered::<RustTestHashMapCustomData>();
2366
2367 let directory = TempDir::new().unwrap();
2368 let staging = directory.path().join("backtest").join("run-custom-ids");
2369 let mut config =
2370 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2371 config.promote_on_close = false;
2372 let mut sink =
2373 parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2374 .unwrap();
2375
2376 let audusd = "AUD/USD.SIM";
2377 let btcusdt = "BTCUSDT.BINANCE";
2378 let records = [
2379 (audusd, sample_custom_data("first", audusd, "1.23456", 11)),
2380 (audusd, sample_custom_data("second", audusd, "1.23457", 21)),
2381 (
2382 btcusdt,
2383 sample_custom_data("third", btcusdt, "65432.10", 31),
2384 ),
2385 ];
2386
2387 for (identifier, record) in &records {
2388 let data_type = DataType::new(
2389 "RustTestHashMapCustomData",
2390 None,
2391 Some((*identifier).to_string()),
2392 );
2393 sink.write_data(Data::Custom(CustomData::new(
2394 Arc::new(record.clone()),
2395 data_type,
2396 )))
2397 .unwrap();
2398 }
2399
2400 sink.close().unwrap();
2401
2402 let mut catalog = ParquetDataCatalog::from_uri(
2403 directory.path().to_str().unwrap(),
2404 None,
2405 None,
2406 None,
2407 None,
2408 )
2409 .unwrap();
2410 catalog
2411 .convert_stream_to_data(
2412 "run-custom-ids",
2413 &NautilusDataType::Custom {
2414 type_name: "RustTestHashMapCustomData".to_string(),
2415 }
2416 .into(),
2417 Environment::Backtest,
2418 Some(&[audusd.to_string()]),
2419 false,
2420 )
2421 .unwrap();
2422
2423 let expected_audusd: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2424 .iter()
2425 .filter(|(identifier, _)| *identifier == audusd)
2426 .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2427 .collect();
2428 assert_eq!(query_custom_records(&mut catalog, None), expected_audusd);
2429
2430 assert_eq!(
2433 query_custom_records(&mut catalog, Some(&["AUDUSD.SIM".to_string()])),
2434 expected_audusd,
2435 );
2436
2437 catalog
2438 .convert_stream_to_data(
2439 "run-custom-ids",
2440 &NautilusDataType::Custom {
2441 type_name: "RustTestHashMapCustomData".to_string(),
2442 }
2443 .into(),
2444 Environment::Backtest,
2445 None,
2446 false,
2447 )
2448 .unwrap();
2449
2450 let expected_all: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2451 .iter()
2452 .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2453 .collect();
2454 assert_eq!(query_custom_records(&mut catalog, None), expected_all);
2455 let expected_btcusdt: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2456 .iter()
2457 .filter(|(identifier, _)| *identifier == btcusdt)
2458 .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2459 .collect();
2460 assert_eq!(
2461 query_custom_records(&mut catalog, Some(&[btcusdt.to_string()])),
2462 expected_btcusdt,
2463 );
2464 }
2465
2466 fn sample_custom_data(
2467 name: &str,
2468 instrument_id: &str,
2469 price: &str,
2470 ts: u64,
2471 ) -> RustTestHashMapCustomData {
2472 RustTestHashMapCustomData {
2473 name: name.to_string(),
2474 prices: [(instrument_id.to_string(), Price::from(price))]
2475 .into_iter()
2476 .collect(),
2477 ts_event: UnixNanos::from(ts - 1),
2478 ts_init: UnixNanos::from(ts),
2479 }
2480 }
2481
2482 fn query_custom_records(
2483 catalog: &mut ParquetDataCatalog,
2484 identifiers: Option<&[String]>,
2485 ) -> Vec<(Option<String>, RustTestHashMapCustomData)> {
2486 let loaded = catalog
2487 .query_custom_data_dynamic(
2488 "RustTestHashMapCustomData",
2489 identifiers,
2490 None,
2491 None,
2492 None,
2493 None,
2494 true,
2495 )
2496 .unwrap();
2497 let mut decoded = Vec::new();
2498
2499 for data in &loaded {
2500 let Data::Custom(custom) = data else {
2501 panic!("expected custom data, found {data:?}")
2502 };
2503
2504 assert_eq!(custom.data_type.type_name(), "RustTestHashMapCustomData");
2505 decoded.push((
2506 custom.data_type.identifier().map(ToString::to_string),
2507 custom
2508 .data
2509 .as_any()
2510 .downcast_ref::<RustTestHashMapCustomData>()
2511 .expect("expected RustTestHashMapCustomData")
2512 .clone(),
2513 ));
2514 }
2515
2516 decoded.sort_by_key(|(_, record)| record.ts_init);
2517 decoded
2518 }
2519
2520 fn local_catalog(directory: &TempDir) -> CatalogConnectConfig {
2521 CatalogConnectConfig::new(directory.path().to_string_lossy(), None)
2522 }
2523
2524 fn stage_per_identifier_quote(staging: &Path) -> (QuoteTick, PathBuf) {
2526 let mut quote = sample_quote();
2527 quote.instrument_id = InstrumentId::from("CAFÉ.SIM");
2528
2529 let mut writer = FeatherWriter::new(
2530 staging.to_path_buf(),
2531 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2532 RotationConfig::NoRotation,
2533 None,
2534 None,
2535 );
2536 writer.write(quote).unwrap();
2537 writer.close().unwrap();
2538
2539 let identifier_directory = staging.join("quotes").join("CAFÉ.SIM");
2540 let staged_file = identifier_directory.join("quotes_0.feather");
2541 fs::create_dir(&identifier_directory).unwrap();
2542 fs::rename(
2543 staging.join("quotes").join("quotes_0.feather"),
2544 &staged_file,
2545 )
2546 .unwrap();
2547
2548 (quote, staged_file)
2549 }
2550
2551 fn sample_quote() -> QuoteTick {
2552 QuoteTick::new(
2553 InstrumentId::from("AUD/USD.SIM"),
2554 Price::from("0.65"),
2555 Price::from("0.67"),
2556 Quantity::from("11"),
2557 Quantity::from("17"),
2558 UnixNanos::from(19),
2559 UnixNanos::from(23),
2560 )
2561 }
2562
2563 #[rstest]
2564 #[case::ascii("AUD/USD.SIM-1-MINUTE-LAST-EXTERNAL")]
2565 #[case::non_ascii("CAFÉ.SIM-1-MINUTE-LAST-EXTERNAL")]
2566 fn bar_replay_across_promotions_is_idempotent(
2567 #[case] bar_type: &str,
2568 #[values(false, true)] restart: bool,
2569 #[values(false, true)] replay_only: bool,
2570 #[values(false, true)] use_ts_event: bool,
2571 ) {
2572 let directory = TempDir::new().unwrap();
2573
2574 let mut config = WriterConnectConfig::new(
2575 directory.path().join("live/run-bars").to_string_lossy(),
2576 Some(local_catalog(&directory)),
2577 );
2578 config.use_ts_event_for_ts_init = use_ts_event;
2579 config.delete_feather_after_promotion = true;
2580 config.promote_on_close = false;
2581 let clock = Arc::new(AtomicU64::new(1));
2582 let mut writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2583
2584 let first = Bar {
2585 bar_type: BarType::from(bar_type),
2586 ..sample_bar()
2587 };
2588
2589 writer.core.write_data(Data::Bar(first)).unwrap();
2590 assert_eq!(writer.promote_now().unwrap().len(), 1);
2591
2592 if restart {
2593 drop(writer);
2594 writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2595 }
2596
2597 clock.store(2, Ordering::Relaxed);
2598
2599 let replay = Bar {
2600 ts_init: if use_ts_event {
2601 UnixNanos::from(63_000_000_000)
2602 } else {
2603 first.ts_init
2604 },
2605 ..first
2606 };
2607
2608 let next = Bar {
2609 ts_event: UnixNanos::from(120_000_000_000),
2610 ts_init: UnixNanos::from(121_000_000_000),
2611 ..first
2612 };
2613
2614 writer.core.write_data(Data::Bar(replay)).unwrap();
2615 if !replay_only {
2616 writer.core.write_data(Data::Bar(next)).unwrap();
2617 }
2618
2619 let promoted = writer.promote_now().unwrap();
2620 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2621
2622 let DataBatch::Bar(rows) = catalog
2623 .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2624 .unwrap()
2625 else {
2626 panic!("expected bars")
2627 };
2628
2629 let mut expected = vec![Bar {
2630 ts_init: if use_ts_event {
2631 first.ts_event
2632 } else {
2633 first.ts_init
2634 },
2635 ..first
2636 }];
2637
2638 if !replay_only {
2639 expected.push(Bar {
2640 ts_init: if use_ts_event {
2641 next.ts_event
2642 } else {
2643 next.ts_init
2644 },
2645 ..next
2646 });
2647 }
2648
2649 assert_eq!(promoted.len(), 1);
2650 assert_eq!(rows.as_ref(), expected.as_slice());
2651 assert_eq!(
2652 catalog
2653 .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2654 .unwrap()
2655 .len(),
2656 expected.len()
2657 );
2658 assert!(
2659 list_session_feather_files(&writer.source.storage, Environment::Live, "run-bars")
2660 .unwrap()
2661 .is_empty()
2662 );
2663 }
2664
2665 #[rstest]
2666 #[case::open("open")]
2667 #[case::high("high")]
2668 #[case::low("low")]
2669 #[case::close("close")]
2670 #[case::volume("volume")]
2671 #[case::price_precision("price_precision")]
2672 #[case::size_precision("size_precision")]
2673 #[case::event_timestamp("ts_event")]
2674 #[case::init_timestamp("ts_init")]
2675 fn bar_replay_rejects_changed_values(#[case] field: &str) {
2676 let directory = TempDir::new().unwrap();
2677
2678 let mut config = WriterConnectConfig::new(
2679 directory.path().join("live/run-conflict").to_string_lossy(),
2680 Some(local_catalog(&directory)),
2681 );
2682 config.use_ts_event_for_ts_init = !matches!(field, "ts_event" | "ts_init");
2683 config.delete_feather_after_promotion = true;
2684 config.promote_on_close = false;
2685 let clock = Arc::new(AtomicU64::new(1));
2686 let mut writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2687 let first = sample_bar();
2688 writer.core.write_data(Data::Bar(first)).unwrap();
2689 writer.promote_now().unwrap();
2690 clock.store(2, Ordering::Relaxed);
2691 let mut correction = first;
2692 match field {
2693 "open" => correction.open = Price::from("1.02"),
2694 "high" => correction.high = Price::from("1.06"),
2695 "low" => correction.low = Price::from("0.98"),
2696 "close" => correction.close = Price::from("1.04"),
2697 "volume" => correction.volume = Quantity::from("18.000"),
2698 "price_precision" => {
2699 correction.open = Price::from("1.010");
2700 correction.high = Price::from("1.050");
2701 correction.low = Price::from("0.990");
2702 correction.close = Price::from("1.030");
2703 }
2704 "size_precision" => correction.volume = Quantity::from("17.0000"),
2705 "ts_event" => correction.ts_event = UnixNanos::from(59_000_000_000),
2706 "ts_init" => correction.ts_init = UnixNanos::from(60_500_000_000),
2707 _ => unreachable!(),
2708 }
2709
2710 let next = Bar {
2711 ts_event: UnixNanos::from(120_000_000_000),
2712 ts_init: UnixNanos::from(121_000_000_000),
2713 ..first
2714 };
2715
2716 writer.core.write_data(Data::Bar(correction)).unwrap();
2717 writer.core.write_data(Data::Bar(next)).unwrap();
2718 let work = writer.prepare_promotion().unwrap().unwrap();
2719 let files = work.files().to_vec();
2720 let error = writer.finalize(work.execute()).unwrap_err();
2721 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2722
2723 let DataBatch::Bar(rows) = catalog
2724 .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2725 .unwrap()
2726 else {
2727 panic!("expected bars")
2728 };
2729
2730 let expected = if config.use_ts_event_for_ts_init {
2731 Bar {
2732 ts_init: first.ts_event,
2733 ..first
2734 }
2735 } else {
2736 first
2737 };
2738
2739 assert!(error.to_string().contains("non-disjoint intervals"));
2740 assert_eq!(rows.as_ref(), &[expected]);
2741 assert_eq!(
2742 catalog
2743 .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2744 .unwrap()
2745 .len(),
2746 1
2747 );
2748 assert_eq!(files.len(), 1);
2749 assert!(directory.path().join(&files[0]).exists());
2750 }
2751
2752 #[rstest]
2753 fn bar_replay_manual_conversion_is_idempotent() {
2754 let directory = TempDir::new().unwrap();
2755 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2756 let first = sample_bar();
2757
2758 let next = Bar {
2759 ts_event: UnixNanos::from(120_000_000_000),
2760 ts_init: UnixNanos::from(121_000_000_000),
2761 ..first
2762 };
2763
2764 for run in ["first", "restart"] {
2765 let mut writer = FeatherWriter::new(
2766 directory.path().join("backtest").join(run),
2767 WriterClock::Test(Arc::new(AtomicU64::new(0))),
2768 RotationConfig::NoRotation,
2769 None,
2770 None,
2771 );
2772 writer.write(first).unwrap();
2773 if run == "restart" {
2774 writer.write(next).unwrap();
2775 }
2776
2777 writer.close().unwrap();
2778 catalog
2779 .convert_stream_to_data(
2780 run,
2781 &NautilusDataType::Bar.into(),
2782 Environment::Backtest,
2783 None,
2784 true,
2785 )
2786 .unwrap();
2787 }
2788
2789 let DataBatch::Bar(rows) = catalog
2790 .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2791 .unwrap()
2792 else {
2793 panic!("expected bars")
2794 };
2795
2796 assert_eq!(
2797 rows.as_ref(),
2798 &[
2799 Bar {
2800 ts_init: first.ts_event,
2801 ..first
2802 },
2803 Bar {
2804 ts_init: next.ts_event,
2805 ..next
2806 },
2807 ]
2808 );
2809 assert_eq!(
2810 catalog
2811 .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2812 .unwrap()
2813 .len(),
2814 2
2815 );
2816 }
2817
2818 fn sample_bar() -> Bar {
2819 Bar::new(
2820 BarType::from("AUD/USD.SIM-1-MINUTE-LAST-EXTERNAL"),
2821 Price::from("1.01"),
2822 Price::from("1.05"),
2823 Price::from("0.99"),
2824 Price::from("1.03"),
2825 Quantity::from("17.000"),
2826 UnixNanos::from(60_000_000_000),
2827 UnixNanos::from(61_000_000_000),
2828 )
2829 }
2830
2831 #[rstest]
2832 fn kept_promotions_release_scheduled_paths() {
2833 let directory = TempDir::new().unwrap();
2834 let config = WriterConnectConfig::new(
2835 directory.path().join("live/run-bounded").to_string_lossy(),
2836 Some(local_catalog(&directory)),
2837 );
2838 let mut writer = ParquetWriter::new(&config, WriterClock::Live).unwrap();
2839
2840 for timestamp in 1..=32 {
2841 writer
2842 .core
2843 .write_data(Data::Quote(QuoteTick {
2844 ts_event: UnixNanos::from(timestamp),
2845 ts_init: UnixNanos::from(timestamp),
2846 ..sample_quote()
2847 }))
2848 .unwrap();
2849 let work = writer.prepare_promotion().unwrap().unwrap();
2850 assert_eq!(work.files().len(), 1);
2851 writer.finalize(work.execute()).unwrap();
2852 assert_scheduled_count(&writer.core.promotion_driver.schedule(), 0);
2853 assert!(writer.prepare_promotion().unwrap().is_none());
2854 }
2855
2856 assert_eq!(
2857 list_session_feather_files(&writer.source.storage, Environment::Live, "run-bounded")
2858 .unwrap()
2859 .len(),
2860 32
2861 );
2862 }
2863
2864 #[rstest]
2865 #[case::timer_seal(RotationConfig::NoRotation, false, 1)]
2866 #[case::close(RotationConfig::NoRotation, true, 1)]
2867 #[case::rotation(RotationConfig::Size { max_size: 1 }, false, 2)]
2868 fn sealed_paths_promote_once_and_do_not_resubmit_in_flight(
2869 #[case] rotation: RotationConfig,
2870 #[case] close: bool,
2871 #[case] expected_files: usize,
2872 ) {
2873 let directory = TempDir::new().unwrap();
2874 let mut config = WriterConnectConfig::new(
2875 directory.path().join("live/run-sealing").to_string_lossy(),
2876 Some(local_catalog(&directory)),
2877 );
2878 config.rotation_config = rotation;
2879 let mut writer = ParquetWriter::new(&config, WriterClock::Live).unwrap();
2880 let first = sample_quote();
2881 let second = QuoteTick {
2882 ts_event: UnixNanos::from(29),
2883 ts_init: UnixNanos::from(31),
2884 ..first
2885 };
2886 writer.core.write_data(Data::Quote(first)).unwrap();
2887 writer.core.write_data(Data::Quote(second)).unwrap();
2888
2889 if close {
2890 writer.core.close().unwrap();
2891 }
2892
2893 let work = writer.prepare_promotion().unwrap().unwrap();
2894 let files = work.files().to_vec();
2895 assert_eq!(files.len(), expected_files);
2896 assert!(writer.prepare_promotion().unwrap().is_none());
2897 assert_scheduled_count(&writer.core.promotion_driver.schedule(), expected_files);
2898 let result = work.execute();
2899 assert_scheduled_count(&writer.core.promotion_driver.schedule(), 0);
2900 assert_eq!(result.committed_paths, files);
2901 assert_eq!(writer.finalize(result).unwrap().len(), expected_files);
2902 assert!(writer.prepare_promotion().unwrap().is_none());
2903
2904 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2905 let DataBatch::Quote(rows) = catalog
2906 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2907 .unwrap()
2908 else {
2909 panic!("expected quotes")
2910 };
2911 assert_eq!(rows.as_ref(), &[first, second]);
2912 }
2913
2914 #[rstest]
2915 #[case::space("my data")]
2916 #[case::non_ascii("Données")]
2917 #[case::fragment("data#1")]
2918 #[case::literal_percent("data%20")]
2919 fn writer_root_path_promotes_and_replays(
2920 #[case] root_name: &str,
2921 #[values(false, true)] file_uri: bool,
2922 ) {
2923 let directory = TempDir::new().unwrap();
2924 let root = directory.path().join(root_name);
2925 let staging = root.join("live/run-root");
2926 let staging_uri = if file_uri {
2927 normalize_path_to_uri(&staging.to_string_lossy()).unwrap()
2928 } else {
2929 staging.to_string_lossy().into_owned()
2930 };
2931 let config = WriterConnectConfig::new(staging_uri, Some(local_catalog(&directory)));
2932 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2933 let quote = sample_quote();
2934 let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
2935 writer.core.write_data(Data::Quote(quote)).unwrap();
2936 let work = writer.prepare_promotion().unwrap().unwrap();
2937 assert_eq!(work.files(), &["live/run-root/quotes/quotes_0.feather"]);
2938 let promoted = writer.finalize(work.execute()).unwrap();
2939 assert_eq!(promoted.len(), 1);
2940 assert!(writer.prepare_promotion().unwrap().is_none());
2941 drop(writer);
2942
2943 let mut restarted = ParquetWriter::new(&config, clock).unwrap();
2944 let work = restarted.prepare_promotion().unwrap().unwrap();
2945 assert_eq!(work.files(), &["live/run-root/quotes/quotes_0.feather"]);
2946 assert_eq!(restarted.finalize(work.execute()).unwrap(), promoted);
2947 assert!(restarted.prepare_promotion().unwrap().is_none());
2948
2949 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2950 let DataBatch::Quote(rows) = catalog
2951 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2952 .unwrap()
2953 else {
2954 panic!("expected quotes")
2955 };
2956 assert_eq!(rows.as_ref(), &[quote]);
2957 }
2958
2959 #[rstest]
2960 fn parent_relative_writer_path_promotes_and_replays() {
2961 let directory = TempDir::new_in(".").unwrap();
2962 let current = std::env::current_dir().unwrap();
2963 let staging = PathBuf::from("..")
2964 .join(current.file_name().unwrap())
2965 .join(directory.path().file_name().unwrap())
2966 .join("live/run-relative");
2967 let config =
2968 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2969 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2970 let quote = sample_quote();
2971 let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
2972 writer.core.write_data(Data::Quote(quote)).unwrap();
2973 let work = writer.prepare_promotion().unwrap().unwrap();
2974 assert_eq!(work.files(), &["live/run-relative/quotes/quotes_0.feather"]);
2975 let promoted = writer.finalize(work.execute()).unwrap();
2976 assert_eq!(promoted.len(), 1);
2977 assert!(writer.prepare_promotion().unwrap().is_none());
2978 drop(writer);
2979
2980 let mut restarted = ParquetWriter::new(&config, clock).unwrap();
2981 let work = restarted.prepare_promotion().unwrap().unwrap();
2982 assert_eq!(work.files(), &["live/run-relative/quotes/quotes_0.feather"]);
2983 assert_eq!(restarted.finalize(work.execute()).unwrap(), promoted);
2984 assert!(restarted.prepare_promotion().unwrap().is_none());
2985
2986 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2987 let DataBatch::Quote(rows) = catalog
2988 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2989 .unwrap()
2990 else {
2991 panic!("expected quotes")
2992 };
2993 assert_eq!(rows.as_ref(), &[quote]);
2994 }
2995
2996 #[rstest]
2997 #[case::sealed(false)]
2998 #[case::partial(true)]
2999 fn restart_discovers_crashed_files_and_deduplicates_promotions(
3000 #[case] partial: bool,
3001 #[values(false, true)] delete_source: bool,
3002 #[values(false, true)] restage: bool,
3003 ) {
3004 let directory = TempDir::new().unwrap();
3005 let staging = directory.path().join("backtest/run-restart");
3006 let quote = sample_quote();
3007 let mut crashed = FeatherWriter::new(
3008 staging.clone(),
3009 WriterClock::Test(Arc::new(AtomicU64::new(0))),
3010 RotationConfig::NoRotation,
3011 None,
3012 None,
3013 );
3014 crashed.write(quote).unwrap();
3015 crashed.close().unwrap();
3016 let sealed = staging.join("quotes/quotes_0.feather");
3017
3018 if partial {
3019 fs::rename(&sealed, sealed.with_extension(FEATHER_PARTIAL_EXTENSION)).unwrap();
3020 }
3021
3022 drop(crashed);
3023 let mut config =
3024 WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
3025 config.delete_feather_after_promotion = restage;
3026 let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
3027 let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
3028 let source_bytes = fs::read(&sealed).unwrap();
3029 let work = writer.prepare_promotion().unwrap().unwrap();
3030 assert_eq!(
3031 work.files(),
3032 &["backtest/run-restart/quotes/quotes_0.feather".to_string()]
3033 );
3034 let promoted = writer.finalize(work.execute()).unwrap();
3035 assert_eq!(promoted.len(), 1);
3036 assert!(writer.prepare_promotion().unwrap().is_none());
3037 drop(writer);
3038
3039 if restage {
3040 assert!(!sealed.exists());
3041 fs::write(&sealed, source_bytes).unwrap();
3042 }
3043
3044 config.delete_feather_after_promotion = delete_source;
3045 let mut restarted = ParquetWriter::new(&config, clock).unwrap();
3046 let work = restarted.prepare_promotion().unwrap().unwrap();
3047 assert_eq!(work.files().len(), 1);
3048 let files = work.files().to_vec();
3049 let result = work.execute();
3050 assert_eq!(result.committed_paths, files);
3051 assert_eq!(
3052 result.deleted_paths,
3053 if delete_source { files } else { Vec::new() }
3054 );
3055 assert_eq!(restarted.finalize(result).unwrap(), promoted);
3056 assert!(restarted.prepare_promotion().unwrap().is_none());
3057 assert_scheduled_count(&restarted.core.promotion_driver.schedule(), 0);
3058 assert_eq!(sealed.exists(), !delete_source);
3059
3060 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
3061 let DataBatch::Quote(rows) = catalog
3062 .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
3063 .unwrap()
3064 else {
3065 panic!("expected quotes")
3066 };
3067 assert_eq!(rows.as_ref(), &[quote]);
3068 }
3069
3070 #[rstest]
3071 #[case::nested_base("/prefix/catalog/", "prefix/catalog/")]
3072 #[case::root_base("", "")]
3073 fn promotion_paths_live_under_the_catalog_base_path(
3074 #[case] base_path: &str,
3075 #[case] expected_prefix: &str,
3076 ) {
3077 let directory = TempDir::new().unwrap();
3078 let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
3079 catalog.base_path = base_path.to_string();
3080
3081 let backend = ParquetPromotionBackend {
3082 catalog,
3083 legacy_manifest_missing: Arc::default(),
3084 };
3085
3086 assert_eq!(
3087 backend.manifest_path(),
3088 ObjectPath::from(format!("{expected_prefix}_nautilus_promotions.json")),
3089 );
3090 assert_eq!(
3091 backend.marker_path("replay-id"),
3092 ObjectPath::from(format!(
3093 "{expected_prefix}_nautilus_promotions/\
3094 cd3112001080bc2d11985ffe2b8b90b324d336d82c17ddb66127d3a05f08c69c.json"
3095 )),
3096 );
3097 }
3098
3099 #[rstest]
3100 fn feather_replay_identity_is_stable_and_ignores_identifier_order() {
3101 let identity = |identifiers: Option<&[String]>| {
3102 feather_replay_identity(
3103 "file:///catalog/backtest/run-1",
3104 "quotes/AUDUSD.SIM/part-0.feather",
3105 "content-hash",
3106 identifiers,
3107 )
3108 };
3109
3110 let unordered = ["B".to_string(), "A".to_string(), "A".to_string()];
3111 let ordered = ["A".to_string(), "B".to_string()];
3112
3113 let expected =
3114 "nautilus-feather:10a9435c28f7536f26653c3fc808571be89bfe769e06c7307cb4743e273a23fd";
3115
3116 assert_eq!(identity(Some(&unordered)), expected);
3117 assert_eq!(identity(Some(&ordered)), expected);
3118 assert_ne!(identity(None), expected);
3119 }
3120}