Skip to main content

nautilus_persistence/backend/parquet/
writer.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautilustrader.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8// -------------------------------------------------------------------------------------------------
9
10//! Feather-staged promoting writer for the Parquet catalog.
11
12use std::{
13    fmt::Debug,
14    sync::{
15        Arc,
16        atomic::{AtomicBool, Ordering},
17    },
18};
19
20use nautilus_common::live::block_on_nautilus_with;
21use object_store::{
22    Error as ObjectStoreError, ObjectStoreExt, PutMode, PutOptions, path::Path as ObjectPath,
23};
24use serde::{Deserialize, Serialize};
25
26use super::catalog::ParquetDataCatalog;
27use crate::{
28    catalog::factory::CatalogConnectConfig,
29    common::{
30        conversion::FeatherConversionSummary,
31        datafusion::identifiers_from_record_batches,
32        paths::{
33            catalog_data_type_from_session_feather_path, create_local_directory,
34            identifier_from_session_feather_path,
35        },
36        storage::create_storage_backend_from_path,
37    },
38    writer::{
39        factory::{PARQUET_WRITER_FACTORY_NAME, WriterConnectConfig, WriterFactoryRegistry},
40        feather::WriterClock,
41        materializer::read_feather_record_batches_with_identity,
42        promotion::{
43            PromotionBackend, PromotionResult, PromotionSession, PromotionWork,
44            StagedPromotionBackend,
45        },
46        run::{FeatherSessionSource, RunStatus},
47        staged::{StagedFeatherWriter, StagedWriter},
48        traits::StreamingDataSink,
49    },
50};
51
52pub(crate) fn register_factory(registry: &mut WriterFactoryRegistry) {
53    registry.insert(
54        PARQUET_WRITER_FACTORY_NAME.to_string(),
55        Arc::new(parquet_writer_factory),
56    );
57}
58
59fn parquet_writer_factory(
60    config: &WriterConnectConfig,
61    clock: WriterClock,
62) -> anyhow::Result<StreamingDataSink> {
63    Ok(Box::new(ParquetWriter::new(config, clock)?))
64}
65
66#[expect(
67    clippy::struct_excessive_bools,
68    reason = "the booleans are independent writer policy and lifecycle flags"
69)]
70struct ParquetWriter {
71    core: StagedFeatherWriter<ParquetPromotionBackend>,
72    session: PromotionSession,
73    source: FeatherSessionSource,
74    catalog: CatalogConnectConfig,
75    interval_ms: Option<u64>,
76    promote_on_close: bool,
77    delete_feather_after_commit: bool,
78    use_ts_event_for_ts_init: bool,
79    legacy_manifest_missing: Arc<AtomicBool>,
80    has_data: bool,
81    run_status: RunStatus,
82}
83
84impl Debug for ParquetWriter {
85    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86        f.debug_struct(stringify!(ParquetWriter))
87            .field("staging_uri", &self.core.storage.original_uri)
88            .field("catalog_uri", &self.catalog.uri)
89            .field("interval_ms", &self.interval_ms)
90            .finish_non_exhaustive()
91    }
92}
93
94impl ParquetWriter {
95    fn new(config: &WriterConnectConfig, clock: WriterClock) -> anyhow::Result<Self> {
96        let catalog = config
97            .required_catalog(PARQUET_WRITER_FACTORY_NAME)?
98            .clone();
99        let storage = create_storage_backend_from_path(&config.uri, None)?;
100        let session = StagedFeatherWriter::<ParquetPromotionBackend>::required_session(
101            &storage.original_uri,
102            "Parquet writer URI",
103        )?;
104        let source_storage = create_storage_backend_from_path(&session.root_uri, None)?;
105
106        let source = FeatherSessionSource::new(
107            source_storage,
108            session.environment,
109            session.instance_id.clone(),
110        );
111
112        // A Parquet catalog opens only an existing local directory, so the writer creates the one
113        // it promotes into, then opens it now rather than failing at the first promotion
114        create_local_directory(&catalog.uri)?;
115
116        let legacy_manifest_missing = Arc::new(AtomicBool::new(false));
117
118        ParquetPromotionBackend::new(&catalog, Arc::clone(&legacy_manifest_missing))?;
119
120        let mut core = StagedFeatherWriter::new(
121            storage.clone(),
122            &source,
123            clock,
124            config.rotation_config.clone(),
125            None,
126            config.flush_interval_ms,
127            config.record_filter.clone(),
128        )?;
129
130        block_on_nautilus_with(|| {
131            storage.write_current_run_manifest(
132                session.environment,
133                &session.instance_id,
134                RunStatus::InProgress,
135                true,
136            )
137        })?;
138
139        let timer_catalog = catalog.clone();
140        core.warn_if_orphan_feather_present("Parquet");
141        let timer_legacy_manifest_missing = Arc::clone(&legacy_manifest_missing);
142        core.start_promotion_timer(
143            config.promotion_interval_ms,
144            source.clone(),
145            "parquet-promotion",
146            move || {
147                ParquetPromotionBackend::new(
148                    &timer_catalog,
149                    Arc::clone(&timer_legacy_manifest_missing),
150                )
151            },
152            config.use_ts_event_for_ts_init,
153            config.delete_feather_after_promotion,
154        )?;
155
156        Ok(Self {
157            core,
158            catalog,
159            session,
160            source,
161            interval_ms: config.promotion_interval_ms,
162            promote_on_close: config.promote_on_close,
163            delete_feather_after_commit: config.delete_feather_after_promotion,
164            use_ts_event_for_ts_init: config.use_ts_event_for_ts_init,
165            legacy_manifest_missing,
166            has_data: false,
167            run_status: RunStatus::InProgress,
168        })
169    }
170
171    fn record_status(&mut self, status: RunStatus) -> anyhow::Result<()> {
172        if self.run_status == status {
173            return Ok(());
174        }
175
176        block_on_nautilus_with(|| {
177            self.core.storage.write_current_run_manifest(
178                self.session.environment,
179                &self.session.instance_id,
180                status,
181                !self.has_data,
182            )
183        })?;
184
185        self.run_status = status;
186        Ok(())
187    }
188
189    fn mark_non_empty(&mut self) -> anyhow::Result<()> {
190        if self.has_data {
191            return Ok(());
192        }
193
194        block_on_nautilus_with(|| {
195            self.core.storage.write_current_run_manifest(
196                self.session.environment,
197                &self.session.instance_id,
198                RunStatus::InProgress,
199                false,
200            )
201        })?;
202
203        self.has_data = true;
204        Ok(())
205    }
206
207    fn interval_ns(&self) -> Option<u64> {
208        self.interval_ms
209            .filter(|interval_ms| *interval_ms > 0)
210            .map(|interval_ms| interval_ms.saturating_mul(1_000_000))
211    }
212
213    fn prepare_promotion(&self) -> anyhow::Result<Option<PromotionWork<ParquetPromotionBackend>>> {
214        let backend =
215            ParquetPromotionBackend::new(&self.catalog, Arc::clone(&self.legacy_manifest_missing))?;
216        self.core.prepare_promotion(
217            backend,
218            self.source.clone(),
219            self.use_ts_event_for_ts_init,
220            self.delete_feather_after_commit,
221        )
222    }
223
224    fn finalize(
225        &mut self,
226        result: PromotionResult,
227    ) -> anyhow::Result<Vec<FeatherConversionSummary>> {
228        if result.converted.is_err() {
229            if result.run_state_recorded {
230                self.run_status = RunStatus::Failed;
231            } else if let Err(e) = self.record_status(RunStatus::Failed) {
232                log::warn!("Failed to record Parquet run Failed state: {e}");
233            }
234        } else if result.run_state_recorded {
235            self.run_status = RunStatus::Promoted;
236        }
237
238        self.core.finalize_promotion(result)
239    }
240
241    fn drain_completed(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
242        let mut converted = Vec::new();
243        for result in self.core.promotion_driver.drain_completed()? {
244            converted.extend(self.finalize(result)?);
245        }
246
247        Ok(converted)
248    }
249
250    fn wait(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
251        let mut converted = Vec::new();
252        for result in self.core.promotion_driver.wait()? {
253            converted.extend(self.finalize(result)?);
254        }
255
256        Ok(converted)
257    }
258
259    fn promote_now(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
260        let mut converted = self.wait()?;
261        if let Some(work) = self.prepare_promotion()? {
262            converted.extend(self.finalize(work.execute())?);
263        }
264
265        Ok(converted)
266    }
267}
268
269impl StagedWriter for ParquetWriter {
270    type Backend = ParquetPromotionBackend;
271
272    fn staged(&mut self) -> &mut StagedFeatherWriter<Self::Backend> {
273        &mut self.core
274    }
275
276    fn mark_non_empty(&mut self) -> anyhow::Result<()> {
277        Self::mark_non_empty(self)
278    }
279
280    fn maybe_promote(&mut self) -> anyhow::Result<()> {
281        self.drain_completed()?;
282
283        let Some(interval_ns) = self.interval_ns() else {
284            return Ok(());
285        };
286
287        if self
288            .core
289            .promotion_driver
290            .is_due(self.core.clock.timestamp_ns(), interval_ns)
291            && let Some(work) = self.prepare_promotion()?
292        {
293            self.core
294                .promotion_driver
295                .submit(work, "parquet-promotion")?;
296            self.core
297                .promotion_driver
298                .mark_committed_at(self.core.clock.timestamp_ns());
299        }
300
301        Ok(())
302    }
303
304    fn wait_for_promotions(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
305        self.wait()
306    }
307
308    fn promote_on_flush(&self) -> bool {
309        self.interval_ns().is_some_and(|interval_ns| {
310            self.core
311                .promotion_driver
312                .is_due(self.core.clock.timestamp_ns(), interval_ns)
313        })
314    }
315
316    fn promote_on_close(&self) -> bool {
317        self.promote_on_close
318    }
319
320    fn promote_now(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
321        Self::promote_now(self)
322    }
323
324    fn record_completed(&mut self) {
325        if self.run_status == RunStatus::InProgress
326            && let Err(e) = self.record_status(RunStatus::Completed)
327        {
328            log::warn!("Failed to complete Parquet run manifest: {e}");
329        }
330    }
331}
332
333impl Drop for ParquetWriter {
334    fn drop(&mut self) {
335        self.core.stop_promotion_timer();
336
337        let promotion_failed = if let Err(e) = self.wait() {
338            log::warn!("ParquetWriter dropped with pending promotion error: {e}");
339            true
340        } else {
341            false
342        };
343
344        if self.run_status == RunStatus::InProgress {
345            let status = if promotion_failed || !self.core.is_closed().unwrap_or(false) {
346                RunStatus::Failed
347            } else {
348                RunStatus::Completed
349            };
350
351            if let Err(e) = self.record_status(status) {
352                log::warn!(
353                    "Failed to record Parquet run {} state during drop: {e}",
354                    status.as_str()
355                );
356            }
357        }
358    }
359}
360
361const PROMOTION_MANIFEST: &str = "_nautilus_promotions.json";
362const PROMOTION_MARKERS: &str = "_nautilus_promotions";
363
364struct ParquetPromotionBackend {
365    catalog: ParquetDataCatalog,
366    legacy_manifest_missing: Arc<AtomicBool>,
367}
368
369impl ParquetPromotionBackend {
370    fn new(
371        catalog: &CatalogConnectConfig,
372        legacy_manifest_missing: Arc<AtomicBool>,
373    ) -> anyhow::Result<Self> {
374        Ok(Self {
375            catalog: super::open_catalog(catalog)?,
376            legacy_manifest_missing,
377        })
378    }
379
380    fn manifest_path(&self) -> ObjectPath {
381        self.catalog_path(PROMOTION_MANIFEST)
382    }
383
384    fn manifest(&self) -> anyhow::Result<ParquetPromotionManifest> {
385        if self.legacy_manifest_missing.load(Ordering::Relaxed) {
386            return Ok(ParquetPromotionManifest::default());
387        }
388
389        let path = self.manifest_path();
390        block_on_nautilus_with(|| async {
391            let result = match self.catalog.object_store.get(&path).await {
392                Ok(result) => result,
393                Err(ObjectStoreError::NotFound { .. }) => {
394                    self.legacy_manifest_missing.store(true, Ordering::Relaxed);
395                    return Ok(ParquetPromotionManifest::default());
396                }
397                Err(e) => return Err(e.into()),
398            };
399
400            Ok(serde_json::from_slice(&result.bytes().await?)?)
401        })
402    }
403
404    fn marker_path(&self, identity: &str) -> ObjectPath {
405        let digest = blake3::hash(identity.as_bytes()).to_hex();
406        self.catalog_path(&format!("{PROMOTION_MARKERS}/{digest}.json"))
407    }
408
409    fn catalog_path(&self, path: &str) -> ObjectPath {
410        let base = self.catalog.base_path.trim_matches('/');
411        if base.is_empty() {
412            ObjectPath::from(path)
413        } else {
414            ObjectPath::from(format!("{base}/{path}"))
415        }
416    }
417
418    fn identity_recorded(&self, identity: &str) -> anyhow::Result<bool> {
419        let path = self.marker_path(identity);
420
421        let marker_exists = block_on_nautilus_with(|| async {
422            match self.catalog.object_store.head(&path).await {
423                Ok(_) => Ok::<bool, anyhow::Error>(true),
424                Err(ObjectStoreError::NotFound { .. }) => Ok(false),
425                Err(e) => Err(anyhow::Error::from(e)),
426            }
427        })?;
428
429        Ok(marker_exists
430            || self
431                .manifest()?
432                .identities
433                .iter()
434                .any(|item| item == identity))
435    }
436
437    fn record_identity(&self, identity: &str) -> anyhow::Result<()> {
438        let bytes = serde_json::to_vec(identity)?;
439        let path = self.marker_path(identity);
440        block_on_nautilus_with(|| async {
441            match self
442                .catalog
443                .object_store
444                .put_opts(
445                    &path,
446                    bytes.into(),
447                    PutOptions {
448                        mode: PutMode::Create,
449                        ..Default::default()
450                    },
451                )
452                .await
453            {
454                Ok(_) | Err(ObjectStoreError::AlreadyExists { .. }) => Ok(()),
455                Err(e) => Err(e.into()),
456            }
457        })
458    }
459}
460
461impl PromotionBackend for ParquetPromotionBackend {
462    type Source = FeatherSessionSource;
463
464    const NAME: &'static str = "Parquet";
465
466    fn convert_file(
467        &mut self,
468        source: &Self::Source,
469        file: &str,
470        use_ts_event_for_ts_init: bool,
471        record_promoted: bool,
472    ) -> anyhow::Result<Option<FeatherConversionSummary>> {
473        let object_path = ObjectPath::parse(file)?;
474
475        let read = block_on_nautilus_with(|| {
476            read_feather_record_batches_with_identity(
477                source.storage.object_store.clone(),
478                &object_path,
479            )
480        })?;
481
482        let identifiers = identifiers_from_record_batches(&read.batches)
483            .ok()
484            .filter(|identifiers| !identifiers.is_empty());
485        let identity = feather_replay_identity(
486            &source.storage.original_uri,
487            file,
488            &read.content_hash,
489            identifiers.as_deref(),
490        );
491
492        if self.identity_recorded(&identity)? {
493            let identifier = match identifiers {
494                Some(mut identifiers) if identifiers.len() == 1 => identifiers.pop(),
495                Some(_) => None,
496                None => identifier_from_session_feather_path(
497                    file,
498                    source.environment,
499                    &source.instance_id,
500                ),
501            };
502
503            return Ok(Some(FeatherConversionSummary {
504                data_type: catalog_data_type_from_session_feather_path(
505                    file,
506                    source.environment,
507                    &source.instance_id,
508                )?,
509                identifier,
510                feather_path: file.to_string(),
511                native_version: None,
512                unmatched_identifiers: None,
513            }));
514        }
515
516        let summary = self.catalog.promote_feather_file(
517            source,
518            file,
519            read.batches,
520            use_ts_event_for_ts_init,
521            &identity,
522        )?;
523
524        if summary.is_some() && record_promoted {
525            self.record_identity(&identity)?;
526        }
527
528        Ok(summary)
529    }
530
531    fn delete_file(&mut self, source: &Self::Source, file: &str) -> anyhow::Result<()> {
532        let object_path = ObjectPath::parse(file)?;
533        block_on_nautilus_with(|| async {
534            source.storage.object_store.delete(&object_path).await?;
535            Ok::<(), anyhow::Error>(())
536        })
537    }
538}
539
540impl StagedPromotionBackend for ParquetPromotionBackend {
541    const REQUIRES_SESSION: bool = true;
542
543    fn record_run_state(
544        &mut self,
545        source: &FeatherSessionSource,
546        _staging_uri: &str,
547        status: RunStatus,
548        empty: bool,
549        _error: Option<&str>,
550    ) -> anyhow::Result<()> {
551        block_on_nautilus_with(|| {
552            source.storage.write_run_manifest(
553                source.environment,
554                &source.instance_id,
555                status,
556                empty,
557            )
558        })
559    }
560}
561
562#[derive(Debug, Default, Deserialize, Serialize)]
563struct ParquetPromotionManifest {
564    identities: Vec<String>,
565}
566
567fn feather_replay_identity(
568    source_uri: &str,
569    source_path: &str,
570    content_hash: &str,
571    identifiers: Option<&[String]>,
572) -> String {
573    let mut identifiers = identifiers.map(<[String]>::to_vec);
574    if let Some(identifiers) = identifiers.as_mut() {
575        identifiers.sort();
576        identifiers.dedup();
577    }
578
579    let identity = serde_json::json!({
580        "source_uri": source_uri,
581        "source_path": source_path,
582        "content_hash": content_hash,
583        "identifiers": identifiers,
584    });
585    format!(
586        "nautilus-feather:{}",
587        blake3::hash(identity.to_string().as_bytes()).to_hex(),
588    )
589}
590
591#[cfg(test)]
592mod tests {
593    use std::{
594        fs,
595        path::{Path, PathBuf},
596        sync::atomic::AtomicU64,
597    };
598
599    use arrow::datatypes::SchemaRef;
600    use nautilus_common::enums::Environment;
601    use nautilus_core::{Params, UnixNanos};
602    use nautilus_model::{
603        data::{
604            Bar, BarType, BookOrder, Data, DataBatch, NautilusDataType, NautilusRecordType,
605            OrderBookDelta, QuoteTick,
606        },
607        enums::{BookAction, OrderSide, RecordFlag},
608        identifiers::InstrumentId,
609        types::{ERROR_PRICE, Price, Quantity},
610    };
611    use nautilus_serialization::arrow::{DecodeTypedFromRecordBatch, EncodeToRecordBatch};
612    use object_store::memory::InMemory;
613    use parquet::{
614        basic::Compression,
615        file::{
616            metadata::RowGroupMetaData,
617            reader::{FileReader, SerializedFileReader},
618        },
619    };
620    use rstest::rstest;
621    use tempfile::TempDir;
622
623    use super::*;
624    use crate::{
625        backend::parquet::io::read_parquet_from_object_store,
626        catalog::traits::{CatalogQuery, CatalogReader},
627        common::{paths::normalize_path_to_uri, storage::RUN_MANIFEST_FILENAME},
628        config::{CatalogCompression, DataCatalogConfig},
629        test_data::RustTestHashMapCustomData,
630        writer::{
631            feather::{FEATHER_PARTIAL_EXTENSION, FeatherWriter, RotationConfig},
632            promotion::{list_session_feather_files, tests::assert_scheduled_count},
633        },
634    };
635
636    #[rstest]
637    fn parquet_default_close_promotes_and_honors_source_retention(
638        #[values(false, true)] delete_source: bool,
639    ) {
640        let directory = TempDir::new().unwrap();
641        let staging = directory.path().join("backtest").join("run-1");
642        let mut config =
643            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
644        config.delete_feather_after_promotion = delete_source;
645        let mut sink =
646            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
647                .unwrap();
648        let quote = sample_quote();
649        let mut second = quote;
650        second.instrument_id = InstrumentId::from("BTC/USD.SIM");
651        second.bid_price = Price::from("100.1234");
652        second.ask_price = Price::from("100.5678");
653        second.ts_init = UnixNanos::from(24);
654        let mut catalog = ParquetDataCatalog::from_uri(
655            directory.path().to_str().unwrap(),
656            None,
657            None,
658            None,
659            None,
660        )
661        .unwrap();
662        sink.write_data(Data::Quote(quote)).unwrap();
663        sink.write_data(Data::Quote(second)).unwrap();
664        sink.flush().unwrap();
665        let before = catalog
666            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
667            .unwrap();
668        sink.close().unwrap();
669        let after = catalog
670            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
671            .unwrap();
672        let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
673        let staged = block_on_nautilus_with(|| storage.list_files("", None))
674            .unwrap()
675            .into_iter()
676            .filter(|path| path.ends_with(".feather"))
677            .count();
678        assert!(before.is_empty());
679
680        let DataBatch::Quote(rows) = after else {
681            panic!("expected quotes")
682        };
683
684        assert_eq!(rows.as_ref(), &[quote, second]);
685        assert_eq!(staged, usize::from(!delete_source));
686    }
687
688    #[rstest]
689    fn parquet_writer_recovers_partial_files_at_startup() {
690        let directory = TempDir::new().unwrap();
691        let staging = directory.path().join("backtest").join("run-recover");
692        let quote = sample_quote();
693
694        // A writer that exited before sealing leaves its flushed stream as a partial file
695        let mut crashed = FeatherWriter::new(
696            staging.clone(),
697            WriterClock::Test(Arc::new(AtomicU64::new(0))),
698            RotationConfig::NoRotation,
699            None,
700            None,
701        );
702        crashed.write(quote).unwrap();
703        crashed.close().unwrap();
704        let sealed = staging.join("quotes").join("quotes_0.feather");
705        std::fs::rename(&sealed, sealed.with_extension(FEATHER_PARTIAL_EXTENSION)).unwrap();
706
707        let config =
708            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
709        let mut sink =
710            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
711                .unwrap();
712        sink.close().unwrap();
713
714        let mut catalog = ParquetDataCatalog::from_uri(
715            directory.path().to_str().unwrap(),
716            None,
717            None,
718            None,
719            None,
720        )
721        .unwrap();
722        let rows = catalog
723            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
724            .unwrap();
725
726        let DataBatch::Quote(rows) = rows else {
727            panic!("expected quotes")
728        };
729
730        assert_eq!(rows.as_ref(), &[quote]);
731    }
732
733    #[rstest]
734    fn parquet_close_promotes_per_identifier_file_with_non_ascii_identifier(
735        #[values(false, true)] delete_source: bool,
736    ) {
737        let directory = TempDir::new().unwrap();
738        let staging = directory.path().join("backtest").join("run-1");
739        let (quote, staged_file) = stage_per_identifier_quote(&staging);
740        let mut config =
741            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
742        config.delete_feather_after_promotion = delete_source;
743        let mut sink =
744            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
745                .unwrap();
746
747        sink.close().unwrap();
748
749        let mut catalog = ParquetDataCatalog::from_uri(
750            directory.path().to_str().unwrap(),
751            None,
752            None,
753            None,
754            None,
755        )
756        .unwrap();
757
758        let DataBatch::Quote(rows) = catalog
759            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
760            .unwrap()
761        else {
762            panic!("expected quotes")
763        };
764
765        assert_eq!(rows.as_ref(), &[quote]);
766        assert_eq!(staged_file.exists(), !delete_source);
767    }
768
769    #[rstest]
770    fn parquet_manual_conversion_reads_per_identifier_file_with_non_ascii_identifier() {
771        let directory = TempDir::new().unwrap();
772        let staging = directory.path().join("backtest").join("run-1");
773        let (quote, _) = stage_per_identifier_quote(&staging);
774        let mut catalog = ParquetDataCatalog::from_uri(
775            directory.path().to_str().unwrap(),
776            None,
777            None,
778            None,
779            None,
780        )
781        .unwrap();
782
783        catalog
784            .convert_stream_to_data(
785                "run-1",
786                &NautilusDataType::QuoteTick.into(),
787                Environment::Backtest,
788                None,
789                false,
790            )
791            .unwrap();
792
793        let DataBatch::Quote(rows) = catalog
794            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
795            .unwrap()
796        else {
797            panic!("expected quotes")
798        };
799
800        assert_eq!(rows.as_ref(), &[quote]);
801    }
802
803    #[rstest]
804    fn parquet_promotion_writes_non_ascii_identifier_to_object_store() {
805        let directory = TempDir::new().unwrap();
806        let staging = directory.path().join("backtest").join("run-1");
807        let (quote, staged_file) = stage_per_identifier_quote(&staging);
808
809        let source = FeatherSessionSource::new(
810            create_storage_backend_from_path(directory.path().to_str().unwrap(), None).unwrap(),
811            Environment::Backtest,
812            "run-1",
813        );
814        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
815        catalog.base_path = "catalog".to_string();
816        catalog.original_uri = "s3://test-bucket/catalog".to_string();
817        catalog.object_store = Arc::new(InMemory::new());
818
819        let mut backend = ParquetPromotionBackend {
820            catalog,
821            legacy_manifest_missing: Arc::default(),
822        };
823
824        let files =
825            list_session_feather_files(&source.storage, Environment::Backtest, "run-1").unwrap();
826
827        for file in &files {
828            backend.convert_file(&source, file, false, true).unwrap();
829            backend.delete_file(&source, file).unwrap();
830        }
831
832        let DataBatch::Quote(rows) = backend
833            .catalog
834            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
835            .unwrap()
836        else {
837            panic!("expected quotes")
838        };
839
840        assert_eq!(files.len(), 1);
841        assert_eq!(rows.as_ref(), &[quote]);
842        assert!(!staged_file.exists());
843    }
844
845    #[rstest]
846    #[case("backtest")]
847    #[case("sandbox")]
848    #[case("live")]
849    fn parquet_close_promotes_into_missing_local_catalog(#[case] environment: &str) {
850        let directory = TempDir::new().unwrap();
851        let staging = directory
852            .path()
853            .join("stream")
854            .join(environment)
855            .join("run-1");
856        let catalog_path = directory.path().join("charts").join("catalog");
857
858        let config = WriterConnectConfig::new(
859            staging.to_string_lossy(),
860            Some(CatalogConnectConfig::new(
861                catalog_path.to_string_lossy(),
862                None,
863            )),
864        );
865        let mut sink =
866            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
867                .unwrap();
868        let quote = sample_quote();
869
870        sink.write_data(Data::Quote(quote)).unwrap();
871        sink.close().unwrap();
872
873        let mut catalog =
874            ParquetDataCatalog::from_uri(catalog_path.to_str().unwrap(), None, None, None, None)
875                .unwrap();
876
877        let DataBatch::Quote(rows) = catalog
878            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
879            .unwrap()
880        else {
881            panic!("expected quotes")
882        };
883
884        let manifest: serde_json::Value =
885            serde_json::from_slice(&std::fs::read(staging.join(RUN_MANIFEST_FILENAME)).unwrap())
886                .unwrap();
887
888        assert_eq!(rows.as_ref(), &[quote]);
889        assert_eq!(manifest["kind"], environment);
890        assert_eq!(manifest["status"], "promoted");
891        assert!(!catalog_path.join(environment).exists());
892    }
893
894    #[rstest]
895    #[case::non_ascii("run-é")]
896    #[case::space("run 1")]
897    #[case::fragment("run#1")]
898    fn parquet_writer_rejects_run_id_that_paths_encode(#[case] run_id: &str) {
899        let directory = TempDir::new().unwrap();
900        let staging = directory.path().join("backtest").join(run_id);
901        let config =
902            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
903
904        let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
905            .unwrap_err();
906
907        let staging_uri = normalize_path_to_uri(&staging.to_string_lossy()).unwrap();
908        assert_eq!(
909            error.to_string(),
910            format!(
911                "Parquet writer URI '{staging_uri}' has a run ID that object-store paths \
912                 percent-encode; use a run ID without spaces, non-ASCII, or reserved characters",
913            )
914        );
915    }
916
917    #[rstest]
918    #[case::windows_separators(r"file:///C:\catalog\backtest\run-1", "file:///C:/catalog")]
919    #[case::windows_trailing_separator(r"file:///C:\catalog\backtest\run-1\", "file:///C:/catalog")]
920    #[case::trailing_separator("file:///tmp/catalog/backtest/run-1/", "file:///tmp/catalog")]
921    #[case::parent_components(
922        "file:///tmp/unused/../catalog/backtest/run-1",
923        "file:///tmp/catalog"
924    )]
925    fn parquet_writer_session_accepts_separator_forms(#[case] uri: &str, #[case] root_uri: &str) {
926        let session = StagedFeatherWriter::<ParquetPromotionBackend>::required_session(
927            uri,
928            "Parquet writer URI",
929        )
930        .unwrap();
931
932        assert_eq!(
933            session,
934            PromotionSession {
935                root_uri: root_uri.to_string(),
936                environment: Environment::Backtest,
937                instance_id: "run-1".to_string(),
938            }
939        );
940    }
941
942    #[rstest]
943    fn parquet_writer_requires_a_catalog() {
944        let directory = TempDir::new().unwrap();
945        let staging = directory.path().join("backtest").join("run-1");
946        let config = WriterConnectConfig::new(staging.to_string_lossy(), None);
947
948        let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
949            .unwrap_err();
950
951        assert_eq!(
952            error.to_string(),
953            "Parquet writer requires a promotion catalog"
954        );
955        assert!(!staging.exists());
956    }
957
958    // Opening an S3 store needs no network, and a run with no data promotes nothing, so the
959    // writer records the run as completed when dropped
960    #[cfg(feature = "cloud")]
961    #[rstest]
962    fn parquet_writer_stages_locally_for_a_remote_catalog() {
963        let directory = TempDir::new().unwrap();
964        let staging = directory.path().join("backtest").join("run-1");
965
966        let config = WriterConnectConfig::new(
967            staging.to_string_lossy(),
968            Some(CatalogConnectConfig::new("s3://bucket/catalog", None)),
969        );
970
971        let mut sink =
972            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
973                .unwrap();
974        sink.close().unwrap();
975        drop(sink);
976
977        let manifest: serde_json::Value =
978            serde_json::from_slice(&std::fs::read(staging.join(RUN_MANIFEST_FILENAME)).unwrap())
979                .unwrap();
980        assert_eq!(manifest["kind"], "backtest");
981        assert_eq!(manifest["instance_id"], "run-1");
982        assert_eq!(manifest["status"], "completed");
983        assert_eq!(manifest["empty"], true);
984    }
985
986    #[rstest]
987    fn parquet_promotion_writes_the_direct_write_schema() {
988        let directory = TempDir::new().unwrap();
989
990        let config = WriterConnectConfig::new(
991            directory
992                .path()
993                .join("backtest/run-schema")
994                .to_string_lossy(),
995            Some(local_catalog(&directory)),
996        );
997        let mut sink =
998            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
999                .unwrap();
1000        let quote = sample_quote();
1001        sink.write_data(Data::Quote(quote)).unwrap();
1002        sink.close().unwrap();
1003
1004        let promoted = ParquetDataCatalog::from_uri(
1005            directory.path().to_str().unwrap(),
1006            None,
1007            None,
1008            None,
1009            None,
1010        )
1011        .unwrap();
1012        let direct_directory = TempDir::new().unwrap();
1013        let direct = ParquetDataCatalog::from_uri(
1014            direct_directory.path().to_str().unwrap(),
1015            None,
1016            None,
1017            None,
1018            None,
1019        )
1020        .unwrap();
1021        direct.write_to_parquet(&[quote], None, None, None).unwrap();
1022
1023        assert_eq!(
1024            quote_file_schema(&promoted).fields(),
1025            quote_file_schema(&direct).fields(),
1026        );
1027    }
1028
1029    #[rstest]
1030    fn parquet_promotion_honors_catalog_settings() {
1031        let directory = TempDir::new().unwrap();
1032        let staging = directory.path().join("backtest").join("run-1");
1033        let catalog = DataCatalogConfig::builder()
1034            .path(directory.path().to_string_lossy().to_string())
1035            .batch_size(7)
1036            .compression(CatalogCompression::Uncompressed)
1037            .max_row_group_size(1)
1038            .build()
1039            .unwrap()
1040            .connect_config();
1041        let backend =
1042            ParquetPromotionBackend::new(&catalog, Arc::new(AtomicBool::new(false))).unwrap();
1043
1044        let config = WriterConnectConfig::new(staging.to_string_lossy(), Some(catalog));
1045        let mut sink =
1046            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1047                .unwrap();
1048        let quote = sample_quote();
1049        sink.write_data(Data::Quote(quote)).unwrap();
1050        sink.write_data(Data::Quote(QuoteTick {
1051            ts_event: UnixNanos::from(29),
1052            ts_init: UnixNanos::from(31),
1053            ..quote
1054        }))
1055        .unwrap();
1056
1057        sink.close().unwrap();
1058
1059        let files = ParquetDataCatalog::new(directory.path(), None, None, None, None)
1060            .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
1061            .unwrap();
1062        let reader = SerializedFileReader::new(
1063            std::fs::File::open(directory.path().join(&files[0])).unwrap(),
1064        )
1065        .unwrap();
1066        let metadata = reader.metadata();
1067
1068        assert_eq!(backend.catalog.batch_size, 7);
1069        assert_eq!(files.len(), 1);
1070        assert_eq!(metadata.num_row_groups(), 2);
1071        assert!(
1072            metadata
1073                .row_groups()
1074                .iter()
1075                .flat_map(RowGroupMetaData::columns)
1076                .all(|column| column.compression() == Compression::UNCOMPRESSED)
1077        );
1078    }
1079
1080    #[rstest]
1081    fn parquet_writer_rejects_unknown_catalog_param() {
1082        let directory = TempDir::new().unwrap();
1083        let staging = directory.path().join("backtest").join("run-1");
1084        let mut catalog = local_catalog(&directory);
1085        let params = Params::new([("compression".to_string(), serde_json::json!(0))].into());
1086        catalog.params = Some(params);
1087
1088        let config = WriterConnectConfig::new(staging.to_string_lossy(), Some(catalog));
1089        let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1090            .unwrap_err();
1091
1092        assert_eq!(
1093            error.to_string(),
1094            "Unknown Parquet catalog param 'compression': this catalog takes no params"
1095        );
1096    }
1097
1098    fn quote_file_schema(catalog: &ParquetDataCatalog) -> SchemaRef {
1099        let files = catalog
1100            .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
1101            .unwrap();
1102        assert_eq!(files.len(), 1);
1103
1104        let path = ObjectPath::from(files[0].as_str());
1105
1106        let (_, schema) = block_on_nautilus_with(|| {
1107            read_parquet_from_object_store(catalog.object_store.clone(), &path)
1108        })
1109        .unwrap();
1110
1111        schema
1112    }
1113
1114    #[rstest]
1115    #[case(None)]
1116    #[case(Some(0))]
1117    #[case(Some(1))]
1118    fn parquet_interval_uses_test_clock_and_close_can_leave_staged_data(
1119        #[case] interval: Option<u64>,
1120    ) {
1121        let directory = TempDir::new().unwrap();
1122        let staging = directory.path().join("backtest").join("run-2");
1123        let mut config =
1124            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1125        config.promotion_interval_ms = interval;
1126        config.promote_on_close = false;
1127        let clock = Arc::new(AtomicU64::new(0));
1128        let mut sink =
1129            parquet_writer_factory(&config, WriterClock::Test(Arc::clone(&clock))).unwrap();
1130        let quote = sample_quote();
1131        sink.write_data(Data::Quote(quote)).unwrap();
1132        clock.store(1_000_000, Ordering::Relaxed);
1133        sink.flush().unwrap();
1134        sink.close().unwrap();
1135        let mut catalog = ParquetDataCatalog::from_uri(
1136            directory.path().to_str().unwrap(),
1137            None,
1138            None,
1139            None,
1140            None,
1141        )
1142        .unwrap();
1143        let before_manual = catalog
1144            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
1145            .unwrap();
1146
1147        if interval != Some(1) {
1148            catalog
1149                .convert_stream_to_data(
1150                    "run-2",
1151                    &NautilusDataType::QuoteTick.into(),
1152                    Environment::Backtest,
1153                    None,
1154                    false,
1155                )
1156                .unwrap();
1157        }
1158
1159        let after = catalog
1160            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
1161            .unwrap();
1162        assert_eq!(before_manual.len(), usize::from(interval == Some(1)));
1163
1164        let DataBatch::Quote(rows) = after else {
1165            panic!("expected quotes")
1166        };
1167
1168        assert_eq!(rows.as_ref(), &[quote]);
1169    }
1170
1171    #[rstest]
1172    fn parquet_streaming_funding_round_trip() {
1173        use nautilus_model::data::FundingRateUpdate;
1174        let directory = TempDir::new().unwrap();
1175
1176        let config = WriterConnectConfig::new(
1177            directory
1178                .path()
1179                .join("backtest/run-funding")
1180                .to_string_lossy(),
1181            Some(local_catalog(&directory)),
1182        );
1183        let mut sink =
1184            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1185                .unwrap();
1186
1187        let funding = FundingRateUpdate::new(
1188            InstrumentId::from("AUD/USD.SIM"),
1189            "0.0012".parse().unwrap(),
1190            Some(480),
1191            Some(789.into()),
1192            123.into(),
1193            456.into(),
1194        );
1195        assert!(sink.write_any(&funding).unwrap());
1196        sink.close().unwrap();
1197        let mut catalog = ParquetDataCatalog::from_uri(
1198            directory.path().to_str().unwrap(),
1199            None,
1200            None,
1201            None,
1202            None,
1203        )
1204        .unwrap();
1205        let rows = catalog
1206            .query_batch(&CatalogQuery::new(NautilusDataType::FundingRateUpdate))
1207            .unwrap();
1208
1209        let DataBatch::FundingRate(rows) = rows else {
1210            panic!("expected funding rates")
1211        };
1212
1213        assert_eq!(rows.as_ref(), &[funding]);
1214    }
1215
1216    #[rstest]
1217    fn parquet_streaming_write_error_reaches_flush() {
1218        let directory = TempDir::new().unwrap();
1219
1220        let config = WriterConnectConfig::new(
1221            directory
1222                .path()
1223                .join("backtest/run-error")
1224                .to_string_lossy(),
1225            Some(local_catalog(&directory)),
1226        );
1227        let mut sink =
1228            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1229                .unwrap();
1230        let mut quote = sample_quote();
1231        quote.bid_price = ERROR_PRICE;
1232        let write_error = sink.write_any(&quote).unwrap_err().to_string();
1233        let flush_error = sink.flush().unwrap_err().to_string();
1234        assert_eq!(flush_error, write_error);
1235    }
1236
1237    #[rstest]
1238    fn parquet_streaming_instrument_promotes() {
1239        use nautilus_model::instruments::{InstrumentAny, stubs::audusd_sim};
1240        let directory = TempDir::new().unwrap();
1241
1242        let config = WriterConnectConfig::new(
1243            directory
1244                .path()
1245                .join("backtest/run-instrument")
1246                .to_string_lossy(),
1247            Some(local_catalog(&directory)),
1248        );
1249        let mut sink =
1250            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1251                .unwrap();
1252        let instrument = InstrumentAny::CurrencyPair(audusd_sim());
1253        assert!(sink.write_any(&instrument).unwrap());
1254        sink.close().unwrap();
1255        let mut catalog = ParquetDataCatalog::from_uri(
1256            directory.path().to_str().unwrap(),
1257            None,
1258            None,
1259            None,
1260            None,
1261        )
1262        .unwrap();
1263        let rows = catalog
1264            .query_batch(&CatalogQuery::new(NautilusDataType::Instrument))
1265            .unwrap();
1266
1267        let DataBatch::Instrument(rows) = rows else {
1268            panic!("expected instruments")
1269        };
1270
1271        assert_eq!(rows.as_ref(), &[instrument]);
1272    }
1273
1274    #[rstest]
1275    fn parquet_promotion_preserves_every_record_family(#[values(false, true)] automatic: bool) {
1276        use nautilus_core::UUID4;
1277        use nautilus_model::{
1278            enums::{
1279                LiquiditySide, OrderSide, OrderStatus, OrderType, PositionAdjustmentType,
1280                PositionSide, TimeInForce,
1281            },
1282            events::{
1283                OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled, OrderDenied,
1284                OrderEmulated, OrderExpired, OrderInitialized, OrderModifyDenied,
1285                OrderModifyRejected, OrderPendingCancel, OrderPendingUpdate, OrderRejected,
1286                OrderReleased, OrderSnapshot, OrderSubmitted, OrderTriggered, OrderUpdated,
1287                PositionAdjusted, PositionChanged, PositionClosed, PositionOpened,
1288                PositionSnapshot,
1289                account::stubs::cash_account_state,
1290                order::spec::{OrderFillVoidedSpec, OrderFilledSpec},
1291            },
1292            identifiers::{
1293                AccountId, ClientId, ClientOrderId, PositionId, TradeId, Venue, VenueOrderId,
1294            },
1295            instruments::{InstrumentAny, stubs::audusd_sim},
1296            orders::OrderTestBuilder,
1297            position::Position,
1298            reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
1299            stubs::TestDefault,
1300            types::Money,
1301        };
1302        use strum::IntoEnumIterator;
1303        let mut families = ahash::AHashSet::new();
1304        macro_rules! promote {
1305            ($record:expr) => {
1306                families.insert(assert_record_promotion($record, automatic));
1307            };
1308        }
1309
1310        promote!(cash_account_state());
1311        promote!(OrderInitialized::test_default());
1312        promote!(OrderDenied::test_default());
1313        promote!(OrderEmulated::test_default());
1314        promote!(OrderSubmitted::test_default());
1315        promote!(OrderAccepted::test_default());
1316        promote!(OrderRejected::test_default());
1317        promote!(OrderPendingCancel::test_default());
1318        promote!(OrderCanceled::test_default());
1319        let mut cancel_denied = OrderCancelDenied::test_default();
1320        cancel_denied.causation_id = Some(UUID4::new());
1321        promote!(cancel_denied);
1322        promote!(OrderCancelRejected::test_default());
1323        promote!(OrderExpired::test_default());
1324        promote!(OrderTriggered::test_default());
1325        promote!(OrderPendingUpdate::test_default());
1326        promote!(OrderReleased::test_default());
1327        let mut modify_denied = OrderModifyDenied::test_default();
1328        modify_denied.causation_id = Some(UUID4::new());
1329        promote!(modify_denied);
1330        promote!(OrderModifyRejected::test_default());
1331        promote!(OrderUpdated::test_default());
1332        promote!(OrderFillVoidedSpec::builder().is_reopened(true).build());
1333        let id = InstrumentId::from("AUD/USD.SIM");
1334        let fill = OrderFilledSpec::builder()
1335            .instrument_id(id)
1336            .position_id(PositionId::from("P-009"))
1337            .last_px(Price::from("1.23456"))
1338            .last_qty(Quantity::from("1234"))
1339            .ts_event(11.into())
1340            .ts_init(13.into())
1341            .build();
1342        promote!(fill.clone());
1343        let mut position = Position::new(&InstrumentAny::CurrencyPair(audusd_sim()), fill.clone());
1344        promote!(PositionOpened::create(
1345            &position,
1346            &fill,
1347            UUID4::default(),
1348            17.into()
1349        ));
1350        promote!(PositionChanged::create(
1351            &position,
1352            &fill,
1353            UUID4::default(),
1354            19.into()
1355        ));
1356        promote!(PositionAdjusted::new(
1357            fill.trader_id,
1358            fill.strategy_id,
1359            id,
1360            position.id,
1361            fill.account_id,
1362            PositionAdjustmentType::Funding,
1363            None,
1364            Some(Money::from("1.23 USD")),
1365            Some("funding".into()),
1366            UUID4::default(),
1367            23.into(),
1368            29.into()
1369        ));
1370        promote!(PositionSnapshot::from(
1371            &position,
1372            Some(Money::from("4.56 USD"))
1373        ));
1374        let closing = OrderFilledSpec::builder()
1375            .instrument_id(id)
1376            .position_id(position.id)
1377            .order_side(OrderSide::Sell)
1378            .last_px(Price::from("1.34567"))
1379            .last_qty(fill.last_qty)
1380            .trade_id(TradeId::from("T-011"))
1381            .ts_event(31.into())
1382            .ts_init(37.into())
1383            .build();
1384        position.apply(&closing);
1385        promote!(PositionClosed::create(
1386            &position,
1387            &closing,
1388            UUID4::default(),
1389            41.into()
1390        ));
1391        promote!(OrderSnapshot::from(
1392            OrderTestBuilder::new(OrderType::Limit)
1393                .instrument_id(id)
1394                .price(Price::from("1.45678"))
1395                .quantity(Quantity::from("5678"))
1396                .build()
1397        ));
1398
1399        let order = OrderStatusReport::new(
1400            AccountId::from("SIM-007"),
1401            id,
1402            Some(ClientOrderId::from("O-013")),
1403            VenueOrderId::from("V-017"),
1404            Some(OrderSide::Buy),
1405            OrderType::Limit,
1406            TimeInForce::Gtc,
1407            OrderStatus::PartiallyFilled,
1408            Quantity::from("123"),
1409            Quantity::from("45"),
1410            43.into(),
1411            47.into(),
1412            53.into(),
1413            None,
1414        );
1415
1416        let report_fill = FillReport::new(
1417            AccountId::from("SIM-007"),
1418            id,
1419            VenueOrderId::from("V-017"),
1420            TradeId::from("T-019"),
1421            OrderSide::Buy,
1422            Quantity::from("45"),
1423            Price::from("1.56789"),
1424            Money::from("2.34 USD"),
1425            LiquiditySide::Maker,
1426            Some(ClientOrderId::from("O-013")),
1427            Some(PositionId::from("P-023")),
1428            59.into(),
1429            61.into(),
1430            None,
1431        );
1432
1433        let report_position = PositionStatusReport::new(
1434            AccountId::from("SIM-007"),
1435            id,
1436            PositionSide::Long,
1437            Quantity::from("789"),
1438            67.into(),
1439            71.into(),
1440            None,
1441            Some(PositionId::from("P-023")),
1442            Some(rust_decimal::Decimal::new(16789, 4)),
1443        );
1444
1445        let mut mass = ExecutionMassStatus::new(
1446            ClientId::from("CLIENT-029"),
1447            AccountId::from("SIM-007"),
1448            Venue::from("SIM"),
1449            73.into(),
1450            None,
1451        );
1452        mass.add_order_reports(vec![order.clone()]);
1453        mass.add_fill_reports(vec![report_fill.clone()]);
1454        mass.add_position_reports(vec![report_position.clone()]);
1455        promote!(order);
1456        promote!(report_fill);
1457        promote!(report_position);
1458        promote!(mass);
1459
1460        let expected = NautilusRecordType::iter()
1461            .filter(|family| {
1462                #[cfg(feature = "defi")]
1463                if *family == NautilusRecordType::Defi {
1464                    return false;
1465                }
1466
1467                let _ = family;
1468                true
1469            })
1470            .collect::<ahash::AHashSet<_>>();
1471
1472        assert_eq!(families, expected);
1473    }
1474
1475    #[rstest]
1476    fn parquet_promotion_preserves_calculated_greeks_as_custom_data(
1477        #[values(false, true)] automatic: bool,
1478        #[values(false, true)] portfolio: bool,
1479    ) {
1480        use nautilus_model::data::{
1481            CustomData, DataType,
1482            greeks::{GreeksData, OptionGreekValues, PortfolioGreeks},
1483        };
1484
1485        let greeks = GreeksData::new(
1486            "ESZ26 C7000.XCME".parse().unwrap(),
1487            false,
1488            7000.5,
1489            20_261_218,
1490            68,
1491            0.186,
1492            50.0,
1493            -3.0,
1494            6950.25,
1495            0.0425,
1496            0.0125,
1497            0.175,
1498            12.5,
1499            98.75,
1500            OptionGreekValues {
1501                delta: 0.41,
1502                gamma: 0.0012,
1503                vega: 11.2,
1504                theta: -2.3,
1505                rho: 1.7,
1506            },
1507            0.62,
1508            11.into(),
1509            13.into(),
1510        );
1511
1512        let mut second = greeks.clone();
1513        second.ts_init = 113.into();
1514        second.instrument_id = "ESZ26 P6900.XCME".parse().unwrap();
1515
1516        let greeks = vec![greeks, second]
1517            .into_iter()
1518            .map(|value| {
1519                let data_type =
1520                    DataType::new("GreeksData", None, Some(value.instrument_id.to_string()));
1521                CustomData::new(Arc::new(value), data_type)
1522            })
1523            .collect::<Vec<_>>();
1524
1525        let portfolio_value = PortfolioGreeks {
1526            pnl: -125.5,
1527            price: 4310.25,
1528            greeks: OptionGreekValues {
1529                delta: 0.83,
1530                gamma: 0.004,
1531                vega: 45.5,
1532                theta: -12.25,
1533                rho: 3.75,
1534            },
1535            ts_event: 19.into(),
1536            ts_init: 17.into(),
1537        };
1538
1539        let mut second = portfolio_value.clone();
1540        second.ts_init = 117.into();
1541        let portfolios = vec![portfolio_value, second]
1542            .into_iter()
1543            .map(|value| CustomData::from_arc(Arc::new(value)))
1544            .collect::<Vec<_>>();
1545
1546        let expected = if portfolio { portfolios } else { greeks };
1547
1548        let directory = TempDir::new().unwrap();
1549        let staging = directory.path().join("backtest/run-greeks-custom");
1550        let mut config =
1551            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1552        config.promote_on_close = automatic;
1553        let mut sink =
1554            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1555                .unwrap();
1556
1557        for custom in &expected {
1558            assert!(sink.write_any(custom.data.as_any()).unwrap());
1559        }
1560
1561        sink.close().unwrap();
1562
1563        let data_type = NautilusDataType::Custom {
1564            type_name: expected[0].data_type.type_name().to_string(),
1565        };
1566
1567        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1568        if !automatic {
1569            catalog
1570                .convert_stream_to_data(
1571                    "run-greeks-custom",
1572                    &data_type.clone().into(),
1573                    Environment::Backtest,
1574                    None,
1575                    false,
1576                )
1577                .unwrap();
1578        }
1579
1580        let DataBatch::Custom(actual) = catalog.query_batch(&CatalogQuery::new(data_type)).unwrap()
1581        else {
1582            panic!("expected custom data")
1583        };
1584
1585        assert_eq!(actual.as_ref(), expected.as_slice());
1586    }
1587
1588    fn assert_record_promotion<T>(record: T, automatic: bool) -> NautilusRecordType
1589    where
1590        T: std::any::Any
1591            + serde::Serialize
1592            + serde::de::DeserializeOwned
1593            + EncodeToRecordBatch
1594            + DecodeTypedFromRecordBatch,
1595    {
1596        let directory = TempDir::new().unwrap();
1597        let staging = directory.path().join("backtest/run-record");
1598        let mut config =
1599            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1600        config.promote_on_close = automatic;
1601        let mut sink =
1602            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1603                .unwrap();
1604        let mut second = serde_json::to_value(&record).unwrap();
1605        second["ts_init"] = serde_json::json!(second["ts_init"].as_u64().unwrap() + 101);
1606        let second: T = serde_json::from_value(second).unwrap();
1607        assert!(sink.write_any(&record).unwrap());
1608        assert!(sink.write_any(&second).unwrap());
1609        sink.close().unwrap();
1610        let record_type = T::get_schema(None).metadata()["type_name"]
1611            .parse::<NautilusRecordType>()
1612            .unwrap();
1613        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1614        if !automatic {
1615            catalog
1616                .convert_stream_to_data(
1617                    "run-record",
1618                    &record_type.into(),
1619                    Environment::Backtest,
1620                    None,
1621                    false,
1622                )
1623                .unwrap();
1624        }
1625
1626        let batches = catalog
1627            .query_record_batches(&record_type.into(), None, None, None, None, true)
1628            .unwrap();
1629
1630        let actual = batches
1631            .into_iter()
1632            .flat_map(|batch| {
1633                let schema = batch.schema();
1634                T::decode_typed_batch(schema.metadata(), batch).unwrap()
1635            })
1636            .map(|record| serde_json::to_value(record).unwrap())
1637            .collect::<Vec<_>>();
1638
1639        assert_eq!(
1640            actual,
1641            vec![
1642                serde_json::to_value(record).unwrap(),
1643                serde_json::to_value(second).unwrap()
1644            ]
1645        );
1646        record_type
1647    }
1648
1649    #[rstest]
1650    fn parquet_streaming_voided_fill_promotes() {
1651        use nautilus_model::events::{OrderFillVoided, order::spec::OrderFillVoidedSpec};
1652        use nautilus_serialization::arrow::DecodeTypedFromRecordBatch;
1653        let directory = TempDir::new().unwrap();
1654
1655        let config = WriterConnectConfig::new(
1656            directory.path().join("backtest/run-void").to_string_lossy(),
1657            Some(local_catalog(&directory)),
1658        );
1659        let mut sink =
1660            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1661                .unwrap();
1662        let event = OrderFillVoidedSpec::builder().is_reopened(true).build();
1663        assert!(sink.write_any(&event).unwrap());
1664        sink.close().unwrap();
1665        let mut catalog = ParquetDataCatalog::from_uri(
1666            directory.path().to_str().unwrap(),
1667            None,
1668            None,
1669            None,
1670            None,
1671        )
1672        .unwrap();
1673        let batches = catalog
1674            .query_record_batches(
1675                &NautilusRecordType::OrderFillVoided.into(),
1676                None,
1677                None,
1678                None,
1679                None,
1680                true,
1681            )
1682            .unwrap();
1683        let mut rows = Vec::new();
1684        for batch in batches {
1685            rows.extend(
1686                OrderFillVoided::decode_typed_batch(batch.schema().metadata(), batch.clone())
1687                    .unwrap(),
1688            );
1689        }
1690
1691        assert_eq!(rows, vec![event]);
1692    }
1693
1694    #[rstest]
1695    fn parquet_promotion_preserves_clear_source_order(
1696        #[values(false, true)] automatic: bool,
1697        #[values(false, true)] batched: bool,
1698        #[values(false, true)] undefined_price: bool,
1699    ) {
1700        use nautilus_model::{
1701            data::{BookOrder, OrderBookDelta},
1702            enums::{BookAction, OrderSide, RecordFlag},
1703            types::PRICE_UNDEF,
1704        };
1705        let directory = TempDir::new().unwrap();
1706        let staging = directory.path().join("backtest/run-clear");
1707        let mut config =
1708            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1709        config.promote_on_close = automatic;
1710        let mut sink =
1711            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1712                .unwrap();
1713        let id = InstrumentId::from("AUD/USD.SIM");
1714        let ts = UnixNanos::from(1_700_000_000_000_000_123);
1715        let mut clear = OrderBookDelta::clear(id, 7, ts, ts);
1716        if undefined_price {
1717            clear.order.price = Price::from_raw(PRICE_UNDEF, 0);
1718        }
1719
1720        let add = OrderBookDelta::new(
1721            id,
1722            BookAction::Add,
1723            BookOrder::new(
1724                OrderSide::Buy,
1725                Price::from("1.23456"),
1726                Quantity::from("7.890"),
1727                19,
1728            ),
1729            RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8,
1730            7,
1731            ts,
1732            ts,
1733        );
1734        let mut second_add = add;
1735        second_add.order.order_id = 29;
1736        second_add.order.price = Price::from("1.34567");
1737        second_add.order.size = Quantity::from("8.901");
1738        let rows = vec![clear, add, clear, second_add];
1739        if batched {
1740            sink.write_batch(rows.iter().copied().map(Data::BookDelta).collect())
1741                .unwrap();
1742        } else {
1743            for row in &rows {
1744                sink.write_data(Data::BookDelta(*row)).unwrap();
1745            }
1746        }
1747
1748        sink.close().unwrap();
1749        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1750        let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
1751        let staged = block_on_nautilus_with(|| storage.list_files("", Some(".feather"))).unwrap();
1752        let mut originals = Vec::new();
1753
1754        for path in &staged {
1755            let bytes = block_on_nautilus_with(|| async {
1756                storage
1757                    .object_store
1758                    .get(&object_store::path::Path::from(path.as_str()))
1759                    .await
1760                    .unwrap()
1761                    .bytes()
1762                    .await
1763                    .unwrap()
1764            });
1765
1766            originals.push(bytes);
1767        }
1768
1769        if !automatic {
1770            catalog
1771                .convert_stream_to_data(
1772                    "run-clear",
1773                    &NautilusDataType::OrderBookDelta.into(),
1774                    Environment::Backtest,
1775                    None,
1776                    false,
1777                )
1778                .unwrap();
1779        }
1780
1781        let actual = catalog
1782            .query::<OrderBookDelta>(None, None, None, None, None, true)
1783            .unwrap();
1784        assert_eq!(actual.len(), rows.len());
1785
1786        for (actual, expected) in actual.iter().zip(&rows) {
1787            assert_eq!(actual.instrument_id, expected.instrument_id);
1788            assert_eq!(actual.action, expected.action);
1789            assert_eq!(actual.order.side, expected.order.side);
1790            assert_eq!(actual.order.price.raw(), expected.order.price.raw());
1791
1792            let price_precision = if expected.order.price.is_undefined() {
1793                0
1794            } else {
1795                5
1796            };
1797
1798            assert_eq!(actual.order.price.precision, price_precision);
1799            assert_eq!(
1800                actual.order.size.as_decimal(),
1801                expected.order.size.as_decimal()
1802            );
1803            assert_eq!(actual.order.size.precision, 3);
1804            assert_eq!(actual.order.order_id, expected.order.order_id);
1805            assert_eq!(actual.flags, expected.flags);
1806            assert_eq!(actual.sequence, expected.sequence);
1807            assert_eq!(actual.ts_event, expected.ts_event);
1808            assert_eq!(actual.ts_init, expected.ts_init);
1809        }
1810
1811        for (path, original) in staged.iter().zip(originals) {
1812            let bytes = block_on_nautilus_with(|| async {
1813                storage
1814                    .object_store
1815                    .get(&object_store::path::Path::from(path.as_str()))
1816                    .await
1817                    .unwrap()
1818                    .bytes()
1819                    .await
1820                    .unwrap()
1821            });
1822
1823            assert_eq!(bytes, original);
1824        }
1825
1826        assert_eq!(staged.len(), 1);
1827    }
1828
1829    #[rstest]
1830    #[case::manual(false, false)]
1831    #[case::automatic_retained(true, false)]
1832    #[case::automatic_delete(true, true)]
1833    fn parquet_failed_clear_promotion_preserves_source(
1834        #[case] automatic: bool,
1835        #[case] delete_source: bool,
1836        #[values(false, true)] batched: bool,
1837    ) {
1838        use nautilus_model::{
1839            data::{BookOrder, OrderBookDelta},
1840            enums::{BookAction, OrderSide, RecordFlag},
1841        };
1842        let directory = TempDir::new().unwrap();
1843        let staging = directory.path().join("backtest/run-invalid-clear");
1844        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
1845
1846        let mut writer = FeatherWriter::new(
1847            staging.clone(),
1848            clock.clone(),
1849            RotationConfig::NoRotation,
1850            None,
1851            None,
1852        );
1853        let id = InstrumentId::from("AUD/USD.SIM");
1854        let ts = UnixNanos::from(23);
1855        let mut clear = OrderBookDelta::clear(id, 7, ts, ts);
1856        clear.order.price = Price::from("1");
1857
1858        let add = OrderBookDelta::new(
1859            id,
1860            BookAction::Add,
1861            BookOrder::new(
1862                OrderSide::Buy,
1863                Price::from("1.23456"),
1864                Quantity::from("7.890"),
1865                19,
1866            ),
1867            RecordFlag::F_LAST as u8,
1868            7,
1869            ts,
1870            ts,
1871        );
1872
1873        if batched {
1874            writer.write_batch(vec![clear, add]).unwrap();
1875        } else {
1876            writer.write(clear).unwrap();
1877            writer.write(add).unwrap();
1878        }
1879
1880        writer.close().unwrap();
1881        let source = staging.join("order_book_deltas/order_book_deltas_0.feather");
1882        let original = fs::read(&source).unwrap();
1883        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1884
1885        let failure = if automatic {
1886            let mut config = WriterConnectConfig::new(
1887                staging.to_string_lossy(),
1888                Some(local_catalog(&directory)),
1889            );
1890            config.delete_feather_after_promotion = delete_source;
1891            let mut sink = parquet_writer_factory(&config, clock).unwrap();
1892            sink.close().unwrap_err()
1893        } else {
1894            catalog
1895                .convert_stream_to_data(
1896                    "run-invalid-clear",
1897                    &NautilusDataType::OrderBookDelta.into(),
1898                    Environment::Backtest,
1899                    None,
1900                    false,
1901                )
1902                .unwrap_err()
1903        };
1904
1905        assert!(
1906            format!("{failure:#}").contains(
1907                "cannot relabel precision metadata for a promotion group that contains decimal values"
1908            ),
1909            "{failure:#}"
1910        );
1911        assert_eq!(fs::read(source).unwrap(), original);
1912        assert_eq!(
1913            catalog
1914                .query_files(&NautilusDataType::OrderBookDelta.into(), None, None, None)
1915                .unwrap(),
1916            Vec::<String>::new()
1917        );
1918    }
1919
1920    #[rstest]
1921    #[case(true)]
1922    #[case(false)]
1923    fn parquet_promotion_unifies_empty_and_populated_depth(#[case] automatic: bool) {
1924        use nautilus_model::{
1925            data::{BookOrder, OrderBookDepth},
1926            enums::OrderSide,
1927        };
1928        let directory = TempDir::new().unwrap();
1929        let staging = directory.path().join("backtest/run-depth-ties");
1930        let mut config =
1931            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1932        config.delete_feather_after_promotion = true;
1933        config.promote_on_close = automatic;
1934        let mut sink =
1935            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1936                .unwrap();
1937        let id = InstrumentId::from("AUD/USD.SIM");
1938        let empty =
1939            OrderBookDepth::new(id, vec![], vec![], vec![], vec![], 1, 2, 3.into(), 4.into());
1940        let order = BookOrder::new(
1941            OrderSide::Buy,
1942            Price::from("1.23"),
1943            Quantity::from("4.5"),
1944            6,
1945        );
1946        let populated = OrderBookDepth::new(
1947            id,
1948            vec![order],
1949            vec![],
1950            vec![7],
1951            vec![],
1952            8,
1953            9,
1954            3.into(),
1955            4.into(),
1956        );
1957        sink.write_any(&empty).unwrap();
1958        sink.write_any(&populated).unwrap();
1959        sink.close().unwrap();
1960
1961        let mut catalog = ParquetDataCatalog::from_uri(
1962            directory.path().to_str().unwrap(),
1963            None,
1964            None,
1965            None,
1966            None,
1967        )
1968        .unwrap();
1969
1970        if !automatic {
1971            catalog
1972                .convert_stream_to_data(
1973                    "run-depth-ties",
1974                    &NautilusDataType::OrderBookDepth.into(),
1975                    Environment::Backtest,
1976                    None,
1977                    false,
1978                )
1979                .unwrap();
1980        }
1981
1982        let batch = catalog
1983            .query_batch(&CatalogQuery::new(NautilusDataType::OrderBookDepth))
1984            .unwrap();
1985        let DataBatch::BookDepth(rows) = batch else {
1986            panic!("expected order book depth");
1987        };
1988        let mut rows = rows.as_ref().to_vec();
1989        rows.sort_by_key(|depth| depth.sequence);
1990        assert_eq!(rows, vec![empty, populated]);
1991
1992        let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
1993        let staged = block_on_nautilus_with(|| storage.list_files("", Some(".feather"))).unwrap();
1994        assert_eq!(staged.len(), usize::from(!automatic));
1995    }
1996
1997    #[rstest]
1998    fn parquet_promotion_preserves_clear_snapshots(
1999        #[values(false, true)] automatic: bool,
2000        #[values(false, true)] batch_write: bool,
2001    ) {
2002        let directory = TempDir::new().unwrap();
2003        let staging = directory.path().join("backtest").join("run-clear");
2004        let mut config =
2005            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2006        config.promote_on_close = automatic;
2007        let mut sink =
2008            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2009                .unwrap();
2010        let id = InstrumentId::from("AUD/USD.SIM");
2011        let mut deltas = Vec::new();
2012        for (sequence, timestamp, price, size) in [
2013            (41, 100, "1.23", "4.500"),
2014            (42, 200, "1.24", "6.700"),
2015            (43, 300, "1.2345", "8.90000"),
2016        ] {
2017            deltas.push(OrderBookDelta::clear(
2018                id,
2019                sequence,
2020                (timestamp - 1).into(),
2021                timestamp.into(),
2022            ));
2023
2024            for (side, order_id, flags) in [
2025                (OrderSide::Buy, 71, RecordFlag::F_SNAPSHOT as u8),
2026                (
2027                    OrderSide::Sell,
2028                    72,
2029                    RecordFlag::F_SNAPSHOT as u8 | RecordFlag::F_LAST as u8,
2030                ),
2031            ] {
2032                deltas.push(OrderBookDelta::new(
2033                    id,
2034                    BookAction::Add,
2035                    BookOrder::new(side, Price::from(price), Quantity::from(size), order_id),
2036                    flags,
2037                    sequence,
2038                    (timestamp - 1).into(),
2039                    timestamp.into(),
2040                ));
2041            }
2042        }
2043
2044        deltas.push(OrderBookDelta::clear(id, 44, 399.into(), 400.into()));
2045        if batch_write {
2046            sink.write_batch(deltas.iter().copied().map(Data::from).collect())
2047                .unwrap();
2048        } else {
2049            for delta in &deltas {
2050                sink.write_data(Data::BookDelta(*delta)).unwrap();
2051            }
2052        }
2053
2054        sink.close().unwrap();
2055        let mut catalog = ParquetDataCatalog::from_uri(
2056            directory.path().to_str().unwrap(),
2057            None,
2058            None,
2059            None,
2060            None,
2061        )
2062        .unwrap();
2063
2064        if !automatic {
2065            catalog
2066                .convert_stream_to_data(
2067                    "run-clear",
2068                    &NautilusDataType::OrderBookDelta.into(),
2069                    Environment::Backtest,
2070                    None,
2071                    false,
2072                )
2073                .unwrap();
2074        }
2075
2076        let batch = catalog
2077            .query_batch(&CatalogQuery::new(NautilusDataType::OrderBookDelta))
2078            .unwrap();
2079
2080        let DataBatch::BookDelta(rows) = batch else {
2081            panic!("expected order book deltas");
2082        };
2083
2084        assert_eq!(rows.len(), deltas.len());
2085
2086        for (actual, expected) in rows.as_ref().iter().zip(&deltas) {
2087            let (price_precision, size_precision) = if expected.sequence < 43 {
2088                (2, 3)
2089            } else {
2090                (4, 5)
2091            };
2092
2093            assert_eq!(actual.instrument_id, expected.instrument_id);
2094            assert_eq!(actual.action, expected.action);
2095            assert_eq!(actual.order.side, expected.order.side);
2096            assert_eq!(actual.order.price.raw(), expected.order.price.raw());
2097            assert_eq!(actual.order.price.precision, price_precision);
2098            assert_eq!(actual.order.size.raw(), expected.order.size.raw());
2099            assert_eq!(actual.order.size.precision, size_precision);
2100            assert_eq!(actual.order.order_id, expected.order.order_id);
2101            assert_eq!(actual.flags, expected.flags);
2102            assert_eq!(actual.sequence, expected.sequence);
2103            assert_eq!(actual.ts_event, expected.ts_event);
2104            assert_eq!(actual.ts_init, expected.ts_init);
2105        }
2106    }
2107
2108    #[rstest]
2109    #[case::real_add(BookAction::Add, Some(OrderSide::Buy), "1", "2", 0)]
2110    #[case::clear_side(BookAction::Clear, Some(OrderSide::Buy), "0", "0", 0)]
2111    #[case::clear_price(BookAction::Clear, None, "1", "0", 0)]
2112    #[case::clear_size(BookAction::Clear, None, "0", "1", 0)]
2113    #[case::clear_order_id(BookAction::Clear, None, "0", "0", 1)]
2114    fn parquet_promotion_rejects_noncanonical_zero_precision_orders(
2115        #[case] action: BookAction,
2116        #[case] side: Option<OrderSide>,
2117        #[case] price: &str,
2118        #[case] size: &str,
2119        #[case] order_id: u64,
2120        #[values(false, true)] automatic: bool,
2121    ) {
2122        let directory = TempDir::new().unwrap();
2123        let staging = directory.path().join("backtest").join("run-clear-invalid");
2124        let mut config =
2125            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2126        config.promote_on_close = automatic;
2127        let mut sink =
2128            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2129                .unwrap();
2130        let id = InstrumentId::from("AUD/USD.SIM");
2131        let deltas = vec![
2132            OrderBookDelta::clear(id, 41, 99.into(), 100.into()),
2133            OrderBookDelta::new(
2134                id,
2135                action,
2136                BookOrder {
2137                    side,
2138                    price: Price::from(price),
2139                    size: Quantity::from(size),
2140                    order_id,
2141                },
2142                0,
2143                41,
2144                99.into(),
2145                100.into(),
2146            ),
2147            OrderBookDelta::new(
2148                id,
2149                BookAction::Add,
2150                BookOrder::new(
2151                    OrderSide::Sell,
2152                    Price::from("1.23"),
2153                    Quantity::from("4.500"),
2154                    72,
2155                ),
2156                RecordFlag::F_LAST as u8,
2157                41,
2158                99.into(),
2159                100.into(),
2160            ),
2161        ];
2162        sink.write_batch(deltas.into_iter().map(Data::from).collect())
2163            .unwrap();
2164        let mut catalog = ParquetDataCatalog::from_uri(
2165            directory.path().to_str().unwrap(),
2166            None,
2167            None,
2168            None,
2169            None,
2170        )
2171        .unwrap();
2172
2173        let error = if automatic {
2174            sink.close().unwrap_err()
2175        } else {
2176            sink.close().unwrap();
2177            catalog
2178                .convert_stream_to_data(
2179                    "run-clear-invalid",
2180                    &NautilusDataType::OrderBookDelta.into(),
2181                    Environment::Backtest,
2182                    None,
2183                    false,
2184                )
2185                .unwrap_err()
2186        };
2187
2188        assert!(error.to_string().contains(
2189            "cannot relabel precision metadata for a promotion group that contains decimal values"
2190        ));
2191        assert!(
2192            catalog
2193                .query_files(&NautilusDataType::OrderBookDelta.into(), None, None, None)
2194                .unwrap()
2195                .is_empty()
2196        );
2197    }
2198
2199    #[rstest]
2200    fn parquet_manual_conversion_promotes_custom_data() {
2201        use nautilus_model::data::{CustomData, DataType};
2202        use nautilus_serialization::ensure_custom_data_registered;
2203
2204        ensure_custom_data_registered::<RustTestHashMapCustomData>();
2205
2206        let directory = TempDir::new().unwrap();
2207        let staging = directory.path().join("backtest").join("run-custom");
2208        let mut config =
2209            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2210        config.promote_on_close = false;
2211        let mut sink =
2212            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2213                .unwrap();
2214
2215        let data_type = DataType::new("RustTestHashMapCustomData", None, None);
2216        let records = [
2217            sample_custom_data("first", "AUD/USD.SIM", "1.23456", 11),
2218            sample_custom_data("second", "BTCUSDT.BINANCE", "65432.10", 21),
2219        ];
2220
2221        for record in &records {
2222            sink.write_data(Data::Custom(CustomData::new(
2223                Arc::new(record.clone()),
2224                data_type.clone(),
2225            )))
2226            .unwrap();
2227        }
2228
2229        sink.close().unwrap();
2230
2231        let mut catalog = ParquetDataCatalog::from_uri(
2232            directory.path().to_str().unwrap(),
2233            None,
2234            None,
2235            None,
2236            None,
2237        )
2238        .unwrap();
2239
2240        for _ in 0..2 {
2241            catalog
2242                .convert_stream_to_data(
2243                    "run-custom",
2244                    &NautilusDataType::Custom {
2245                        type_name: "RustTestHashMapCustomData".to_string(),
2246                    }
2247                    .into(),
2248                    Environment::Backtest,
2249                    None,
2250                    false,
2251                )
2252                .unwrap();
2253        }
2254
2255        let expected: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2256            .iter()
2257            .map(|record| (None, record.clone()))
2258            .collect();
2259        assert_eq!(query_custom_records(&mut catalog, None), expected);
2260    }
2261
2262    #[rstest]
2263    fn parquet_manual_conversion_filters_custom_data_identifiers() {
2264        use nautilus_model::data::{CustomData, DataType};
2265        use nautilus_serialization::ensure_custom_data_registered;
2266
2267        ensure_custom_data_registered::<RustTestHashMapCustomData>();
2268
2269        let directory = TempDir::new().unwrap();
2270        let staging = directory.path().join("backtest").join("run-custom-ids");
2271        let mut config =
2272            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2273        config.promote_on_close = false;
2274        let mut sink =
2275            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2276                .unwrap();
2277
2278        let audusd = "AUD/USD.SIM";
2279        let btcusdt = "BTCUSDT.BINANCE";
2280        let records = [
2281            (audusd, sample_custom_data("first", audusd, "1.23456", 11)),
2282            (audusd, sample_custom_data("second", audusd, "1.23457", 21)),
2283            (
2284                btcusdt,
2285                sample_custom_data("third", btcusdt, "65432.10", 31),
2286            ),
2287        ];
2288
2289        for (identifier, record) in &records {
2290            let data_type = DataType::new(
2291                "RustTestHashMapCustomData",
2292                None,
2293                Some((*identifier).to_string()),
2294            );
2295            sink.write_data(Data::Custom(CustomData::new(
2296                Arc::new(record.clone()),
2297                data_type,
2298            )))
2299            .unwrap();
2300        }
2301
2302        sink.close().unwrap();
2303
2304        let mut catalog = ParquetDataCatalog::from_uri(
2305            directory.path().to_str().unwrap(),
2306            None,
2307            None,
2308            None,
2309            None,
2310        )
2311        .unwrap();
2312        catalog
2313            .convert_stream_to_data(
2314                "run-custom-ids",
2315                &NautilusDataType::Custom {
2316                    type_name: "RustTestHashMapCustomData".to_string(),
2317                }
2318                .into(),
2319                Environment::Backtest,
2320                Some(&[audusd.to_string()]),
2321                false,
2322            )
2323            .unwrap();
2324
2325        let expected_audusd: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2326            .iter()
2327            .filter(|(identifier, _)| *identifier == audusd)
2328            .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2329            .collect();
2330        assert_eq!(query_custom_records(&mut catalog, None), expected_audusd);
2331
2332        // Catalog identifier directories are urisafe, so a scoped query pins that spelling and
2333        // proves the converted records landed under their own identifier rather than together.
2334        assert_eq!(
2335            query_custom_records(&mut catalog, Some(&["AUDUSD.SIM".to_string()])),
2336            expected_audusd,
2337        );
2338
2339        catalog
2340            .convert_stream_to_data(
2341                "run-custom-ids",
2342                &NautilusDataType::Custom {
2343                    type_name: "RustTestHashMapCustomData".to_string(),
2344                }
2345                .into(),
2346                Environment::Backtest,
2347                None,
2348                false,
2349            )
2350            .unwrap();
2351
2352        let expected_all: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2353            .iter()
2354            .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2355            .collect();
2356        assert_eq!(query_custom_records(&mut catalog, None), expected_all);
2357        let expected_btcusdt: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2358            .iter()
2359            .filter(|(identifier, _)| *identifier == btcusdt)
2360            .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2361            .collect();
2362        assert_eq!(
2363            query_custom_records(&mut catalog, Some(&[btcusdt.to_string()])),
2364            expected_btcusdt,
2365        );
2366    }
2367
2368    fn sample_custom_data(
2369        name: &str,
2370        instrument_id: &str,
2371        price: &str,
2372        ts: u64,
2373    ) -> RustTestHashMapCustomData {
2374        RustTestHashMapCustomData {
2375            name: name.to_string(),
2376            prices: [(instrument_id.to_string(), Price::from(price))]
2377                .into_iter()
2378                .collect(),
2379            ts_event: UnixNanos::from(ts - 1),
2380            ts_init: UnixNanos::from(ts),
2381        }
2382    }
2383
2384    fn query_custom_records(
2385        catalog: &mut ParquetDataCatalog,
2386        identifiers: Option<&[String]>,
2387    ) -> Vec<(Option<String>, RustTestHashMapCustomData)> {
2388        let loaded = catalog
2389            .query_custom_data_dynamic(
2390                "RustTestHashMapCustomData",
2391                identifiers,
2392                None,
2393                None,
2394                None,
2395                None,
2396                true,
2397            )
2398            .unwrap();
2399        let mut decoded = Vec::new();
2400
2401        for data in &loaded {
2402            let Data::Custom(custom) = data else {
2403                panic!("expected custom data, found {data:?}")
2404            };
2405
2406            assert_eq!(custom.data_type.type_name(), "RustTestHashMapCustomData");
2407            decoded.push((
2408                custom.data_type.identifier().map(ToString::to_string),
2409                custom
2410                    .data
2411                    .as_any()
2412                    .downcast_ref::<RustTestHashMapCustomData>()
2413                    .expect("expected RustTestHashMapCustomData")
2414                    .clone(),
2415            ));
2416        }
2417
2418        decoded.sort_by_key(|(_, record)| record.ts_init);
2419        decoded
2420    }
2421
2422    fn local_catalog(directory: &TempDir) -> CatalogConnectConfig {
2423        CatalogConnectConfig::new(directory.path().to_string_lossy(), None)
2424    }
2425
2426    // Earlier per-instrument writers staged files in identifier directories
2427    fn stage_per_identifier_quote(staging: &Path) -> (QuoteTick, PathBuf) {
2428        let mut quote = sample_quote();
2429        quote.instrument_id = InstrumentId::from("CAFÉ.SIM");
2430
2431        let mut writer = FeatherWriter::new(
2432            staging.to_path_buf(),
2433            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2434            RotationConfig::NoRotation,
2435            None,
2436            None,
2437        );
2438        writer.write(quote).unwrap();
2439        writer.close().unwrap();
2440
2441        let identifier_directory = staging.join("quotes").join("CAFÉ.SIM");
2442        let staged_file = identifier_directory.join("quotes_0.feather");
2443        fs::create_dir(&identifier_directory).unwrap();
2444        fs::rename(
2445            staging.join("quotes").join("quotes_0.feather"),
2446            &staged_file,
2447        )
2448        .unwrap();
2449
2450        (quote, staged_file)
2451    }
2452
2453    fn sample_quote() -> QuoteTick {
2454        QuoteTick::new(
2455            InstrumentId::from("AUD/USD.SIM"),
2456            Price::from("0.65"),
2457            Price::from("0.67"),
2458            Quantity::from("11"),
2459            Quantity::from("17"),
2460            UnixNanos::from(19),
2461            UnixNanos::from(23),
2462        )
2463    }
2464
2465    #[rstest]
2466    #[case::ascii("AUD/USD.SIM-1-MINUTE-LAST-EXTERNAL")]
2467    #[case::non_ascii("CAFÉ.SIM-1-MINUTE-LAST-EXTERNAL")]
2468    fn bar_replay_across_promotions_is_idempotent(
2469        #[case] bar_type: &str,
2470        #[values(false, true)] restart: bool,
2471        #[values(false, true)] replay_only: bool,
2472        #[values(false, true)] use_ts_event: bool,
2473    ) {
2474        let directory = TempDir::new().unwrap();
2475
2476        let mut config = WriterConnectConfig::new(
2477            directory.path().join("live/run-bars").to_string_lossy(),
2478            Some(local_catalog(&directory)),
2479        );
2480        config.use_ts_event_for_ts_init = use_ts_event;
2481        config.delete_feather_after_promotion = true;
2482        config.promote_on_close = false;
2483        let clock = Arc::new(AtomicU64::new(1));
2484        let mut writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2485
2486        let first = Bar {
2487            bar_type: BarType::from(bar_type),
2488            ..sample_bar()
2489        };
2490
2491        writer.core.write_data(Data::Bar(first)).unwrap();
2492        assert_eq!(writer.promote_now().unwrap().len(), 1);
2493
2494        if restart {
2495            drop(writer);
2496            writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2497        }
2498
2499        clock.store(2, Ordering::Relaxed);
2500
2501        let replay = Bar {
2502            ts_init: if use_ts_event {
2503                UnixNanos::from(63_000_000_000)
2504            } else {
2505                first.ts_init
2506            },
2507            ..first
2508        };
2509
2510        let next = Bar {
2511            ts_event: UnixNanos::from(120_000_000_000),
2512            ts_init: UnixNanos::from(121_000_000_000),
2513            ..first
2514        };
2515
2516        writer.core.write_data(Data::Bar(replay)).unwrap();
2517        if !replay_only {
2518            writer.core.write_data(Data::Bar(next)).unwrap();
2519        }
2520
2521        let promoted = writer.promote_now().unwrap();
2522        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2523
2524        let DataBatch::Bar(rows) = catalog
2525            .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2526            .unwrap()
2527        else {
2528            panic!("expected bars")
2529        };
2530
2531        let mut expected = vec![Bar {
2532            ts_init: if use_ts_event {
2533                first.ts_event
2534            } else {
2535                first.ts_init
2536            },
2537            ..first
2538        }];
2539
2540        if !replay_only {
2541            expected.push(Bar {
2542                ts_init: if use_ts_event {
2543                    next.ts_event
2544                } else {
2545                    next.ts_init
2546                },
2547                ..next
2548            });
2549        }
2550
2551        assert_eq!(promoted.len(), 1);
2552        assert_eq!(rows.as_ref(), expected.as_slice());
2553        assert_eq!(
2554            catalog
2555                .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2556                .unwrap()
2557                .len(),
2558            expected.len()
2559        );
2560        assert!(
2561            list_session_feather_files(&writer.source.storage, Environment::Live, "run-bars")
2562                .unwrap()
2563                .is_empty()
2564        );
2565    }
2566
2567    #[rstest]
2568    #[case::open("open")]
2569    #[case::high("high")]
2570    #[case::low("low")]
2571    #[case::close("close")]
2572    #[case::volume("volume")]
2573    #[case::price_precision("price_precision")]
2574    #[case::size_precision("size_precision")]
2575    #[case::event_timestamp("ts_event")]
2576    #[case::init_timestamp("ts_init")]
2577    fn bar_replay_rejects_changed_values(#[case] field: &str) {
2578        let directory = TempDir::new().unwrap();
2579
2580        let mut config = WriterConnectConfig::new(
2581            directory.path().join("live/run-conflict").to_string_lossy(),
2582            Some(local_catalog(&directory)),
2583        );
2584        config.use_ts_event_for_ts_init = !matches!(field, "ts_event" | "ts_init");
2585        config.delete_feather_after_promotion = true;
2586        config.promote_on_close = false;
2587        let clock = Arc::new(AtomicU64::new(1));
2588        let mut writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2589        let first = sample_bar();
2590        writer.core.write_data(Data::Bar(first)).unwrap();
2591        writer.promote_now().unwrap();
2592        clock.store(2, Ordering::Relaxed);
2593        let mut correction = first;
2594        match field {
2595            "open" => correction.open = Price::from("1.02"),
2596            "high" => correction.high = Price::from("1.06"),
2597            "low" => correction.low = Price::from("0.98"),
2598            "close" => correction.close = Price::from("1.04"),
2599            "volume" => correction.volume = Quantity::from("18.000"),
2600            "price_precision" => {
2601                correction.open = Price::from("1.010");
2602                correction.high = Price::from("1.050");
2603                correction.low = Price::from("0.990");
2604                correction.close = Price::from("1.030");
2605            }
2606            "size_precision" => correction.volume = Quantity::from("17.0000"),
2607            "ts_event" => correction.ts_event = UnixNanos::from(59_000_000_000),
2608            "ts_init" => correction.ts_init = UnixNanos::from(60_500_000_000),
2609            _ => unreachable!(),
2610        }
2611
2612        let next = Bar {
2613            ts_event: UnixNanos::from(120_000_000_000),
2614            ts_init: UnixNanos::from(121_000_000_000),
2615            ..first
2616        };
2617
2618        writer.core.write_data(Data::Bar(correction)).unwrap();
2619        writer.core.write_data(Data::Bar(next)).unwrap();
2620        let work = writer.prepare_promotion().unwrap().unwrap();
2621        let files = work.files().to_vec();
2622        let error = writer.finalize(work.execute()).unwrap_err();
2623        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2624
2625        let DataBatch::Bar(rows) = catalog
2626            .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2627            .unwrap()
2628        else {
2629            panic!("expected bars")
2630        };
2631
2632        let expected = if config.use_ts_event_for_ts_init {
2633            Bar {
2634                ts_init: first.ts_event,
2635                ..first
2636            }
2637        } else {
2638            first
2639        };
2640
2641        assert!(error.to_string().contains("non-disjoint intervals"));
2642        assert_eq!(rows.as_ref(), &[expected]);
2643        assert_eq!(
2644            catalog
2645                .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2646                .unwrap()
2647                .len(),
2648            1
2649        );
2650        assert_eq!(files.len(), 1);
2651        assert!(directory.path().join(&files[0]).exists());
2652    }
2653
2654    #[rstest]
2655    fn bar_replay_manual_conversion_is_idempotent() {
2656        let directory = TempDir::new().unwrap();
2657        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2658        let first = sample_bar();
2659
2660        let next = Bar {
2661            ts_event: UnixNanos::from(120_000_000_000),
2662            ts_init: UnixNanos::from(121_000_000_000),
2663            ..first
2664        };
2665
2666        for run in ["first", "restart"] {
2667            let mut writer = FeatherWriter::new(
2668                directory.path().join("backtest").join(run),
2669                WriterClock::Test(Arc::new(AtomicU64::new(0))),
2670                RotationConfig::NoRotation,
2671                None,
2672                None,
2673            );
2674            writer.write(first).unwrap();
2675            if run == "restart" {
2676                writer.write(next).unwrap();
2677            }
2678
2679            writer.close().unwrap();
2680            catalog
2681                .convert_stream_to_data(
2682                    run,
2683                    &NautilusDataType::Bar.into(),
2684                    Environment::Backtest,
2685                    None,
2686                    true,
2687                )
2688                .unwrap();
2689        }
2690
2691        let DataBatch::Bar(rows) = catalog
2692            .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2693            .unwrap()
2694        else {
2695            panic!("expected bars")
2696        };
2697
2698        assert_eq!(
2699            rows.as_ref(),
2700            &[
2701                Bar {
2702                    ts_init: first.ts_event,
2703                    ..first
2704                },
2705                Bar {
2706                    ts_init: next.ts_event,
2707                    ..next
2708                },
2709            ]
2710        );
2711        assert_eq!(
2712            catalog
2713                .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2714                .unwrap()
2715                .len(),
2716            2
2717        );
2718    }
2719
2720    fn sample_bar() -> Bar {
2721        Bar::new(
2722            BarType::from("AUD/USD.SIM-1-MINUTE-LAST-EXTERNAL"),
2723            Price::from("1.01"),
2724            Price::from("1.05"),
2725            Price::from("0.99"),
2726            Price::from("1.03"),
2727            Quantity::from("17.000"),
2728            UnixNanos::from(60_000_000_000),
2729            UnixNanos::from(61_000_000_000),
2730        )
2731    }
2732
2733    #[rstest]
2734    fn kept_promotions_release_scheduled_paths() {
2735        let directory = TempDir::new().unwrap();
2736        let config = WriterConnectConfig::new(
2737            directory.path().join("live/run-bounded").to_string_lossy(),
2738            Some(local_catalog(&directory)),
2739        );
2740        let mut writer = ParquetWriter::new(&config, WriterClock::Live).unwrap();
2741
2742        for timestamp in 1..=32 {
2743            writer
2744                .core
2745                .write_data(Data::Quote(QuoteTick {
2746                    ts_event: UnixNanos::from(timestamp),
2747                    ts_init: UnixNanos::from(timestamp),
2748                    ..sample_quote()
2749                }))
2750                .unwrap();
2751            let work = writer.prepare_promotion().unwrap().unwrap();
2752            assert_eq!(work.files().len(), 1);
2753            writer.finalize(work.execute()).unwrap();
2754            assert_scheduled_count(&writer.core.promotion_driver.schedule(), 0);
2755            assert!(writer.prepare_promotion().unwrap().is_none());
2756        }
2757
2758        assert_eq!(
2759            list_session_feather_files(&writer.source.storage, Environment::Live, "run-bounded")
2760                .unwrap()
2761                .len(),
2762            32
2763        );
2764    }
2765
2766    #[rstest]
2767    #[case::timer_seal(RotationConfig::NoRotation, false, 1)]
2768    #[case::close(RotationConfig::NoRotation, true, 1)]
2769    #[case::rotation(RotationConfig::Size { max_size: 1 }, false, 2)]
2770    fn sealed_paths_promote_once_and_do_not_resubmit_in_flight(
2771        #[case] rotation: RotationConfig,
2772        #[case] close: bool,
2773        #[case] expected_files: usize,
2774    ) {
2775        let directory = TempDir::new().unwrap();
2776        let mut config = WriterConnectConfig::new(
2777            directory.path().join("live/run-sealing").to_string_lossy(),
2778            Some(local_catalog(&directory)),
2779        );
2780        config.rotation_config = rotation;
2781        let mut writer = ParquetWriter::new(&config, WriterClock::Live).unwrap();
2782        let first = sample_quote();
2783        let second = QuoteTick {
2784            ts_event: UnixNanos::from(29),
2785            ts_init: UnixNanos::from(31),
2786            ..first
2787        };
2788        writer.core.write_data(Data::Quote(first)).unwrap();
2789        writer.core.write_data(Data::Quote(second)).unwrap();
2790
2791        if close {
2792            writer.core.close().unwrap();
2793        }
2794
2795        let work = writer.prepare_promotion().unwrap().unwrap();
2796        let files = work.files().to_vec();
2797        assert_eq!(files.len(), expected_files);
2798        assert!(writer.prepare_promotion().unwrap().is_none());
2799        assert_scheduled_count(&writer.core.promotion_driver.schedule(), expected_files);
2800        let result = work.execute();
2801        assert_scheduled_count(&writer.core.promotion_driver.schedule(), 0);
2802        assert_eq!(result.committed_paths, files);
2803        assert_eq!(writer.finalize(result).unwrap().len(), expected_files);
2804        assert!(writer.prepare_promotion().unwrap().is_none());
2805
2806        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2807        let DataBatch::Quote(rows) = catalog
2808            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2809            .unwrap()
2810        else {
2811            panic!("expected quotes")
2812        };
2813        assert_eq!(rows.as_ref(), &[first, second]);
2814    }
2815
2816    #[rstest]
2817    #[case::space("my data")]
2818    #[case::non_ascii("Données")]
2819    #[case::fragment("data#1")]
2820    #[case::literal_percent("data%20")]
2821    fn writer_root_path_promotes_and_replays(
2822        #[case] root_name: &str,
2823        #[values(false, true)] file_uri: bool,
2824    ) {
2825        let directory = TempDir::new().unwrap();
2826        let root = directory.path().join(root_name);
2827        let staging = root.join("live/run-root");
2828        let staging_uri = if file_uri {
2829            normalize_path_to_uri(&staging.to_string_lossy()).unwrap()
2830        } else {
2831            staging.to_string_lossy().into_owned()
2832        };
2833        let config = WriterConnectConfig::new(staging_uri, Some(local_catalog(&directory)));
2834        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2835        let quote = sample_quote();
2836        let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
2837        writer.core.write_data(Data::Quote(quote)).unwrap();
2838        let work = writer.prepare_promotion().unwrap().unwrap();
2839        assert_eq!(work.files(), &["live/run-root/quotes/quotes_0.feather"]);
2840        let promoted = writer.finalize(work.execute()).unwrap();
2841        assert_eq!(promoted.len(), 1);
2842        assert!(writer.prepare_promotion().unwrap().is_none());
2843        drop(writer);
2844
2845        let mut restarted = ParquetWriter::new(&config, clock).unwrap();
2846        let work = restarted.prepare_promotion().unwrap().unwrap();
2847        assert_eq!(work.files(), &["live/run-root/quotes/quotes_0.feather"]);
2848        assert_eq!(restarted.finalize(work.execute()).unwrap(), promoted);
2849        assert!(restarted.prepare_promotion().unwrap().is_none());
2850
2851        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2852        let DataBatch::Quote(rows) = catalog
2853            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2854            .unwrap()
2855        else {
2856            panic!("expected quotes")
2857        };
2858        assert_eq!(rows.as_ref(), &[quote]);
2859    }
2860
2861    #[rstest]
2862    fn parent_relative_writer_path_promotes_and_replays() {
2863        let directory = TempDir::new_in(".").unwrap();
2864        let current = std::env::current_dir().unwrap();
2865        let staging = PathBuf::from("..")
2866            .join(current.file_name().unwrap())
2867            .join(directory.path().file_name().unwrap())
2868            .join("live/run-relative");
2869        let config =
2870            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2871        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2872        let quote = sample_quote();
2873        let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
2874        writer.core.write_data(Data::Quote(quote)).unwrap();
2875        let work = writer.prepare_promotion().unwrap().unwrap();
2876        assert_eq!(work.files(), &["live/run-relative/quotes/quotes_0.feather"]);
2877        let promoted = writer.finalize(work.execute()).unwrap();
2878        assert_eq!(promoted.len(), 1);
2879        assert!(writer.prepare_promotion().unwrap().is_none());
2880        drop(writer);
2881
2882        let mut restarted = ParquetWriter::new(&config, clock).unwrap();
2883        let work = restarted.prepare_promotion().unwrap().unwrap();
2884        assert_eq!(work.files(), &["live/run-relative/quotes/quotes_0.feather"]);
2885        assert_eq!(restarted.finalize(work.execute()).unwrap(), promoted);
2886        assert!(restarted.prepare_promotion().unwrap().is_none());
2887
2888        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2889        let DataBatch::Quote(rows) = catalog
2890            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2891            .unwrap()
2892        else {
2893            panic!("expected quotes")
2894        };
2895        assert_eq!(rows.as_ref(), &[quote]);
2896    }
2897
2898    #[rstest]
2899    #[case::sealed(false)]
2900    #[case::partial(true)]
2901    fn restart_discovers_crashed_files_and_deduplicates_promotions(
2902        #[case] partial: bool,
2903        #[values(false, true)] delete_source: bool,
2904        #[values(false, true)] restage: bool,
2905    ) {
2906        let directory = TempDir::new().unwrap();
2907        let staging = directory.path().join("backtest/run-restart");
2908        let quote = sample_quote();
2909        let mut crashed = FeatherWriter::new(
2910            staging.clone(),
2911            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2912            RotationConfig::NoRotation,
2913            None,
2914            None,
2915        );
2916        crashed.write(quote).unwrap();
2917        crashed.close().unwrap();
2918        let sealed = staging.join("quotes/quotes_0.feather");
2919
2920        if partial {
2921            fs::rename(&sealed, sealed.with_extension(FEATHER_PARTIAL_EXTENSION)).unwrap();
2922        }
2923
2924        drop(crashed);
2925        let mut config =
2926            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2927        config.delete_feather_after_promotion = restage;
2928        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2929        let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
2930        let source_bytes = fs::read(&sealed).unwrap();
2931        let work = writer.prepare_promotion().unwrap().unwrap();
2932        assert_eq!(
2933            work.files(),
2934            &["backtest/run-restart/quotes/quotes_0.feather".to_string()]
2935        );
2936        let promoted = writer.finalize(work.execute()).unwrap();
2937        assert_eq!(promoted.len(), 1);
2938        assert!(writer.prepare_promotion().unwrap().is_none());
2939        drop(writer);
2940
2941        if restage {
2942            assert!(!sealed.exists());
2943            fs::write(&sealed, source_bytes).unwrap();
2944        }
2945
2946        config.delete_feather_after_promotion = delete_source;
2947        let mut restarted = ParquetWriter::new(&config, clock).unwrap();
2948        let work = restarted.prepare_promotion().unwrap().unwrap();
2949        assert_eq!(work.files().len(), 1);
2950        let files = work.files().to_vec();
2951        let result = work.execute();
2952        assert_eq!(result.committed_paths, files);
2953        assert_eq!(
2954            result.deleted_paths,
2955            if delete_source { files } else { Vec::new() }
2956        );
2957        assert_eq!(restarted.finalize(result).unwrap(), promoted);
2958        assert!(restarted.prepare_promotion().unwrap().is_none());
2959        assert_scheduled_count(&restarted.core.promotion_driver.schedule(), 0);
2960        assert_eq!(sealed.exists(), !delete_source);
2961
2962        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2963        let DataBatch::Quote(rows) = catalog
2964            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2965            .unwrap()
2966        else {
2967            panic!("expected quotes")
2968        };
2969        assert_eq!(rows.as_ref(), &[quote]);
2970    }
2971
2972    #[rstest]
2973    #[case::nested_base("/prefix/catalog/", "prefix/catalog/")]
2974    #[case::root_base("", "")]
2975    fn promotion_paths_live_under_the_catalog_base_path(
2976        #[case] base_path: &str,
2977        #[case] expected_prefix: &str,
2978    ) {
2979        let directory = TempDir::new().unwrap();
2980        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2981        catalog.base_path = base_path.to_string();
2982
2983        let backend = ParquetPromotionBackend {
2984            catalog,
2985            legacy_manifest_missing: Arc::default(),
2986        };
2987
2988        assert_eq!(
2989            backend.manifest_path(),
2990            ObjectPath::from(format!("{expected_prefix}_nautilus_promotions.json")),
2991        );
2992        assert_eq!(
2993            backend.marker_path("replay-id"),
2994            ObjectPath::from(format!(
2995                "{expected_prefix}_nautilus_promotions/\
2996                 cd3112001080bc2d11985ffe2b8b90b324d336d82c17ddb66127d3a05f08c69c.json"
2997            )),
2998        );
2999    }
3000
3001    #[rstest]
3002    fn feather_replay_identity_is_stable_and_ignores_identifier_order() {
3003        let identity = |identifiers: Option<&[String]>| {
3004            feather_replay_identity(
3005                "file:///catalog/backtest/run-1",
3006                "quotes/AUDUSD.SIM/part-0.feather",
3007                "content-hash",
3008                identifiers,
3009            )
3010        };
3011
3012        let unordered = ["B".to_string(), "A".to_string(), "A".to_string()];
3013        let ordered = ["A".to_string(), "B".to_string()];
3014
3015        let expected =
3016            "nautilus-feather:10a9435c28f7536f26653c3fc808571be89bfe769e06c7307cb4743e273a23fd";
3017
3018        assert_eq!(identity(Some(&unordered)), expected);
3019        assert_eq!(identity(Some(&ordered)), expected);
3020        assert_ne!(identity(None), expected);
3021    }
3022}