Skip to main content

nautilus_persistence/backend/parquet/
writer.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautilustrader.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8// -------------------------------------------------------------------------------------------------
9
10//! Feather-staged promoting writer for the Parquet catalog.
11
12use std::{
13    fmt::Debug,
14    sync::{
15        Arc,
16        atomic::{AtomicBool, Ordering},
17    },
18};
19
20use nautilus_common::live::block_on_nautilus_with;
21use object_store::{
22    Error as ObjectStoreError, ObjectStoreExt, PutMode, PutOptions, path::Path as ObjectPath,
23};
24use serde::{Deserialize, Serialize};
25
26use super::catalog::ParquetDataCatalog;
27use crate::{
28    catalog::factory::CatalogConnectConfig,
29    common::{
30        conversion::FeatherConversionSummary,
31        datafusion::identifiers_from_record_batches,
32        paths::{
33            catalog_data_type_from_session_feather_path, create_local_directory,
34            identifier_from_session_feather_path,
35        },
36        storage::create_storage_backend_from_path,
37    },
38    writer::{
39        factory::{PARQUET_WRITER_FACTORY_NAME, WriterConnectConfig, WriterFactoryRegistry},
40        feather::WriterClock,
41        materializer::read_feather_record_batches_with_identity,
42        promotion::{
43            PromotionBackend, PromotionResult, PromotionSession, PromotionWork,
44            StagedPromotionBackend,
45        },
46        run::{FeatherSessionSource, RunStatus},
47        staged::{StagedFeatherWriter, StagedWriter},
48        traits::StreamingDataSink,
49    },
50};
51
52pub(crate) fn register_factory(registry: &mut WriterFactoryRegistry) {
53    registry.insert(
54        PARQUET_WRITER_FACTORY_NAME.to_string(),
55        Arc::new(parquet_writer_factory),
56    );
57}
58
59fn parquet_writer_factory(
60    config: &WriterConnectConfig,
61    clock: WriterClock,
62) -> anyhow::Result<StreamingDataSink> {
63    Ok(Box::new(ParquetWriter::new(config, clock)?))
64}
65
66#[expect(
67    clippy::struct_excessive_bools,
68    reason = "the booleans are independent writer policy and lifecycle flags"
69)]
70struct ParquetWriter {
71    core: StagedFeatherWriter<ParquetPromotionBackend>,
72    session: PromotionSession,
73    source: FeatherSessionSource,
74    catalog: CatalogConnectConfig,
75    interval_ms: Option<u64>,
76    promote_on_close: bool,
77    delete_feather_after_commit: bool,
78    use_ts_event_for_ts_init: bool,
79    legacy_manifest_missing: Arc<AtomicBool>,
80    has_data: bool,
81    run_status: RunStatus,
82}
83
84impl Debug for ParquetWriter {
85    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86        f.debug_struct(stringify!(ParquetWriter))
87            .field("staging_uri", &self.core.storage.original_uri)
88            .field("catalog_uri", &self.catalog.uri)
89            .field("interval_ms", &self.interval_ms)
90            .finish_non_exhaustive()
91    }
92}
93
94impl ParquetWriter {
95    fn new(config: &WriterConnectConfig, clock: WriterClock) -> anyhow::Result<Self> {
96        let catalog = config
97            .required_catalog(PARQUET_WRITER_FACTORY_NAME)?
98            .clone();
99        let storage = create_storage_backend_from_path(&config.uri, None)?;
100        let session = StagedFeatherWriter::<ParquetPromotionBackend>::required_session(
101            &storage.original_uri,
102            "Parquet writer URI",
103        )?;
104        let source_storage = create_storage_backend_from_path(&session.root_uri, None)?;
105
106        let source = FeatherSessionSource::new(
107            source_storage,
108            session.environment,
109            session.instance_id.clone(),
110        );
111
112        // A Parquet catalog opens only an existing local directory, so the writer creates the one
113        // it promotes into, then opens it now rather than failing at the first promotion
114        create_local_directory(&catalog.uri)?;
115
116        let legacy_manifest_missing = Arc::new(AtomicBool::new(false));
117
118        ParquetPromotionBackend::new(&catalog, Arc::clone(&legacy_manifest_missing))?;
119
120        let mut core = StagedFeatherWriter::new(
121            storage.clone(),
122            &source,
123            clock,
124            config.rotation_config.clone(),
125            None,
126            config.flush_interval_ms,
127            config.record_filter.clone(),
128        )?;
129
130        block_on_nautilus_with(|| {
131            storage.write_current_run_manifest(
132                session.environment,
133                &session.instance_id,
134                RunStatus::InProgress,
135                true,
136            )
137        })?;
138
139        let timer_catalog = catalog.clone();
140        core.warn_if_orphan_feather_present("Parquet");
141        let timer_legacy_manifest_missing = Arc::clone(&legacy_manifest_missing);
142        core.start_promotion_timer(
143            config.promotion_interval_ms,
144            source.clone(),
145            "parquet-promotion",
146            move || {
147                ParquetPromotionBackend::new(
148                    &timer_catalog,
149                    Arc::clone(&timer_legacy_manifest_missing),
150                )
151            },
152            config.use_ts_event_for_ts_init,
153            config.delete_feather_after_promotion,
154        )?;
155
156        Ok(Self {
157            core,
158            catalog,
159            session,
160            source,
161            interval_ms: config.promotion_interval_ms,
162            promote_on_close: config.promote_on_close,
163            delete_feather_after_commit: config.delete_feather_after_promotion,
164            use_ts_event_for_ts_init: config.use_ts_event_for_ts_init,
165            legacy_manifest_missing,
166            has_data: false,
167            run_status: RunStatus::InProgress,
168        })
169    }
170
171    fn record_status(&mut self, status: RunStatus) -> anyhow::Result<()> {
172        if self.run_status == status {
173            return Ok(());
174        }
175
176        block_on_nautilus_with(|| {
177            self.core.storage.write_current_run_manifest(
178                self.session.environment,
179                &self.session.instance_id,
180                status,
181                !self.has_data,
182            )
183        })?;
184
185        self.run_status = status;
186        Ok(())
187    }
188
189    fn mark_non_empty(&mut self) -> anyhow::Result<()> {
190        if self.has_data {
191            return Ok(());
192        }
193
194        block_on_nautilus_with(|| {
195            self.core.storage.write_current_run_manifest(
196                self.session.environment,
197                &self.session.instance_id,
198                RunStatus::InProgress,
199                false,
200            )
201        })?;
202
203        self.has_data = true;
204        Ok(())
205    }
206
207    fn interval_ns(&self) -> Option<u64> {
208        self.interval_ms
209            .filter(|interval_ms| *interval_ms > 0)
210            .map(|interval_ms| interval_ms.saturating_mul(1_000_000))
211    }
212
213    fn prepare_promotion(&self) -> anyhow::Result<Option<PromotionWork<ParquetPromotionBackend>>> {
214        self.core.prepare_promotion(
215            || {
216                ParquetPromotionBackend::new(
217                    &self.catalog,
218                    Arc::clone(&self.legacy_manifest_missing),
219                )
220            },
221            self.source.clone(),
222            self.use_ts_event_for_ts_init,
223            self.delete_feather_after_commit,
224        )
225    }
226
227    fn finalize(
228        &mut self,
229        result: PromotionResult,
230    ) -> anyhow::Result<Vec<FeatherConversionSummary>> {
231        if result.converted.is_err() {
232            if result.run_state_recorded {
233                self.run_status = RunStatus::Failed;
234            } else if let Err(e) = self.record_status(RunStatus::Failed) {
235                log::warn!("Failed to record Parquet run Failed state: {e}");
236            }
237        } else if result.run_state_recorded {
238            self.run_status = RunStatus::Promoted;
239        }
240
241        self.core.finalize_promotion(result)
242    }
243
244    fn drain_completed(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
245        let mut converted = Vec::new();
246        for result in self.core.promotion_driver.drain_completed()? {
247            converted.extend(self.finalize(result)?);
248        }
249
250        Ok(converted)
251    }
252
253    fn wait(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
254        let mut converted = Vec::new();
255        for result in self.core.promotion_driver.wait()? {
256            converted.extend(self.finalize(result)?);
257        }
258
259        Ok(converted)
260    }
261
262    fn promote_now(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
263        let mut converted = self.wait()?;
264        if let Some(work) = self.prepare_promotion()? {
265            converted.extend(self.finalize(work.execute())?);
266        }
267
268        Ok(converted)
269    }
270}
271
272impl StagedWriter for ParquetWriter {
273    type Backend = ParquetPromotionBackend;
274
275    fn staged(&mut self) -> &mut StagedFeatherWriter<Self::Backend> {
276        &mut self.core
277    }
278
279    fn mark_non_empty(&mut self) -> anyhow::Result<()> {
280        Self::mark_non_empty(self)
281    }
282
283    fn maybe_promote(&mut self) -> anyhow::Result<()> {
284        self.drain_completed()?;
285
286        let Some(interval_ns) = self.interval_ns() else {
287            return Ok(());
288        };
289
290        let now = self.core.clock.timestamp_ns();
291        if !self.core.promotion_driver.is_due(now, interval_ns) {
292            return Ok(());
293        }
294
295        // A due check starts the next interval even when it finds nothing or fails,
296        // otherwise every later message would seal and open the catalog again
297        self.core.promotion_driver.mark_committed_at(now);
298
299        if let Some(work) = self.prepare_promotion()? {
300            self.core
301                .promotion_driver
302                .submit(work, "parquet-promotion")?;
303        }
304
305        Ok(())
306    }
307
308    fn wait_for_promotions(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
309        self.wait()
310    }
311
312    fn promote_on_flush(&self) -> bool {
313        self.interval_ns().is_some_and(|interval_ns| {
314            self.core
315                .promotion_driver
316                .is_due(self.core.clock.timestamp_ns(), interval_ns)
317        })
318    }
319
320    fn promote_on_close(&self) -> bool {
321        self.promote_on_close
322    }
323
324    fn promote_now(&mut self) -> anyhow::Result<Vec<FeatherConversionSummary>> {
325        Self::promote_now(self)
326    }
327
328    fn record_completed(&mut self) {
329        if self.run_status == RunStatus::InProgress
330            && let Err(e) = self.record_status(RunStatus::Completed)
331        {
332            log::warn!("Failed to complete Parquet run manifest: {e}");
333        }
334    }
335}
336
337impl Drop for ParquetWriter {
338    fn drop(&mut self) {
339        self.core.stop_promotion_timer();
340
341        let promotion_failed = if let Err(e) = self.wait() {
342            log::warn!("ParquetWriter dropped with pending promotion error: {e}");
343            true
344        } else {
345            false
346        };
347
348        if self.run_status == RunStatus::InProgress {
349            let status = if promotion_failed || !self.core.is_closed().unwrap_or(false) {
350                RunStatus::Failed
351            } else {
352                RunStatus::Completed
353            };
354
355            if let Err(e) = self.record_status(status) {
356                log::warn!(
357                    "Failed to record Parquet run {} state during drop: {e}",
358                    status.as_str()
359                );
360            }
361        }
362    }
363}
364
365const PROMOTION_MANIFEST: &str = "_nautilus_promotions.json";
366const PROMOTION_MARKERS: &str = "_nautilus_promotions";
367
368struct ParquetPromotionBackend {
369    catalog: ParquetDataCatalog,
370    legacy_manifest_missing: Arc<AtomicBool>,
371}
372
373impl ParquetPromotionBackend {
374    fn new(
375        catalog: &CatalogConnectConfig,
376        legacy_manifest_missing: Arc<AtomicBool>,
377    ) -> anyhow::Result<Self> {
378        Ok(Self {
379            catalog: super::open_catalog(catalog)?,
380            legacy_manifest_missing,
381        })
382    }
383
384    fn manifest_path(&self) -> ObjectPath {
385        self.catalog_path(PROMOTION_MANIFEST)
386    }
387
388    fn manifest(&self) -> anyhow::Result<ParquetPromotionManifest> {
389        if self.legacy_manifest_missing.load(Ordering::Relaxed) {
390            return Ok(ParquetPromotionManifest::default());
391        }
392
393        let path = self.manifest_path();
394        block_on_nautilus_with(|| async {
395            let result = match self.catalog.object_store.get(&path).await {
396                Ok(result) => result,
397                Err(ObjectStoreError::NotFound { .. }) => {
398                    self.legacy_manifest_missing.store(true, Ordering::Relaxed);
399                    return Ok(ParquetPromotionManifest::default());
400                }
401                Err(e) => return Err(e.into()),
402            };
403
404            Ok(serde_json::from_slice(&result.bytes().await?)?)
405        })
406    }
407
408    fn marker_path(&self, identity: &str) -> ObjectPath {
409        let digest = blake3::hash(identity.as_bytes()).to_hex();
410        self.catalog_path(&format!("{PROMOTION_MARKERS}/{digest}.json"))
411    }
412
413    fn catalog_path(&self, path: &str) -> ObjectPath {
414        let base = self.catalog.base_path.trim_matches('/');
415        if base.is_empty() {
416            ObjectPath::from(path)
417        } else {
418            ObjectPath::from(format!("{base}/{path}"))
419        }
420    }
421
422    fn identity_recorded(&self, identity: &str) -> anyhow::Result<bool> {
423        let path = self.marker_path(identity);
424
425        let marker_exists = block_on_nautilus_with(|| async {
426            match self.catalog.object_store.head(&path).await {
427                Ok(_) => Ok::<bool, anyhow::Error>(true),
428                Err(ObjectStoreError::NotFound { .. }) => Ok(false),
429                Err(e) => Err(anyhow::Error::from(e)),
430            }
431        })?;
432
433        Ok(marker_exists
434            || self
435                .manifest()?
436                .identities
437                .iter()
438                .any(|item| item == identity))
439    }
440
441    fn record_identity(&self, identity: &str) -> anyhow::Result<()> {
442        let bytes = serde_json::to_vec(identity)?;
443        let path = self.marker_path(identity);
444        block_on_nautilus_with(|| async {
445            match self
446                .catalog
447                .object_store
448                .put_opts(
449                    &path,
450                    bytes.into(),
451                    PutOptions {
452                        mode: PutMode::Create,
453                        ..Default::default()
454                    },
455                )
456                .await
457            {
458                Ok(_) | Err(ObjectStoreError::AlreadyExists { .. }) => Ok(()),
459                Err(e) => Err(e.into()),
460            }
461        })
462    }
463}
464
465impl PromotionBackend for ParquetPromotionBackend {
466    type Source = FeatherSessionSource;
467
468    const NAME: &'static str = "Parquet";
469
470    fn convert_file(
471        &mut self,
472        source: &Self::Source,
473        file: &str,
474        use_ts_event_for_ts_init: bool,
475        record_promoted: bool,
476    ) -> anyhow::Result<Option<FeatherConversionSummary>> {
477        let object_path = ObjectPath::parse(file)?;
478
479        let read = block_on_nautilus_with(|| {
480            read_feather_record_batches_with_identity(
481                source.storage.object_store.clone(),
482                &object_path,
483            )
484        })?;
485
486        let identifiers = identifiers_from_record_batches(&read.batches)
487            .ok()
488            .filter(|identifiers| !identifiers.is_empty());
489        let identity = feather_replay_identity(
490            &source.storage.original_uri,
491            file,
492            &read.content_hash,
493            identifiers.as_deref(),
494        );
495
496        if self.identity_recorded(&identity)? {
497            let identifier = match identifiers {
498                Some(mut identifiers) if identifiers.len() == 1 => identifiers.pop(),
499                Some(_) => None,
500                None => identifier_from_session_feather_path(
501                    file,
502                    source.environment,
503                    &source.instance_id,
504                ),
505            };
506
507            return Ok(Some(FeatherConversionSummary {
508                data_type: catalog_data_type_from_session_feather_path(
509                    file,
510                    source.environment,
511                    &source.instance_id,
512                )?,
513                identifier,
514                feather_path: file.to_string(),
515                native_version: None,
516                unmatched_identifiers: None,
517            }));
518        }
519
520        let summary = self.catalog.promote_feather_file(
521            source,
522            file,
523            read.batches,
524            use_ts_event_for_ts_init,
525            &identity,
526        )?;
527
528        if summary.is_some() && record_promoted {
529            self.record_identity(&identity)?;
530        }
531
532        Ok(summary)
533    }
534
535    fn delete_file(&mut self, source: &Self::Source, file: &str) -> anyhow::Result<()> {
536        let object_path = ObjectPath::parse(file)?;
537        block_on_nautilus_with(|| async {
538            source.storage.object_store.delete(&object_path).await?;
539            Ok::<(), anyhow::Error>(())
540        })
541    }
542}
543
544impl StagedPromotionBackend for ParquetPromotionBackend {
545    const REQUIRES_SESSION: bool = true;
546
547    fn record_run_state(
548        &mut self,
549        source: &FeatherSessionSource,
550        _staging_uri: &str,
551        status: RunStatus,
552        empty: bool,
553        _error: Option<&str>,
554    ) -> anyhow::Result<()> {
555        block_on_nautilus_with(|| {
556            source.storage.write_run_manifest(
557                source.environment,
558                &source.instance_id,
559                status,
560                empty,
561            )
562        })
563    }
564}
565
566#[derive(Debug, Default, Deserialize, Serialize)]
567struct ParquetPromotionManifest {
568    identities: Vec<String>,
569}
570
571fn feather_replay_identity(
572    source_uri: &str,
573    source_path: &str,
574    content_hash: &str,
575    identifiers: Option<&[String]>,
576) -> String {
577    let mut identifiers = identifiers.map(<[String]>::to_vec);
578    if let Some(identifiers) = identifiers.as_mut() {
579        identifiers.sort();
580        identifiers.dedup();
581    }
582
583    let identity = serde_json::json!({
584        "source_uri": source_uri,
585        "source_path": source_path,
586        "content_hash": content_hash,
587        "identifiers": identifiers,
588    });
589    format!(
590        "nautilus-feather:{}",
591        blake3::hash(identity.to_string().as_bytes()).to_hex(),
592    )
593}
594
595#[cfg(test)]
596mod tests {
597    use std::{
598        fs,
599        path::{Path, PathBuf},
600        sync::atomic::AtomicU64,
601    };
602
603    use arrow::datatypes::SchemaRef;
604    use nautilus_common::enums::Environment;
605    use nautilus_core::{Params, UnixNanos};
606    use nautilus_model::{
607        data::{
608            Bar, BarType, BookOrder, Data, DataBatch, NautilusDataType, NautilusRecordType,
609            OrderBookDelta, QuoteTick,
610        },
611        enums::{BookAction, OrderSide, RecordFlag},
612        identifiers::InstrumentId,
613        types::{ERROR_PRICE, Price, Quantity},
614    };
615    use nautilus_serialization::arrow::{DecodeTypedFromRecordBatch, EncodeToRecordBatch};
616    use object_store::memory::InMemory;
617    use parquet::{
618        basic::Compression,
619        file::{
620            metadata::RowGroupMetaData,
621            reader::{FileReader, SerializedFileReader},
622        },
623    };
624    use rstest::rstest;
625    use tempfile::TempDir;
626
627    use super::*;
628    use crate::{
629        backend::parquet::io::read_parquet_from_object_store,
630        catalog::traits::{CatalogQuery, CatalogReader},
631        common::{paths::normalize_path_to_uri, storage::RUN_MANIFEST_FILENAME},
632        config::{CatalogCompression, DataCatalogConfig},
633        test_data::RustTestHashMapCustomData,
634        writer::{
635            feather::{FEATHER_PARTIAL_EXTENSION, FeatherWriter, RotationConfig},
636            promotion::{list_session_feather_files, tests::assert_scheduled_count},
637            traits::StreamingSink,
638        },
639    };
640
641    #[rstest]
642    fn parquet_default_close_promotes_and_honors_source_retention(
643        #[values(false, true)] delete_source: bool,
644    ) {
645        let directory = TempDir::new().unwrap();
646        let staging = directory.path().join("backtest").join("run-1");
647        let mut config =
648            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
649        config.delete_feather_after_promotion = delete_source;
650        let mut sink =
651            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
652                .unwrap();
653        let quote = sample_quote();
654        let mut second = quote;
655        second.instrument_id = InstrumentId::from("BTC/USD.SIM");
656        second.bid_price = Price::from("100.1234");
657        second.ask_price = Price::from("100.5678");
658        second.ts_init = UnixNanos::from(24);
659        let mut catalog = ParquetDataCatalog::from_uri(
660            directory.path().to_str().unwrap(),
661            None,
662            None,
663            None,
664            None,
665        )
666        .unwrap();
667        sink.write_data(Data::Quote(quote)).unwrap();
668        sink.write_data(Data::Quote(second)).unwrap();
669        sink.flush().unwrap();
670        let before = catalog
671            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
672            .unwrap();
673        sink.close().unwrap();
674        let after = catalog
675            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
676            .unwrap();
677        let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
678        let staged = block_on_nautilus_with(|| storage.list_files("", None))
679            .unwrap()
680            .into_iter()
681            .filter(|path| path.ends_with(".feather"))
682            .count();
683        assert!(before.is_empty());
684
685        let DataBatch::Quote(rows) = after else {
686            panic!("expected quotes")
687        };
688
689        assert_eq!(rows.as_ref(), &[quote, second]);
690        assert_eq!(staged, usize::from(!delete_source));
691    }
692
693    #[rstest]
694    fn parquet_writer_recovers_partial_files_at_startup() {
695        let directory = TempDir::new().unwrap();
696        let staging = directory.path().join("backtest").join("run-recover");
697        let quote = sample_quote();
698
699        // A writer that exited before sealing leaves its flushed stream as a partial file
700        let mut crashed = FeatherWriter::new(
701            staging.clone(),
702            WriterClock::Test(Arc::new(AtomicU64::new(0))),
703            RotationConfig::NoRotation,
704            None,
705            None,
706        );
707        crashed.write(quote).unwrap();
708        crashed.close().unwrap();
709        let sealed = staging.join("quotes").join("quotes_0.feather");
710        std::fs::rename(&sealed, sealed.with_extension(FEATHER_PARTIAL_EXTENSION)).unwrap();
711
712        let config =
713            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
714        let mut sink =
715            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
716                .unwrap();
717        sink.close().unwrap();
718
719        let mut catalog = ParquetDataCatalog::from_uri(
720            directory.path().to_str().unwrap(),
721            None,
722            None,
723            None,
724            None,
725        )
726        .unwrap();
727        let rows = catalog
728            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
729            .unwrap();
730
731        let DataBatch::Quote(rows) = rows else {
732            panic!("expected quotes")
733        };
734
735        assert_eq!(rows.as_ref(), &[quote]);
736    }
737
738    #[rstest]
739    fn parquet_close_promotes_per_identifier_file_with_non_ascii_identifier(
740        #[values(false, true)] delete_source: bool,
741    ) {
742        let directory = TempDir::new().unwrap();
743        let staging = directory.path().join("backtest").join("run-1");
744        let (quote, staged_file) = stage_per_identifier_quote(&staging);
745        let mut config =
746            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
747        config.delete_feather_after_promotion = delete_source;
748        let mut sink =
749            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
750                .unwrap();
751
752        sink.close().unwrap();
753
754        let mut catalog = ParquetDataCatalog::from_uri(
755            directory.path().to_str().unwrap(),
756            None,
757            None,
758            None,
759            None,
760        )
761        .unwrap();
762
763        let DataBatch::Quote(rows) = catalog
764            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
765            .unwrap()
766        else {
767            panic!("expected quotes")
768        };
769
770        assert_eq!(rows.as_ref(), &[quote]);
771        assert_eq!(staged_file.exists(), !delete_source);
772    }
773
774    #[rstest]
775    fn parquet_manual_conversion_reads_per_identifier_file_with_non_ascii_identifier() {
776        let directory = TempDir::new().unwrap();
777        let staging = directory.path().join("backtest").join("run-1");
778        let (quote, _) = stage_per_identifier_quote(&staging);
779        let mut catalog = ParquetDataCatalog::from_uri(
780            directory.path().to_str().unwrap(),
781            None,
782            None,
783            None,
784            None,
785        )
786        .unwrap();
787
788        catalog
789            .convert_stream_to_data(
790                "run-1",
791                &NautilusDataType::QuoteTick.into(),
792                Environment::Backtest,
793                None,
794                false,
795            )
796            .unwrap();
797
798        let DataBatch::Quote(rows) = catalog
799            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
800            .unwrap()
801        else {
802            panic!("expected quotes")
803        };
804
805        assert_eq!(rows.as_ref(), &[quote]);
806    }
807
808    #[rstest]
809    fn parquet_promotion_writes_non_ascii_identifier_to_object_store() {
810        let directory = TempDir::new().unwrap();
811        let staging = directory.path().join("backtest").join("run-1");
812        let (quote, staged_file) = stage_per_identifier_quote(&staging);
813
814        let source = FeatherSessionSource::new(
815            create_storage_backend_from_path(directory.path().to_str().unwrap(), None).unwrap(),
816            Environment::Backtest,
817            "run-1",
818        );
819        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
820        catalog.base_path = "catalog".to_string();
821        catalog.original_uri = "s3://test-bucket/catalog".to_string();
822        catalog.object_store = Arc::new(InMemory::new());
823
824        let mut backend = ParquetPromotionBackend {
825            catalog,
826            legacy_manifest_missing: Arc::default(),
827        };
828
829        let files =
830            list_session_feather_files(&source.storage, Environment::Backtest, "run-1").unwrap();
831
832        for file in &files {
833            backend.convert_file(&source, file, false, true).unwrap();
834            backend.delete_file(&source, file).unwrap();
835        }
836
837        let DataBatch::Quote(rows) = backend
838            .catalog
839            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
840            .unwrap()
841        else {
842            panic!("expected quotes")
843        };
844
845        assert_eq!(files.len(), 1);
846        assert_eq!(rows.as_ref(), &[quote]);
847        assert!(!staged_file.exists());
848    }
849
850    #[rstest]
851    #[case("backtest")]
852    #[case("sandbox")]
853    #[case("live")]
854    fn parquet_close_promotes_into_missing_local_catalog(#[case] environment: &str) {
855        let directory = TempDir::new().unwrap();
856        let staging = directory
857            .path()
858            .join("stream")
859            .join(environment)
860            .join("run-1");
861        let catalog_path = directory.path().join("charts").join("catalog");
862
863        let config = WriterConnectConfig::new(
864            staging.to_string_lossy(),
865            Some(CatalogConnectConfig::new(
866                catalog_path.to_string_lossy(),
867                None,
868            )),
869        );
870        let mut sink =
871            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
872                .unwrap();
873        let quote = sample_quote();
874
875        sink.write_data(Data::Quote(quote)).unwrap();
876        sink.close().unwrap();
877
878        let mut catalog =
879            ParquetDataCatalog::from_uri(catalog_path.to_str().unwrap(), None, None, None, None)
880                .unwrap();
881
882        let DataBatch::Quote(rows) = catalog
883            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
884            .unwrap()
885        else {
886            panic!("expected quotes")
887        };
888
889        let manifest: serde_json::Value =
890            serde_json::from_slice(&std::fs::read(staging.join(RUN_MANIFEST_FILENAME)).unwrap())
891                .unwrap();
892
893        assert_eq!(rows.as_ref(), &[quote]);
894        assert_eq!(manifest["kind"], environment);
895        assert_eq!(manifest["status"], "promoted");
896        assert!(!catalog_path.join(environment).exists());
897    }
898
899    #[rstest]
900    #[case::non_ascii("run-é")]
901    #[case::space("run 1")]
902    #[case::fragment("run#1")]
903    fn parquet_writer_rejects_run_id_that_paths_encode(#[case] run_id: &str) {
904        let directory = TempDir::new().unwrap();
905        let staging = directory.path().join("backtest").join(run_id);
906        let config =
907            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
908
909        let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
910            .unwrap_err();
911
912        let staging_uri = normalize_path_to_uri(&staging.to_string_lossy()).unwrap();
913        assert_eq!(
914            error.to_string(),
915            format!(
916                "Parquet writer URI '{staging_uri}' has a run ID that object-store paths \
917                 percent-encode; use a run ID without spaces, non-ASCII, or reserved characters",
918            )
919        );
920    }
921
922    #[rstest]
923    #[case::windows_separators(r"file:///C:\catalog\backtest\run-1", "file:///C:/catalog")]
924    #[case::windows_trailing_separator(r"file:///C:\catalog\backtest\run-1\", "file:///C:/catalog")]
925    #[case::trailing_separator("file:///tmp/catalog/backtest/run-1/", "file:///tmp/catalog")]
926    #[case::parent_components(
927        "file:///tmp/unused/../catalog/backtest/run-1",
928        "file:///tmp/catalog"
929    )]
930    fn parquet_writer_session_accepts_separator_forms(#[case] uri: &str, #[case] root_uri: &str) {
931        let session = StagedFeatherWriter::<ParquetPromotionBackend>::required_session(
932            uri,
933            "Parquet writer URI",
934        )
935        .unwrap();
936
937        assert_eq!(
938            session,
939            PromotionSession {
940                root_uri: root_uri.to_string(),
941                environment: Environment::Backtest,
942                instance_id: "run-1".to_string(),
943            }
944        );
945    }
946
947    #[rstest]
948    fn parquet_writer_requires_a_catalog() {
949        let directory = TempDir::new().unwrap();
950        let staging = directory.path().join("backtest").join("run-1");
951        let config = WriterConnectConfig::new(staging.to_string_lossy(), None);
952
953        let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
954            .unwrap_err();
955
956        assert_eq!(
957            error.to_string(),
958            "Parquet writer requires a promotion catalog"
959        );
960        assert!(!staging.exists());
961    }
962
963    // Opening an S3 store needs no network, and a run with no data promotes nothing, so the
964    // writer records the run as completed when dropped
965    #[cfg(feature = "cloud")]
966    #[rstest]
967    fn parquet_writer_stages_locally_for_a_remote_catalog() {
968        let directory = TempDir::new().unwrap();
969        let staging = directory.path().join("backtest").join("run-1");
970
971        let config = WriterConnectConfig::new(
972            staging.to_string_lossy(),
973            Some(CatalogConnectConfig::new("s3://bucket/catalog", None)),
974        );
975
976        let mut sink =
977            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
978                .unwrap();
979        sink.close().unwrap();
980        drop(sink);
981
982        let manifest: serde_json::Value =
983            serde_json::from_slice(&std::fs::read(staging.join(RUN_MANIFEST_FILENAME)).unwrap())
984                .unwrap();
985        assert_eq!(manifest["kind"], "backtest");
986        assert_eq!(manifest["instance_id"], "run-1");
987        assert_eq!(manifest["status"], "completed");
988        assert_eq!(manifest["empty"], true);
989    }
990
991    #[rstest]
992    fn parquet_promotion_writes_the_direct_write_schema() {
993        let directory = TempDir::new().unwrap();
994
995        let config = WriterConnectConfig::new(
996            directory
997                .path()
998                .join("backtest/run-schema")
999                .to_string_lossy(),
1000            Some(local_catalog(&directory)),
1001        );
1002        let mut sink =
1003            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1004                .unwrap();
1005        let quote = sample_quote();
1006        sink.write_data(Data::Quote(quote)).unwrap();
1007        sink.close().unwrap();
1008
1009        let promoted = ParquetDataCatalog::from_uri(
1010            directory.path().to_str().unwrap(),
1011            None,
1012            None,
1013            None,
1014            None,
1015        )
1016        .unwrap();
1017        let direct_directory = TempDir::new().unwrap();
1018        let direct = ParquetDataCatalog::from_uri(
1019            direct_directory.path().to_str().unwrap(),
1020            None,
1021            None,
1022            None,
1023            None,
1024        )
1025        .unwrap();
1026        direct.write_to_parquet(&[quote], None, None, None).unwrap();
1027
1028        assert_eq!(
1029            quote_file_schema(&promoted).fields(),
1030            quote_file_schema(&direct).fields(),
1031        );
1032    }
1033
1034    #[rstest]
1035    fn parquet_promotion_honors_catalog_settings() {
1036        let directory = TempDir::new().unwrap();
1037        let staging = directory.path().join("backtest").join("run-1");
1038        let catalog = DataCatalogConfig::builder()
1039            .path(directory.path().to_string_lossy().to_string())
1040            .batch_size(7)
1041            .compression(CatalogCompression::Uncompressed)
1042            .max_row_group_size(1)
1043            .build()
1044            .unwrap()
1045            .connect_config();
1046        let backend =
1047            ParquetPromotionBackend::new(&catalog, Arc::new(AtomicBool::new(false))).unwrap();
1048
1049        let config = WriterConnectConfig::new(staging.to_string_lossy(), Some(catalog));
1050        let mut sink =
1051            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1052                .unwrap();
1053        let quote = sample_quote();
1054        sink.write_data(Data::Quote(quote)).unwrap();
1055        sink.write_data(Data::Quote(QuoteTick {
1056            ts_event: UnixNanos::from(29),
1057            ts_init: UnixNanos::from(31),
1058            ..quote
1059        }))
1060        .unwrap();
1061
1062        sink.close().unwrap();
1063
1064        let files = ParquetDataCatalog::new(directory.path(), None, None, None, None)
1065            .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
1066            .unwrap();
1067        let reader = SerializedFileReader::new(
1068            std::fs::File::open(directory.path().join(&files[0])).unwrap(),
1069        )
1070        .unwrap();
1071        let metadata = reader.metadata();
1072
1073        assert_eq!(backend.catalog.batch_size, 7);
1074        assert_eq!(files.len(), 1);
1075        assert_eq!(metadata.num_row_groups(), 2);
1076        assert!(
1077            metadata
1078                .row_groups()
1079                .iter()
1080                .flat_map(RowGroupMetaData::columns)
1081                .all(|column| column.compression() == Compression::UNCOMPRESSED)
1082        );
1083    }
1084
1085    #[rstest]
1086    fn parquet_writer_rejects_unknown_catalog_param() {
1087        let directory = TempDir::new().unwrap();
1088        let staging = directory.path().join("backtest").join("run-1");
1089        let mut catalog = local_catalog(&directory);
1090        let params = Params::new([("compression".to_string(), serde_json::json!(0))].into());
1091        catalog.params = Some(params);
1092
1093        let config = WriterConnectConfig::new(staging.to_string_lossy(), Some(catalog));
1094        let error = parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1095            .unwrap_err();
1096
1097        assert_eq!(
1098            error.to_string(),
1099            "Unknown Parquet catalog param 'compression': this catalog takes no params"
1100        );
1101    }
1102
1103    fn quote_file_schema(catalog: &ParquetDataCatalog) -> SchemaRef {
1104        let files = catalog
1105            .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
1106            .unwrap();
1107        assert_eq!(files.len(), 1);
1108
1109        let path = ObjectPath::from(files[0].as_str());
1110
1111        let (_, schema) = block_on_nautilus_with(|| {
1112            read_parquet_from_object_store(catalog.object_store.clone(), &path)
1113        })
1114        .unwrap();
1115
1116        schema
1117    }
1118
1119    #[rstest]
1120    #[case(None)]
1121    #[case(Some(0))]
1122    #[case(Some(1))]
1123    fn parquet_interval_uses_test_clock_and_close_can_leave_staged_data(
1124        #[case] interval: Option<u64>,
1125    ) {
1126        let directory = TempDir::new().unwrap();
1127        let staging = directory.path().join("backtest").join("run-2");
1128        let mut config =
1129            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1130        config.promotion_interval_ms = interval;
1131        config.promote_on_close = false;
1132        let clock = Arc::new(AtomicU64::new(0));
1133        let mut sink =
1134            parquet_writer_factory(&config, WriterClock::Test(Arc::clone(&clock))).unwrap();
1135        let quote = sample_quote();
1136        sink.write_data(Data::Quote(quote)).unwrap();
1137        clock.store(1_000_000, Ordering::Relaxed);
1138        sink.flush().unwrap();
1139        sink.close().unwrap();
1140        let mut catalog = ParquetDataCatalog::from_uri(
1141            directory.path().to_str().unwrap(),
1142            None,
1143            None,
1144            None,
1145            None,
1146        )
1147        .unwrap();
1148        let before_manual = catalog
1149            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
1150            .unwrap();
1151
1152        if interval != Some(1) {
1153            catalog
1154                .convert_stream_to_data(
1155                    "run-2",
1156                    &NautilusDataType::QuoteTick.into(),
1157                    Environment::Backtest,
1158                    None,
1159                    false,
1160                )
1161                .unwrap();
1162        }
1163
1164        let after = catalog
1165            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
1166            .unwrap();
1167        assert_eq!(before_manual.len(), usize::from(interval == Some(1)));
1168
1169        let DataBatch::Quote(rows) = after else {
1170            panic!("expected quotes")
1171        };
1172
1173        assert_eq!(rows.as_ref(), &[quote]);
1174    }
1175
1176    #[rstest]
1177    fn due_promotion_without_staged_files_skips_catalog_and_waits_for_next_interval() {
1178        let directory = TempDir::new().unwrap();
1179        let staging = directory.path().join("backtest").join("run-idle");
1180        let mut config =
1181            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1182        config.promotion_interval_ms = Some(1);
1183        let clock = Arc::new(AtomicU64::new(0));
1184        let mut writer =
1185            ParquetWriter::new(&config, WriterClock::Test(Arc::clone(&clock))).unwrap();
1186
1187        // Any catalog open fails from here, so an empty promotion must not open one
1188        writer.catalog.params = Some(Params::new(
1189            [("compression".to_string(), serde_json::json!(0))].into(),
1190        ));
1191
1192        clock.store(1_000_000, Ordering::Relaxed);
1193        writer.maybe_promote().unwrap();
1194
1195        // Data staged during the idle interval promotes once the next interval is due
1196        writer.catalog.params = None;
1197        writer.core.write_data(Data::Quote(sample_quote())).unwrap();
1198        writer.maybe_promote().unwrap();
1199        assert!(writer.wait().unwrap().is_empty());
1200
1201        clock.store(2_000_000, Ordering::Relaxed);
1202        writer.maybe_promote().unwrap();
1203        assert_eq!(writer.wait().unwrap().len(), 1);
1204    }
1205
1206    #[rstest]
1207    fn failed_catalog_open_seals_once_per_interval_and_keeps_files_retryable() {
1208        let directory = TempDir::new().unwrap();
1209        let staging = directory.path().join("backtest").join("run-failing");
1210        let mut config =
1211            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1212        config.promotion_interval_ms = Some(1);
1213        let clock = Arc::new(AtomicU64::new(0));
1214        let mut writer =
1215            ParquetWriter::new(&config, WriterClock::Test(Arc::clone(&clock))).unwrap();
1216
1217        let quotes = (1..=5)
1218            .map(|timestamp| QuoteTick {
1219                ts_event: UnixNanos::from(timestamp),
1220                ts_init: UnixNanos::from(timestamp),
1221                ..sample_quote()
1222            })
1223            .collect::<Vec<_>>();
1224
1225        let sealed_count = |writer: &ParquetWriter| {
1226            list_session_feather_files(&writer.source.storage, Environment::Backtest, "run-failing")
1227                .unwrap()
1228                .len()
1229        };
1230
1231        // Any catalog open fails from here
1232        writer.catalog.params = Some(Params::new(
1233            [("compression".to_string(), serde_json::json!(0))].into(),
1234        ));
1235
1236        clock.store(1_000_000, Ordering::Relaxed);
1237        let first_error = writer.write_data(Data::Quote(quotes[0])).unwrap_err();
1238        writer.write_data(Data::Quote(quotes[1])).unwrap();
1239        writer.write_data(Data::Quote(quotes[2])).unwrap();
1240        let sealed_first_interval = sealed_count(&writer);
1241
1242        clock.store(2_000_000, Ordering::Relaxed);
1243        let second_error = writer.write_data(Data::Quote(quotes[3])).unwrap_err();
1244        writer.write_data(Data::Quote(quotes[4])).unwrap();
1245        let sealed_second_interval = sealed_count(&writer);
1246
1247        writer.catalog.params = None;
1248        clock.store(3_000_000, Ordering::Relaxed);
1249        writer.maybe_promote().unwrap();
1250        writer.wait().unwrap();
1251
1252        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1253
1254        let DataBatch::Quote(rows) = catalog
1255            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
1256            .unwrap()
1257        else {
1258            panic!("expected quotes")
1259        };
1260
1261        let open_error =
1262            "Unknown Parquet catalog param 'compression': this catalog takes no params";
1263        assert_eq!(first_error.to_string(), open_error);
1264        assert_eq!(second_error.to_string(), open_error);
1265        assert_eq!(sealed_first_interval, 1);
1266        assert_eq!(sealed_second_interval, 2);
1267        assert_eq!(rows.as_ref(), &quotes[..]);
1268    }
1269
1270    #[rstest]
1271    fn parquet_streaming_funding_round_trip() {
1272        use nautilus_model::data::FundingRateUpdate;
1273        let directory = TempDir::new().unwrap();
1274
1275        let config = WriterConnectConfig::new(
1276            directory
1277                .path()
1278                .join("backtest/run-funding")
1279                .to_string_lossy(),
1280            Some(local_catalog(&directory)),
1281        );
1282        let mut sink =
1283            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1284                .unwrap();
1285
1286        let funding = FundingRateUpdate::new(
1287            InstrumentId::from("AUD/USD.SIM"),
1288            "0.0012".parse().unwrap(),
1289            Some(480),
1290            Some(789.into()),
1291            123.into(),
1292            456.into(),
1293        );
1294        assert!(sink.write_any(&funding).unwrap());
1295        sink.close().unwrap();
1296        let mut catalog = ParquetDataCatalog::from_uri(
1297            directory.path().to_str().unwrap(),
1298            None,
1299            None,
1300            None,
1301            None,
1302        )
1303        .unwrap();
1304        let rows = catalog
1305            .query_batch(&CatalogQuery::new(NautilusDataType::FundingRateUpdate))
1306            .unwrap();
1307
1308        let DataBatch::FundingRate(rows) = rows else {
1309            panic!("expected funding rates")
1310        };
1311
1312        assert_eq!(rows.as_ref(), &[funding]);
1313    }
1314
1315    #[rstest]
1316    fn parquet_streaming_write_error_reaches_flush() {
1317        let directory = TempDir::new().unwrap();
1318
1319        let config = WriterConnectConfig::new(
1320            directory
1321                .path()
1322                .join("backtest/run-error")
1323                .to_string_lossy(),
1324            Some(local_catalog(&directory)),
1325        );
1326        let mut sink =
1327            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1328                .unwrap();
1329        let mut quote = sample_quote();
1330        quote.bid_price = ERROR_PRICE;
1331        let write_error = sink.write_any(&quote).unwrap_err().to_string();
1332        let flush_error = sink.flush().unwrap_err().to_string();
1333        assert_eq!(flush_error, write_error);
1334    }
1335
1336    #[rstest]
1337    fn parquet_streaming_instrument_promotes() {
1338        use nautilus_model::instruments::{InstrumentAny, stubs::audusd_sim};
1339        let directory = TempDir::new().unwrap();
1340
1341        let config = WriterConnectConfig::new(
1342            directory
1343                .path()
1344                .join("backtest/run-instrument")
1345                .to_string_lossy(),
1346            Some(local_catalog(&directory)),
1347        );
1348        let mut sink =
1349            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1350                .unwrap();
1351        let instrument = InstrumentAny::CurrencyPair(audusd_sim());
1352        assert!(sink.write_any(&instrument).unwrap());
1353        sink.close().unwrap();
1354        let mut catalog = ParquetDataCatalog::from_uri(
1355            directory.path().to_str().unwrap(),
1356            None,
1357            None,
1358            None,
1359            None,
1360        )
1361        .unwrap();
1362        let rows = catalog
1363            .query_batch(&CatalogQuery::new(NautilusDataType::Instrument))
1364            .unwrap();
1365
1366        let DataBatch::Instrument(rows) = rows else {
1367            panic!("expected instruments")
1368        };
1369
1370        assert_eq!(rows.as_ref(), &[instrument]);
1371    }
1372
1373    #[rstest]
1374    fn parquet_promotion_preserves_every_record_family(#[values(false, true)] automatic: bool) {
1375        use nautilus_core::UUID4;
1376        use nautilus_model::{
1377            enums::{
1378                LiquiditySide, OrderSide, OrderStatus, OrderType, PositionAdjustmentType,
1379                PositionSide, TimeInForce,
1380            },
1381            events::{
1382                OrderAccepted, OrderCancelDenied, OrderCancelRejected, OrderCanceled, OrderDenied,
1383                OrderEmulated, OrderExpired, OrderInitialized, OrderModifyDenied,
1384                OrderModifyRejected, OrderPendingCancel, OrderPendingUpdate, OrderRejected,
1385                OrderReleased, OrderSnapshot, OrderSubmitted, OrderTriggered, OrderUpdated,
1386                PositionAdjusted, PositionChanged, PositionClosed, PositionOpened,
1387                PositionSnapshot,
1388                account::stubs::cash_account_state,
1389                order::spec::{OrderFillVoidedSpec, OrderFilledSpec},
1390            },
1391            identifiers::{
1392                AccountId, ClientId, ClientOrderId, PositionId, TradeId, Venue, VenueOrderId,
1393            },
1394            instruments::{InstrumentAny, stubs::audusd_sim},
1395            orders::OrderTestBuilder,
1396            position::Position,
1397            reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
1398            stubs::TestDefault,
1399            types::Money,
1400        };
1401        use strum::IntoEnumIterator;
1402        let mut families = ahash::AHashSet::new();
1403        macro_rules! promote {
1404            ($record:expr) => {
1405                families.insert(assert_record_promotion($record, automatic));
1406            };
1407        }
1408
1409        promote!(cash_account_state());
1410        promote!(OrderInitialized::test_default());
1411        promote!(OrderDenied::test_default());
1412        promote!(OrderEmulated::test_default());
1413        promote!(OrderSubmitted::test_default());
1414        promote!(OrderAccepted::test_default());
1415        promote!(OrderRejected::test_default());
1416        promote!(OrderPendingCancel::test_default());
1417        promote!(OrderCanceled::test_default());
1418        let mut cancel_denied = OrderCancelDenied::test_default();
1419        cancel_denied.causation_id = Some(UUID4::new());
1420        promote!(cancel_denied);
1421        promote!(OrderCancelRejected::test_default());
1422        promote!(OrderExpired::test_default());
1423        promote!(OrderTriggered::test_default());
1424        promote!(OrderPendingUpdate::test_default());
1425        promote!(OrderReleased::test_default());
1426        let mut modify_denied = OrderModifyDenied::test_default();
1427        modify_denied.causation_id = Some(UUID4::new());
1428        promote!(modify_denied);
1429        promote!(OrderModifyRejected::test_default());
1430        promote!(OrderUpdated::test_default());
1431        promote!(OrderFillVoidedSpec::builder().is_reopened(true).build());
1432        let id = InstrumentId::from("AUD/USD.SIM");
1433        let fill = OrderFilledSpec::builder()
1434            .instrument_id(id)
1435            .position_id(PositionId::from("P-009"))
1436            .last_px(Price::from("1.23456"))
1437            .last_qty(Quantity::from("1234"))
1438            .ts_event(11.into())
1439            .ts_init(13.into())
1440            .build();
1441        promote!(fill.clone());
1442        let mut position = Position::new(&InstrumentAny::CurrencyPair(audusd_sim()), fill.clone());
1443        promote!(PositionOpened::create(
1444            &position,
1445            &fill,
1446            UUID4::default(),
1447            17.into()
1448        ));
1449        promote!(PositionChanged::create(
1450            &position,
1451            &fill,
1452            UUID4::default(),
1453            19.into()
1454        ));
1455        promote!(PositionAdjusted::new(
1456            fill.trader_id,
1457            fill.strategy_id,
1458            id,
1459            position.id,
1460            fill.account_id,
1461            PositionAdjustmentType::Funding,
1462            None,
1463            Some(Money::from("1.23 USD")),
1464            Some("funding".into()),
1465            UUID4::default(),
1466            23.into(),
1467            29.into()
1468        ));
1469        promote!(PositionSnapshot::from(
1470            &position,
1471            Some(Money::from("4.56 USD"))
1472        ));
1473        let closing = OrderFilledSpec::builder()
1474            .instrument_id(id)
1475            .position_id(position.id)
1476            .order_side(OrderSide::Sell)
1477            .last_px(Price::from("1.34567"))
1478            .last_qty(fill.last_qty)
1479            .trade_id(TradeId::from("T-011"))
1480            .ts_event(31.into())
1481            .ts_init(37.into())
1482            .build();
1483        position.apply(&closing);
1484        promote!(PositionClosed::create(
1485            &position,
1486            &closing,
1487            UUID4::default(),
1488            41.into()
1489        ));
1490        promote!(OrderSnapshot::from(
1491            OrderTestBuilder::new(OrderType::Limit)
1492                .instrument_id(id)
1493                .price(Price::from("1.45678"))
1494                .quantity(Quantity::from("5678"))
1495                .build()
1496        ));
1497
1498        let order = OrderStatusReport::new(
1499            AccountId::from("SIM-007"),
1500            id,
1501            Some(ClientOrderId::from("O-013")),
1502            VenueOrderId::from("V-017"),
1503            Some(OrderSide::Buy),
1504            OrderType::Limit,
1505            TimeInForce::Gtc,
1506            OrderStatus::PartiallyFilled,
1507            Quantity::from("123"),
1508            Quantity::from("45"),
1509            43.into(),
1510            47.into(),
1511            53.into(),
1512            None,
1513        );
1514
1515        let report_fill = FillReport::new(
1516            AccountId::from("SIM-007"),
1517            id,
1518            VenueOrderId::from("V-017"),
1519            TradeId::from("T-019"),
1520            OrderSide::Buy,
1521            Quantity::from("45"),
1522            Price::from("1.56789"),
1523            Money::from("2.34 USD"),
1524            LiquiditySide::Maker,
1525            Some(ClientOrderId::from("O-013")),
1526            Some(PositionId::from("P-023")),
1527            59.into(),
1528            61.into(),
1529            None,
1530        );
1531
1532        let report_position = PositionStatusReport::new(
1533            AccountId::from("SIM-007"),
1534            id,
1535            PositionSide::Long,
1536            Quantity::from("789"),
1537            67.into(),
1538            71.into(),
1539            None,
1540            Some(PositionId::from("P-023")),
1541            Some(rust_decimal::Decimal::new(16789, 4)),
1542        );
1543
1544        let mut mass = ExecutionMassStatus::new(
1545            ClientId::from("CLIENT-029"),
1546            AccountId::from("SIM-007"),
1547            Venue::from("SIM"),
1548            73.into(),
1549            None,
1550        );
1551        mass.add_order_reports(vec![order.clone()]);
1552        mass.add_fill_reports(vec![report_fill.clone()]);
1553        mass.add_position_reports(vec![report_position.clone()]);
1554        promote!(order);
1555        promote!(report_fill);
1556        promote!(report_position);
1557        promote!(mass);
1558
1559        let expected = NautilusRecordType::iter()
1560            .filter(|family| {
1561                #[cfg(feature = "defi")]
1562                if *family == NautilusRecordType::Defi {
1563                    return false;
1564                }
1565
1566                let _ = family;
1567                true
1568            })
1569            .collect::<ahash::AHashSet<_>>();
1570
1571        assert_eq!(families, expected);
1572    }
1573
1574    #[rstest]
1575    fn parquet_promotion_preserves_calculated_greeks_as_custom_data(
1576        #[values(false, true)] automatic: bool,
1577        #[values(false, true)] portfolio: bool,
1578    ) {
1579        use nautilus_model::data::{
1580            CustomData, DataType,
1581            greeks::{GreeksData, OptionGreekValues, PortfolioGreeks},
1582        };
1583
1584        let greeks = GreeksData::new(
1585            "ESZ26 C7000.XCME".parse().unwrap(),
1586            false,
1587            7000.5,
1588            20_261_218,
1589            68,
1590            0.186,
1591            50.0,
1592            -3.0,
1593            6950.25,
1594            0.0425,
1595            0.0125,
1596            0.175,
1597            12.5,
1598            98.75,
1599            OptionGreekValues {
1600                delta: 0.41,
1601                gamma: 0.0012,
1602                vega: 11.2,
1603                theta: -2.3,
1604                rho: 1.7,
1605            },
1606            0.62,
1607            11.into(),
1608            13.into(),
1609        );
1610
1611        let mut second = greeks.clone();
1612        second.ts_init = 113.into();
1613        second.instrument_id = "ESZ26 P6900.XCME".parse().unwrap();
1614
1615        let greeks = vec![greeks, second]
1616            .into_iter()
1617            .map(|value| {
1618                let data_type =
1619                    DataType::new("GreeksData", None, Some(value.instrument_id.to_string()));
1620                CustomData::new(Arc::new(value), data_type)
1621            })
1622            .collect::<Vec<_>>();
1623
1624        let portfolio_value = PortfolioGreeks {
1625            pnl: -125.5,
1626            price: 4310.25,
1627            greeks: OptionGreekValues {
1628                delta: 0.83,
1629                gamma: 0.004,
1630                vega: 45.5,
1631                theta: -12.25,
1632                rho: 3.75,
1633            },
1634            ts_event: 19.into(),
1635            ts_init: 17.into(),
1636        };
1637
1638        let mut second = portfolio_value.clone();
1639        second.ts_init = 117.into();
1640        let portfolios = vec![portfolio_value, second]
1641            .into_iter()
1642            .map(|value| CustomData::from_arc(Arc::new(value)))
1643            .collect::<Vec<_>>();
1644
1645        let expected = if portfolio { portfolios } else { greeks };
1646
1647        let directory = TempDir::new().unwrap();
1648        let staging = directory.path().join("backtest/run-greeks-custom");
1649        let mut config =
1650            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1651        config.promote_on_close = automatic;
1652        let mut sink =
1653            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1654                .unwrap();
1655
1656        for custom in &expected {
1657            assert!(sink.write_any(custom.data.as_any()).unwrap());
1658        }
1659
1660        sink.close().unwrap();
1661
1662        let data_type = NautilusDataType::Custom {
1663            type_name: expected[0].data_type.type_name().to_string(),
1664        };
1665
1666        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1667        if !automatic {
1668            catalog
1669                .convert_stream_to_data(
1670                    "run-greeks-custom",
1671                    &data_type.clone().into(),
1672                    Environment::Backtest,
1673                    None,
1674                    false,
1675                )
1676                .unwrap();
1677        }
1678
1679        let DataBatch::Custom(actual) = catalog.query_batch(&CatalogQuery::new(data_type)).unwrap()
1680        else {
1681            panic!("expected custom data")
1682        };
1683
1684        assert_eq!(actual.as_ref(), expected.as_slice());
1685    }
1686
1687    fn assert_record_promotion<T>(record: T, automatic: bool) -> NautilusRecordType
1688    where
1689        T: std::any::Any
1690            + serde::Serialize
1691            + serde::de::DeserializeOwned
1692            + EncodeToRecordBatch
1693            + DecodeTypedFromRecordBatch,
1694    {
1695        let directory = TempDir::new().unwrap();
1696        let staging = directory.path().join("backtest/run-record");
1697        let mut config =
1698            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1699        config.promote_on_close = automatic;
1700        let mut sink =
1701            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1702                .unwrap();
1703        let mut second = serde_json::to_value(&record).unwrap();
1704        second["ts_init"] = serde_json::json!(second["ts_init"].as_u64().unwrap() + 101);
1705        let second: T = serde_json::from_value(second).unwrap();
1706        assert!(sink.write_any(&record).unwrap());
1707        assert!(sink.write_any(&second).unwrap());
1708        sink.close().unwrap();
1709        let record_type = T::get_schema(None).metadata()["type_name"]
1710            .parse::<NautilusRecordType>()
1711            .unwrap();
1712        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1713        if !automatic {
1714            catalog
1715                .convert_stream_to_data(
1716                    "run-record",
1717                    &record_type.into(),
1718                    Environment::Backtest,
1719                    None,
1720                    false,
1721                )
1722                .unwrap();
1723        }
1724
1725        let batches = catalog
1726            .query_record_batches(&record_type.into(), None, None, None, None, true)
1727            .unwrap();
1728
1729        let actual = batches
1730            .into_iter()
1731            .flat_map(|batch| {
1732                let schema = batch.schema();
1733                T::decode_typed_batch(schema.metadata(), batch).unwrap()
1734            })
1735            .map(|record| serde_json::to_value(record).unwrap())
1736            .collect::<Vec<_>>();
1737
1738        assert_eq!(
1739            actual,
1740            vec![
1741                serde_json::to_value(record).unwrap(),
1742                serde_json::to_value(second).unwrap()
1743            ]
1744        );
1745        record_type
1746    }
1747
1748    #[rstest]
1749    fn parquet_streaming_voided_fill_promotes() {
1750        use nautilus_model::events::{OrderFillVoided, order::spec::OrderFillVoidedSpec};
1751        use nautilus_serialization::arrow::DecodeTypedFromRecordBatch;
1752        let directory = TempDir::new().unwrap();
1753
1754        let config = WriterConnectConfig::new(
1755            directory.path().join("backtest/run-void").to_string_lossy(),
1756            Some(local_catalog(&directory)),
1757        );
1758        let mut sink =
1759            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1760                .unwrap();
1761        let event = OrderFillVoidedSpec::builder().is_reopened(true).build();
1762        assert!(sink.write_any(&event).unwrap());
1763        sink.close().unwrap();
1764        let mut catalog = ParquetDataCatalog::from_uri(
1765            directory.path().to_str().unwrap(),
1766            None,
1767            None,
1768            None,
1769            None,
1770        )
1771        .unwrap();
1772        let batches = catalog
1773            .query_record_batches(
1774                &NautilusRecordType::OrderFillVoided.into(),
1775                None,
1776                None,
1777                None,
1778                None,
1779                true,
1780            )
1781            .unwrap();
1782        let mut rows = Vec::new();
1783        for batch in batches {
1784            rows.extend(
1785                OrderFillVoided::decode_typed_batch(batch.schema().metadata(), batch).unwrap(),
1786            );
1787        }
1788
1789        assert_eq!(rows, vec![event]);
1790    }
1791
1792    #[rstest]
1793    fn parquet_promotion_preserves_clear_source_order(
1794        #[values(false, true)] automatic: bool,
1795        #[values(false, true)] batched: bool,
1796        #[values(false, true)] undefined_price: bool,
1797    ) {
1798        use nautilus_model::{
1799            data::{BookOrder, OrderBookDelta},
1800            enums::{BookAction, OrderSide, RecordFlag},
1801            types::PRICE_UNDEF,
1802        };
1803        let directory = TempDir::new().unwrap();
1804        let staging = directory.path().join("backtest/run-clear");
1805        let mut config =
1806            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
1807        config.promote_on_close = automatic;
1808        let mut sink =
1809            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
1810                .unwrap();
1811        let id = InstrumentId::from("AUD/USD.SIM");
1812        let ts = UnixNanos::from(1_700_000_000_000_000_123);
1813        let mut clear = OrderBookDelta::clear(id, 7, ts, ts);
1814        if undefined_price {
1815            clear.order.price = Price::from_raw(PRICE_UNDEF, 0);
1816        }
1817
1818        let add = OrderBookDelta::new(
1819            id,
1820            BookAction::Add,
1821            BookOrder::new(
1822                OrderSide::Buy,
1823                Price::from("1.23456"),
1824                Quantity::from("7.890"),
1825                19,
1826            ),
1827            RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8,
1828            7,
1829            ts,
1830            ts,
1831        );
1832        let mut second_add = add;
1833        second_add.order.order_id = 29;
1834        second_add.order.price = Price::from("1.34567");
1835        second_add.order.size = Quantity::from("8.901");
1836        let rows = vec![clear, add, clear, second_add];
1837        if batched {
1838            sink.write_batch(rows.iter().copied().map(Data::BookDelta).collect())
1839                .unwrap();
1840        } else {
1841            for row in &rows {
1842                sink.write_data(Data::BookDelta(*row)).unwrap();
1843            }
1844        }
1845
1846        sink.close().unwrap();
1847        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1848        let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
1849        let staged = block_on_nautilus_with(|| storage.list_files("", Some(".feather"))).unwrap();
1850        let mut originals = Vec::new();
1851
1852        for path in &staged {
1853            let bytes = block_on_nautilus_with(|| async {
1854                storage
1855                    .object_store
1856                    .get(&object_store::path::Path::from(path.as_str()))
1857                    .await
1858                    .unwrap()
1859                    .bytes()
1860                    .await
1861                    .unwrap()
1862            });
1863
1864            originals.push(bytes);
1865        }
1866
1867        if !automatic {
1868            catalog
1869                .convert_stream_to_data(
1870                    "run-clear",
1871                    &NautilusDataType::OrderBookDelta.into(),
1872                    Environment::Backtest,
1873                    None,
1874                    false,
1875                )
1876                .unwrap();
1877        }
1878
1879        let actual = catalog
1880            .query::<OrderBookDelta>(None, None, None, None, None, true)
1881            .unwrap();
1882        assert_eq!(actual.len(), rows.len());
1883
1884        for (actual, expected) in actual.iter().zip(&rows) {
1885            assert_eq!(actual.instrument_id, expected.instrument_id);
1886            assert_eq!(actual.action, expected.action);
1887            assert_eq!(actual.order.side, expected.order.side);
1888            assert_eq!(actual.order.price.raw(), expected.order.price.raw());
1889
1890            let price_precision = if expected.order.price.is_undefined() {
1891                0
1892            } else {
1893                5
1894            };
1895
1896            assert_eq!(actual.order.price.precision, price_precision);
1897            assert_eq!(
1898                actual.order.size.as_decimal(),
1899                expected.order.size.as_decimal()
1900            );
1901            assert_eq!(actual.order.size.precision, 3);
1902            assert_eq!(actual.order.order_id, expected.order.order_id);
1903            assert_eq!(actual.flags, expected.flags);
1904            assert_eq!(actual.sequence, expected.sequence);
1905            assert_eq!(actual.ts_event, expected.ts_event);
1906            assert_eq!(actual.ts_init, expected.ts_init);
1907        }
1908
1909        for (path, original) in staged.iter().zip(originals) {
1910            let bytes = block_on_nautilus_with(|| async {
1911                storage
1912                    .object_store
1913                    .get(&object_store::path::Path::from(path.as_str()))
1914                    .await
1915                    .unwrap()
1916                    .bytes()
1917                    .await
1918                    .unwrap()
1919            });
1920
1921            assert_eq!(bytes, original);
1922        }
1923
1924        assert_eq!(staged.len(), 1);
1925    }
1926
1927    #[rstest]
1928    #[case::manual(false, false)]
1929    #[case::automatic_retained(true, false)]
1930    #[case::automatic_delete(true, true)]
1931    fn parquet_failed_clear_promotion_preserves_source(
1932        #[case] automatic: bool,
1933        #[case] delete_source: bool,
1934        #[values(false, true)] batched: bool,
1935    ) {
1936        use nautilus_model::{
1937            data::{BookOrder, OrderBookDelta},
1938            enums::{BookAction, OrderSide, RecordFlag},
1939        };
1940        let directory = TempDir::new().unwrap();
1941        let staging = directory.path().join("backtest/run-invalid-clear");
1942        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
1943
1944        let mut writer = FeatherWriter::new(
1945            staging.clone(),
1946            clock.clone(),
1947            RotationConfig::NoRotation,
1948            None,
1949            None,
1950        );
1951        let id = InstrumentId::from("AUD/USD.SIM");
1952        let ts = UnixNanos::from(23);
1953        let mut clear = OrderBookDelta::clear(id, 7, ts, ts);
1954        clear.order.price = Price::from("1");
1955
1956        let add = OrderBookDelta::new(
1957            id,
1958            BookAction::Add,
1959            BookOrder::new(
1960                OrderSide::Buy,
1961                Price::from("1.23456"),
1962                Quantity::from("7.890"),
1963                19,
1964            ),
1965            RecordFlag::F_LAST as u8,
1966            7,
1967            ts,
1968            ts,
1969        );
1970
1971        if batched {
1972            writer.write_batch(vec![clear, add]).unwrap();
1973        } else {
1974            writer.write(clear).unwrap();
1975            writer.write(add).unwrap();
1976        }
1977
1978        writer.close().unwrap();
1979        let source = staging.join("order_book_deltas/order_book_deltas_0.feather");
1980        let original = fs::read(&source).unwrap();
1981        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1982
1983        let failure = if automatic {
1984            let mut config = WriterConnectConfig::new(
1985                staging.to_string_lossy(),
1986                Some(local_catalog(&directory)),
1987            );
1988            config.delete_feather_after_promotion = delete_source;
1989            let mut sink = parquet_writer_factory(&config, clock).unwrap();
1990            sink.close().unwrap_err()
1991        } else {
1992            catalog
1993                .convert_stream_to_data(
1994                    "run-invalid-clear",
1995                    &NautilusDataType::OrderBookDelta.into(),
1996                    Environment::Backtest,
1997                    None,
1998                    false,
1999                )
2000                .unwrap_err()
2001        };
2002
2003        assert!(
2004            format!("{failure:#}").contains(
2005                "cannot relabel precision metadata for a promotion group that contains decimal values"
2006            ),
2007            "{failure:#}"
2008        );
2009        assert_eq!(fs::read(source).unwrap(), original);
2010        assert_eq!(
2011            catalog
2012                .query_files(&NautilusDataType::OrderBookDelta.into(), None, None, None)
2013                .unwrap(),
2014            Vec::<String>::new()
2015        );
2016    }
2017
2018    #[rstest]
2019    #[case(true)]
2020    #[case(false)]
2021    fn parquet_promotion_unifies_empty_and_populated_depth(#[case] automatic: bool) {
2022        use nautilus_model::{
2023            data::{BookOrder, OrderBookDepth},
2024            enums::OrderSide,
2025        };
2026        let directory = TempDir::new().unwrap();
2027        let staging = directory.path().join("backtest/run-depth-ties");
2028        let mut config =
2029            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2030        config.delete_feather_after_promotion = true;
2031        config.promote_on_close = automatic;
2032        let mut sink =
2033            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2034                .unwrap();
2035        let id = InstrumentId::from("AUD/USD.SIM");
2036        let empty =
2037            OrderBookDepth::new(id, vec![], vec![], vec![], vec![], 1, 2, 3.into(), 4.into());
2038        let order = BookOrder::new(
2039            OrderSide::Buy,
2040            Price::from("1.23"),
2041            Quantity::from("4.5"),
2042            6,
2043        );
2044        let populated = OrderBookDepth::new(
2045            id,
2046            vec![order],
2047            vec![],
2048            vec![7],
2049            vec![],
2050            8,
2051            9,
2052            3.into(),
2053            4.into(),
2054        );
2055        sink.write_any(&empty).unwrap();
2056        sink.write_any(&populated).unwrap();
2057        sink.close().unwrap();
2058
2059        let mut catalog = ParquetDataCatalog::from_uri(
2060            directory.path().to_str().unwrap(),
2061            None,
2062            None,
2063            None,
2064            None,
2065        )
2066        .unwrap();
2067
2068        if !automatic {
2069            catalog
2070                .convert_stream_to_data(
2071                    "run-depth-ties",
2072                    &NautilusDataType::OrderBookDepth.into(),
2073                    Environment::Backtest,
2074                    None,
2075                    false,
2076                )
2077                .unwrap();
2078        }
2079
2080        let batch = catalog
2081            .query_batch(&CatalogQuery::new(NautilusDataType::OrderBookDepth))
2082            .unwrap();
2083        let DataBatch::BookDepth(rows) = batch else {
2084            panic!("expected order book depth");
2085        };
2086        let mut rows = rows.as_ref().to_vec();
2087        rows.sort_by_key(|depth| depth.sequence);
2088        assert_eq!(rows, vec![empty, populated]);
2089
2090        let storage = create_storage_backend_from_path(staging.to_str().unwrap(), None).unwrap();
2091        let staged = block_on_nautilus_with(|| storage.list_files("", Some(".feather"))).unwrap();
2092        assert_eq!(staged.len(), usize::from(!automatic));
2093    }
2094
2095    #[rstest]
2096    fn parquet_promotion_preserves_clear_snapshots(
2097        #[values(false, true)] automatic: bool,
2098        #[values(false, true)] batch_write: bool,
2099    ) {
2100        let directory = TempDir::new().unwrap();
2101        let staging = directory.path().join("backtest").join("run-clear");
2102        let mut config =
2103            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2104        config.promote_on_close = automatic;
2105        let mut sink =
2106            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2107                .unwrap();
2108        let id = InstrumentId::from("AUD/USD.SIM");
2109        let mut deltas = Vec::new();
2110        for (sequence, timestamp, price, size) in [
2111            (41, 100, "1.23", "4.500"),
2112            (42, 200, "1.24", "6.700"),
2113            (43, 300, "1.2345", "8.90000"),
2114        ] {
2115            deltas.push(OrderBookDelta::clear(
2116                id,
2117                sequence,
2118                (timestamp - 1).into(),
2119                timestamp.into(),
2120            ));
2121
2122            for (side, order_id, flags) in [
2123                (OrderSide::Buy, 71, RecordFlag::F_SNAPSHOT as u8),
2124                (
2125                    OrderSide::Sell,
2126                    72,
2127                    RecordFlag::F_SNAPSHOT as u8 | RecordFlag::F_LAST as u8,
2128                ),
2129            ] {
2130                deltas.push(OrderBookDelta::new(
2131                    id,
2132                    BookAction::Add,
2133                    BookOrder::new(side, Price::from(price), Quantity::from(size), order_id),
2134                    flags,
2135                    sequence,
2136                    (timestamp - 1).into(),
2137                    timestamp.into(),
2138                ));
2139            }
2140        }
2141
2142        deltas.push(OrderBookDelta::clear(id, 44, 399.into(), 400.into()));
2143        if batch_write {
2144            sink.write_batch(deltas.iter().copied().map(Data::from).collect())
2145                .unwrap();
2146        } else {
2147            for delta in &deltas {
2148                sink.write_data(Data::BookDelta(*delta)).unwrap();
2149            }
2150        }
2151
2152        sink.close().unwrap();
2153        let mut catalog = ParquetDataCatalog::from_uri(
2154            directory.path().to_str().unwrap(),
2155            None,
2156            None,
2157            None,
2158            None,
2159        )
2160        .unwrap();
2161
2162        if !automatic {
2163            catalog
2164                .convert_stream_to_data(
2165                    "run-clear",
2166                    &NautilusDataType::OrderBookDelta.into(),
2167                    Environment::Backtest,
2168                    None,
2169                    false,
2170                )
2171                .unwrap();
2172        }
2173
2174        let batch = catalog
2175            .query_batch(&CatalogQuery::new(NautilusDataType::OrderBookDelta))
2176            .unwrap();
2177
2178        let DataBatch::BookDelta(rows) = batch else {
2179            panic!("expected order book deltas");
2180        };
2181
2182        assert_eq!(rows.len(), deltas.len());
2183
2184        for (actual, expected) in rows.as_ref().iter().zip(&deltas) {
2185            let (price_precision, size_precision) = if expected.sequence < 43 {
2186                (2, 3)
2187            } else {
2188                (4, 5)
2189            };
2190
2191            assert_eq!(actual.instrument_id, expected.instrument_id);
2192            assert_eq!(actual.action, expected.action);
2193            assert_eq!(actual.order.side, expected.order.side);
2194            assert_eq!(actual.order.price.raw(), expected.order.price.raw());
2195            assert_eq!(actual.order.price.precision, price_precision);
2196            assert_eq!(actual.order.size.raw(), expected.order.size.raw());
2197            assert_eq!(actual.order.size.precision, size_precision);
2198            assert_eq!(actual.order.order_id, expected.order.order_id);
2199            assert_eq!(actual.flags, expected.flags);
2200            assert_eq!(actual.sequence, expected.sequence);
2201            assert_eq!(actual.ts_event, expected.ts_event);
2202            assert_eq!(actual.ts_init, expected.ts_init);
2203        }
2204    }
2205
2206    #[rstest]
2207    #[case::real_add(BookAction::Add, Some(OrderSide::Buy), "1", "2", 0)]
2208    #[case::clear_side(BookAction::Clear, Some(OrderSide::Buy), "0", "0", 0)]
2209    #[case::clear_price(BookAction::Clear, None, "1", "0", 0)]
2210    #[case::clear_size(BookAction::Clear, None, "0", "1", 0)]
2211    #[case::clear_order_id(BookAction::Clear, None, "0", "0", 1)]
2212    fn parquet_promotion_rejects_noncanonical_zero_precision_orders(
2213        #[case] action: BookAction,
2214        #[case] side: Option<OrderSide>,
2215        #[case] price: &str,
2216        #[case] size: &str,
2217        #[case] order_id: u64,
2218        #[values(false, true)] automatic: bool,
2219    ) {
2220        let directory = TempDir::new().unwrap();
2221        let staging = directory.path().join("backtest").join("run-clear-invalid");
2222        let mut config =
2223            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2224        config.promote_on_close = automatic;
2225        let mut sink =
2226            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2227                .unwrap();
2228        let id = InstrumentId::from("AUD/USD.SIM");
2229        let deltas = vec![
2230            OrderBookDelta::clear(id, 41, 99.into(), 100.into()),
2231            OrderBookDelta::new(
2232                id,
2233                action,
2234                BookOrder {
2235                    side,
2236                    price: Price::from(price),
2237                    size: Quantity::from(size),
2238                    order_id,
2239                },
2240                0,
2241                41,
2242                99.into(),
2243                100.into(),
2244            ),
2245            OrderBookDelta::new(
2246                id,
2247                BookAction::Add,
2248                BookOrder::new(
2249                    OrderSide::Sell,
2250                    Price::from("1.23"),
2251                    Quantity::from("4.500"),
2252                    72,
2253                ),
2254                RecordFlag::F_LAST as u8,
2255                41,
2256                99.into(),
2257                100.into(),
2258            ),
2259        ];
2260        sink.write_batch(deltas.into_iter().map(Data::from).collect())
2261            .unwrap();
2262        let mut catalog = ParquetDataCatalog::from_uri(
2263            directory.path().to_str().unwrap(),
2264            None,
2265            None,
2266            None,
2267            None,
2268        )
2269        .unwrap();
2270
2271        let error = if automatic {
2272            sink.close().unwrap_err()
2273        } else {
2274            sink.close().unwrap();
2275            catalog
2276                .convert_stream_to_data(
2277                    "run-clear-invalid",
2278                    &NautilusDataType::OrderBookDelta.into(),
2279                    Environment::Backtest,
2280                    None,
2281                    false,
2282                )
2283                .unwrap_err()
2284        };
2285
2286        assert!(error.to_string().contains(
2287            "cannot relabel precision metadata for a promotion group that contains decimal values"
2288        ));
2289        assert!(
2290            catalog
2291                .query_files(&NautilusDataType::OrderBookDelta.into(), None, None, None)
2292                .unwrap()
2293                .is_empty()
2294        );
2295    }
2296
2297    #[rstest]
2298    fn parquet_manual_conversion_promotes_custom_data() {
2299        use nautilus_model::data::{CustomData, DataType};
2300        use nautilus_serialization::ensure_custom_data_registered;
2301
2302        ensure_custom_data_registered::<RustTestHashMapCustomData>();
2303
2304        let directory = TempDir::new().unwrap();
2305        let staging = directory.path().join("backtest").join("run-custom");
2306        let mut config =
2307            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2308        config.promote_on_close = false;
2309        let mut sink =
2310            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2311                .unwrap();
2312
2313        let data_type = DataType::new("RustTestHashMapCustomData", None, None);
2314        let records = [
2315            sample_custom_data("first", "AUD/USD.SIM", "1.23456", 11),
2316            sample_custom_data("second", "BTCUSDT.BINANCE", "65432.10", 21),
2317        ];
2318
2319        for record in &records {
2320            sink.write_data(Data::Custom(CustomData::new(
2321                Arc::new(record.clone()),
2322                data_type.clone(),
2323            )))
2324            .unwrap();
2325        }
2326
2327        sink.close().unwrap();
2328
2329        let mut catalog = ParquetDataCatalog::from_uri(
2330            directory.path().to_str().unwrap(),
2331            None,
2332            None,
2333            None,
2334            None,
2335        )
2336        .unwrap();
2337
2338        for _ in 0..2 {
2339            catalog
2340                .convert_stream_to_data(
2341                    "run-custom",
2342                    &NautilusDataType::Custom {
2343                        type_name: "RustTestHashMapCustomData".to_string(),
2344                    }
2345                    .into(),
2346                    Environment::Backtest,
2347                    None,
2348                    false,
2349                )
2350                .unwrap();
2351        }
2352
2353        let expected: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2354            .iter()
2355            .map(|record| (None, record.clone()))
2356            .collect();
2357        assert_eq!(query_custom_records(&mut catalog, None), expected);
2358    }
2359
2360    #[rstest]
2361    fn parquet_manual_conversion_filters_custom_data_identifiers() {
2362        use nautilus_model::data::{CustomData, DataType};
2363        use nautilus_serialization::ensure_custom_data_registered;
2364
2365        ensure_custom_data_registered::<RustTestHashMapCustomData>();
2366
2367        let directory = TempDir::new().unwrap();
2368        let staging = directory.path().join("backtest").join("run-custom-ids");
2369        let mut config =
2370            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2371        config.promote_on_close = false;
2372        let mut sink =
2373            parquet_writer_factory(&config, WriterClock::Test(Arc::new(AtomicU64::new(0))))
2374                .unwrap();
2375
2376        let audusd = "AUD/USD.SIM";
2377        let btcusdt = "BTCUSDT.BINANCE";
2378        let records = [
2379            (audusd, sample_custom_data("first", audusd, "1.23456", 11)),
2380            (audusd, sample_custom_data("second", audusd, "1.23457", 21)),
2381            (
2382                btcusdt,
2383                sample_custom_data("third", btcusdt, "65432.10", 31),
2384            ),
2385        ];
2386
2387        for (identifier, record) in &records {
2388            let data_type = DataType::new(
2389                "RustTestHashMapCustomData",
2390                None,
2391                Some((*identifier).to_string()),
2392            );
2393            sink.write_data(Data::Custom(CustomData::new(
2394                Arc::new(record.clone()),
2395                data_type,
2396            )))
2397            .unwrap();
2398        }
2399
2400        sink.close().unwrap();
2401
2402        let mut catalog = ParquetDataCatalog::from_uri(
2403            directory.path().to_str().unwrap(),
2404            None,
2405            None,
2406            None,
2407            None,
2408        )
2409        .unwrap();
2410        catalog
2411            .convert_stream_to_data(
2412                "run-custom-ids",
2413                &NautilusDataType::Custom {
2414                    type_name: "RustTestHashMapCustomData".to_string(),
2415                }
2416                .into(),
2417                Environment::Backtest,
2418                Some(&[audusd.to_string()]),
2419                false,
2420            )
2421            .unwrap();
2422
2423        let expected_audusd: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2424            .iter()
2425            .filter(|(identifier, _)| *identifier == audusd)
2426            .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2427            .collect();
2428        assert_eq!(query_custom_records(&mut catalog, None), expected_audusd);
2429
2430        // Catalog identifier directories are urisafe, so a scoped query pins that spelling and
2431        // proves the converted records landed under their own identifier rather than together.
2432        assert_eq!(
2433            query_custom_records(&mut catalog, Some(&["AUDUSD.SIM".to_string()])),
2434            expected_audusd,
2435        );
2436
2437        catalog
2438            .convert_stream_to_data(
2439                "run-custom-ids",
2440                &NautilusDataType::Custom {
2441                    type_name: "RustTestHashMapCustomData".to_string(),
2442                }
2443                .into(),
2444                Environment::Backtest,
2445                None,
2446                false,
2447            )
2448            .unwrap();
2449
2450        let expected_all: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2451            .iter()
2452            .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2453            .collect();
2454        assert_eq!(query_custom_records(&mut catalog, None), expected_all);
2455        let expected_btcusdt: Vec<(Option<String>, RustTestHashMapCustomData)> = records
2456            .iter()
2457            .filter(|(identifier, _)| *identifier == btcusdt)
2458            .map(|(identifier, record)| (Some((*identifier).to_string()), record.clone()))
2459            .collect();
2460        assert_eq!(
2461            query_custom_records(&mut catalog, Some(&[btcusdt.to_string()])),
2462            expected_btcusdt,
2463        );
2464    }
2465
2466    fn sample_custom_data(
2467        name: &str,
2468        instrument_id: &str,
2469        price: &str,
2470        ts: u64,
2471    ) -> RustTestHashMapCustomData {
2472        RustTestHashMapCustomData {
2473            name: name.to_string(),
2474            prices: [(instrument_id.to_string(), Price::from(price))]
2475                .into_iter()
2476                .collect(),
2477            ts_event: UnixNanos::from(ts - 1),
2478            ts_init: UnixNanos::from(ts),
2479        }
2480    }
2481
2482    fn query_custom_records(
2483        catalog: &mut ParquetDataCatalog,
2484        identifiers: Option<&[String]>,
2485    ) -> Vec<(Option<String>, RustTestHashMapCustomData)> {
2486        let loaded = catalog
2487            .query_custom_data_dynamic(
2488                "RustTestHashMapCustomData",
2489                identifiers,
2490                None,
2491                None,
2492                None,
2493                None,
2494                true,
2495            )
2496            .unwrap();
2497        let mut decoded = Vec::new();
2498
2499        for data in &loaded {
2500            let Data::Custom(custom) = data else {
2501                panic!("expected custom data, found {data:?}")
2502            };
2503
2504            assert_eq!(custom.data_type.type_name(), "RustTestHashMapCustomData");
2505            decoded.push((
2506                custom.data_type.identifier().map(ToString::to_string),
2507                custom
2508                    .data
2509                    .as_any()
2510                    .downcast_ref::<RustTestHashMapCustomData>()
2511                    .expect("expected RustTestHashMapCustomData")
2512                    .clone(),
2513            ));
2514        }
2515
2516        decoded.sort_by_key(|(_, record)| record.ts_init);
2517        decoded
2518    }
2519
2520    fn local_catalog(directory: &TempDir) -> CatalogConnectConfig {
2521        CatalogConnectConfig::new(directory.path().to_string_lossy(), None)
2522    }
2523
2524    // Earlier per-instrument writers staged files in identifier directories
2525    fn stage_per_identifier_quote(staging: &Path) -> (QuoteTick, PathBuf) {
2526        let mut quote = sample_quote();
2527        quote.instrument_id = InstrumentId::from("CAFÉ.SIM");
2528
2529        let mut writer = FeatherWriter::new(
2530            staging.to_path_buf(),
2531            WriterClock::Test(Arc::new(AtomicU64::new(0))),
2532            RotationConfig::NoRotation,
2533            None,
2534            None,
2535        );
2536        writer.write(quote).unwrap();
2537        writer.close().unwrap();
2538
2539        let identifier_directory = staging.join("quotes").join("CAFÉ.SIM");
2540        let staged_file = identifier_directory.join("quotes_0.feather");
2541        fs::create_dir(&identifier_directory).unwrap();
2542        fs::rename(
2543            staging.join("quotes").join("quotes_0.feather"),
2544            &staged_file,
2545        )
2546        .unwrap();
2547
2548        (quote, staged_file)
2549    }
2550
2551    fn sample_quote() -> QuoteTick {
2552        QuoteTick::new(
2553            InstrumentId::from("AUD/USD.SIM"),
2554            Price::from("0.65"),
2555            Price::from("0.67"),
2556            Quantity::from("11"),
2557            Quantity::from("17"),
2558            UnixNanos::from(19),
2559            UnixNanos::from(23),
2560        )
2561    }
2562
2563    #[rstest]
2564    #[case::ascii("AUD/USD.SIM-1-MINUTE-LAST-EXTERNAL")]
2565    #[case::non_ascii("CAFÉ.SIM-1-MINUTE-LAST-EXTERNAL")]
2566    fn bar_replay_across_promotions_is_idempotent(
2567        #[case] bar_type: &str,
2568        #[values(false, true)] restart: bool,
2569        #[values(false, true)] replay_only: bool,
2570        #[values(false, true)] use_ts_event: bool,
2571    ) {
2572        let directory = TempDir::new().unwrap();
2573
2574        let mut config = WriterConnectConfig::new(
2575            directory.path().join("live/run-bars").to_string_lossy(),
2576            Some(local_catalog(&directory)),
2577        );
2578        config.use_ts_event_for_ts_init = use_ts_event;
2579        config.delete_feather_after_promotion = true;
2580        config.promote_on_close = false;
2581        let clock = Arc::new(AtomicU64::new(1));
2582        let mut writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2583
2584        let first = Bar {
2585            bar_type: BarType::from(bar_type),
2586            ..sample_bar()
2587        };
2588
2589        writer.core.write_data(Data::Bar(first)).unwrap();
2590        assert_eq!(writer.promote_now().unwrap().len(), 1);
2591
2592        if restart {
2593            drop(writer);
2594            writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2595        }
2596
2597        clock.store(2, Ordering::Relaxed);
2598
2599        let replay = Bar {
2600            ts_init: if use_ts_event {
2601                UnixNanos::from(63_000_000_000)
2602            } else {
2603                first.ts_init
2604            },
2605            ..first
2606        };
2607
2608        let next = Bar {
2609            ts_event: UnixNanos::from(120_000_000_000),
2610            ts_init: UnixNanos::from(121_000_000_000),
2611            ..first
2612        };
2613
2614        writer.core.write_data(Data::Bar(replay)).unwrap();
2615        if !replay_only {
2616            writer.core.write_data(Data::Bar(next)).unwrap();
2617        }
2618
2619        let promoted = writer.promote_now().unwrap();
2620        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2621
2622        let DataBatch::Bar(rows) = catalog
2623            .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2624            .unwrap()
2625        else {
2626            panic!("expected bars")
2627        };
2628
2629        let mut expected = vec![Bar {
2630            ts_init: if use_ts_event {
2631                first.ts_event
2632            } else {
2633                first.ts_init
2634            },
2635            ..first
2636        }];
2637
2638        if !replay_only {
2639            expected.push(Bar {
2640                ts_init: if use_ts_event {
2641                    next.ts_event
2642                } else {
2643                    next.ts_init
2644                },
2645                ..next
2646            });
2647        }
2648
2649        assert_eq!(promoted.len(), 1);
2650        assert_eq!(rows.as_ref(), expected.as_slice());
2651        assert_eq!(
2652            catalog
2653                .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2654                .unwrap()
2655                .len(),
2656            expected.len()
2657        );
2658        assert!(
2659            list_session_feather_files(&writer.source.storage, Environment::Live, "run-bars")
2660                .unwrap()
2661                .is_empty()
2662        );
2663    }
2664
2665    #[rstest]
2666    #[case::open("open")]
2667    #[case::high("high")]
2668    #[case::low("low")]
2669    #[case::close("close")]
2670    #[case::volume("volume")]
2671    #[case::price_precision("price_precision")]
2672    #[case::size_precision("size_precision")]
2673    #[case::event_timestamp("ts_event")]
2674    #[case::init_timestamp("ts_init")]
2675    fn bar_replay_rejects_changed_values(#[case] field: &str) {
2676        let directory = TempDir::new().unwrap();
2677
2678        let mut config = WriterConnectConfig::new(
2679            directory.path().join("live/run-conflict").to_string_lossy(),
2680            Some(local_catalog(&directory)),
2681        );
2682        config.use_ts_event_for_ts_init = !matches!(field, "ts_event" | "ts_init");
2683        config.delete_feather_after_promotion = true;
2684        config.promote_on_close = false;
2685        let clock = Arc::new(AtomicU64::new(1));
2686        let mut writer = ParquetWriter::new(&config, WriterClock::Test(clock.clone())).unwrap();
2687        let first = sample_bar();
2688        writer.core.write_data(Data::Bar(first)).unwrap();
2689        writer.promote_now().unwrap();
2690        clock.store(2, Ordering::Relaxed);
2691        let mut correction = first;
2692        match field {
2693            "open" => correction.open = Price::from("1.02"),
2694            "high" => correction.high = Price::from("1.06"),
2695            "low" => correction.low = Price::from("0.98"),
2696            "close" => correction.close = Price::from("1.04"),
2697            "volume" => correction.volume = Quantity::from("18.000"),
2698            "price_precision" => {
2699                correction.open = Price::from("1.010");
2700                correction.high = Price::from("1.050");
2701                correction.low = Price::from("0.990");
2702                correction.close = Price::from("1.030");
2703            }
2704            "size_precision" => correction.volume = Quantity::from("17.0000"),
2705            "ts_event" => correction.ts_event = UnixNanos::from(59_000_000_000),
2706            "ts_init" => correction.ts_init = UnixNanos::from(60_500_000_000),
2707            _ => unreachable!(),
2708        }
2709
2710        let next = Bar {
2711            ts_event: UnixNanos::from(120_000_000_000),
2712            ts_init: UnixNanos::from(121_000_000_000),
2713            ..first
2714        };
2715
2716        writer.core.write_data(Data::Bar(correction)).unwrap();
2717        writer.core.write_data(Data::Bar(next)).unwrap();
2718        let work = writer.prepare_promotion().unwrap().unwrap();
2719        let files = work.files().to_vec();
2720        let error = writer.finalize(work.execute()).unwrap_err();
2721        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2722
2723        let DataBatch::Bar(rows) = catalog
2724            .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2725            .unwrap()
2726        else {
2727            panic!("expected bars")
2728        };
2729
2730        let expected = if config.use_ts_event_for_ts_init {
2731            Bar {
2732                ts_init: first.ts_event,
2733                ..first
2734            }
2735        } else {
2736            first
2737        };
2738
2739        assert!(error.to_string().contains("non-disjoint intervals"));
2740        assert_eq!(rows.as_ref(), &[expected]);
2741        assert_eq!(
2742            catalog
2743                .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2744                .unwrap()
2745                .len(),
2746            1
2747        );
2748        assert_eq!(files.len(), 1);
2749        assert!(directory.path().join(&files[0]).exists());
2750    }
2751
2752    #[rstest]
2753    fn bar_replay_manual_conversion_is_idempotent() {
2754        let directory = TempDir::new().unwrap();
2755        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2756        let first = sample_bar();
2757
2758        let next = Bar {
2759            ts_event: UnixNanos::from(120_000_000_000),
2760            ts_init: UnixNanos::from(121_000_000_000),
2761            ..first
2762        };
2763
2764        for run in ["first", "restart"] {
2765            let mut writer = FeatherWriter::new(
2766                directory.path().join("backtest").join(run),
2767                WriterClock::Test(Arc::new(AtomicU64::new(0))),
2768                RotationConfig::NoRotation,
2769                None,
2770                None,
2771            );
2772            writer.write(first).unwrap();
2773            if run == "restart" {
2774                writer.write(next).unwrap();
2775            }
2776
2777            writer.close().unwrap();
2778            catalog
2779                .convert_stream_to_data(
2780                    run,
2781                    &NautilusDataType::Bar.into(),
2782                    Environment::Backtest,
2783                    None,
2784                    true,
2785                )
2786                .unwrap();
2787        }
2788
2789        let DataBatch::Bar(rows) = catalog
2790            .query_batch(&CatalogQuery::new(NautilusDataType::Bar))
2791            .unwrap()
2792        else {
2793            panic!("expected bars")
2794        };
2795
2796        assert_eq!(
2797            rows.as_ref(),
2798            &[
2799                Bar {
2800                    ts_init: first.ts_event,
2801                    ..first
2802                },
2803                Bar {
2804                    ts_init: next.ts_event,
2805                    ..next
2806                },
2807            ]
2808        );
2809        assert_eq!(
2810            catalog
2811                .get_file_list_from_data_cls(&NautilusDataType::Bar.into())
2812                .unwrap()
2813                .len(),
2814            2
2815        );
2816    }
2817
2818    fn sample_bar() -> Bar {
2819        Bar::new(
2820            BarType::from("AUD/USD.SIM-1-MINUTE-LAST-EXTERNAL"),
2821            Price::from("1.01"),
2822            Price::from("1.05"),
2823            Price::from("0.99"),
2824            Price::from("1.03"),
2825            Quantity::from("17.000"),
2826            UnixNanos::from(60_000_000_000),
2827            UnixNanos::from(61_000_000_000),
2828        )
2829    }
2830
2831    #[rstest]
2832    fn kept_promotions_release_scheduled_paths() {
2833        let directory = TempDir::new().unwrap();
2834        let config = WriterConnectConfig::new(
2835            directory.path().join("live/run-bounded").to_string_lossy(),
2836            Some(local_catalog(&directory)),
2837        );
2838        let mut writer = ParquetWriter::new(&config, WriterClock::Live).unwrap();
2839
2840        for timestamp in 1..=32 {
2841            writer
2842                .core
2843                .write_data(Data::Quote(QuoteTick {
2844                    ts_event: UnixNanos::from(timestamp),
2845                    ts_init: UnixNanos::from(timestamp),
2846                    ..sample_quote()
2847                }))
2848                .unwrap();
2849            let work = writer.prepare_promotion().unwrap().unwrap();
2850            assert_eq!(work.files().len(), 1);
2851            writer.finalize(work.execute()).unwrap();
2852            assert_scheduled_count(&writer.core.promotion_driver.schedule(), 0);
2853            assert!(writer.prepare_promotion().unwrap().is_none());
2854        }
2855
2856        assert_eq!(
2857            list_session_feather_files(&writer.source.storage, Environment::Live, "run-bounded")
2858                .unwrap()
2859                .len(),
2860            32
2861        );
2862    }
2863
2864    #[rstest]
2865    #[case::timer_seal(RotationConfig::NoRotation, false, 1)]
2866    #[case::close(RotationConfig::NoRotation, true, 1)]
2867    #[case::rotation(RotationConfig::Size { max_size: 1 }, false, 2)]
2868    fn sealed_paths_promote_once_and_do_not_resubmit_in_flight(
2869        #[case] rotation: RotationConfig,
2870        #[case] close: bool,
2871        #[case] expected_files: usize,
2872    ) {
2873        let directory = TempDir::new().unwrap();
2874        let mut config = WriterConnectConfig::new(
2875            directory.path().join("live/run-sealing").to_string_lossy(),
2876            Some(local_catalog(&directory)),
2877        );
2878        config.rotation_config = rotation;
2879        let mut writer = ParquetWriter::new(&config, WriterClock::Live).unwrap();
2880        let first = sample_quote();
2881        let second = QuoteTick {
2882            ts_event: UnixNanos::from(29),
2883            ts_init: UnixNanos::from(31),
2884            ..first
2885        };
2886        writer.core.write_data(Data::Quote(first)).unwrap();
2887        writer.core.write_data(Data::Quote(second)).unwrap();
2888
2889        if close {
2890            writer.core.close().unwrap();
2891        }
2892
2893        let work = writer.prepare_promotion().unwrap().unwrap();
2894        let files = work.files().to_vec();
2895        assert_eq!(files.len(), expected_files);
2896        assert!(writer.prepare_promotion().unwrap().is_none());
2897        assert_scheduled_count(&writer.core.promotion_driver.schedule(), expected_files);
2898        let result = work.execute();
2899        assert_scheduled_count(&writer.core.promotion_driver.schedule(), 0);
2900        assert_eq!(result.committed_paths, files);
2901        assert_eq!(writer.finalize(result).unwrap().len(), expected_files);
2902        assert!(writer.prepare_promotion().unwrap().is_none());
2903
2904        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2905        let DataBatch::Quote(rows) = catalog
2906            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2907            .unwrap()
2908        else {
2909            panic!("expected quotes")
2910        };
2911        assert_eq!(rows.as_ref(), &[first, second]);
2912    }
2913
2914    #[rstest]
2915    #[case::space("my data")]
2916    #[case::non_ascii("Données")]
2917    #[case::fragment("data#1")]
2918    #[case::literal_percent("data%20")]
2919    fn writer_root_path_promotes_and_replays(
2920        #[case] root_name: &str,
2921        #[values(false, true)] file_uri: bool,
2922    ) {
2923        let directory = TempDir::new().unwrap();
2924        let root = directory.path().join(root_name);
2925        let staging = root.join("live/run-root");
2926        let staging_uri = if file_uri {
2927            normalize_path_to_uri(&staging.to_string_lossy()).unwrap()
2928        } else {
2929            staging.to_string_lossy().into_owned()
2930        };
2931        let config = WriterConnectConfig::new(staging_uri, Some(local_catalog(&directory)));
2932        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2933        let quote = sample_quote();
2934        let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
2935        writer.core.write_data(Data::Quote(quote)).unwrap();
2936        let work = writer.prepare_promotion().unwrap().unwrap();
2937        assert_eq!(work.files(), &["live/run-root/quotes/quotes_0.feather"]);
2938        let promoted = writer.finalize(work.execute()).unwrap();
2939        assert_eq!(promoted.len(), 1);
2940        assert!(writer.prepare_promotion().unwrap().is_none());
2941        drop(writer);
2942
2943        let mut restarted = ParquetWriter::new(&config, clock).unwrap();
2944        let work = restarted.prepare_promotion().unwrap().unwrap();
2945        assert_eq!(work.files(), &["live/run-root/quotes/quotes_0.feather"]);
2946        assert_eq!(restarted.finalize(work.execute()).unwrap(), promoted);
2947        assert!(restarted.prepare_promotion().unwrap().is_none());
2948
2949        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2950        let DataBatch::Quote(rows) = catalog
2951            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2952            .unwrap()
2953        else {
2954            panic!("expected quotes")
2955        };
2956        assert_eq!(rows.as_ref(), &[quote]);
2957    }
2958
2959    #[rstest]
2960    fn parent_relative_writer_path_promotes_and_replays() {
2961        let directory = TempDir::new_in(".").unwrap();
2962        let current = std::env::current_dir().unwrap();
2963        let staging = PathBuf::from("..")
2964            .join(current.file_name().unwrap())
2965            .join(directory.path().file_name().unwrap())
2966            .join("live/run-relative");
2967        let config =
2968            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
2969        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
2970        let quote = sample_quote();
2971        let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
2972        writer.core.write_data(Data::Quote(quote)).unwrap();
2973        let work = writer.prepare_promotion().unwrap().unwrap();
2974        assert_eq!(work.files(), &["live/run-relative/quotes/quotes_0.feather"]);
2975        let promoted = writer.finalize(work.execute()).unwrap();
2976        assert_eq!(promoted.len(), 1);
2977        assert!(writer.prepare_promotion().unwrap().is_none());
2978        drop(writer);
2979
2980        let mut restarted = ParquetWriter::new(&config, clock).unwrap();
2981        let work = restarted.prepare_promotion().unwrap().unwrap();
2982        assert_eq!(work.files(), &["live/run-relative/quotes/quotes_0.feather"]);
2983        assert_eq!(restarted.finalize(work.execute()).unwrap(), promoted);
2984        assert!(restarted.prepare_promotion().unwrap().is_none());
2985
2986        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
2987        let DataBatch::Quote(rows) = catalog
2988            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
2989            .unwrap()
2990        else {
2991            panic!("expected quotes")
2992        };
2993        assert_eq!(rows.as_ref(), &[quote]);
2994    }
2995
2996    #[rstest]
2997    #[case::sealed(false)]
2998    #[case::partial(true)]
2999    fn restart_discovers_crashed_files_and_deduplicates_promotions(
3000        #[case] partial: bool,
3001        #[values(false, true)] delete_source: bool,
3002        #[values(false, true)] restage: bool,
3003    ) {
3004        let directory = TempDir::new().unwrap();
3005        let staging = directory.path().join("backtest/run-restart");
3006        let quote = sample_quote();
3007        let mut crashed = FeatherWriter::new(
3008            staging.clone(),
3009            WriterClock::Test(Arc::new(AtomicU64::new(0))),
3010            RotationConfig::NoRotation,
3011            None,
3012            None,
3013        );
3014        crashed.write(quote).unwrap();
3015        crashed.close().unwrap();
3016        let sealed = staging.join("quotes/quotes_0.feather");
3017
3018        if partial {
3019            fs::rename(&sealed, sealed.with_extension(FEATHER_PARTIAL_EXTENSION)).unwrap();
3020        }
3021
3022        drop(crashed);
3023        let mut config =
3024            WriterConnectConfig::new(staging.to_string_lossy(), Some(local_catalog(&directory)));
3025        config.delete_feather_after_promotion = restage;
3026        let clock = WriterClock::Test(Arc::new(AtomicU64::new(0)));
3027        let mut writer = ParquetWriter::new(&config, clock.clone()).unwrap();
3028        let source_bytes = fs::read(&sealed).unwrap();
3029        let work = writer.prepare_promotion().unwrap().unwrap();
3030        assert_eq!(
3031            work.files(),
3032            &["backtest/run-restart/quotes/quotes_0.feather".to_string()]
3033        );
3034        let promoted = writer.finalize(work.execute()).unwrap();
3035        assert_eq!(promoted.len(), 1);
3036        assert!(writer.prepare_promotion().unwrap().is_none());
3037        drop(writer);
3038
3039        if restage {
3040            assert!(!sealed.exists());
3041            fs::write(&sealed, source_bytes).unwrap();
3042        }
3043
3044        config.delete_feather_after_promotion = delete_source;
3045        let mut restarted = ParquetWriter::new(&config, clock).unwrap();
3046        let work = restarted.prepare_promotion().unwrap().unwrap();
3047        assert_eq!(work.files().len(), 1);
3048        let files = work.files().to_vec();
3049        let result = work.execute();
3050        assert_eq!(result.committed_paths, files);
3051        assert_eq!(
3052            result.deleted_paths,
3053            if delete_source { files } else { Vec::new() }
3054        );
3055        assert_eq!(restarted.finalize(result).unwrap(), promoted);
3056        assert!(restarted.prepare_promotion().unwrap().is_none());
3057        assert_scheduled_count(&restarted.core.promotion_driver.schedule(), 0);
3058        assert_eq!(sealed.exists(), !delete_source);
3059
3060        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
3061        let DataBatch::Quote(rows) = catalog
3062            .query_batch(&CatalogQuery::new(NautilusDataType::QuoteTick))
3063            .unwrap()
3064        else {
3065            panic!("expected quotes")
3066        };
3067        assert_eq!(rows.as_ref(), &[quote]);
3068    }
3069
3070    #[rstest]
3071    #[case::nested_base("/prefix/catalog/", "prefix/catalog/")]
3072    #[case::root_base("", "")]
3073    fn promotion_paths_live_under_the_catalog_base_path(
3074        #[case] base_path: &str,
3075        #[case] expected_prefix: &str,
3076    ) {
3077        let directory = TempDir::new().unwrap();
3078        let mut catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
3079        catalog.base_path = base_path.to_string();
3080
3081        let backend = ParquetPromotionBackend {
3082            catalog,
3083            legacy_manifest_missing: Arc::default(),
3084        };
3085
3086        assert_eq!(
3087            backend.manifest_path(),
3088            ObjectPath::from(format!("{expected_prefix}_nautilus_promotions.json")),
3089        );
3090        assert_eq!(
3091            backend.marker_path("replay-id"),
3092            ObjectPath::from(format!(
3093                "{expected_prefix}_nautilus_promotions/\
3094                 cd3112001080bc2d11985ffe2b8b90b324d336d82c17ddb66127d3a05f08c69c.json"
3095            )),
3096        );
3097    }
3098
3099    #[rstest]
3100    fn feather_replay_identity_is_stable_and_ignores_identifier_order() {
3101        let identity = |identifiers: Option<&[String]>| {
3102            feather_replay_identity(
3103                "file:///catalog/backtest/run-1",
3104                "quotes/AUDUSD.SIM/part-0.feather",
3105                "content-hash",
3106                identifiers,
3107            )
3108        };
3109
3110        let unordered = ["B".to_string(), "A".to_string(), "A".to_string()];
3111        let ordered = ["A".to_string(), "B".to_string()];
3112
3113        let expected =
3114            "nautilus-feather:10a9435c28f7536f26653c3fc808571be89bfe769e06c7307cb4743e273a23fd";
3115
3116        assert_eq!(identity(Some(&unordered)), expected);
3117        assert_eq!(identity(Some(&ordered)), expected);
3118        assert_ne!(identity(None), expected);
3119    }
3120}