1#![expect(
19 clippy::missing_errors_doc,
20 reason = "catalog write functions validate catalog-controlled batches and forward storage errors"
21)]
22
23use nautilus_serialization::arrow::catalog_identifier_from_metadata;
24
25use super::{
26 BTreeMap, CustomData, Data, DataBatch, EncodeToRecordBatch, HasCatalogDataType, HasTsInit,
27 Instrument, InstrumentAny, NautilusDataType, NautilusRecordType, ObjectPath, ObjectStoreExt,
28 Params, ParquetDataCatalog, PathBuf, RecordBatch, Serialize, UnixNanos,
29 WRITE_SKIP_DISJOINT_CHECK, are_intervals_disjoint, group_custom_data_by_type,
30 instrument_any_type, instrument_path_prefix, local_to_object_store_path,
31 parquet_data_path_prefix, prepare_custom_data_batch, record_batch_without_identifier_column,
32 record_path_prefix, timestamps_to_filename, to_snake_case, write_batches_to_object_store,
33 write_catalog_batch,
34};
35use crate::{
36 backend::parquet::{io::write_batches_to_object_store_create, paths::catalog_filename},
37 common::metadata::record_batch_ts_init_range,
38};
39
40impl ParquetDataCatalog {
41 pub fn write_data_enum(
78 &self,
79 data: &[Data],
80 start: Option<UnixNanos>,
81 end: Option<UnixNanos>,
82 skip_disjoint_check: Option<bool>,
83 ) -> anyhow::Result<()> {
84 let mut batches = Vec::new();
85 let mut custom = Vec::new();
86
87 for batch in DataBatch::from_data_vec_grouped(data)? {
88 match batch {
89 DataBatch::Custom(data) => custom.extend(data.iter().cloned()),
90 batch => batches.push(batch),
91 }
92 }
93
94 let custom_writes =
96 self.prepare_custom_data_groups(&custom, start, end, skip_disjoint_check)?;
97
98 for batch in &batches {
99 write_catalog_batch(self, batch, start, end, skip_disjoint_check)?;
100 }
101
102 for write in &custom_writes {
103 self.write_custom_data_prepared(write, skip_disjoint_check)?;
104 }
105
106 Ok(())
107 }
108
109 pub(super) fn write_grouped_to_parquet<T>(
110 &self,
111 data: &[T],
112 start: Option<UnixNanos>,
113 end: Option<UnixNanos>,
114 skip_disjoint_check: Option<bool>,
115 ) -> anyhow::Result<()>
116 where
117 T: Clone + HasTsInit + EncodeToRecordBatch + HasCatalogDataType,
118 {
119 let mut groups: BTreeMap<Option<String>, Vec<T>> = BTreeMap::new();
120
121 for item in data {
122 let identifier = catalog_identifier_from_metadata(&item.metadata());
123 groups.entry(identifier).or_default().push(item.clone());
124 }
125
126 for items in groups.into_values() {
127 self.write_to_parquet(&items, start, end, skip_disjoint_check)?;
128 }
129
130 Ok(())
131 }
132
133 pub fn write_record_batches(
139 &mut self,
140 record_type: &NautilusRecordType,
141 identifier: Option<&str>,
142 batches: &[RecordBatch],
143 params: &Params,
144 ) -> anyhow::Result<()> {
145 if batches.is_empty() || batches.iter().all(|batch| batch.num_rows() == 0) {
146 return Ok(());
147 }
148
149 let (start_ts, end_ts) = record_batch_ts_init_range(batches)?;
150 let record_prefix = record_path_prefix(record_type);
151 let directory = self.make_path(record_prefix.as_ref(), identifier)?;
152 self.recover_consolidation(&self.to_object_path(&directory)?)?;
153 let filename = timestamps_to_filename(UnixNanos::from(start_ts), UnixNanos::from(end_ts));
154 let path = PathBuf::from(directory.clone()).join(&filename);
155 let object_path = self.to_object_path(&path.to_string_lossy())?;
156 let skip_disjoint_check = params.get_bool(WRITE_SKIP_DISJOINT_CHECK).unwrap_or(false);
157
158 if !skip_disjoint_check {
159 let current_intervals = self.get_directory_intervals(&directory)?;
160 let mut intervals = current_intervals.clone();
161 intervals.push((start_ts, end_ts));
162 anyhow::ensure!(
163 are_intervals_disjoint(&intervals),
164 "Writing file {filename} interval ({start_ts}, {end_ts}) would create non-disjoint intervals. Existing intervals: {current_intervals:?}",
165 );
166 }
167
168 self.execute_async(|| async {
169 write_batches_to_object_store(
170 batches,
171 self.object_store.clone(),
172 &object_path,
173 Some(self.compression),
174 Some(self.max_row_group_size),
175 None,
176 )
177 .await
178 })
179 }
180
181 pub fn write_to_parquet<T>(
237 &self,
238 data: &[T],
239 start: Option<UnixNanos>,
240 end: Option<UnixNanos>,
241 skip_disjoint_check: Option<bool>,
242 ) -> anyhow::Result<PathBuf>
243 where
244 T: HasTsInit + EncodeToRecordBatch + HasCatalogDataType,
245 {
246 let (Some(first), Some(last)) = (data.first(), data.last()) else {
247 return Ok(PathBuf::new());
248 };
249
250 let type_name = to_snake_case(std::any::type_name::<T>());
251 Self::check_ascending_timestamps(data, &type_name)?;
252 Self::check_chunk_identity(data, &type_name)?;
253
254 let start_ts = start.unwrap_or(first.ts_init());
255 let end_ts = end.unwrap_or(last.ts_init());
256
257 let batches = self.data_to_record_batches(data)?;
258 let schema = batches.first().expect("Batches are empty.").schema();
259
260 let data_type = T::catalog_data_type();
261 let path_prefix = parquet_data_path_prefix(&data_type);
262
263 let identifier = if matches!(data_type, NautilusDataType::Bar) {
264 schema.metadata.get("bar_type").cloned()
265 } else {
266 schema.metadata.get("instrument_id").cloned()
267 };
268
269 let directory = self.make_path(path_prefix.as_ref(), identifier.as_deref())?;
270 self.write_parquet_file_checked(
271 &directory,
272 start_ts,
273 end_ts,
274 &batches,
275 skip_disjoint_check.unwrap_or(false),
276 "File",
277 Some(&format!("{type_name} data")),
278 None,
279 )
280 }
281
282 pub fn write_custom_data_batch<D>(
307 &self,
308 data: D,
309 start: Option<UnixNanos>,
310 end: Option<UnixNanos>,
311 skip_disjoint_check: Option<bool>,
312 ) -> anyhow::Result<PathBuf>
313 where
314 D: AsRef<[CustomData]>,
315 {
316 let data = data.as_ref().iter().collect::<Vec<_>>();
317 self.write_custom_data_refs_batch(&data, start, end, skip_disjoint_check)
318 }
319
320 fn write_custom_data_refs_batch(
321 &self,
322 data: &[&CustomData],
323 start: Option<UnixNanos>,
324 end: Option<UnixNanos>,
325 skip_disjoint_check: Option<bool>,
326 ) -> anyhow::Result<PathBuf> {
327 if data.is_empty() {
328 return Ok(PathBuf::new());
329 }
330
331 let write = self.prepare_custom_data_write(data, start, end)?;
332 self.write_custom_data_prepared(&write, skip_disjoint_check)
333 }
334
335 pub(crate) fn write_custom_data_groups(
341 &self,
342 data: &[CustomData],
343 start: Option<UnixNanos>,
344 end: Option<UnixNanos>,
345 skip_disjoint_check: Option<bool>,
346 ) -> anyhow::Result<()> {
347 for write in &self.prepare_custom_data_groups(data, start, end, skip_disjoint_check)? {
348 self.write_custom_data_prepared(write, skip_disjoint_check)?;
349 }
350
351 Ok(())
352 }
353
354 pub(crate) fn prepare_custom_data_groups(
359 &self,
360 data: &[CustomData],
361 start: Option<UnixNanos>,
362 end: Option<UnixNanos>,
363 skip_disjoint_check: Option<bool>,
364 ) -> anyhow::Result<Vec<CustomDataWrite>> {
365 let check_disjoint = !skip_disjoint_check.unwrap_or(false);
366 let mut writes = Vec::new();
367 let mut directory_intervals: BTreeMap<String, Vec<(u64, u64)>> = BTreeMap::new();
368
369 for group in group_custom_data_by_type(data) {
370 let write = self.prepare_custom_data_write(&group, start, end)?;
371 let interval = (write.start_ts.as_u64(), write.end_ts.as_u64());
372 let intervals = directory_intervals
373 .entry(write.directory.clone())
374 .or_default();
375
376 anyhow::ensure!(
377 !intervals.contains(&interval),
378 "Cannot write custom data for {} with different metadata to the same file {}: \
379 a file holds one data type",
380 write.type_name,
381 PathBuf::from(&write.directory)
382 .join(catalog_filename(write.start_ts, write.end_ts, None))
383 .display(),
384 );
385 intervals.push(interval);
386 anyhow::ensure!(
387 !check_disjoint || are_intervals_disjoint(intervals),
388 "Cannot write custom data for {} with different metadata over overlapping \
389 intervals {intervals:?} in {}",
390 write.type_name,
391 write.directory,
392 );
393
394 writes.push(write);
395 }
396
397 Ok(writes)
398 }
399
400 pub(crate) fn prepare_custom_data_write(
401 &self,
402 data: &[&CustomData],
403 start: Option<UnixNanos>,
404 end: Option<UnixNanos>,
405 ) -> anyhow::Result<CustomDataWrite> {
406 let (batch, type_name, identifier, start_ts, end_ts) = prepare_custom_data_batch(data)?;
407 let directory = self.make_path_custom_data(&type_name, identifier.as_deref())?;
408
409 Ok(CustomDataWrite {
410 type_name,
411 directory,
412 start_ts: start.unwrap_or(start_ts),
413 end_ts: end.unwrap_or(end_ts),
414 batches: vec![record_batch_without_identifier_column(batch)?],
415 })
416 }
417
418 pub(crate) fn write_custom_data_prepared(
419 &self,
420 write: &CustomDataWrite,
421 skip_disjoint_check: Option<bool>,
422 ) -> anyhow::Result<PathBuf> {
423 self.write_parquet_file_checked(
424 &write.directory,
425 write.start_ts,
426 write.end_ts,
427 &write.batches,
428 skip_disjoint_check.unwrap_or(false),
429 "File",
430 None,
431 None,
432 )
433 }
434
435 pub fn write_instruments(
476 &self,
477 instruments: Vec<InstrumentAny>,
478 ) -> anyhow::Result<Vec<PathBuf>> {
479 if instruments.is_empty() {
480 return Ok(Vec::new());
481 }
482
483 let mut by_type_and_id: BTreeMap<(String, String), Vec<InstrumentAny>> = BTreeMap::new();
486
487 for instrument in instruments {
488 let instrument_type = instrument_any_type(&instrument);
489 let instrument_prefix = instrument_path_prefix(&instrument_type).to_string();
490 let instrument_id = Instrument::id(&instrument).to_string();
491 by_type_and_id
492 .entry((instrument_prefix, instrument_id))
493 .or_default()
494 .push(instrument);
495 }
496
497 let mut paths = Vec::new();
498
499 for ((instrument_prefix, instrument_id), instrument_group) in by_type_and_id {
500 Self::check_ascending_timestamps(&instrument_group, "instrument")?;
501
502 let (Some(first), Some(last)) = (instrument_group.first(), instrument_group.last())
503 else {
504 continue;
505 };
506
507 let start_ts = HasTsInit::ts_init(first);
508 let end_ts = HasTsInit::ts_init(last);
509 let batches = self.data_to_record_batches(&instrument_group)?;
510
511 let directory = self.make_path(&instrument_prefix, Some(instrument_id.as_str()))?;
512
513 let path = self.write_parquet_file_checked(
516 &directory,
517 start_ts,
518 end_ts,
519 &batches,
520 false,
521 "Instrument file",
522 Some(&format!("instrument data for {instrument_id}")),
523 None,
524 )?;
525
526 paths.push(path);
527 }
528
529 Ok(paths)
530 }
531
532 #[expect(clippy::too_many_arguments)]
538 pub(crate) fn write_parquet_file_checked(
539 &self,
540 directory: &str,
541 start_ts: UnixNanos,
542 end_ts: UnixNanos,
543 batches: &[RecordBatch],
544 skip_disjoint_check: bool,
545 file_label: &str,
546 data_description: Option<&str>,
547 replay_identity: Option<&str>,
548 ) -> anyhow::Result<PathBuf> {
549 self.recover_consolidation(&self.to_object_path(directory)?)?;
550 let filename = catalog_filename(start_ts, end_ts, replay_identity);
551
552 let path = PathBuf::from(directory).join(&filename);
553 let object_path = self.to_object_path(&path.to_string_lossy())?;
554
555 if self.file_exists(&path.to_string_lossy())? {
556 log::info!(
557 "{file_label} {} already exists, skipping write",
558 path.display()
559 );
560 return Ok(path);
561 }
562
563 if !skip_disjoint_check {
564 let current_intervals = self.get_directory_intervals(directory)?;
565 let new_interval = (start_ts.as_u64(), end_ts.as_u64());
566 let mut new_intervals = current_intervals.clone();
567 new_intervals.push(new_interval);
568
569 anyhow::ensure!(
570 are_intervals_disjoint(&new_intervals),
571 "Writing file {filename} with interval ({start_ts}, {end_ts}) would create \
572 non-disjoint intervals. Existing intervals: {current_intervals:?}"
573 );
574 }
575
576 if let Some(data_description) = data_description {
577 log::info!(
578 "Writing {} batches of {data_description} to {}",
579 batches.len(),
580 path.display(),
581 );
582 }
583
584 self.execute_async(|| async {
585 if replay_identity.is_none() {
586 return write_batches_to_object_store(
587 batches,
588 self.object_store.clone(),
589 &object_path,
590 Some(self.compression),
591 Some(self.max_row_group_size),
592 None,
593 )
594 .await;
595 }
596
597 let result = write_batches_to_object_store_create(
598 batches,
599 self.object_store.clone(),
600 &object_path,
601 Some(self.compression),
602 Some(self.max_row_group_size),
603 None,
604 )
605 .await;
606
607 match result {
608 Err(e)
609 if matches!(
610 e.downcast_ref::<object_store::Error>(),
611 Some(object_store::Error::AlreadyExists { .. })
612 ) =>
613 {
614 Ok(())
615 }
616 result => result,
617 }
618 })?;
619
620 Ok(path)
621 }
622
623 pub fn write_to_json<T>(
673 &self,
674 data: Vec<T>,
675 path: Option<PathBuf>,
676 write_metadata: bool,
677 ) -> anyhow::Result<PathBuf>
678 where
679 T: HasTsInit + Serialize + HasCatalogDataType + EncodeToRecordBatch,
680 {
681 let (Some(first), Some(last)) = (data.first(), data.last()) else {
682 return Ok(PathBuf::new());
683 };
684
685 let type_name = to_snake_case(std::any::type_name::<T>());
686 Self::check_ascending_timestamps(&data, &type_name)?;
687
688 let start_ts = first.ts_init();
689 let end_ts = last.ts_init();
690
691 let data_type = T::catalog_data_type();
692 let path_prefix = parquet_data_path_prefix(&data_type);
693
694 let directory = match path {
695 Some(path) => path,
696 None => PathBuf::from(self.make_path(path_prefix.as_ref(), None)?),
697 };
698
699 let filename = timestamps_to_filename(start_ts, end_ts).replace(".parquet", ".json");
700 let json_path = directory.join(&filename);
701
702 log::info!(
703 "Writing {} records of {type_name} data to {}",
704 data.len(),
705 json_path.display(),
706 );
707
708 if write_metadata {
709 let metadata = T::chunk_metadata(&data);
710 let metadata_path = json_path.with_extension("metadata.json");
711 log::info!("Writing metadata to {}", metadata_path.display());
712
713 let metadata_object_path = ObjectPath::from(local_to_object_store_path(&metadata_path));
715 let metadata_json = serde_json::to_vec_pretty(&metadata)?;
716 self.execute_async(|| async {
717 let _: object_store::PutResult = self
718 .object_store
719 .put(&metadata_object_path, metadata_json.into())
720 .await?;
721 Ok(())
722 })?;
723 }
724
725 let json_object_path = ObjectPath::from(local_to_object_store_path(&json_path));
727 let json_data = serde_json::to_vec_pretty(&serde_json::to_value(data)?)?;
728 self.execute_async(|| async {
729 let _: object_store::PutResult = self
730 .object_store
731 .put(&json_object_path, json_data.into())
732 .await?;
733 Ok(())
734 })?;
735
736 Ok(json_path)
737 }
738
739 pub fn check_ascending_timestamps<T: HasTsInit>(
746 data: &[T],
747 type_name: &str,
748 ) -> anyhow::Result<()> {
749 anyhow::ensure!(
750 data.array_windows()
751 .all(|[a, b]| a.ts_init() <= b.ts_init()),
752 "{type_name} timestamps must be in ascending order"
753 );
754 Ok(())
755 }
756
757 pub(crate) fn check_chunk_identity<T: EncodeToRecordBatch>(
760 data: &[T],
761 type_name: &str,
762 ) -> anyhow::Result<()> {
763 if data.is_empty() {
764 return Ok(());
765 }
766 let chunk_metadata = T::chunk_metadata(data);
767
768 if let Some(position) = data
769 .iter()
770 .position(|item| !item.matches_chunk_metadata(&chunk_metadata))
771 {
772 anyhow::bail!(
773 "Cannot write {type_name} data with mixed identities: element {position} has \
774 metadata {:?} but the chunk has {chunk_metadata:?}; write each \
775 instrument or bar type separately",
776 data[position].metadata(),
777 );
778 }
779
780 Ok(())
781 }
782
783 pub fn data_to_record_batches<T>(&self, data: &[T]) -> anyhow::Result<Vec<RecordBatch>>
804 where
805 T: HasTsInit + EncodeToRecordBatch,
806 {
807 if data.is_empty() {
808 return Ok(Vec::new());
809 }
810
811 let mut batches = Vec::new();
812 let metadata = EncodeToRecordBatch::chunk_metadata(data);
813
814 for chunk in data.chunks(self.batch_size) {
815 let record_batch = T::encode_batch(&metadata, chunk)?;
816 let record_batch = record_batch_without_identifier_column(record_batch)?;
817 batches.push(record_batch);
818 }
819
820 Ok(batches)
821 }
822}
823
824pub(crate) struct CustomDataWrite {
826 type_name: String,
827 directory: String,
828 start_ts: UnixNanos,
829 end_ts: UnixNanos,
830 pub(crate) batches: Vec<RecordBatch>,
831}
832
833impl CustomDataWrite {
834 pub(crate) fn path(&self) -> PathBuf {
835 let filename = catalog_filename(self.start_ts, self.end_ts, None);
836 PathBuf::from(&self.directory).join(filename)
837 }
838}
839
840#[cfg(test)]
841mod tests {
842 use std::{
843 fmt::Display,
844 fs::File,
845 sync::{
846 Arc,
847 atomic::{AtomicBool, AtomicUsize, Ordering},
848 },
849 };
850
851 use arrow::{
852 array::Int64Array,
853 datatypes::{DataType, Field, Schema},
854 record_batch::RecordBatch,
855 };
856 use futures::{StreamExt, TryStreamExt, stream::BoxStream};
857 use nautilus_core::UnixNanos;
858 use nautilus_model::{
859 data::{
860 NautilusDataType, OrderBookDelta, OrderBookDepth, QuoteTick,
861 stubs::{stub_delta, stub_depth10},
862 },
863 identifiers::InstrumentId,
864 types::{Price, Quantity},
865 };
866 use nautilus_serialization::arrow::{
867 DecodeFromRecordBatch, KEY_PRICE_PRECISION, KEY_SIZE_PRECISION,
868 };
869 use object_store::{
870 CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
871 ObjectStoreExt, PutMode, PutMultipartOptions, PutOptions, PutPayload, PutResult,
872 Result as ObjectStoreResult, memory::InMemory, path::Path as ObjectPath,
873 };
874 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
875 use rstest::rstest;
876 use tempfile::TempDir;
877
878 use super::ParquetDataCatalog;
879 use crate::common::datafusion::DataBackendSession;
880
881 #[rstest]
882 fn depth_write_shares_file_metadata_across_chunks(stub_depth10: OrderBookDepth) {
883 let mut empty = stub_depth10.clone();
884 empty.bids.clear();
885 empty.asks.clear();
886 empty.bid_counts.clear();
887 empty.ask_counts.clear();
888 let directory = TempDir::new().unwrap();
889 let catalog = ParquetDataCatalog::from_uri(
890 directory.path().to_str().unwrap(),
891 None,
892 Some(2),
893 None,
894 Some(2),
895 )
896 .unwrap();
897 let data = vec![empty.clone(), empty, stub_depth10];
898
899 let path = catalog.write_to_parquet(&data, None, None, None).unwrap();
900 let builder = ParquetRecordBatchReaderBuilder::try_new(
901 File::open(directory.path().join(path)).unwrap(),
902 )
903 .unwrap();
904 let metadata = builder.schema().metadata().clone();
905 let batches = builder
906 .build()
907 .unwrap()
908 .collect::<Result<Vec<_>, _>>()
909 .unwrap();
910 let decoded = batches
911 .iter()
912 .cloned()
913 .flat_map(|batch| OrderBookDepth::decode_batch(&metadata, batch).unwrap())
914 .collect::<Vec<_>>();
915
916 assert_eq!(metadata[KEY_PRICE_PRECISION], "2");
917 assert_eq!(metadata[KEY_SIZE_PRECISION], "0");
918 assert_eq!(decoded.len(), 3);
919 assert_eq!(decoded, data);
920 assert_eq!(decoded[2].bids[0].price.precision, 2);
921 }
922
923 #[rstest]
924 fn leading_clear_delta_writes_with_following_precision(stub_delta: OrderBookDelta) {
925 let directory = TempDir::new().unwrap();
926 let catalog = ParquetDataCatalog::from_uri(
927 directory.path().to_str().unwrap(),
928 None,
929 Some(2),
930 None,
931 None,
932 )
933 .unwrap();
934 let clear = OrderBookDelta::clear(
935 stub_delta.instrument_id,
936 0,
937 UnixNanos::from(1),
938 UnixNanos::from(1),
939 );
940 let second_clear = OrderBookDelta::clear(
941 stub_delta.instrument_id,
942 0,
943 UnixNanos::from(2),
944 UnixNanos::from(2),
945 );
946
947 let path = catalog
948 .write_to_parquet(&[clear, second_clear, stub_delta], None, None, None)
949 .unwrap();
950 let builder = ParquetRecordBatchReaderBuilder::try_new(
951 File::open(directory.path().join(&path)).unwrap(),
952 )
953 .unwrap();
954 let metadata = builder.schema().metadata().clone();
955 let decoded = builder
956 .build()
957 .unwrap()
958 .map(|batch| OrderBookDelta::decode_batch(&metadata, batch.unwrap()).unwrap())
959 .collect::<Vec<_>>()
960 .concat();
961
962 assert!(directory.path().join(path).exists());
963 assert_eq!(metadata[KEY_PRICE_PRECISION], "2");
964 assert_eq!(decoded[2].order.price.precision, 2);
965 }
966
967 #[derive(Debug)]
968 struct CreateRaceStore {
969 inner: InMemory,
970 create_calls: AtomicUsize,
971 already_exists: bool,
972 fail_head: AtomicBool,
973 }
974
975 impl Display for CreateRaceStore {
976 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
977 f.write_str("create-race")
978 }
979 }
980
981 #[async_trait::async_trait]
982 impl ObjectStore for CreateRaceStore {
983 async fn put_opts(
984 &self,
985 location: &ObjectPath,
986 payload: PutPayload,
987 opts: PutOptions,
988 ) -> ObjectStoreResult<PutResult> {
989 if opts.mode == PutMode::Create {
990 self.create_calls.fetch_add(1, Ordering::Relaxed);
991
992 if !self.already_exists {
993 return Err(object_store::Error::Generic {
994 store: "create-race",
995 source: "injected create failure".into(),
996 });
997 }
998
999 return Err(object_store::Error::AlreadyExists {
1000 path: location.to_string(),
1001 source: Box::new(std::io::Error::new(
1002 std::io::ErrorKind::AlreadyExists,
1003 "injected create race",
1004 )),
1005 });
1006 }
1007
1008 self.inner.put_opts(location, payload, opts).await
1009 }
1010
1011 async fn put_multipart_opts(
1012 &self,
1013 location: &ObjectPath,
1014 opts: PutMultipartOptions,
1015 ) -> ObjectStoreResult<Box<dyn MultipartUpload>> {
1016 self.inner.put_multipart_opts(location, opts).await
1017 }
1018
1019 async fn get_opts(
1020 &self,
1021 location: &ObjectPath,
1022 options: GetOptions,
1023 ) -> ObjectStoreResult<GetResult> {
1024 if options.head && self.fail_head.load(Ordering::Relaxed) {
1025 return Err(object_store::Error::Generic {
1026 store: "create-race",
1027 source: "injected head failure".into(),
1028 });
1029 }
1030
1031 self.inner.get_opts(location, options).await
1032 }
1033
1034 fn list(
1035 &self,
1036 prefix: Option<&ObjectPath>,
1037 ) -> BoxStream<'static, ObjectStoreResult<ObjectMeta>> {
1038 self.inner.list(prefix)
1039 }
1040
1041 async fn list_with_delimiter(
1042 &self,
1043 prefix: Option<&ObjectPath>,
1044 ) -> ObjectStoreResult<ListResult> {
1045 self.inner.list_with_delimiter(prefix).await
1046 }
1047
1048 fn delete_stream(
1049 &self,
1050 locations: BoxStream<'static, ObjectStoreResult<ObjectPath>>,
1051 ) -> BoxStream<'static, ObjectStoreResult<ObjectPath>> {
1052 self.inner.delete_stream(locations)
1053 }
1054
1055 async fn copy_opts(
1056 &self,
1057 from: &ObjectPath,
1058 to: &ObjectPath,
1059 opts: CopyOptions,
1060 ) -> ObjectStoreResult<()> {
1061 self.inner.copy_opts(from, to, opts).await
1062 }
1063 }
1064
1065 #[rstest]
1066 fn promotion_writes_to_memory_store_without_copy_support() {
1067 let catalog = ParquetDataCatalog {
1068 base_path: "catalog".to_string(),
1069 original_uri: "memory://".to_string(),
1070 object_store: Arc::new(InMemory::new()),
1071 session: DataBackendSession::new(5_000),
1072 batch_size: 5_000,
1073 compression: parquet::basic::Compression::SNAPPY,
1074 max_row_group_size: 5_000,
1075 };
1076
1077 let batch = RecordBatch::try_new(
1078 Arc::new(Schema::new(vec![Field::new(
1079 "ts_init",
1080 DataType::Int64,
1081 false,
1082 )])),
1083 vec![Arc::new(Int64Array::from(vec![1]))],
1084 )
1085 .unwrap();
1086
1087 let path = catalog
1088 .write_parquet_file_checked(
1089 "quotes/TEST",
1090 UnixNanos::from(1),
1091 UnixNanos::from(1),
1092 &[batch],
1093 false,
1094 "Promoted file",
1095 None,
1096 Some("memory-replay"),
1097 )
1098 .unwrap();
1099 let object_path = catalog.to_object_path(&path.to_string_lossy()).unwrap();
1100 catalog
1101 .execute_async(|| async {
1102 catalog.object_store.head(&object_path).await?;
1103 Ok(())
1104 })
1105 .unwrap();
1106 }
1107
1108 #[rstest]
1109 fn promotion_accepts_already_exists_after_head_miss() {
1110 let object_store = Arc::new(CreateRaceStore {
1111 inner: InMemory::new(),
1112 create_calls: AtomicUsize::new(0),
1113 already_exists: true,
1114 fail_head: AtomicBool::new(false),
1115 });
1116
1117 let catalog = ParquetDataCatalog {
1118 base_path: "catalog".to_string(),
1119 original_uri: "memory://".to_string(),
1120 object_store: object_store.clone(),
1121 session: DataBackendSession::new(5_000),
1122 batch_size: 5_000,
1123 compression: parquet::basic::Compression::SNAPPY,
1124 max_row_group_size: 5_000,
1125 };
1126
1127 let batch = RecordBatch::try_new(
1128 Arc::new(Schema::new(vec![Field::new(
1129 "ts_init",
1130 DataType::Int64,
1131 false,
1132 )])),
1133 vec![Arc::new(Int64Array::from(vec![1]))],
1134 )
1135 .unwrap();
1136
1137 catalog
1138 .write_parquet_file_checked(
1139 "quotes/TEST",
1140 UnixNanos::from(1),
1141 UnixNanos::from(1),
1142 &[batch],
1143 true,
1144 "Promoted file",
1145 None,
1146 Some("racing-replay"),
1147 )
1148 .unwrap();
1149
1150 assert_eq!(object_store.create_calls.load(Ordering::Relaxed), 1);
1151 }
1152
1153 #[rstest]
1154 fn promotion_propagates_create_errors_other_than_already_exists() {
1155 let object_store = Arc::new(CreateRaceStore {
1156 inner: InMemory::new(),
1157 create_calls: AtomicUsize::new(0),
1158 already_exists: false,
1159 fail_head: AtomicBool::new(false),
1160 });
1161
1162 let catalog = ParquetDataCatalog {
1163 base_path: "catalog".to_string(),
1164 original_uri: "memory://".to_string(),
1165 object_store: object_store.clone(),
1166 session: DataBackendSession::new(5_000),
1167 batch_size: 5_000,
1168 compression: parquet::basic::Compression::SNAPPY,
1169 max_row_group_size: 5_000,
1170 };
1171
1172 let batch = RecordBatch::try_new(
1173 Arc::new(Schema::new(vec![Field::new(
1174 "ts_init",
1175 DataType::Int64,
1176 false,
1177 )])),
1178 vec![Arc::new(Int64Array::from(vec![1]))],
1179 )
1180 .unwrap();
1181
1182 let error = catalog
1183 .write_parquet_file_checked(
1184 "quotes/TEST",
1185 UnixNanos::from(1),
1186 UnixNanos::from(1),
1187 &[batch],
1188 true,
1189 "Promoted file",
1190 None,
1191 Some("failing-replay"),
1192 )
1193 .unwrap_err();
1194
1195 assert!(
1196 format!("{error:#}").contains("injected create failure"),
1197 "unexpected error: {error:#}"
1198 );
1199 assert_eq!(object_store.create_calls.load(Ordering::Relaxed), 1);
1200 }
1201
1202 #[rstest]
1203 #[case::head_failure(true, None)]
1204 #[case::missing_file(false, Some(false))]
1205 fn file_exists_propagates_storage_errors_other_than_not_found(
1206 #[case] fail_head: bool,
1207 #[case] expected: Option<bool>,
1208 ) {
1209 let catalog = ParquetDataCatalog {
1210 base_path: "catalog".to_string(),
1211 original_uri: "memory://".to_string(),
1212 object_store: Arc::new(CreateRaceStore {
1213 inner: InMemory::new(),
1214 create_calls: AtomicUsize::new(0),
1215 already_exists: true,
1216 fail_head: AtomicBool::new(fail_head),
1217 }),
1218 session: DataBackendSession::new(5_000),
1219 batch_size: 5_000,
1220 compression: parquet::basic::Compression::SNAPPY,
1221 max_row_group_size: 5_000,
1222 };
1223
1224 let result = catalog.file_exists("quotes/TEST/missing.parquet");
1225
1226 match expected {
1227 Some(exists) => assert_eq!(result.unwrap(), exists),
1228 None => assert!(
1229 format!("{:#}", result.unwrap_err()).contains("injected head failure"),
1230 "head failure must propagate"
1231 ),
1232 }
1233 }
1234
1235 #[rstest]
1236 fn write_propagates_head_failure_instead_of_writing() {
1237 let object_store = Arc::new(CreateRaceStore {
1238 inner: InMemory::new(),
1239 create_calls: AtomicUsize::new(0),
1240 already_exists: true,
1241 fail_head: AtomicBool::new(true),
1242 });
1243
1244 let catalog = ParquetDataCatalog {
1245 base_path: "catalog".to_string(),
1246 original_uri: "memory://".to_string(),
1247 object_store: object_store.clone(),
1248 session: DataBackendSession::new(5_000),
1249 batch_size: 5_000,
1250 compression: parquet::basic::Compression::SNAPPY,
1251 max_row_group_size: 5_000,
1252 };
1253
1254 let batch = RecordBatch::try_new(
1255 Arc::new(Schema::new(vec![Field::new(
1256 "ts_init",
1257 DataType::Int64,
1258 false,
1259 )])),
1260 vec![Arc::new(Int64Array::from(vec![1]))],
1261 )
1262 .unwrap();
1263
1264 let error = catalog
1265 .write_parquet_file_checked(
1266 "quotes/TEST",
1267 UnixNanos::from(1),
1268 UnixNanos::from(1),
1269 &[batch],
1270 true,
1271 "Data file",
1272 None,
1273 None,
1274 )
1275 .unwrap_err();
1276
1277 let stored = catalog
1278 .execute_async(|| async {
1279 Ok(object_store
1280 .inner
1281 .list(None)
1282 .collect::<Vec<_>>()
1283 .await
1284 .len())
1285 })
1286 .unwrap();
1287
1288 assert!(
1289 format!("{error:#}").contains("injected head failure"),
1290 "unexpected error: {error:#}"
1291 );
1292 assert_eq!(stored, 0);
1293 }
1294
1295 #[rstest]
1296 fn period_consolidation_stops_on_head_failure() {
1297 let object_store = Arc::new(CreateRaceStore {
1298 inner: InMemory::new(),
1299 create_calls: AtomicUsize::new(0),
1300 already_exists: true,
1301 fail_head: AtomicBool::new(false),
1302 });
1303
1304 let mut catalog = ParquetDataCatalog {
1305 base_path: String::new(),
1306 original_uri: "s3://test-bucket".to_string(),
1307 object_store: object_store.clone(),
1308 session: DataBackendSession::new(5_000),
1309 batch_size: 5_000,
1310 compression: parquet::basic::Compression::SNAPPY,
1311 max_row_group_size: 5_000,
1312 };
1313
1314 let quote = |ts: u64| {
1315 QuoteTick::new(
1316 InstrumentId::from("ETH/USDT.BINANCE"),
1317 Price::from("1.0001"),
1318 Price::from("1.0002"),
1319 Quantity::from("100"),
1320 Quantity::from("100"),
1321 UnixNanos::from(ts),
1322 UnixNanos::from(ts),
1323 )
1324 };
1325
1326 catalog
1327 .write_to_parquet(&[quote(1)], None, None, None)
1328 .unwrap();
1329 catalog
1330 .write_to_parquet(&[quote(2)], None, None, None)
1331 .unwrap();
1332 object_store.fail_head.store(true, Ordering::Relaxed);
1333
1334 let error = catalog
1335 .consolidate_data_by_period(
1336 &NautilusDataType::QuoteTick.into(),
1337 Some("ETH/USDT.BINANCE"),
1338 Some(86_400_000_000_000),
1339 None,
1340 None,
1341 Some(true),
1342 )
1343 .unwrap_err();
1344
1345 let files = catalog
1346 .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
1347 .unwrap();
1348 assert!(
1349 format!("{error:#}").contains("injected head failure"),
1350 "unexpected error: {error:#}"
1351 );
1352 assert_eq!(files.len(), 2);
1353 }
1354
1355 #[rstest]
1356 fn write_to_json_normalizes_windows_separators() {
1357 let catalog = ParquetDataCatalog {
1358 base_path: String::new(),
1359 original_uri: "memory://".to_string(),
1360 object_store: Arc::new(InMemory::new()),
1361 session: DataBackendSession::new(5_000),
1362 batch_size: 5_000,
1363 compression: parquet::basic::Compression::SNAPPY,
1364 max_row_group_size: 5_000,
1365 };
1366
1367 let quote = QuoteTick::new(
1368 InstrumentId::from("ETH/USDT.BINANCE"),
1369 Price::from("1.0001"),
1370 Price::from("1.0002"),
1371 Quantity::from("100"),
1372 Quantity::from("100"),
1373 UnixNanos::from(1),
1374 UnixNanos::from(1),
1375 );
1376
1377 catalog
1378 .write_to_json(vec![quote], Some(r"data\quotes".into()), true)
1379 .unwrap();
1380
1381 let mut keys = catalog
1382 .execute_async(|| async {
1383 Ok(catalog
1384 .object_store
1385 .list(None)
1386 .map_ok(|object| object.location.to_string())
1387 .try_collect::<Vec<_>>()
1388 .await?)
1389 })
1390 .unwrap();
1391
1392 keys.sort();
1393
1394 assert_eq!(
1395 keys,
1396 vec![
1397 "data/quotes/1970-01-01T00-00-00-000000001Z_1970-01-01T00-00-00-000000001Z.json",
1398 "data/quotes/1970-01-01T00-00-00-000000001Z_1970-01-01T00-00-00-000000001Z.metadata.json",
1399 ]
1400 );
1401 }
1402}