1#![expect(
17 clippy::missing_panics_doc,
18 reason = "Parquet I/O functions use validated schema paths"
19)]
20
21use std::{collections::HashMap, sync::Arc};
22
23use ahash::AHashMap;
24use anyhow::Context;
25use arrow::{
26 array::UInt64Array,
27 compute::{SortColumn, SortOptions, concat_batches, lexsort_to_indices, take_record_batch},
28 datatypes::Schema,
29 record_batch::RecordBatch,
30};
31use nautilus_serialization::arrow::{
32 KEY_IDENTIFIER, KEY_PRICE_PRECISION, KEY_SIZE_PRECISION, U64ColumnRef,
33};
34use object_store::{
35 ObjectMeta, ObjectStore, ObjectStoreExt, PutMode, PutOptions, buffered::BufReader,
36 path::Path as ObjectPath,
37};
38use parquet::{
39 arrow::{
40 ArrowSchemaConverter, ArrowWriter, ParquetRecordBatchStreamBuilder,
41 arrow_reader::ParquetRecordBatchReaderBuilder,
42 },
43 basic::{Compression, Encoding, ZstdLevel},
44 file::{
45 metadata::{KeyValue, SortingColumn},
46 properties::WriterProperties,
47 reader::{FileReader, SerializedFileReader},
48 statistics::Statistics,
49 },
50 schema::types::ColumnPath,
51};
52use url::Url;
53
54use super::consolidation_state;
55pub(crate) use crate::common::paths::file_uri_to_native_path;
56pub use crate::common::paths::normalize_path_to_uri;
57
58pub(crate) struct ObjectStoreLocation {
59 pub object_store: Arc<dyn ObjectStore>,
60 pub base_path: String,
61 pub original_uri: String,
62}
63
64pub async fn read_parquet_schema_from_object_store(
72 object_store: Arc<dyn ObjectStore>,
73 path: &ObjectPath,
74) -> anyhow::Result<Arc<Schema>> {
75 let object = object_store.head(path).await?;
76 if object.size == 0 {
77 return Ok(Arc::new(Schema::empty()));
78 }
79
80 let reader = BufReader::new(object_store, &object);
81 let builder = ParquetRecordBatchStreamBuilder::new(reader).await?;
82 Ok(builder.schema().clone())
83}
84
85pub async fn read_parquet_from_object_store(
94 object_store: Arc<dyn ObjectStore>,
95 path: &ObjectPath,
96) -> anyhow::Result<(Vec<RecordBatch>, Arc<Schema>)> {
97 let data = object_store.get(path).await?.bytes().await?;
98 if data.is_empty() {
99 return Ok((Vec::new(), Arc::new(Schema::empty())));
100 }
101
102 let builder = ParquetRecordBatchReaderBuilder::try_new(data)?;
103 let schema = builder.schema().clone();
104 let batches = builder.build()?.collect::<Result<Vec<_>, _>>()?;
105
106 Ok((batches, schema))
107}
108
109pub async fn write_batches_to_object_store(
116 batches: &[RecordBatch],
117 object_store: Arc<dyn ObjectStore>,
118 path: &ObjectPath,
119 compression: Option<Compression>,
120 max_row_group_size: Option<usize>,
121 key_value_metadata: Option<Vec<KeyValue>>,
122) -> anyhow::Result<()> {
123 write_batches_to_object_store_with_mode(
124 batches,
125 object_store,
126 path,
127 compression,
128 max_row_group_size,
129 key_value_metadata,
130 PutMode::Overwrite,
131 None,
132 )
133 .await
134}
135
136pub(crate) async fn write_batches_to_object_store_create(
137 batches: &[RecordBatch],
138 object_store: Arc<dyn ObjectStore>,
139 path: &ObjectPath,
140 compression: Option<Compression>,
141 max_row_group_size: Option<usize>,
142 key_value_metadata: Option<Vec<KeyValue>>,
143) -> anyhow::Result<()> {
144 write_batches_to_object_store_with_mode(
145 batches,
146 object_store,
147 path,
148 compression,
149 max_row_group_size,
150 key_value_metadata,
151 PutMode::Create,
152 None,
153 )
154 .await
155}
156
157#[expect(
158 clippy::too_many_arguments,
159 reason = "private writer keeps storage mode and footer identity separate"
160)]
161async fn write_batches_to_object_store_with_mode(
162 batches: &[RecordBatch],
163 object_store: Arc<dyn ObjectStore>,
164 path: &ObjectPath,
165 compression: Option<Compression>,
166 max_row_group_size: Option<usize>,
167 key_value_metadata: Option<Vec<KeyValue>>,
168 put_mode: PutMode,
169 created_by: Option<String>,
170) -> anyhow::Result<()> {
171 let Some(first) = batches.first() else {
172 anyhow::bail!("Cannot write Parquet file {path} with no record batches");
173 };
174
175 let mut buffer = Vec::new();
177
178 let schema = first.schema();
179 let sorting_columns = parquet_sorting_columns(schema.as_ref())?;
180 let mut props_builder = WriterProperties::builder()
181 .set_compression(compression.unwrap_or(Compression::ZSTD(ZstdLevel::default())))
182 .set_max_row_group_row_count(Some(
183 max_row_group_size.unwrap_or(super::DEFAULT_ROW_GROUP_SIZE),
184 ))
185 .set_sorting_columns(sorting_columns)
186 .set_key_value_metadata(key_value_metadata);
187
188 if let Some(created_by) = created_by {
189 props_builder = props_builder.set_created_by(created_by);
190 }
191
192 if schema.index_of(KEY_IDENTIFIER).is_ok() {
193 props_builder =
194 props_builder.set_column_bloom_filter_enabled(ColumnPath::from(KEY_IDENTIFIER), true);
195 }
196
197 for name in ["ts_event", "ts_init"] {
199 if schema.index_of(name).is_ok() {
200 props_builder = props_builder
201 .set_column_dictionary_enabled(ColumnPath::from(name), false)
202 .set_column_encoding(ColumnPath::from(name), Encoding::DELTA_BINARY_PACKED);
203 }
204 }
205
206 let writer_props = props_builder.build();
207
208 let mut writer = ArrowWriter::try_new(&mut buffer, schema, Some(writer_props))?;
209 for batch in batches {
210 writer.write(batch)?;
211 }
212
213 writer.close()?;
214
215 object_store
217 .put_opts(
218 path,
219 buffer.into(),
220 PutOptions {
221 mode: put_mode,
222 ..Default::default()
223 },
224 )
225 .await?;
226
227 Ok(())
228}
229
230fn parquet_sorting_columns(schema: &Schema) -> anyhow::Result<Option<Vec<SortingColumn>>> {
231 if schema.index_of("ts_init").is_err() {
232 return Ok(None);
233 }
234
235 let parquet_schema = ArrowSchemaConverter::new().convert(schema)?;
236 let mut names = Vec::with_capacity(2);
237 names.push("ts_init");
238 if schema.index_of(KEY_IDENTIFIER).is_ok() {
239 names.push(KEY_IDENTIFIER);
240 }
241
242 let columns = names
243 .into_iter()
244 .map(|name| {
245 let index = parquet_schema
246 .columns()
247 .iter()
248 .position(|column| {
249 column
250 .path()
251 .parts()
252 .first()
253 .is_some_and(|part| part == name)
254 })
255 .ok_or_else(|| anyhow::anyhow!("Parquet schema is missing sort column {name}"))?;
256
257 Ok(SortingColumn {
258 column_idx: i32::try_from(index)?,
259 descending: false,
260 nulls_first: false,
261 })
262 })
263 .collect::<anyhow::Result<Vec<_>>>()?;
264
265 Ok(Some(columns))
266}
267
268fn deduplicate_record_batches(batches: &[RecordBatch]) -> anyhow::Result<Vec<RecordBatch>> {
278 if batches.is_empty() {
279 return Ok(Vec::new());
280 }
281
282 let schema = batches[0].schema();
283
284 let fields: Vec<arrow::row::SortField> = schema
285 .fields()
286 .iter()
287 .map(|f| arrow::row::SortField::new(f.data_type().clone()))
288 .collect();
289
290 let converter = arrow::row::RowConverter::new(fields)?;
291 let mut seen: std::collections::HashSet<Vec<u8>> = std::collections::HashSet::new();
292 let mut result: Vec<RecordBatch> = Vec::new();
293
294 for batch in batches {
295 let rows = converter.convert_columns(batch.columns())?;
296 let mut indices: Vec<u32> = Vec::new();
297
298 for (i, row) in rows.iter().enumerate() {
299 if seen.insert(row.as_ref().to_vec()) {
300 indices.push(u32::try_from(i)?);
301 }
302 }
303
304 if !indices.is_empty() {
305 let index_array = arrow::array::UInt32Array::from(indices);
306 let deduped_columns: Vec<arrow::array::ArrayRef> = batch
307 .columns()
308 .iter()
309 .map(|col| arrow::compute::take(col.as_ref(), &index_array, None))
310 .collect::<Result<_, _>>()?;
311 result.push(RecordBatch::try_new(schema.clone(), deduped_columns)?);
312 }
313 }
314
315 Ok(result)
316}
317
318pub async fn combine_parquet_files_from_object_store(
324 object_store: Arc<dyn ObjectStore>,
325 file_paths: Vec<ObjectPath>,
326 new_file_path: &ObjectPath,
327 compression: Option<Compression>,
328 max_row_group_size: Option<usize>,
329 deduplicate: Option<bool>,
330) -> anyhow::Result<()> {
331 let Some(merged) =
332 read_combined_batches(object_store.clone(), &file_paths, deduplicate).await?
333 else {
334 return Ok(());
335 };
336
337 write_batches_to_object_store(
338 &merged.batches,
339 object_store.clone(),
340 new_file_path,
341 compression,
342 max_row_group_size,
343 None,
344 )
345 .await?;
346
347 for source in merged.sources {
348 if source.location != *new_file_path {
349 object_store.delete(&source.location).await?;
350 }
351 }
352
353 Ok(())
354}
355
356pub(crate) async fn consolidate_parquet_files_from_object_store(
357 object_store: Arc<dyn ObjectStore>,
358 file_paths: Vec<ObjectPath>,
359 new_file_path: &ObjectPath,
360 compression: Option<Compression>,
361 max_row_group_size: Option<usize>,
362 deduplicate: Option<bool>,
363) -> anyhow::Result<()> {
364 let Some(merged) =
365 read_combined_batches(object_store.clone(), &file_paths, deduplicate).await?
366 else {
367 return Ok(());
368 };
369
370 let created_by =
371 consolidation_state::begin(&object_store, new_file_path, &merged.sources).await?;
372 write_batches_to_object_store_with_mode(
373 &merged.batches,
374 object_store.clone(),
375 new_file_path,
376 compression,
377 max_row_group_size,
378 None,
379 PutMode::Overwrite,
380 Some(created_by),
381 )
382 .await?;
383 let directory = new_file_path.parent().unwrap_or_default();
384 consolidation_state::recover(&object_store, &directory).await
385}
386
387struct MergedParquet {
388 batches: Vec<RecordBatch>,
389 sources: Vec<ObjectMeta>,
390}
391
392async fn read_combined_batches(
393 object_store: Arc<dyn ObjectStore>,
394 file_paths: &[ObjectPath],
395 deduplicate: Option<bool>,
396) -> anyhow::Result<Option<MergedParquet>> {
397 if file_paths.len() <= 1 {
398 return Ok(None);
399 }
400
401 let mut all_batches: Vec<RecordBatch> = Vec::new();
402 let mut schema_with_metadata: Option<Arc<arrow::datatypes::Schema>> = None;
403 let mut schema_source: Option<&ObjectPath> = None;
404 let mut field_metadata_sources = HashMap::new();
405 let mut sources = Vec::with_capacity(file_paths.len());
406
407 for path in file_paths {
409 let object = object_store.get(path).await?;
410 sources.push(object.meta.clone());
411 let data = object.bytes().await?;
412 let builder = ParquetRecordBatchReaderBuilder::try_new(data)?;
413
414 let candidate_schema = builder.schema().clone();
415 schema_with_metadata = Some(
416 if let (Some(schema), Some(source)) = (&schema_with_metadata, schema_source) {
417 let reconciled = reconcile_consolidation_schema_with_sources(
418 schema,
419 source,
420 &field_metadata_sources,
421 &candidate_schema,
422 path,
423 )?;
424
425 if reconciled.schema_source == ConsolidationSchemaSource::Candidate {
426 schema_source = Some(path);
427 }
428
429 for key in reconciled.candidate_field_metadata {
430 field_metadata_sources.insert(key, path.clone());
431 }
432
433 reconciled.schema
434 } else {
435 schema_source = Some(path);
436 field_metadata_sources
437 .extend(field_metadata_keys(&candidate_schema).map(|key| (key, path.clone())));
438 candidate_schema
439 },
440 );
441
442 for batch in builder.build()? {
443 all_batches.push(batch?);
444 }
445 }
446
447 let schema = schema_with_metadata.expect("non-empty source files provide a schema");
451 all_batches = all_batches
452 .into_iter()
453 .map(|b| RecordBatch::try_new(schema.clone(), b.columns().to_vec()))
454 .collect::<Result<Vec<_>, _>>()?;
455
456 let batches_to_write = if deduplicate.unwrap_or(false) {
458 deduplicate_record_batches(&all_batches)?
459 } else {
460 all_batches
461 };
462
463 let batches_to_write = sort_consolidation_batches(batches_to_write)?;
464 Ok(Some(MergedParquet {
465 batches: batches_to_write,
466 sources,
467 }))
468}
469
470fn sort_consolidation_batches(batches: Vec<RecordBatch>) -> anyhow::Result<Vec<RecordBatch>> {
471 let Some(first) = batches.first() else {
472 return Ok(batches);
473 };
474
475 if first.column_by_name("ts_init").is_none() {
476 return Ok(batches);
477 }
478
479 let mut previous = 0;
480 let mut ordered = true;
481
482 for batch in &batches {
483 let array = batch
484 .column_by_name("ts_init")
485 .expect("consistent batch schema");
486 let timestamps = U64ColumnRef::try_from_array(array.as_ref())
487 .ok_or_else(|| anyhow::anyhow!("Unsupported consolidation ts_init type"))?;
488
489 for row in 0..batch.num_rows() {
490 let timestamp = timestamps
491 .value(row)
492 .ok_or_else(|| anyhow::anyhow!("Invalid consolidation ts_init value"))?;
493 ordered &= previous <= timestamp;
494 previous = timestamp;
495 }
496 }
497
498 if ordered {
499 return Ok(batches);
500 }
501
502 let schema = first.schema();
503 let batch = concat_batches(&schema, &batches)?;
504 let original_index = Arc::new(UInt64Array::from_iter_values(0..batch.num_rows() as u64));
505
506 let options = Some(SortOptions {
507 descending: false,
508 nulls_first: false,
509 });
510
511 let indices = lexsort_to_indices(
512 &[
513 SortColumn {
514 values: batch
515 .column_by_name("ts_init")
516 .expect("ts_init exists")
517 .clone(),
518 options,
519 },
520 SortColumn {
521 values: original_index,
522 options,
523 },
524 ],
525 None,
526 )?;
527
528 Ok(vec![take_record_batch(&batch, &indices)?])
529}
530
531#[derive(Clone, Copy, Debug, Eq, PartialEq)]
532enum ConsolidationSchemaSource {
533 Current,
534 Candidate,
535}
536
537#[derive(Debug)]
538struct ReconciledConsolidationSchema {
539 schema: Arc<Schema>,
540 schema_source: ConsolidationSchemaSource,
541 candidate_field_metadata: Vec<(String, String)>,
542}
543
544fn reconcile_consolidation_schema_with_sources(
545 current: &Arc<Schema>,
546 current_path: &ObjectPath,
547 current_field_metadata_sources: &HashMap<(String, String), ObjectPath>,
548 candidate: &Arc<Schema>,
549 candidate_path: &ObjectPath,
550) -> anyhow::Result<ReconciledConsolidationSchema> {
551 anyhow::ensure!(
552 current.fields().len() == candidate.fields().len(),
553 "Cannot consolidate Parquet files {current_path} and {candidate_path}: field schemas differ"
554 );
555 let mut candidate_field_metadata = Vec::new();
556
557 let fields = current
558 .fields()
559 .iter()
560 .zip(candidate.fields())
561 .map(|(current, candidate)| {
562 anyhow::ensure!(
563 current.name() == candidate.name()
564 && current.data_type() == candidate.data_type()
565 && current.is_nullable() == candidate.is_nullable(),
566 "Cannot consolidate Parquet files {current_path} and {candidate_path}: field schemas differ"
567 );
568 let mut metadata = current.metadata().clone();
569 for (key, value) in candidate.metadata() {
570 if let Some(current_value) = metadata.get(key) {
571 let source = current_field_metadata_sources
572 .get(&(current.name().clone(), key.clone()))
573 .unwrap_or(current_path);
574 anyhow::ensure!(
575 current_value == value,
576 "Cannot consolidate Parquet files {source} and {candidate_path}: field '{}' metadata differs",
577 current.name(),
578 );
579 } else {
580 metadata.insert(key.clone(), value.clone());
581 candidate_field_metadata.push((current.name().clone(), key.clone()));
582 }
583 }
584
585 Ok(Arc::new(current.as_ref().clone().with_metadata(metadata)))
586 })
587 .collect::<anyhow::Result<Vec<_>>>()?;
588
589 let schema_with_fields =
590 |metadata| Arc::new(Schema::new_with_metadata(fields.clone(), metadata));
591
592 if current.metadata() == candidate.metadata() {
593 return Ok(ReconciledConsolidationSchema {
594 schema: schema_with_fields(current.metadata().clone()),
595 schema_source: ConsolidationSchemaSource::Current,
596 candidate_field_metadata,
597 });
598 }
599
600 let without_precision = |schema: &Schema| {
601 let mut metadata = schema.metadata().clone();
602 metadata.remove(KEY_PRICE_PRECISION);
603 metadata.remove(KEY_SIZE_PRECISION);
604 metadata
605 };
606
607 let is_precision_fallback = |metadata: &HashMap<String, String>| {
608 metadata.get(KEY_PRICE_PRECISION).map(String::as_str) == Some("0")
609 && metadata.get(KEY_SIZE_PRECISION).map(String::as_str) == Some("0")
610 };
611
612 let has_precision = |metadata: &HashMap<String, String>| {
613 metadata.contains_key(KEY_PRICE_PRECISION) && metadata.contains_key(KEY_SIZE_PRECISION)
614 };
615
616 let current_metadata = current.metadata();
617 let candidate_metadata = candidate.metadata();
618
619 if without_precision(current) == without_precision(candidate) {
620 match (
621 is_precision_fallback(current_metadata),
622 is_precision_fallback(candidate_metadata),
623 ) {
624 (true, false) if has_precision(candidate_metadata) => {
625 return Ok(ReconciledConsolidationSchema {
626 schema: schema_with_fields(candidate.metadata().clone()),
627 schema_source: ConsolidationSchemaSource::Candidate,
628 candidate_field_metadata,
629 });
630 }
631 (false, true) if has_precision(current_metadata) => {
632 return Ok(ReconciledConsolidationSchema {
633 schema: schema_with_fields(current.metadata().clone()),
634 schema_source: ConsolidationSchemaSource::Current,
635 candidate_field_metadata,
636 });
637 }
638 _ => {}
639 }
640 }
641
642 anyhow::bail!(
643 "Cannot consolidate Parquet files {current_path} and {candidate_path}: schema metadata differs: {current_metadata:?} versus {candidate_metadata:?}"
644 )
645}
646
647fn field_metadata_keys(schema: &Schema) -> impl Iterator<Item = (String, String)> + '_ {
648 schema.fields().iter().flat_map(|field| {
649 field
650 .metadata()
651 .keys()
652 .map(|key| (field.name().clone(), key.clone()))
653 })
654}
655
656pub async fn min_max_from_parquet_metadata_object_store(
662 object_store: Arc<dyn ObjectStore>,
663 file_path: &ObjectPath,
664 column_name: &str,
665) -> anyhow::Result<(u64, u64)> {
666 let data = object_store.get(file_path).await?.bytes().await?;
668 let reader = SerializedFileReader::new(data)?;
669
670 let metadata = reader.metadata();
671 let mut overall_min_value: Option<i64> = None;
672 let mut overall_max_value: Option<i64> = None;
673
674 for i in 0..metadata.num_row_groups() {
676 let row_group = metadata.row_group(i);
677
678 for j in 0..row_group.num_columns() {
680 let col_metadata = row_group.column(j);
681
682 if col_metadata.column_path().string() == column_name {
683 if let Some(stats) = col_metadata.statistics() {
684 if let Statistics::Int64(int64_stats) = stats {
686 if let Some(&min_value) = int64_stats.min_opt()
688 && (overall_min_value.is_none()
689 || min_value < overall_min_value.unwrap())
690 {
691 overall_min_value = Some(min_value);
692 }
693
694 if let Some(&max_value) = int64_stats.max_opt()
696 && (overall_max_value.is_none()
697 || max_value > overall_max_value.unwrap())
698 {
699 overall_max_value = Some(max_value);
700 }
701 } else {
702 anyhow::bail!("Warning: Column name '{column_name}' is not of type i64.");
703 }
704 } else {
705 anyhow::bail!(
706 "Warning: Statistics not available for column '{column_name}' in row group {i}."
707 );
708 }
709 }
710 }
711 }
712
713 if let (Some(min), Some(max)) = (overall_min_value, overall_max_value) {
715 Ok((u64::try_from(min)?, u64::try_from(max)?))
716 } else {
717 anyhow::bail!(
718 "Column '{column_name}' not found or has no Int64 statistics in any row group."
719 )
720 }
721}
722
723pub fn create_object_store_from_path(
750 path: &str,
751 storage_options: Option<AHashMap<String, String>>,
752) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
753 let location = create_object_store_location_from_path(path, storage_options)?;
754 Ok((
755 location.object_store,
756 location.base_path,
757 location.original_uri,
758 ))
759}
760
761#[cfg_attr(
764 not(feature = "cloud"),
765 allow(unused_variables, clippy::needless_pass_by_value)
766)]
767pub(crate) fn create_object_store_location_from_path(
768 path: &str,
769 storage_options: Option<AHashMap<String, String>>,
770) -> anyhow::Result<ObjectStoreLocation> {
771 let uri = normalize_path_to_uri(path)?;
772
773 let (object_store, base_path, original_uri) = match uri.as_str() {
774 #[cfg(feature = "cloud")]
775 s if s.starts_with("s3://") => create_s3_store(&uri, storage_options),
776 #[cfg(feature = "cloud")]
777 s if s.starts_with("gs://") || s.starts_with("gcs://") => {
778 create_gcs_store(&uri, storage_options)
779 }
780 #[cfg(feature = "cloud")]
781 s if s.starts_with("az://") => create_azure_store(&uri, storage_options),
782 #[cfg(feature = "cloud")]
783 s if s.starts_with("abfs://") => create_abfs_store(&uri, storage_options),
784 #[cfg(feature = "cloud")]
785 s if s.starts_with("http://") || s.starts_with("https://") => {
786 create_http_store(&uri, storage_options)
787 }
788 #[cfg(not(feature = "cloud"))]
789 s if s.starts_with("s3://")
790 || s.starts_with("gs://")
791 || s.starts_with("gcs://")
792 || s.starts_with("az://")
793 || s.starts_with("abfs://")
794 || s.starts_with("http://")
795 || s.starts_with("https://") =>
796 {
797 anyhow::bail!("Cloud storage support requires the 'cloud' feature: {uri}")
798 }
799 s if s.starts_with("file://") => create_local_store(&uri, true),
800 _ => create_local_store(&uri, false), }?;
802
803 Ok(ObjectStoreLocation {
804 object_store,
805 base_path,
806 original_uri,
807 })
808}
809
810pub(crate) fn is_remote_uri_scheme(scheme: &str) -> bool {
811 matches!(
812 scheme,
813 "s3" | "gs" | "gcs" | "az" | "abfs" | "http" | "https"
814 )
815}
816
817pub(crate) fn remote_store_root_url(uri: &str) -> anyhow::Result<Url> {
818 let mut url = Url::parse(uri)?;
819 url.set_path("");
820 url.set_query(None);
821 url.set_fragment(None);
822 Ok(url)
823}
824
825pub(crate) fn remote_full_uri(uri: &str, object_path: &str) -> anyhow::Result<String> {
826 let root = remote_store_root_url(uri)?;
827 let root = root.as_str().trim_end_matches('/');
828 let object_path = object_path.trim_start_matches('/');
829
830 if object_path.is_empty() {
831 Ok(root.to_string())
832 } else {
833 Ok(format!("{root}/{object_path}"))
834 }
835}
836
837pub(crate) fn append_path_to_file_uri(base_uri: &str, path: &str) -> String {
840 if let Ok(mut url) = Url::parse(base_uri) {
841 if let Ok(mut segments) = url.path_segments_mut() {
842 segments.pop_if_empty();
843 segments.extend(
844 path.trim_end_matches('/')
845 .split('/')
846 .filter(|segment| !segment.is_empty()),
847 );
848 }
849
850 return url.to_string();
851 }
852
853 format!(
854 "{}/{}",
855 base_uri.trim_end_matches('/'),
856 path.trim_end_matches('/')
857 )
858}
859
860pub(crate) fn decode_object_store_segment(segment: &str) -> String {
867 ObjectPath::from_url_path(segment).map_or_else(|_| segment.to_string(), String::from)
868}
869
870fn create_local_store(
871 uri: &str,
872 is_file_uri: bool,
873) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
874 let path = if is_file_uri {
875 file_uri_to_native_path(uri)
876 } else {
877 uri.to_string()
878 };
879
880 let local_store =
881 object_store::local::LocalFileSystem::new_with_prefix(&path).with_context(|| {
882 format!(
883 "failed to open local storage directory '{path}'; \
884 create it if it does not exist and check access permissions"
885 )
886 })?;
887
888 Ok((Arc::new(local_store), String::new(), uri.to_string()))
889}
890
891#[cfg(feature = "cloud")]
893fn create_s3_store(
894 uri: &str,
895 storage_options: Option<AHashMap<String, String>>,
896) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
897 let (url, path) = parse_url_and_path(uri)?;
898 let bucket = extract_host(&url, "Invalid S3 URI: missing bucket")?;
899
900 let mut builder = object_store::aws::AmazonS3Builder::new().with_bucket_name(&bucket);
901
902 if let Some(options) = storage_options {
904 for (key, value) in options {
905 let config_key = match key.as_str() {
907 "key" => object_store::aws::AmazonS3ConfigKey::AccessKeyId,
908 "secret" => object_store::aws::AmazonS3ConfigKey::SecretAccessKey,
909 _ => key.parse()?,
910 };
911 builder = builder.with_config(config_key, value);
912 }
913 }
914
915 let s3_store = builder.build()?;
916 Ok((Arc::new(s3_store), path, uri.to_string()))
917}
918
919#[cfg(feature = "cloud")]
921fn create_gcs_store(
922 uri: &str,
923 storage_options: Option<AHashMap<String, String>>,
924) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
925 let (url, path) = parse_url_and_path(uri)?;
926 let bucket = extract_host(&url, "Invalid GCS URI: missing bucket")?;
927
928 let mut builder = object_store::gcp::GoogleCloudStorageBuilder::new().with_bucket_name(&bucket);
929
930 if let Some(options) = storage_options {
932 for (key, value) in options {
933 builder =
934 builder.with_config(key.parse::<object_store::gcp::GoogleConfigKey>()?, value);
935 }
936 }
937
938 let gcs_store = builder.build()?;
939 Ok((Arc::new(gcs_store), path, uri.to_string()))
940}
941
942#[cfg(feature = "cloud")]
944fn create_azure_store(
945 uri: &str,
946 storage_options: Option<AHashMap<String, String>>,
947) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
948 let (url, path) = parse_url_and_path(uri)?;
949 let container = extract_host(&url, "Invalid Azure URI: missing container")?;
950
951 let mut builder =
952 object_store::azure::MicrosoftAzureBuilder::new().with_container_name(container);
953
954 if let Some(options) = storage_options {
956 builder = apply_azure_storage_options(builder, options)?;
957 }
958
959 let azure_store = builder.build()?;
960 Ok((Arc::new(azure_store), path, uri.to_string()))
961}
962
963#[cfg(feature = "cloud")]
965fn create_abfs_store(
966 uri: &str,
967 storage_options: Option<AHashMap<String, String>>,
968) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
969 let (url, path) = parse_url_and_path(uri)?;
970 let host = extract_host(&url, "Invalid ABFS URI: missing host")?;
971
972 let account = host
974 .split('.')
975 .next()
976 .ok_or_else(|| anyhow::anyhow!("Invalid ABFS URI: cannot extract account from host"))?;
977
978 let container = url
980 .username()
981 .split('@')
982 .next()
983 .ok_or_else(|| anyhow::anyhow!("Invalid ABFS URI: missing container"))?;
984
985 let mut builder = object_store::azure::MicrosoftAzureBuilder::new()
986 .with_account(account)
987 .with_container_name(container);
988
989 if let Some(options) = storage_options {
991 builder = apply_azure_storage_options(builder, options)?;
992 }
993
994 let azure_store = builder.build()?;
995 Ok((Arc::new(azure_store), path, uri.to_string()))
996}
997
998#[cfg(feature = "cloud")]
1000fn apply_azure_storage_options(
1001 mut builder: object_store::azure::MicrosoftAzureBuilder,
1002 options: AHashMap<String, String>,
1003) -> anyhow::Result<object_store::azure::MicrosoftAzureBuilder> {
1004 for (key, value) in options {
1005 builder = builder.with_config(key.parse::<object_store::azure::AzureConfigKey>()?, value);
1006 }
1007
1008 Ok(builder)
1009}
1010
1011#[cfg(feature = "cloud")]
1013fn create_http_store(
1014 uri: &str,
1015 storage_options: Option<AHashMap<String, String>>,
1016) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
1017 let (_, path) = parse_url_and_path(uri)?;
1018 let base_url = remote_store_root_url(uri)?
1019 .as_str()
1020 .trim_end_matches('/')
1021 .to_string();
1022
1023 let mut builder = object_store::http::HttpBuilder::new().with_url(base_url);
1024
1025 if let Some(options) = storage_options {
1027 for (key, value) in options {
1028 builder = builder.with_config(key.parse::<object_store::ClientConfigKey>()?, value);
1029 }
1030 }
1031
1032 let http_store = builder.build()?;
1033 Ok((Arc::new(http_store), path, uri.to_string()))
1034}
1035
1036#[cfg(feature = "cloud")]
1041fn parse_url_and_path(uri: &str) -> anyhow::Result<(Url, String)> {
1042 let url = Url::parse(uri)?;
1043 let path = url.path().trim_start_matches('/').to_string();
1044 let raw_path = uri
1045 .split_once("://")
1046 .and_then(|(_, rest)| rest.split_once('/'))
1047 .map_or("", |(_, raw)| raw.trim_start_matches('/'));
1048
1049 anyhow::ensure!(
1050 raw_path == path && ObjectPath::from(path.as_str()).as_ref() == path.trim_end_matches('/'),
1051 "Storage URI '{uri}' has a base path that URL parsing or object-store path encoding \
1052 changes; use a base path without spaces, non-ASCII, or reserved characters",
1053 );
1054
1055 Ok((url, path))
1056}
1057
1058#[cfg(feature = "cloud")]
1060fn extract_host(url: &Url, error_msg: &str) -> anyhow::Result<String> {
1061 url.host_str()
1062 .map(ToString::to_string)
1063 .ok_or_else(|| anyhow::anyhow!("{error_msg}"))
1064}
1065
1066#[cfg(test)]
1067mod tests {
1068 #[cfg(feature = "cloud")]
1069 use std::io::{BufRead, Write};
1070 use std::{collections::HashMap, sync::Arc};
1071
1072 #[cfg(feature = "cloud")]
1073 use ahash::AHashMap;
1074 use arrow::{
1075 array::{Array, ArrayRef, StringArray, TimestampNanosecondArray, UInt64Array},
1076 datatypes::{DataType, Field, Schema},
1077 };
1078 use nautilus_serialization::arrow::{json_string_field, timestamp_data_type};
1079 use parquet::file::{properties::ReaderProperties, serialized_reader::ReadOptionsBuilder};
1080 use rstest::rstest;
1081
1082 use super::*;
1083
1084 #[rstest]
1085 #[tokio::test]
1086 async fn combine_accepts_an_output_outside_the_source_directory() {
1087 use futures::TryStreamExt;
1088
1089 let store: Arc<dyn ObjectStore> = Arc::new(object_store::memory::InMemory::new());
1090
1091 let schema = Arc::new(Schema::new(vec![Field::new(
1092 "value",
1093 DataType::UInt64,
1094 false,
1095 )]));
1096
1097 let paths = [
1098 ObjectPath::from("inputs/one.parquet"),
1099 ObjectPath::from("inputs/two.parquet"),
1100 ];
1101
1102 for (path, value) in paths.iter().zip([11, 22]) {
1103 let batch = RecordBatch::try_new(
1104 schema.clone(),
1105 vec![Arc::new(UInt64Array::from(vec![value]))],
1106 )
1107 .unwrap();
1108 write_batches_to_object_store(&[batch], store.clone(), path, None, None, None)
1109 .await
1110 .unwrap();
1111 }
1112
1113 let target = ObjectPath::from("outputs/combined.parquet");
1114 combine_parquet_files_from_object_store(
1115 store.clone(),
1116 paths.to_vec(),
1117 &target,
1118 None,
1119 None,
1120 None,
1121 )
1122 .await
1123 .unwrap();
1124 let (batches, actual_schema) = read_parquet_from_object_store(store.clone(), &target)
1125 .await
1126 .unwrap();
1127 let actual = concat_batches(&actual_schema, &batches).unwrap();
1128 let expected =
1129 RecordBatch::try_new(schema, vec![Arc::new(UInt64Array::from(vec![11, 22]))]).unwrap();
1130 let remaining = store.list(None).try_collect::<Vec<_>>().await.unwrap();
1131
1132 assert_eq!(actual, expected);
1133 assert_eq!(
1134 remaining
1135 .iter()
1136 .map(|meta| &meta.location)
1137 .collect::<Vec<_>>(),
1138 vec![&target]
1139 );
1140 }
1141
1142 fn consolidation_depth_schema(
1143 price_precision: &str,
1144 size_precision: &str,
1145 instrument_id: &str,
1146 ) -> Arc<Schema> {
1147 Arc::new(Schema::new_with_metadata(
1148 vec![Field::new("bids", DataType::Utf8, false)],
1149 HashMap::from([
1150 (KEY_PRICE_PRECISION.to_string(), price_precision.to_string()),
1151 (KEY_SIZE_PRECISION.to_string(), size_precision.to_string()),
1152 (KEY_IDENTIFIER.to_string(), instrument_id.to_string()),
1153 ]),
1154 ))
1155 }
1156
1157 fn reconcile_consolidation_schema(
1158 current: &Arc<Schema>,
1159 current_path: &ObjectPath,
1160 candidate: &Arc<Schema>,
1161 candidate_path: &ObjectPath,
1162 ) -> anyhow::Result<ReconciledConsolidationSchema> {
1163 let field_metadata_sources = field_metadata_keys(current)
1164 .map(|key| (key, current_path.clone()))
1165 .collect();
1166 reconcile_consolidation_schema_with_sources(
1167 current,
1168 current_path,
1169 &field_metadata_sources,
1170 candidate,
1171 candidate_path,
1172 )
1173 }
1174
1175 #[rstest]
1176 fn consolidation_schema_prefers_populated_depth_precision_in_either_order() {
1177 let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1178 let populated = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1179 let fallback_path = ObjectPath::from("empty.parquet");
1180 let populated_path = ObjectPath::from("populated.parquet");
1181
1182 let fallback_first =
1183 reconcile_consolidation_schema(&fallback, &fallback_path, &populated, &populated_path)
1184 .unwrap();
1185 let populated_first =
1186 reconcile_consolidation_schema(&populated, &populated_path, &fallback, &fallback_path)
1187 .unwrap();
1188
1189 assert_eq!(
1190 fallback_first.schema_source,
1191 ConsolidationSchemaSource::Candidate,
1192 );
1193 assert_eq!(
1194 populated_first.schema_source,
1195 ConsolidationSchemaSource::Current,
1196 );
1197
1198 for schema in [fallback_first.schema, populated_first.schema] {
1199 assert_eq!(schema.metadata()[KEY_PRICE_PRECISION], "2");
1200 assert_eq!(schema.metadata()[KEY_SIZE_PRECISION], "3");
1201 }
1202 }
1203
1204 #[rstest]
1205 fn consolidation_schema_rejects_other_metadata_mismatches() {
1206 let current = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1207 let candidate = consolidation_depth_schema("2", "3", "BTCUSDT.BINANCE");
1208 let current_path = ObjectPath::from("eth.parquet");
1209 let candidate_path = ObjectPath::from("btc.parquet");
1210
1211 let error =
1212 reconcile_consolidation_schema(¤t, ¤t_path, &candidate, &candidate_path)
1213 .unwrap_err();
1214
1215 assert!(error.to_string().contains("eth.parquet and btc.parquet"));
1216 assert!(error.to_string().contains("ETHUSDT.BINANCE"));
1217 assert!(error.to_string().contains("BTCUSDT.BINANCE"));
1218 }
1219
1220 #[rstest]
1221 fn consolidation_schema_rejects_two_populated_precisions() {
1222 let current = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1223 let candidate = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1224 let current_path = ObjectPath::from("precision-2.parquet");
1225 let candidate_path = ObjectPath::from("precision-4.parquet");
1226
1227 let error =
1228 reconcile_consolidation_schema(¤t, ¤t_path, &candidate, &candidate_path)
1229 .unwrap_err();
1230
1231 assert!(error.to_string().contains("precision-2.parquet"));
1232 assert!(error.to_string().contains("precision-4.parquet"));
1233 }
1234
1235 #[rstest]
1236 fn consolidation_schema_keeps_fallback_for_all_empty_files() {
1237 let first = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1238 let second = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1239
1240 let reconciled = reconcile_consolidation_schema(
1241 &first,
1242 &ObjectPath::from("first-empty.parquet"),
1243 &second,
1244 &ObjectPath::from("second-empty.parquet"),
1245 )
1246 .unwrap();
1247
1248 assert_eq!(reconciled.schema.metadata()[KEY_PRICE_PRECISION], "0");
1249 assert_eq!(reconciled.schema.metadata()[KEY_SIZE_PRECISION], "0");
1250 }
1251
1252 #[rstest]
1253 #[case::missing_first(true)]
1254 #[case::fallback_first(false)]
1255 fn consolidation_schema_rejects_missing_precision_against_fallback(
1256 #[case] missing_first: bool,
1257 ) {
1258 let missing = Arc::new(Schema::new_with_metadata(
1259 vec![Field::new("bids", DataType::Utf8, false)],
1260 HashMap::from([(KEY_IDENTIFIER.to_string(), "ETHUSDT.BINANCE".to_string())]),
1261 ));
1262 let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1263 let missing_path = ObjectPath::from("missing.parquet");
1264 let fallback_path = ObjectPath::from("fallback.parquet");
1265
1266 let error = if missing_first {
1267 reconcile_consolidation_schema(&missing, &missing_path, &fallback, &fallback_path)
1268 } else {
1269 reconcile_consolidation_schema(&fallback, &fallback_path, &missing, &missing_path)
1270 }
1271 .unwrap_err();
1272
1273 assert!(error.to_string().contains("schema metadata differs"));
1274 }
1275
1276 #[rstest]
1277 fn consolidation_schema_merges_json_field_annotation_in_either_order() {
1278 let bare = Arc::new(Schema::new(vec![Field::new("info", DataType::Utf8, true)]));
1279 let annotated = Arc::new(Schema::new(vec![json_string_field("info", true)]));
1280 let bare_path = ObjectPath::from("bare.parquet");
1281 let annotated_path = ObjectPath::from("annotated.parquet");
1282
1283 let bare_first =
1284 reconcile_consolidation_schema(&bare, &bare_path, &annotated, &annotated_path).unwrap();
1285 let annotated_first =
1286 reconcile_consolidation_schema(&annotated, &annotated_path, &bare, &bare_path).unwrap();
1287
1288 assert_eq!(bare_first.schema, annotated_first.schema);
1289 assert_eq!(
1290 bare_first.schema.field_with_name("info").unwrap(),
1291 &json_string_field("info", true),
1292 );
1293 }
1294
1295 #[rstest]
1296 fn consolidation_field_metadata_conflict_names_the_winning_file() {
1297 let bare = Arc::new(Schema::new(vec![Field::new("info", DataType::Utf8, true)]));
1298 let annotated = Arc::new(Schema::new(vec![json_string_field("info", true)]));
1299
1300 let conflicting = Arc::new(Schema::new(vec![
1301 Field::new("info", DataType::Utf8, true).with_metadata(HashMap::from([(
1302 "ARROW:extension:name".to_string(),
1303 "other.extension".to_string(),
1304 )])),
1305 ]));
1306 let bare_path = ObjectPath::from("bare.parquet");
1307 let annotated_path = ObjectPath::from("annotated.parquet");
1308 let conflicting_path = ObjectPath::from("conflicting.parquet");
1309 let reconciled =
1310 reconcile_consolidation_schema(&bare, &bare_path, &annotated, &annotated_path).unwrap();
1311
1312 assert_eq!(reconciled.candidate_field_metadata.len(), 2);
1313 assert!(
1314 reconciled
1315 .candidate_field_metadata
1316 .contains(&("info".to_string(), "ARROW:extension:name".to_string())),
1317 );
1318 assert!(
1319 reconciled
1320 .candidate_field_metadata
1321 .contains(&("info".to_string(), "ARROW:extension:metadata".to_string())),
1322 );
1323 let field_metadata_sources = reconciled
1324 .candidate_field_metadata
1325 .iter()
1326 .cloned()
1327 .map(|key| (key, annotated_path.clone()))
1328 .collect();
1329 let error = reconcile_consolidation_schema_with_sources(
1330 &reconciled.schema,
1331 &bare_path,
1332 &field_metadata_sources,
1333 &conflicting,
1334 &conflicting_path,
1335 )
1336 .unwrap_err();
1337 assert!(error.to_string().contains("annotated.parquet"));
1338 assert!(error.to_string().contains("conflicting.parquet"));
1339 assert!(error.to_string().contains("field 'info' metadata differs"));
1340 }
1341
1342 #[rstest]
1343 fn consolidation_conflict_names_the_last_winning_schema() {
1344 let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1345 let precision_2 = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1346 let precision_4 = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1347 let fallback_path = ObjectPath::from("fallback.parquet");
1348 let precision_2_path = ObjectPath::from("precision-2.parquet");
1349 let precision_4_path = ObjectPath::from("precision-4.parquet");
1350 let reconciled = reconcile_consolidation_schema(
1351 &fallback,
1352 &fallback_path,
1353 &precision_2,
1354 &precision_2_path,
1355 )
1356 .unwrap();
1357
1358 let error = reconcile_consolidation_schema(
1359 &reconciled.schema,
1360 &precision_2_path,
1361 &precision_4,
1362 &precision_4_path,
1363 )
1364 .unwrap_err();
1365
1366 assert!(error.to_string().contains("precision-2.parquet"));
1367 assert!(error.to_string().contains("precision-4.parquet"));
1368 }
1369
1370 #[rstest]
1371 fn consolidation_field_metadata_does_not_replace_precision_source() {
1372 let precision_2 = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1373
1374 let fallback = Arc::new(Schema::new_with_metadata(
1375 vec![json_string_field("bids", false)],
1376 consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE")
1377 .metadata()
1378 .clone(),
1379 ));
1380 let precision_4 = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1381 let precision_2_path = ObjectPath::from("precision-2.parquet");
1382 let fallback_path = ObjectPath::from("fallback-annotated.parquet");
1383 let precision_4_path = ObjectPath::from("precision-4.parquet");
1384 let reconciled = reconcile_consolidation_schema(
1385 &precision_2,
1386 &precision_2_path,
1387 &fallback,
1388 &fallback_path,
1389 )
1390 .unwrap();
1391
1392 assert_eq!(reconciled.schema_source, ConsolidationSchemaSource::Current,);
1393 assert_eq!(reconciled.candidate_field_metadata.len(), 2);
1394 assert!(
1395 reconciled
1396 .candidate_field_metadata
1397 .contains(&("bids".to_string(), "ARROW:extension:name".to_string())),
1398 );
1399 assert!(
1400 reconciled
1401 .candidate_field_metadata
1402 .contains(&("bids".to_string(), "ARROW:extension:metadata".to_string())),
1403 );
1404 let field_metadata_sources = reconciled
1405 .candidate_field_metadata
1406 .iter()
1407 .cloned()
1408 .map(|key| (key, fallback_path.clone()))
1409 .collect();
1410 let error = reconcile_consolidation_schema_with_sources(
1411 &reconciled.schema,
1412 &precision_2_path,
1413 &field_metadata_sources,
1414 &precision_4,
1415 &precision_4_path,
1416 )
1417 .unwrap_err();
1418 let message = error.to_string();
1419
1420 assert!(message.contains("precision-2.parquet"));
1421 assert!(message.contains("precision-4.parquet"));
1422 assert!(!message.contains("fallback-annotated.parquet"));
1423 }
1424
1425 #[rstest]
1426 fn consolidation_field_metadata_tracks_each_origin() {
1427 let bare = Arc::new(Schema::new(vec![
1428 Field::new("info", DataType::Utf8, true),
1429 Field::new("balances", DataType::Utf8, true),
1430 ]));
1431
1432 let info = Arc::new(Schema::new(vec![
1433 json_string_field("info", true),
1434 Field::new("balances", DataType::Utf8, true),
1435 ]));
1436
1437 let balances = Arc::new(Schema::new(vec![
1438 json_string_field("info", true),
1439 json_string_field("balances", true),
1440 ]));
1441
1442 let conflicting = Arc::new(Schema::new(vec![
1443 Field::new("info", DataType::Utf8, true).with_metadata(HashMap::from([(
1444 "ARROW:extension:name".to_string(),
1445 "other.extension".to_string(),
1446 )])),
1447 json_string_field("balances", true),
1448 ]));
1449 let bare_path = ObjectPath::from("bare.parquet");
1450 let info_path = ObjectPath::from("info.parquet");
1451 let balances_path = ObjectPath::from("balances.parquet");
1452 let conflicting_path = ObjectPath::from("conflicting.parquet");
1453 let with_info =
1454 reconcile_consolidation_schema(&bare, &bare_path, &info, &info_path).unwrap();
1455 let mut field_metadata_sources = with_info
1456 .candidate_field_metadata
1457 .iter()
1458 .cloned()
1459 .map(|key| (key, info_path.clone()))
1460 .collect::<HashMap<_, _>>();
1461 let with_balances = reconcile_consolidation_schema_with_sources(
1462 &with_info.schema,
1463 &bare_path,
1464 &field_metadata_sources,
1465 &balances,
1466 &balances_path,
1467 )
1468 .unwrap();
1469
1470 for key in with_balances.candidate_field_metadata {
1471 field_metadata_sources.insert(key, balances_path.clone());
1472 }
1473
1474 let error = reconcile_consolidation_schema_with_sources(
1475 &with_balances.schema,
1476 &bare_path,
1477 &field_metadata_sources,
1478 &conflicting,
1479 &conflicting_path,
1480 )
1481 .unwrap_err();
1482 let message = error.to_string();
1483
1484 assert!(message.contains("info.parquet"));
1485 assert!(message.contains("conflicting.parquet"));
1486 assert!(!message.contains("balances.parquet"));
1487 }
1488
1489 #[tokio::test]
1490 async fn write_batches_to_object_store_rejects_empty_input() {
1491 let object_store = Arc::new(object_store::memory::InMemory::new());
1492
1493 let error = write_batches_to_object_store(
1494 &[],
1495 object_store,
1496 &ObjectPath::from("empty.parquet"),
1497 None,
1498 None,
1499 None,
1500 )
1501 .await
1502 .unwrap_err();
1503
1504 assert_eq!(
1505 error.to_string(),
1506 "Cannot write Parquet file empty.parquet with no record batches"
1507 );
1508 }
1509
1510 #[tokio::test]
1511 async fn default_writer_sets_zstd_sorting_and_identifier_bloom_filter() {
1512 let directory = tempfile::tempdir().unwrap();
1513 let path = directory.path().join("layout.parquet");
1514
1515 let object_store = Arc::new(
1516 object_store::local::LocalFileSystem::new_with_prefix(directory.path()).unwrap(),
1517 );
1518 let object_path = ObjectPath::from("layout.parquet");
1519
1520 let schema = Arc::new(Schema::new(vec![
1521 Field::new("identifier", DataType::Utf8, false),
1522 Field::new("ts_init", DataType::UInt64, false),
1523 ]));
1524 let batch = RecordBatch::try_new(
1525 schema,
1526 vec![
1527 Arc::new(StringArray::from(vec!["AUD/USD.SIM", "AUD/USD.SIM"])) as ArrayRef,
1528 Arc::new(UInt64Array::from(vec![1_u64, 2])) as ArrayRef,
1529 ],
1530 )
1531 .unwrap();
1532
1533 write_batches_to_object_store(&[batch], object_store, &object_path, None, None, None)
1534 .await
1535 .unwrap();
1536
1537 let read_options = ReadOptionsBuilder::new()
1538 .with_reader_properties(
1539 ReaderProperties::builder()
1540 .set_read_bloom_filter(true)
1541 .build(),
1542 )
1543 .build();
1544 let reader = SerializedFileReader::new_with_options(
1545 std::fs::File::open(path).unwrap(),
1546 read_options,
1547 )
1548 .unwrap();
1549 let row_group = reader.metadata().row_group(0);
1550 let sorting = row_group.sorting_columns().unwrap();
1551
1552 assert_eq!(crate::backend::parquet::DEFAULT_ROW_GROUP_SIZE, 131_072);
1553 assert_eq!(
1554 sorting,
1555 &vec![
1556 SortingColumn {
1557 column_idx: 1,
1558 descending: false,
1559 nulls_first: false,
1560 },
1561 SortingColumn {
1562 column_idx: 0,
1563 descending: false,
1564 nulls_first: false,
1565 },
1566 ],
1567 );
1568 assert!(
1569 row_group
1570 .columns()
1571 .iter()
1572 .all(|column| column.compression() == Compression::ZSTD(ZstdLevel::default())),
1573 );
1574 assert!(
1575 reader
1576 .get_row_group(0)
1577 .unwrap()
1578 .get_column_bloom_filter(0)
1579 .is_some(),
1580 );
1581 }
1582
1583 #[tokio::test]
1584 async fn default_writer_delta_encodes_timestamps_without_dictionary() {
1585 let directory = tempfile::tempdir().unwrap();
1586 let path = directory.path().join("timestamps.parquet");
1587
1588 let object_store = Arc::new(
1589 object_store::local::LocalFileSystem::new_with_prefix(directory.path()).unwrap(),
1590 );
1591 let object_path = ObjectPath::from("timestamps.parquet");
1592
1593 let schema = Arc::new(Schema::new(vec![
1594 Field::new("identifier", DataType::Utf8, false),
1595 Field::new("ts_event", timestamp_data_type(), false),
1596 Field::new("ts_init", timestamp_data_type(), false),
1597 ]));
1598 let identifiers = StringArray::from(vec!["AUD/USD.SIM", "AUD/USD.SIM", "AUD/USD.SIM"]);
1599 let ts_event = TimestampNanosecondArray::from(vec![
1600 1_700_000_000_123_456_001_i64,
1601 1_700_000_000_123_456_789,
1602 1_700_000_001_000_000_000,
1603 ])
1604 .with_timezone("UTC");
1605 let ts_init = TimestampNanosecondArray::from(vec![
1606 1_700_000_000_123_457_000_i64,
1607 1_700_000_000_123_458_000,
1608 1_700_000_001_000_001_000,
1609 ])
1610 .with_timezone("UTC");
1611 let batch = RecordBatch::try_new(
1612 schema,
1613 vec![
1614 Arc::new(identifiers.clone()) as ArrayRef,
1615 Arc::new(ts_event.clone()) as ArrayRef,
1616 Arc::new(ts_init.clone()) as ArrayRef,
1617 ],
1618 )
1619 .unwrap();
1620
1621 write_batches_to_object_store(&[batch], object_store, &object_path, None, None, None)
1622 .await
1623 .unwrap();
1624
1625 let reader = SerializedFileReader::new(std::fs::File::open(&path).unwrap()).unwrap();
1626 let columns = reader.metadata().row_group(0).columns();
1627 let timestamp_encodings = columns[1..]
1628 .iter()
1629 .map(|column| {
1630 (
1631 column.column_path().string(),
1632 column.dictionary_page_offset(),
1633 column.encodings().collect::<Vec<_>>(),
1634 )
1635 })
1636 .collect::<Vec<_>>();
1637 let read = ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(&path).unwrap())
1638 .unwrap()
1639 .build()
1640 .unwrap()
1641 .collect::<Result<Vec<_>, _>>()
1642 .unwrap();
1643
1644 assert!(columns[0].dictionary_page_offset().is_some());
1645 assert_eq!(
1646 timestamp_encodings,
1647 vec![
1648 (
1649 "ts_event".to_string(),
1650 None,
1651 vec![Encoding::RLE, Encoding::DELTA_BINARY_PACKED],
1652 ),
1653 (
1654 "ts_init".to_string(),
1655 None,
1656 vec![Encoding::RLE, Encoding::DELTA_BINARY_PACKED],
1657 ),
1658 ],
1659 );
1660 assert_eq!(read.len(), 1);
1661 assert_eq!(read[0].column(0).as_ref(), &identifiers as &dyn Array);
1662 assert_eq!(read[0].column(1).as_ref(), &ts_event as &dyn Array);
1663 assert_eq!(read[0].column(2).as_ref(), &ts_init as &dyn Array);
1664 }
1665
1666 #[rstest]
1667 fn test_create_object_store_from_path_local() {
1668 let temp_dir = std::env::temp_dir().join("nautilus_test");
1670 std::fs::create_dir_all(&temp_dir).unwrap();
1671
1672 let result = create_object_store_from_path(temp_dir.to_str().unwrap(), None);
1673 if let Err(e) = &result {
1674 println!("Error: {e:?}");
1675 }
1676
1677 assert!(result.is_ok());
1678 let (_, base_path, uri) = result.unwrap();
1679 assert_eq!(base_path, "");
1680 assert_eq!(uri, format!("file://{}", temp_dir.to_str().unwrap()));
1682
1683 std::fs::remove_dir_all(&temp_dir).ok();
1685 }
1686
1687 #[rstest]
1688 fn append_path_to_file_uri_extends_windows_drive_catalog() {
1689 assert_eq!(
1690 append_path_to_file_uri(
1691 "file:///C:/data/catalog",
1692 "data/quotes/EURUSD.SIM/file.parquet",
1693 ),
1694 "file:///C:/data/catalog/data/quotes/EURUSD.SIM/file.parquet",
1695 );
1696 }
1697
1698 #[rstest]
1699 #[cfg(feature = "cloud")]
1700 fn test_create_object_store_from_path_s3() {
1701 let mut options = AHashMap::new();
1702 options.insert(
1703 "endpoint_url".to_string(),
1704 "https://test.endpoint.com".to_string(),
1705 );
1706 options.insert("region".to_string(), "us-west-2".to_string());
1707 options.insert("access_key_id".to_string(), "test_key".to_string());
1708 options.insert("secret_access_key".to_string(), "test_secret".to_string());
1709
1710 let result = create_object_store_from_path("s3://test-bucket/path", Some(options));
1711 assert!(result.is_ok());
1712 let (_, base_path, uri) = result.unwrap();
1713 assert_eq!(base_path, "path");
1714 assert_eq!(uri, "s3://test-bucket/path");
1715 }
1716
1717 #[rstest]
1718 #[cfg(feature = "cloud")]
1719 fn test_create_object_store_from_path_azure() {
1720 let mut options = AHashMap::new();
1721 options.insert("account_name".to_string(), "testaccount".to_string());
1722 options.insert("account_key".to_string(), "dGVzdGtleQ==".to_string()); let result = create_object_store_from_path("az://container/path", Some(options));
1726 if let Err(e) = &result {
1727 println!("Azure Error: {e:?}");
1728 }
1729
1730 assert!(result.is_ok());
1731 let (_, base_path, uri) = result.unwrap();
1732 assert_eq!(base_path, "path");
1733 assert_eq!(uri, "az://container/path");
1734 }
1735
1736 #[rstest]
1737 #[cfg(feature = "cloud")]
1738 fn test_create_object_store_from_path_gcs() {
1739 let credentials_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
1740 .join("test_data/gcs_authorized_user_credentials.json");
1741 let options = AHashMap::from([
1742 ("skip_signature".to_string(), "true".to_string()),
1743 (
1744 "application_credentials".to_string(),
1745 credentials_path.to_str().unwrap().to_string(),
1746 ),
1747 ]);
1748
1749 let (_, base_path, uri) =
1750 create_object_store_from_path("gs://test-bucket/path", Some(options)).unwrap();
1751
1752 assert_eq!(base_path, "path");
1753 assert_eq!(uri, "gs://test-bucket/path");
1754 }
1755
1756 #[rstest]
1757 #[cfg(feature = "cloud")]
1758 fn test_create_object_store_gcs_reads_application_credentials_option() {
1759 let directory = tempfile::tempdir().unwrap();
1760 let credentials_path = directory.path().join("missing-credentials.json");
1761 let credentials_path = credentials_path.to_str().unwrap();
1762 let options = AHashMap::from([(
1763 "application_credentials".to_string(),
1764 credentials_path.to_string(),
1765 )]);
1766
1767 let error = create_object_store_from_path("gs://test-bucket/path", Some(options))
1768 .err()
1769 .unwrap()
1770 .to_string();
1771
1772 assert!(
1773 error.contains(&format!(
1774 "Unable to open service account file from {credentials_path}"
1775 )),
1776 "{error}"
1777 );
1778 }
1779
1780 #[rstest]
1781 #[cfg(feature = "cloud")]
1782 #[case::s3("s3://test-bucket/path", "no_such_option", "S3")]
1783 #[case::gcs_project_id("gs://test-bucket/path", "project_id", "GCS")]
1784 #[case::azure("az://container/path", "no_such_option", "MicrosoftAzure")]
1785 #[case::abfs(
1786 "abfs://container@account.dfs.core.windows.net/path",
1787 "no_such_option",
1788 "MicrosoftAzure"
1789 )]
1790 #[case::http("https://example.com/path", "no_such_option", "HTTP")]
1791 fn test_create_object_store_rejects_unknown_storage_option(
1792 #[case] uri: &str,
1793 #[case] key: &str,
1794 #[case] store: &str,
1795 ) {
1796 let options = AHashMap::from([(key.to_string(), "value".to_string())]);
1797
1798 let error = create_object_store_from_path(uri, Some(options))
1799 .err()
1800 .unwrap()
1801 .to_string();
1802
1803 assert_eq!(
1804 error,
1805 format!("Configuration key: '{key}' is not valid for store '{store}'.")
1806 );
1807 }
1808
1809 #[rstest]
1810 #[cfg(feature = "cloud")]
1811 #[case::non_ascii("s3://test-bucket/préfix")]
1812 #[case::space("s3://test-bucket/my prefix")]
1813 #[case::tilde("s3://test-bucket/~prefix")]
1814 #[case::fragment("s3://test-bucket/pre#fix")]
1815 #[case::percent_encoded("s3://test-bucket/pr%C3%A9fix")]
1816 #[case::empty_segment("s3://test-bucket/base//path")]
1817 #[case::gcs("gs://test-bucket/préfix")]
1818 #[case::azure("az://container/préfix")]
1819 #[case::abfs("abfs://container@account.dfs.core.windows.net/préfix")]
1820 #[case::http("https://example.com/préfix")]
1821 fn test_create_object_store_rejects_base_path_that_paths_change(#[case] uri: &str) {
1822 let error = create_object_store_from_path(uri, None)
1823 .err()
1824 .unwrap()
1825 .to_string();
1826
1827 assert_eq!(
1828 error,
1829 format!(
1830 "Storage URI '{uri}' has a base path that URL parsing or object-store path \
1831 encoding changes; use a base path without spaces, non-ASCII, or reserved \
1832 characters"
1833 )
1834 );
1835 }
1836
1837 #[rstest]
1838 #[cfg(feature = "cloud")]
1839 #[case::bucket_root("s3://test-bucket", "")]
1840 #[case::trailing_slash("s3://test-bucket/nautilus-data/", "nautilus-data/")]
1841 #[case::unreserved(
1842 "s3://test-bucket/team_a/v2.catalog/year=2026",
1843 "team_a/v2.catalog/year=2026"
1844 )]
1845 fn test_create_object_store_accepts_ascii_base_path(
1846 #[case] uri: &str,
1847 #[case] expected_base_path: &str,
1848 ) {
1849 let (_, base_path, original_uri) = create_object_store_from_path(uri, None).unwrap();
1850
1851 assert_eq!(base_path, expected_base_path);
1852 assert_eq!(original_uri, uri);
1853 }
1854
1855 #[rstest]
1856 #[case::native("aws_access_key_id", "aws_secret_access_key")]
1857 #[case::short("access_key_id", "secret_access_key")]
1858 #[case::legacy("key", "secret")]
1859 #[tokio::test]
1860 #[cfg(feature = "cloud")]
1861 async fn test_create_object_store_s3_applies_credential_option_names(
1862 #[case] key_id_option: &str,
1863 #[case] secret_option: &str,
1864 ) {
1865 let (endpoint, server) = capture_request_head();
1866 let options = AHashMap::from([
1867 ("aws_endpoint".to_string(), endpoint),
1868 ("aws_allow_http".to_string(), "true".to_string()),
1869 (key_id_option.to_string(), "TESTKEYID".to_string()),
1870 (secret_option.to_string(), "test-secret".to_string()),
1871 ]);
1872 let (store, base_path, _) =
1873 create_object_store_from_path("s3://test-bucket/path", Some(options)).unwrap();
1874
1875 let result = store
1876 .head(&ObjectPath::from(format!("{base_path}/probe")))
1877 .await;
1878
1879 assert!(
1880 matches!(result, Err(object_store::Error::NotFound { .. })),
1881 "{result:?}"
1882 );
1883 let head = server.join().unwrap();
1884 assert_eq!(
1885 head.lines().next(),
1886 Some("HEAD /test-bucket/path/probe HTTP/1.1")
1887 );
1888 assert!(
1889 head.contains("authorization: AWS4-HMAC-SHA256 Credential=TESTKEYID/"),
1890 "{head}"
1891 );
1892 }
1893
1894 #[tokio::test]
1895 #[cfg(feature = "cloud")]
1896 async fn test_create_object_store_http_applies_client_option() {
1897 let (endpoint, server) = capture_request_head();
1898 let options = AHashMap::from([
1899 ("allow_http".to_string(), "true".to_string()),
1900 (
1901 "user_agent".to_string(),
1902 "nautilus-catalog-test".to_string(),
1903 ),
1904 ]);
1905 let (store, base_path, _) =
1906 create_object_store_from_path(&format!("{endpoint}/path"), Some(options)).unwrap();
1907
1908 let result = store
1909 .head(&ObjectPath::from(format!("{base_path}/probe")))
1910 .await;
1911
1912 assert!(
1913 matches!(result, Err(object_store::Error::NotFound { .. })),
1914 "{result:?}"
1915 );
1916 let head = server.join().unwrap();
1917 assert_eq!(head.lines().next(), Some("HEAD /path/probe HTTP/1.1"));
1918 assert!(
1919 head.contains("user-agent: nautilus-catalog-test\r\n"),
1920 "{head}"
1921 );
1922 }
1923
1924 #[tokio::test]
1925 #[cfg(feature = "cloud")]
1926 async fn test_create_object_store_azure_sends_sas_token_signature_encoded_once() {
1927 let (endpoint, server) = capture_request_head();
1928 let options = AHashMap::from([
1929 ("account_name".to_string(), "account".to_string()),
1930 ("azure_storage_endpoint".to_string(), endpoint),
1931 ("azure_allow_http".to_string(), "true".to_string()),
1932 (
1933 "sas_token".to_string(),
1934 "sv=2022-11-02&se=2030-01-01T00:00:00Z&sig=abc%2Bdef%3D".to_string(),
1935 ),
1936 ]);
1937 let (store, base_path, _) =
1938 create_object_store_from_path("az://container/path", Some(options)).unwrap();
1939
1940 let result = store
1941 .head(&ObjectPath::from(format!("{base_path}/probe")))
1942 .await;
1943
1944 assert!(
1945 matches!(result, Err(object_store::Error::NotFound { .. })),
1946 "{result:?}"
1947 );
1948 let head = server.join().unwrap();
1949 assert_eq!(
1950 head.lines().next(),
1951 Some(
1952 "HEAD /container/path/probe?sv=2022-11-02&se=2030-01-01T00%3A00%3A00Z\
1953 &sig=abc%2Bdef%3D HTTP/1.1"
1954 )
1955 );
1956 }
1957
1958 #[rstest]
1959 #[cfg(feature = "cloud")]
1960 fn test_create_object_store_from_path_empty_options() {
1961 let result = create_object_store_from_path("s3://test-bucket/path", None);
1962 assert!(result.is_ok());
1963 let (_, base_path, uri) = result.unwrap();
1964 assert_eq!(base_path, "path");
1965 assert_eq!(uri, "s3://test-bucket/path");
1966 }
1967
1968 #[rstest]
1969 #[cfg(feature = "cloud")]
1970 fn test_remote_store_root_url_preserves_authority() {
1971 let https_root = remote_store_root_url("https://example.com:9000/base/path").unwrap();
1972 assert_eq!(
1973 https_root.as_str().trim_end_matches('/'),
1974 "https://example.com:9000"
1975 );
1976
1977 let abfs_root =
1978 remote_store_root_url("abfs://container@account.dfs.core.windows.net/base/path")
1979 .unwrap();
1980 assert_eq!(
1981 abfs_root.as_str().trim_end_matches('/'),
1982 "abfs://container@account.dfs.core.windows.net"
1983 );
1984
1985 let full_uri = remote_full_uri(
1986 "https://example.com:9000/base/path",
1987 "base/path/data/%5E/file.parquet",
1988 )
1989 .unwrap();
1990 assert_eq!(
1991 full_uri,
1992 "https://example.com:9000/base/path/data/%5E/file.parquet"
1993 );
1994
1995 let location = create_object_store_location_from_path("s3://test-bucket/path", None)
1996 .expect("S3 location should be created");
1997 assert_eq!(location.base_path, "path");
1998 assert_eq!(
1999 remote_store_root_url(&location.original_uri)
2000 .expect("S3 should be remote")
2001 .as_str()
2002 .trim_end_matches('/'),
2003 "s3://test-bucket"
2004 );
2005 }
2006
2007 #[cfg(feature = "cloud")]
2008 fn capture_request_head() -> (String, std::thread::JoinHandle<String>) {
2009 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2010 let endpoint = format!("http://{}", listener.local_addr().unwrap());
2011 let handle = std::thread::spawn(move || {
2012 let (mut stream, _) = listener.accept().unwrap();
2013 let mut reader = std::io::BufReader::new(stream.try_clone().unwrap());
2014 let mut head = String::new();
2015 loop {
2016 let mut line = String::new();
2017 reader.read_line(&mut line).unwrap();
2018 if line.trim_end().is_empty() {
2019 break;
2020 }
2021 head.push_str(&line);
2022 }
2023 stream
2024 .write_all(
2025 b"HTTP/1.1 404 Not Found\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
2026 )
2027 .unwrap();
2028 head
2029 });
2030 (endpoint, handle)
2031 }
2032}