1#![expect(
17 clippy::missing_panics_doc,
18 reason = "Parquet I/O functions use validated schema paths"
19)]
20
21use std::{collections::HashMap, sync::Arc};
22
23use ahash::AHashMap;
24use anyhow::Context;
25use arrow::{
26 array::UInt64Array,
27 compute::{SortColumn, SortOptions, concat_batches, lexsort_to_indices, take_record_batch},
28 datatypes::Schema,
29 record_batch::RecordBatch,
30};
31use nautilus_serialization::arrow::{
32 KEY_IDENTIFIER, KEY_PRICE_PRECISION, KEY_SIZE_PRECISION, U64ColumnRef,
33};
34use object_store::{
35 ObjectMeta, ObjectStore, ObjectStoreExt, PutMode, PutOptions, buffered::BufReader,
36 path::Path as ObjectPath,
37};
38use parquet::{
39 arrow::{
40 ArrowSchemaConverter, ArrowWriter, ParquetRecordBatchStreamBuilder,
41 arrow_reader::ParquetRecordBatchReaderBuilder,
42 },
43 basic::{Compression, Encoding, ZstdLevel},
44 file::{
45 metadata::{KeyValue, SortingColumn},
46 properties::WriterProperties,
47 reader::{FileReader, SerializedFileReader},
48 statistics::Statistics,
49 },
50 schema::types::ColumnPath,
51};
52use url::Url;
53
54use super::consolidation_state;
55pub(crate) use crate::common::paths::file_uri_to_native_path;
56pub use crate::common::paths::normalize_path_to_uri;
57
58pub(crate) struct ObjectStoreLocation {
59 pub object_store: Arc<dyn ObjectStore>,
60 pub base_path: String,
61 pub original_uri: String,
62}
63
64pub async fn read_parquet_schema_from_object_store(
72 object_store: Arc<dyn ObjectStore>,
73 path: &ObjectPath,
74) -> anyhow::Result<Arc<Schema>> {
75 let object = object_store.head(path).await?;
76 if object.size == 0 {
77 return Ok(Arc::new(Schema::empty()));
78 }
79
80 let reader = BufReader::new(object_store, &object);
81 let builder = ParquetRecordBatchStreamBuilder::new(reader).await?;
82 Ok(builder.schema().clone())
83}
84
85pub async fn read_parquet_from_object_store(
94 object_store: Arc<dyn ObjectStore>,
95 path: &ObjectPath,
96) -> anyhow::Result<(Vec<RecordBatch>, Arc<Schema>)> {
97 let data = object_store.get(path).await?.bytes().await?;
98 if data.is_empty() {
99 return Ok((Vec::new(), Arc::new(Schema::empty())));
100 }
101
102 let builder = ParquetRecordBatchReaderBuilder::try_new(data)?;
103 let schema = builder.schema().clone();
104 let batches = builder.build()?.collect::<Result<Vec<_>, _>>()?;
105
106 Ok((batches, schema))
107}
108
109pub async fn write_batches_to_object_store(
116 batches: &[RecordBatch],
117 object_store: Arc<dyn ObjectStore>,
118 path: &ObjectPath,
119 compression: Option<Compression>,
120 max_row_group_size: Option<usize>,
121 key_value_metadata: Option<Vec<KeyValue>>,
122) -> anyhow::Result<()> {
123 write_batches_to_object_store_with_mode(
124 batches,
125 object_store,
126 path,
127 compression,
128 max_row_group_size,
129 key_value_metadata,
130 PutMode::Overwrite,
131 None,
132 )
133 .await
134}
135
136pub(crate) async fn write_batches_to_object_store_create(
137 batches: &[RecordBatch],
138 object_store: Arc<dyn ObjectStore>,
139 path: &ObjectPath,
140 compression: Option<Compression>,
141 max_row_group_size: Option<usize>,
142 key_value_metadata: Option<Vec<KeyValue>>,
143) -> anyhow::Result<()> {
144 write_batches_to_object_store_with_mode(
145 batches,
146 object_store,
147 path,
148 compression,
149 max_row_group_size,
150 key_value_metadata,
151 PutMode::Create,
152 None,
153 )
154 .await
155}
156
157#[expect(
158 clippy::too_many_arguments,
159 reason = "private writer keeps storage mode and footer identity separate"
160)]
161async fn write_batches_to_object_store_with_mode(
162 batches: &[RecordBatch],
163 object_store: Arc<dyn ObjectStore>,
164 path: &ObjectPath,
165 compression: Option<Compression>,
166 max_row_group_size: Option<usize>,
167 key_value_metadata: Option<Vec<KeyValue>>,
168 put_mode: PutMode,
169 created_by: Option<String>,
170) -> anyhow::Result<()> {
171 let Some(first) = batches.first() else {
172 anyhow::bail!("Cannot write Parquet file {path} with no record batches");
173 };
174
175 let mut buffer = Vec::new();
177
178 let schema = first.schema();
179 let sorting_columns = parquet_sorting_columns(schema.as_ref())?;
180 let mut props_builder = WriterProperties::builder()
181 .set_compression(compression.unwrap_or(Compression::ZSTD(ZstdLevel::default())))
182 .set_max_row_group_row_count(Some(
183 max_row_group_size.unwrap_or(super::DEFAULT_ROW_GROUP_SIZE),
184 ))
185 .set_sorting_columns(sorting_columns)
186 .set_key_value_metadata(key_value_metadata);
187
188 if let Some(created_by) = created_by {
189 props_builder = props_builder.set_created_by(created_by);
190 }
191
192 if schema.index_of(KEY_IDENTIFIER).is_ok() {
193 props_builder =
194 props_builder.set_column_bloom_filter_enabled(ColumnPath::from(KEY_IDENTIFIER), true);
195 }
196
197 for name in ["ts_event", "ts_init"] {
199 if schema.index_of(name).is_ok() {
200 props_builder = props_builder
201 .set_column_dictionary_enabled(ColumnPath::from(name), false)
202 .set_column_encoding(ColumnPath::from(name), Encoding::DELTA_BINARY_PACKED);
203 }
204 }
205
206 let writer_props = props_builder.build();
207
208 let mut writer = ArrowWriter::try_new(&mut buffer, schema, Some(writer_props))?;
209 for batch in batches {
210 writer.write(batch)?;
211 }
212
213 writer.close()?;
214
215 object_store
217 .put_opts(
218 path,
219 buffer.into(),
220 PutOptions {
221 mode: put_mode,
222 ..Default::default()
223 },
224 )
225 .await?;
226
227 Ok(())
228}
229
230fn parquet_sorting_columns(schema: &Schema) -> anyhow::Result<Option<Vec<SortingColumn>>> {
231 if schema.index_of("ts_init").is_err() {
232 return Ok(None);
233 }
234
235 let parquet_schema = ArrowSchemaConverter::new().convert(schema)?;
236 let mut names = Vec::with_capacity(2);
237 names.push("ts_init");
238 if schema.index_of(KEY_IDENTIFIER).is_ok() {
239 names.push(KEY_IDENTIFIER);
240 }
241
242 let columns = names
243 .into_iter()
244 .map(|name| {
245 let index = parquet_schema
246 .columns()
247 .iter()
248 .position(|column| {
249 column
250 .path()
251 .parts()
252 .first()
253 .is_some_and(|part| part == name)
254 })
255 .ok_or_else(|| anyhow::anyhow!("Parquet schema is missing sort column {name}"))?;
256
257 Ok(SortingColumn {
258 column_idx: i32::try_from(index)?,
259 descending: false,
260 nulls_first: false,
261 })
262 })
263 .collect::<anyhow::Result<Vec<_>>>()?;
264
265 Ok(Some(columns))
266}
267
268fn deduplicate_record_batches(batches: &[RecordBatch]) -> anyhow::Result<Vec<RecordBatch>> {
278 if batches.is_empty() {
279 return Ok(Vec::new());
280 }
281
282 let schema = batches[0].schema();
283
284 let fields: Vec<arrow::row::SortField> = schema
285 .fields()
286 .iter()
287 .map(|f| arrow::row::SortField::new(f.data_type().clone()))
288 .collect();
289
290 let converter = arrow::row::RowConverter::new(fields)?;
291 let mut seen: std::collections::HashSet<Vec<u8>> = std::collections::HashSet::new();
292 let mut result: Vec<RecordBatch> = Vec::new();
293
294 for batch in batches {
295 let rows = converter.convert_columns(batch.columns())?;
296 let mut indices: Vec<u32> = Vec::new();
297
298 for (i, row) in rows.iter().enumerate() {
299 if seen.insert(row.as_ref().to_vec()) {
300 indices.push(u32::try_from(i)?);
301 }
302 }
303
304 if !indices.is_empty() {
305 let index_array = arrow::array::UInt32Array::from(indices);
306 let deduped_columns: Vec<arrow::array::ArrayRef> = batch
307 .columns()
308 .iter()
309 .map(|col| arrow::compute::take(col.as_ref(), &index_array, None))
310 .collect::<Result<_, _>>()?;
311 result.push(RecordBatch::try_new(schema.clone(), deduped_columns)?);
312 }
313 }
314
315 Ok(result)
316}
317
318pub async fn combine_parquet_files_from_object_store(
324 object_store: Arc<dyn ObjectStore>,
325 file_paths: Vec<ObjectPath>,
326 new_file_path: &ObjectPath,
327 compression: Option<Compression>,
328 max_row_group_size: Option<usize>,
329 deduplicate: Option<bool>,
330) -> anyhow::Result<()> {
331 let Some(merged) =
332 read_combined_batches(object_store.clone(), &file_paths, deduplicate).await?
333 else {
334 return Ok(());
335 };
336
337 write_batches_to_object_store(
338 &merged.batches,
339 object_store.clone(),
340 new_file_path,
341 compression,
342 max_row_group_size,
343 None,
344 )
345 .await?;
346
347 for source in merged.sources {
348 if source.location != *new_file_path {
349 object_store.delete(&source.location).await?;
350 }
351 }
352
353 Ok(())
354}
355
356pub(crate) async fn consolidate_parquet_files_from_object_store(
357 object_store: Arc<dyn ObjectStore>,
358 file_paths: Vec<ObjectPath>,
359 new_file_path: &ObjectPath,
360 compression: Option<Compression>,
361 max_row_group_size: Option<usize>,
362 deduplicate: Option<bool>,
363) -> anyhow::Result<()> {
364 let Some(merged) =
365 read_combined_batches(object_store.clone(), &file_paths, deduplicate).await?
366 else {
367 return Ok(());
368 };
369
370 let created_by =
371 consolidation_state::begin(&object_store, new_file_path, &merged.sources).await?;
372 write_batches_to_object_store_with_mode(
373 &merged.batches,
374 object_store.clone(),
375 new_file_path,
376 compression,
377 max_row_group_size,
378 None,
379 PutMode::Overwrite,
380 Some(created_by),
381 )
382 .await?;
383 let directory = new_file_path.parent().unwrap_or_default();
384 consolidation_state::recover(&object_store, &directory).await
385}
386
387struct MergedParquet {
388 batches: Vec<RecordBatch>,
389 sources: Vec<ObjectMeta>,
390}
391
392async fn read_combined_batches(
393 object_store: Arc<dyn ObjectStore>,
394 file_paths: &[ObjectPath],
395 deduplicate: Option<bool>,
396) -> anyhow::Result<Option<MergedParquet>> {
397 if file_paths.len() <= 1 {
398 return Ok(None);
399 }
400
401 let mut all_batches: Vec<RecordBatch> = Vec::new();
402 let mut schema_with_metadata: Option<Arc<arrow::datatypes::Schema>> = None;
403 let mut schema_source: Option<&ObjectPath> = None;
404 let mut field_metadata_sources = HashMap::new();
405 let mut sources = Vec::with_capacity(file_paths.len());
406
407 for path in file_paths {
409 let object = object_store.get(path).await?;
410 sources.push(object.meta.clone());
411 let data = object.bytes().await?;
412 let builder = ParquetRecordBatchReaderBuilder::try_new(data)?;
413
414 let candidate_schema = builder.schema().clone();
415 schema_with_metadata = Some(
416 if let (Some(schema), Some(source)) = (&schema_with_metadata, schema_source) {
417 let reconciled = reconcile_consolidation_schema_with_sources(
418 schema,
419 source,
420 &field_metadata_sources,
421 &candidate_schema,
422 path,
423 )?;
424
425 if reconciled.schema_source == ConsolidationSchemaSource::Candidate {
426 schema_source = Some(path);
427 }
428
429 for key in reconciled.candidate_field_metadata {
430 field_metadata_sources.insert(key, path.clone());
431 }
432
433 reconciled.schema
434 } else {
435 schema_source = Some(path);
436 field_metadata_sources
437 .extend(field_metadata_keys(&candidate_schema).map(|key| (key, path.clone())));
438 candidate_schema
439 },
440 );
441
442 for batch in builder.build()? {
443 all_batches.push(batch?);
444 }
445 }
446
447 let schema = schema_with_metadata.expect("non-empty source files provide a schema");
451 all_batches = all_batches
452 .into_iter()
453 .map(|b| RecordBatch::try_new(schema.clone(), b.columns().to_vec()))
454 .collect::<Result<Vec<_>, _>>()?;
455
456 let batches_to_write = if deduplicate.unwrap_or(false) {
458 deduplicate_record_batches(&all_batches)?
459 } else {
460 all_batches
461 };
462
463 let batches_to_write = sort_consolidation_batches(batches_to_write)?;
464 Ok(Some(MergedParquet {
465 batches: batches_to_write,
466 sources,
467 }))
468}
469
470fn sort_consolidation_batches(batches: Vec<RecordBatch>) -> anyhow::Result<Vec<RecordBatch>> {
471 let Some(first) = batches.first() else {
472 return Ok(batches);
473 };
474
475 if first.column_by_name("ts_init").is_none() {
476 return Ok(batches);
477 }
478
479 let mut previous = 0;
480 let mut ordered = true;
481
482 for batch in &batches {
483 let array = batch
484 .column_by_name("ts_init")
485 .expect("consistent batch schema");
486 let timestamps = U64ColumnRef::try_from_array(array.as_ref())
487 .ok_or_else(|| anyhow::anyhow!("Unsupported consolidation ts_init type"))?;
488
489 for row in 0..batch.num_rows() {
490 let timestamp = timestamps
491 .value(row)
492 .ok_or_else(|| anyhow::anyhow!("Invalid consolidation ts_init value"))?;
493 ordered &= previous <= timestamp;
494 previous = timestamp;
495 }
496 }
497
498 if ordered {
499 return Ok(batches);
500 }
501
502 let schema = first.schema();
503 let batch = concat_batches(&schema, &batches)?;
504 let original_index = Arc::new(UInt64Array::from_iter_values(0..batch.num_rows() as u64));
505
506 let options = Some(SortOptions {
507 descending: false,
508 nulls_first: false,
509 });
510
511 let indices = lexsort_to_indices(
512 &[
513 SortColumn {
514 values: batch
515 .column_by_name("ts_init")
516 .expect("ts_init exists")
517 .clone(),
518 options,
519 },
520 SortColumn {
521 values: original_index,
522 options,
523 },
524 ],
525 None,
526 )?;
527
528 Ok(vec![take_record_batch(&batch, &indices)?])
529}
530
531#[derive(Clone, Copy, Debug, Eq, PartialEq)]
532enum ConsolidationSchemaSource {
533 Current,
534 Candidate,
535}
536
537#[derive(Debug)]
538struct ReconciledConsolidationSchema {
539 schema: Arc<Schema>,
540 schema_source: ConsolidationSchemaSource,
541 candidate_field_metadata: Vec<(String, String)>,
542}
543
544fn reconcile_consolidation_schema_with_sources(
545 current: &Arc<Schema>,
546 current_path: &ObjectPath,
547 current_field_metadata_sources: &HashMap<(String, String), ObjectPath>,
548 candidate: &Arc<Schema>,
549 candidate_path: &ObjectPath,
550) -> anyhow::Result<ReconciledConsolidationSchema> {
551 anyhow::ensure!(
552 current.fields().len() == candidate.fields().len(),
553 "Cannot consolidate Parquet files {current_path} and {candidate_path}: field schemas differ"
554 );
555 let mut candidate_field_metadata = Vec::new();
556
557 let fields = current
558 .fields()
559 .iter()
560 .zip(candidate.fields())
561 .map(|(current, candidate)| {
562 anyhow::ensure!(
563 current.name() == candidate.name()
564 && current.data_type() == candidate.data_type()
565 && current.is_nullable() == candidate.is_nullable(),
566 "Cannot consolidate Parquet files {current_path} and {candidate_path}: field schemas differ"
567 );
568 let mut metadata = current.metadata().clone();
569 for (key, value) in candidate.metadata() {
570 if let Some(current_value) = metadata.get(key) {
571 let source = current_field_metadata_sources
572 .get(&(current.name().clone(), key.clone()))
573 .unwrap_or(current_path);
574 anyhow::ensure!(
575 current_value == value,
576 "Cannot consolidate Parquet files {source} and {candidate_path}: field '{}' metadata differs",
577 current.name(),
578 );
579 } else {
580 metadata.insert(key.clone(), value.clone());
581 candidate_field_metadata.push((current.name().clone(), key.clone()));
582 }
583 }
584
585 Ok(Arc::new(current.as_ref().clone().with_metadata(metadata)))
586 })
587 .collect::<anyhow::Result<Vec<_>>>()?;
588
589 let schema_with_fields =
590 |metadata| Arc::new(Schema::new_with_metadata(fields.clone(), metadata));
591
592 if current.metadata() == candidate.metadata() {
593 return Ok(ReconciledConsolidationSchema {
594 schema: schema_with_fields(current.metadata().clone()),
595 schema_source: ConsolidationSchemaSource::Current,
596 candidate_field_metadata,
597 });
598 }
599
600 let without_precision = |schema: &Schema| {
601 let mut metadata = schema.metadata().clone();
602 metadata.remove(KEY_PRICE_PRECISION);
603 metadata.remove(KEY_SIZE_PRECISION);
604 metadata
605 };
606
607 let is_precision_fallback = |metadata: &HashMap<String, String>| {
608 metadata.get(KEY_PRICE_PRECISION).map(String::as_str) == Some("0")
609 && metadata.get(KEY_SIZE_PRECISION).map(String::as_str) == Some("0")
610 };
611
612 let has_precision = |metadata: &HashMap<String, String>| {
613 metadata.contains_key(KEY_PRICE_PRECISION) && metadata.contains_key(KEY_SIZE_PRECISION)
614 };
615
616 let current_metadata = current.metadata();
617 let candidate_metadata = candidate.metadata();
618
619 if without_precision(current) == without_precision(candidate) {
620 match (
621 is_precision_fallback(current_metadata),
622 is_precision_fallback(candidate_metadata),
623 ) {
624 (true, false) if has_precision(candidate_metadata) => {
625 return Ok(ReconciledConsolidationSchema {
626 schema: schema_with_fields(candidate.metadata().clone()),
627 schema_source: ConsolidationSchemaSource::Candidate,
628 candidate_field_metadata,
629 });
630 }
631 (false, true) if has_precision(current_metadata) => {
632 return Ok(ReconciledConsolidationSchema {
633 schema: schema_with_fields(current.metadata().clone()),
634 schema_source: ConsolidationSchemaSource::Current,
635 candidate_field_metadata,
636 });
637 }
638 _ => {}
639 }
640 }
641
642 anyhow::bail!(
643 "Cannot consolidate Parquet files {current_path} and {candidate_path}: schema metadata differs: {current_metadata:?} versus {candidate_metadata:?}"
644 )
645}
646
647fn field_metadata_keys(schema: &Schema) -> impl Iterator<Item = (String, String)> + '_ {
648 schema.fields().iter().flat_map(|field| {
649 field
650 .metadata()
651 .keys()
652 .map(|key| (field.name().clone(), key.clone()))
653 })
654}
655
656pub async fn min_max_from_parquet_metadata_object_store(
662 object_store: Arc<dyn ObjectStore>,
663 file_path: &ObjectPath,
664 column_name: &str,
665) -> anyhow::Result<(u64, u64)> {
666 let data = object_store.get(file_path).await?.bytes().await?;
668 let reader = SerializedFileReader::new(data)?;
669
670 let metadata = reader.metadata();
671 let mut overall_min_value: Option<i64> = None;
672 let mut overall_max_value: Option<i64> = None;
673
674 for i in 0..metadata.num_row_groups() {
676 let row_group = metadata.row_group(i);
677
678 for j in 0..row_group.num_columns() {
680 let col_metadata = row_group.column(j);
681
682 if col_metadata.column_path().string() == column_name {
683 if let Some(stats) = col_metadata.statistics() {
684 if let Statistics::Int64(int64_stats) = stats {
686 if let Some(&min_value) = int64_stats.min_opt()
688 && (overall_min_value.is_none()
689 || min_value < overall_min_value.unwrap())
690 {
691 overall_min_value = Some(min_value);
692 }
693
694 if let Some(&max_value) = int64_stats.max_opt()
696 && (overall_max_value.is_none()
697 || max_value > overall_max_value.unwrap())
698 {
699 overall_max_value = Some(max_value);
700 }
701 } else {
702 anyhow::bail!("Warning: Column name '{column_name}' is not of type i64.");
703 }
704 } else {
705 anyhow::bail!(
706 "Warning: Statistics not available for column '{column_name}' in row group {i}."
707 );
708 }
709 }
710 }
711 }
712
713 if let (Some(min), Some(max)) = (overall_min_value, overall_max_value) {
715 Ok((u64::try_from(min)?, u64::try_from(max)?))
716 } else {
717 anyhow::bail!(
718 "Column '{column_name}' not found or has no Int64 statistics in any row group."
719 )
720 }
721}
722
723pub fn create_object_store_from_path(
750 path: &str,
751 storage_options: Option<AHashMap<String, String>>,
752) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
753 let location = create_object_store_location_from_path(path, storage_options)?;
754 Ok((
755 location.object_store,
756 location.base_path,
757 location.original_uri,
758 ))
759}
760
761#[cfg_attr(
764 not(feature = "cloud"),
765 allow(unused_variables, clippy::needless_pass_by_value)
766)]
767pub(crate) fn create_object_store_location_from_path(
768 path: &str,
769 storage_options: Option<AHashMap<String, String>>,
770) -> anyhow::Result<ObjectStoreLocation> {
771 let uri = normalize_path_to_uri(path)?;
772
773 let (object_store, base_path, original_uri) = match uri.as_str() {
774 #[cfg(feature = "cloud")]
775 s if s.starts_with("s3://") => create_s3_store(&uri, storage_options),
776 #[cfg(feature = "cloud")]
777 s if s.starts_with("gs://") || s.starts_with("gcs://") => {
778 create_gcs_store(&uri, storage_options)
779 }
780 #[cfg(feature = "cloud")]
781 s if s.starts_with("az://") => create_azure_store(&uri, storage_options),
782 #[cfg(feature = "cloud")]
783 s if s.starts_with("abfs://") => create_abfs_store(&uri, storage_options),
784 #[cfg(feature = "cloud")]
785 s if s.starts_with("http://") || s.starts_with("https://") => {
786 create_http_store(&uri, storage_options)
787 }
788 #[cfg(not(feature = "cloud"))]
789 s if s.starts_with("s3://")
790 || s.starts_with("gs://")
791 || s.starts_with("gcs://")
792 || s.starts_with("az://")
793 || s.starts_with("abfs://")
794 || s.starts_with("http://")
795 || s.starts_with("https://") =>
796 {
797 anyhow::bail!("Cloud storage support requires the 'cloud' feature: {uri}")
798 }
799 s if s.starts_with("file://") => create_local_store(&uri, true),
800 _ => create_local_store(&uri, false), }?;
802
803 Ok(ObjectStoreLocation {
804 object_store,
805 base_path,
806 original_uri,
807 })
808}
809
810pub(crate) fn is_remote_uri_scheme(scheme: &str) -> bool {
811 matches!(
812 scheme,
813 "s3" | "gs" | "gcs" | "az" | "abfs" | "http" | "https"
814 )
815}
816
817pub(crate) fn remote_store_root_url(uri: &str) -> anyhow::Result<Url> {
818 let mut url = Url::parse(uri)?;
819 url.set_path("");
820 url.set_query(None);
821 url.set_fragment(None);
822 Ok(url)
823}
824
825pub(crate) fn remote_full_uri(uri: &str, object_path: &str) -> anyhow::Result<String> {
826 let root = remote_store_root_url(uri)?;
827 let root = root.as_str().trim_end_matches('/');
828 let object_path = object_path.trim_start_matches('/');
829
830 if object_path.is_empty() {
831 Ok(root.to_string())
832 } else {
833 Ok(format!("{root}/{object_path}"))
834 }
835}
836
837pub(crate) fn append_path_to_file_uri(base_uri: &str, path: &str) -> String {
840 if let Ok(mut url) = Url::parse(base_uri) {
841 if let Ok(mut segments) = url.path_segments_mut() {
842 segments.pop_if_empty();
843 segments.extend(
844 path.trim_end_matches('/')
845 .split('/')
846 .filter(|segment| !segment.is_empty()),
847 );
848 }
849
850 return url.to_string();
851 }
852
853 format!(
854 "{}/{}",
855 base_uri.trim_end_matches('/'),
856 path.trim_end_matches('/')
857 )
858}
859
860pub(crate) fn decode_object_store_segment(segment: &str) -> String {
867 ObjectPath::from_url_path(segment).map_or_else(|_| segment.to_string(), String::from)
868}
869
870fn create_local_store(
871 uri: &str,
872 is_file_uri: bool,
873) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
874 let path = if is_file_uri {
875 file_uri_to_native_path(uri)
876 } else {
877 uri.to_string()
878 };
879
880 let local_store =
881 object_store::local::LocalFileSystem::new_with_prefix(&path).with_context(|| {
882 format!(
883 "failed to open local storage directory '{path}'; \
884 create it if it does not exist and check access permissions"
885 )
886 })?;
887
888 Ok((Arc::new(local_store), String::new(), uri.to_string()))
889}
890
891#[cfg(feature = "cloud")]
893fn create_s3_store(
894 uri: &str,
895 storage_options: Option<AHashMap<String, String>>,
896) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
897 let (url, path) = parse_url_and_path(uri)?;
898 let bucket = extract_host(&url, "Invalid S3 URI: missing bucket")?;
899
900 let mut builder = object_store::aws::AmazonS3Builder::new().with_bucket_name(&bucket);
901
902 if let Some(options) = storage_options {
904 for (key, value) in options {
905 let config_key = match key.as_str() {
907 "key" => object_store::aws::AmazonS3ConfigKey::AccessKeyId,
908 "secret" => object_store::aws::AmazonS3ConfigKey::SecretAccessKey,
909 _ => key.parse()?,
910 };
911 builder = builder.with_config(config_key, value);
912 }
913 }
914
915 let s3_store = builder.build()?;
916 Ok((Arc::new(s3_store), path, uri.to_string()))
917}
918
919#[cfg(feature = "cloud")]
921fn create_gcs_store(
922 uri: &str,
923 storage_options: Option<AHashMap<String, String>>,
924) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
925 let (url, path) = parse_url_and_path(uri)?;
926 let bucket = extract_host(&url, "Invalid GCS URI: missing bucket")?;
927
928 let mut builder = object_store::gcp::GoogleCloudStorageBuilder::new().with_bucket_name(&bucket);
929
930 if let Some(options) = storage_options {
932 for (key, value) in options {
933 builder =
934 builder.with_config(key.parse::<object_store::gcp::GoogleConfigKey>()?, value);
935 }
936 }
937
938 let gcs_store = builder.build()?;
939 Ok((Arc::new(gcs_store), path, uri.to_string()))
940}
941
942#[cfg(feature = "cloud")]
944fn create_azure_store(
945 uri: &str,
946 storage_options: Option<AHashMap<String, String>>,
947) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
948 let (url, path) = parse_url_and_path(uri)?;
949 let container = extract_host(&url, "Invalid Azure URI: missing container")?;
950
951 let mut builder =
952 object_store::azure::MicrosoftAzureBuilder::new().with_container_name(container);
953
954 if let Some(options) = storage_options {
956 builder = apply_azure_storage_options(builder, options)?;
957 }
958
959 let azure_store = builder.build()?;
960 Ok((Arc::new(azure_store), path, uri.to_string()))
961}
962
963#[cfg(feature = "cloud")]
965fn create_abfs_store(
966 uri: &str,
967 storage_options: Option<AHashMap<String, String>>,
968) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
969 let (url, path) = parse_url_and_path(uri)?;
970 let host = extract_host(&url, "Invalid ABFS URI: missing host")?;
971
972 let account = host
974 .split('.')
975 .next()
976 .ok_or_else(|| anyhow::anyhow!("Invalid ABFS URI: cannot extract account from host"))?;
977
978 let container = url
980 .username()
981 .split('@')
982 .next()
983 .ok_or_else(|| anyhow::anyhow!("Invalid ABFS URI: missing container"))?;
984
985 let mut builder = object_store::azure::MicrosoftAzureBuilder::new()
986 .with_account(account)
987 .with_container_name(container);
988
989 if let Some(options) = storage_options {
991 builder = apply_azure_storage_options(builder, options)?;
992 }
993
994 let azure_store = builder.build()?;
995 Ok((Arc::new(azure_store), path, uri.to_string()))
996}
997
998#[cfg(feature = "cloud")]
1000fn apply_azure_storage_options(
1001 mut builder: object_store::azure::MicrosoftAzureBuilder,
1002 options: AHashMap<String, String>,
1003) -> anyhow::Result<object_store::azure::MicrosoftAzureBuilder> {
1004 for (key, value) in options {
1005 builder = builder.with_config(key.parse::<object_store::azure::AzureConfigKey>()?, value);
1006 }
1007
1008 Ok(builder)
1009}
1010
1011#[cfg(feature = "cloud")]
1013fn create_http_store(
1014 uri: &str,
1015 storage_options: Option<AHashMap<String, String>>,
1016) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
1017 let (_, path) = parse_url_and_path(uri)?;
1018 let base_url = remote_store_root_url(uri)?
1019 .as_str()
1020 .trim_end_matches('/')
1021 .to_string();
1022
1023 let mut builder = object_store::http::HttpBuilder::new().with_url(base_url);
1024
1025 if let Some(options) = storage_options {
1027 for (key, value) in options {
1028 builder = builder.with_config(key.parse::<object_store::ClientConfigKey>()?, value);
1029 }
1030 }
1031
1032 let http_store = builder.build()?;
1033 Ok((Arc::new(http_store), path, uri.to_string()))
1034}
1035
1036#[cfg(feature = "cloud")]
1041fn parse_url_and_path(uri: &str) -> anyhow::Result<(Url, String)> {
1042 let url = Url::parse(uri)?;
1043 let path = url.path().trim_start_matches('/').to_string();
1044 let raw_path = uri
1045 .split_once("://")
1046 .and_then(|(_, rest)| rest.split_once('/'))
1047 .map_or("", |(_, raw)| raw.trim_start_matches('/'));
1048
1049 anyhow::ensure!(
1050 raw_path == path && ObjectPath::from(path.as_str()).as_ref() == path.trim_end_matches('/'),
1051 "Storage URI '{uri}' has a base path that URL parsing or object-store path encoding \
1052 changes; use a base path without spaces, non-ASCII, or reserved characters",
1053 );
1054
1055 Ok((url, path))
1056}
1057
1058#[cfg(feature = "cloud")]
1060fn extract_host(url: &Url, error_msg: &str) -> anyhow::Result<String> {
1061 url.host_str()
1062 .map(ToString::to_string)
1063 .ok_or_else(|| anyhow::anyhow!("{error_msg}"))
1064}
1065
1066#[cfg(test)]
1067mod tests {
1068 #[cfg(feature = "cloud")]
1069 use std::io::{BufRead, Write};
1070 use std::{collections::HashMap, sync::Arc};
1071
1072 #[cfg(feature = "cloud")]
1073 use ahash::AHashMap;
1074 use arrow::{
1075 array::{Array, ArrayRef, StringArray, TimestampNanosecondArray, UInt64Array},
1076 datatypes::{DataType, Field, Schema},
1077 };
1078 use nautilus_serialization::arrow::{json_string_field, timestamp_data_type};
1079 use parquet::file::{properties::ReaderProperties, serialized_reader::ReadOptionsBuilder};
1080 use rstest::rstest;
1081
1082 use super::*;
1083 use crate::common::paths::escape_local_file_uri;
1084
1085 #[rstest]
1086 #[tokio::test]
1087 async fn combine_accepts_an_output_outside_the_source_directory() {
1088 use futures::TryStreamExt;
1089
1090 let store: Arc<dyn ObjectStore> = Arc::new(object_store::memory::InMemory::new());
1091
1092 let schema = Arc::new(Schema::new(vec![Field::new(
1093 "value",
1094 DataType::UInt64,
1095 false,
1096 )]));
1097
1098 let paths = [
1099 ObjectPath::from("inputs/one.parquet"),
1100 ObjectPath::from("inputs/two.parquet"),
1101 ];
1102
1103 for (path, value) in paths.iter().zip([11, 22]) {
1104 let batch = RecordBatch::try_new(
1105 schema.clone(),
1106 vec![Arc::new(UInt64Array::from(vec![value]))],
1107 )
1108 .unwrap();
1109 write_batches_to_object_store(&[batch], store.clone(), path, None, None, None)
1110 .await
1111 .unwrap();
1112 }
1113
1114 let target = ObjectPath::from("outputs/combined.parquet");
1115 combine_parquet_files_from_object_store(
1116 store.clone(),
1117 paths.to_vec(),
1118 &target,
1119 None,
1120 None,
1121 None,
1122 )
1123 .await
1124 .unwrap();
1125 let (batches, actual_schema) = read_parquet_from_object_store(store.clone(), &target)
1126 .await
1127 .unwrap();
1128 let actual = concat_batches(&actual_schema, &batches).unwrap();
1129 let expected =
1130 RecordBatch::try_new(schema, vec![Arc::new(UInt64Array::from(vec![11, 22]))]).unwrap();
1131 let remaining = store.list(None).try_collect::<Vec<_>>().await.unwrap();
1132
1133 assert_eq!(actual, expected);
1134 assert_eq!(
1135 remaining
1136 .iter()
1137 .map(|meta| &meta.location)
1138 .collect::<Vec<_>>(),
1139 vec![&target]
1140 );
1141 }
1142
1143 fn consolidation_depth_schema(
1144 price_precision: &str,
1145 size_precision: &str,
1146 instrument_id: &str,
1147 ) -> Arc<Schema> {
1148 Arc::new(Schema::new_with_metadata(
1149 vec![Field::new("bids", DataType::Utf8, false)],
1150 HashMap::from([
1151 (KEY_PRICE_PRECISION.to_string(), price_precision.to_string()),
1152 (KEY_SIZE_PRECISION.to_string(), size_precision.to_string()),
1153 (KEY_IDENTIFIER.to_string(), instrument_id.to_string()),
1154 ]),
1155 ))
1156 }
1157
1158 fn reconcile_consolidation_schema(
1159 current: &Arc<Schema>,
1160 current_path: &ObjectPath,
1161 candidate: &Arc<Schema>,
1162 candidate_path: &ObjectPath,
1163 ) -> anyhow::Result<ReconciledConsolidationSchema> {
1164 let field_metadata_sources = field_metadata_keys(current)
1165 .map(|key| (key, current_path.clone()))
1166 .collect();
1167 reconcile_consolidation_schema_with_sources(
1168 current,
1169 current_path,
1170 &field_metadata_sources,
1171 candidate,
1172 candidate_path,
1173 )
1174 }
1175
1176 #[rstest]
1177 fn consolidation_schema_prefers_populated_depth_precision_in_either_order() {
1178 let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1179 let populated = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1180 let fallback_path = ObjectPath::from("empty.parquet");
1181 let populated_path = ObjectPath::from("populated.parquet");
1182
1183 let fallback_first =
1184 reconcile_consolidation_schema(&fallback, &fallback_path, &populated, &populated_path)
1185 .unwrap();
1186 let populated_first =
1187 reconcile_consolidation_schema(&populated, &populated_path, &fallback, &fallback_path)
1188 .unwrap();
1189
1190 assert_eq!(
1191 fallback_first.schema_source,
1192 ConsolidationSchemaSource::Candidate,
1193 );
1194 assert_eq!(
1195 populated_first.schema_source,
1196 ConsolidationSchemaSource::Current,
1197 );
1198
1199 for schema in [fallback_first.schema, populated_first.schema] {
1200 assert_eq!(schema.metadata()[KEY_PRICE_PRECISION], "2");
1201 assert_eq!(schema.metadata()[KEY_SIZE_PRECISION], "3");
1202 }
1203 }
1204
1205 #[rstest]
1206 fn consolidation_schema_rejects_other_metadata_mismatches() {
1207 let current = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1208 let candidate = consolidation_depth_schema("2", "3", "BTCUSDT.BINANCE");
1209 let current_path = ObjectPath::from("eth.parquet");
1210 let candidate_path = ObjectPath::from("btc.parquet");
1211
1212 let error =
1213 reconcile_consolidation_schema(¤t, ¤t_path, &candidate, &candidate_path)
1214 .unwrap_err();
1215
1216 assert!(error.to_string().contains("eth.parquet and btc.parquet"));
1217 assert!(error.to_string().contains("ETHUSDT.BINANCE"));
1218 assert!(error.to_string().contains("BTCUSDT.BINANCE"));
1219 }
1220
1221 #[rstest]
1222 fn consolidation_schema_rejects_two_populated_precisions() {
1223 let current = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1224 let candidate = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1225 let current_path = ObjectPath::from("precision-2.parquet");
1226 let candidate_path = ObjectPath::from("precision-4.parquet");
1227
1228 let error =
1229 reconcile_consolidation_schema(¤t, ¤t_path, &candidate, &candidate_path)
1230 .unwrap_err();
1231
1232 assert!(error.to_string().contains("precision-2.parquet"));
1233 assert!(error.to_string().contains("precision-4.parquet"));
1234 }
1235
1236 #[rstest]
1237 fn consolidation_schema_keeps_fallback_for_all_empty_files() {
1238 let first = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1239 let second = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1240
1241 let reconciled = reconcile_consolidation_schema(
1242 &first,
1243 &ObjectPath::from("first-empty.parquet"),
1244 &second,
1245 &ObjectPath::from("second-empty.parquet"),
1246 )
1247 .unwrap();
1248
1249 assert_eq!(reconciled.schema.metadata()[KEY_PRICE_PRECISION], "0");
1250 assert_eq!(reconciled.schema.metadata()[KEY_SIZE_PRECISION], "0");
1251 }
1252
1253 #[rstest]
1254 #[case::missing_first(true)]
1255 #[case::fallback_first(false)]
1256 fn consolidation_schema_rejects_missing_precision_against_fallback(
1257 #[case] missing_first: bool,
1258 ) {
1259 let missing = Arc::new(Schema::new_with_metadata(
1260 vec![Field::new("bids", DataType::Utf8, false)],
1261 HashMap::from([(KEY_IDENTIFIER.to_string(), "ETHUSDT.BINANCE".to_string())]),
1262 ));
1263 let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1264 let missing_path = ObjectPath::from("missing.parquet");
1265 let fallback_path = ObjectPath::from("fallback.parquet");
1266
1267 let error = if missing_first {
1268 reconcile_consolidation_schema(&missing, &missing_path, &fallback, &fallback_path)
1269 } else {
1270 reconcile_consolidation_schema(&fallback, &fallback_path, &missing, &missing_path)
1271 }
1272 .unwrap_err();
1273
1274 assert!(error.to_string().contains("schema metadata differs"));
1275 }
1276
1277 #[rstest]
1278 fn consolidation_schema_merges_json_field_annotation_in_either_order() {
1279 let bare = Arc::new(Schema::new(vec![Field::new("info", DataType::Utf8, true)]));
1280 let annotated = Arc::new(Schema::new(vec![json_string_field("info", true)]));
1281 let bare_path = ObjectPath::from("bare.parquet");
1282 let annotated_path = ObjectPath::from("annotated.parquet");
1283
1284 let bare_first =
1285 reconcile_consolidation_schema(&bare, &bare_path, &annotated, &annotated_path).unwrap();
1286 let annotated_first =
1287 reconcile_consolidation_schema(&annotated, &annotated_path, &bare, &bare_path).unwrap();
1288
1289 assert_eq!(bare_first.schema, annotated_first.schema);
1290 assert_eq!(
1291 bare_first.schema.field_with_name("info").unwrap(),
1292 &json_string_field("info", true),
1293 );
1294 }
1295
1296 #[rstest]
1297 fn consolidation_field_metadata_conflict_names_the_winning_file() {
1298 let bare = Arc::new(Schema::new(vec![Field::new("info", DataType::Utf8, true)]));
1299 let annotated = Arc::new(Schema::new(vec![json_string_field("info", true)]));
1300
1301 let conflicting = Arc::new(Schema::new(vec![
1302 Field::new("info", DataType::Utf8, true).with_metadata(HashMap::from([(
1303 "ARROW:extension:name".to_string(),
1304 "other.extension".to_string(),
1305 )])),
1306 ]));
1307 let bare_path = ObjectPath::from("bare.parquet");
1308 let annotated_path = ObjectPath::from("annotated.parquet");
1309 let conflicting_path = ObjectPath::from("conflicting.parquet");
1310 let reconciled =
1311 reconcile_consolidation_schema(&bare, &bare_path, &annotated, &annotated_path).unwrap();
1312
1313 assert_eq!(reconciled.candidate_field_metadata.len(), 2);
1314 assert!(
1315 reconciled
1316 .candidate_field_metadata
1317 .contains(&("info".to_string(), "ARROW:extension:name".to_string())),
1318 );
1319 assert!(
1320 reconciled
1321 .candidate_field_metadata
1322 .contains(&("info".to_string(), "ARROW:extension:metadata".to_string())),
1323 );
1324 let field_metadata_sources = reconciled
1325 .candidate_field_metadata
1326 .iter()
1327 .cloned()
1328 .map(|key| (key, annotated_path.clone()))
1329 .collect();
1330 let error = reconcile_consolidation_schema_with_sources(
1331 &reconciled.schema,
1332 &bare_path,
1333 &field_metadata_sources,
1334 &conflicting,
1335 &conflicting_path,
1336 )
1337 .unwrap_err();
1338 assert!(error.to_string().contains("annotated.parquet"));
1339 assert!(error.to_string().contains("conflicting.parquet"));
1340 assert!(error.to_string().contains("field 'info' metadata differs"));
1341 }
1342
1343 #[rstest]
1344 fn consolidation_conflict_names_the_last_winning_schema() {
1345 let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1346 let precision_2 = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1347 let precision_4 = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1348 let fallback_path = ObjectPath::from("fallback.parquet");
1349 let precision_2_path = ObjectPath::from("precision-2.parquet");
1350 let precision_4_path = ObjectPath::from("precision-4.parquet");
1351 let reconciled = reconcile_consolidation_schema(
1352 &fallback,
1353 &fallback_path,
1354 &precision_2,
1355 &precision_2_path,
1356 )
1357 .unwrap();
1358
1359 let error = reconcile_consolidation_schema(
1360 &reconciled.schema,
1361 &precision_2_path,
1362 &precision_4,
1363 &precision_4_path,
1364 )
1365 .unwrap_err();
1366
1367 assert!(error.to_string().contains("precision-2.parquet"));
1368 assert!(error.to_string().contains("precision-4.parquet"));
1369 }
1370
1371 #[rstest]
1372 fn consolidation_field_metadata_does_not_replace_precision_source() {
1373 let precision_2 = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1374
1375 let fallback = Arc::new(Schema::new_with_metadata(
1376 vec![json_string_field("bids", false)],
1377 consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE")
1378 .metadata()
1379 .clone(),
1380 ));
1381 let precision_4 = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1382 let precision_2_path = ObjectPath::from("precision-2.parquet");
1383 let fallback_path = ObjectPath::from("fallback-annotated.parquet");
1384 let precision_4_path = ObjectPath::from("precision-4.parquet");
1385 let reconciled = reconcile_consolidation_schema(
1386 &precision_2,
1387 &precision_2_path,
1388 &fallback,
1389 &fallback_path,
1390 )
1391 .unwrap();
1392
1393 assert_eq!(reconciled.schema_source, ConsolidationSchemaSource::Current,);
1394 assert_eq!(reconciled.candidate_field_metadata.len(), 2);
1395 assert!(
1396 reconciled
1397 .candidate_field_metadata
1398 .contains(&("bids".to_string(), "ARROW:extension:name".to_string())),
1399 );
1400 assert!(
1401 reconciled
1402 .candidate_field_metadata
1403 .contains(&("bids".to_string(), "ARROW:extension:metadata".to_string())),
1404 );
1405 let field_metadata_sources = reconciled
1406 .candidate_field_metadata
1407 .iter()
1408 .cloned()
1409 .map(|key| (key, fallback_path.clone()))
1410 .collect();
1411 let error = reconcile_consolidation_schema_with_sources(
1412 &reconciled.schema,
1413 &precision_2_path,
1414 &field_metadata_sources,
1415 &precision_4,
1416 &precision_4_path,
1417 )
1418 .unwrap_err();
1419 let message = error.to_string();
1420
1421 assert!(message.contains("precision-2.parquet"));
1422 assert!(message.contains("precision-4.parquet"));
1423 assert!(!message.contains("fallback-annotated.parquet"));
1424 }
1425
1426 #[rstest]
1427 fn consolidation_field_metadata_tracks_each_origin() {
1428 let bare = Arc::new(Schema::new(vec![
1429 Field::new("info", DataType::Utf8, true),
1430 Field::new("balances", DataType::Utf8, true),
1431 ]));
1432
1433 let info = Arc::new(Schema::new(vec![
1434 json_string_field("info", true),
1435 Field::new("balances", DataType::Utf8, true),
1436 ]));
1437
1438 let balances = Arc::new(Schema::new(vec![
1439 json_string_field("info", true),
1440 json_string_field("balances", true),
1441 ]));
1442
1443 let conflicting = Arc::new(Schema::new(vec![
1444 Field::new("info", DataType::Utf8, true).with_metadata(HashMap::from([(
1445 "ARROW:extension:name".to_string(),
1446 "other.extension".to_string(),
1447 )])),
1448 json_string_field("balances", true),
1449 ]));
1450 let bare_path = ObjectPath::from("bare.parquet");
1451 let info_path = ObjectPath::from("info.parquet");
1452 let balances_path = ObjectPath::from("balances.parquet");
1453 let conflicting_path = ObjectPath::from("conflicting.parquet");
1454 let with_info =
1455 reconcile_consolidation_schema(&bare, &bare_path, &info, &info_path).unwrap();
1456 let mut field_metadata_sources = with_info
1457 .candidate_field_metadata
1458 .iter()
1459 .cloned()
1460 .map(|key| (key, info_path.clone()))
1461 .collect::<HashMap<_, _>>();
1462 let with_balances = reconcile_consolidation_schema_with_sources(
1463 &with_info.schema,
1464 &bare_path,
1465 &field_metadata_sources,
1466 &balances,
1467 &balances_path,
1468 )
1469 .unwrap();
1470
1471 for key in with_balances.candidate_field_metadata {
1472 field_metadata_sources.insert(key, balances_path.clone());
1473 }
1474
1475 let error = reconcile_consolidation_schema_with_sources(
1476 &with_balances.schema,
1477 &bare_path,
1478 &field_metadata_sources,
1479 &conflicting,
1480 &conflicting_path,
1481 )
1482 .unwrap_err();
1483 let message = error.to_string();
1484
1485 assert!(message.contains("info.parquet"));
1486 assert!(message.contains("conflicting.parquet"));
1487 assert!(!message.contains("balances.parquet"));
1488 }
1489
1490 #[tokio::test]
1491 async fn write_batches_to_object_store_rejects_empty_input() {
1492 let object_store = Arc::new(object_store::memory::InMemory::new());
1493
1494 let error = write_batches_to_object_store(
1495 &[],
1496 object_store,
1497 &ObjectPath::from("empty.parquet"),
1498 None,
1499 None,
1500 None,
1501 )
1502 .await
1503 .unwrap_err();
1504
1505 assert_eq!(
1506 error.to_string(),
1507 "Cannot write Parquet file empty.parquet with no record batches"
1508 );
1509 }
1510
1511 #[tokio::test]
1512 async fn default_writer_sets_zstd_sorting_and_identifier_bloom_filter() {
1513 let directory = tempfile::tempdir().unwrap();
1514 let path = directory.path().join("layout.parquet");
1515
1516 let object_store = Arc::new(
1517 object_store::local::LocalFileSystem::new_with_prefix(directory.path()).unwrap(),
1518 );
1519 let object_path = ObjectPath::from("layout.parquet");
1520
1521 let schema = Arc::new(Schema::new(vec![
1522 Field::new("identifier", DataType::Utf8, false),
1523 Field::new("ts_init", DataType::UInt64, false),
1524 ]));
1525 let batch = RecordBatch::try_new(
1526 schema,
1527 vec![
1528 Arc::new(StringArray::from(vec!["AUD/USD.SIM", "AUD/USD.SIM"])) as ArrayRef,
1529 Arc::new(UInt64Array::from(vec![1_u64, 2])) as ArrayRef,
1530 ],
1531 )
1532 .unwrap();
1533
1534 write_batches_to_object_store(&[batch], object_store, &object_path, None, None, None)
1535 .await
1536 .unwrap();
1537
1538 let read_options = ReadOptionsBuilder::new()
1539 .with_reader_properties(
1540 ReaderProperties::builder()
1541 .set_read_bloom_filter(true)
1542 .build(),
1543 )
1544 .build();
1545 let reader = SerializedFileReader::new_with_options(
1546 std::fs::File::open(path).unwrap(),
1547 read_options,
1548 )
1549 .unwrap();
1550 let row_group = reader.metadata().row_group(0);
1551 let sorting = row_group.sorting_columns().unwrap();
1552
1553 assert_eq!(crate::backend::parquet::DEFAULT_ROW_GROUP_SIZE, 131_072);
1554 assert_eq!(
1555 sorting,
1556 &vec![
1557 SortingColumn {
1558 column_idx: 1,
1559 descending: false,
1560 nulls_first: false,
1561 },
1562 SortingColumn {
1563 column_idx: 0,
1564 descending: false,
1565 nulls_first: false,
1566 },
1567 ],
1568 );
1569 assert!(
1570 row_group
1571 .columns()
1572 .iter()
1573 .all(|column| column.compression() == Compression::ZSTD(ZstdLevel::default())),
1574 );
1575 assert!(
1576 reader
1577 .get_row_group(0)
1578 .unwrap()
1579 .get_column_bloom_filter(0)
1580 .is_some(),
1581 );
1582 }
1583
1584 #[tokio::test]
1585 async fn default_writer_delta_encodes_timestamps_without_dictionary() {
1586 let directory = tempfile::tempdir().unwrap();
1587 let path = directory.path().join("timestamps.parquet");
1588
1589 let object_store = Arc::new(
1590 object_store::local::LocalFileSystem::new_with_prefix(directory.path()).unwrap(),
1591 );
1592 let object_path = ObjectPath::from("timestamps.parquet");
1593
1594 let schema = Arc::new(Schema::new(vec![
1595 Field::new("identifier", DataType::Utf8, false),
1596 Field::new("ts_event", timestamp_data_type(), false),
1597 Field::new("ts_init", timestamp_data_type(), false),
1598 ]));
1599 let identifiers = StringArray::from(vec!["AUD/USD.SIM", "AUD/USD.SIM", "AUD/USD.SIM"]);
1600 let ts_event = TimestampNanosecondArray::from(vec![
1601 1_700_000_000_123_456_001_i64,
1602 1_700_000_000_123_456_789,
1603 1_700_000_001_000_000_000,
1604 ])
1605 .with_timezone("UTC");
1606 let ts_init = TimestampNanosecondArray::from(vec![
1607 1_700_000_000_123_457_000_i64,
1608 1_700_000_000_123_458_000,
1609 1_700_000_001_000_001_000,
1610 ])
1611 .with_timezone("UTC");
1612 let batch = RecordBatch::try_new(
1613 schema,
1614 vec![
1615 Arc::new(identifiers.clone()) as ArrayRef,
1616 Arc::new(ts_event.clone()) as ArrayRef,
1617 Arc::new(ts_init.clone()) as ArrayRef,
1618 ],
1619 )
1620 .unwrap();
1621
1622 write_batches_to_object_store(&[batch], object_store, &object_path, None, None, None)
1623 .await
1624 .unwrap();
1625
1626 let reader = SerializedFileReader::new(std::fs::File::open(&path).unwrap()).unwrap();
1627 let columns = reader.metadata().row_group(0).columns();
1628 let timestamp_encodings = columns[1..]
1629 .iter()
1630 .map(|column| {
1631 (
1632 column.column_path().string(),
1633 column.dictionary_page_offset(),
1634 column.encodings().collect::<Vec<_>>(),
1635 )
1636 })
1637 .collect::<Vec<_>>();
1638 let read = ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(&path).unwrap())
1639 .unwrap()
1640 .build()
1641 .unwrap()
1642 .collect::<Result<Vec<_>, _>>()
1643 .unwrap();
1644
1645 assert!(columns[0].dictionary_page_offset().is_some());
1646 assert_eq!(
1647 timestamp_encodings,
1648 vec![
1649 (
1650 "ts_event".to_string(),
1651 None,
1652 vec![Encoding::RLE, Encoding::DELTA_BINARY_PACKED],
1653 ),
1654 (
1655 "ts_init".to_string(),
1656 None,
1657 vec![Encoding::RLE, Encoding::DELTA_BINARY_PACKED],
1658 ),
1659 ],
1660 );
1661 assert_eq!(read.len(), 1);
1662 assert_eq!(read[0].column(0).as_ref(), &identifiers as &dyn Array);
1663 assert_eq!(read[0].column(1).as_ref(), &ts_event as &dyn Array);
1664 assert_eq!(read[0].column(2).as_ref(), &ts_init as &dyn Array);
1665 }
1666
1667 #[rstest]
1668 fn test_create_object_store_from_path_local() {
1669 let temp_dir = std::env::temp_dir().join("nautilus_test");
1671 std::fs::create_dir_all(&temp_dir).unwrap();
1672
1673 let result = create_object_store_from_path(temp_dir.to_str().unwrap(), None);
1674 if let Err(e) = &result {
1675 println!("Error: {e:?}");
1676 }
1677
1678 assert!(result.is_ok());
1679 let (_, base_path, uri) = result.unwrap();
1680 assert_eq!(base_path, "");
1681 assert_eq!(uri, format!("file://{}", temp_dir.to_str().unwrap()));
1683
1684 std::fs::remove_dir_all(&temp_dir).ok();
1686 }
1687
1688 #[rstest]
1689 fn append_path_to_file_uri_extends_windows_drive_catalog() {
1690 assert_eq!(
1691 append_path_to_file_uri(
1692 "file:///C:/data/catalog",
1693 "data/quotes/EURUSD.SIM/file.parquet",
1694 ),
1695 "file:///C:/data/catalog/data/quotes/EURUSD.SIM/file.parquet",
1696 );
1697 }
1698
1699 #[rstest]
1700 #[case::hash(
1701 "file:///C:/data/C# work",
1702 "file:///C:/data/C%23%20work/data/bars/file.parquet"
1703 )]
1704 #[case::percent_escape("file:///tmp/a%20b", "file:///tmp/a%2520b/data/bars/file.parquet")]
1705 #[case::question_mark("file:///tmp/a?b", "file:///tmp/a%3Fb/data/bars/file.parquet")]
1706 fn append_path_to_file_uri_keeps_escaped_local_root(
1707 #[case] base: &str,
1708 #[case] expected: &str,
1709 ) {
1710 let base = escape_local_file_uri(base);
1711
1712 assert_eq!(
1713 append_path_to_file_uri(&base, "data/bars/file.parquet"),
1714 expected,
1715 );
1716 }
1717
1718 #[rstest]
1719 #[cfg(feature = "cloud")]
1720 fn test_create_object_store_from_path_s3() {
1721 let mut options = AHashMap::new();
1722 options.insert(
1723 "endpoint_url".to_string(),
1724 "https://test.endpoint.com".to_string(),
1725 );
1726 options.insert("region".to_string(), "us-west-2".to_string());
1727 options.insert("access_key_id".to_string(), "test_key".to_string());
1728 options.insert("secret_access_key".to_string(), "test_secret".to_string());
1729
1730 let result = create_object_store_from_path("s3://test-bucket/path", Some(options));
1731 assert!(result.is_ok());
1732 let (_, base_path, uri) = result.unwrap();
1733 assert_eq!(base_path, "path");
1734 assert_eq!(uri, "s3://test-bucket/path");
1735 }
1736
1737 #[rstest]
1738 #[cfg(feature = "cloud")]
1739 fn test_create_object_store_from_path_azure() {
1740 let mut options = AHashMap::new();
1741 options.insert("account_name".to_string(), "testaccount".to_string());
1742 options.insert("account_key".to_string(), "dGVzdGtleQ==".to_string()); let result = create_object_store_from_path("az://container/path", Some(options));
1746 if let Err(e) = &result {
1747 println!("Azure Error: {e:?}");
1748 }
1749
1750 assert!(result.is_ok());
1751 let (_, base_path, uri) = result.unwrap();
1752 assert_eq!(base_path, "path");
1753 assert_eq!(uri, "az://container/path");
1754 }
1755
1756 #[rstest]
1757 #[cfg(feature = "cloud")]
1758 fn test_create_object_store_from_path_gcs() {
1759 let credentials_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
1760 .join("test_data/gcs_authorized_user_credentials.json");
1761 let options = AHashMap::from([
1762 ("skip_signature".to_string(), "true".to_string()),
1763 (
1764 "application_credentials".to_string(),
1765 credentials_path.to_str().unwrap().to_string(),
1766 ),
1767 ]);
1768
1769 let (_, base_path, uri) =
1770 create_object_store_from_path("gs://test-bucket/path", Some(options)).unwrap();
1771
1772 assert_eq!(base_path, "path");
1773 assert_eq!(uri, "gs://test-bucket/path");
1774 }
1775
1776 #[rstest]
1777 #[cfg(feature = "cloud")]
1778 fn test_create_object_store_gcs_reads_application_credentials_option() {
1779 let directory = tempfile::tempdir().unwrap();
1780 let credentials_path = directory.path().join("missing-credentials.json");
1781 let credentials_path = credentials_path.to_str().unwrap();
1782 let options = AHashMap::from([(
1783 "application_credentials".to_string(),
1784 credentials_path.to_string(),
1785 )]);
1786
1787 let error = create_object_store_from_path("gs://test-bucket/path", Some(options))
1788 .err()
1789 .unwrap()
1790 .to_string();
1791
1792 assert!(
1793 error.contains(&format!(
1794 "Unable to open service account file from {credentials_path}"
1795 )),
1796 "{error}"
1797 );
1798 }
1799
1800 #[rstest]
1801 #[cfg(feature = "cloud")]
1802 #[case::s3("s3://test-bucket/path", "no_such_option", "S3")]
1803 #[case::gcs_project_id("gs://test-bucket/path", "project_id", "GCS")]
1804 #[case::azure("az://container/path", "no_such_option", "MicrosoftAzure")]
1805 #[case::abfs(
1806 "abfs://container@account.dfs.core.windows.net/path",
1807 "no_such_option",
1808 "MicrosoftAzure"
1809 )]
1810 #[case::http("https://example.com/path", "no_such_option", "HTTP")]
1811 fn test_create_object_store_rejects_unknown_storage_option(
1812 #[case] uri: &str,
1813 #[case] key: &str,
1814 #[case] store: &str,
1815 ) {
1816 let options = AHashMap::from([(key.to_string(), "value".to_string())]);
1817
1818 let error = create_object_store_from_path(uri, Some(options))
1819 .err()
1820 .unwrap()
1821 .to_string();
1822
1823 assert_eq!(
1824 error,
1825 format!("Configuration key: '{key}' is not valid for store '{store}'.")
1826 );
1827 }
1828
1829 #[rstest]
1830 #[cfg(feature = "cloud")]
1831 #[case::non_ascii("s3://test-bucket/préfix")]
1832 #[case::space("s3://test-bucket/my prefix")]
1833 #[case::tilde("s3://test-bucket/~prefix")]
1834 #[case::fragment("s3://test-bucket/pre#fix")]
1835 #[case::percent_encoded("s3://test-bucket/pr%C3%A9fix")]
1836 #[case::empty_segment("s3://test-bucket/base//path")]
1837 #[case::gcs("gs://test-bucket/préfix")]
1838 #[case::azure("az://container/préfix")]
1839 #[case::abfs("abfs://container@account.dfs.core.windows.net/préfix")]
1840 #[case::http("https://example.com/préfix")]
1841 fn test_create_object_store_rejects_base_path_that_paths_change(#[case] uri: &str) {
1842 let error = create_object_store_from_path(uri, None)
1843 .err()
1844 .unwrap()
1845 .to_string();
1846
1847 assert_eq!(
1848 error,
1849 format!(
1850 "Storage URI '{uri}' has a base path that URL parsing or object-store path \
1851 encoding changes; use a base path without spaces, non-ASCII, or reserved \
1852 characters"
1853 )
1854 );
1855 }
1856
1857 #[rstest]
1858 #[cfg(feature = "cloud")]
1859 #[case::bucket_root("s3://test-bucket", "")]
1860 #[case::trailing_slash("s3://test-bucket/nautilus-data/", "nautilus-data/")]
1861 #[case::unreserved(
1862 "s3://test-bucket/team_a/v2.catalog/year=2026",
1863 "team_a/v2.catalog/year=2026"
1864 )]
1865 fn test_create_object_store_accepts_ascii_base_path(
1866 #[case] uri: &str,
1867 #[case] expected_base_path: &str,
1868 ) {
1869 let (_, base_path, original_uri) = create_object_store_from_path(uri, None).unwrap();
1870
1871 assert_eq!(base_path, expected_base_path);
1872 assert_eq!(original_uri, uri);
1873 }
1874
1875 #[rstest]
1876 #[case::native("aws_access_key_id", "aws_secret_access_key")]
1877 #[case::short("access_key_id", "secret_access_key")]
1878 #[case::legacy("key", "secret")]
1879 #[tokio::test]
1880 #[cfg(feature = "cloud")]
1881 async fn test_create_object_store_s3_applies_credential_option_names(
1882 #[case] key_id_option: &str,
1883 #[case] secret_option: &str,
1884 ) {
1885 let (endpoint, server) = capture_request_head();
1886 let options = AHashMap::from([
1887 ("aws_endpoint".to_string(), endpoint),
1888 ("aws_allow_http".to_string(), "true".to_string()),
1889 (key_id_option.to_string(), "TESTKEYID".to_string()),
1890 (secret_option.to_string(), "test-secret".to_string()),
1891 ]);
1892 let (store, base_path, _) =
1893 create_object_store_from_path("s3://test-bucket/path", Some(options)).unwrap();
1894
1895 let result = store
1896 .head(&ObjectPath::from(format!("{base_path}/probe")))
1897 .await;
1898
1899 assert!(
1900 matches!(result, Err(object_store::Error::NotFound { .. })),
1901 "{result:?}"
1902 );
1903 let head = server.join().unwrap();
1904 assert_eq!(
1905 head.lines().next(),
1906 Some("HEAD /test-bucket/path/probe HTTP/1.1")
1907 );
1908 assert!(
1909 head.contains("authorization: AWS4-HMAC-SHA256 Credential=TESTKEYID/"),
1910 "{head}"
1911 );
1912 }
1913
1914 #[tokio::test]
1915 #[cfg(feature = "cloud")]
1916 async fn test_create_object_store_http_applies_client_option() {
1917 let (endpoint, server) = capture_request_head();
1918 let options = AHashMap::from([
1919 ("allow_http".to_string(), "true".to_string()),
1920 (
1921 "user_agent".to_string(),
1922 "nautilus-catalog-test".to_string(),
1923 ),
1924 ]);
1925 let (store, base_path, _) =
1926 create_object_store_from_path(&format!("{endpoint}/path"), Some(options)).unwrap();
1927
1928 let result = store
1929 .head(&ObjectPath::from(format!("{base_path}/probe")))
1930 .await;
1931
1932 assert!(
1933 matches!(result, Err(object_store::Error::NotFound { .. })),
1934 "{result:?}"
1935 );
1936 let head = server.join().unwrap();
1937 assert_eq!(head.lines().next(), Some("HEAD /path/probe HTTP/1.1"));
1938 assert!(
1939 head.contains("user-agent: nautilus-catalog-test\r\n"),
1940 "{head}"
1941 );
1942 }
1943
1944 #[tokio::test]
1945 #[cfg(feature = "cloud")]
1946 async fn test_create_object_store_azure_sends_sas_token_signature_encoded_once() {
1947 let (endpoint, server) = capture_request_head();
1948 let options = AHashMap::from([
1949 ("account_name".to_string(), "account".to_string()),
1950 ("azure_storage_endpoint".to_string(), endpoint),
1951 ("azure_allow_http".to_string(), "true".to_string()),
1952 (
1953 "sas_token".to_string(),
1954 "sv=2022-11-02&se=2030-01-01T00:00:00Z&sig=abc%2Bdef%3D".to_string(),
1955 ),
1956 ]);
1957 let (store, base_path, _) =
1958 create_object_store_from_path("az://container/path", Some(options)).unwrap();
1959
1960 let result = store
1961 .head(&ObjectPath::from(format!("{base_path}/probe")))
1962 .await;
1963
1964 assert!(
1965 matches!(result, Err(object_store::Error::NotFound { .. })),
1966 "{result:?}"
1967 );
1968 let head = server.join().unwrap();
1969 assert_eq!(
1970 head.lines().next(),
1971 Some(
1972 "HEAD /container/path/probe?sv=2022-11-02&se=2030-01-01T00%3A00%3A00Z\
1973 &sig=abc%2Bdef%3D HTTP/1.1"
1974 )
1975 );
1976 }
1977
1978 #[rstest]
1979 #[cfg(feature = "cloud")]
1980 fn test_create_object_store_from_path_empty_options() {
1981 let result = create_object_store_from_path("s3://test-bucket/path", None);
1982 assert!(result.is_ok());
1983 let (_, base_path, uri) = result.unwrap();
1984 assert_eq!(base_path, "path");
1985 assert_eq!(uri, "s3://test-bucket/path");
1986 }
1987
1988 #[rstest]
1989 #[cfg(feature = "cloud")]
1990 fn test_remote_store_root_url_preserves_authority() {
1991 let https_root = remote_store_root_url("https://example.com:9000/base/path").unwrap();
1992 assert_eq!(
1993 https_root.as_str().trim_end_matches('/'),
1994 "https://example.com:9000"
1995 );
1996
1997 let abfs_root =
1998 remote_store_root_url("abfs://container@account.dfs.core.windows.net/base/path")
1999 .unwrap();
2000 assert_eq!(
2001 abfs_root.as_str().trim_end_matches('/'),
2002 "abfs://container@account.dfs.core.windows.net"
2003 );
2004
2005 let full_uri = remote_full_uri(
2006 "https://example.com:9000/base/path",
2007 "base/path/data/%5E/file.parquet",
2008 )
2009 .unwrap();
2010 assert_eq!(
2011 full_uri,
2012 "https://example.com:9000/base/path/data/%5E/file.parquet"
2013 );
2014
2015 let location = create_object_store_location_from_path("s3://test-bucket/path", None)
2016 .expect("S3 location should be created");
2017 assert_eq!(location.base_path, "path");
2018 assert_eq!(
2019 remote_store_root_url(&location.original_uri)
2020 .expect("S3 should be remote")
2021 .as_str()
2022 .trim_end_matches('/'),
2023 "s3://test-bucket"
2024 );
2025 }
2026
2027 #[cfg(feature = "cloud")]
2028 fn capture_request_head() -> (String, std::thread::JoinHandle<String>) {
2029 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2030 let endpoint = format!("http://{}", listener.local_addr().unwrap());
2031 let handle = std::thread::spawn(move || {
2032 let (mut stream, _) = listener.accept().unwrap();
2033 let mut reader = std::io::BufReader::new(stream.try_clone().unwrap());
2034 let mut head = String::new();
2035 loop {
2036 let mut line = String::new();
2037 reader.read_line(&mut line).unwrap();
2038 if line.trim_end().is_empty() {
2039 break;
2040 }
2041 head.push_str(&line);
2042 }
2043 stream
2044 .write_all(
2045 b"HTTP/1.1 404 Not Found\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
2046 )
2047 .unwrap();
2048 head
2049 });
2050 (endpoint, handle)
2051 }
2052}