Skip to main content

nautilus_persistence/backend/parquet/
io.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16#![expect(
17    clippy::missing_panics_doc,
18    reason = "Parquet I/O functions use validated schema paths"
19)]
20
21use std::{collections::HashMap, sync::Arc};
22
23use ahash::AHashMap;
24use anyhow::Context;
25use arrow::{
26    array::UInt64Array,
27    compute::{SortColumn, SortOptions, concat_batches, lexsort_to_indices, take_record_batch},
28    datatypes::Schema,
29    record_batch::RecordBatch,
30};
31use nautilus_serialization::arrow::{
32    KEY_IDENTIFIER, KEY_PRICE_PRECISION, KEY_SIZE_PRECISION, U64ColumnRef,
33};
34use object_store::{
35    ObjectMeta, ObjectStore, ObjectStoreExt, PutMode, PutOptions, buffered::BufReader,
36    path::Path as ObjectPath,
37};
38use parquet::{
39    arrow::{
40        ArrowSchemaConverter, ArrowWriter, ParquetRecordBatchStreamBuilder,
41        arrow_reader::ParquetRecordBatchReaderBuilder,
42    },
43    basic::{Compression, Encoding, ZstdLevel},
44    file::{
45        metadata::{KeyValue, SortingColumn},
46        properties::WriterProperties,
47        reader::{FileReader, SerializedFileReader},
48        statistics::Statistics,
49    },
50    schema::types::ColumnPath,
51};
52use url::Url;
53
54use super::consolidation_state;
55pub(crate) use crate::common::paths::file_uri_to_native_path;
56pub use crate::common::paths::normalize_path_to_uri;
57
58pub(crate) struct ObjectStoreLocation {
59    pub object_store: Arc<dyn ObjectStore>,
60    pub base_path: String,
61    pub original_uri: String,
62}
63
64/// Reads only the Arrow schema (including key/value metadata) of a Parquet object.
65///
66/// Avoids decoding any record batches; use when only schema metadata is needed.
67///
68/// # Errors
69///
70/// Returns an error if the object cannot be fetched or its footer cannot be parsed.
71pub async fn read_parquet_schema_from_object_store(
72    object_store: Arc<dyn ObjectStore>,
73    path: &ObjectPath,
74) -> anyhow::Result<Arc<Schema>> {
75    let object = object_store.head(path).await?;
76    if object.size == 0 {
77        return Ok(Arc::new(Schema::empty()));
78    }
79
80    let reader = BufReader::new(object_store, &object);
81    let builder = ParquetRecordBatchStreamBuilder::new(reader).await?;
82    Ok(builder.schema().clone())
83}
84
85/// Reads a Parquet file from an object store and returns all record batches plus
86/// the Arrow schema from the builder. The builder's schema includes metadata restored
87/// from the file's `ARROW:schema` `key_value_metadata`; use it for decoding instead of
88/// each batch's schema (which has metadata stripped).
89///
90/// # Errors
91///
92/// Returns an error if the path cannot be read or Parquet parsing fails.
93pub async fn read_parquet_from_object_store(
94    object_store: Arc<dyn ObjectStore>,
95    path: &ObjectPath,
96) -> anyhow::Result<(Vec<RecordBatch>, Arc<Schema>)> {
97    let data = object_store.get(path).await?.bytes().await?;
98    if data.is_empty() {
99        return Ok((Vec::new(), Arc::new(Schema::empty())));
100    }
101
102    let builder = ParquetRecordBatchReaderBuilder::try_new(data)?;
103    let schema = builder.schema().clone();
104    let batches = builder.build()?.collect::<Result<Vec<_>, _>>()?;
105
106    Ok((batches, schema))
107}
108
109/// Writes multiple `RecordBatch` items to an object store URI, with optional compression,
110/// row group sizing, and `key_value_metadata` (e.g. for instrument `type_name` so it survives roundtrip).
111///
112/// # Errors
113///
114/// Returns an error if `batches` is empty, writing to Parquet fails, or any I/O operation fails.
115pub async fn write_batches_to_object_store(
116    batches: &[RecordBatch],
117    object_store: Arc<dyn ObjectStore>,
118    path: &ObjectPath,
119    compression: Option<Compression>,
120    max_row_group_size: Option<usize>,
121    key_value_metadata: Option<Vec<KeyValue>>,
122) -> anyhow::Result<()> {
123    write_batches_to_object_store_with_mode(
124        batches,
125        object_store,
126        path,
127        compression,
128        max_row_group_size,
129        key_value_metadata,
130        PutMode::Overwrite,
131        None,
132    )
133    .await
134}
135
136pub(crate) async fn write_batches_to_object_store_create(
137    batches: &[RecordBatch],
138    object_store: Arc<dyn ObjectStore>,
139    path: &ObjectPath,
140    compression: Option<Compression>,
141    max_row_group_size: Option<usize>,
142    key_value_metadata: Option<Vec<KeyValue>>,
143) -> anyhow::Result<()> {
144    write_batches_to_object_store_with_mode(
145        batches,
146        object_store,
147        path,
148        compression,
149        max_row_group_size,
150        key_value_metadata,
151        PutMode::Create,
152        None,
153    )
154    .await
155}
156
157#[expect(
158    clippy::too_many_arguments,
159    reason = "private writer keeps storage mode and footer identity separate"
160)]
161async fn write_batches_to_object_store_with_mode(
162    batches: &[RecordBatch],
163    object_store: Arc<dyn ObjectStore>,
164    path: &ObjectPath,
165    compression: Option<Compression>,
166    max_row_group_size: Option<usize>,
167    key_value_metadata: Option<Vec<KeyValue>>,
168    put_mode: PutMode,
169    created_by: Option<String>,
170) -> anyhow::Result<()> {
171    let Some(first) = batches.first() else {
172        anyhow::bail!("Cannot write Parquet file {path} with no record batches");
173    };
174
175    // Create a temporary buffer to write the parquet data
176    let mut buffer = Vec::new();
177
178    let schema = first.schema();
179    let sorting_columns = parquet_sorting_columns(schema.as_ref())?;
180    let mut props_builder = WriterProperties::builder()
181        .set_compression(compression.unwrap_or(Compression::ZSTD(ZstdLevel::default())))
182        .set_max_row_group_row_count(Some(
183            max_row_group_size.unwrap_or(super::DEFAULT_ROW_GROUP_SIZE),
184        ))
185        .set_sorting_columns(sorting_columns)
186        .set_key_value_metadata(key_value_metadata);
187
188    if let Some(created_by) = created_by {
189        props_builder = props_builder.set_created_by(created_by);
190    }
191
192    if schema.index_of(KEY_IDENTIFIER).is_ok() {
193        props_builder =
194            props_builder.set_column_bloom_filter_enabled(ColumnPath::from(KEY_IDENTIFIER), true);
195    }
196
197    // Delta encoding stores near-unique timestamps more compactly than a dictionary
198    for name in ["ts_event", "ts_init"] {
199        if schema.index_of(name).is_ok() {
200            props_builder = props_builder
201                .set_column_dictionary_enabled(ColumnPath::from(name), false)
202                .set_column_encoding(ColumnPath::from(name), Encoding::DELTA_BINARY_PACKED);
203        }
204    }
205
206    let writer_props = props_builder.build();
207
208    let mut writer = ArrowWriter::try_new(&mut buffer, schema, Some(writer_props))?;
209    for batch in batches {
210        writer.write(batch)?;
211    }
212
213    writer.close()?;
214
215    // Upload the buffer to object store
216    object_store
217        .put_opts(
218            path,
219            buffer.into(),
220            PutOptions {
221                mode: put_mode,
222                ..Default::default()
223            },
224        )
225        .await?;
226
227    Ok(())
228}
229
230fn parquet_sorting_columns(schema: &Schema) -> anyhow::Result<Option<Vec<SortingColumn>>> {
231    if schema.index_of("ts_init").is_err() {
232        return Ok(None);
233    }
234
235    let parquet_schema = ArrowSchemaConverter::new().convert(schema)?;
236    let mut names = Vec::with_capacity(2);
237    names.push("ts_init");
238    if schema.index_of(KEY_IDENTIFIER).is_ok() {
239        names.push(KEY_IDENTIFIER);
240    }
241
242    let columns = names
243        .into_iter()
244        .map(|name| {
245            let index = parquet_schema
246                .columns()
247                .iter()
248                .position(|column| {
249                    column
250                        .path()
251                        .parts()
252                        .first()
253                        .is_some_and(|part| part == name)
254                })
255                .ok_or_else(|| anyhow::anyhow!("Parquet schema is missing sort column {name}"))?;
256
257            Ok(SortingColumn {
258                column_idx: i32::try_from(index)?,
259                descending: false,
260                nulls_first: false,
261            })
262        })
263        .collect::<anyhow::Result<Vec<_>>>()?;
264
265    Ok(Some(columns))
266}
267
268/// Deduplicates a slice of `RecordBatch` items, removing rows that are identical across all columns.
269///
270/// Rows are compared by encoding each row to a canonical byte sequence using Arrow's row format.
271/// Only the first occurrence of each unique row is retained; the relative order of unique rows
272/// is preserved.
273///
274/// # Errors
275///
276/// Returns an error if the row converter cannot be constructed or if the `take` kernel fails.
277fn deduplicate_record_batches(batches: &[RecordBatch]) -> anyhow::Result<Vec<RecordBatch>> {
278    if batches.is_empty() {
279        return Ok(Vec::new());
280    }
281
282    let schema = batches[0].schema();
283
284    let fields: Vec<arrow::row::SortField> = schema
285        .fields()
286        .iter()
287        .map(|f| arrow::row::SortField::new(f.data_type().clone()))
288        .collect();
289
290    let converter = arrow::row::RowConverter::new(fields)?;
291    let mut seen: std::collections::HashSet<Vec<u8>> = std::collections::HashSet::new();
292    let mut result: Vec<RecordBatch> = Vec::new();
293
294    for batch in batches {
295        let rows = converter.convert_columns(batch.columns())?;
296        let mut indices: Vec<u32> = Vec::new();
297
298        for (i, row) in rows.iter().enumerate() {
299            if seen.insert(row.as_ref().to_vec()) {
300                indices.push(u32::try_from(i)?);
301            }
302        }
303
304        if !indices.is_empty() {
305            let index_array = arrow::array::UInt32Array::from(indices);
306            let deduped_columns: Vec<arrow::array::ArrayRef> = batch
307                .columns()
308                .iter()
309                .map(|col| arrow::compute::take(col.as_ref(), &index_array, None))
310                .collect::<Result<_, _>>()?;
311            result.push(RecordBatch::try_new(schema.clone(), deduped_columns)?);
312        }
313    }
314
315    Ok(result)
316}
317
318/// Combines multiple Parquet files from object store
319///
320/// # Errors
321///
322/// Returns an error if file reading or writing fails.
323pub async fn combine_parquet_files_from_object_store(
324    object_store: Arc<dyn ObjectStore>,
325    file_paths: Vec<ObjectPath>,
326    new_file_path: &ObjectPath,
327    compression: Option<Compression>,
328    max_row_group_size: Option<usize>,
329    deduplicate: Option<bool>,
330) -> anyhow::Result<()> {
331    let Some(merged) =
332        read_combined_batches(object_store.clone(), &file_paths, deduplicate).await?
333    else {
334        return Ok(());
335    };
336
337    write_batches_to_object_store(
338        &merged.batches,
339        object_store.clone(),
340        new_file_path,
341        compression,
342        max_row_group_size,
343        None,
344    )
345    .await?;
346
347    for source in merged.sources {
348        if source.location != *new_file_path {
349            object_store.delete(&source.location).await?;
350        }
351    }
352
353    Ok(())
354}
355
356pub(crate) async fn consolidate_parquet_files_from_object_store(
357    object_store: Arc<dyn ObjectStore>,
358    file_paths: Vec<ObjectPath>,
359    new_file_path: &ObjectPath,
360    compression: Option<Compression>,
361    max_row_group_size: Option<usize>,
362    deduplicate: Option<bool>,
363) -> anyhow::Result<()> {
364    let Some(merged) =
365        read_combined_batches(object_store.clone(), &file_paths, deduplicate).await?
366    else {
367        return Ok(());
368    };
369
370    let created_by =
371        consolidation_state::begin(&object_store, new_file_path, &merged.sources).await?;
372    write_batches_to_object_store_with_mode(
373        &merged.batches,
374        object_store.clone(),
375        new_file_path,
376        compression,
377        max_row_group_size,
378        None,
379        PutMode::Overwrite,
380        Some(created_by),
381    )
382    .await?;
383    let directory = new_file_path.parent().unwrap_or_default();
384    consolidation_state::recover(&object_store, &directory).await
385}
386
387struct MergedParquet {
388    batches: Vec<RecordBatch>,
389    sources: Vec<ObjectMeta>,
390}
391
392async fn read_combined_batches(
393    object_store: Arc<dyn ObjectStore>,
394    file_paths: &[ObjectPath],
395    deduplicate: Option<bool>,
396) -> anyhow::Result<Option<MergedParquet>> {
397    if file_paths.len() <= 1 {
398        return Ok(None);
399    }
400
401    let mut all_batches: Vec<RecordBatch> = Vec::new();
402    let mut schema_with_metadata: Option<Arc<arrow::datatypes::Schema>> = None;
403    let mut schema_source: Option<&ObjectPath> = None;
404    let mut field_metadata_sources = HashMap::new();
405    let mut sources = Vec::with_capacity(file_paths.len());
406
407    // Read all files from object store
408    for path in file_paths {
409        let object = object_store.get(path).await?;
410        sources.push(object.meta.clone());
411        let data = object.bytes().await?;
412        let builder = ParquetRecordBatchReaderBuilder::try_new(data)?;
413
414        let candidate_schema = builder.schema().clone();
415        schema_with_metadata = Some(
416            if let (Some(schema), Some(source)) = (&schema_with_metadata, schema_source) {
417                let reconciled = reconcile_consolidation_schema_with_sources(
418                    schema,
419                    source,
420                    &field_metadata_sources,
421                    &candidate_schema,
422                    path,
423                )?;
424
425                if reconciled.schema_source == ConsolidationSchemaSource::Candidate {
426                    schema_source = Some(path);
427                }
428
429                for key in reconciled.candidate_field_metadata {
430                    field_metadata_sources.insert(key, path.clone());
431                }
432
433                reconciled.schema
434            } else {
435                schema_source = Some(path);
436                field_metadata_sources
437                    .extend(field_metadata_keys(&candidate_schema).map(|key| (key, path.clone())));
438                candidate_schema
439            },
440        );
441
442        for batch in builder.build()? {
443            all_batches.push(batch?);
444        }
445    }
446
447    // Re-apply the preserved schema metadata to all collected batches so that
448    // write_batches_to_object_store (which uses batches[0].schema()) can encode
449    // the correct Arrow schema metadata into the combined output file.
450    let schema = schema_with_metadata.expect("non-empty source files provide a schema");
451    all_batches = all_batches
452        .into_iter()
453        .map(|b| RecordBatch::try_new(schema.clone(), b.columns().to_vec()))
454        .collect::<Result<Vec<_>, _>>()?;
455
456    // Deduplicate rows if requested
457    let batches_to_write = if deduplicate.unwrap_or(false) {
458        deduplicate_record_batches(&all_batches)?
459    } else {
460        all_batches
461    };
462
463    let batches_to_write = sort_consolidation_batches(batches_to_write)?;
464    Ok(Some(MergedParquet {
465        batches: batches_to_write,
466        sources,
467    }))
468}
469
470fn sort_consolidation_batches(batches: Vec<RecordBatch>) -> anyhow::Result<Vec<RecordBatch>> {
471    let Some(first) = batches.first() else {
472        return Ok(batches);
473    };
474
475    if first.column_by_name("ts_init").is_none() {
476        return Ok(batches);
477    }
478
479    let mut previous = 0;
480    let mut ordered = true;
481
482    for batch in &batches {
483        let array = batch
484            .column_by_name("ts_init")
485            .expect("consistent batch schema");
486        let timestamps = U64ColumnRef::try_from_array(array.as_ref())
487            .ok_or_else(|| anyhow::anyhow!("Unsupported consolidation ts_init type"))?;
488
489        for row in 0..batch.num_rows() {
490            let timestamp = timestamps
491                .value(row)
492                .ok_or_else(|| anyhow::anyhow!("Invalid consolidation ts_init value"))?;
493            ordered &= previous <= timestamp;
494            previous = timestamp;
495        }
496    }
497
498    if ordered {
499        return Ok(batches);
500    }
501
502    let schema = first.schema();
503    let batch = concat_batches(&schema, &batches)?;
504    let original_index = Arc::new(UInt64Array::from_iter_values(0..batch.num_rows() as u64));
505
506    let options = Some(SortOptions {
507        descending: false,
508        nulls_first: false,
509    });
510
511    let indices = lexsort_to_indices(
512        &[
513            SortColumn {
514                values: batch
515                    .column_by_name("ts_init")
516                    .expect("ts_init exists")
517                    .clone(),
518                options,
519            },
520            SortColumn {
521                values: original_index,
522                options,
523            },
524        ],
525        None,
526    )?;
527
528    Ok(vec![take_record_batch(&batch, &indices)?])
529}
530
531#[derive(Clone, Copy, Debug, Eq, PartialEq)]
532enum ConsolidationSchemaSource {
533    Current,
534    Candidate,
535}
536
537#[derive(Debug)]
538struct ReconciledConsolidationSchema {
539    schema: Arc<Schema>,
540    schema_source: ConsolidationSchemaSource,
541    candidate_field_metadata: Vec<(String, String)>,
542}
543
544fn reconcile_consolidation_schema_with_sources(
545    current: &Arc<Schema>,
546    current_path: &ObjectPath,
547    current_field_metadata_sources: &HashMap<(String, String), ObjectPath>,
548    candidate: &Arc<Schema>,
549    candidate_path: &ObjectPath,
550) -> anyhow::Result<ReconciledConsolidationSchema> {
551    anyhow::ensure!(
552        current.fields().len() == candidate.fields().len(),
553        "Cannot consolidate Parquet files {current_path} and {candidate_path}: field schemas differ"
554    );
555    let mut candidate_field_metadata = Vec::new();
556
557    let fields = current
558        .fields()
559        .iter()
560        .zip(candidate.fields())
561        .map(|(current, candidate)| {
562            anyhow::ensure!(
563                current.name() == candidate.name()
564                    && current.data_type() == candidate.data_type()
565                    && current.is_nullable() == candidate.is_nullable(),
566                "Cannot consolidate Parquet files {current_path} and {candidate_path}: field schemas differ"
567            );
568            let mut metadata = current.metadata().clone();
569            for (key, value) in candidate.metadata() {
570                if let Some(current_value) = metadata.get(key) {
571                    let source = current_field_metadata_sources
572                        .get(&(current.name().clone(), key.clone()))
573                        .unwrap_or(current_path);
574                    anyhow::ensure!(
575                        current_value == value,
576                        "Cannot consolidate Parquet files {source} and {candidate_path}: field '{}' metadata differs",
577                        current.name(),
578                    );
579                } else {
580                    metadata.insert(key.clone(), value.clone());
581                    candidate_field_metadata.push((current.name().clone(), key.clone()));
582                }
583            }
584
585            Ok(Arc::new(current.as_ref().clone().with_metadata(metadata)))
586        })
587        .collect::<anyhow::Result<Vec<_>>>()?;
588
589    let schema_with_fields =
590        |metadata| Arc::new(Schema::new_with_metadata(fields.clone(), metadata));
591
592    if current.metadata() == candidate.metadata() {
593        return Ok(ReconciledConsolidationSchema {
594            schema: schema_with_fields(current.metadata().clone()),
595            schema_source: ConsolidationSchemaSource::Current,
596            candidate_field_metadata,
597        });
598    }
599
600    let without_precision = |schema: &Schema| {
601        let mut metadata = schema.metadata().clone();
602        metadata.remove(KEY_PRICE_PRECISION);
603        metadata.remove(KEY_SIZE_PRECISION);
604        metadata
605    };
606
607    let is_precision_fallback = |metadata: &HashMap<String, String>| {
608        metadata.get(KEY_PRICE_PRECISION).map(String::as_str) == Some("0")
609            && metadata.get(KEY_SIZE_PRECISION).map(String::as_str) == Some("0")
610    };
611
612    let has_precision = |metadata: &HashMap<String, String>| {
613        metadata.contains_key(KEY_PRICE_PRECISION) && metadata.contains_key(KEY_SIZE_PRECISION)
614    };
615
616    let current_metadata = current.metadata();
617    let candidate_metadata = candidate.metadata();
618
619    if without_precision(current) == without_precision(candidate) {
620        match (
621            is_precision_fallback(current_metadata),
622            is_precision_fallback(candidate_metadata),
623        ) {
624            (true, false) if has_precision(candidate_metadata) => {
625                return Ok(ReconciledConsolidationSchema {
626                    schema: schema_with_fields(candidate.metadata().clone()),
627                    schema_source: ConsolidationSchemaSource::Candidate,
628                    candidate_field_metadata,
629                });
630            }
631            (false, true) if has_precision(current_metadata) => {
632                return Ok(ReconciledConsolidationSchema {
633                    schema: schema_with_fields(current.metadata().clone()),
634                    schema_source: ConsolidationSchemaSource::Current,
635                    candidate_field_metadata,
636                });
637            }
638            _ => {}
639        }
640    }
641
642    anyhow::bail!(
643        "Cannot consolidate Parquet files {current_path} and {candidate_path}: schema metadata differs: {current_metadata:?} versus {candidate_metadata:?}"
644    )
645}
646
647fn field_metadata_keys(schema: &Schema) -> impl Iterator<Item = (String, String)> + '_ {
648    schema.fields().iter().flat_map(|field| {
649        field
650            .metadata()
651            .keys()
652            .map(|key| (field.name().clone(), key.clone()))
653    })
654}
655
656/// Extracts the minimum and maximum i64 values for the specified `column_name` from a Parquet file's metadata in object store.
657///
658/// # Errors
659///
660/// Returns an error if the file cannot be read, metadata parsing fails, or the column is missing or has no statistics.
661pub async fn min_max_from_parquet_metadata_object_store(
662    object_store: Arc<dyn ObjectStore>,
663    file_path: &ObjectPath,
664    column_name: &str,
665) -> anyhow::Result<(u64, u64)> {
666    // Download the parquet file from object store
667    let data = object_store.get(file_path).await?.bytes().await?;
668    let reader = SerializedFileReader::new(data)?;
669
670    let metadata = reader.metadata();
671    let mut overall_min_value: Option<i64> = None;
672    let mut overall_max_value: Option<i64> = None;
673
674    // Iterate through all row groups
675    for i in 0..metadata.num_row_groups() {
676        let row_group = metadata.row_group(i);
677
678        // Iterate through all columns in this row group
679        for j in 0..row_group.num_columns() {
680            let col_metadata = row_group.column(j);
681
682            if col_metadata.column_path().string() == column_name {
683                if let Some(stats) = col_metadata.statistics() {
684                    // Check if we have Int64 statistics
685                    if let Statistics::Int64(int64_stats) = stats {
686                        // Extract min value if available
687                        if let Some(&min_value) = int64_stats.min_opt()
688                            && (overall_min_value.is_none()
689                                || min_value < overall_min_value.unwrap())
690                        {
691                            overall_min_value = Some(min_value);
692                        }
693
694                        // Extract max value if available
695                        if let Some(&max_value) = int64_stats.max_opt()
696                            && (overall_max_value.is_none()
697                                || max_value > overall_max_value.unwrap())
698                        {
699                            overall_max_value = Some(max_value);
700                        }
701                    } else {
702                        anyhow::bail!("Warning: Column name '{column_name}' is not of type i64.");
703                    }
704                } else {
705                    anyhow::bail!(
706                        "Warning: Statistics not available for column '{column_name}' in row group {i}."
707                    );
708                }
709            }
710        }
711    }
712
713    // Return the min/max pair if both are available
714    if let (Some(min), Some(max)) = (overall_min_value, overall_max_value) {
715        Ok((u64::try_from(min)?, u64::try_from(max)?))
716    } else {
717        anyhow::bail!(
718            "Column '{column_name}' not found or has no Int64 statistics in any row group."
719        )
720    }
721}
722
723/// Creates an object store from a URI string with optional storage options.
724///
725/// Supports multiple cloud storage providers:
726/// - AWS S3: `s3://bucket/path`
727/// - Google Cloud Storage: `gs://bucket/path` or `gcs://bucket/path`
728/// - Azure Blob Storage: `az://account/container/path` or `abfs://container@account.dfs.core.windows.net/path`
729/// - HTTP/WebDAV: `http://` or `https://`
730/// - Local files: `file://path` or plain paths
731///
732/// # Parameters
733///
734/// - `path`: The URI string for the storage location.
735/// - `storage_options`: Optional `HashMap` of `object_store` configuration keys for the scheme,
736///   under their prefixed or short names (`aws_region` or `region`):
737///   - For S3: `endpoint_url`, region, `access_key_id`, `secret_access_key`, `session_token`, etc.
738///     The legacy `key` and `secret` names are also accepted.
739///   - For GCS: `service_account_path`, `service_account_key`, `application_credentials`, etc.
740///   - For Azure: `account_name`, `account_key`, `sas_token`, etc.
741///   - For HTTP: client options such as `timeout`.
742///
743/// Returns a tuple of (`ObjectStore`, `base_path`, `normalized_uri`)
744///
745/// # Errors
746///
747/// Returns an error if the URI is invalid, a storage option key is unknown for the scheme, or the
748/// object store cannot be built.
749pub fn create_object_store_from_path(
750    path: &str,
751    storage_options: Option<AHashMap<String, String>>,
752) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
753    let location = create_object_store_location_from_path(path, storage_options)?;
754    Ok((
755        location.object_store,
756        location.base_path,
757        location.original_uri,
758    ))
759}
760
761// `storage_options` is only consumed by the cloud-feature arms,
762// so keep the allow scoped to the no-cloud build.
763#[cfg_attr(
764    not(feature = "cloud"),
765    allow(unused_variables, clippy::needless_pass_by_value)
766)]
767pub(crate) fn create_object_store_location_from_path(
768    path: &str,
769    storage_options: Option<AHashMap<String, String>>,
770) -> anyhow::Result<ObjectStoreLocation> {
771    let uri = normalize_path_to_uri(path)?;
772
773    let (object_store, base_path, original_uri) = match uri.as_str() {
774        #[cfg(feature = "cloud")]
775        s if s.starts_with("s3://") => create_s3_store(&uri, storage_options),
776        #[cfg(feature = "cloud")]
777        s if s.starts_with("gs://") || s.starts_with("gcs://") => {
778            create_gcs_store(&uri, storage_options)
779        }
780        #[cfg(feature = "cloud")]
781        s if s.starts_with("az://") => create_azure_store(&uri, storage_options),
782        #[cfg(feature = "cloud")]
783        s if s.starts_with("abfs://") => create_abfs_store(&uri, storage_options),
784        #[cfg(feature = "cloud")]
785        s if s.starts_with("http://") || s.starts_with("https://") => {
786            create_http_store(&uri, storage_options)
787        }
788        #[cfg(not(feature = "cloud"))]
789        s if s.starts_with("s3://")
790            || s.starts_with("gs://")
791            || s.starts_with("gcs://")
792            || s.starts_with("az://")
793            || s.starts_with("abfs://")
794            || s.starts_with("http://")
795            || s.starts_with("https://") =>
796        {
797            anyhow::bail!("Cloud storage support requires the 'cloud' feature: {uri}")
798        }
799        s if s.starts_with("file://") => create_local_store(&uri, true),
800        _ => create_local_store(&uri, false), // Fallback: assume local path
801    }?;
802
803    Ok(ObjectStoreLocation {
804        object_store,
805        base_path,
806        original_uri,
807    })
808}
809
810pub(crate) fn is_remote_uri_scheme(scheme: &str) -> bool {
811    matches!(
812        scheme,
813        "s3" | "gs" | "gcs" | "az" | "abfs" | "http" | "https"
814    )
815}
816
817pub(crate) fn remote_store_root_url(uri: &str) -> anyhow::Result<Url> {
818    let mut url = Url::parse(uri)?;
819    url.set_path("");
820    url.set_query(None);
821    url.set_fragment(None);
822    Ok(url)
823}
824
825pub(crate) fn remote_full_uri(uri: &str, object_path: &str) -> anyhow::Result<String> {
826    let root = remote_store_root_url(uri)?;
827    let root = root.as_str().trim_end_matches('/');
828    let object_path = object_path.trim_start_matches('/');
829
830    if object_path.is_empty() {
831        Ok(root.to_string())
832    } else {
833        Ok(format!("{root}/{object_path}"))
834    }
835}
836
837/// Appends an encoded object-store path to a local or remote storage URI.
838/// Preserve the encoded names used by the native object-store backend.
839pub(crate) fn append_path_to_file_uri(base_uri: &str, path: &str) -> String {
840    if let Ok(mut url) = Url::parse(base_uri) {
841        if let Ok(mut segments) = url.path_segments_mut() {
842            segments.pop_if_empty();
843            segments.extend(
844                path.trim_end_matches('/')
845                    .split('/')
846                    .filter(|segment| !segment.is_empty()),
847            );
848        }
849
850        return url.to_string();
851    }
852
853    format!(
854        "{}/{}",
855        base_uri.trim_end_matches('/'),
856        path.trim_end_matches('/')
857    )
858}
859
860/// Decodes a percent-encoded `object_store` path segment back to its logical form.
861///
862/// `object_store` lists path segments in URL-encoded form, so a non-ASCII instrument
863/// directory reads back with each non-ASCII byte as a `%XX` sequence. Decoding recovers the
864/// original id for matching against `urisafe_instrument_id`. Returns the input unchanged when
865/// it is not valid percent-encoded UTF-8.
866pub(crate) fn decode_object_store_segment(segment: &str) -> String {
867    ObjectPath::from_url_path(segment).map_or_else(|_| segment.to_string(), String::from)
868}
869
870fn create_local_store(
871    uri: &str,
872    is_file_uri: bool,
873) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
874    let path = if is_file_uri {
875        file_uri_to_native_path(uri)
876    } else {
877        uri.to_string()
878    };
879
880    let local_store =
881        object_store::local::LocalFileSystem::new_with_prefix(&path).with_context(|| {
882            format!(
883                "failed to open local storage directory '{path}'; \
884             create it if it does not exist and check access permissions"
885            )
886        })?;
887
888    Ok((Arc::new(local_store), String::new(), uri.to_string()))
889}
890
891/// Helper function to create S3 object store with options.
892#[cfg(feature = "cloud")]
893fn create_s3_store(
894    uri: &str,
895    storage_options: Option<AHashMap<String, String>>,
896) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
897    let (url, path) = parse_url_and_path(uri)?;
898    let bucket = extract_host(&url, "Invalid S3 URI: missing bucket")?;
899
900    let mut builder = object_store::aws::AmazonS3Builder::new().with_bucket_name(&bucket);
901
902    // Apply storage options if provided
903    if let Some(options) = storage_options {
904        for (key, value) in options {
905            // The `object_store` parser lacks the legacy fsspec `key` and `secret` names
906            let config_key = match key.as_str() {
907                "key" => object_store::aws::AmazonS3ConfigKey::AccessKeyId,
908                "secret" => object_store::aws::AmazonS3ConfigKey::SecretAccessKey,
909                _ => key.parse()?,
910            };
911            builder = builder.with_config(config_key, value);
912        }
913    }
914
915    let s3_store = builder.build()?;
916    Ok((Arc::new(s3_store), path, uri.to_string()))
917}
918
919/// Helper function to create GCS object store with options.
920#[cfg(feature = "cloud")]
921fn create_gcs_store(
922    uri: &str,
923    storage_options: Option<AHashMap<String, String>>,
924) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
925    let (url, path) = parse_url_and_path(uri)?;
926    let bucket = extract_host(&url, "Invalid GCS URI: missing bucket")?;
927
928    let mut builder = object_store::gcp::GoogleCloudStorageBuilder::new().with_bucket_name(&bucket);
929
930    // Apply storage options if provided
931    if let Some(options) = storage_options {
932        for (key, value) in options {
933            builder =
934                builder.with_config(key.parse::<object_store::gcp::GoogleConfigKey>()?, value);
935        }
936    }
937
938    let gcs_store = builder.build()?;
939    Ok((Arc::new(gcs_store), path, uri.to_string()))
940}
941
942/// Helper function to create Azure object store with options.
943#[cfg(feature = "cloud")]
944fn create_azure_store(
945    uri: &str,
946    storage_options: Option<AHashMap<String, String>>,
947) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
948    let (url, path) = parse_url_and_path(uri)?;
949    let container = extract_host(&url, "Invalid Azure URI: missing container")?;
950
951    let mut builder =
952        object_store::azure::MicrosoftAzureBuilder::new().with_container_name(container);
953
954    // Apply storage options if provided
955    if let Some(options) = storage_options {
956        builder = apply_azure_storage_options(builder, options)?;
957    }
958
959    let azure_store = builder.build()?;
960    Ok((Arc::new(azure_store), path, uri.to_string()))
961}
962
963/// Helper function to create Azure object store from abfs:// URI with options.
964#[cfg(feature = "cloud")]
965fn create_abfs_store(
966    uri: &str,
967    storage_options: Option<AHashMap<String, String>>,
968) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
969    let (url, path) = parse_url_and_path(uri)?;
970    let host = extract_host(&url, "Invalid ABFS URI: missing host")?;
971
972    // Extract account from host (account.dfs.core.windows.net)
973    let account = host
974        .split('.')
975        .next()
976        .ok_or_else(|| anyhow::anyhow!("Invalid ABFS URI: cannot extract account from host"))?;
977
978    // Extract container from username part
979    let container = url
980        .username()
981        .split('@')
982        .next()
983        .ok_or_else(|| anyhow::anyhow!("Invalid ABFS URI: missing container"))?;
984
985    let mut builder = object_store::azure::MicrosoftAzureBuilder::new()
986        .with_account(account)
987        .with_container_name(container);
988
989    // Apply storage options if provided (same as Azure store)
990    if let Some(options) = storage_options {
991        builder = apply_azure_storage_options(builder, options)?;
992    }
993
994    let azure_store = builder.build()?;
995    Ok((Arc::new(azure_store), path, uri.to_string()))
996}
997
998/// Applies shared Azure storage options to the builder for the `az` and `abfs` schemes.
999#[cfg(feature = "cloud")]
1000fn apply_azure_storage_options(
1001    mut builder: object_store::azure::MicrosoftAzureBuilder,
1002    options: AHashMap<String, String>,
1003) -> anyhow::Result<object_store::azure::MicrosoftAzureBuilder> {
1004    for (key, value) in options {
1005        builder = builder.with_config(key.parse::<object_store::azure::AzureConfigKey>()?, value);
1006    }
1007
1008    Ok(builder)
1009}
1010
1011/// Helper function to create HTTP object store with options.
1012#[cfg(feature = "cloud")]
1013fn create_http_store(
1014    uri: &str,
1015    storage_options: Option<AHashMap<String, String>>,
1016) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
1017    let (_, path) = parse_url_and_path(uri)?;
1018    let base_url = remote_store_root_url(uri)?
1019        .as_str()
1020        .trim_end_matches('/')
1021        .to_string();
1022
1023    let mut builder = object_store::http::HttpBuilder::new().with_url(base_url);
1024
1025    // Apply storage options if provided
1026    if let Some(options) = storage_options {
1027        for (key, value) in options {
1028            builder = builder.with_config(key.parse::<object_store::ClientConfigKey>()?, value);
1029        }
1030    }
1031
1032    let http_store = builder.build()?;
1033    Ok((Arc::new(http_store), path, uri.to_string()))
1034}
1035
1036/// Parses a remote storage URI into its URL and object-store base path.
1037///
1038/// Catalog listings return keys under the encoded base path while lookups use the path as
1039/// written, so a base path that URL parsing or object-store path encoding changes is rejected.
1040#[cfg(feature = "cloud")]
1041fn parse_url_and_path(uri: &str) -> anyhow::Result<(Url, String)> {
1042    let url = Url::parse(uri)?;
1043    let path = url.path().trim_start_matches('/').to_string();
1044    let raw_path = uri
1045        .split_once("://")
1046        .and_then(|(_, rest)| rest.split_once('/'))
1047        .map_or("", |(_, raw)| raw.trim_start_matches('/'));
1048
1049    anyhow::ensure!(
1050        raw_path == path && ObjectPath::from(path.as_str()).as_ref() == path.trim_end_matches('/'),
1051        "Storage URI '{uri}' has a base path that URL parsing or object-store path encoding \
1052         changes; use a base path without spaces, non-ASCII, or reserved characters",
1053    );
1054
1055    Ok((url, path))
1056}
1057
1058/// Helper function to extract host from URL with error handling.
1059#[cfg(feature = "cloud")]
1060fn extract_host(url: &Url, error_msg: &str) -> anyhow::Result<String> {
1061    url.host_str()
1062        .map(ToString::to_string)
1063        .ok_or_else(|| anyhow::anyhow!("{error_msg}"))
1064}
1065
1066#[cfg(test)]
1067mod tests {
1068    #[cfg(feature = "cloud")]
1069    use std::io::{BufRead, Write};
1070    use std::{collections::HashMap, sync::Arc};
1071
1072    #[cfg(feature = "cloud")]
1073    use ahash::AHashMap;
1074    use arrow::{
1075        array::{Array, ArrayRef, StringArray, TimestampNanosecondArray, UInt64Array},
1076        datatypes::{DataType, Field, Schema},
1077    };
1078    use nautilus_serialization::arrow::{json_string_field, timestamp_data_type};
1079    use parquet::file::{properties::ReaderProperties, serialized_reader::ReadOptionsBuilder};
1080    use rstest::rstest;
1081
1082    use super::*;
1083    use crate::common::paths::escape_local_file_uri;
1084
1085    #[rstest]
1086    #[tokio::test]
1087    async fn combine_accepts_an_output_outside_the_source_directory() {
1088        use futures::TryStreamExt;
1089
1090        let store: Arc<dyn ObjectStore> = Arc::new(object_store::memory::InMemory::new());
1091
1092        let schema = Arc::new(Schema::new(vec![Field::new(
1093            "value",
1094            DataType::UInt64,
1095            false,
1096        )]));
1097
1098        let paths = [
1099            ObjectPath::from("inputs/one.parquet"),
1100            ObjectPath::from("inputs/two.parquet"),
1101        ];
1102
1103        for (path, value) in paths.iter().zip([11, 22]) {
1104            let batch = RecordBatch::try_new(
1105                schema.clone(),
1106                vec![Arc::new(UInt64Array::from(vec![value]))],
1107            )
1108            .unwrap();
1109            write_batches_to_object_store(&[batch], store.clone(), path, None, None, None)
1110                .await
1111                .unwrap();
1112        }
1113
1114        let target = ObjectPath::from("outputs/combined.parquet");
1115        combine_parquet_files_from_object_store(
1116            store.clone(),
1117            paths.to_vec(),
1118            &target,
1119            None,
1120            None,
1121            None,
1122        )
1123        .await
1124        .unwrap();
1125        let (batches, actual_schema) = read_parquet_from_object_store(store.clone(), &target)
1126            .await
1127            .unwrap();
1128        let actual = concat_batches(&actual_schema, &batches).unwrap();
1129        let expected =
1130            RecordBatch::try_new(schema, vec![Arc::new(UInt64Array::from(vec![11, 22]))]).unwrap();
1131        let remaining = store.list(None).try_collect::<Vec<_>>().await.unwrap();
1132
1133        assert_eq!(actual, expected);
1134        assert_eq!(
1135            remaining
1136                .iter()
1137                .map(|meta| &meta.location)
1138                .collect::<Vec<_>>(),
1139            vec![&target]
1140        );
1141    }
1142
1143    fn consolidation_depth_schema(
1144        price_precision: &str,
1145        size_precision: &str,
1146        instrument_id: &str,
1147    ) -> Arc<Schema> {
1148        Arc::new(Schema::new_with_metadata(
1149            vec![Field::new("bids", DataType::Utf8, false)],
1150            HashMap::from([
1151                (KEY_PRICE_PRECISION.to_string(), price_precision.to_string()),
1152                (KEY_SIZE_PRECISION.to_string(), size_precision.to_string()),
1153                (KEY_IDENTIFIER.to_string(), instrument_id.to_string()),
1154            ]),
1155        ))
1156    }
1157
1158    fn reconcile_consolidation_schema(
1159        current: &Arc<Schema>,
1160        current_path: &ObjectPath,
1161        candidate: &Arc<Schema>,
1162        candidate_path: &ObjectPath,
1163    ) -> anyhow::Result<ReconciledConsolidationSchema> {
1164        let field_metadata_sources = field_metadata_keys(current)
1165            .map(|key| (key, current_path.clone()))
1166            .collect();
1167        reconcile_consolidation_schema_with_sources(
1168            current,
1169            current_path,
1170            &field_metadata_sources,
1171            candidate,
1172            candidate_path,
1173        )
1174    }
1175
1176    #[rstest]
1177    fn consolidation_schema_prefers_populated_depth_precision_in_either_order() {
1178        let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1179        let populated = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1180        let fallback_path = ObjectPath::from("empty.parquet");
1181        let populated_path = ObjectPath::from("populated.parquet");
1182
1183        let fallback_first =
1184            reconcile_consolidation_schema(&fallback, &fallback_path, &populated, &populated_path)
1185                .unwrap();
1186        let populated_first =
1187            reconcile_consolidation_schema(&populated, &populated_path, &fallback, &fallback_path)
1188                .unwrap();
1189
1190        assert_eq!(
1191            fallback_first.schema_source,
1192            ConsolidationSchemaSource::Candidate,
1193        );
1194        assert_eq!(
1195            populated_first.schema_source,
1196            ConsolidationSchemaSource::Current,
1197        );
1198
1199        for schema in [fallback_first.schema, populated_first.schema] {
1200            assert_eq!(schema.metadata()[KEY_PRICE_PRECISION], "2");
1201            assert_eq!(schema.metadata()[KEY_SIZE_PRECISION], "3");
1202        }
1203    }
1204
1205    #[rstest]
1206    fn consolidation_schema_rejects_other_metadata_mismatches() {
1207        let current = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1208        let candidate = consolidation_depth_schema("2", "3", "BTCUSDT.BINANCE");
1209        let current_path = ObjectPath::from("eth.parquet");
1210        let candidate_path = ObjectPath::from("btc.parquet");
1211
1212        let error =
1213            reconcile_consolidation_schema(&current, &current_path, &candidate, &candidate_path)
1214                .unwrap_err();
1215
1216        assert!(error.to_string().contains("eth.parquet and btc.parquet"));
1217        assert!(error.to_string().contains("ETHUSDT.BINANCE"));
1218        assert!(error.to_string().contains("BTCUSDT.BINANCE"));
1219    }
1220
1221    #[rstest]
1222    fn consolidation_schema_rejects_two_populated_precisions() {
1223        let current = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1224        let candidate = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1225        let current_path = ObjectPath::from("precision-2.parquet");
1226        let candidate_path = ObjectPath::from("precision-4.parquet");
1227
1228        let error =
1229            reconcile_consolidation_schema(&current, &current_path, &candidate, &candidate_path)
1230                .unwrap_err();
1231
1232        assert!(error.to_string().contains("precision-2.parquet"));
1233        assert!(error.to_string().contains("precision-4.parquet"));
1234    }
1235
1236    #[rstest]
1237    fn consolidation_schema_keeps_fallback_for_all_empty_files() {
1238        let first = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1239        let second = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1240
1241        let reconciled = reconcile_consolidation_schema(
1242            &first,
1243            &ObjectPath::from("first-empty.parquet"),
1244            &second,
1245            &ObjectPath::from("second-empty.parquet"),
1246        )
1247        .unwrap();
1248
1249        assert_eq!(reconciled.schema.metadata()[KEY_PRICE_PRECISION], "0");
1250        assert_eq!(reconciled.schema.metadata()[KEY_SIZE_PRECISION], "0");
1251    }
1252
1253    #[rstest]
1254    #[case::missing_first(true)]
1255    #[case::fallback_first(false)]
1256    fn consolidation_schema_rejects_missing_precision_against_fallback(
1257        #[case] missing_first: bool,
1258    ) {
1259        let missing = Arc::new(Schema::new_with_metadata(
1260            vec![Field::new("bids", DataType::Utf8, false)],
1261            HashMap::from([(KEY_IDENTIFIER.to_string(), "ETHUSDT.BINANCE".to_string())]),
1262        ));
1263        let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1264        let missing_path = ObjectPath::from("missing.parquet");
1265        let fallback_path = ObjectPath::from("fallback.parquet");
1266
1267        let error = if missing_first {
1268            reconcile_consolidation_schema(&missing, &missing_path, &fallback, &fallback_path)
1269        } else {
1270            reconcile_consolidation_schema(&fallback, &fallback_path, &missing, &missing_path)
1271        }
1272        .unwrap_err();
1273
1274        assert!(error.to_string().contains("schema metadata differs"));
1275    }
1276
1277    #[rstest]
1278    fn consolidation_schema_merges_json_field_annotation_in_either_order() {
1279        let bare = Arc::new(Schema::new(vec![Field::new("info", DataType::Utf8, true)]));
1280        let annotated = Arc::new(Schema::new(vec![json_string_field("info", true)]));
1281        let bare_path = ObjectPath::from("bare.parquet");
1282        let annotated_path = ObjectPath::from("annotated.parquet");
1283
1284        let bare_first =
1285            reconcile_consolidation_schema(&bare, &bare_path, &annotated, &annotated_path).unwrap();
1286        let annotated_first =
1287            reconcile_consolidation_schema(&annotated, &annotated_path, &bare, &bare_path).unwrap();
1288
1289        assert_eq!(bare_first.schema, annotated_first.schema);
1290        assert_eq!(
1291            bare_first.schema.field_with_name("info").unwrap(),
1292            &json_string_field("info", true),
1293        );
1294    }
1295
1296    #[rstest]
1297    fn consolidation_field_metadata_conflict_names_the_winning_file() {
1298        let bare = Arc::new(Schema::new(vec![Field::new("info", DataType::Utf8, true)]));
1299        let annotated = Arc::new(Schema::new(vec![json_string_field("info", true)]));
1300
1301        let conflicting = Arc::new(Schema::new(vec![
1302            Field::new("info", DataType::Utf8, true).with_metadata(HashMap::from([(
1303                "ARROW:extension:name".to_string(),
1304                "other.extension".to_string(),
1305            )])),
1306        ]));
1307        let bare_path = ObjectPath::from("bare.parquet");
1308        let annotated_path = ObjectPath::from("annotated.parquet");
1309        let conflicting_path = ObjectPath::from("conflicting.parquet");
1310        let reconciled =
1311            reconcile_consolidation_schema(&bare, &bare_path, &annotated, &annotated_path).unwrap();
1312
1313        assert_eq!(reconciled.candidate_field_metadata.len(), 2);
1314        assert!(
1315            reconciled
1316                .candidate_field_metadata
1317                .contains(&("info".to_string(), "ARROW:extension:name".to_string())),
1318        );
1319        assert!(
1320            reconciled
1321                .candidate_field_metadata
1322                .contains(&("info".to_string(), "ARROW:extension:metadata".to_string())),
1323        );
1324        let field_metadata_sources = reconciled
1325            .candidate_field_metadata
1326            .iter()
1327            .cloned()
1328            .map(|key| (key, annotated_path.clone()))
1329            .collect();
1330        let error = reconcile_consolidation_schema_with_sources(
1331            &reconciled.schema,
1332            &bare_path,
1333            &field_metadata_sources,
1334            &conflicting,
1335            &conflicting_path,
1336        )
1337        .unwrap_err();
1338        assert!(error.to_string().contains("annotated.parquet"));
1339        assert!(error.to_string().contains("conflicting.parquet"));
1340        assert!(error.to_string().contains("field 'info' metadata differs"));
1341    }
1342
1343    #[rstest]
1344    fn consolidation_conflict_names_the_last_winning_schema() {
1345        let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1346        let precision_2 = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1347        let precision_4 = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1348        let fallback_path = ObjectPath::from("fallback.parquet");
1349        let precision_2_path = ObjectPath::from("precision-2.parquet");
1350        let precision_4_path = ObjectPath::from("precision-4.parquet");
1351        let reconciled = reconcile_consolidation_schema(
1352            &fallback,
1353            &fallback_path,
1354            &precision_2,
1355            &precision_2_path,
1356        )
1357        .unwrap();
1358
1359        let error = reconcile_consolidation_schema(
1360            &reconciled.schema,
1361            &precision_2_path,
1362            &precision_4,
1363            &precision_4_path,
1364        )
1365        .unwrap_err();
1366
1367        assert!(error.to_string().contains("precision-2.parquet"));
1368        assert!(error.to_string().contains("precision-4.parquet"));
1369    }
1370
1371    #[rstest]
1372    fn consolidation_field_metadata_does_not_replace_precision_source() {
1373        let precision_2 = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1374
1375        let fallback = Arc::new(Schema::new_with_metadata(
1376            vec![json_string_field("bids", false)],
1377            consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE")
1378                .metadata()
1379                .clone(),
1380        ));
1381        let precision_4 = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1382        let precision_2_path = ObjectPath::from("precision-2.parquet");
1383        let fallback_path = ObjectPath::from("fallback-annotated.parquet");
1384        let precision_4_path = ObjectPath::from("precision-4.parquet");
1385        let reconciled = reconcile_consolidation_schema(
1386            &precision_2,
1387            &precision_2_path,
1388            &fallback,
1389            &fallback_path,
1390        )
1391        .unwrap();
1392
1393        assert_eq!(reconciled.schema_source, ConsolidationSchemaSource::Current,);
1394        assert_eq!(reconciled.candidate_field_metadata.len(), 2);
1395        assert!(
1396            reconciled
1397                .candidate_field_metadata
1398                .contains(&("bids".to_string(), "ARROW:extension:name".to_string())),
1399        );
1400        assert!(
1401            reconciled
1402                .candidate_field_metadata
1403                .contains(&("bids".to_string(), "ARROW:extension:metadata".to_string())),
1404        );
1405        let field_metadata_sources = reconciled
1406            .candidate_field_metadata
1407            .iter()
1408            .cloned()
1409            .map(|key| (key, fallback_path.clone()))
1410            .collect();
1411        let error = reconcile_consolidation_schema_with_sources(
1412            &reconciled.schema,
1413            &precision_2_path,
1414            &field_metadata_sources,
1415            &precision_4,
1416            &precision_4_path,
1417        )
1418        .unwrap_err();
1419        let message = error.to_string();
1420
1421        assert!(message.contains("precision-2.parquet"));
1422        assert!(message.contains("precision-4.parquet"));
1423        assert!(!message.contains("fallback-annotated.parquet"));
1424    }
1425
1426    #[rstest]
1427    fn consolidation_field_metadata_tracks_each_origin() {
1428        let bare = Arc::new(Schema::new(vec![
1429            Field::new("info", DataType::Utf8, true),
1430            Field::new("balances", DataType::Utf8, true),
1431        ]));
1432
1433        let info = Arc::new(Schema::new(vec![
1434            json_string_field("info", true),
1435            Field::new("balances", DataType::Utf8, true),
1436        ]));
1437
1438        let balances = Arc::new(Schema::new(vec![
1439            json_string_field("info", true),
1440            json_string_field("balances", true),
1441        ]));
1442
1443        let conflicting = Arc::new(Schema::new(vec![
1444            Field::new("info", DataType::Utf8, true).with_metadata(HashMap::from([(
1445                "ARROW:extension:name".to_string(),
1446                "other.extension".to_string(),
1447            )])),
1448            json_string_field("balances", true),
1449        ]));
1450        let bare_path = ObjectPath::from("bare.parquet");
1451        let info_path = ObjectPath::from("info.parquet");
1452        let balances_path = ObjectPath::from("balances.parquet");
1453        let conflicting_path = ObjectPath::from("conflicting.parquet");
1454        let with_info =
1455            reconcile_consolidation_schema(&bare, &bare_path, &info, &info_path).unwrap();
1456        let mut field_metadata_sources = with_info
1457            .candidate_field_metadata
1458            .iter()
1459            .cloned()
1460            .map(|key| (key, info_path.clone()))
1461            .collect::<HashMap<_, _>>();
1462        let with_balances = reconcile_consolidation_schema_with_sources(
1463            &with_info.schema,
1464            &bare_path,
1465            &field_metadata_sources,
1466            &balances,
1467            &balances_path,
1468        )
1469        .unwrap();
1470
1471        for key in with_balances.candidate_field_metadata {
1472            field_metadata_sources.insert(key, balances_path.clone());
1473        }
1474
1475        let error = reconcile_consolidation_schema_with_sources(
1476            &with_balances.schema,
1477            &bare_path,
1478            &field_metadata_sources,
1479            &conflicting,
1480            &conflicting_path,
1481        )
1482        .unwrap_err();
1483        let message = error.to_string();
1484
1485        assert!(message.contains("info.parquet"));
1486        assert!(message.contains("conflicting.parquet"));
1487        assert!(!message.contains("balances.parquet"));
1488    }
1489
1490    #[tokio::test]
1491    async fn write_batches_to_object_store_rejects_empty_input() {
1492        let object_store = Arc::new(object_store::memory::InMemory::new());
1493
1494        let error = write_batches_to_object_store(
1495            &[],
1496            object_store,
1497            &ObjectPath::from("empty.parquet"),
1498            None,
1499            None,
1500            None,
1501        )
1502        .await
1503        .unwrap_err();
1504
1505        assert_eq!(
1506            error.to_string(),
1507            "Cannot write Parquet file empty.parquet with no record batches"
1508        );
1509    }
1510
1511    #[tokio::test]
1512    async fn default_writer_sets_zstd_sorting_and_identifier_bloom_filter() {
1513        let directory = tempfile::tempdir().unwrap();
1514        let path = directory.path().join("layout.parquet");
1515
1516        let object_store = Arc::new(
1517            object_store::local::LocalFileSystem::new_with_prefix(directory.path()).unwrap(),
1518        );
1519        let object_path = ObjectPath::from("layout.parquet");
1520
1521        let schema = Arc::new(Schema::new(vec![
1522            Field::new("identifier", DataType::Utf8, false),
1523            Field::new("ts_init", DataType::UInt64, false),
1524        ]));
1525        let batch = RecordBatch::try_new(
1526            schema,
1527            vec![
1528                Arc::new(StringArray::from(vec!["AUD/USD.SIM", "AUD/USD.SIM"])) as ArrayRef,
1529                Arc::new(UInt64Array::from(vec![1_u64, 2])) as ArrayRef,
1530            ],
1531        )
1532        .unwrap();
1533
1534        write_batches_to_object_store(&[batch], object_store, &object_path, None, None, None)
1535            .await
1536            .unwrap();
1537
1538        let read_options = ReadOptionsBuilder::new()
1539            .with_reader_properties(
1540                ReaderProperties::builder()
1541                    .set_read_bloom_filter(true)
1542                    .build(),
1543            )
1544            .build();
1545        let reader = SerializedFileReader::new_with_options(
1546            std::fs::File::open(path).unwrap(),
1547            read_options,
1548        )
1549        .unwrap();
1550        let row_group = reader.metadata().row_group(0);
1551        let sorting = row_group.sorting_columns().unwrap();
1552
1553        assert_eq!(crate::backend::parquet::DEFAULT_ROW_GROUP_SIZE, 131_072);
1554        assert_eq!(
1555            sorting,
1556            &vec![
1557                SortingColumn {
1558                    column_idx: 1,
1559                    descending: false,
1560                    nulls_first: false,
1561                },
1562                SortingColumn {
1563                    column_idx: 0,
1564                    descending: false,
1565                    nulls_first: false,
1566                },
1567            ],
1568        );
1569        assert!(
1570            row_group
1571                .columns()
1572                .iter()
1573                .all(|column| column.compression() == Compression::ZSTD(ZstdLevel::default())),
1574        );
1575        assert!(
1576            reader
1577                .get_row_group(0)
1578                .unwrap()
1579                .get_column_bloom_filter(0)
1580                .is_some(),
1581        );
1582    }
1583
1584    #[tokio::test]
1585    async fn default_writer_delta_encodes_timestamps_without_dictionary() {
1586        let directory = tempfile::tempdir().unwrap();
1587        let path = directory.path().join("timestamps.parquet");
1588
1589        let object_store = Arc::new(
1590            object_store::local::LocalFileSystem::new_with_prefix(directory.path()).unwrap(),
1591        );
1592        let object_path = ObjectPath::from("timestamps.parquet");
1593
1594        let schema = Arc::new(Schema::new(vec![
1595            Field::new("identifier", DataType::Utf8, false),
1596            Field::new("ts_event", timestamp_data_type(), false),
1597            Field::new("ts_init", timestamp_data_type(), false),
1598        ]));
1599        let identifiers = StringArray::from(vec!["AUD/USD.SIM", "AUD/USD.SIM", "AUD/USD.SIM"]);
1600        let ts_event = TimestampNanosecondArray::from(vec![
1601            1_700_000_000_123_456_001_i64,
1602            1_700_000_000_123_456_789,
1603            1_700_000_001_000_000_000,
1604        ])
1605        .with_timezone("UTC");
1606        let ts_init = TimestampNanosecondArray::from(vec![
1607            1_700_000_000_123_457_000_i64,
1608            1_700_000_000_123_458_000,
1609            1_700_000_001_000_001_000,
1610        ])
1611        .with_timezone("UTC");
1612        let batch = RecordBatch::try_new(
1613            schema,
1614            vec![
1615                Arc::new(identifiers.clone()) as ArrayRef,
1616                Arc::new(ts_event.clone()) as ArrayRef,
1617                Arc::new(ts_init.clone()) as ArrayRef,
1618            ],
1619        )
1620        .unwrap();
1621
1622        write_batches_to_object_store(&[batch], object_store, &object_path, None, None, None)
1623            .await
1624            .unwrap();
1625
1626        let reader = SerializedFileReader::new(std::fs::File::open(&path).unwrap()).unwrap();
1627        let columns = reader.metadata().row_group(0).columns();
1628        let timestamp_encodings = columns[1..]
1629            .iter()
1630            .map(|column| {
1631                (
1632                    column.column_path().string(),
1633                    column.dictionary_page_offset(),
1634                    column.encodings().collect::<Vec<_>>(),
1635                )
1636            })
1637            .collect::<Vec<_>>();
1638        let read = ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(&path).unwrap())
1639            .unwrap()
1640            .build()
1641            .unwrap()
1642            .collect::<Result<Vec<_>, _>>()
1643            .unwrap();
1644
1645        assert!(columns[0].dictionary_page_offset().is_some());
1646        assert_eq!(
1647            timestamp_encodings,
1648            vec![
1649                (
1650                    "ts_event".to_string(),
1651                    None,
1652                    vec![Encoding::RLE, Encoding::DELTA_BINARY_PACKED],
1653                ),
1654                (
1655                    "ts_init".to_string(),
1656                    None,
1657                    vec![Encoding::RLE, Encoding::DELTA_BINARY_PACKED],
1658                ),
1659            ],
1660        );
1661        assert_eq!(read.len(), 1);
1662        assert_eq!(read[0].column(0).as_ref(), &identifiers as &dyn Array);
1663        assert_eq!(read[0].column(1).as_ref(), &ts_event as &dyn Array);
1664        assert_eq!(read[0].column(2).as_ref(), &ts_init as &dyn Array);
1665    }
1666
1667    #[rstest]
1668    fn test_create_object_store_from_path_local() {
1669        // Create a temporary directory for testing
1670        let temp_dir = std::env::temp_dir().join("nautilus_test");
1671        std::fs::create_dir_all(&temp_dir).unwrap();
1672
1673        let result = create_object_store_from_path(temp_dir.to_str().unwrap(), None);
1674        if let Err(e) = &result {
1675            println!("Error: {e:?}");
1676        }
1677
1678        assert!(result.is_ok());
1679        let (_, base_path, uri) = result.unwrap();
1680        assert_eq!(base_path, "");
1681        // The URI should be normalized to file:// format
1682        assert_eq!(uri, format!("file://{}", temp_dir.to_str().unwrap()));
1683
1684        // Clean up
1685        std::fs::remove_dir_all(&temp_dir).ok();
1686    }
1687
1688    #[rstest]
1689    fn append_path_to_file_uri_extends_windows_drive_catalog() {
1690        assert_eq!(
1691            append_path_to_file_uri(
1692                "file:///C:/data/catalog",
1693                "data/quotes/EURUSD.SIM/file.parquet",
1694            ),
1695            "file:///C:/data/catalog/data/quotes/EURUSD.SIM/file.parquet",
1696        );
1697    }
1698
1699    #[rstest]
1700    #[case::hash(
1701        "file:///C:/data/C# work",
1702        "file:///C:/data/C%23%20work/data/bars/file.parquet"
1703    )]
1704    #[case::percent_escape("file:///tmp/a%20b", "file:///tmp/a%2520b/data/bars/file.parquet")]
1705    #[case::question_mark("file:///tmp/a?b", "file:///tmp/a%3Fb/data/bars/file.parquet")]
1706    fn append_path_to_file_uri_keeps_escaped_local_root(
1707        #[case] base: &str,
1708        #[case] expected: &str,
1709    ) {
1710        let base = escape_local_file_uri(base);
1711
1712        assert_eq!(
1713            append_path_to_file_uri(&base, "data/bars/file.parquet"),
1714            expected,
1715        );
1716    }
1717
1718    #[rstest]
1719    #[cfg(feature = "cloud")]
1720    fn test_create_object_store_from_path_s3() {
1721        let mut options = AHashMap::new();
1722        options.insert(
1723            "endpoint_url".to_string(),
1724            "https://test.endpoint.com".to_string(),
1725        );
1726        options.insert("region".to_string(), "us-west-2".to_string());
1727        options.insert("access_key_id".to_string(), "test_key".to_string());
1728        options.insert("secret_access_key".to_string(), "test_secret".to_string());
1729
1730        let result = create_object_store_from_path("s3://test-bucket/path", Some(options));
1731        assert!(result.is_ok());
1732        let (_, base_path, uri) = result.unwrap();
1733        assert_eq!(base_path, "path");
1734        assert_eq!(uri, "s3://test-bucket/path");
1735    }
1736
1737    #[rstest]
1738    #[cfg(feature = "cloud")]
1739    fn test_create_object_store_from_path_azure() {
1740        let mut options = AHashMap::new();
1741        options.insert("account_name".to_string(), "testaccount".to_string());
1742        // Use a valid base64 encoded key for testing
1743        options.insert("account_key".to_string(), "dGVzdGtleQ==".to_string()); // "testkey" in base64
1744
1745        let result = create_object_store_from_path("az://container/path", Some(options));
1746        if let Err(e) = &result {
1747            println!("Azure Error: {e:?}");
1748        }
1749
1750        assert!(result.is_ok());
1751        let (_, base_path, uri) = result.unwrap();
1752        assert_eq!(base_path, "path");
1753        assert_eq!(uri, "az://container/path");
1754    }
1755
1756    #[rstest]
1757    #[cfg(feature = "cloud")]
1758    fn test_create_object_store_from_path_gcs() {
1759        let credentials_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
1760            .join("test_data/gcs_authorized_user_credentials.json");
1761        let options = AHashMap::from([
1762            ("skip_signature".to_string(), "true".to_string()),
1763            (
1764                "application_credentials".to_string(),
1765                credentials_path.to_str().unwrap().to_string(),
1766            ),
1767        ]);
1768
1769        let (_, base_path, uri) =
1770            create_object_store_from_path("gs://test-bucket/path", Some(options)).unwrap();
1771
1772        assert_eq!(base_path, "path");
1773        assert_eq!(uri, "gs://test-bucket/path");
1774    }
1775
1776    #[rstest]
1777    #[cfg(feature = "cloud")]
1778    fn test_create_object_store_gcs_reads_application_credentials_option() {
1779        let directory = tempfile::tempdir().unwrap();
1780        let credentials_path = directory.path().join("missing-credentials.json");
1781        let credentials_path = credentials_path.to_str().unwrap();
1782        let options = AHashMap::from([(
1783            "application_credentials".to_string(),
1784            credentials_path.to_string(),
1785        )]);
1786
1787        let error = create_object_store_from_path("gs://test-bucket/path", Some(options))
1788            .err()
1789            .unwrap()
1790            .to_string();
1791
1792        assert!(
1793            error.contains(&format!(
1794                "Unable to open service account file from {credentials_path}"
1795            )),
1796            "{error}"
1797        );
1798    }
1799
1800    #[rstest]
1801    #[cfg(feature = "cloud")]
1802    #[case::s3("s3://test-bucket/path", "no_such_option", "S3")]
1803    #[case::gcs_project_id("gs://test-bucket/path", "project_id", "GCS")]
1804    #[case::azure("az://container/path", "no_such_option", "MicrosoftAzure")]
1805    #[case::abfs(
1806        "abfs://container@account.dfs.core.windows.net/path",
1807        "no_such_option",
1808        "MicrosoftAzure"
1809    )]
1810    #[case::http("https://example.com/path", "no_such_option", "HTTP")]
1811    fn test_create_object_store_rejects_unknown_storage_option(
1812        #[case] uri: &str,
1813        #[case] key: &str,
1814        #[case] store: &str,
1815    ) {
1816        let options = AHashMap::from([(key.to_string(), "value".to_string())]);
1817
1818        let error = create_object_store_from_path(uri, Some(options))
1819            .err()
1820            .unwrap()
1821            .to_string();
1822
1823        assert_eq!(
1824            error,
1825            format!("Configuration key: '{key}' is not valid for store '{store}'.")
1826        );
1827    }
1828
1829    #[rstest]
1830    #[cfg(feature = "cloud")]
1831    #[case::non_ascii("s3://test-bucket/préfix")]
1832    #[case::space("s3://test-bucket/my prefix")]
1833    #[case::tilde("s3://test-bucket/~prefix")]
1834    #[case::fragment("s3://test-bucket/pre#fix")]
1835    #[case::percent_encoded("s3://test-bucket/pr%C3%A9fix")]
1836    #[case::empty_segment("s3://test-bucket/base//path")]
1837    #[case::gcs("gs://test-bucket/préfix")]
1838    #[case::azure("az://container/préfix")]
1839    #[case::abfs("abfs://container@account.dfs.core.windows.net/préfix")]
1840    #[case::http("https://example.com/préfix")]
1841    fn test_create_object_store_rejects_base_path_that_paths_change(#[case] uri: &str) {
1842        let error = create_object_store_from_path(uri, None)
1843            .err()
1844            .unwrap()
1845            .to_string();
1846
1847        assert_eq!(
1848            error,
1849            format!(
1850                "Storage URI '{uri}' has a base path that URL parsing or object-store path \
1851                 encoding changes; use a base path without spaces, non-ASCII, or reserved \
1852                 characters"
1853            )
1854        );
1855    }
1856
1857    #[rstest]
1858    #[cfg(feature = "cloud")]
1859    #[case::bucket_root("s3://test-bucket", "")]
1860    #[case::trailing_slash("s3://test-bucket/nautilus-data/", "nautilus-data/")]
1861    #[case::unreserved(
1862        "s3://test-bucket/team_a/v2.catalog/year=2026",
1863        "team_a/v2.catalog/year=2026"
1864    )]
1865    fn test_create_object_store_accepts_ascii_base_path(
1866        #[case] uri: &str,
1867        #[case] expected_base_path: &str,
1868    ) {
1869        let (_, base_path, original_uri) = create_object_store_from_path(uri, None).unwrap();
1870
1871        assert_eq!(base_path, expected_base_path);
1872        assert_eq!(original_uri, uri);
1873    }
1874
1875    #[rstest]
1876    #[case::native("aws_access_key_id", "aws_secret_access_key")]
1877    #[case::short("access_key_id", "secret_access_key")]
1878    #[case::legacy("key", "secret")]
1879    #[tokio::test]
1880    #[cfg(feature = "cloud")]
1881    async fn test_create_object_store_s3_applies_credential_option_names(
1882        #[case] key_id_option: &str,
1883        #[case] secret_option: &str,
1884    ) {
1885        let (endpoint, server) = capture_request_head();
1886        let options = AHashMap::from([
1887            ("aws_endpoint".to_string(), endpoint),
1888            ("aws_allow_http".to_string(), "true".to_string()),
1889            (key_id_option.to_string(), "TESTKEYID".to_string()),
1890            (secret_option.to_string(), "test-secret".to_string()),
1891        ]);
1892        let (store, base_path, _) =
1893            create_object_store_from_path("s3://test-bucket/path", Some(options)).unwrap();
1894
1895        let result = store
1896            .head(&ObjectPath::from(format!("{base_path}/probe")))
1897            .await;
1898
1899        assert!(
1900            matches!(result, Err(object_store::Error::NotFound { .. })),
1901            "{result:?}"
1902        );
1903        let head = server.join().unwrap();
1904        assert_eq!(
1905            head.lines().next(),
1906            Some("HEAD /test-bucket/path/probe HTTP/1.1")
1907        );
1908        assert!(
1909            head.contains("authorization: AWS4-HMAC-SHA256 Credential=TESTKEYID/"),
1910            "{head}"
1911        );
1912    }
1913
1914    #[tokio::test]
1915    #[cfg(feature = "cloud")]
1916    async fn test_create_object_store_http_applies_client_option() {
1917        let (endpoint, server) = capture_request_head();
1918        let options = AHashMap::from([
1919            ("allow_http".to_string(), "true".to_string()),
1920            (
1921                "user_agent".to_string(),
1922                "nautilus-catalog-test".to_string(),
1923            ),
1924        ]);
1925        let (store, base_path, _) =
1926            create_object_store_from_path(&format!("{endpoint}/path"), Some(options)).unwrap();
1927
1928        let result = store
1929            .head(&ObjectPath::from(format!("{base_path}/probe")))
1930            .await;
1931
1932        assert!(
1933            matches!(result, Err(object_store::Error::NotFound { .. })),
1934            "{result:?}"
1935        );
1936        let head = server.join().unwrap();
1937        assert_eq!(head.lines().next(), Some("HEAD /path/probe HTTP/1.1"));
1938        assert!(
1939            head.contains("user-agent: nautilus-catalog-test\r\n"),
1940            "{head}"
1941        );
1942    }
1943
1944    #[tokio::test]
1945    #[cfg(feature = "cloud")]
1946    async fn test_create_object_store_azure_sends_sas_token_signature_encoded_once() {
1947        let (endpoint, server) = capture_request_head();
1948        let options = AHashMap::from([
1949            ("account_name".to_string(), "account".to_string()),
1950            ("azure_storage_endpoint".to_string(), endpoint),
1951            ("azure_allow_http".to_string(), "true".to_string()),
1952            (
1953                "sas_token".to_string(),
1954                "sv=2022-11-02&se=2030-01-01T00:00:00Z&sig=abc%2Bdef%3D".to_string(),
1955            ),
1956        ]);
1957        let (store, base_path, _) =
1958            create_object_store_from_path("az://container/path", Some(options)).unwrap();
1959
1960        let result = store
1961            .head(&ObjectPath::from(format!("{base_path}/probe")))
1962            .await;
1963
1964        assert!(
1965            matches!(result, Err(object_store::Error::NotFound { .. })),
1966            "{result:?}"
1967        );
1968        let head = server.join().unwrap();
1969        assert_eq!(
1970            head.lines().next(),
1971            Some(
1972                "HEAD /container/path/probe?sv=2022-11-02&se=2030-01-01T00%3A00%3A00Z\
1973                 &sig=abc%2Bdef%3D HTTP/1.1"
1974            )
1975        );
1976    }
1977
1978    #[rstest]
1979    #[cfg(feature = "cloud")]
1980    fn test_create_object_store_from_path_empty_options() {
1981        let result = create_object_store_from_path("s3://test-bucket/path", None);
1982        assert!(result.is_ok());
1983        let (_, base_path, uri) = result.unwrap();
1984        assert_eq!(base_path, "path");
1985        assert_eq!(uri, "s3://test-bucket/path");
1986    }
1987
1988    #[rstest]
1989    #[cfg(feature = "cloud")]
1990    fn test_remote_store_root_url_preserves_authority() {
1991        let https_root = remote_store_root_url("https://example.com:9000/base/path").unwrap();
1992        assert_eq!(
1993            https_root.as_str().trim_end_matches('/'),
1994            "https://example.com:9000"
1995        );
1996
1997        let abfs_root =
1998            remote_store_root_url("abfs://container@account.dfs.core.windows.net/base/path")
1999                .unwrap();
2000        assert_eq!(
2001            abfs_root.as_str().trim_end_matches('/'),
2002            "abfs://container@account.dfs.core.windows.net"
2003        );
2004
2005        let full_uri = remote_full_uri(
2006            "https://example.com:9000/base/path",
2007            "base/path/data/%5E/file.parquet",
2008        )
2009        .unwrap();
2010        assert_eq!(
2011            full_uri,
2012            "https://example.com:9000/base/path/data/%5E/file.parquet"
2013        );
2014
2015        let location = create_object_store_location_from_path("s3://test-bucket/path", None)
2016            .expect("S3 location should be created");
2017        assert_eq!(location.base_path, "path");
2018        assert_eq!(
2019            remote_store_root_url(&location.original_uri)
2020                .expect("S3 should be remote")
2021                .as_str()
2022                .trim_end_matches('/'),
2023            "s3://test-bucket"
2024        );
2025    }
2026
2027    #[cfg(feature = "cloud")]
2028    fn capture_request_head() -> (String, std::thread::JoinHandle<String>) {
2029        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2030        let endpoint = format!("http://{}", listener.local_addr().unwrap());
2031        let handle = std::thread::spawn(move || {
2032            let (mut stream, _) = listener.accept().unwrap();
2033            let mut reader = std::io::BufReader::new(stream.try_clone().unwrap());
2034            let mut head = String::new();
2035            loop {
2036                let mut line = String::new();
2037                reader.read_line(&mut line).unwrap();
2038                if line.trim_end().is_empty() {
2039                    break;
2040                }
2041                head.push_str(&line);
2042            }
2043            stream
2044                .write_all(
2045                    b"HTTP/1.1 404 Not Found\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
2046                )
2047                .unwrap();
2048            head
2049        });
2050        (endpoint, handle)
2051    }
2052}