Skip to main content

nautilus_persistence/backend/parquet/
io.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16#![expect(
17    clippy::missing_panics_doc,
18    reason = "Parquet I/O functions use validated schema paths"
19)]
20
21use std::{collections::HashMap, sync::Arc};
22
23use ahash::AHashMap;
24use anyhow::Context;
25use arrow::{
26    array::UInt64Array,
27    compute::{SortColumn, SortOptions, concat_batches, lexsort_to_indices, take_record_batch},
28    datatypes::Schema,
29    record_batch::RecordBatch,
30};
31use nautilus_serialization::arrow::{
32    KEY_IDENTIFIER, KEY_PRICE_PRECISION, KEY_SIZE_PRECISION, U64ColumnRef,
33};
34use object_store::{
35    ObjectMeta, ObjectStore, ObjectStoreExt, PutMode, PutOptions, buffered::BufReader,
36    path::Path as ObjectPath,
37};
38use parquet::{
39    arrow::{
40        ArrowSchemaConverter, ArrowWriter, ParquetRecordBatchStreamBuilder,
41        arrow_reader::ParquetRecordBatchReaderBuilder,
42    },
43    basic::{Compression, Encoding, ZstdLevel},
44    file::{
45        metadata::{KeyValue, SortingColumn},
46        properties::WriterProperties,
47        reader::{FileReader, SerializedFileReader},
48        statistics::Statistics,
49    },
50    schema::types::ColumnPath,
51};
52use url::Url;
53
54use super::consolidation_state;
55pub(crate) use crate::common::paths::file_uri_to_native_path;
56pub use crate::common::paths::normalize_path_to_uri;
57
58pub(crate) struct ObjectStoreLocation {
59    pub object_store: Arc<dyn ObjectStore>,
60    pub base_path: String,
61    pub original_uri: String,
62}
63
64/// Reads only the Arrow schema (including key/value metadata) of a Parquet object.
65///
66/// Avoids decoding any record batches; use when only schema metadata is needed.
67///
68/// # Errors
69///
70/// Returns an error if the object cannot be fetched or its footer cannot be parsed.
71pub async fn read_parquet_schema_from_object_store(
72    object_store: Arc<dyn ObjectStore>,
73    path: &ObjectPath,
74) -> anyhow::Result<Arc<Schema>> {
75    let object = object_store.head(path).await?;
76    if object.size == 0 {
77        return Ok(Arc::new(Schema::empty()));
78    }
79
80    let reader = BufReader::new(object_store, &object);
81    let builder = ParquetRecordBatchStreamBuilder::new(reader).await?;
82    Ok(builder.schema().clone())
83}
84
85/// Reads a Parquet file from an object store and returns all record batches plus
86/// the Arrow schema from the builder. The builder's schema includes metadata restored
87/// from the file's `ARROW:schema` `key_value_metadata`; use it for decoding instead of
88/// each batch's schema (which has metadata stripped).
89///
90/// # Errors
91///
92/// Returns an error if the path cannot be read or Parquet parsing fails.
93pub async fn read_parquet_from_object_store(
94    object_store: Arc<dyn ObjectStore>,
95    path: &ObjectPath,
96) -> anyhow::Result<(Vec<RecordBatch>, Arc<Schema>)> {
97    let data = object_store.get(path).await?.bytes().await?;
98    if data.is_empty() {
99        return Ok((Vec::new(), Arc::new(Schema::empty())));
100    }
101
102    let builder = ParquetRecordBatchReaderBuilder::try_new(data)?;
103    let schema = builder.schema().clone();
104    let batches = builder.build()?.collect::<Result<Vec<_>, _>>()?;
105
106    Ok((batches, schema))
107}
108
109/// Writes multiple `RecordBatch` items to an object store URI, with optional compression,
110/// row group sizing, and `key_value_metadata` (e.g. for instrument `type_name` so it survives roundtrip).
111///
112/// # Errors
113///
114/// Returns an error if `batches` is empty, writing to Parquet fails, or any I/O operation fails.
115pub async fn write_batches_to_object_store(
116    batches: &[RecordBatch],
117    object_store: Arc<dyn ObjectStore>,
118    path: &ObjectPath,
119    compression: Option<Compression>,
120    max_row_group_size: Option<usize>,
121    key_value_metadata: Option<Vec<KeyValue>>,
122) -> anyhow::Result<()> {
123    write_batches_to_object_store_with_mode(
124        batches,
125        object_store,
126        path,
127        compression,
128        max_row_group_size,
129        key_value_metadata,
130        PutMode::Overwrite,
131        None,
132    )
133    .await
134}
135
136pub(crate) async fn write_batches_to_object_store_create(
137    batches: &[RecordBatch],
138    object_store: Arc<dyn ObjectStore>,
139    path: &ObjectPath,
140    compression: Option<Compression>,
141    max_row_group_size: Option<usize>,
142    key_value_metadata: Option<Vec<KeyValue>>,
143) -> anyhow::Result<()> {
144    write_batches_to_object_store_with_mode(
145        batches,
146        object_store,
147        path,
148        compression,
149        max_row_group_size,
150        key_value_metadata,
151        PutMode::Create,
152        None,
153    )
154    .await
155}
156
157#[expect(
158    clippy::too_many_arguments,
159    reason = "private writer keeps storage mode and footer identity separate"
160)]
161async fn write_batches_to_object_store_with_mode(
162    batches: &[RecordBatch],
163    object_store: Arc<dyn ObjectStore>,
164    path: &ObjectPath,
165    compression: Option<Compression>,
166    max_row_group_size: Option<usize>,
167    key_value_metadata: Option<Vec<KeyValue>>,
168    put_mode: PutMode,
169    created_by: Option<String>,
170) -> anyhow::Result<()> {
171    let Some(first) = batches.first() else {
172        anyhow::bail!("Cannot write Parquet file {path} with no record batches");
173    };
174
175    // Create a temporary buffer to write the parquet data
176    let mut buffer = Vec::new();
177
178    let schema = first.schema();
179    let sorting_columns = parquet_sorting_columns(schema.as_ref())?;
180    let mut props_builder = WriterProperties::builder()
181        .set_compression(compression.unwrap_or(Compression::ZSTD(ZstdLevel::default())))
182        .set_max_row_group_row_count(Some(
183            max_row_group_size.unwrap_or(super::DEFAULT_ROW_GROUP_SIZE),
184        ))
185        .set_sorting_columns(sorting_columns)
186        .set_key_value_metadata(key_value_metadata);
187
188    if let Some(created_by) = created_by {
189        props_builder = props_builder.set_created_by(created_by);
190    }
191
192    if schema.index_of(KEY_IDENTIFIER).is_ok() {
193        props_builder =
194            props_builder.set_column_bloom_filter_enabled(ColumnPath::from(KEY_IDENTIFIER), true);
195    }
196
197    // Delta encoding stores near-unique timestamps more compactly than a dictionary
198    for name in ["ts_event", "ts_init"] {
199        if schema.index_of(name).is_ok() {
200            props_builder = props_builder
201                .set_column_dictionary_enabled(ColumnPath::from(name), false)
202                .set_column_encoding(ColumnPath::from(name), Encoding::DELTA_BINARY_PACKED);
203        }
204    }
205
206    let writer_props = props_builder.build();
207
208    let mut writer = ArrowWriter::try_new(&mut buffer, schema, Some(writer_props))?;
209    for batch in batches {
210        writer.write(batch)?;
211    }
212
213    writer.close()?;
214
215    // Upload the buffer to object store
216    object_store
217        .put_opts(
218            path,
219            buffer.into(),
220            PutOptions {
221                mode: put_mode,
222                ..Default::default()
223            },
224        )
225        .await?;
226
227    Ok(())
228}
229
230fn parquet_sorting_columns(schema: &Schema) -> anyhow::Result<Option<Vec<SortingColumn>>> {
231    if schema.index_of("ts_init").is_err() {
232        return Ok(None);
233    }
234
235    let parquet_schema = ArrowSchemaConverter::new().convert(schema)?;
236    let mut names = Vec::with_capacity(2);
237    names.push("ts_init");
238    if schema.index_of(KEY_IDENTIFIER).is_ok() {
239        names.push(KEY_IDENTIFIER);
240    }
241
242    let columns = names
243        .into_iter()
244        .map(|name| {
245            let index = parquet_schema
246                .columns()
247                .iter()
248                .position(|column| {
249                    column
250                        .path()
251                        .parts()
252                        .first()
253                        .is_some_and(|part| part == name)
254                })
255                .ok_or_else(|| anyhow::anyhow!("Parquet schema is missing sort column {name}"))?;
256
257            Ok(SortingColumn {
258                column_idx: i32::try_from(index)?,
259                descending: false,
260                nulls_first: false,
261            })
262        })
263        .collect::<anyhow::Result<Vec<_>>>()?;
264
265    Ok(Some(columns))
266}
267
268/// Deduplicates a slice of `RecordBatch` items, removing rows that are identical across all columns.
269///
270/// Rows are compared by encoding each row to a canonical byte sequence using Arrow's row format.
271/// Only the first occurrence of each unique row is retained; the relative order of unique rows
272/// is preserved.
273///
274/// # Errors
275///
276/// Returns an error if the row converter cannot be constructed or if the `take` kernel fails.
277fn deduplicate_record_batches(batches: &[RecordBatch]) -> anyhow::Result<Vec<RecordBatch>> {
278    if batches.is_empty() {
279        return Ok(Vec::new());
280    }
281
282    let schema = batches[0].schema();
283
284    let fields: Vec<arrow::row::SortField> = schema
285        .fields()
286        .iter()
287        .map(|f| arrow::row::SortField::new(f.data_type().clone()))
288        .collect();
289
290    let converter = arrow::row::RowConverter::new(fields)?;
291    let mut seen: std::collections::HashSet<Vec<u8>> = std::collections::HashSet::new();
292    let mut result: Vec<RecordBatch> = Vec::new();
293
294    for batch in batches {
295        let rows = converter.convert_columns(batch.columns())?;
296        let mut indices: Vec<u32> = Vec::new();
297
298        for (i, row) in rows.iter().enumerate() {
299            if seen.insert(row.as_ref().to_vec()) {
300                indices.push(u32::try_from(i)?);
301            }
302        }
303
304        if !indices.is_empty() {
305            let index_array = arrow::array::UInt32Array::from(indices);
306            let deduped_columns: Vec<arrow::array::ArrayRef> = batch
307                .columns()
308                .iter()
309                .map(|col| arrow::compute::take(col.as_ref(), &index_array, None))
310                .collect::<Result<_, _>>()?;
311            result.push(RecordBatch::try_new(schema.clone(), deduped_columns)?);
312        }
313    }
314
315    Ok(result)
316}
317
318/// Combines multiple Parquet files from object store
319///
320/// # Errors
321///
322/// Returns an error if file reading or writing fails.
323pub async fn combine_parquet_files_from_object_store(
324    object_store: Arc<dyn ObjectStore>,
325    file_paths: Vec<ObjectPath>,
326    new_file_path: &ObjectPath,
327    compression: Option<Compression>,
328    max_row_group_size: Option<usize>,
329    deduplicate: Option<bool>,
330) -> anyhow::Result<()> {
331    let Some(merged) =
332        read_combined_batches(object_store.clone(), &file_paths, deduplicate).await?
333    else {
334        return Ok(());
335    };
336
337    write_batches_to_object_store(
338        &merged.batches,
339        object_store.clone(),
340        new_file_path,
341        compression,
342        max_row_group_size,
343        None,
344    )
345    .await?;
346
347    for source in merged.sources {
348        if source.location != *new_file_path {
349            object_store.delete(&source.location).await?;
350        }
351    }
352
353    Ok(())
354}
355
356pub(crate) async fn consolidate_parquet_files_from_object_store(
357    object_store: Arc<dyn ObjectStore>,
358    file_paths: Vec<ObjectPath>,
359    new_file_path: &ObjectPath,
360    compression: Option<Compression>,
361    max_row_group_size: Option<usize>,
362    deduplicate: Option<bool>,
363) -> anyhow::Result<()> {
364    let Some(merged) =
365        read_combined_batches(object_store.clone(), &file_paths, deduplicate).await?
366    else {
367        return Ok(());
368    };
369
370    let created_by =
371        consolidation_state::begin(&object_store, new_file_path, &merged.sources).await?;
372    write_batches_to_object_store_with_mode(
373        &merged.batches,
374        object_store.clone(),
375        new_file_path,
376        compression,
377        max_row_group_size,
378        None,
379        PutMode::Overwrite,
380        Some(created_by),
381    )
382    .await?;
383    let directory = new_file_path.parent().unwrap_or_default();
384    consolidation_state::recover(&object_store, &directory).await
385}
386
387struct MergedParquet {
388    batches: Vec<RecordBatch>,
389    sources: Vec<ObjectMeta>,
390}
391
392async fn read_combined_batches(
393    object_store: Arc<dyn ObjectStore>,
394    file_paths: &[ObjectPath],
395    deduplicate: Option<bool>,
396) -> anyhow::Result<Option<MergedParquet>> {
397    if file_paths.len() <= 1 {
398        return Ok(None);
399    }
400
401    let mut all_batches: Vec<RecordBatch> = Vec::new();
402    let mut schema_with_metadata: Option<Arc<arrow::datatypes::Schema>> = None;
403    let mut schema_source: Option<&ObjectPath> = None;
404    let mut field_metadata_sources = HashMap::new();
405    let mut sources = Vec::with_capacity(file_paths.len());
406
407    // Read all files from object store
408    for path in file_paths {
409        let object = object_store.get(path).await?;
410        sources.push(object.meta.clone());
411        let data = object.bytes().await?;
412        let builder = ParquetRecordBatchReaderBuilder::try_new(data)?;
413
414        let candidate_schema = builder.schema().clone();
415        schema_with_metadata = Some(
416            if let (Some(schema), Some(source)) = (&schema_with_metadata, schema_source) {
417                let reconciled = reconcile_consolidation_schema_with_sources(
418                    schema,
419                    source,
420                    &field_metadata_sources,
421                    &candidate_schema,
422                    path,
423                )?;
424
425                if reconciled.schema_source == ConsolidationSchemaSource::Candidate {
426                    schema_source = Some(path);
427                }
428
429                for key in reconciled.candidate_field_metadata {
430                    field_metadata_sources.insert(key, path.clone());
431                }
432
433                reconciled.schema
434            } else {
435                schema_source = Some(path);
436                field_metadata_sources
437                    .extend(field_metadata_keys(&candidate_schema).map(|key| (key, path.clone())));
438                candidate_schema
439            },
440        );
441
442        for batch in builder.build()? {
443            all_batches.push(batch?);
444        }
445    }
446
447    // Re-apply the preserved schema metadata to all collected batches so that
448    // write_batches_to_object_store (which uses batches[0].schema()) can encode
449    // the correct Arrow schema metadata into the combined output file.
450    let schema = schema_with_metadata.expect("non-empty source files provide a schema");
451    all_batches = all_batches
452        .into_iter()
453        .map(|b| RecordBatch::try_new(schema.clone(), b.columns().to_vec()))
454        .collect::<Result<Vec<_>, _>>()?;
455
456    // Deduplicate rows if requested
457    let batches_to_write = if deduplicate.unwrap_or(false) {
458        deduplicate_record_batches(&all_batches)?
459    } else {
460        all_batches
461    };
462
463    let batches_to_write = sort_consolidation_batches(batches_to_write)?;
464    Ok(Some(MergedParquet {
465        batches: batches_to_write,
466        sources,
467    }))
468}
469
470fn sort_consolidation_batches(batches: Vec<RecordBatch>) -> anyhow::Result<Vec<RecordBatch>> {
471    let Some(first) = batches.first() else {
472        return Ok(batches);
473    };
474
475    if first.column_by_name("ts_init").is_none() {
476        return Ok(batches);
477    }
478
479    let mut previous = 0;
480    let mut ordered = true;
481
482    for batch in &batches {
483        let array = batch
484            .column_by_name("ts_init")
485            .expect("consistent batch schema");
486        let timestamps = U64ColumnRef::try_from_array(array.as_ref())
487            .ok_or_else(|| anyhow::anyhow!("Unsupported consolidation ts_init type"))?;
488
489        for row in 0..batch.num_rows() {
490            let timestamp = timestamps
491                .value(row)
492                .ok_or_else(|| anyhow::anyhow!("Invalid consolidation ts_init value"))?;
493            ordered &= previous <= timestamp;
494            previous = timestamp;
495        }
496    }
497
498    if ordered {
499        return Ok(batches);
500    }
501
502    let schema = first.schema();
503    let batch = concat_batches(&schema, &batches)?;
504    let original_index = Arc::new(UInt64Array::from_iter_values(0..batch.num_rows() as u64));
505
506    let options = Some(SortOptions {
507        descending: false,
508        nulls_first: false,
509    });
510
511    let indices = lexsort_to_indices(
512        &[
513            SortColumn {
514                values: batch
515                    .column_by_name("ts_init")
516                    .expect("ts_init exists")
517                    .clone(),
518                options,
519            },
520            SortColumn {
521                values: original_index,
522                options,
523            },
524        ],
525        None,
526    )?;
527
528    Ok(vec![take_record_batch(&batch, &indices)?])
529}
530
531#[derive(Clone, Copy, Debug, Eq, PartialEq)]
532enum ConsolidationSchemaSource {
533    Current,
534    Candidate,
535}
536
537#[derive(Debug)]
538struct ReconciledConsolidationSchema {
539    schema: Arc<Schema>,
540    schema_source: ConsolidationSchemaSource,
541    candidate_field_metadata: Vec<(String, String)>,
542}
543
544fn reconcile_consolidation_schema_with_sources(
545    current: &Arc<Schema>,
546    current_path: &ObjectPath,
547    current_field_metadata_sources: &HashMap<(String, String), ObjectPath>,
548    candidate: &Arc<Schema>,
549    candidate_path: &ObjectPath,
550) -> anyhow::Result<ReconciledConsolidationSchema> {
551    anyhow::ensure!(
552        current.fields().len() == candidate.fields().len(),
553        "Cannot consolidate Parquet files {current_path} and {candidate_path}: field schemas differ"
554    );
555    let mut candidate_field_metadata = Vec::new();
556
557    let fields = current
558        .fields()
559        .iter()
560        .zip(candidate.fields())
561        .map(|(current, candidate)| {
562            anyhow::ensure!(
563                current.name() == candidate.name()
564                    && current.data_type() == candidate.data_type()
565                    && current.is_nullable() == candidate.is_nullable(),
566                "Cannot consolidate Parquet files {current_path} and {candidate_path}: field schemas differ"
567            );
568            let mut metadata = current.metadata().clone();
569            for (key, value) in candidate.metadata() {
570                if let Some(current_value) = metadata.get(key) {
571                    let source = current_field_metadata_sources
572                        .get(&(current.name().clone(), key.clone()))
573                        .unwrap_or(current_path);
574                    anyhow::ensure!(
575                        current_value == value,
576                        "Cannot consolidate Parquet files {source} and {candidate_path}: field '{}' metadata differs",
577                        current.name(),
578                    );
579                } else {
580                    metadata.insert(key.clone(), value.clone());
581                    candidate_field_metadata.push((current.name().clone(), key.clone()));
582                }
583            }
584
585            Ok(Arc::new(current.as_ref().clone().with_metadata(metadata)))
586        })
587        .collect::<anyhow::Result<Vec<_>>>()?;
588
589    let schema_with_fields =
590        |metadata| Arc::new(Schema::new_with_metadata(fields.clone(), metadata));
591
592    if current.metadata() == candidate.metadata() {
593        return Ok(ReconciledConsolidationSchema {
594            schema: schema_with_fields(current.metadata().clone()),
595            schema_source: ConsolidationSchemaSource::Current,
596            candidate_field_metadata,
597        });
598    }
599
600    let without_precision = |schema: &Schema| {
601        let mut metadata = schema.metadata().clone();
602        metadata.remove(KEY_PRICE_PRECISION);
603        metadata.remove(KEY_SIZE_PRECISION);
604        metadata
605    };
606
607    let is_precision_fallback = |metadata: &HashMap<String, String>| {
608        metadata.get(KEY_PRICE_PRECISION).map(String::as_str) == Some("0")
609            && metadata.get(KEY_SIZE_PRECISION).map(String::as_str) == Some("0")
610    };
611
612    let has_precision = |metadata: &HashMap<String, String>| {
613        metadata.contains_key(KEY_PRICE_PRECISION) && metadata.contains_key(KEY_SIZE_PRECISION)
614    };
615
616    let current_metadata = current.metadata();
617    let candidate_metadata = candidate.metadata();
618
619    if without_precision(current) == without_precision(candidate) {
620        match (
621            is_precision_fallback(current_metadata),
622            is_precision_fallback(candidate_metadata),
623        ) {
624            (true, false) if has_precision(candidate_metadata) => {
625                return Ok(ReconciledConsolidationSchema {
626                    schema: schema_with_fields(candidate.metadata().clone()),
627                    schema_source: ConsolidationSchemaSource::Candidate,
628                    candidate_field_metadata,
629                });
630            }
631            (false, true) if has_precision(current_metadata) => {
632                return Ok(ReconciledConsolidationSchema {
633                    schema: schema_with_fields(current.metadata().clone()),
634                    schema_source: ConsolidationSchemaSource::Current,
635                    candidate_field_metadata,
636                });
637            }
638            _ => {}
639        }
640    }
641
642    anyhow::bail!(
643        "Cannot consolidate Parquet files {current_path} and {candidate_path}: schema metadata differs: {current_metadata:?} versus {candidate_metadata:?}"
644    )
645}
646
647fn field_metadata_keys(schema: &Schema) -> impl Iterator<Item = (String, String)> + '_ {
648    schema.fields().iter().flat_map(|field| {
649        field
650            .metadata()
651            .keys()
652            .map(|key| (field.name().clone(), key.clone()))
653    })
654}
655
656/// Extracts the minimum and maximum i64 values for the specified `column_name` from a Parquet file's metadata in object store.
657///
658/// # Errors
659///
660/// Returns an error if the file cannot be read, metadata parsing fails, or the column is missing or has no statistics.
661pub async fn min_max_from_parquet_metadata_object_store(
662    object_store: Arc<dyn ObjectStore>,
663    file_path: &ObjectPath,
664    column_name: &str,
665) -> anyhow::Result<(u64, u64)> {
666    // Download the parquet file from object store
667    let data = object_store.get(file_path).await?.bytes().await?;
668    let reader = SerializedFileReader::new(data)?;
669
670    let metadata = reader.metadata();
671    let mut overall_min_value: Option<i64> = None;
672    let mut overall_max_value: Option<i64> = None;
673
674    // Iterate through all row groups
675    for i in 0..metadata.num_row_groups() {
676        let row_group = metadata.row_group(i);
677
678        // Iterate through all columns in this row group
679        for j in 0..row_group.num_columns() {
680            let col_metadata = row_group.column(j);
681
682            if col_metadata.column_path().string() == column_name {
683                if let Some(stats) = col_metadata.statistics() {
684                    // Check if we have Int64 statistics
685                    if let Statistics::Int64(int64_stats) = stats {
686                        // Extract min value if available
687                        if let Some(&min_value) = int64_stats.min_opt()
688                            && (overall_min_value.is_none()
689                                || min_value < overall_min_value.unwrap())
690                        {
691                            overall_min_value = Some(min_value);
692                        }
693
694                        // Extract max value if available
695                        if let Some(&max_value) = int64_stats.max_opt()
696                            && (overall_max_value.is_none()
697                                || max_value > overall_max_value.unwrap())
698                        {
699                            overall_max_value = Some(max_value);
700                        }
701                    } else {
702                        anyhow::bail!("Warning: Column name '{column_name}' is not of type i64.");
703                    }
704                } else {
705                    anyhow::bail!(
706                        "Warning: Statistics not available for column '{column_name}' in row group {i}."
707                    );
708                }
709            }
710        }
711    }
712
713    // Return the min/max pair if both are available
714    if let (Some(min), Some(max)) = (overall_min_value, overall_max_value) {
715        Ok((u64::try_from(min)?, u64::try_from(max)?))
716    } else {
717        anyhow::bail!(
718            "Column '{column_name}' not found or has no Int64 statistics in any row group."
719        )
720    }
721}
722
723/// Creates an object store from a URI string with optional storage options.
724///
725/// Supports multiple cloud storage providers:
726/// - AWS S3: `s3://bucket/path`
727/// - Google Cloud Storage: `gs://bucket/path` or `gcs://bucket/path`
728/// - Azure Blob Storage: `az://account/container/path` or `abfs://container@account.dfs.core.windows.net/path`
729/// - HTTP/WebDAV: `http://` or `https://`
730/// - Local files: `file://path` or plain paths
731///
732/// # Parameters
733///
734/// - `path`: The URI string for the storage location.
735/// - `storage_options`: Optional `HashMap` of `object_store` configuration keys for the scheme,
736///   under their prefixed or short names (`aws_region` or `region`):
737///   - For S3: `endpoint_url`, region, `access_key_id`, `secret_access_key`, `session_token`, etc.
738///     The legacy `key` and `secret` names are also accepted.
739///   - For GCS: `service_account_path`, `service_account_key`, `application_credentials`, etc.
740///   - For Azure: `account_name`, `account_key`, `sas_token`, etc.
741///   - For HTTP: client options such as `timeout`.
742///
743/// Returns a tuple of (`ObjectStore`, `base_path`, `normalized_uri`)
744///
745/// # Errors
746///
747/// Returns an error if the URI is invalid, a storage option key is unknown for the scheme, or the
748/// object store cannot be built.
749pub fn create_object_store_from_path(
750    path: &str,
751    storage_options: Option<AHashMap<String, String>>,
752) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
753    let location = create_object_store_location_from_path(path, storage_options)?;
754    Ok((
755        location.object_store,
756        location.base_path,
757        location.original_uri,
758    ))
759}
760
761// `storage_options` is only consumed by the cloud-feature arms,
762// so keep the allow scoped to the no-cloud build.
763#[cfg_attr(
764    not(feature = "cloud"),
765    allow(unused_variables, clippy::needless_pass_by_value)
766)]
767pub(crate) fn create_object_store_location_from_path(
768    path: &str,
769    storage_options: Option<AHashMap<String, String>>,
770) -> anyhow::Result<ObjectStoreLocation> {
771    let uri = normalize_path_to_uri(path)?;
772
773    let (object_store, base_path, original_uri) = match uri.as_str() {
774        #[cfg(feature = "cloud")]
775        s if s.starts_with("s3://") => create_s3_store(&uri, storage_options),
776        #[cfg(feature = "cloud")]
777        s if s.starts_with("gs://") || s.starts_with("gcs://") => {
778            create_gcs_store(&uri, storage_options)
779        }
780        #[cfg(feature = "cloud")]
781        s if s.starts_with("az://") => create_azure_store(&uri, storage_options),
782        #[cfg(feature = "cloud")]
783        s if s.starts_with("abfs://") => create_abfs_store(&uri, storage_options),
784        #[cfg(feature = "cloud")]
785        s if s.starts_with("http://") || s.starts_with("https://") => {
786            create_http_store(&uri, storage_options)
787        }
788        #[cfg(not(feature = "cloud"))]
789        s if s.starts_with("s3://")
790            || s.starts_with("gs://")
791            || s.starts_with("gcs://")
792            || s.starts_with("az://")
793            || s.starts_with("abfs://")
794            || s.starts_with("http://")
795            || s.starts_with("https://") =>
796        {
797            anyhow::bail!("Cloud storage support requires the 'cloud' feature: {uri}")
798        }
799        s if s.starts_with("file://") => create_local_store(&uri, true),
800        _ => create_local_store(&uri, false), // Fallback: assume local path
801    }?;
802
803    Ok(ObjectStoreLocation {
804        object_store,
805        base_path,
806        original_uri,
807    })
808}
809
810pub(crate) fn is_remote_uri_scheme(scheme: &str) -> bool {
811    matches!(
812        scheme,
813        "s3" | "gs" | "gcs" | "az" | "abfs" | "http" | "https"
814    )
815}
816
817pub(crate) fn remote_store_root_url(uri: &str) -> anyhow::Result<Url> {
818    let mut url = Url::parse(uri)?;
819    url.set_path("");
820    url.set_query(None);
821    url.set_fragment(None);
822    Ok(url)
823}
824
825pub(crate) fn remote_full_uri(uri: &str, object_path: &str) -> anyhow::Result<String> {
826    let root = remote_store_root_url(uri)?;
827    let root = root.as_str().trim_end_matches('/');
828    let object_path = object_path.trim_start_matches('/');
829
830    if object_path.is_empty() {
831        Ok(root.to_string())
832    } else {
833        Ok(format!("{root}/{object_path}"))
834    }
835}
836
837/// Appends an encoded object-store path to a local or remote storage URI.
838/// Preserve the encoded names used by the native object-store backend.
839pub(crate) fn append_path_to_file_uri(base_uri: &str, path: &str) -> String {
840    if let Ok(mut url) = Url::parse(base_uri) {
841        if let Ok(mut segments) = url.path_segments_mut() {
842            segments.pop_if_empty();
843            segments.extend(
844                path.trim_end_matches('/')
845                    .split('/')
846                    .filter(|segment| !segment.is_empty()),
847            );
848        }
849
850        return url.to_string();
851    }
852
853    format!(
854        "{}/{}",
855        base_uri.trim_end_matches('/'),
856        path.trim_end_matches('/')
857    )
858}
859
860/// Decodes a percent-encoded `object_store` path segment back to its logical form.
861///
862/// `object_store` lists path segments in URL-encoded form, so a non-ASCII instrument
863/// directory reads back with each non-ASCII byte as a `%XX` sequence. Decoding recovers the
864/// original id for matching against `urisafe_instrument_id`. Returns the input unchanged when
865/// it is not valid percent-encoded UTF-8.
866pub(crate) fn decode_object_store_segment(segment: &str) -> String {
867    ObjectPath::from_url_path(segment).map_or_else(|_| segment.to_string(), String::from)
868}
869
870fn create_local_store(
871    uri: &str,
872    is_file_uri: bool,
873) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
874    let path = if is_file_uri {
875        file_uri_to_native_path(uri)
876    } else {
877        uri.to_string()
878    };
879
880    let local_store =
881        object_store::local::LocalFileSystem::new_with_prefix(&path).with_context(|| {
882            format!(
883                "failed to open local storage directory '{path}'; \
884             create it if it does not exist and check access permissions"
885            )
886        })?;
887
888    Ok((Arc::new(local_store), String::new(), uri.to_string()))
889}
890
891/// Helper function to create S3 object store with options.
892#[cfg(feature = "cloud")]
893fn create_s3_store(
894    uri: &str,
895    storage_options: Option<AHashMap<String, String>>,
896) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
897    let (url, path) = parse_url_and_path(uri)?;
898    let bucket = extract_host(&url, "Invalid S3 URI: missing bucket")?;
899
900    let mut builder = object_store::aws::AmazonS3Builder::new().with_bucket_name(&bucket);
901
902    // Apply storage options if provided
903    if let Some(options) = storage_options {
904        for (key, value) in options {
905            // The `object_store` parser lacks the legacy fsspec `key` and `secret` names
906            let config_key = match key.as_str() {
907                "key" => object_store::aws::AmazonS3ConfigKey::AccessKeyId,
908                "secret" => object_store::aws::AmazonS3ConfigKey::SecretAccessKey,
909                _ => key.parse()?,
910            };
911            builder = builder.with_config(config_key, value);
912        }
913    }
914
915    let s3_store = builder.build()?;
916    Ok((Arc::new(s3_store), path, uri.to_string()))
917}
918
919/// Helper function to create GCS object store with options.
920#[cfg(feature = "cloud")]
921fn create_gcs_store(
922    uri: &str,
923    storage_options: Option<AHashMap<String, String>>,
924) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
925    let (url, path) = parse_url_and_path(uri)?;
926    let bucket = extract_host(&url, "Invalid GCS URI: missing bucket")?;
927
928    let mut builder = object_store::gcp::GoogleCloudStorageBuilder::new().with_bucket_name(&bucket);
929
930    // Apply storage options if provided
931    if let Some(options) = storage_options {
932        for (key, value) in options {
933            builder =
934                builder.with_config(key.parse::<object_store::gcp::GoogleConfigKey>()?, value);
935        }
936    }
937
938    let gcs_store = builder.build()?;
939    Ok((Arc::new(gcs_store), path, uri.to_string()))
940}
941
942/// Helper function to create Azure object store with options.
943#[cfg(feature = "cloud")]
944fn create_azure_store(
945    uri: &str,
946    storage_options: Option<AHashMap<String, String>>,
947) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
948    let (url, path) = parse_url_and_path(uri)?;
949    let container = extract_host(&url, "Invalid Azure URI: missing container")?;
950
951    let mut builder =
952        object_store::azure::MicrosoftAzureBuilder::new().with_container_name(container);
953
954    // Apply storage options if provided
955    if let Some(options) = storage_options {
956        builder = apply_azure_storage_options(builder, options)?;
957    }
958
959    let azure_store = builder.build()?;
960    Ok((Arc::new(azure_store), path, uri.to_string()))
961}
962
963/// Helper function to create Azure object store from abfs:// URI with options.
964#[cfg(feature = "cloud")]
965fn create_abfs_store(
966    uri: &str,
967    storage_options: Option<AHashMap<String, String>>,
968) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
969    let (url, path) = parse_url_and_path(uri)?;
970    let host = extract_host(&url, "Invalid ABFS URI: missing host")?;
971
972    // Extract account from host (account.dfs.core.windows.net)
973    let account = host
974        .split('.')
975        .next()
976        .ok_or_else(|| anyhow::anyhow!("Invalid ABFS URI: cannot extract account from host"))?;
977
978    // Extract container from username part
979    let container = url
980        .username()
981        .split('@')
982        .next()
983        .ok_or_else(|| anyhow::anyhow!("Invalid ABFS URI: missing container"))?;
984
985    let mut builder = object_store::azure::MicrosoftAzureBuilder::new()
986        .with_account(account)
987        .with_container_name(container);
988
989    // Apply storage options if provided (same as Azure store)
990    if let Some(options) = storage_options {
991        builder = apply_azure_storage_options(builder, options)?;
992    }
993
994    let azure_store = builder.build()?;
995    Ok((Arc::new(azure_store), path, uri.to_string()))
996}
997
998/// Applies shared Azure storage options to the builder for the `az` and `abfs` schemes.
999#[cfg(feature = "cloud")]
1000fn apply_azure_storage_options(
1001    mut builder: object_store::azure::MicrosoftAzureBuilder,
1002    options: AHashMap<String, String>,
1003) -> anyhow::Result<object_store::azure::MicrosoftAzureBuilder> {
1004    for (key, value) in options {
1005        builder = builder.with_config(key.parse::<object_store::azure::AzureConfigKey>()?, value);
1006    }
1007
1008    Ok(builder)
1009}
1010
1011/// Helper function to create HTTP object store with options.
1012#[cfg(feature = "cloud")]
1013fn create_http_store(
1014    uri: &str,
1015    storage_options: Option<AHashMap<String, String>>,
1016) -> anyhow::Result<(Arc<dyn ObjectStore>, String, String)> {
1017    let (_, path) = parse_url_and_path(uri)?;
1018    let base_url = remote_store_root_url(uri)?
1019        .as_str()
1020        .trim_end_matches('/')
1021        .to_string();
1022
1023    let mut builder = object_store::http::HttpBuilder::new().with_url(base_url);
1024
1025    // Apply storage options if provided
1026    if let Some(options) = storage_options {
1027        for (key, value) in options {
1028            builder = builder.with_config(key.parse::<object_store::ClientConfigKey>()?, value);
1029        }
1030    }
1031
1032    let http_store = builder.build()?;
1033    Ok((Arc::new(http_store), path, uri.to_string()))
1034}
1035
1036/// Parses a remote storage URI into its URL and object-store base path.
1037///
1038/// Catalog listings return keys under the encoded base path while lookups use the path as
1039/// written, so a base path that URL parsing or object-store path encoding changes is rejected.
1040#[cfg(feature = "cloud")]
1041fn parse_url_and_path(uri: &str) -> anyhow::Result<(Url, String)> {
1042    let url = Url::parse(uri)?;
1043    let path = url.path().trim_start_matches('/').to_string();
1044    let raw_path = uri
1045        .split_once("://")
1046        .and_then(|(_, rest)| rest.split_once('/'))
1047        .map_or("", |(_, raw)| raw.trim_start_matches('/'));
1048
1049    anyhow::ensure!(
1050        raw_path == path && ObjectPath::from(path.as_str()).as_ref() == path.trim_end_matches('/'),
1051        "Storage URI '{uri}' has a base path that URL parsing or object-store path encoding \
1052         changes; use a base path without spaces, non-ASCII, or reserved characters",
1053    );
1054
1055    Ok((url, path))
1056}
1057
1058/// Helper function to extract host from URL with error handling.
1059#[cfg(feature = "cloud")]
1060fn extract_host(url: &Url, error_msg: &str) -> anyhow::Result<String> {
1061    url.host_str()
1062        .map(ToString::to_string)
1063        .ok_or_else(|| anyhow::anyhow!("{error_msg}"))
1064}
1065
1066#[cfg(test)]
1067mod tests {
1068    #[cfg(feature = "cloud")]
1069    use std::io::{BufRead, Write};
1070    use std::{collections::HashMap, sync::Arc};
1071
1072    #[cfg(feature = "cloud")]
1073    use ahash::AHashMap;
1074    use arrow::{
1075        array::{Array, ArrayRef, StringArray, TimestampNanosecondArray, UInt64Array},
1076        datatypes::{DataType, Field, Schema},
1077    };
1078    use nautilus_serialization::arrow::{json_string_field, timestamp_data_type};
1079    use parquet::file::{properties::ReaderProperties, serialized_reader::ReadOptionsBuilder};
1080    use rstest::rstest;
1081
1082    use super::*;
1083
1084    #[rstest]
1085    #[tokio::test]
1086    async fn combine_accepts_an_output_outside_the_source_directory() {
1087        use futures::TryStreamExt;
1088
1089        let store: Arc<dyn ObjectStore> = Arc::new(object_store::memory::InMemory::new());
1090
1091        let schema = Arc::new(Schema::new(vec![Field::new(
1092            "value",
1093            DataType::UInt64,
1094            false,
1095        )]));
1096
1097        let paths = [
1098            ObjectPath::from("inputs/one.parquet"),
1099            ObjectPath::from("inputs/two.parquet"),
1100        ];
1101
1102        for (path, value) in paths.iter().zip([11, 22]) {
1103            let batch = RecordBatch::try_new(
1104                schema.clone(),
1105                vec![Arc::new(UInt64Array::from(vec![value]))],
1106            )
1107            .unwrap();
1108            write_batches_to_object_store(&[batch], store.clone(), path, None, None, None)
1109                .await
1110                .unwrap();
1111        }
1112
1113        let target = ObjectPath::from("outputs/combined.parquet");
1114        combine_parquet_files_from_object_store(
1115            store.clone(),
1116            paths.to_vec(),
1117            &target,
1118            None,
1119            None,
1120            None,
1121        )
1122        .await
1123        .unwrap();
1124        let (batches, actual_schema) = read_parquet_from_object_store(store.clone(), &target)
1125            .await
1126            .unwrap();
1127        let actual = concat_batches(&actual_schema, &batches).unwrap();
1128        let expected =
1129            RecordBatch::try_new(schema, vec![Arc::new(UInt64Array::from(vec![11, 22]))]).unwrap();
1130        let remaining = store.list(None).try_collect::<Vec<_>>().await.unwrap();
1131
1132        assert_eq!(actual, expected);
1133        assert_eq!(
1134            remaining
1135                .iter()
1136                .map(|meta| &meta.location)
1137                .collect::<Vec<_>>(),
1138            vec![&target]
1139        );
1140    }
1141
1142    fn consolidation_depth_schema(
1143        price_precision: &str,
1144        size_precision: &str,
1145        instrument_id: &str,
1146    ) -> Arc<Schema> {
1147        Arc::new(Schema::new_with_metadata(
1148            vec![Field::new("bids", DataType::Utf8, false)],
1149            HashMap::from([
1150                (KEY_PRICE_PRECISION.to_string(), price_precision.to_string()),
1151                (KEY_SIZE_PRECISION.to_string(), size_precision.to_string()),
1152                (KEY_IDENTIFIER.to_string(), instrument_id.to_string()),
1153            ]),
1154        ))
1155    }
1156
1157    fn reconcile_consolidation_schema(
1158        current: &Arc<Schema>,
1159        current_path: &ObjectPath,
1160        candidate: &Arc<Schema>,
1161        candidate_path: &ObjectPath,
1162    ) -> anyhow::Result<ReconciledConsolidationSchema> {
1163        let field_metadata_sources = field_metadata_keys(current)
1164            .map(|key| (key, current_path.clone()))
1165            .collect();
1166        reconcile_consolidation_schema_with_sources(
1167            current,
1168            current_path,
1169            &field_metadata_sources,
1170            candidate,
1171            candidate_path,
1172        )
1173    }
1174
1175    #[rstest]
1176    fn consolidation_schema_prefers_populated_depth_precision_in_either_order() {
1177        let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1178        let populated = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1179        let fallback_path = ObjectPath::from("empty.parquet");
1180        let populated_path = ObjectPath::from("populated.parquet");
1181
1182        let fallback_first =
1183            reconcile_consolidation_schema(&fallback, &fallback_path, &populated, &populated_path)
1184                .unwrap();
1185        let populated_first =
1186            reconcile_consolidation_schema(&populated, &populated_path, &fallback, &fallback_path)
1187                .unwrap();
1188
1189        assert_eq!(
1190            fallback_first.schema_source,
1191            ConsolidationSchemaSource::Candidate,
1192        );
1193        assert_eq!(
1194            populated_first.schema_source,
1195            ConsolidationSchemaSource::Current,
1196        );
1197
1198        for schema in [fallback_first.schema, populated_first.schema] {
1199            assert_eq!(schema.metadata()[KEY_PRICE_PRECISION], "2");
1200            assert_eq!(schema.metadata()[KEY_SIZE_PRECISION], "3");
1201        }
1202    }
1203
1204    #[rstest]
1205    fn consolidation_schema_rejects_other_metadata_mismatches() {
1206        let current = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1207        let candidate = consolidation_depth_schema("2", "3", "BTCUSDT.BINANCE");
1208        let current_path = ObjectPath::from("eth.parquet");
1209        let candidate_path = ObjectPath::from("btc.parquet");
1210
1211        let error =
1212            reconcile_consolidation_schema(&current, &current_path, &candidate, &candidate_path)
1213                .unwrap_err();
1214
1215        assert!(error.to_string().contains("eth.parquet and btc.parquet"));
1216        assert!(error.to_string().contains("ETHUSDT.BINANCE"));
1217        assert!(error.to_string().contains("BTCUSDT.BINANCE"));
1218    }
1219
1220    #[rstest]
1221    fn consolidation_schema_rejects_two_populated_precisions() {
1222        let current = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1223        let candidate = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1224        let current_path = ObjectPath::from("precision-2.parquet");
1225        let candidate_path = ObjectPath::from("precision-4.parquet");
1226
1227        let error =
1228            reconcile_consolidation_schema(&current, &current_path, &candidate, &candidate_path)
1229                .unwrap_err();
1230
1231        assert!(error.to_string().contains("precision-2.parquet"));
1232        assert!(error.to_string().contains("precision-4.parquet"));
1233    }
1234
1235    #[rstest]
1236    fn consolidation_schema_keeps_fallback_for_all_empty_files() {
1237        let first = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1238        let second = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1239
1240        let reconciled = reconcile_consolidation_schema(
1241            &first,
1242            &ObjectPath::from("first-empty.parquet"),
1243            &second,
1244            &ObjectPath::from("second-empty.parquet"),
1245        )
1246        .unwrap();
1247
1248        assert_eq!(reconciled.schema.metadata()[KEY_PRICE_PRECISION], "0");
1249        assert_eq!(reconciled.schema.metadata()[KEY_SIZE_PRECISION], "0");
1250    }
1251
1252    #[rstest]
1253    #[case::missing_first(true)]
1254    #[case::fallback_first(false)]
1255    fn consolidation_schema_rejects_missing_precision_against_fallback(
1256        #[case] missing_first: bool,
1257    ) {
1258        let missing = Arc::new(Schema::new_with_metadata(
1259            vec![Field::new("bids", DataType::Utf8, false)],
1260            HashMap::from([(KEY_IDENTIFIER.to_string(), "ETHUSDT.BINANCE".to_string())]),
1261        ));
1262        let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1263        let missing_path = ObjectPath::from("missing.parquet");
1264        let fallback_path = ObjectPath::from("fallback.parquet");
1265
1266        let error = if missing_first {
1267            reconcile_consolidation_schema(&missing, &missing_path, &fallback, &fallback_path)
1268        } else {
1269            reconcile_consolidation_schema(&fallback, &fallback_path, &missing, &missing_path)
1270        }
1271        .unwrap_err();
1272
1273        assert!(error.to_string().contains("schema metadata differs"));
1274    }
1275
1276    #[rstest]
1277    fn consolidation_schema_merges_json_field_annotation_in_either_order() {
1278        let bare = Arc::new(Schema::new(vec![Field::new("info", DataType::Utf8, true)]));
1279        let annotated = Arc::new(Schema::new(vec![json_string_field("info", true)]));
1280        let bare_path = ObjectPath::from("bare.parquet");
1281        let annotated_path = ObjectPath::from("annotated.parquet");
1282
1283        let bare_first =
1284            reconcile_consolidation_schema(&bare, &bare_path, &annotated, &annotated_path).unwrap();
1285        let annotated_first =
1286            reconcile_consolidation_schema(&annotated, &annotated_path, &bare, &bare_path).unwrap();
1287
1288        assert_eq!(bare_first.schema, annotated_first.schema);
1289        assert_eq!(
1290            bare_first.schema.field_with_name("info").unwrap(),
1291            &json_string_field("info", true),
1292        );
1293    }
1294
1295    #[rstest]
1296    fn consolidation_field_metadata_conflict_names_the_winning_file() {
1297        let bare = Arc::new(Schema::new(vec![Field::new("info", DataType::Utf8, true)]));
1298        let annotated = Arc::new(Schema::new(vec![json_string_field("info", true)]));
1299
1300        let conflicting = Arc::new(Schema::new(vec![
1301            Field::new("info", DataType::Utf8, true).with_metadata(HashMap::from([(
1302                "ARROW:extension:name".to_string(),
1303                "other.extension".to_string(),
1304            )])),
1305        ]));
1306        let bare_path = ObjectPath::from("bare.parquet");
1307        let annotated_path = ObjectPath::from("annotated.parquet");
1308        let conflicting_path = ObjectPath::from("conflicting.parquet");
1309        let reconciled =
1310            reconcile_consolidation_schema(&bare, &bare_path, &annotated, &annotated_path).unwrap();
1311
1312        assert_eq!(reconciled.candidate_field_metadata.len(), 2);
1313        assert!(
1314            reconciled
1315                .candidate_field_metadata
1316                .contains(&("info".to_string(), "ARROW:extension:name".to_string())),
1317        );
1318        assert!(
1319            reconciled
1320                .candidate_field_metadata
1321                .contains(&("info".to_string(), "ARROW:extension:metadata".to_string())),
1322        );
1323        let field_metadata_sources = reconciled
1324            .candidate_field_metadata
1325            .iter()
1326            .cloned()
1327            .map(|key| (key, annotated_path.clone()))
1328            .collect();
1329        let error = reconcile_consolidation_schema_with_sources(
1330            &reconciled.schema,
1331            &bare_path,
1332            &field_metadata_sources,
1333            &conflicting,
1334            &conflicting_path,
1335        )
1336        .unwrap_err();
1337        assert!(error.to_string().contains("annotated.parquet"));
1338        assert!(error.to_string().contains("conflicting.parquet"));
1339        assert!(error.to_string().contains("field 'info' metadata differs"));
1340    }
1341
1342    #[rstest]
1343    fn consolidation_conflict_names_the_last_winning_schema() {
1344        let fallback = consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE");
1345        let precision_2 = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1346        let precision_4 = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1347        let fallback_path = ObjectPath::from("fallback.parquet");
1348        let precision_2_path = ObjectPath::from("precision-2.parquet");
1349        let precision_4_path = ObjectPath::from("precision-4.parquet");
1350        let reconciled = reconcile_consolidation_schema(
1351            &fallback,
1352            &fallback_path,
1353            &precision_2,
1354            &precision_2_path,
1355        )
1356        .unwrap();
1357
1358        let error = reconcile_consolidation_schema(
1359            &reconciled.schema,
1360            &precision_2_path,
1361            &precision_4,
1362            &precision_4_path,
1363        )
1364        .unwrap_err();
1365
1366        assert!(error.to_string().contains("precision-2.parquet"));
1367        assert!(error.to_string().contains("precision-4.parquet"));
1368    }
1369
1370    #[rstest]
1371    fn consolidation_field_metadata_does_not_replace_precision_source() {
1372        let precision_2 = consolidation_depth_schema("2", "3", "ETHUSDT.BINANCE");
1373
1374        let fallback = Arc::new(Schema::new_with_metadata(
1375            vec![json_string_field("bids", false)],
1376            consolidation_depth_schema("0", "0", "ETHUSDT.BINANCE")
1377                .metadata()
1378                .clone(),
1379        ));
1380        let precision_4 = consolidation_depth_schema("4", "5", "ETHUSDT.BINANCE");
1381        let precision_2_path = ObjectPath::from("precision-2.parquet");
1382        let fallback_path = ObjectPath::from("fallback-annotated.parquet");
1383        let precision_4_path = ObjectPath::from("precision-4.parquet");
1384        let reconciled = reconcile_consolidation_schema(
1385            &precision_2,
1386            &precision_2_path,
1387            &fallback,
1388            &fallback_path,
1389        )
1390        .unwrap();
1391
1392        assert_eq!(reconciled.schema_source, ConsolidationSchemaSource::Current,);
1393        assert_eq!(reconciled.candidate_field_metadata.len(), 2);
1394        assert!(
1395            reconciled
1396                .candidate_field_metadata
1397                .contains(&("bids".to_string(), "ARROW:extension:name".to_string())),
1398        );
1399        assert!(
1400            reconciled
1401                .candidate_field_metadata
1402                .contains(&("bids".to_string(), "ARROW:extension:metadata".to_string())),
1403        );
1404        let field_metadata_sources = reconciled
1405            .candidate_field_metadata
1406            .iter()
1407            .cloned()
1408            .map(|key| (key, fallback_path.clone()))
1409            .collect();
1410        let error = reconcile_consolidation_schema_with_sources(
1411            &reconciled.schema,
1412            &precision_2_path,
1413            &field_metadata_sources,
1414            &precision_4,
1415            &precision_4_path,
1416        )
1417        .unwrap_err();
1418        let message = error.to_string();
1419
1420        assert!(message.contains("precision-2.parquet"));
1421        assert!(message.contains("precision-4.parquet"));
1422        assert!(!message.contains("fallback-annotated.parquet"));
1423    }
1424
1425    #[rstest]
1426    fn consolidation_field_metadata_tracks_each_origin() {
1427        let bare = Arc::new(Schema::new(vec![
1428            Field::new("info", DataType::Utf8, true),
1429            Field::new("balances", DataType::Utf8, true),
1430        ]));
1431
1432        let info = Arc::new(Schema::new(vec![
1433            json_string_field("info", true),
1434            Field::new("balances", DataType::Utf8, true),
1435        ]));
1436
1437        let balances = Arc::new(Schema::new(vec![
1438            json_string_field("info", true),
1439            json_string_field("balances", true),
1440        ]));
1441
1442        let conflicting = Arc::new(Schema::new(vec![
1443            Field::new("info", DataType::Utf8, true).with_metadata(HashMap::from([(
1444                "ARROW:extension:name".to_string(),
1445                "other.extension".to_string(),
1446            )])),
1447            json_string_field("balances", true),
1448        ]));
1449        let bare_path = ObjectPath::from("bare.parquet");
1450        let info_path = ObjectPath::from("info.parquet");
1451        let balances_path = ObjectPath::from("balances.parquet");
1452        let conflicting_path = ObjectPath::from("conflicting.parquet");
1453        let with_info =
1454            reconcile_consolidation_schema(&bare, &bare_path, &info, &info_path).unwrap();
1455        let mut field_metadata_sources = with_info
1456            .candidate_field_metadata
1457            .iter()
1458            .cloned()
1459            .map(|key| (key, info_path.clone()))
1460            .collect::<HashMap<_, _>>();
1461        let with_balances = reconcile_consolidation_schema_with_sources(
1462            &with_info.schema,
1463            &bare_path,
1464            &field_metadata_sources,
1465            &balances,
1466            &balances_path,
1467        )
1468        .unwrap();
1469
1470        for key in with_balances.candidate_field_metadata {
1471            field_metadata_sources.insert(key, balances_path.clone());
1472        }
1473
1474        let error = reconcile_consolidation_schema_with_sources(
1475            &with_balances.schema,
1476            &bare_path,
1477            &field_metadata_sources,
1478            &conflicting,
1479            &conflicting_path,
1480        )
1481        .unwrap_err();
1482        let message = error.to_string();
1483
1484        assert!(message.contains("info.parquet"));
1485        assert!(message.contains("conflicting.parquet"));
1486        assert!(!message.contains("balances.parquet"));
1487    }
1488
1489    #[tokio::test]
1490    async fn write_batches_to_object_store_rejects_empty_input() {
1491        let object_store = Arc::new(object_store::memory::InMemory::new());
1492
1493        let error = write_batches_to_object_store(
1494            &[],
1495            object_store,
1496            &ObjectPath::from("empty.parquet"),
1497            None,
1498            None,
1499            None,
1500        )
1501        .await
1502        .unwrap_err();
1503
1504        assert_eq!(
1505            error.to_string(),
1506            "Cannot write Parquet file empty.parquet with no record batches"
1507        );
1508    }
1509
1510    #[tokio::test]
1511    async fn default_writer_sets_zstd_sorting_and_identifier_bloom_filter() {
1512        let directory = tempfile::tempdir().unwrap();
1513        let path = directory.path().join("layout.parquet");
1514
1515        let object_store = Arc::new(
1516            object_store::local::LocalFileSystem::new_with_prefix(directory.path()).unwrap(),
1517        );
1518        let object_path = ObjectPath::from("layout.parquet");
1519
1520        let schema = Arc::new(Schema::new(vec![
1521            Field::new("identifier", DataType::Utf8, false),
1522            Field::new("ts_init", DataType::UInt64, false),
1523        ]));
1524        let batch = RecordBatch::try_new(
1525            schema,
1526            vec![
1527                Arc::new(StringArray::from(vec!["AUD/USD.SIM", "AUD/USD.SIM"])) as ArrayRef,
1528                Arc::new(UInt64Array::from(vec![1_u64, 2])) as ArrayRef,
1529            ],
1530        )
1531        .unwrap();
1532
1533        write_batches_to_object_store(&[batch], object_store, &object_path, None, None, None)
1534            .await
1535            .unwrap();
1536
1537        let read_options = ReadOptionsBuilder::new()
1538            .with_reader_properties(
1539                ReaderProperties::builder()
1540                    .set_read_bloom_filter(true)
1541                    .build(),
1542            )
1543            .build();
1544        let reader = SerializedFileReader::new_with_options(
1545            std::fs::File::open(path).unwrap(),
1546            read_options,
1547        )
1548        .unwrap();
1549        let row_group = reader.metadata().row_group(0);
1550        let sorting = row_group.sorting_columns().unwrap();
1551
1552        assert_eq!(crate::backend::parquet::DEFAULT_ROW_GROUP_SIZE, 131_072);
1553        assert_eq!(
1554            sorting,
1555            &vec![
1556                SortingColumn {
1557                    column_idx: 1,
1558                    descending: false,
1559                    nulls_first: false,
1560                },
1561                SortingColumn {
1562                    column_idx: 0,
1563                    descending: false,
1564                    nulls_first: false,
1565                },
1566            ],
1567        );
1568        assert!(
1569            row_group
1570                .columns()
1571                .iter()
1572                .all(|column| column.compression() == Compression::ZSTD(ZstdLevel::default())),
1573        );
1574        assert!(
1575            reader
1576                .get_row_group(0)
1577                .unwrap()
1578                .get_column_bloom_filter(0)
1579                .is_some(),
1580        );
1581    }
1582
1583    #[tokio::test]
1584    async fn default_writer_delta_encodes_timestamps_without_dictionary() {
1585        let directory = tempfile::tempdir().unwrap();
1586        let path = directory.path().join("timestamps.parquet");
1587
1588        let object_store = Arc::new(
1589            object_store::local::LocalFileSystem::new_with_prefix(directory.path()).unwrap(),
1590        );
1591        let object_path = ObjectPath::from("timestamps.parquet");
1592
1593        let schema = Arc::new(Schema::new(vec![
1594            Field::new("identifier", DataType::Utf8, false),
1595            Field::new("ts_event", timestamp_data_type(), false),
1596            Field::new("ts_init", timestamp_data_type(), false),
1597        ]));
1598        let identifiers = StringArray::from(vec!["AUD/USD.SIM", "AUD/USD.SIM", "AUD/USD.SIM"]);
1599        let ts_event = TimestampNanosecondArray::from(vec![
1600            1_700_000_000_123_456_001_i64,
1601            1_700_000_000_123_456_789,
1602            1_700_000_001_000_000_000,
1603        ])
1604        .with_timezone("UTC");
1605        let ts_init = TimestampNanosecondArray::from(vec![
1606            1_700_000_000_123_457_000_i64,
1607            1_700_000_000_123_458_000,
1608            1_700_000_001_000_001_000,
1609        ])
1610        .with_timezone("UTC");
1611        let batch = RecordBatch::try_new(
1612            schema,
1613            vec![
1614                Arc::new(identifiers.clone()) as ArrayRef,
1615                Arc::new(ts_event.clone()) as ArrayRef,
1616                Arc::new(ts_init.clone()) as ArrayRef,
1617            ],
1618        )
1619        .unwrap();
1620
1621        write_batches_to_object_store(&[batch], object_store, &object_path, None, None, None)
1622            .await
1623            .unwrap();
1624
1625        let reader = SerializedFileReader::new(std::fs::File::open(&path).unwrap()).unwrap();
1626        let columns = reader.metadata().row_group(0).columns();
1627        let timestamp_encodings = columns[1..]
1628            .iter()
1629            .map(|column| {
1630                (
1631                    column.column_path().string(),
1632                    column.dictionary_page_offset(),
1633                    column.encodings().collect::<Vec<_>>(),
1634                )
1635            })
1636            .collect::<Vec<_>>();
1637        let read = ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(&path).unwrap())
1638            .unwrap()
1639            .build()
1640            .unwrap()
1641            .collect::<Result<Vec<_>, _>>()
1642            .unwrap();
1643
1644        assert!(columns[0].dictionary_page_offset().is_some());
1645        assert_eq!(
1646            timestamp_encodings,
1647            vec![
1648                (
1649                    "ts_event".to_string(),
1650                    None,
1651                    vec![Encoding::RLE, Encoding::DELTA_BINARY_PACKED],
1652                ),
1653                (
1654                    "ts_init".to_string(),
1655                    None,
1656                    vec![Encoding::RLE, Encoding::DELTA_BINARY_PACKED],
1657                ),
1658            ],
1659        );
1660        assert_eq!(read.len(), 1);
1661        assert_eq!(read[0].column(0).as_ref(), &identifiers as &dyn Array);
1662        assert_eq!(read[0].column(1).as_ref(), &ts_event as &dyn Array);
1663        assert_eq!(read[0].column(2).as_ref(), &ts_init as &dyn Array);
1664    }
1665
1666    #[rstest]
1667    fn test_create_object_store_from_path_local() {
1668        // Create a temporary directory for testing
1669        let temp_dir = std::env::temp_dir().join("nautilus_test");
1670        std::fs::create_dir_all(&temp_dir).unwrap();
1671
1672        let result = create_object_store_from_path(temp_dir.to_str().unwrap(), None);
1673        if let Err(e) = &result {
1674            println!("Error: {e:?}");
1675        }
1676
1677        assert!(result.is_ok());
1678        let (_, base_path, uri) = result.unwrap();
1679        assert_eq!(base_path, "");
1680        // The URI should be normalized to file:// format
1681        assert_eq!(uri, format!("file://{}", temp_dir.to_str().unwrap()));
1682
1683        // Clean up
1684        std::fs::remove_dir_all(&temp_dir).ok();
1685    }
1686
1687    #[rstest]
1688    fn append_path_to_file_uri_extends_windows_drive_catalog() {
1689        assert_eq!(
1690            append_path_to_file_uri(
1691                "file:///C:/data/catalog",
1692                "data/quotes/EURUSD.SIM/file.parquet",
1693            ),
1694            "file:///C:/data/catalog/data/quotes/EURUSD.SIM/file.parquet",
1695        );
1696    }
1697
1698    #[rstest]
1699    #[cfg(feature = "cloud")]
1700    fn test_create_object_store_from_path_s3() {
1701        let mut options = AHashMap::new();
1702        options.insert(
1703            "endpoint_url".to_string(),
1704            "https://test.endpoint.com".to_string(),
1705        );
1706        options.insert("region".to_string(), "us-west-2".to_string());
1707        options.insert("access_key_id".to_string(), "test_key".to_string());
1708        options.insert("secret_access_key".to_string(), "test_secret".to_string());
1709
1710        let result = create_object_store_from_path("s3://test-bucket/path", Some(options));
1711        assert!(result.is_ok());
1712        let (_, base_path, uri) = result.unwrap();
1713        assert_eq!(base_path, "path");
1714        assert_eq!(uri, "s3://test-bucket/path");
1715    }
1716
1717    #[rstest]
1718    #[cfg(feature = "cloud")]
1719    fn test_create_object_store_from_path_azure() {
1720        let mut options = AHashMap::new();
1721        options.insert("account_name".to_string(), "testaccount".to_string());
1722        // Use a valid base64 encoded key for testing
1723        options.insert("account_key".to_string(), "dGVzdGtleQ==".to_string()); // "testkey" in base64
1724
1725        let result = create_object_store_from_path("az://container/path", Some(options));
1726        if let Err(e) = &result {
1727            println!("Azure Error: {e:?}");
1728        }
1729
1730        assert!(result.is_ok());
1731        let (_, base_path, uri) = result.unwrap();
1732        assert_eq!(base_path, "path");
1733        assert_eq!(uri, "az://container/path");
1734    }
1735
1736    #[rstest]
1737    #[cfg(feature = "cloud")]
1738    fn test_create_object_store_from_path_gcs() {
1739        let credentials_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
1740            .join("test_data/gcs_authorized_user_credentials.json");
1741        let options = AHashMap::from([
1742            ("skip_signature".to_string(), "true".to_string()),
1743            (
1744                "application_credentials".to_string(),
1745                credentials_path.to_str().unwrap().to_string(),
1746            ),
1747        ]);
1748
1749        let (_, base_path, uri) =
1750            create_object_store_from_path("gs://test-bucket/path", Some(options)).unwrap();
1751
1752        assert_eq!(base_path, "path");
1753        assert_eq!(uri, "gs://test-bucket/path");
1754    }
1755
1756    #[rstest]
1757    #[cfg(feature = "cloud")]
1758    fn test_create_object_store_gcs_reads_application_credentials_option() {
1759        let directory = tempfile::tempdir().unwrap();
1760        let credentials_path = directory.path().join("missing-credentials.json");
1761        let credentials_path = credentials_path.to_str().unwrap();
1762        let options = AHashMap::from([(
1763            "application_credentials".to_string(),
1764            credentials_path.to_string(),
1765        )]);
1766
1767        let error = create_object_store_from_path("gs://test-bucket/path", Some(options))
1768            .err()
1769            .unwrap()
1770            .to_string();
1771
1772        assert!(
1773            error.contains(&format!(
1774                "Unable to open service account file from {credentials_path}"
1775            )),
1776            "{error}"
1777        );
1778    }
1779
1780    #[rstest]
1781    #[cfg(feature = "cloud")]
1782    #[case::s3("s3://test-bucket/path", "no_such_option", "S3")]
1783    #[case::gcs_project_id("gs://test-bucket/path", "project_id", "GCS")]
1784    #[case::azure("az://container/path", "no_such_option", "MicrosoftAzure")]
1785    #[case::abfs(
1786        "abfs://container@account.dfs.core.windows.net/path",
1787        "no_such_option",
1788        "MicrosoftAzure"
1789    )]
1790    #[case::http("https://example.com/path", "no_such_option", "HTTP")]
1791    fn test_create_object_store_rejects_unknown_storage_option(
1792        #[case] uri: &str,
1793        #[case] key: &str,
1794        #[case] store: &str,
1795    ) {
1796        let options = AHashMap::from([(key.to_string(), "value".to_string())]);
1797
1798        let error = create_object_store_from_path(uri, Some(options))
1799            .err()
1800            .unwrap()
1801            .to_string();
1802
1803        assert_eq!(
1804            error,
1805            format!("Configuration key: '{key}' is not valid for store '{store}'.")
1806        );
1807    }
1808
1809    #[rstest]
1810    #[cfg(feature = "cloud")]
1811    #[case::non_ascii("s3://test-bucket/préfix")]
1812    #[case::space("s3://test-bucket/my prefix")]
1813    #[case::tilde("s3://test-bucket/~prefix")]
1814    #[case::fragment("s3://test-bucket/pre#fix")]
1815    #[case::percent_encoded("s3://test-bucket/pr%C3%A9fix")]
1816    #[case::empty_segment("s3://test-bucket/base//path")]
1817    #[case::gcs("gs://test-bucket/préfix")]
1818    #[case::azure("az://container/préfix")]
1819    #[case::abfs("abfs://container@account.dfs.core.windows.net/préfix")]
1820    #[case::http("https://example.com/préfix")]
1821    fn test_create_object_store_rejects_base_path_that_paths_change(#[case] uri: &str) {
1822        let error = create_object_store_from_path(uri, None)
1823            .err()
1824            .unwrap()
1825            .to_string();
1826
1827        assert_eq!(
1828            error,
1829            format!(
1830                "Storage URI '{uri}' has a base path that URL parsing or object-store path \
1831                 encoding changes; use a base path without spaces, non-ASCII, or reserved \
1832                 characters"
1833            )
1834        );
1835    }
1836
1837    #[rstest]
1838    #[cfg(feature = "cloud")]
1839    #[case::bucket_root("s3://test-bucket", "")]
1840    #[case::trailing_slash("s3://test-bucket/nautilus-data/", "nautilus-data/")]
1841    #[case::unreserved(
1842        "s3://test-bucket/team_a/v2.catalog/year=2026",
1843        "team_a/v2.catalog/year=2026"
1844    )]
1845    fn test_create_object_store_accepts_ascii_base_path(
1846        #[case] uri: &str,
1847        #[case] expected_base_path: &str,
1848    ) {
1849        let (_, base_path, original_uri) = create_object_store_from_path(uri, None).unwrap();
1850
1851        assert_eq!(base_path, expected_base_path);
1852        assert_eq!(original_uri, uri);
1853    }
1854
1855    #[rstest]
1856    #[case::native("aws_access_key_id", "aws_secret_access_key")]
1857    #[case::short("access_key_id", "secret_access_key")]
1858    #[case::legacy("key", "secret")]
1859    #[tokio::test]
1860    #[cfg(feature = "cloud")]
1861    async fn test_create_object_store_s3_applies_credential_option_names(
1862        #[case] key_id_option: &str,
1863        #[case] secret_option: &str,
1864    ) {
1865        let (endpoint, server) = capture_request_head();
1866        let options = AHashMap::from([
1867            ("aws_endpoint".to_string(), endpoint),
1868            ("aws_allow_http".to_string(), "true".to_string()),
1869            (key_id_option.to_string(), "TESTKEYID".to_string()),
1870            (secret_option.to_string(), "test-secret".to_string()),
1871        ]);
1872        let (store, base_path, _) =
1873            create_object_store_from_path("s3://test-bucket/path", Some(options)).unwrap();
1874
1875        let result = store
1876            .head(&ObjectPath::from(format!("{base_path}/probe")))
1877            .await;
1878
1879        assert!(
1880            matches!(result, Err(object_store::Error::NotFound { .. })),
1881            "{result:?}"
1882        );
1883        let head = server.join().unwrap();
1884        assert_eq!(
1885            head.lines().next(),
1886            Some("HEAD /test-bucket/path/probe HTTP/1.1")
1887        );
1888        assert!(
1889            head.contains("authorization: AWS4-HMAC-SHA256 Credential=TESTKEYID/"),
1890            "{head}"
1891        );
1892    }
1893
1894    #[tokio::test]
1895    #[cfg(feature = "cloud")]
1896    async fn test_create_object_store_http_applies_client_option() {
1897        let (endpoint, server) = capture_request_head();
1898        let options = AHashMap::from([
1899            ("allow_http".to_string(), "true".to_string()),
1900            (
1901                "user_agent".to_string(),
1902                "nautilus-catalog-test".to_string(),
1903            ),
1904        ]);
1905        let (store, base_path, _) =
1906            create_object_store_from_path(&format!("{endpoint}/path"), Some(options)).unwrap();
1907
1908        let result = store
1909            .head(&ObjectPath::from(format!("{base_path}/probe")))
1910            .await;
1911
1912        assert!(
1913            matches!(result, Err(object_store::Error::NotFound { .. })),
1914            "{result:?}"
1915        );
1916        let head = server.join().unwrap();
1917        assert_eq!(head.lines().next(), Some("HEAD /path/probe HTTP/1.1"));
1918        assert!(
1919            head.contains("user-agent: nautilus-catalog-test\r\n"),
1920            "{head}"
1921        );
1922    }
1923
1924    #[tokio::test]
1925    #[cfg(feature = "cloud")]
1926    async fn test_create_object_store_azure_sends_sas_token_signature_encoded_once() {
1927        let (endpoint, server) = capture_request_head();
1928        let options = AHashMap::from([
1929            ("account_name".to_string(), "account".to_string()),
1930            ("azure_storage_endpoint".to_string(), endpoint),
1931            ("azure_allow_http".to_string(), "true".to_string()),
1932            (
1933                "sas_token".to_string(),
1934                "sv=2022-11-02&se=2030-01-01T00:00:00Z&sig=abc%2Bdef%3D".to_string(),
1935            ),
1936        ]);
1937        let (store, base_path, _) =
1938            create_object_store_from_path("az://container/path", Some(options)).unwrap();
1939
1940        let result = store
1941            .head(&ObjectPath::from(format!("{base_path}/probe")))
1942            .await;
1943
1944        assert!(
1945            matches!(result, Err(object_store::Error::NotFound { .. })),
1946            "{result:?}"
1947        );
1948        let head = server.join().unwrap();
1949        assert_eq!(
1950            head.lines().next(),
1951            Some(
1952                "HEAD /container/path/probe?sv=2022-11-02&se=2030-01-01T00%3A00%3A00Z\
1953                 &sig=abc%2Bdef%3D HTTP/1.1"
1954            )
1955        );
1956    }
1957
1958    #[rstest]
1959    #[cfg(feature = "cloud")]
1960    fn test_create_object_store_from_path_empty_options() {
1961        let result = create_object_store_from_path("s3://test-bucket/path", None);
1962        assert!(result.is_ok());
1963        let (_, base_path, uri) = result.unwrap();
1964        assert_eq!(base_path, "path");
1965        assert_eq!(uri, "s3://test-bucket/path");
1966    }
1967
1968    #[rstest]
1969    #[cfg(feature = "cloud")]
1970    fn test_remote_store_root_url_preserves_authority() {
1971        let https_root = remote_store_root_url("https://example.com:9000/base/path").unwrap();
1972        assert_eq!(
1973            https_root.as_str().trim_end_matches('/'),
1974            "https://example.com:9000"
1975        );
1976
1977        let abfs_root =
1978            remote_store_root_url("abfs://container@account.dfs.core.windows.net/base/path")
1979                .unwrap();
1980        assert_eq!(
1981            abfs_root.as_str().trim_end_matches('/'),
1982            "abfs://container@account.dfs.core.windows.net"
1983        );
1984
1985        let full_uri = remote_full_uri(
1986            "https://example.com:9000/base/path",
1987            "base/path/data/%5E/file.parquet",
1988        )
1989        .unwrap();
1990        assert_eq!(
1991            full_uri,
1992            "https://example.com:9000/base/path/data/%5E/file.parquet"
1993        );
1994
1995        let location = create_object_store_location_from_path("s3://test-bucket/path", None)
1996            .expect("S3 location should be created");
1997        assert_eq!(location.base_path, "path");
1998        assert_eq!(
1999            remote_store_root_url(&location.original_uri)
2000                .expect("S3 should be remote")
2001                .as_str()
2002                .trim_end_matches('/'),
2003            "s3://test-bucket"
2004        );
2005    }
2006
2007    #[cfg(feature = "cloud")]
2008    fn capture_request_head() -> (String, std::thread::JoinHandle<String>) {
2009        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2010        let endpoint = format!("http://{}", listener.local_addr().unwrap());
2011        let handle = std::thread::spawn(move || {
2012            let (mut stream, _) = listener.accept().unwrap();
2013            let mut reader = std::io::BufReader::new(stream.try_clone().unwrap());
2014            let mut head = String::new();
2015            loop {
2016                let mut line = String::new();
2017                reader.read_line(&mut line).unwrap();
2018                if line.trim_end().is_empty() {
2019                    break;
2020                }
2021                head.push_str(&line);
2022            }
2023            stream
2024                .write_all(
2025                    b"HTTP/1.1 404 Not Found\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
2026                )
2027                .unwrap();
2028            head
2029        });
2030        (endpoint, handle)
2031    }
2032}