1#![expect(
19 clippy::missing_errors_doc,
20 reason = "query methods forward DataFusion errors"
21)]
22
23use nautilus_model::instruments::NautilusInstrumentType;
24use nautilus_serialization::arrow::{
25 catalog_identifier_from_metadata, instrument::decode_instrument_any_batch,
26 record_batch_with_identifier_column,
27};
28
29use super::{
30 ArrowSchemaProvider, Bar, CustomDataDecoder, Data, DecodeDataFromRecordBatch,
31 DecodeTypedFromRecordBatch, FundingRateUpdate, HasCatalogDataType, HasTsInit, HashMap,
32 INSTRUMENT_PATH_PREFIXES, InstrumentAny, InstrumentClose, NautilusDataType, OptionGreeks,
33 OrderBookDelta, OrderBookDepth, ParquetDataCatalog, Path, QuoteTick, RecordBatch, TradeTick,
34 UnixNanos, build_query, catalog_record_batch_to_display, datafusion,
35 decode_object_store_segment, extract_bar_type_instrument_id, extract_identifier_from_path,
36 extract_sql_safe_filename, filter_instruments_for_request_range, instrument_path_prefix,
37 is_monotonically_increasing_by_init, make_object_store_path, make_sql_safe_identifier,
38 parquet_data_path_prefix, parse_filename_timestamps, query_intersects_filename,
39 read_parquet_from_object_store, read_parquet_schema_from_object_store,
40 session::{MergedPages, TypedPages, decode_typed_pages},
41 urisafe_instrument_id,
42};
43use crate::{
44 catalog::types::{
45 CatalogDataType, parquet_catalog_data_type_path_prefixes,
46 parquet_catalog_data_type_table_stem,
47 },
48 common::arrow::{empty_display_batch_with_identifier, validate_catalog_schema},
49};
50
51impl ParquetDataCatalog {
52 pub fn query_instruments(
95 &self,
96 instrument_ids: Option<&[String]>,
97 ) -> anyhow::Result<Vec<InstrumentAny>> {
98 self.query_instruments_filtered(instrument_ids, None, None)
99 }
100
101 pub fn query_instruments_filtered(
107 &self,
108 instrument_ids: Option<&[String]>,
109 start: Option<UnixNanos>,
110 end: Option<UnixNanos>,
111 ) -> anyhow::Result<Vec<InstrumentAny>> {
112 let instrument_files = self.discover_instrument_files(instrument_ids, end, None)?;
113 self.decode_instrument_files(instrument_files, start, end)
114 }
115
116 pub fn query_instruments_filtered_with_where(
121 &mut self,
122 instrument_ids: Option<&[String]>,
123 start: Option<UnixNanos>,
124 end: Option<UnixNanos>,
125 where_clause: Option<&str>,
126 ) -> anyhow::Result<Vec<InstrumentAny>> {
127 self.query_instruments_filtered_with_where_and_type(
128 instrument_ids,
129 start,
130 end,
131 where_clause,
132 None,
133 )
134 }
135
136 pub fn query_instruments_filtered_with_where_and_type(
141 &mut self,
142 instrument_ids: Option<&[String]>,
143 start: Option<UnixNanos>,
144 end: Option<UnixNanos>,
145 where_clause: Option<&str>,
146 instrument_type: Option<&NautilusInstrumentType>,
147 ) -> anyhow::Result<Vec<InstrumentAny>> {
148 let Some(where_clause) = where_clause else {
149 let instrument_files =
150 self.discover_instrument_files(instrument_ids, end, instrument_type)?;
151 return self.decode_instrument_files(instrument_files, start, end);
152 };
153
154 self.clear_session_tables();
155 self.register_remote_object_store()?;
156
157 let mut all_instruments = Vec::new();
158 let instrument_files =
159 self.discover_instrument_files(instrument_ids, end, instrument_type)?;
160
161 for (index, file_path) in instrument_files.into_iter().enumerate() {
162 let object_path = self.to_object_path_parsed(&file_path)?;
163
164 let (_, builder_schema) = self.execute_async(|| async {
165 read_parquet_from_object_store(self.object_store.clone(), &object_path).await
166 })?;
167
168 validate_catalog_schema(&builder_schema)?;
169 let metadata = builder_schema.metadata().clone();
170 let target_schema = InstrumentAny::get_schema(Some(metadata.clone()));
171
172 let table_name = format!(
173 "instruments_{}_{}",
174 index,
175 extract_sql_safe_filename(&file_path)
176 );
177
178 let query = build_query(&table_name, None, end, Some(where_clause));
180 let resolved_path = self.resolve_path_for_datafusion(&file_path);
181 let batches = self.session.collect_parquet_files_batches(
182 &table_name,
183 vec![resolved_path],
184 Some(&query),
185 )?;
186
187 for batch in batches {
188 let batch = datafusion::cast_record_batch_to_schema(&batch, &target_schema)?;
189 all_instruments.extend(decode_instrument_any_batch(&metadata, &batch)?);
190 }
191 }
192
193 Ok(filter_instruments_for_request_range(
194 all_instruments,
195 start,
196 end,
197 ))
198 }
199
200 fn discover_instrument_files(
203 &self,
204 instrument_ids: Option<&[String]>,
205 end: Option<UnixNanos>,
206 instrument_type: Option<&NautilusInstrumentType>,
207 ) -> anyhow::Result<Vec<String>> {
208 let base_dir = make_object_store_path(&self.base_path, ["data"]);
209 let end_u64 = end.map(|ts| ts.as_u64());
210 let list_result = self.list_objects(&base_dir)?;
211
212 let mut instrument_files = Vec::new();
213
214 for object in list_result {
215 let path_str = object.location.to_string();
216 if !path_str.ends_with(".parquet") {
217 continue;
218 }
219
220 let directory = path_str
221 .rsplit_once('/')
222 .map_or(path_str.as_str(), |(directory, _)| directory);
223
224 let (Some(type_dir), Some(identifier)) =
225 self.extract_data_cls_and_identifier_from_path(directory)?
226 else {
227 continue;
228 };
229
230 let type_dir = decode_object_store_segment(&type_dir);
231 if !is_parquet_instrument_type_prefix(&type_dir)
232 || instrument_type.is_some_and(|value| instrument_path_prefix(value) != type_dir)
233 {
234 continue;
235 }
236
237 let instrument_id_dir = decode_object_store_segment(&identifier);
238
239 if let Some(ids) = instrument_ids
240 && !ids
241 .iter()
242 .map(|id| urisafe_instrument_id(id))
243 .any(|x| x.as_str() == urisafe_instrument_id(&instrument_id_dir))
244 {
245 continue;
246 }
247
248 let include_file = if path_str.ends_with("/instrument.parquet") {
249 true
250 } else if let Some((file_start, _)) = parse_filename_timestamps(&path_str) {
251 end_u64.is_none_or(|end| file_start <= end)
252 } else {
253 log::warn!(
256 "Including instrument file with unparsable interval filename: {path_str}"
257 );
258 true
259 };
260
261 if include_file {
262 instrument_files.push(path_str);
263 }
264 }
265
266 instrument_files.sort();
267 Ok(instrument_files)
268 }
269
270 fn decode_instrument_files(
271 &self,
272 instrument_files: Vec<String>,
273 start: Option<UnixNanos>,
274 end: Option<UnixNanos>,
275 ) -> anyhow::Result<Vec<InstrumentAny>> {
276 let mut instruments = Vec::new();
277
278 for file_path in instrument_files {
279 let object_path = self.to_object_path_parsed(&file_path)?;
280
281 let (batches, builder_schema) = self.execute_async(|| async {
282 read_parquet_from_object_store(self.object_store.clone(), &object_path).await
283 })?;
284
285 validate_catalog_schema(&builder_schema)?;
286 let metadata = builder_schema.metadata().clone();
287 let target_schema = InstrumentAny::get_schema(Some(metadata.clone()));
288
289 for batch in batches {
290 let batch = datafusion::cast_record_batch_to_schema(&batch, &target_schema)?;
291 instruments.extend(decode_instrument_any_batch(&metadata, &batch)?);
292 }
293 }
294
295 Ok(filter_instruments_for_request_range(
296 instruments,
297 start,
298 end,
299 ))
300 }
301
302 pub fn query<T>(
412 &mut self,
413 identifiers: Option<Vec<String>>,
414 start: Option<UnixNanos>,
415 end: Option<UnixNanos>,
416 where_clause: Option<&str>,
417 files: Option<Vec<String>>,
418 optimize_file_loading: bool,
419 ) -> anyhow::Result<Vec<T>>
420 where
421 T: DecodeTypedFromRecordBatch + HasCatalogDataType + HasTsInit,
422 {
423 self.clear_session_tables();
424
425 self.register_remote_object_store()?;
426
427 let data_type = T::catalog_data_type();
428 let path_prefix = parquet_data_path_prefix(&data_type);
429
430 let files_list = if let Some(files) = files {
431 files
432 } else {
433 self.query_files(
434 &CatalogDataType::Data(data_type.clone()),
435 identifiers,
436 start,
437 end,
438 )?
439 };
440
441 let table_prefix = make_sql_safe_identifier(path_prefix.as_ref());
442 let tables =
443 self.resolve_tables_for_datafusion(&table_prefix, &files_list, optimize_file_loading)?;
444 let mut all_records = Vec::new();
445
446 for table in tables {
447 let query = build_query(&table.name, start, end, where_clause);
448 let batches = self.session.collect_parquet_files_batches(
449 &table.name,
450 vec![table.path],
451 Some(&query),
452 )?;
453
454 all_records.extend(self.convert_record_batches_to_typed::<T>(batches)?);
455 }
456
457 if !is_monotonically_increasing_by_init(&all_records) {
458 all_records.sort_by_key(HasTsInit::ts_init);
459 }
460
461 Ok(all_records)
462 }
463
464 pub(super) fn query_typed_pages<T>(
465 &mut self,
466 identifiers: Option<Vec<String>>,
467 start: Option<UnixNanos>,
468 end: Option<UnixNanos>,
469 where_clause: Option<&str>,
470 files: Option<Vec<String>>,
471 optimize_file_loading: bool,
472 ) -> anyhow::Result<TypedPages<T>>
473 where
474 T: DecodeTypedFromRecordBatch + HasCatalogDataType + HasTsInit + Send + 'static,
475 {
476 self.clear_session_tables();
477 self.register_remote_object_store()?;
478 let data_type = T::catalog_data_type();
479
480 let files = match files {
481 Some(files) => files,
482 None => self.query_files(&CatalogDataType::Data(data_type), identifiers, start, end)?,
483 };
484
485 let tables =
486 self.resolve_tables_for_datafusion("parquet", &files, optimize_file_loading)?;
487 let mut sources = Vec::with_capacity(tables.len());
488 for table in tables {
489 let sql = build_query(&table.name, start, end, where_clause);
490 let stream = self.session.parquet_files_batch_stream(
491 &table.name,
492 vec![table.path],
493 Some(&sql),
494 )?;
495 let pages = decode_typed_pages::<T>(stream);
496 sources.push(
497 Box::new(datafusion::BlockingBatchStream::from_stream_with_runtime(
498 pages,
499 &self.session.runtime,
500 )) as TypedPages<T>,
501 );
502 }
503
504 Ok(Box::new(MergedPages::new(sources, self.batch_size)))
505 }
506
507 pub fn query_record_batches(
513 &mut self,
514 data_type: &CatalogDataType,
515 identifier: Option<String>,
516 start: Option<UnixNanos>,
517 end: Option<UnixNanos>,
518 where_clause: Option<&str>,
519 optimize_file_loading: bool,
520 ) -> anyhow::Result<Vec<RecordBatch>> {
521 self.clear_session_tables();
522 self.register_remote_object_store()?;
523
524 let identifiers = identifier.map(|value| vec![value]);
525 let files_list = self.query_files(data_type, identifiers, start, end)?;
526 let mut record_batches = Vec::new();
527 let table_prefix =
528 make_sql_safe_identifier(&parquet_catalog_data_type_table_stem(data_type));
529
530 let tables =
531 self.resolve_tables_for_datafusion(&table_prefix, &files_list, optimize_file_loading)?;
532
533 for table in tables {
534 let query = build_query(&table.name, start, end, where_clause);
535 record_batches.extend(self.session.collect_parquet_files_batches(
536 &table.name,
537 vec![table.path],
538 Some(&query),
539 )?);
540 }
541
542 Ok(record_batches)
543 }
544
545 pub fn query_display_record_batches(
552 &mut self,
553 data_type: &NautilusDataType,
554 identifiers: Option<Vec<String>>,
555 start: Option<UnixNanos>,
556 end: Option<UnixNanos>,
557 where_clause: Option<&str>,
558 optimize_file_loading: bool,
559 ) -> anyhow::Result<Vec<RecordBatch>> {
560 self.clear_session_tables();
561 self.register_remote_object_store()?;
562
563 let data_path_prefix = parquet_data_path_prefix(data_type);
564 let files_list = self.query_files(
565 &CatalogDataType::Data(data_type.clone()),
566 identifiers,
567 start,
568 end,
569 )?;
570 let mut display_batches = Vec::new();
571 let table_prefix = make_sql_safe_identifier(data_path_prefix.as_ref());
572 let tables =
573 self.resolve_tables_for_datafusion(&table_prefix, &files_list, optimize_file_loading)?;
574
575 for table in tables {
576 let path_identifier = display_identifier(data_type, &table.directory);
577 let query = build_query(&table.name, start, end, where_clause);
578 let batches = self.session.collect_parquet_files_batches(
579 &table.name,
580 vec![table.path],
581 Some(&query),
582 )?;
583
584 for batch in batches {
585 let identifier =
586 display_batch_identifier(data_type, &batch, path_identifier.as_deref());
587 let batch = record_batch_with_identifier_column(batch, identifier.as_deref())?;
588 let metadata = batch.schema().metadata().clone();
589 display_batches.push(catalog_record_batch_to_display(
590 data_type, &metadata, &batch,
591 )?);
592 }
593 }
594
595 if display_batches.is_empty() {
596 display_batches.push(empty_display_batch_with_identifier(data_type)?);
597 }
598
599 Ok(display_batches)
600 }
601
602 pub fn query_identifiers(
604 &mut self,
605 data_type: &CatalogDataType,
606 identifiers: Option<Vec<String>>,
607 start: Option<UnixNanos>,
608 end: Option<UnixNanos>,
609 where_clause: Option<&str>,
610 _optimize_file_loading: bool,
611 ) -> anyhow::Result<Vec<String>> {
612 self.clear_session_tables();
613 self.register_remote_object_store()?;
614
615 let files_list = self.query_files(data_type, identifiers, start, end)?;
616 let table_prefix =
617 make_sql_safe_identifier(&parquet_catalog_data_type_table_stem(data_type));
618 let tables = self.resolve_tables_for_datafusion(&table_prefix, &files_list, true)?;
619 let mut identifiers = Vec::new();
620
621 for table in tables {
622 let identifier = dir_identifier(&table.directory);
623 let query = format!(
624 "{} LIMIT 1",
625 build_query(&table.name, start, end, where_clause)
626 );
627 let batches = self.session.collect_parquet_files_batches(
628 &table.name,
629 vec![table.path],
630 Some(&query),
631 )?;
632
633 if batches.iter().any(|batch| batch.num_rows() != 0) {
634 identifiers.push(decode_object_store_segment(&identifier));
635 }
636 }
637
638 identifiers.sort();
639 identifiers.dedup();
640 Ok(identifiers)
641 }
642
643 fn resolve_tables_for_datafusion(
647 &mut self,
648 table_prefix: &str,
649 files: &[String],
650 optimize_file_loading: bool,
651 ) -> anyhow::Result<Vec<DataFusionTable>> {
652 let table_name = |index: usize| format!("{table_prefix}_{index}");
653 let mut tables = Vec::new();
654
655 if !optimize_file_loading {
656 for file in files {
657 tables.push(self.file_table(table_name(tables.len()), file)?);
658 }
659 return Ok(tables);
660 }
661
662 for directory in parent_directories(files) {
664 let name = table_name(tables.len());
665 let path = self.resolve_directory_for_datafusion(&directory);
666
667 if !self.has_pending_consolidation(&directory)?
668 && self
669 .session
670 .try_register_parquet_files_table(&name, vec![path.clone()])?
671 {
672 tables.push(DataFusionTable {
673 name,
674 directory,
675 path,
676 });
677 continue;
678 }
679
680 for file in files
681 .iter()
682 .filter(|file| parent_directory(file).as_ref() == Some(&directory))
683 {
684 tables.push(self.file_table(table_name(tables.len()), file)?);
685 }
686 }
687
688 Ok(tables)
689 }
690
691 fn file_table(&self, name: String, file: &str) -> anyhow::Result<DataFusionTable> {
692 let directory = parent_directory(file)
693 .ok_or_else(|| anyhow::anyhow!("Cannot extract directory from '{file}'"))?;
694 let path = self.resolve_path_for_datafusion(file);
695
696 Ok(DataFusionTable {
697 name,
698 directory,
699 path,
700 })
701 }
702
703 #[expect(clippy::too_many_arguments)]
729 pub fn query_custom_data_dynamic(
730 &mut self,
731 type_name: &str,
732 identifiers: Option<&[String]>,
733 start: Option<UnixNanos>,
734 end: Option<UnixNanos>,
735 where_clause: Option<&str>,
736 files: Option<Vec<String>>,
737 _optimize_file_loading: bool,
738 ) -> anyhow::Result<Vec<Data>> {
739 self.clear_session_tables();
740
741 self.register_remote_object_store()?;
742
743 let files = if let Some(f) = files {
744 f.into_iter()
745 .map(|p| self.to_object_path_parsed(&p).map(|op| op.to_string()))
746 .collect::<anyhow::Result<Vec<_>>>()?
747 } else {
748 self.list_parquet_files_with_criteria(
749 &CatalogDataType::Data(NautilusDataType::Custom {
750 type_name: type_name.to_string(),
751 }),
752 identifiers,
753 start,
754 end,
755 )?
756 };
757
758 if files.is_empty() {
759 return Ok(Vec::new());
760 }
761
762 let mut lookup_metadata = HashMap::new();
766 lookup_metadata.insert("type_name".to_string(), type_name.to_string());
767 let registered_schema = CustomDataDecoder::get_schema(Some(lookup_metadata.clone()));
768 registered_schema.field_with_name("ts_init").map_err(|_| {
769 anyhow::anyhow!(
770 "custom data type '{type_name}' is not registered with an Arrow schema containing ts_init; \
771 call ensure_custom_data_registered::<T>() before querying"
772 )
773 })?;
774
775 let mut all_data = Vec::new();
776
777 for (index, file) in files.into_iter().enumerate() {
778 let object_path = self.to_object_path_parsed(&file)?;
779
780 let mut decode_metadata = self.execute_async(|| async {
781 let schema =
782 read_parquet_schema_from_object_store(self.object_store.clone(), &object_path)
783 .await?;
784 validate_catalog_schema(&schema)?;
785 Ok::<HashMap<String, String>, anyhow::Error>(schema.metadata().clone())
786 })?;
787
788 decode_metadata.extend(lookup_metadata.clone());
789 let identifier = extract_identifier_from_path(&file)
790 .ok_or_else(|| anyhow::anyhow!("Cannot extract identifier from path '{file}'"))?;
791 let safe_type_name = make_sql_safe_identifier(type_name);
792 let safe_sql_identifier = make_sql_safe_identifier(identifier);
793 let safe_filename = extract_sql_safe_filename(&file);
794 let table_name =
795 format!("custom_{safe_type_name}_{safe_sql_identifier}_{safe_filename}_{index}");
796 let resolved_path = self.resolve_path_for_datafusion(&file);
797 let sql_query = build_query(&table_name, start, end, where_clause);
798
799 let batches = self.session.collect_parquet_files_batches(
803 &table_name,
804 vec![resolved_path],
805 Some(&sql_query),
806 )?;
807
808 for batch in batches {
809 all_data.extend(CustomDataDecoder::decode_data_batch(
810 &decode_metadata,
811 batch,
812 )?);
813 }
814 }
815
816 all_data.sort_by_key(HasTsInit::ts_init);
817 Ok(all_data)
818 }
819
820 pub fn query_files(
875 &self,
876 data_type: &CatalogDataType,
877 identifiers: Option<Vec<String>>,
878 start: Option<UnixNanos>,
879 end: Option<UnixNanos>,
880 ) -> anyhow::Result<Vec<String>> {
881 let identifiers = identifiers.map(Vec::into_boxed_slice);
883
884 let mut files = Vec::new();
885
886 for prefix in parquet_catalog_data_type_path_prefixes(data_type) {
887 files.extend(self.query_prefix_files(
888 prefix.as_ref(),
889 identifiers.as_deref(),
890 start,
891 end,
892 )?);
893 }
894
895 files.sort();
896 files.dedup();
897
898 Ok(files)
899 }
900
901 fn query_prefix_files(
902 &self,
903 data_cls: &str,
904 identifiers: Option<&[String]>,
905 start: Option<UnixNanos>,
906 end: Option<UnixNanos>,
907 ) -> anyhow::Result<Vec<String>> {
908 let mut files = Vec::new();
909
910 let start_u64 = start.map(|s| s.as_u64());
911 let end_u64 = end.map(|e| e.as_u64());
912
913 let base_dir = self.make_path(data_cls, None)?;
914
915 let list_result = self.list_objects(&base_dir)?;
917
918 let mut file_paths: Vec<String> = list_result
919 .into_iter()
920 .filter_map(|object| {
921 let path_str = object.location.to_string();
922 if path_str.ends_with(".parquet") {
923 Some(path_str)
924 } else {
925 None
926 }
927 })
928 .collect();
929
930 if let Some(identifiers) = identifiers {
931 file_paths =
932 filter_identifier_files(file_paths, identifiers, is_parquet_bar_prefix(data_cls));
933 }
934
935 file_paths.retain(|file_path| query_intersects_filename(file_path, start_u64, end_u64));
937
938 for file_path in file_paths {
939 files.push(self.path_for_query_list(&file_path));
940 }
941
942 Ok(files)
943 }
944
945 pub fn quote_ticks(
947 &mut self,
948 instrument_ids: Option<Vec<String>>,
949 start: Option<UnixNanos>,
950 end: Option<UnixNanos>,
951 ) -> anyhow::Result<Vec<QuoteTick>> {
952 self.query::<QuoteTick>(instrument_ids, start, end, None, None, true)
953 }
954
955 pub fn trade_ticks(
957 &mut self,
958 instrument_ids: Option<Vec<String>>,
959 start: Option<UnixNanos>,
960 end: Option<UnixNanos>,
961 ) -> anyhow::Result<Vec<TradeTick>> {
962 self.query::<TradeTick>(instrument_ids, start, end, None, None, true)
963 }
964
965 pub fn bars(
967 &mut self,
968 instrument_ids: Option<Vec<String>>,
969 start: Option<UnixNanos>,
970 end: Option<UnixNanos>,
971 ) -> anyhow::Result<Vec<Bar>> {
972 self.query::<Bar>(instrument_ids, start, end, None, None, true)
973 }
974
975 pub fn order_book_deltas(
977 &mut self,
978 instrument_ids: Option<Vec<String>>,
979 start: Option<UnixNanos>,
980 end: Option<UnixNanos>,
981 ) -> anyhow::Result<Vec<OrderBookDelta>> {
982 self.query::<OrderBookDelta>(instrument_ids, start, end, None, None, true)
983 }
984
985 pub fn order_book_depths(
987 &mut self,
988 instrument_ids: Option<Vec<String>>,
989 start: Option<UnixNanos>,
990 end: Option<UnixNanos>,
991 ) -> anyhow::Result<Vec<OrderBookDepth>> {
992 self.query::<OrderBookDepth>(instrument_ids, start, end, None, None, true)
993 }
994
995 pub fn funding_rates(
997 &mut self,
998 instrument_ids: Option<Vec<String>>,
999 start: Option<UnixNanos>,
1000 end: Option<UnixNanos>,
1001 ) -> anyhow::Result<Vec<FundingRateUpdate>> {
1002 self.query::<FundingRateUpdate>(instrument_ids, start, end, None, None, true)
1003 }
1004
1005 pub fn instrument_closes(
1007 &mut self,
1008 instrument_ids: Option<Vec<String>>,
1009 start: Option<UnixNanos>,
1010 end: Option<UnixNanos>,
1011 ) -> anyhow::Result<Vec<InstrumentClose>> {
1012 self.query::<InstrumentClose>(instrument_ids, start, end, None, None, true)
1013 }
1014
1015 pub fn option_greeks(
1017 &mut self,
1018 instrument_ids: Option<Vec<String>>,
1019 start: Option<UnixNanos>,
1020 end: Option<UnixNanos>,
1021 ) -> anyhow::Result<Vec<OptionGreeks>> {
1022 self.query::<OptionGreeks>(instrument_ids, start, end, None, None, true)
1023 }
1024
1025 pub fn instruments(
1027 &self,
1028 instrument_ids: Option<&[String]>,
1029 start: Option<UnixNanos>,
1030 end: Option<UnixNanos>,
1031 ) -> anyhow::Result<Vec<InstrumentAny>> {
1032 self.query_instruments_filtered(instrument_ids, start, end)
1033 }
1034
1035 pub fn get_file_list_from_data_cls(
1075 &self,
1076 data_type: &CatalogDataType,
1077 ) -> anyhow::Result<Vec<String>> {
1078 let mut file_paths = Vec::new();
1079
1080 for data_cls in parquet_catalog_data_type_path_prefixes(data_type) {
1081 let directory = self.make_path(data_cls.as_ref(), None)?;
1082 file_paths.extend(self.list_parquet_files(&directory)?);
1083 }
1084
1085 Ok(file_paths)
1086 }
1087
1088 pub fn filter_files(
1138 &self,
1139 data_type: &CatalogDataType,
1140 file_paths: Vec<String>,
1141 identifiers: Option<Vec<String>>,
1142 start: Option<UnixNanos>,
1143 end: Option<UnixNanos>,
1144 ) -> anyhow::Result<Vec<String>> {
1145 let has_bar_prefix = parquet_catalog_data_type_path_prefixes(data_type)
1146 .iter()
1147 .any(|data_cls| is_parquet_bar_prefix(data_cls.as_ref()));
1148 let mut filtered_paths = file_paths;
1149
1150 if let Some(identifiers) = identifiers {
1151 filtered_paths = filter_identifier_files(filtered_paths, &identifiers, has_bar_prefix);
1152 }
1153
1154 let start_u64 = start.map(|s| s.as_u64());
1156 let end_u64 = end.map(|e| e.as_u64());
1157 filtered_paths.retain(|file_path| query_intersects_filename(file_path, start_u64, end_u64));
1158
1159 Ok(filtered_paths)
1160 }
1161}
1162
1163fn is_parquet_instrument_type_prefix(prefix: &str) -> bool {
1164 INSTRUMENT_PATH_PREFIXES.contains(&prefix)
1165}
1166
1167pub(super) fn is_parquet_bar_prefix(data_cls: &str) -> bool {
1168 data_cls == parquet_data_path_prefix(&NautilusDataType::Bar).as_ref()
1169}
1170
1171pub(super) fn filter_identifier_files(
1175 file_paths: Vec<String>,
1176 identifiers: &[String],
1177 bars: bool,
1178) -> Vec<String> {
1179 let directories = file_paths
1180 .iter()
1181 .map(|file_path| identifier_directory(file_path))
1182 .collect::<Vec<_>>();
1183 let mut keep = vec![false; file_paths.len()];
1184
1185 for safe_id in identifiers.iter().map(|id| urisafe_instrument_id(id)) {
1186 let has_exact = directories
1187 .iter()
1188 .flatten()
1189 .any(|directory| *directory == safe_id);
1190 let partial = bars && !has_exact;
1191
1192 for (keep, directory) in keep.iter_mut().zip(&directories) {
1193 let Some(directory) = directory else {
1194 continue;
1195 };
1196
1197 *keep |= if partial {
1198 extract_bar_type_instrument_id(directory) == Some(safe_id.as_str())
1199 } else {
1200 *directory == safe_id
1201 };
1202 }
1203 }
1204
1205 file_paths
1206 .into_iter()
1207 .zip(keep)
1208 .filter_map(|(file_path, keep)| keep.then_some(file_path))
1209 .collect()
1210}
1211
1212fn identifier_directory(file_path: &str) -> Option<String> {
1213 let mut segments = file_path.rsplit('/');
1214 segments.next()?;
1215 segments.next().map(decode_object_store_segment)
1216}
1217
1218struct DataFusionTable {
1221 name: String,
1222 directory: String,
1223 path: String,
1224}
1225
1226fn parent_directories(files: &[String]) -> Vec<String> {
1229 let mut directories: Vec<String> = files
1230 .iter()
1231 .filter_map(|file_uri| parent_directory(file_uri))
1232 .collect();
1233
1234 directories.sort();
1235 directories.dedup();
1236 directories
1237}
1238
1239fn parent_directory(file_uri: &str) -> Option<String> {
1240 Path::new(file_uri)
1241 .parent()
1242 .map(|path| path.to_string_lossy().to_string())
1243}
1244
1245fn dir_identifier(directory: &str) -> String {
1247 directory
1248 .rsplit_once('/')
1249 .map_or(directory, |(_, name)| name)
1250 .to_string()
1251}
1252
1253fn display_identifier(data_type: &NautilusDataType, directory: &str) -> Option<String> {
1254 let identifier = dir_identifier(directory);
1255 let is_unpartitioned_custom = matches!(data_type, NautilusDataType::Custom { .. })
1256 && Path::new(directory)
1257 .ends_with(Path::new("data").join(parquet_data_path_prefix(data_type).as_ref()));
1258
1259 (!is_unpartitioned_custom).then(|| decode_object_store_segment(&identifier))
1260}
1261
1262fn display_batch_identifier(
1263 data_type: &NautilusDataType,
1264 batch: &RecordBatch,
1265 path_identifier: Option<&str>,
1266) -> Option<String> {
1267 if matches!(data_type, NautilusDataType::Custom { .. }) {
1268 path_identifier.map(str::to_string)
1269 } else {
1270 catalog_identifier_from_metadata(batch.schema().metadata())
1271 .or_else(|| path_identifier.map(str::to_string))
1272 }
1273}