nautilus_persistence/common/
paths.rs1use std::{fs, path::PathBuf};
29
30use nautilus_common::enums::Environment;
31use nautilus_model::data::NautilusDataType;
32use strum::IntoEnumIterator;
33
34use crate::catalog::types::{CatalogDataType, catalog_data_type_from_path};
35
36pub trait CatalogPathPrefix {
38 fn path_prefix() -> &'static str;
40}
41
42#[must_use]
48pub fn normalize_path_separators(path: &str) -> String {
49 path.replace('\\', "/")
50}
51
52#[must_use]
54pub fn extract_path_components(path_str: &str) -> Vec<String> {
55 normalize_path_separators(path_str)
56 .split('/')
57 .filter(|s| !s.is_empty())
58 .map(ToString::to_string)
59 .collect()
60}
61
62#[must_use]
64pub fn local_to_object_store_path(local_path: &std::path::Path) -> String {
65 normalize_path_separators(&local_path.to_string_lossy())
66}
67
68#[must_use]
72pub fn make_object_store_path<I, S>(base_path: &str, components: I) -> String
73where
74 I: IntoIterator<Item = S>,
75 S: AsRef<str>,
76{
77 let mut parts = Vec::new();
78 let normalized_base = normalize_path_separators(base_path)
79 .trim_end_matches('/')
80 .to_string();
81
82 if !normalized_base.is_empty() {
83 parts.push(normalized_base);
84 }
85
86 for component in components {
87 let normalized_component = normalize_path_separators(component.as_ref())
88 .trim_start_matches('/')
89 .trim_end_matches('/')
90 .to_string();
91
92 if !normalized_component.is_empty() {
93 parts.push(normalized_component);
94 }
95 }
96
97 parts.join("/")
98}
99
100#[must_use]
104pub fn urisafe_instrument_id(instrument_id: &str) -> String {
105 instrument_id.replace('/', "").replace('^', "_")
106}
107
108#[must_use]
113pub fn safe_directory_identifier(identifier: &str) -> String {
114 let normalized = normalize_path_separators(identifier).replace("//", "/");
115 let segments: Vec<&str> = normalized
116 .split('/')
117 .filter(|s| !s.is_empty() && *s != "..")
118 .collect();
119 segments.join("/")
120}
121
122#[must_use]
127pub fn extract_identifier_from_path(file_path: &str) -> Option<&str> {
128 let parent = file_path.rfind(['/', '\\']).map(|idx| &file_path[..idx])?;
129 let identifier = parent
130 .rfind(['/', '\\'])
131 .map_or(parent, |idx| &parent[idx + 1..]);
132 (!identifier.is_empty()).then_some(identifier)
133}
134
135#[must_use]
139pub fn make_sql_safe_identifier(identifier: &str) -> String {
140 urisafe_instrument_id(identifier)
141 .chars()
142 .map(|c| {
143 if c.is_ascii_alphanumeric() {
144 c.to_ascii_lowercase()
145 } else {
146 '_'
147 }
148 })
149 .collect()
150}
151
152pub fn normalize_path_to_uri(path: &str) -> anyhow::Result<String> {
176 if path.contains("://") {
177 Ok(path.to_string())
179 } else if is_absolute_path(path) {
180 Ok(path_to_file_uri(path))
181 } else {
182 let current_dir = std::env::current_dir().map_err(|e| {
184 anyhow::anyhow!("Failed to resolve current directory for relative path '{path}': {e}")
185 })?;
186
187 let absolute_path = current_dir.join(path);
188 Ok(path_to_file_uri(&absolute_path.to_string_lossy()))
189 }
190}
191
192pub(crate) fn local_writer_directory(path: &str) -> anyhow::Result<PathBuf> {
201 let uri = normalize_path_to_uri(path)?;
202 anyhow::ensure!(
203 uri.starts_with("file://"),
204 "Streaming writers append to local files, writer path must be local, was {path}"
205 );
206 Ok(PathBuf::from(file_uri_to_native_path(&uri)))
207}
208
209pub fn create_local_directory(path: &str) -> anyhow::Result<()> {
218 let uri = normalize_path_to_uri(path)?;
219
220 if uri.starts_with("file://") {
221 fs::create_dir_all(file_uri_to_native_path(&uri))?;
222 }
223
224 Ok(())
225}
226
227#[must_use]
229pub const fn environment_directory(environment: Environment) -> &'static str {
230 match environment {
231 Environment::Backtest => "backtest",
232 Environment::Sandbox => "sandbox",
233 Environment::Live => "live",
234 }
235}
236
237#[must_use]
239pub fn environment_from_directory(directory: &str) -> Option<Environment> {
240 Environment::iter().find(|environment| environment_directory(*environment) == directory)
241}
242
243#[must_use]
245fn is_absolute_path(path: &str) -> bool {
246 path.starts_with('/') || path.starts_with("\\\\") || is_windows_drive_path(path)
247}
248
249#[must_use]
251fn is_windows_drive_path(path: &str) -> bool {
252 path.len() >= 3
253 && path.chars().nth(1) == Some(':')
254 && matches!(path.chars().nth(2), Some('\\' | '/'))
255}
256
257#[must_use]
266pub(crate) fn file_protocol_uri(path: &str) -> String {
267 if is_windows_drive_path(path) {
268 path_to_file_uri(path)
269 } else if path.is_empty() || is_absolute_path(path) {
270 format!("file://{path}")
271 } else {
272 path.to_string()
273 }
274}
275
276#[must_use]
278pub(crate) fn path_to_file_uri(path: &str) -> String {
279 if path.starts_with('/') {
280 format!("file://{path}")
282 } else if path.len() >= 3 && path.chars().nth(1) == Some(':') {
283 let normalized = normalize_path_separators(path);
285 format!("file:///{normalized}")
286 } else if let Some(without_prefix) = path.strip_prefix("\\\\") {
287 let normalized = normalize_path_separators(without_prefix);
289 format!("file://{normalized}")
290 } else {
291 format!("file://{path}")
293 }
294}
295
296#[cfg(windows)]
299pub(crate) fn file_uri_to_native_path(uri: &str) -> String {
300 let without_scheme = uri
301 .strip_prefix("file://")
302 .or_else(|| uri.strip_prefix("file:"))
303 .unwrap_or(uri);
304 let without_leading = without_scheme.trim_start_matches('/');
306 without_leading.replace('/', "\\")
307}
308
309#[cfg(not(windows))]
311pub(crate) fn file_uri_to_native_path(uri: &str) -> String {
312 uri.strip_prefix("file://").unwrap_or(uri).to_string()
313}
314
315pub(crate) fn catalog_data_type_from_session_feather_path(
326 path: &str,
327 environment: Environment,
328 instance_id: &str,
329) -> anyhow::Result<CatalogDataType> {
330 let components = extract_path_components(path);
331 let type_index = session_type_index(&components, environment, instance_id, path)?;
332 if has_custom_data_prefix(&components, type_index) {
333 let type_name = components.get(type_index + 2).ok_or_else(|| {
334 anyhow::anyhow!(
335 "Cannot infer custom data type from Feather session path '{path}' for {}/{instance_id}",
336 environment_directory(environment)
337 )
338 })?;
339
340 return Ok(CatalogDataType::from(NautilusDataType::Custom {
341 type_name: type_name.clone(),
342 }));
343 }
344
345 let type_segment = components[type_index].as_str();
346 let file_name = components.last().map_or(type_segment, String::as_str);
347
348 let type_name = if type_segment.ends_with(".feather") {
349 file_name
350 .strip_suffix(".feather")
351 .and_then(|stem| stem.rsplit_once('_').map(|(type_name, _)| type_name))
352 .unwrap_or(type_segment)
353 } else {
354 type_segment
355 };
356
357 catalog_data_type_from_path(type_name)
358}
359
360pub(crate) fn identifier_from_session_feather_path(
364 path: &str,
365 environment: Environment,
366 instance_id: &str,
367) -> Option<String> {
368 let components = extract_path_components(path);
369 let type_index = session_type_index(&components, environment, instance_id, path).ok()?;
370 if has_custom_data_prefix(&components, type_index) {
371 let identifier_start = type_index + 3;
372 let file_index = components.len().checked_sub(1)?;
373 if identifier_start >= file_index {
374 return None;
375 }
376
377 return Some(components[identifier_start].clone());
378 }
379
380 if components[type_index] == "instruments" {
382 return None;
383 }
384
385 let identifier = components.get(type_index + 1)?;
386 let file_name = components.last()?;
387
388 if identifier.ends_with(".feather") {
389 return None;
390 }
391
392 (identifier != file_name).then(|| identifier.clone())
393}
394
395fn session_type_index(
396 components: &[String],
397 environment: Environment,
398 instance_id: &str,
399 path: &str,
400) -> anyhow::Result<usize> {
401 let directory = environment_directory(environment);
402 components
403 .windows(2)
404 .position(|window| window[0] == directory && window[1] == instance_id)
405 .and_then(|environment_index| environment_index.checked_add(2))
406 .filter(|type_index| *type_index < components.len())
407 .ok_or_else(|| {
408 anyhow::anyhow!(
409 "Cannot infer data type from Feather session path '{path}' for {directory}/{instance_id}"
410 )
411 })
412}
413
414fn has_custom_data_prefix(components: &[String], type_index: usize) -> bool {
415 components.get(type_index).is_some_and(|c| c == "data")
416 && components
417 .get(type_index + 1)
418 .is_some_and(|c| c == "custom")
419}
420
421#[cfg(test)]
422mod tests {
423 use nautilus_core::UUID4;
424 use rstest::rstest;
425 use tempfile::TempDir;
426
427 use super::*;
428
429 #[rstest]
430 fn create_local_directory_creates_local_path_and_skips_remote_uri() {
431 let directory = TempDir::new().unwrap();
432 let local = directory.path().join("catalog").join("nested");
433 let bucket = format!("nautilus-{}", UUID4::new());
434
435 create_local_directory(&local.to_string_lossy()).unwrap();
436 create_local_directory(&format!("s3://{bucket}/catalog")).unwrap();
437
438 assert!(local.is_dir());
439 assert!(!PathBuf::from("s3:").join(&bucket).exists());
441 }
442
443 #[rstest]
444 fn normalize_path_separators_converts_backslashes() {
445 assert_eq!(
446 normalize_path_separators(r"C:\catalog\backtest\run-1"),
447 "C:/catalog/backtest/run-1",
448 );
449 assert_eq!(
450 normalize_path_separators("C:/catalog/backtest/run-1"),
451 "C:/catalog/backtest/run-1",
452 );
453 assert_eq!(
454 normalize_path_separators(r"\\server\share\live\run-2"),
455 "//server/share/live/run-2",
456 );
457 }
458
459 #[rstest]
460 fn extract_path_components_handles_platform_separators() {
461 assert_eq!(
462 extract_path_components(r"C:\catalog\backtest\run-1"),
463 vec!["C:", "catalog", "backtest", "run-1"],
464 );
465 assert_eq!(
466 extract_path_components("/catalog/backtest/run-1/"),
467 vec!["catalog", "backtest", "run-1"],
468 );
469 assert!(extract_path_components("").is_empty());
470 }
471
472 #[rstest]
473 fn extract_identifier_from_path_handles_platform_separators() {
474 assert_eq!(
475 extract_identifier_from_path("data/quotes/EURUSD/file.parquet"),
476 Some("EURUSD"),
477 );
478 assert_eq!(
479 extract_identifier_from_path(r"data\quotes\EURUSD\file.parquet"),
480 Some("EURUSD"),
481 );
482 assert_eq!(
483 extract_identifier_from_path(r"C:\data\quotes\EURUSD\file.parquet"),
484 Some("EURUSD"),
485 );
486 assert_eq!(extract_identifier_from_path("file.parquet"), None);
487 assert_eq!(extract_identifier_from_path(""), None);
488 }
489
490 #[rstest]
491 fn safe_directory_identifier_blocks_windows_traversal() {
492 assert_eq!(safe_directory_identifier(r"..\\..\\etc"), "etc");
493 assert_eq!(safe_directory_identifier("../../etc"), "etc");
494 assert_eq!(safe_directory_identifier("run-1"), "run-1");
495 }
496
497 #[rstest]
498 fn test_normalize_path_to_uri() {
499 assert_eq!(
501 normalize_path_to_uri("/tmp/test").unwrap(),
502 "file:///tmp/test"
503 );
504
505 assert_eq!(
507 normalize_path_to_uri("C:\\tmp\\test").unwrap(),
508 "file:///C:/tmp/test"
509 );
510 assert_eq!(
511 normalize_path_to_uri("C:/tmp/test").unwrap(),
512 "file:///C:/tmp/test"
513 );
514 assert_eq!(
515 normalize_path_to_uri("D:\\data\\file.txt").unwrap(),
516 "file:///D:/data/file.txt"
517 );
518
519 assert_eq!(
521 normalize_path_to_uri("\\\\server\\share\\file").unwrap(),
522 "file://server/share/file"
523 );
524
525 assert_eq!(
527 normalize_path_to_uri("s3://bucket/path").unwrap(),
528 "s3://bucket/path"
529 );
530 assert_eq!(
531 normalize_path_to_uri("file:///tmp/test").unwrap(),
532 "file:///tmp/test"
533 );
534 assert_eq!(
535 normalize_path_to_uri("https://example.com/path").unwrap(),
536 "https://example.com/path"
537 );
538 }
539
540 #[rstest]
541 fn test_is_absolute_path() {
542 assert!(is_absolute_path("/tmp/test"));
544 assert!(is_absolute_path("/"));
545
546 assert!(is_absolute_path("C:\\tmp\\test"));
548 assert!(is_absolute_path("C:/tmp/test"));
549 assert!(is_absolute_path("D:\\"));
550 assert!(is_absolute_path("Z:/"));
551
552 assert!(is_absolute_path("\\\\server\\share"));
554 assert!(is_absolute_path("\\\\localhost\\c$"));
555
556 assert!(!is_absolute_path("tmp/test"));
558 assert!(!is_absolute_path("./test"));
559 assert!(!is_absolute_path("../test"));
560 assert!(!is_absolute_path("test.txt"));
561
562 assert!(!is_absolute_path(""));
564 assert!(!is_absolute_path("C"));
565 assert!(!is_absolute_path("C:"));
566 assert!(!is_absolute_path("\\"));
567 }
568
569 #[rstest]
570 fn file_protocol_uri_normalizes_windows_drive_path() {
571 assert_eq!(
572 file_protocol_uri(r"C:\data\catalog"),
573 "file:///C:/data/catalog",
574 );
575 assert_eq!(
576 file_protocol_uri("C:/data/catalog"),
577 "file:///C:/data/catalog"
578 );
579 assert_eq!(file_protocol_uri("/tmp/cat"), "file:///tmp/cat");
580 assert_eq!(
581 file_protocol_uri(r"\\server\share\catalog"),
582 r"file://\\server\share\catalog",
583 );
584 }
585
586 #[rstest]
587 fn test_path_to_file_uri() {
588 assert_eq!(path_to_file_uri("/tmp/test"), "file:///tmp/test");
590 assert_eq!(path_to_file_uri("/"), "file:///");
591
592 assert_eq!(path_to_file_uri("C:\\tmp\\test"), "file:///C:/tmp/test");
594 assert_eq!(path_to_file_uri("C:/tmp/test"), "file:///C:/tmp/test");
595 assert_eq!(path_to_file_uri("D:\\"), "file:///D:/");
596
597 assert_eq!(
599 path_to_file_uri("\\\\server\\share\\file"),
600 "file://server/share/file"
601 );
602 assert_eq!(
603 path_to_file_uri("\\\\localhost\\c$\\test"),
604 "file://localhost/c$/test"
605 );
606 }
607
608 #[rstest]
609 #[case(Environment::Backtest, "backtest")]
610 #[case(Environment::Sandbox, "sandbox")]
611 #[case(Environment::Live, "live")]
612 fn environment_directory_round_trips(
613 #[case] environment: Environment,
614 #[case] directory: &str,
615 ) {
616 assert_eq!(environment_directory(environment), directory);
617 assert_eq!(environment_from_directory(directory), Some(environment));
618 }
619
620 #[rstest]
621 #[case("BACKTEST")]
622 #[case("data")]
623 #[case("")]
624 fn environment_from_directory_rejects_other_folders(#[case] directory: &str) {
625 assert_eq!(environment_from_directory(directory), None);
626 }
627
628 #[rstest]
629 fn session_feather_paths_recover_type_and_identifier() {
630 let path = "backtest/run-1/quotes/EURUSD.SIM/0001.feather";
634 assert_eq!(
635 catalog_data_type_from_session_feather_path(path, Environment::Backtest, "run-1")
636 .unwrap(),
637 CatalogDataType::from(NautilusDataType::QuoteTick),
638 );
639 assert_eq!(
640 identifier_from_session_feather_path(path, Environment::Backtest, "run-1").as_deref(),
641 Some("EURUSD.SIM"),
642 );
643 assert_eq!(
644 catalog_data_type_from_session_feather_path(
645 "backtest/run-1/quotes_1000-1.feather",
646 Environment::Backtest,
647 "run-1",
648 )
649 .unwrap(),
650 CatalogDataType::from(NautilusDataType::QuoteTick),
651 );
652
653 let custom = "backtest/run-1/data/custom/MyType/inst/0001.feather";
654 assert_eq!(
655 catalog_data_type_from_session_feather_path(custom, Environment::Backtest, "run-1")
656 .unwrap(),
657 CatalogDataType::from(NautilusDataType::Custom {
658 type_name: "MyType".to_string(),
659 }),
660 );
661 assert_eq!(
662 identifier_from_session_feather_path(custom, Environment::Backtest, "run-1").as_deref(),
663 Some("inst"),
664 );
665 }
666
667 #[rstest]
668 #[case::prefixed_custom(
669 "catalog/backtest/run-1/data/custom/MyType/inst/0001.feather",
670 "custom/MyType",
671 Some("inst")
672 )]
673 #[case::custom_without_identifier(
674 "backtest/run-1/data/custom/MyType/0001.feather",
675 "custom/MyType",
676 None
677 )]
678 #[case::identifier_named_custom(
679 "backtest/run-1/bars/custom/0001.feather",
680 "bars",
681 Some("custom")
682 )]
683 #[case::type_without_identifier("backtest/run-1/quotes/0001.feather", "quotes", None)]
684 #[case::instrument_class_folder(
685 "backtest/run-1/instruments/currency_pair/instruments_1.feather",
686 "instruments",
687 None
688 )]
689 fn session_feather_paths_recover_type_and_identifier_from_layouts(
690 #[case] path: &str,
691 #[case] expected_type: &str,
692 #[case] expected_identifier: Option<&str>,
693 ) {
694 assert_eq!(
695 catalog_data_type_from_session_feather_path(path, Environment::Backtest, "run-1")
696 .unwrap(),
697 catalog_data_type_from_path(expected_type).unwrap(),
698 );
699 assert_eq!(
700 identifier_from_session_feather_path(path, Environment::Backtest, "run-1").as_deref(),
701 expected_identifier,
702 );
703 }
704
705 #[rstest]
706 #[case::missing_type_segment(
707 "backtest/run-1",
708 "Cannot infer data type from Feather session path 'backtest/run-1' for backtest/run-1"
709 )]
710 #[case::other_instance(
711 "backtest/run-2/quotes/0001.feather",
712 "Cannot infer data type from Feather session path 'backtest/run-2/quotes/0001.feather' for backtest/run-1"
713 )]
714 #[case::missing_custom_type(
715 "backtest/run-1/data/custom",
716 "Cannot infer custom data type from Feather session path 'backtest/run-1/data/custom' for backtest/run-1"
717 )]
718 fn session_feather_paths_reject_unrecognized_layouts(
719 #[case] path: &str,
720 #[case] expected: &str,
721 ) {
722 let error =
723 catalog_data_type_from_session_feather_path(path, Environment::Backtest, "run-1")
724 .unwrap_err();
725
726 assert_eq!(error.to_string(), expected);
727 assert_eq!(
728 identifier_from_session_feather_path(path, Environment::Backtest, "run-1"),
729 None
730 );
731 }
732
733 #[rstest]
734 fn session_feather_paths_recover_type_and_identifier_with_backslashes() {
735 let path = r"backtest\run-1\quotes\EURUSD.SIM\0001.feather";
736 assert_eq!(
737 catalog_data_type_from_session_feather_path(path, Environment::Backtest, "run-1")
738 .unwrap(),
739 CatalogDataType::from(NautilusDataType::QuoteTick),
740 );
741 assert_eq!(
742 identifier_from_session_feather_path(path, Environment::Backtest, "run-1").as_deref(),
743 Some("EURUSD.SIM"),
744 );
745 }
746}