1use std::{
17 collections::BTreeMap,
18 ffi::OsString,
19 fmt::Display,
20 fs::{File, remove_file},
21 io::{BufReader, BufWriter, Read, Write},
22 path::{Path, PathBuf},
23 sync::OnceLock,
24 thread::sleep,
25 time::Duration,
26};
27
28use anyhow::Context;
29use aws_lc_rs::digest;
30use nautilus_core::hex;
31use nautilus_network::{
32 http::{HttpClient, HttpResponseStream},
33 retry::{RetryConfig, RetryManager},
34};
35use parking_lot::Mutex;
36use rand::{RngExt, rng};
37use serde_json::Value;
38
39static LARGE_CHECKSUMS_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
40
41fn lock_large_checksums() -> parking_lot::MutexGuard<'static, ()> {
42 LARGE_CHECKSUMS_LOCK.get_or_init(|| Mutex::new(())).lock()
43}
44
45#[derive(Debug)]
46enum DownloadError {
47 Retryable(String),
48 NonRetryable(String),
49}
50
51impl Display for DownloadError {
52 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
53 match self {
54 Self::Retryable(msg) => write!(f, "Retryable error: {msg}"),
55 Self::NonRetryable(msg) => write!(f, "Non-retryable error: {msg}"),
56 }
57 }
58}
59
60impl std::error::Error for DownloadError {}
61
62pub fn prepare_test_data() -> anyhow::Result<()> {
70 let checksums = crate::common::get_test_data_large_checksums_filepath();
71 let manifest: BTreeMap<String, String> =
72 serde_json::from_reader(BufReader::new(File::open(&checksums)?))?;
73 let directory = nautilus_core::paths::get_test_data_path().join("large");
74
75 for filename in manifest.keys() {
76 let filepath = directory.join(filename);
77 let url = format!("https://test-data.nautechsystems.io/large/{filename}");
78 prepare_test_data_file(&filepath, &url, &checksums)?;
79 }
80 Ok(())
81}
82
83fn prepare_test_data_file(filepath: &Path, url: &str, checksums: &Path) -> anyhow::Result<()> {
84 if filepath.exists() {
85 if verify_sha256_checksum(filepath, checksums)? {
86 return Ok(());
87 }
88 remove_file(filepath)?;
89 }
90
91 download_file(filepath, url, 30, None)?;
92 if !verify_sha256_checksum(filepath, checksums)? {
93 remove_file(filepath)?;
94 anyhow::bail!("Checksum mismatch for {}", filepath.display());
95 }
96 Ok(())
97}
98
99pub fn ensure_file_exists_or_download_http(
120 filepath: &Path,
121 url: &str,
122 checksums: Option<&Path>,
123 timeout_secs: Option<u64>,
124) -> anyhow::Result<()> {
125 ensure_file_exists_or_download_http_with_config(
126 filepath,
127 url,
128 checksums,
129 timeout_secs.unwrap_or(30),
130 None,
131 None,
132 )
133}
134
135pub fn ensure_file_exists_or_download_http_with_timeout(
144 filepath: &Path,
145 url: &str,
146 checksums: Option<&Path>,
147 timeout_secs: u64,
148) -> anyhow::Result<()> {
149 ensure_file_exists_or_download_http_with_config(
150 filepath,
151 url,
152 checksums,
153 timeout_secs,
154 None,
155 None,
156 )
157}
158
159pub fn ensure_file_exists_or_download_http_with_config(
179 filepath: &Path,
180 url: &str,
181 checksums: Option<&Path>,
182 timeout_secs: u64,
183 retry_config: Option<RetryConfig>,
184 initial_jitter_ms: Option<u64>,
185) -> anyhow::Result<()> {
186 if filepath.exists() {
190 println!("File already exists (local/cached): {}", filepath.display());
191
192 if let Some(checksums_file) = checksums {
193 let _guard = lock_large_checksums();
194
195 if verify_sha256_checksum(filepath, checksums_file)? {
196 println!("Checksum verified");
197 return Ok(());
198 }
199
200 let new_checksum = calculate_sha256(filepath)?;
201 println!("Updating checksum for local file: {new_checksum}");
202 update_sha256_checksums(filepath, checksums_file, &new_checksum)?;
203 return Ok(());
204 }
205 return Ok(());
206 }
207
208 if let Some(jitter_ms) = initial_jitter_ms {
211 if jitter_ms > 0 {
212 sleep(Duration::from_millis(jitter_ms));
213 }
214 } else {
215 let jitter_delay = {
216 let mut r = rng();
217 Duration::from_millis(r.random_range(100..=600))
218 };
219 sleep(jitter_delay);
220 }
221
222 download_file(filepath, url, timeout_secs, retry_config.clone())?;
223
224 if let Some(checksums_file) = checksums {
226 let guard = lock_large_checksums();
227
228 if !verify_sha256_checksum(filepath, checksums_file)? {
229 let actual = calculate_sha256(filepath)?;
230 println!("Checksum mismatch after download (calculated {actual}), retrying...");
231 remove_file(filepath)?;
232 drop(guard);
233
234 download_file(filepath, url, timeout_secs, retry_config)?;
235
236 let _guard = lock_large_checksums();
237
238 if !verify_sha256_checksum(filepath, checksums_file)? {
239 let actual = calculate_sha256(filepath)?;
240 remove_file(filepath)?;
241 anyhow::bail!(
242 "Checksum mismatch after retry for {} (calculated {actual})",
243 filepath.file_name().unwrap_or_default().display(),
244 );
245 }
246 }
247 }
248
249 Ok(())
250}
251
252fn download_file(
253 filepath: &Path,
254 url: &str,
255 timeout_secs: u64,
256 retry_config: Option<RetryConfig>,
257) -> anyhow::Result<()> {
258 #[cfg(not(test))]
262 if !url.starts_with("https://") {
263 anyhow::bail!("URL must use HTTPS protocol for security: {url}");
264 }
265
266 println!("Downloading file from {url} to {}", filepath.display());
267
268 if let Some(parent) = filepath.parent() {
269 std::fs::create_dir_all(parent)?;
270 }
271
272 let runtime = tokio::runtime::Builder::new_current_thread()
273 .enable_all()
274 .build()?;
275 let client = HttpClient::builder().build()?;
276 let timeout = Duration::from_secs(timeout_secs);
277
278 let cfg = retry_config.unwrap_or(RetryConfig {
279 max_retries: 5,
280 initial_delay_ms: 1_000,
281 max_delay_ms: 10_000,
282 backoff_factor: 2.0,
283 jitter_ms: 1_000,
284 operation_timeout_ms: None,
285 immediate_first: false,
286 max_elapsed_ms: None,
287 });
288
289 remove_stale_partials(filepath);
290 let partial_path = partial_path_for(filepath);
291
292 let op = || async {
293 let _ = remove_file(&partial_path);
295
296 match async {
297 Ok::<_, anyhow::Error>(
298 tokio::time::timeout(timeout, client.get_stream(url.to_owned())).await??,
299 )
300 }
301 .await
302 {
303 Ok(mut response) => {
304 let status = response.status();
305 if status.is_success() {
306 store_response_body(&mut response, timeout, &partial_path, filepath).await
307 } else if status.is_server_error()
308 || status.as_u16() == 429
309 || status.as_u16() == 408
310 {
311 println!("HTTP error {status}, retrying...");
312 Err(DownloadError::Retryable(format!("HTTP {status}")))
313 } else {
314 Err(DownloadError::NonRetryable(format!(
316 "Client error: HTTP {status}"
317 )))
318 }
319 }
320 Err(e) => {
321 println!("Request failed: {e}");
322 Err(DownloadError::Retryable(e.to_string()))
323 }
324 }
325 };
326
327 let should_retry = |e: &DownloadError| matches!(e, DownloadError::Retryable(_));
328
329 let manager = RetryManager::new(cfg);
330
331 let result = runtime.block_on(
332 manager
333 .invocation("download_file", op, should_retry, |e| {
334 DownloadError::Retryable(e.to_string())
335 })
336 .execute(),
337 );
338
339 if result.is_err() {
340 let _ = remove_file(&partial_path);
341 }
342
343 result.map_err(|e| anyhow::anyhow!(e.to_string()))
344}
345
346async fn store_response_body(
348 response: &mut HttpResponseStream,
349 timeout: Duration,
350 partial_path: &Path,
351 filepath: &Path,
352) -> Result<(), DownloadError> {
353 let mut out =
354 File::create(partial_path).map_err(|e| DownloadError::NonRetryable(e.to_string()))?;
355
356 let copied = async {
357 while let Some(chunk) = tokio::time::timeout(timeout, response.chunk())
358 .await
359 .map_err(|e| DownloadError::Retryable(format!("body stream error: {e}")))?
360 .map_err(|e| DownloadError::Retryable(format!("body stream error: {e}")))?
361 {
362 out.write_all(&chunk).map_err(|e| {
363 DownloadError::NonRetryable(format!("write {} failed: {e}", partial_path.display()))
364 })?;
365 }
366
367 out.sync_all().map_err(|e| {
369 DownloadError::NonRetryable(format!("sync {} failed: {e}", partial_path.display()))
370 })
371 }
372 .await;
373
374 drop(out);
375
376 if let Err(e) = copied {
377 let _ = remove_file(partial_path);
378 return Err(e);
379 }
380
381 if let Err(e) = std::fs::rename(partial_path, filepath) {
382 let _ = remove_file(partial_path);
383 return Err(DownloadError::NonRetryable(format!(
384 "rename {} -> {} failed: {e}",
385 partial_path.display(),
386 filepath.display(),
387 )));
388 }
389
390 println!("File downloaded to {}", filepath.display());
391 Ok(())
392}
393
394fn remove_stale_partials(filepath: &Path) {
396 const STALE_PARTIAL_AGE: Duration = Duration::from_secs(3600);
397
398 let (Some(parent), Some(file_name)) = (filepath.parent(), filepath.file_name()) else {
399 return;
400 };
401
402 let parent = if parent.as_os_str().is_empty() {
404 Path::new(".")
405 } else {
406 parent
407 };
408
409 let prefix = format!("{}.partial.", file_name.to_string_lossy());
410
411 let entries = match std::fs::read_dir(parent) {
412 Ok(entries) => entries,
413 Err(e) => {
414 println!(
415 "Failed to scan {} for stale partial downloads: {e}",
416 parent.display()
417 );
418 return;
419 }
420 };
421
422 for entry in entries.flatten() {
423 let name = entry.file_name();
424
425 let Some(suffix) = name.to_str().and_then(|name| name.strip_prefix(&prefix)) else {
426 continue;
427 };
428
429 let is_partial_name = suffix.split_once('.').is_some_and(|(pid, nanos)| {
431 pid.parse::<u32>().is_ok() && nanos.parse::<u128>().is_ok()
432 });
433
434 if !is_partial_name {
435 continue;
436 }
437
438 let is_stale = entry
439 .metadata()
440 .and_then(|metadata| metadata.modified())
441 .ok()
442 .and_then(|modified| modified.elapsed().ok())
443 .is_some_and(|age| age >= STALE_PARTIAL_AGE);
444
445 if !is_stale {
446 continue;
447 }
448
449 let path = entry.path();
450 match remove_file(&path) {
451 Ok(()) => println!("Removed stale partial download {}", path.display()),
452 Err(e) => println!(
453 "Failed to remove stale partial download {}: {e}",
454 path.display()
455 ),
456 }
457 }
458}
459
460fn partial_path_for(filepath: &Path) -> PathBuf {
461 let nanos = std::time::SystemTime::now()
464 .duration_since(std::time::UNIX_EPOCH)
465 .map_or(0, |d| d.as_nanos());
466 let mut p: OsString = filepath.as_os_str().to_owned();
467 p.push(format!(".partial.{}.{}", std::process::id(), nanos));
468 PathBuf::from(p)
469}
470
471fn calculate_sha256(filepath: &Path) -> anyhow::Result<String> {
472 let mut file = File::open(filepath)?;
473 let mut ctx = digest::Context::new(&digest::SHA256);
474 let mut buffer = [0u8; 4096];
475
476 loop {
477 let count = file.read(&mut buffer)?;
478 if count == 0 {
479 break;
480 }
481 ctx.update(&buffer[..count]);
482 }
483
484 let digest = ctx.finish();
485 Ok(hex::encode(digest.as_ref()))
486}
487
488fn verify_sha256_checksum(filepath: &Path, checksums: &Path) -> anyhow::Result<bool> {
489 let file = File::open(checksums)?;
490 let reader = BufReader::new(file);
491 let manifest: Value = serde_json::from_reader(reader)?;
492
493 let filename = checksum_key(filepath)?;
494 if let Some(expected_checksum) = manifest.get(filename) {
495 let expected_checksum_str = expected_checksum.as_str().with_context(|| {
496 format!(
497 "checksum for {filename} in {} is not a string",
498 checksums.display()
499 )
500 })?;
501
502 let expected_hash = expected_checksum_str
503 .strip_prefix("sha256:")
504 .unwrap_or(expected_checksum_str);
505 let calculated_checksum = calculate_sha256(filepath)?;
506 if expected_hash == calculated_checksum {
507 return Ok(true);
508 }
509 }
510
511 Ok(false)
512}
513
514fn update_sha256_checksums(
515 filepath: &Path,
516 checksums_file: &Path,
517 new_checksum: &str,
518) -> anyhow::Result<()> {
519 let mut checksums_map = if checksums_file.exists() {
520 let file = File::open(checksums_file)?;
521 let reader = BufReader::new(file);
522 match serde_json::from_reader(reader)? {
523 Value::Object(map) => map,
524 _ => anyhow::bail!(
525 "checksums file {} is not a JSON object",
526 checksums_file.display()
527 ),
528 }
529 } else {
530 serde_json::Map::new()
531 };
532
533 let filename = checksum_key(filepath)?.to_string();
535 let prefixed_checksum = format!("sha256:{new_checksum}");
536 checksums_map.insert(filename, Value::String(prefixed_checksum));
537
538 let temp_path = partial_path_for(checksums_file);
540 let result = write_checksums(&temp_path, &Value::Object(checksums_map))
541 .and_then(|()| std::fs::rename(&temp_path, checksums_file).map_err(Into::into));
542
543 if result.is_err() {
544 let _ = remove_file(&temp_path);
545 }
546
547 result
548}
549
550fn write_checksums(path: &Path, checksums: &Value) -> anyhow::Result<()> {
551 let mut writer = BufWriter::new(File::create(path)?);
552 serde_json::to_writer_pretty(&mut writer, checksums)?;
553 writer.into_inner()?.sync_all()?;
554 Ok(())
555}
556
557fn checksum_key(filepath: &Path) -> anyhow::Result<&str> {
558 filepath
559 .file_name()
560 .and_then(|name| name.to_str())
561 .with_context(|| format!("no UTF-8 file name in {}", filepath.display()))
562}
563
564#[cfg(test)]
565mod tests {
566 use std::{
567 fs,
568 io::{BufWriter, Write},
569 net::SocketAddr,
570 sync::{
571 Arc,
572 atomic::{AtomicUsize, Ordering},
573 },
574 };
575
576 use axum::{Router, http::StatusCode, routing::get, serve};
577 use rstest::*;
578 use serde_json::{json, to_writer};
579 use tempfile::TempDir;
580 use tokio::{
581 io::{AsyncReadExt, AsyncWriteExt},
582 net::TcpListener,
583 task,
584 time::{Duration, sleep},
585 };
586
587 use super::*;
588
589 fn test_retry_config() -> RetryConfig {
592 RetryConfig {
593 max_retries: 5,
594 initial_delay_ms: 10,
595 max_delay_ms: 50,
596 backoff_factor: 2.0,
597 jitter_ms: 5,
598 operation_timeout_ms: Some(500),
599 immediate_first: false,
600 max_elapsed_ms: Some(2000),
601 }
602 }
603
604 async fn setup_test_server(
605 server_content: Option<String>,
606 status_code: StatusCode,
607 ) -> SocketAddr {
608 let server_content = Arc::new(server_content);
609 let server_content_clone = server_content.clone();
610 let app = Router::new().route(
611 "/testfile.txt",
612 get(move || {
613 let server_content = server_content_clone.clone();
614 async move {
615 let response_body = match &*server_content {
616 Some(content) => content.clone(),
617 None => "File not found".to_string(),
618 };
619 (status_code, response_body)
620 }
621 }),
622 );
623
624 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
625 let addr = listener.local_addr().unwrap();
626 let server = serve(listener, app);
627
628 task::spawn(async move {
629 if let Err(e) = server.await {
630 eprintln!("server error: {e}");
631 }
632 });
633
634 sleep(Duration::from_millis(100)).await;
635
636 addr
637 }
638
639 #[rstest]
640 #[case::missing(None, "verified fixture", true)]
641 #[case::cached(Some("verified fixture"), "verified fixture", true)]
642 #[case::stale_cached(Some("stale fixture"), "verified fixture", true)]
643 #[case::corrupt_download(None, "corrupt download", false)]
644 #[tokio::test]
645 async fn test_prepare_test_data_file(
646 #[case] cached: Option<&str>,
647 #[case] downloaded: &str,
648 #[case] valid: bool,
649 ) {
650 let temp_dir = TempDir::new().unwrap();
651 let filepath = temp_dir.path().join("testfile.txt");
652 let checksums = temp_dir.path().join("checksums.json");
653 let content = "verified fixture";
654 let manifest = serde_json::to_vec(&json!({
655 "testfile.txt": format!("sha256:{}", calculate_sha256_bytes(content.as_bytes()))
656 }))
657 .unwrap();
658 fs::write(&checksums, &manifest).unwrap();
659 if let Some(cached) = cached {
660 fs::write(&filepath, cached).unwrap();
661 }
662
663 let url = if cached == Some(content) {
664 "http://127.0.0.1:0/testfile.txt".to_string()
665 } else {
666 let addr = setup_test_server(Some(downloaded.to_string()), StatusCode::OK).await;
667 format!("http://{addr}/testfile.txt")
668 };
669 let filepath_clone = filepath.clone();
670 let checksums_clone = checksums.clone();
671 let result = tokio::task::spawn_blocking(move || {
672 prepare_test_data_file(&filepath_clone, &url, &checksums_clone)
673 })
674 .await
675 .unwrap();
676
677 if valid {
678 result.unwrap();
679 assert_eq!(fs::read_to_string(&filepath).unwrap(), content);
680 } else {
681 assert_eq!(
682 result.unwrap_err().to_string(),
683 format!("Checksum mismatch for {}", filepath.display()),
684 );
685 assert!(!filepath.exists());
686 }
687 assert_eq!(fs::read(&checksums).unwrap(), manifest);
688 }
689
690 #[tokio::test]
691 async fn test_file_already_exists() {
692 let temp_dir = TempDir::new().unwrap();
693 let file_path = temp_dir.path().join("testfile.txt");
694 fs::write(&file_path, "Existing file content").unwrap();
695
696 let url = "http://example.com/testfile.txt".to_string();
697 let result = ensure_file_exists_or_download_http(&file_path, &url, None, Some(5));
698
699 assert!(result.is_ok());
700 let content = fs::read_to_string(&file_path).unwrap();
701 assert_eq!(content, "Existing file content");
702 }
703
704 #[tokio::test]
705 async fn test_download_file_success() {
706 let temp_dir = TempDir::new().unwrap();
707 let filepath = temp_dir.path().join("testfile.txt");
708 let filepath_clone = filepath.clone();
709
710 let server_content = "Server file content".to_string();
711 let status_code = StatusCode::OK;
712 let addr = setup_test_server(Some(server_content.clone()), status_code).await;
713 let url = format!("http://{addr}/testfile.txt");
714
715 let result = tokio::task::spawn_blocking(move || {
716 ensure_file_exists_or_download_http_with_config(
717 &filepath_clone,
718 &url,
719 None,
720 5,
721 Some(test_retry_config()),
722 Some(0),
723 )
724 })
725 .await
726 .unwrap();
727
728 assert!(result.is_ok());
729 let content = fs::read_to_string(&filepath).unwrap();
730 assert_eq!(content, server_content);
731 }
732
733 #[tokio::test]
734 async fn test_download_file_not_found() {
735 let temp_dir = TempDir::new().unwrap();
736 let file_path = temp_dir.path().join("testfile.txt");
737
738 let server_content = None;
739 let status_code = StatusCode::NOT_FOUND;
740 let addr = setup_test_server(server_content, status_code).await;
741 let url = format!("http://{addr}/testfile.txt");
742
743 let result = tokio::task::spawn_blocking(move || {
744 ensure_file_exists_or_download_http_with_config(
745 &file_path,
746 &url,
747 None,
748 1,
749 Some(test_retry_config()),
750 Some(0),
751 )
752 })
753 .await
754 .unwrap();
755
756 assert!(result.is_err());
757 let err_msg = format!("{}", result.unwrap_err());
758 assert!(
759 err_msg.contains("Client error: HTTP"),
760 "Unexpected error message: {err_msg}"
761 );
762 }
763
764 #[tokio::test]
765 async fn test_network_error() {
766 let temp_dir = TempDir::new().unwrap();
767 let file_path = temp_dir.path().join("testfile.txt");
768
769 let url = "http://127.0.0.1:0/testfile.txt".to_string();
771
772 let result = tokio::task::spawn_blocking(move || {
773 ensure_file_exists_or_download_http_with_config(
774 &file_path,
775 &url,
776 None,
777 2,
778 Some(test_retry_config()),
779 Some(0),
780 )
781 })
782 .await
783 .unwrap();
784
785 assert!(result.is_err());
786 let err_msg = format!("{}", result.unwrap_err());
787 assert!(
788 err_msg.contains("error"),
789 "Unexpected error message: {err_msg}"
790 );
791 }
792
793 #[tokio::test]
794 async fn test_retry_then_success_on_500() {
795 let temp_dir = TempDir::new().unwrap();
796 let filepath = temp_dir.path().join("testfile.txt");
797 let filepath_clone = filepath.clone();
798
799 let counter = Arc::new(AtomicUsize::new(0));
800 let counter_clone = counter.clone();
801
802 let app = Router::new().route(
803 "/testfile.txt",
804 get(move || {
805 let c = counter_clone.clone();
806 async move {
807 let n = c.fetch_add(1, Ordering::SeqCst);
808 if n < 2 {
809 (StatusCode::INTERNAL_SERVER_ERROR, "temporary error")
810 } else {
811 (StatusCode::OK, "eventual success")
812 }
813 }
814 }),
815 );
816
817 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
818 let addr = listener.local_addr().unwrap();
819 let server = serve(listener, app);
820 task::spawn(async move {
821 let _ = server.await;
822 });
823 sleep(Duration::from_millis(100)).await;
824
825 let url = format!("http://{addr}/testfile.txt");
826
827 let result = tokio::task::spawn_blocking(move || {
828 ensure_file_exists_or_download_http_with_config(
829 &filepath_clone,
830 &url,
831 None,
832 5,
833 Some(test_retry_config()),
834 Some(0),
835 )
836 })
837 .await
838 .unwrap();
839
840 assert!(result.is_ok());
841 let content = std::fs::read_to_string(&filepath).unwrap();
842 assert_eq!(content, "eventual success");
843 assert!(counter.load(Ordering::SeqCst) >= 2);
844 }
845
846 #[tokio::test]
847 async fn test_retry_then_success_on_429() {
848 let temp_dir = TempDir::new().unwrap();
849 let filepath = temp_dir.path().join("testfile.txt");
850 let filepath_clone = filepath.clone();
851
852 let counter = Arc::new(AtomicUsize::new(0));
853 let counter_clone = counter.clone();
854
855 let app = Router::new().route(
856 "/testfile.txt",
857 get(move || {
858 let c = counter_clone.clone();
859 async move {
860 let n = c.fetch_add(1, Ordering::SeqCst);
861 if n < 1 {
862 (StatusCode::TOO_MANY_REQUESTS, "rate limited")
863 } else {
864 (StatusCode::OK, "ok after retry")
865 }
866 }
867 }),
868 );
869
870 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
871 let addr = listener.local_addr().unwrap();
872 let server = serve(listener, app);
873 task::spawn(async move {
874 let _ = server.await;
875 });
876 sleep(Duration::from_millis(100)).await;
877
878 let url = format!("http://{addr}/testfile.txt");
879
880 let result = tokio::task::spawn_blocking(move || {
881 ensure_file_exists_or_download_http_with_config(
882 &filepath_clone,
883 &url,
884 None,
885 5,
886 Some(test_retry_config()),
887 Some(0),
888 )
889 })
890 .await
891 .unwrap();
892
893 assert!(result.is_ok());
894 let content = std::fs::read_to_string(&filepath).unwrap();
895 assert_eq!(content, "ok after retry");
896 assert!(counter.load(Ordering::SeqCst) >= 2);
897 }
898
899 #[tokio::test]
900 async fn test_no_retry_on_404() {
901 let temp_dir = TempDir::new().unwrap();
902 let filepath = temp_dir.path().join("testfile.txt");
903 let filepath_clone = filepath.clone();
904
905 let counter = Arc::new(AtomicUsize::new(0));
906 let counter_clone = counter.clone();
907
908 let app = Router::new().route(
909 "/testfile.txt",
910 get(move || {
911 let c = counter_clone.clone();
912 async move {
913 c.fetch_add(1, Ordering::SeqCst);
914 (StatusCode::NOT_FOUND, "missing")
915 }
916 }),
917 );
918
919 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
920 let addr = listener.local_addr().unwrap();
921 let server = serve(listener, app);
922 task::spawn(async move {
923 let _ = server.await;
924 });
925 sleep(Duration::from_millis(100)).await;
926
927 let url = format!("http://{addr}/testfile.txt");
928
929 let result = tokio::task::spawn_blocking(move || {
930 ensure_file_exists_or_download_http_with_config(
931 &filepath_clone,
932 &url,
933 None,
934 5,
935 Some(test_retry_config()),
936 Some(0),
937 )
938 })
939 .await
940 .unwrap();
941
942 assert!(result.is_err());
943 assert_eq!(counter.load(Ordering::SeqCst), 1, "should not retry on 404");
944 }
945
946 #[tokio::test]
947 async fn test_checksum_mismatch_retry_then_success() {
948 let temp_dir = TempDir::new().unwrap();
949 let filepath = temp_dir.path().join("testfile.txt");
950 let filepath_clone = filepath.clone();
951
952 let good_content = "correct content";
953 let good_checksum = calculate_sha256_bytes(good_content.as_bytes());
954
955 let checksums_path = temp_dir.path().join("checksums.json");
956 let checksums_data = json!({
957 "testfile.txt": format!("sha256:{good_checksum}")
958 });
959 let checksums_file = File::create(&checksums_path).unwrap();
960 to_writer(BufWriter::new(checksums_file), &checksums_data).unwrap();
961 let checksums_clone = checksums_path.clone();
962
963 let counter = Arc::new(AtomicUsize::new(0));
965 let counter_clone = counter.clone();
966
967 let app = Router::new().route(
968 "/testfile.txt",
969 get(move || {
970 let c = counter_clone.clone();
971 async move {
972 let n = c.fetch_add(1, Ordering::SeqCst);
973 if n == 0 {
974 (StatusCode::OK, "corrupt data")
975 } else {
976 (StatusCode::OK, "correct content")
977 }
978 }
979 }),
980 );
981
982 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
983 let addr = listener.local_addr().unwrap();
984 let server = serve(listener, app);
985 task::spawn(async move {
986 let _ = server.await;
987 });
988 sleep(Duration::from_millis(100)).await;
989
990 let url = format!("http://{addr}/testfile.txt");
991
992 let result = tokio::task::spawn_blocking(move || {
993 ensure_file_exists_or_download_http_with_config(
994 &filepath_clone,
995 &url,
996 Some(&checksums_clone),
997 5,
998 Some(test_retry_config()),
999 Some(0),
1000 )
1001 })
1002 .await
1003 .unwrap();
1004
1005 assert!(result.is_ok());
1006 let content = fs::read_to_string(&filepath).unwrap();
1007 assert_eq!(content, good_content);
1008 assert_eq!(counter.load(Ordering::SeqCst), 2);
1009 }
1010
1011 #[tokio::test]
1012 async fn test_checksum_mismatch_retry_then_fail() {
1013 let temp_dir = TempDir::new().unwrap();
1014 let filepath = temp_dir.path().join("testfile.txt");
1015 let filepath_clone = filepath.clone();
1016
1017 let checksums_path = temp_dir.path().join("checksums.json");
1019 let checksums_data = json!({
1020 "testfile.txt": "sha256:0000000000000000000000000000000000000000000000000000000000000000"
1021 });
1022 let checksums_file = File::create(&checksums_path).unwrap();
1023 to_writer(BufWriter::new(checksums_file), &checksums_data).unwrap();
1024 let checksums_clone = checksums_path.clone();
1025
1026 let counter = Arc::new(AtomicUsize::new(0));
1027 let counter_clone = counter.clone();
1028
1029 let app = Router::new().route(
1030 "/testfile.txt",
1031 get(move || {
1032 let c = counter_clone.clone();
1033 async move {
1034 c.fetch_add(1, Ordering::SeqCst);
1035 (StatusCode::OK, "always wrong content")
1036 }
1037 }),
1038 );
1039
1040 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1041 let addr = listener.local_addr().unwrap();
1042 let server = serve(listener, app);
1043 task::spawn(async move {
1044 let _ = server.await;
1045 });
1046 sleep(Duration::from_millis(100)).await;
1047
1048 let url = format!("http://{addr}/testfile.txt");
1049
1050 let result = tokio::task::spawn_blocking(move || {
1051 ensure_file_exists_or_download_http_with_config(
1052 &filepath_clone,
1053 &url,
1054 Some(&checksums_clone),
1055 5,
1056 Some(test_retry_config()),
1057 Some(0),
1058 )
1059 })
1060 .await
1061 .unwrap();
1062
1063 assert!(result.is_err());
1064 let err_msg = format!("{}", result.unwrap_err());
1065 assert!(err_msg.contains("Checksum mismatch after retry"));
1066 assert_eq!(
1067 counter.load(Ordering::SeqCst),
1068 2,
1069 "should download exactly twice"
1070 );
1071 assert!(!filepath.exists(), "corrupt file should be cleaned up");
1072 }
1073
1074 async fn truncated_then_full_server(good_body: &'static str) -> (SocketAddr, Arc<AtomicUsize>) {
1076 use tokio::io::{AsyncReadExt, AsyncWriteExt};
1077
1078 let counter = Arc::new(AtomicUsize::new(0));
1079 let counter_clone = counter.clone();
1080
1081 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1082 let addr = listener.local_addr().unwrap();
1083
1084 task::spawn(async move {
1085 loop {
1086 let Ok((mut sock, _)) = listener.accept().await else {
1087 break;
1088 };
1089 let counter = counter_clone.clone();
1090
1091 task::spawn(async move {
1092 let mut buf = [0u8; 1024];
1093 let _ = sock.read(&mut buf).await;
1094 let n = counter.fetch_add(1, Ordering::SeqCst);
1095
1096 if n == 0 {
1097 let resp = b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\nshort";
1098 let _ = sock.write_all(resp).await;
1099 } else {
1100 let resp = format!(
1101 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
1102 good_body.len(),
1103 good_body,
1104 );
1105 let _ = sock.write_all(resp.as_bytes()).await;
1106 }
1107 let _ = sock.shutdown().await;
1108 });
1109 }
1110 });
1111
1112 sleep(Duration::from_millis(100)).await;
1113 (addr, counter)
1114 }
1115
1116 fn count_partial_siblings(filepath: &Path) -> usize {
1117 let parent = filepath.parent().unwrap();
1118 let stem = filepath.file_name().unwrap().to_string_lossy().into_owned();
1119 let prefix = format!("{stem}.partial.");
1120 fs::read_dir(parent)
1121 .unwrap()
1122 .filter_map(Result::ok)
1123 .filter(|e| e.file_name().to_string_lossy().starts_with(&prefix))
1124 .count()
1125 }
1126
1127 #[rstest]
1128 #[case::operation(false, "Retryable error: Timed out after 40ms")]
1129 #[case::elapsed(true, "Retryable error: Retry budget exceeded (1/1)")]
1130 #[tokio::test]
1131 async fn test_download_retry_deadline_cleans_partial(
1132 #[case] elapsed_budget: bool,
1133 #[case] expected_error: &str,
1134 #[values(false, true)] body_started: bool,
1135 ) {
1136 let directory = TempDir::new().unwrap();
1137 let filepath = directory.path().join("deadline.txt");
1138 let destination = filepath.clone();
1139 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1140 let addr = listener.local_addr().unwrap();
1141 let peer = tokio::spawn(async move {
1142 let (mut socket, _) = listener.accept().await.unwrap();
1143 let mut request = Vec::new();
1144 while !request.ends_with(b"\r\n\r\n") {
1145 request.push(socket.read_u8().await.unwrap());
1146 }
1147
1148 if body_started {
1149 socket
1150 .write_all(
1151 b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\na",
1152 )
1153 .await
1154 .unwrap();
1155 }
1156
1157 std::future::pending::<()>().await;
1158 });
1159
1160 let result = tokio::task::spawn_blocking(move || {
1161 let config = RetryConfig {
1162 max_retries: 0,
1163 operation_timeout_ms: if elapsed_budget { None } else { Some(40) },
1164 max_elapsed_ms: if elapsed_budget { Some(40) } else { None },
1165 ..test_retry_config()
1166 };
1167
1168 download_file(
1169 &destination,
1170 &format!("http://{addr}/deadline"),
1171 1,
1172 Some(config),
1173 )
1174 .map_err(|e| e.to_string())
1175 })
1176 .await
1177 .unwrap();
1178
1179 peer.abort();
1180 let _ = peer.await;
1181
1182 assert_eq!(result, Err(expected_error.to_string()));
1183 assert!(!filepath.exists());
1184 assert_eq!(count_partial_siblings(&filepath), 0);
1185 }
1186
1187 #[tokio::test]
1188 async fn test_download_retry_budget_prevents_another_request() {
1189 let directory = TempDir::new().unwrap();
1190 let filepath = directory.path().join("budget.txt");
1191 let destination = filepath.clone();
1192 let requests = Arc::new(AtomicUsize::new(0));
1193 let counter = requests.clone();
1194 let app = Router::new().route(
1195 "/budget",
1196 get(move || {
1197 counter.fetch_add(1, Ordering::SeqCst);
1198 async { StatusCode::INTERNAL_SERVER_ERROR }
1199 }),
1200 );
1201
1202 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1203 let addr = listener.local_addr().unwrap();
1204 let server = tokio::spawn(async move { serve(listener, app).await.unwrap() });
1205
1206 let result = tokio::task::spawn_blocking(move || {
1207 let config = RetryConfig {
1208 max_retries: 2,
1209 initial_delay_ms: 100,
1210 max_delay_ms: 100,
1211 jitter_ms: 0,
1212 operation_timeout_ms: None,
1213 max_elapsed_ms: Some(40),
1214 ..test_retry_config()
1215 };
1216
1217 download_file(
1218 &destination,
1219 &format!("http://{addr}/budget"),
1220 1,
1221 Some(config),
1222 )
1223 .map_err(|e| e.to_string())
1224 })
1225 .await
1226 .unwrap();
1227
1228 server.abort();
1229 let _ = server.await;
1230
1231 assert_eq!(requests.load(Ordering::SeqCst), 1);
1232 assert_eq!(
1233 result,
1234 Err("Retryable error: Retry budget exceeded (2/3)".to_string())
1235 );
1236 assert!(!filepath.exists());
1237 assert_eq!(count_partial_siblings(&filepath), 0);
1238 }
1239
1240 #[tokio::test]
1241 async fn test_download_immediate_first_retains_later_delay() {
1242 let directory = TempDir::new().unwrap();
1243 let filepath = directory.path().join("immediate.txt");
1244 let destination = filepath.clone();
1245 let requests = Arc::new(AtomicUsize::new(0));
1246 let counter = requests.clone();
1247 let app = Router::new().route(
1248 "/immediate",
1249 get(move || {
1250 let attempt = counter.fetch_add(1, Ordering::SeqCst);
1251 async move {
1252 if attempt < 2 {
1253 (StatusCode::INTERNAL_SERVER_ERROR, "retry")
1254 } else {
1255 (StatusCode::OK, "complete")
1256 }
1257 }
1258 }),
1259 );
1260
1261 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1262 let addr = listener.local_addr().unwrap();
1263 let server = tokio::spawn(async move { serve(listener, app).await.unwrap() });
1264
1265 let (result, elapsed) = tokio::task::spawn_blocking(move || {
1266 let config = RetryConfig {
1267 max_retries: 2,
1268 initial_delay_ms: 100,
1269 max_delay_ms: 100,
1270 jitter_ms: 0,
1271 immediate_first: true,
1272 operation_timeout_ms: None,
1273 max_elapsed_ms: None,
1274 ..test_retry_config()
1275 };
1276
1277 let start = std::time::Instant::now();
1278 let result = download_file(
1279 &destination,
1280 &format!("http://{addr}/immediate"),
1281 1,
1282 Some(config),
1283 );
1284 (result, start.elapsed())
1285 })
1286 .await
1287 .unwrap();
1288
1289 server.abort();
1290 let _ = server.await;
1291
1292 result.unwrap();
1293 assert_eq!(requests.load(Ordering::SeqCst), 3);
1294 assert!(elapsed >= Duration::from_millis(100));
1295 assert_eq!(fs::read_to_string(&filepath).unwrap(), "complete");
1296 assert_eq!(count_partial_siblings(&filepath), 0);
1297 }
1298
1299 #[rstest]
1300 #[case::progressing(0, 400, Ok(()), false)]
1301 #[case::progressing_default(0, 400, Ok(()), true)]
1302 #[case::stalled_headers(1200, 0, Err("Retryable error: deadline has elapsed"), false)]
1303 #[case::stalled_body(
1304 0,
1305 1200,
1306 Err("Retryable error: body stream error: deadline has elapsed"),
1307 false
1308 )]
1309 #[tokio::test]
1310 async fn download_timeout_applies_to_each_read(
1311 #[case] header_delay_ms: u64,
1312 #[case] chunk_delay_ms: u64,
1313 #[case] expected: Result<(), &str>,
1314 #[case] default_retry: bool,
1315 ) {
1316 use tokio::io::{AsyncReadExt, AsyncWriteExt};
1317
1318 let temp_dir = TempDir::new().unwrap();
1319 let filepath = temp_dir.path().join("streamed.txt");
1320 let destination = filepath.clone();
1321 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1322 let addr = listener.local_addr().unwrap();
1323 let peer = task::spawn(async move {
1324 let (mut socket, _) = listener.accept().await.unwrap();
1325 let mut request = Vec::new();
1326 while !request.ends_with(b"\r\n\r\n") {
1327 request.push(socket.read_u8().await.unwrap());
1328 }
1329 sleep(Duration::from_millis(header_delay_ms)).await;
1330
1331 if socket
1332 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\na")
1333 .await
1334 .is_err()
1335 {
1336 return;
1337 }
1338
1339 for byte in b"bcd" {
1340 sleep(Duration::from_millis(chunk_delay_ms)).await;
1341
1342 if socket.write_all(&[*byte]).await.is_err() {
1343 return;
1344 }
1345 }
1346 });
1347 let result = task::spawn_blocking(move || {
1348 let cfg = RetryConfig {
1349 max_retries: 0,
1350 operation_timeout_ms: None,
1351 max_elapsed_ms: None,
1352 ..test_retry_config()
1353 };
1354
1355 let config = if default_retry { None } else { Some(cfg) };
1356 download_file(&destination, &format!("http://{addr}/streamed"), 1, config)
1357 .map_err(|e| e.to_string())
1358 })
1359 .await
1360 .unwrap();
1361 peer.await.unwrap();
1362
1363 assert_eq!(
1364 result.as_ref().map_err(String::as_str),
1365 expected.as_ref().map_err(|e| *e)
1366 );
1367 assert_eq!(count_partial_siblings(&filepath), 0);
1368 if expected.is_ok() {
1369 assert_eq!(fs::read(&filepath).unwrap(), b"abcd");
1370 } else {
1371 assert!(!filepath.exists());
1372 }
1373 }
1374
1375 #[tokio::test]
1376 async fn test_body_truncation_retries_and_recovers() {
1377 let temp_dir = TempDir::new().unwrap();
1378 let filepath = temp_dir.path().join("testfile.txt");
1379 let filepath_clone = filepath.clone();
1380
1381 let (addr, counter) = truncated_then_full_server("complete payload").await;
1382 let url = format!("http://{addr}/testfile.txt");
1383
1384 let result = tokio::task::spawn_blocking(move || {
1385 ensure_file_exists_or_download_http_with_config(
1386 &filepath_clone,
1387 &url,
1388 None,
1389 5,
1390 Some(test_retry_config()),
1391 Some(0),
1392 )
1393 })
1394 .await
1395 .unwrap();
1396
1397 assert!(result.is_ok(), "should retry past the truncated response");
1398 assert_eq!(counter.load(Ordering::SeqCst), 2);
1399 assert_eq!(
1400 count_partial_siblings(&filepath),
1401 0,
1402 "no .partial siblings must remain after success",
1403 );
1404 let content = fs::read_to_string(&filepath).unwrap();
1405 assert_eq!(content, "complete payload");
1406 }
1407
1408 #[tokio::test]
1409 async fn test_body_truncation_exhausts_retries_leaves_no_corrupt_file() {
1410 use tokio::io::{AsyncReadExt, AsyncWriteExt};
1411
1412 let temp_dir = TempDir::new().unwrap();
1413 let filepath = temp_dir.path().join("testfile.txt");
1414 let filepath_clone = filepath.clone();
1415
1416 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1417 let addr = listener.local_addr().unwrap();
1418
1419 task::spawn(async move {
1420 loop {
1421 let Ok((mut sock, _)) = listener.accept().await else {
1422 break;
1423 };
1424
1425 task::spawn(async move {
1426 let mut buf = [0u8; 1024];
1427 let _ = sock.read(&mut buf).await;
1428 let resp =
1429 b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\nshort";
1430 let _ = sock.write_all(resp).await;
1431 let _ = sock.shutdown().await;
1432 });
1433 }
1434 });
1435
1436 sleep(Duration::from_millis(100)).await;
1437 let url = format!("http://{addr}/testfile.txt");
1438
1439 let result = tokio::task::spawn_blocking(move || {
1440 ensure_file_exists_or_download_http_with_config(
1441 &filepath_clone,
1442 &url,
1443 None,
1444 5,
1445 Some(test_retry_config()),
1446 Some(0),
1447 )
1448 })
1449 .await
1450 .unwrap();
1451
1452 assert!(result.is_err(), "all retries should fail");
1453 assert!(!filepath.exists(), "no corrupt file at final path");
1454 assert_eq!(
1455 count_partial_siblings(&filepath),
1456 0,
1457 "no .partial siblings may leak after exhausted retries",
1458 );
1459 }
1460
1461 #[rstest]
1462 fn test_partial_path_for_is_unique_and_marked() {
1463 let target = Path::new("/tmp/data.parquet");
1464 let stem = target.file_name().unwrap().to_string_lossy().into_owned();
1465 let expected_prefix = format!("{stem}.partial.");
1466
1467 let a = partial_path_for(target);
1468 let b = partial_path_for(target);
1469 assert_ne!(a, b, "partial paths must be unique per call");
1470 for p in [&a, &b] {
1471 let name = p.file_name().unwrap().to_string_lossy().into_owned();
1472 assert!(name.starts_with(&expected_prefix), "got {name}");
1473 }
1474 }
1475
1476 #[tokio::test]
1477 async fn test_unrelated_partial_sibling_not_clobbered() {
1478 let temp_dir = TempDir::new().unwrap();
1479 let filepath = temp_dir.path().join("testfile.txt");
1480 let filepath_clone = filepath.clone();
1481 let bystander = temp_dir.path().join("testfile.txt.partial");
1483 fs::write(&bystander, b"do not touch").unwrap();
1484
1485 let server_content = "downloaded".to_string();
1486 let addr = setup_test_server(Some(server_content.clone()), StatusCode::OK).await;
1487 let url = format!("http://{addr}/testfile.txt");
1488
1489 let result = tokio::task::spawn_blocking(move || {
1490 ensure_file_exists_or_download_http_with_config(
1491 &filepath_clone,
1492 &url,
1493 None,
1494 5,
1495 Some(test_retry_config()),
1496 Some(0),
1497 )
1498 })
1499 .await
1500 .unwrap();
1501
1502 assert!(result.is_ok());
1503 assert_eq!(fs::read_to_string(&filepath).unwrap(), server_content);
1504 assert_eq!(
1505 fs::read_to_string(&bystander).unwrap(),
1506 "do not touch",
1507 "unrelated sibling file must be preserved",
1508 );
1509 }
1510
1511 #[tokio::test]
1512 async fn test_download_removes_stale_partial_downloads() {
1513 let temp_dir = TempDir::new().unwrap();
1514 let filepath = temp_dir.path().join("testfile.txt");
1515 let filepath_clone = filepath.clone();
1516 let stale_partial = temp_dir.path().join("testfile.txt.partial.123.456");
1517 File::create(&stale_partial)
1518 .unwrap()
1519 .set_modified(std::time::SystemTime::now() - Duration::from_secs(7200))
1520 .unwrap();
1521
1522 let server_content = "downloaded".to_string();
1523 let addr = setup_test_server(Some(server_content.clone()), StatusCode::OK).await;
1524 let url = format!("http://{addr}/testfile.txt");
1525
1526 let result = tokio::task::spawn_blocking(move || {
1527 ensure_file_exists_or_download_http_with_config(
1528 &filepath_clone,
1529 &url,
1530 None,
1531 5,
1532 Some(test_retry_config()),
1533 Some(0),
1534 )
1535 })
1536 .await
1537 .unwrap();
1538
1539 result.unwrap();
1540 assert_eq!(fs::read_to_string(&filepath).unwrap(), server_content);
1541 assert!(!stale_partial.exists());
1542 }
1543
1544 fn calculate_sha256_bytes(data: &[u8]) -> String {
1545 let mut ctx = digest::Context::new(&digest::SHA256);
1546 ctx.update(data);
1547 hex::encode(ctx.finish().as_ref())
1548 }
1549
1550 #[rstest]
1551 #[expect(clippy::panic_in_result_fn)]
1552 fn test_calculate_sha256() -> anyhow::Result<()> {
1553 let temp_dir = TempDir::new()?;
1554 let test_file_path = temp_dir.path().join("test_file.txt");
1555 let mut test_file = File::create(&test_file_path)?;
1556 let content = b"Hello, world!";
1557 test_file.write_all(content)?;
1558
1559 let expected_hash = "315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3";
1560 let calculated_hash = calculate_sha256(&test_file_path)?;
1561
1562 assert_eq!(calculated_hash, expected_hash);
1563 Ok(())
1564 }
1565
1566 #[rstest]
1567 #[expect(clippy::panic_in_result_fn)]
1568 fn test_verify_sha256_checksum() -> anyhow::Result<()> {
1569 let temp_dir = TempDir::new()?;
1570 let test_file_path = temp_dir.path().join("test_file.txt");
1571 let mut test_file = File::create(&test_file_path)?;
1572 let content = b"Hello, world!";
1573 test_file.write_all(content)?;
1574
1575 let calculated_checksum = calculate_sha256(&test_file_path)?;
1576
1577 let checksums_path = temp_dir.path().join("checksums.json");
1579 let checksums_data = json!({
1580 "test_file.txt": format!("sha256:{}", calculated_checksum)
1581 });
1582 let checksums_file = File::create(&checksums_path)?;
1583 let writer = BufWriter::new(checksums_file);
1584 to_writer(writer, &checksums_data)?;
1585
1586 let is_valid = verify_sha256_checksum(&test_file_path, &checksums_path)?;
1587 assert!(is_valid, "The checksum should be valid");
1588 Ok(())
1589 }
1590
1591 #[rstest]
1592 fn test_verify_sha256_checksum_rejects_non_string_entry() {
1593 let temp_dir = TempDir::new().unwrap();
1594 let test_file_path = temp_dir.path().join("test_file.txt");
1595 fs::write(&test_file_path, b"Hello, world!").unwrap();
1596 let checksums_path = temp_dir.path().join("checksums.json");
1597 fs::write(&checksums_path, r#"{"test_file.txt": 5}"#).unwrap();
1598
1599 let error = verify_sha256_checksum(&test_file_path, &checksums_path).unwrap_err();
1600
1601 assert_eq!(
1602 error.to_string(),
1603 format!(
1604 "checksum for test_file.txt in {} is not a string",
1605 checksums_path.display()
1606 )
1607 );
1608 }
1609
1610 #[cfg(unix)]
1611 #[rstest]
1612 fn test_verify_sha256_checksum_rejects_non_utf8_file_name() {
1613 use std::os::unix::ffi::OsStrExt;
1614
1615 let temp_dir = TempDir::new().unwrap();
1616 let test_file_path = temp_dir
1617 .path()
1618 .join(std::ffi::OsStr::from_bytes(b"bad\xff.txt"));
1619 let checksums_path = temp_dir.path().join("checksums.json");
1620 fs::write(&checksums_path, "{}").unwrap();
1621
1622 let error = verify_sha256_checksum(&test_file_path, &checksums_path).unwrap_err();
1623
1624 assert_eq!(
1625 error.to_string(),
1626 format!("no UTF-8 file name in {}", test_file_path.display())
1627 );
1628 }
1629
1630 #[rstest]
1631 fn test_update_sha256_checksums_replaces_manifest_without_leftovers() {
1632 let temp_dir = TempDir::new().unwrap();
1633 let checksums_path = temp_dir.path().join("checksums.json");
1634 fs::write(&checksums_path, r#"{"existing.txt": "sha256:aaa"}"#).unwrap();
1635
1636 update_sha256_checksums(&temp_dir.path().join("new.txt"), &checksums_path, "bbb").unwrap();
1637
1638 let manifest: Value =
1639 serde_json::from_str(&fs::read_to_string(&checksums_path).unwrap()).unwrap();
1640 assert_eq!(
1641 manifest,
1642 json!({"existing.txt": "sha256:aaa", "new.txt": "sha256:bbb"})
1643 );
1644 let entries: Vec<_> = fs::read_dir(temp_dir.path())
1645 .unwrap()
1646 .map(|entry| entry.unwrap().file_name())
1647 .collect();
1648 assert_eq!(entries, vec![OsString::from("checksums.json")]);
1649 }
1650
1651 #[rstest]
1652 fn test_update_sha256_checksums_rejects_non_object_manifest() {
1653 let temp_dir = TempDir::new().unwrap();
1654 let checksums_path = temp_dir.path().join("checksums.json");
1655 fs::write(&checksums_path, "[]").unwrap();
1656
1657 let error =
1658 update_sha256_checksums(&temp_dir.path().join("new.txt"), &checksums_path, "bbb")
1659 .unwrap_err();
1660
1661 assert_eq!(
1662 error.to_string(),
1663 format!(
1664 "checksums file {} is not a JSON object",
1665 checksums_path.display()
1666 )
1667 );
1668 assert_eq!(fs::read_to_string(&checksums_path).unwrap(), "[]");
1669 }
1670
1671 #[rstest]
1672 fn test_remove_stale_partials_only_removes_old_download_partials() {
1673 let temp_dir = TempDir::new().unwrap();
1674 let filepath = temp_dir.path().join("testfile.txt");
1675 let two_hours_ago = std::time::SystemTime::now() - Duration::from_secs(7200);
1676
1677 let write_file = |name: &str, stale: bool| {
1678 let path = temp_dir.path().join(name);
1679 let file = File::create(&path).unwrap();
1680
1681 if stale {
1682 file.set_modified(two_hours_ago).unwrap();
1683 }
1684
1685 path
1686 };
1687
1688 let stale_partial = write_file("testfile.txt.partial.123.456", true);
1689 let fresh_partial = write_file("testfile.txt.partial.124.457", false);
1690 let bystander = write_file("testfile.txt.partial", true);
1691 let non_numeric = write_file("testfile.txt.partial.abc.def", true);
1692 let other_target = write_file("other.txt.partial.123.456", true);
1693
1694 remove_stale_partials(&filepath);
1695
1696 assert!(!stale_partial.exists());
1697 assert!(fresh_partial.exists());
1698 assert!(bystander.exists());
1699 assert!(non_numeric.exists());
1700 assert!(other_target.exists());
1701 }
1702}