Skip to main content

nautilus_testkit/
files.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    collections::BTreeMap,
18    ffi::OsString,
19    fmt::Display,
20    fs::{File, remove_file},
21    io::{BufReader, BufWriter, Read, Write},
22    path::{Path, PathBuf},
23    sync::OnceLock,
24    thread::sleep,
25    time::Duration,
26};
27
28use anyhow::Context;
29use aws_lc_rs::digest;
30use nautilus_core::hex;
31use nautilus_network::{
32    http::{HttpClient, HttpResponseStream},
33    retry::{RetryConfig, RetryManager},
34};
35use parking_lot::Mutex;
36use rand::{RngExt, rng};
37use serde_json::Value;
38
39static LARGE_CHECKSUMS_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
40
41fn lock_large_checksums() -> parking_lot::MutexGuard<'static, ()> {
42    LARGE_CHECKSUMS_LOCK.get_or_init(|| Mutex::new(())).lock()
43}
44
45#[derive(Debug)]
46enum DownloadError {
47    Retryable(String),
48    NonRetryable(String),
49}
50
51impl Display for DownloadError {
52    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
53        match self {
54            Self::Retryable(msg) => write!(f, "Retryable error: {msg}"),
55            Self::NonRetryable(msg) => write!(f, "Non-retryable error: {msg}"),
56        }
57    }
58}
59
60impl std::error::Error for DownloadError {}
61
62/// Downloads missing large test fixtures and verifies every file against the checksum manifest.
63///
64/// Replaces cached files whose checksums differ. The manifest remains unchanged.
65///
66/// # Errors
67///
68/// Returns an error if the manifest cannot be read, a download fails, or a checksum differs.
69pub fn prepare_test_data() -> anyhow::Result<()> {
70    let checksums = crate::common::get_test_data_large_checksums_filepath();
71    let manifest: BTreeMap<String, String> =
72        serde_json::from_reader(BufReader::new(File::open(&checksums)?))?;
73    let directory = nautilus_core::paths::get_test_data_path().join("large");
74
75    for filename in manifest.keys() {
76        let filepath = directory.join(filename);
77        let url = format!("https://test-data.nautechsystems.io/large/{filename}");
78        prepare_test_data_file(&filepath, &url, &checksums)?;
79    }
80    Ok(())
81}
82
83fn prepare_test_data_file(filepath: &Path, url: &str, checksums: &Path) -> anyhow::Result<()> {
84    if filepath.exists() {
85        if verify_sha256_checksum(filepath, checksums)? {
86            return Ok(());
87        }
88        remove_file(filepath)?;
89    }
90
91    download_file(filepath, url, 30, None)?;
92    if !verify_sha256_checksum(filepath, checksums)? {
93        remove_file(filepath)?;
94        anyhow::bail!("Checksum mismatch for {}", filepath.display());
95    }
96    Ok(())
97}
98
99/// Ensures that a file exists at the specified path by downloading it if necessary.
100///
101/// If the file already exists, it checks the integrity of the file using a SHA-256 checksum
102/// from the optional `checksums` file. If the checksum is valid, the function exits early. If
103/// the checksum is invalid or missing, the function updates the checksums file with the correct
104/// hash for the existing file without redownloading it.
105///
106/// If the file does not exist, it downloads the file from the specified `url`. When a
107/// `checksums` file is provided, the download must match its entry for the file (retried once on
108/// mismatch), and a missing entry counts as a mismatch. Downloads never update the checksums file.
109///
110/// The `timeout_secs` parameter specifies the timeout in seconds for the HTTP request.
111/// If `None` is provided, a default timeout of 30 seconds will be used.
112///
113/// # Errors
114///
115/// Returns an error if:
116/// - The HTTP request cannot be sent or returns a non-success status code.
117/// - Any I/O operation fails during file creation, reading, or writing.
118/// - Checksum verification or JSON parsing fails.
119pub fn ensure_file_exists_or_download_http(
120    filepath: &Path,
121    url: &str,
122    checksums: Option<&Path>,
123    timeout_secs: Option<u64>,
124) -> anyhow::Result<()> {
125    ensure_file_exists_or_download_http_with_config(
126        filepath,
127        url,
128        checksums,
129        timeout_secs.unwrap_or(30),
130        None,
131        None,
132    )
133}
134
135/// Ensures that a file exists at the specified path by downloading it if necessary, with a custom timeout.
136///
137/// # Errors
138///
139/// Returns an error if:
140/// - The HTTP request cannot be sent or returns a non-success status code after retries.
141/// - Any I/O operation fails during file creation, reading, or writing.
142/// - Checksum verification or JSON parsing fails.
143pub fn ensure_file_exists_or_download_http_with_timeout(
144    filepath: &Path,
145    url: &str,
146    checksums: Option<&Path>,
147    timeout_secs: u64,
148) -> anyhow::Result<()> {
149    ensure_file_exists_or_download_http_with_config(
150        filepath,
151        url,
152        checksums,
153        timeout_secs,
154        None,
155        None,
156    )
157}
158
159/// Ensures that a file exists at the specified path by downloading it if necessary,
160/// with custom timeout, retry config, and initial jitter delay.
161///
162/// # Parameters
163///
164/// - `filepath`: The path where the file should exist.
165/// - `url`: The URL to download from if the file doesn't exist.
166/// - `checksums`: Optional path to checksums file for verification.
167/// - `timeout_secs`: Timeout in seconds for response headers and each body read.
168/// - `retry_config`: Optional retry configuration. Defaults retry up to five times without a total deadline.
169/// - `initial_jitter_ms`: Optional initial jitter delay in milliseconds before download (defaults to 100-600ms if None).
170///
171/// # Errors
172///
173/// Returns an error if:
174/// - The HTTP request cannot be sent or returns a non-success status code after retries.
175/// - Any I/O operation fails during file creation, reading, or writing.
176/// - Checksum verification or JSON parsing fails.
177/// - A download is required and the retry configuration is invalid.
178pub fn ensure_file_exists_or_download_http_with_config(
179    filepath: &Path,
180    url: &str,
181    checksums: Option<&Path>,
182    timeout_secs: u64,
183    retry_config: Option<RetryConfig>,
184    initial_jitter_ms: Option<u64>,
185) -> anyhow::Result<()> {
186    // Local/cached file path: accept the file if it exists, updating the
187    // checksum record when it differs (e.g. after local regeneration).
188    // This is intentionally lenient - download verification below is strict.
189    if filepath.exists() {
190        println!("File already exists (local/cached): {}", filepath.display());
191
192        if let Some(checksums_file) = checksums {
193            let _guard = lock_large_checksums();
194
195            if verify_sha256_checksum(filepath, checksums_file)? {
196                println!("Checksum verified");
197                return Ok(());
198            }
199
200            let new_checksum = calculate_sha256(filepath)?;
201            println!("Updating checksum for local file: {new_checksum}");
202            update_sha256_checksums(filepath, checksums_file, &new_checksum)?;
203            return Ok(());
204        }
205        return Ok(());
206    }
207
208    // Add a small random delay to avoid bursting the remote server when
209    // many downloads start concurrently. Can be disabled by passing Some(0).
210    if let Some(jitter_ms) = initial_jitter_ms {
211        if jitter_ms > 0 {
212            sleep(Duration::from_millis(jitter_ms));
213        }
214    } else {
215        let jitter_delay = {
216            let mut r = rng();
217            Duration::from_millis(r.random_range(100..=600))
218        };
219        sleep(jitter_delay);
220    }
221
222    download_file(filepath, url, timeout_secs, retry_config.clone())?;
223
224    // Verify checksum after download, retry once on mismatch (corrupt download)
225    if let Some(checksums_file) = checksums {
226        let guard = lock_large_checksums();
227
228        if !verify_sha256_checksum(filepath, checksums_file)? {
229            let actual = calculate_sha256(filepath)?;
230            println!("Checksum mismatch after download (calculated {actual}), retrying...");
231            remove_file(filepath)?;
232            drop(guard);
233
234            download_file(filepath, url, timeout_secs, retry_config)?;
235
236            let _guard = lock_large_checksums();
237
238            if !verify_sha256_checksum(filepath, checksums_file)? {
239                let actual = calculate_sha256(filepath)?;
240                remove_file(filepath)?;
241                anyhow::bail!(
242                    "Checksum mismatch after retry for {} (calculated {actual})",
243                    filepath.file_name().unwrap_or_default().display(),
244                );
245            }
246        }
247    }
248
249    Ok(())
250}
251
252fn download_file(
253    filepath: &Path,
254    url: &str,
255    timeout_secs: u64,
256    retry_config: Option<RetryConfig>,
257) -> anyhow::Result<()> {
258    // Validate HTTPS for security in production builds,
259    // HTTP is intentionally allowed in test builds for local test servers (127.0.0.1),
260    // CodeQL flags this as "non-https-url" but it's a deliberate design choice for testkit.
261    #[cfg(not(test))]
262    if !url.starts_with("https://") {
263        anyhow::bail!("URL must use HTTPS protocol for security: {url}");
264    }
265
266    println!("Downloading file from {url} to {}", filepath.display());
267
268    if let Some(parent) = filepath.parent() {
269        std::fs::create_dir_all(parent)?;
270    }
271
272    let runtime = tokio::runtime::Builder::new_current_thread()
273        .enable_all()
274        .build()?;
275    let client = HttpClient::builder().build()?;
276    let timeout = Duration::from_secs(timeout_secs);
277
278    let cfg = retry_config.unwrap_or(RetryConfig {
279        max_retries: 5,
280        initial_delay_ms: 1_000,
281        max_delay_ms: 10_000,
282        backoff_factor: 2.0,
283        jitter_ms: 1_000,
284        operation_timeout_ms: None,
285        immediate_first: false,
286        max_elapsed_ms: None,
287    });
288
289    remove_stale_partials(filepath);
290    let partial_path = partial_path_for(filepath);
291
292    let op = || async {
293        // Discard any leftover partial from a prior attempt before writing
294        let _ = remove_file(&partial_path);
295
296        match async {
297            Ok::<_, anyhow::Error>(
298                tokio::time::timeout(timeout, client.get_stream(url.to_owned())).await??,
299            )
300        }
301        .await
302        {
303            Ok(mut response) => {
304                let status = response.status();
305                if status.is_success() {
306                    store_response_body(&mut response, timeout, &partial_path, filepath).await
307                } else if status.is_server_error()
308                    || status.as_u16() == 429
309                    || status.as_u16() == 408
310                {
311                    println!("HTTP error {status}, retrying...");
312                    Err(DownloadError::Retryable(format!("HTTP {status}")))
313                } else {
314                    // Preserve existing error text used by tests
315                    Err(DownloadError::NonRetryable(format!(
316                        "Client error: HTTP {status}"
317                    )))
318                }
319            }
320            Err(e) => {
321                println!("Request failed: {e}");
322                Err(DownloadError::Retryable(e.to_string()))
323            }
324        }
325    };
326
327    let should_retry = |e: &DownloadError| matches!(e, DownloadError::Retryable(_));
328
329    let manager = RetryManager::new(cfg);
330
331    let result = runtime.block_on(
332        manager
333            .invocation("download_file", op, should_retry, |e| {
334                DownloadError::Retryable(e.to_string())
335            })
336            .execute(),
337    );
338
339    if result.is_err() {
340        let _ = remove_file(&partial_path);
341    }
342
343    result.map_err(|e| anyhow::anyhow!(e.to_string()))
344}
345
346// Stream errors are transient (Retryable), local write errors like disk full are not
347async fn store_response_body(
348    response: &mut HttpResponseStream,
349    timeout: Duration,
350    partial_path: &Path,
351    filepath: &Path,
352) -> Result<(), DownloadError> {
353    let mut out =
354        File::create(partial_path).map_err(|e| DownloadError::NonRetryable(e.to_string()))?;
355
356    let copied = async {
357        while let Some(chunk) = tokio::time::timeout(timeout, response.chunk())
358            .await
359            .map_err(|e| DownloadError::Retryable(format!("body stream error: {e}")))?
360            .map_err(|e| DownloadError::Retryable(format!("body stream error: {e}")))?
361        {
362            out.write_all(&chunk).map_err(|e| {
363                DownloadError::NonRetryable(format!("write {} failed: {e}", partial_path.display()))
364            })?;
365        }
366
367        // Persist before the rename makes the file visible at `filepath`
368        out.sync_all().map_err(|e| {
369            DownloadError::NonRetryable(format!("sync {} failed: {e}", partial_path.display()))
370        })
371    }
372    .await;
373
374    drop(out);
375
376    if let Err(e) = copied {
377        let _ = remove_file(partial_path);
378        return Err(e);
379    }
380
381    if let Err(e) = std::fs::rename(partial_path, filepath) {
382        let _ = remove_file(partial_path);
383        return Err(DownloadError::NonRetryable(format!(
384            "rename {} -> {} failed: {e}",
385            partial_path.display(),
386            filepath.display(),
387        )));
388    }
389
390    println!("File downloaded to {}", filepath.display());
391    Ok(())
392}
393
394// Active downloads keep writing and time out well within this age, so old partials are orphaned
395fn remove_stale_partials(filepath: &Path) {
396    const STALE_PARTIAL_AGE: Duration = Duration::from_secs(3600);
397
398    let (Some(parent), Some(file_name)) = (filepath.parent(), filepath.file_name()) else {
399        return;
400    };
401
402    // A bare file name has an empty parent, which names the current directory
403    let parent = if parent.as_os_str().is_empty() {
404        Path::new(".")
405    } else {
406        parent
407    };
408
409    let prefix = format!("{}.partial.", file_name.to_string_lossy());
410
411    let entries = match std::fs::read_dir(parent) {
412        Ok(entries) => entries,
413        Err(e) => {
414            println!(
415                "Failed to scan {} for stale partial downloads: {e}",
416                parent.display()
417            );
418            return;
419        }
420    };
421
422    for entry in entries.flatten() {
423        let name = entry.file_name();
424
425        let Some(suffix) = name.to_str().and_then(|name| name.strip_prefix(&prefix)) else {
426            continue;
427        };
428
429        // Only names produced by `partial_path_for`: `<pid>.<nanos>`
430        let is_partial_name = suffix.split_once('.').is_some_and(|(pid, nanos)| {
431            pid.parse::<u32>().is_ok() && nanos.parse::<u128>().is_ok()
432        });
433
434        if !is_partial_name {
435            continue;
436        }
437
438        let is_stale = entry
439            .metadata()
440            .and_then(|metadata| metadata.modified())
441            .ok()
442            .and_then(|modified| modified.elapsed().ok())
443            .is_some_and(|age| age >= STALE_PARTIAL_AGE);
444
445        if !is_stale {
446            continue;
447        }
448
449        let path = entry.path();
450        match remove_file(&path) {
451            Ok(()) => println!("Removed stale partial download {}", path.display()),
452            Err(e) => println!(
453                "Failed to remove stale partial download {}: {e}",
454                path.display()
455            ),
456        }
457    }
458}
459
460fn partial_path_for(filepath: &Path) -> PathBuf {
461    // Suffix with pid + nanos so the staging path is unique per invocation,
462    // preventing collision with a real user file or another concurrent downloader.
463    let nanos = std::time::SystemTime::now()
464        .duration_since(std::time::UNIX_EPOCH)
465        .map_or(0, |d| d.as_nanos());
466    let mut p: OsString = filepath.as_os_str().to_owned();
467    p.push(format!(".partial.{}.{}", std::process::id(), nanos));
468    PathBuf::from(p)
469}
470
471fn calculate_sha256(filepath: &Path) -> anyhow::Result<String> {
472    let mut file = File::open(filepath)?;
473    let mut ctx = digest::Context::new(&digest::SHA256);
474    let mut buffer = [0u8; 4096];
475
476    loop {
477        let count = file.read(&mut buffer)?;
478        if count == 0 {
479            break;
480        }
481        ctx.update(&buffer[..count]);
482    }
483
484    let digest = ctx.finish();
485    Ok(hex::encode(digest.as_ref()))
486}
487
488fn verify_sha256_checksum(filepath: &Path, checksums: &Path) -> anyhow::Result<bool> {
489    let file = File::open(checksums)?;
490    let reader = BufReader::new(file);
491    let manifest: Value = serde_json::from_reader(reader)?;
492
493    let filename = checksum_key(filepath)?;
494    if let Some(expected_checksum) = manifest.get(filename) {
495        let expected_checksum_str = expected_checksum.as_str().with_context(|| {
496            format!(
497                "checksum for {filename} in {} is not a string",
498                checksums.display()
499            )
500        })?;
501
502        let expected_hash = expected_checksum_str
503            .strip_prefix("sha256:")
504            .unwrap_or(expected_checksum_str);
505        let calculated_checksum = calculate_sha256(filepath)?;
506        if expected_hash == calculated_checksum {
507            return Ok(true);
508        }
509    }
510
511    Ok(false)
512}
513
514fn update_sha256_checksums(
515    filepath: &Path,
516    checksums_file: &Path,
517    new_checksum: &str,
518) -> anyhow::Result<()> {
519    let mut checksums_map = if checksums_file.exists() {
520        let file = File::open(checksums_file)?;
521        let reader = BufReader::new(file);
522        match serde_json::from_reader(reader)? {
523            Value::Object(map) => map,
524            _ => anyhow::bail!(
525                "checksums file {} is not a JSON object",
526                checksums_file.display()
527            ),
528        }
529    } else {
530        serde_json::Map::new()
531    };
532
533    // Add or update the checksum
534    let filename = checksum_key(filepath)?.to_string();
535    let prefixed_checksum = format!("sha256:{new_checksum}");
536    checksums_map.insert(filename, Value::String(prefixed_checksum));
537
538    // Replace via a synced sibling file so a crash or full disk never truncates the manifest
539    let temp_path = partial_path_for(checksums_file);
540    let result = write_checksums(&temp_path, &Value::Object(checksums_map))
541        .and_then(|()| std::fs::rename(&temp_path, checksums_file).map_err(Into::into));
542
543    if result.is_err() {
544        let _ = remove_file(&temp_path);
545    }
546
547    result
548}
549
550fn write_checksums(path: &Path, checksums: &Value) -> anyhow::Result<()> {
551    let mut writer = BufWriter::new(File::create(path)?);
552    serde_json::to_writer_pretty(&mut writer, checksums)?;
553    writer.into_inner()?.sync_all()?;
554    Ok(())
555}
556
557fn checksum_key(filepath: &Path) -> anyhow::Result<&str> {
558    filepath
559        .file_name()
560        .and_then(|name| name.to_str())
561        .with_context(|| format!("no UTF-8 file name in {}", filepath.display()))
562}
563
564#[cfg(test)]
565mod tests {
566    use std::{
567        fs,
568        io::{BufWriter, Write},
569        net::SocketAddr,
570        sync::{
571            Arc,
572            atomic::{AtomicUsize, Ordering},
573        },
574    };
575
576    use axum::{Router, http::StatusCode, routing::get, serve};
577    use rstest::*;
578    use serde_json::{json, to_writer};
579    use tempfile::TempDir;
580    use tokio::{
581        io::{AsyncReadExt, AsyncWriteExt},
582        net::TcpListener,
583        task,
584        time::{Duration, sleep},
585    };
586
587    use super::*;
588
589    /// Creates a fast, deterministic retry config for tests.
590    /// Uses very short delays to make tests run quickly without introducing flakiness.
591    fn test_retry_config() -> RetryConfig {
592        RetryConfig {
593            max_retries: 5,
594            initial_delay_ms: 10,
595            max_delay_ms: 50,
596            backoff_factor: 2.0,
597            jitter_ms: 5,
598            operation_timeout_ms: Some(500),
599            immediate_first: false,
600            max_elapsed_ms: Some(2000),
601        }
602    }
603
604    async fn setup_test_server(
605        server_content: Option<String>,
606        status_code: StatusCode,
607    ) -> SocketAddr {
608        let server_content = Arc::new(server_content);
609        let server_content_clone = server_content.clone();
610        let app = Router::new().route(
611            "/testfile.txt",
612            get(move || {
613                let server_content = server_content_clone.clone();
614                async move {
615                    let response_body = match &*server_content {
616                        Some(content) => content.clone(),
617                        None => "File not found".to_string(),
618                    };
619                    (status_code, response_body)
620                }
621            }),
622        );
623
624        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
625        let addr = listener.local_addr().unwrap();
626        let server = serve(listener, app);
627
628        task::spawn(async move {
629            if let Err(e) = server.await {
630                eprintln!("server error: {e}");
631            }
632        });
633
634        sleep(Duration::from_millis(100)).await;
635
636        addr
637    }
638
639    #[rstest]
640    #[case::missing(None, "verified fixture", true)]
641    #[case::cached(Some("verified fixture"), "verified fixture", true)]
642    #[case::stale_cached(Some("stale fixture"), "verified fixture", true)]
643    #[case::corrupt_download(None, "corrupt download", false)]
644    #[tokio::test]
645    async fn test_prepare_test_data_file(
646        #[case] cached: Option<&str>,
647        #[case] downloaded: &str,
648        #[case] valid: bool,
649    ) {
650        let temp_dir = TempDir::new().unwrap();
651        let filepath = temp_dir.path().join("testfile.txt");
652        let checksums = temp_dir.path().join("checksums.json");
653        let content = "verified fixture";
654        let manifest = serde_json::to_vec(&json!({
655            "testfile.txt": format!("sha256:{}", calculate_sha256_bytes(content.as_bytes()))
656        }))
657        .unwrap();
658        fs::write(&checksums, &manifest).unwrap();
659        if let Some(cached) = cached {
660            fs::write(&filepath, cached).unwrap();
661        }
662
663        let url = if cached == Some(content) {
664            "http://127.0.0.1:0/testfile.txt".to_string()
665        } else {
666            let addr = setup_test_server(Some(downloaded.to_string()), StatusCode::OK).await;
667            format!("http://{addr}/testfile.txt")
668        };
669        let filepath_clone = filepath.clone();
670        let checksums_clone = checksums.clone();
671        let result = tokio::task::spawn_blocking(move || {
672            prepare_test_data_file(&filepath_clone, &url, &checksums_clone)
673        })
674        .await
675        .unwrap();
676
677        if valid {
678            result.unwrap();
679            assert_eq!(fs::read_to_string(&filepath).unwrap(), content);
680        } else {
681            assert_eq!(
682                result.unwrap_err().to_string(),
683                format!("Checksum mismatch for {}", filepath.display()),
684            );
685            assert!(!filepath.exists());
686        }
687        assert_eq!(fs::read(&checksums).unwrap(), manifest);
688    }
689
690    #[tokio::test]
691    async fn test_file_already_exists() {
692        let temp_dir = TempDir::new().unwrap();
693        let file_path = temp_dir.path().join("testfile.txt");
694        fs::write(&file_path, "Existing file content").unwrap();
695
696        let url = "http://example.com/testfile.txt".to_string();
697        let result = ensure_file_exists_or_download_http(&file_path, &url, None, Some(5));
698
699        assert!(result.is_ok());
700        let content = fs::read_to_string(&file_path).unwrap();
701        assert_eq!(content, "Existing file content");
702    }
703
704    #[tokio::test]
705    async fn test_download_file_success() {
706        let temp_dir = TempDir::new().unwrap();
707        let filepath = temp_dir.path().join("testfile.txt");
708        let filepath_clone = filepath.clone();
709
710        let server_content = "Server file content".to_string();
711        let status_code = StatusCode::OK;
712        let addr = setup_test_server(Some(server_content.clone()), status_code).await;
713        let url = format!("http://{addr}/testfile.txt");
714
715        let result = tokio::task::spawn_blocking(move || {
716            ensure_file_exists_or_download_http_with_config(
717                &filepath_clone,
718                &url,
719                None,
720                5,
721                Some(test_retry_config()),
722                Some(0),
723            )
724        })
725        .await
726        .unwrap();
727
728        assert!(result.is_ok());
729        let content = fs::read_to_string(&filepath).unwrap();
730        assert_eq!(content, server_content);
731    }
732
733    #[tokio::test]
734    async fn test_download_file_not_found() {
735        let temp_dir = TempDir::new().unwrap();
736        let file_path = temp_dir.path().join("testfile.txt");
737
738        let server_content = None;
739        let status_code = StatusCode::NOT_FOUND;
740        let addr = setup_test_server(server_content, status_code).await;
741        let url = format!("http://{addr}/testfile.txt");
742
743        let result = tokio::task::spawn_blocking(move || {
744            ensure_file_exists_or_download_http_with_config(
745                &file_path,
746                &url,
747                None,
748                1,
749                Some(test_retry_config()),
750                Some(0),
751            )
752        })
753        .await
754        .unwrap();
755
756        assert!(result.is_err());
757        let err_msg = format!("{}", result.unwrap_err());
758        assert!(
759            err_msg.contains("Client error: HTTP"),
760            "Unexpected error message: {err_msg}"
761        );
762    }
763
764    #[tokio::test]
765    async fn test_network_error() {
766        let temp_dir = TempDir::new().unwrap();
767        let file_path = temp_dir.path().join("testfile.txt");
768
769        // Use an unreachable address to simulate a network error
770        let url = "http://127.0.0.1:0/testfile.txt".to_string();
771
772        let result = tokio::task::spawn_blocking(move || {
773            ensure_file_exists_or_download_http_with_config(
774                &file_path,
775                &url,
776                None,
777                2,
778                Some(test_retry_config()),
779                Some(0),
780            )
781        })
782        .await
783        .unwrap();
784
785        assert!(result.is_err());
786        let err_msg = format!("{}", result.unwrap_err());
787        assert!(
788            err_msg.contains("error"),
789            "Unexpected error message: {err_msg}"
790        );
791    }
792
793    #[tokio::test]
794    async fn test_retry_then_success_on_500() {
795        let temp_dir = TempDir::new().unwrap();
796        let filepath = temp_dir.path().join("testfile.txt");
797        let filepath_clone = filepath.clone();
798
799        let counter = Arc::new(AtomicUsize::new(0));
800        let counter_clone = counter.clone();
801
802        let app = Router::new().route(
803            "/testfile.txt",
804            get(move || {
805                let c = counter_clone.clone();
806                async move {
807                    let n = c.fetch_add(1, Ordering::SeqCst);
808                    if n < 2 {
809                        (StatusCode::INTERNAL_SERVER_ERROR, "temporary error")
810                    } else {
811                        (StatusCode::OK, "eventual success")
812                    }
813                }
814            }),
815        );
816
817        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
818        let addr = listener.local_addr().unwrap();
819        let server = serve(listener, app);
820        task::spawn(async move {
821            let _ = server.await;
822        });
823        sleep(Duration::from_millis(100)).await;
824
825        let url = format!("http://{addr}/testfile.txt");
826
827        let result = tokio::task::spawn_blocking(move || {
828            ensure_file_exists_or_download_http_with_config(
829                &filepath_clone,
830                &url,
831                None,
832                5,
833                Some(test_retry_config()),
834                Some(0),
835            )
836        })
837        .await
838        .unwrap();
839
840        assert!(result.is_ok());
841        let content = std::fs::read_to_string(&filepath).unwrap();
842        assert_eq!(content, "eventual success");
843        assert!(counter.load(Ordering::SeqCst) >= 2);
844    }
845
846    #[tokio::test]
847    async fn test_retry_then_success_on_429() {
848        let temp_dir = TempDir::new().unwrap();
849        let filepath = temp_dir.path().join("testfile.txt");
850        let filepath_clone = filepath.clone();
851
852        let counter = Arc::new(AtomicUsize::new(0));
853        let counter_clone = counter.clone();
854
855        let app = Router::new().route(
856            "/testfile.txt",
857            get(move || {
858                let c = counter_clone.clone();
859                async move {
860                    let n = c.fetch_add(1, Ordering::SeqCst);
861                    if n < 1 {
862                        (StatusCode::TOO_MANY_REQUESTS, "rate limited")
863                    } else {
864                        (StatusCode::OK, "ok after retry")
865                    }
866                }
867            }),
868        );
869
870        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
871        let addr = listener.local_addr().unwrap();
872        let server = serve(listener, app);
873        task::spawn(async move {
874            let _ = server.await;
875        });
876        sleep(Duration::from_millis(100)).await;
877
878        let url = format!("http://{addr}/testfile.txt");
879
880        let result = tokio::task::spawn_blocking(move || {
881            ensure_file_exists_or_download_http_with_config(
882                &filepath_clone,
883                &url,
884                None,
885                5,
886                Some(test_retry_config()),
887                Some(0),
888            )
889        })
890        .await
891        .unwrap();
892
893        assert!(result.is_ok());
894        let content = std::fs::read_to_string(&filepath).unwrap();
895        assert_eq!(content, "ok after retry");
896        assert!(counter.load(Ordering::SeqCst) >= 2);
897    }
898
899    #[tokio::test]
900    async fn test_no_retry_on_404() {
901        let temp_dir = TempDir::new().unwrap();
902        let filepath = temp_dir.path().join("testfile.txt");
903        let filepath_clone = filepath.clone();
904
905        let counter = Arc::new(AtomicUsize::new(0));
906        let counter_clone = counter.clone();
907
908        let app = Router::new().route(
909            "/testfile.txt",
910            get(move || {
911                let c = counter_clone.clone();
912                async move {
913                    c.fetch_add(1, Ordering::SeqCst);
914                    (StatusCode::NOT_FOUND, "missing")
915                }
916            }),
917        );
918
919        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
920        let addr = listener.local_addr().unwrap();
921        let server = serve(listener, app);
922        task::spawn(async move {
923            let _ = server.await;
924        });
925        sleep(Duration::from_millis(100)).await;
926
927        let url = format!("http://{addr}/testfile.txt");
928
929        let result = tokio::task::spawn_blocking(move || {
930            ensure_file_exists_or_download_http_with_config(
931                &filepath_clone,
932                &url,
933                None,
934                5,
935                Some(test_retry_config()),
936                Some(0),
937            )
938        })
939        .await
940        .unwrap();
941
942        assert!(result.is_err());
943        assert_eq!(counter.load(Ordering::SeqCst), 1, "should not retry on 404");
944    }
945
946    #[tokio::test]
947    async fn test_checksum_mismatch_retry_then_success() {
948        let temp_dir = TempDir::new().unwrap();
949        let filepath = temp_dir.path().join("testfile.txt");
950        let filepath_clone = filepath.clone();
951
952        let good_content = "correct content";
953        let good_checksum = calculate_sha256_bytes(good_content.as_bytes());
954
955        let checksums_path = temp_dir.path().join("checksums.json");
956        let checksums_data = json!({
957            "testfile.txt": format!("sha256:{good_checksum}")
958        });
959        let checksums_file = File::create(&checksums_path).unwrap();
960        to_writer(BufWriter::new(checksums_file), &checksums_data).unwrap();
961        let checksums_clone = checksums_path.clone();
962
963        // First request returns corrupt data, second returns correct data
964        let counter = Arc::new(AtomicUsize::new(0));
965        let counter_clone = counter.clone();
966
967        let app = Router::new().route(
968            "/testfile.txt",
969            get(move || {
970                let c = counter_clone.clone();
971                async move {
972                    let n = c.fetch_add(1, Ordering::SeqCst);
973                    if n == 0 {
974                        (StatusCode::OK, "corrupt data")
975                    } else {
976                        (StatusCode::OK, "correct content")
977                    }
978                }
979            }),
980        );
981
982        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
983        let addr = listener.local_addr().unwrap();
984        let server = serve(listener, app);
985        task::spawn(async move {
986            let _ = server.await;
987        });
988        sleep(Duration::from_millis(100)).await;
989
990        let url = format!("http://{addr}/testfile.txt");
991
992        let result = tokio::task::spawn_blocking(move || {
993            ensure_file_exists_or_download_http_with_config(
994                &filepath_clone,
995                &url,
996                Some(&checksums_clone),
997                5,
998                Some(test_retry_config()),
999                Some(0),
1000            )
1001        })
1002        .await
1003        .unwrap();
1004
1005        assert!(result.is_ok());
1006        let content = fs::read_to_string(&filepath).unwrap();
1007        assert_eq!(content, good_content);
1008        assert_eq!(counter.load(Ordering::SeqCst), 2);
1009    }
1010
1011    #[tokio::test]
1012    async fn test_checksum_mismatch_retry_then_fail() {
1013        let temp_dir = TempDir::new().unwrap();
1014        let filepath = temp_dir.path().join("testfile.txt");
1015        let filepath_clone = filepath.clone();
1016
1017        // Checksum for content that the server will never return
1018        let checksums_path = temp_dir.path().join("checksums.json");
1019        let checksums_data = json!({
1020            "testfile.txt": "sha256:0000000000000000000000000000000000000000000000000000000000000000"
1021        });
1022        let checksums_file = File::create(&checksums_path).unwrap();
1023        to_writer(BufWriter::new(checksums_file), &checksums_data).unwrap();
1024        let checksums_clone = checksums_path.clone();
1025
1026        let counter = Arc::new(AtomicUsize::new(0));
1027        let counter_clone = counter.clone();
1028
1029        let app = Router::new().route(
1030            "/testfile.txt",
1031            get(move || {
1032                let c = counter_clone.clone();
1033                async move {
1034                    c.fetch_add(1, Ordering::SeqCst);
1035                    (StatusCode::OK, "always wrong content")
1036                }
1037            }),
1038        );
1039
1040        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1041        let addr = listener.local_addr().unwrap();
1042        let server = serve(listener, app);
1043        task::spawn(async move {
1044            let _ = server.await;
1045        });
1046        sleep(Duration::from_millis(100)).await;
1047
1048        let url = format!("http://{addr}/testfile.txt");
1049
1050        let result = tokio::task::spawn_blocking(move || {
1051            ensure_file_exists_or_download_http_with_config(
1052                &filepath_clone,
1053                &url,
1054                Some(&checksums_clone),
1055                5,
1056                Some(test_retry_config()),
1057                Some(0),
1058            )
1059        })
1060        .await
1061        .unwrap();
1062
1063        assert!(result.is_err());
1064        let err_msg = format!("{}", result.unwrap_err());
1065        assert!(err_msg.contains("Checksum mismatch after retry"));
1066        assert_eq!(
1067            counter.load(Ordering::SeqCst),
1068            2,
1069            "should download exactly twice"
1070        );
1071        assert!(!filepath.exists(), "corrupt file should be cleaned up");
1072    }
1073
1074    /// First call overstates Content-Length to fail the body read mid-stream; subsequent calls succeed.
1075    async fn truncated_then_full_server(good_body: &'static str) -> (SocketAddr, Arc<AtomicUsize>) {
1076        use tokio::io::{AsyncReadExt, AsyncWriteExt};
1077
1078        let counter = Arc::new(AtomicUsize::new(0));
1079        let counter_clone = counter.clone();
1080
1081        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1082        let addr = listener.local_addr().unwrap();
1083
1084        task::spawn(async move {
1085            loop {
1086                let Ok((mut sock, _)) = listener.accept().await else {
1087                    break;
1088                };
1089                let counter = counter_clone.clone();
1090
1091                task::spawn(async move {
1092                    let mut buf = [0u8; 1024];
1093                    let _ = sock.read(&mut buf).await;
1094                    let n = counter.fetch_add(1, Ordering::SeqCst);
1095
1096                    if n == 0 {
1097                        let resp = b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\nshort";
1098                        let _ = sock.write_all(resp).await;
1099                    } else {
1100                        let resp = format!(
1101                            "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
1102                            good_body.len(),
1103                            good_body,
1104                        );
1105                        let _ = sock.write_all(resp.as_bytes()).await;
1106                    }
1107                    let _ = sock.shutdown().await;
1108                });
1109            }
1110        });
1111
1112        sleep(Duration::from_millis(100)).await;
1113        (addr, counter)
1114    }
1115
1116    fn count_partial_siblings(filepath: &Path) -> usize {
1117        let parent = filepath.parent().unwrap();
1118        let stem = filepath.file_name().unwrap().to_string_lossy().into_owned();
1119        let prefix = format!("{stem}.partial.");
1120        fs::read_dir(parent)
1121            .unwrap()
1122            .filter_map(Result::ok)
1123            .filter(|e| e.file_name().to_string_lossy().starts_with(&prefix))
1124            .count()
1125    }
1126
1127    #[rstest]
1128    #[case::operation(false, "Retryable error: Timed out after 40ms")]
1129    #[case::elapsed(true, "Retryable error: Retry budget exceeded (1/1)")]
1130    #[tokio::test]
1131    async fn test_download_retry_deadline_cleans_partial(
1132        #[case] elapsed_budget: bool,
1133        #[case] expected_error: &str,
1134        #[values(false, true)] body_started: bool,
1135    ) {
1136        let directory = TempDir::new().unwrap();
1137        let filepath = directory.path().join("deadline.txt");
1138        let destination = filepath.clone();
1139        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1140        let addr = listener.local_addr().unwrap();
1141        let peer = tokio::spawn(async move {
1142            let (mut socket, _) = listener.accept().await.unwrap();
1143            let mut request = Vec::new();
1144            while !request.ends_with(b"\r\n\r\n") {
1145                request.push(socket.read_u8().await.unwrap());
1146            }
1147
1148            if body_started {
1149                socket
1150                    .write_all(
1151                        b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\na",
1152                    )
1153                    .await
1154                    .unwrap();
1155            }
1156
1157            std::future::pending::<()>().await;
1158        });
1159
1160        let result = tokio::task::spawn_blocking(move || {
1161            let config = RetryConfig {
1162                max_retries: 0,
1163                operation_timeout_ms: if elapsed_budget { None } else { Some(40) },
1164                max_elapsed_ms: if elapsed_budget { Some(40) } else { None },
1165                ..test_retry_config()
1166            };
1167
1168            download_file(
1169                &destination,
1170                &format!("http://{addr}/deadline"),
1171                1,
1172                Some(config),
1173            )
1174            .map_err(|e| e.to_string())
1175        })
1176        .await
1177        .unwrap();
1178
1179        peer.abort();
1180        let _ = peer.await;
1181
1182        assert_eq!(result, Err(expected_error.to_string()));
1183        assert!(!filepath.exists());
1184        assert_eq!(count_partial_siblings(&filepath), 0);
1185    }
1186
1187    #[tokio::test]
1188    async fn test_download_retry_budget_prevents_another_request() {
1189        let directory = TempDir::new().unwrap();
1190        let filepath = directory.path().join("budget.txt");
1191        let destination = filepath.clone();
1192        let requests = Arc::new(AtomicUsize::new(0));
1193        let counter = requests.clone();
1194        let app = Router::new().route(
1195            "/budget",
1196            get(move || {
1197                counter.fetch_add(1, Ordering::SeqCst);
1198                async { StatusCode::INTERNAL_SERVER_ERROR }
1199            }),
1200        );
1201
1202        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1203        let addr = listener.local_addr().unwrap();
1204        let server = tokio::spawn(async move { serve(listener, app).await.unwrap() });
1205
1206        let result = tokio::task::spawn_blocking(move || {
1207            let config = RetryConfig {
1208                max_retries: 2,
1209                initial_delay_ms: 100,
1210                max_delay_ms: 100,
1211                jitter_ms: 0,
1212                operation_timeout_ms: None,
1213                max_elapsed_ms: Some(40),
1214                ..test_retry_config()
1215            };
1216
1217            download_file(
1218                &destination,
1219                &format!("http://{addr}/budget"),
1220                1,
1221                Some(config),
1222            )
1223            .map_err(|e| e.to_string())
1224        })
1225        .await
1226        .unwrap();
1227
1228        server.abort();
1229        let _ = server.await;
1230
1231        assert_eq!(requests.load(Ordering::SeqCst), 1);
1232        assert_eq!(
1233            result,
1234            Err("Retryable error: Retry budget exceeded (2/3)".to_string())
1235        );
1236        assert!(!filepath.exists());
1237        assert_eq!(count_partial_siblings(&filepath), 0);
1238    }
1239
1240    #[tokio::test]
1241    async fn test_download_immediate_first_retains_later_delay() {
1242        let directory = TempDir::new().unwrap();
1243        let filepath = directory.path().join("immediate.txt");
1244        let destination = filepath.clone();
1245        let requests = Arc::new(AtomicUsize::new(0));
1246        let counter = requests.clone();
1247        let app = Router::new().route(
1248            "/immediate",
1249            get(move || {
1250                let attempt = counter.fetch_add(1, Ordering::SeqCst);
1251                async move {
1252                    if attempt < 2 {
1253                        (StatusCode::INTERNAL_SERVER_ERROR, "retry")
1254                    } else {
1255                        (StatusCode::OK, "complete")
1256                    }
1257                }
1258            }),
1259        );
1260
1261        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1262        let addr = listener.local_addr().unwrap();
1263        let server = tokio::spawn(async move { serve(listener, app).await.unwrap() });
1264
1265        let (result, elapsed) = tokio::task::spawn_blocking(move || {
1266            let config = RetryConfig {
1267                max_retries: 2,
1268                initial_delay_ms: 100,
1269                max_delay_ms: 100,
1270                jitter_ms: 0,
1271                immediate_first: true,
1272                operation_timeout_ms: None,
1273                max_elapsed_ms: None,
1274                ..test_retry_config()
1275            };
1276
1277            let start = std::time::Instant::now();
1278            let result = download_file(
1279                &destination,
1280                &format!("http://{addr}/immediate"),
1281                1,
1282                Some(config),
1283            );
1284            (result, start.elapsed())
1285        })
1286        .await
1287        .unwrap();
1288
1289        server.abort();
1290        let _ = server.await;
1291
1292        result.unwrap();
1293        assert_eq!(requests.load(Ordering::SeqCst), 3);
1294        assert!(elapsed >= Duration::from_millis(100));
1295        assert_eq!(fs::read_to_string(&filepath).unwrap(), "complete");
1296        assert_eq!(count_partial_siblings(&filepath), 0);
1297    }
1298
1299    #[rstest]
1300    #[case::progressing(0, 400, Ok(()), false)]
1301    #[case::progressing_default(0, 400, Ok(()), true)]
1302    #[case::stalled_headers(1200, 0, Err("Retryable error: deadline has elapsed"), false)]
1303    #[case::stalled_body(
1304        0,
1305        1200,
1306        Err("Retryable error: body stream error: deadline has elapsed"),
1307        false
1308    )]
1309    #[tokio::test]
1310    async fn download_timeout_applies_to_each_read(
1311        #[case] header_delay_ms: u64,
1312        #[case] chunk_delay_ms: u64,
1313        #[case] expected: Result<(), &str>,
1314        #[case] default_retry: bool,
1315    ) {
1316        use tokio::io::{AsyncReadExt, AsyncWriteExt};
1317
1318        let temp_dir = TempDir::new().unwrap();
1319        let filepath = temp_dir.path().join("streamed.txt");
1320        let destination = filepath.clone();
1321        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1322        let addr = listener.local_addr().unwrap();
1323        let peer = task::spawn(async move {
1324            let (mut socket, _) = listener.accept().await.unwrap();
1325            let mut request = Vec::new();
1326            while !request.ends_with(b"\r\n\r\n") {
1327                request.push(socket.read_u8().await.unwrap());
1328            }
1329            sleep(Duration::from_millis(header_delay_ms)).await;
1330
1331            if socket
1332                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\na")
1333                .await
1334                .is_err()
1335            {
1336                return;
1337            }
1338
1339            for byte in b"bcd" {
1340                sleep(Duration::from_millis(chunk_delay_ms)).await;
1341
1342                if socket.write_all(&[*byte]).await.is_err() {
1343                    return;
1344                }
1345            }
1346        });
1347        let result = task::spawn_blocking(move || {
1348            let cfg = RetryConfig {
1349                max_retries: 0,
1350                operation_timeout_ms: None,
1351                max_elapsed_ms: None,
1352                ..test_retry_config()
1353            };
1354
1355            let config = if default_retry { None } else { Some(cfg) };
1356            download_file(&destination, &format!("http://{addr}/streamed"), 1, config)
1357                .map_err(|e| e.to_string())
1358        })
1359        .await
1360        .unwrap();
1361        peer.await.unwrap();
1362
1363        assert_eq!(
1364            result.as_ref().map_err(String::as_str),
1365            expected.as_ref().map_err(|e| *e)
1366        );
1367        assert_eq!(count_partial_siblings(&filepath), 0);
1368        if expected.is_ok() {
1369            assert_eq!(fs::read(&filepath).unwrap(), b"abcd");
1370        } else {
1371            assert!(!filepath.exists());
1372        }
1373    }
1374
1375    #[tokio::test]
1376    async fn test_body_truncation_retries_and_recovers() {
1377        let temp_dir = TempDir::new().unwrap();
1378        let filepath = temp_dir.path().join("testfile.txt");
1379        let filepath_clone = filepath.clone();
1380
1381        let (addr, counter) = truncated_then_full_server("complete payload").await;
1382        let url = format!("http://{addr}/testfile.txt");
1383
1384        let result = tokio::task::spawn_blocking(move || {
1385            ensure_file_exists_or_download_http_with_config(
1386                &filepath_clone,
1387                &url,
1388                None,
1389                5,
1390                Some(test_retry_config()),
1391                Some(0),
1392            )
1393        })
1394        .await
1395        .unwrap();
1396
1397        assert!(result.is_ok(), "should retry past the truncated response");
1398        assert_eq!(counter.load(Ordering::SeqCst), 2);
1399        assert_eq!(
1400            count_partial_siblings(&filepath),
1401            0,
1402            "no .partial siblings must remain after success",
1403        );
1404        let content = fs::read_to_string(&filepath).unwrap();
1405        assert_eq!(content, "complete payload");
1406    }
1407
1408    #[tokio::test]
1409    async fn test_body_truncation_exhausts_retries_leaves_no_corrupt_file() {
1410        use tokio::io::{AsyncReadExt, AsyncWriteExt};
1411
1412        let temp_dir = TempDir::new().unwrap();
1413        let filepath = temp_dir.path().join("testfile.txt");
1414        let filepath_clone = filepath.clone();
1415
1416        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1417        let addr = listener.local_addr().unwrap();
1418
1419        task::spawn(async move {
1420            loop {
1421                let Ok((mut sock, _)) = listener.accept().await else {
1422                    break;
1423                };
1424
1425                task::spawn(async move {
1426                    let mut buf = [0u8; 1024];
1427                    let _ = sock.read(&mut buf).await;
1428                    let resp =
1429                        b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\nshort";
1430                    let _ = sock.write_all(resp).await;
1431                    let _ = sock.shutdown().await;
1432                });
1433            }
1434        });
1435
1436        sleep(Duration::from_millis(100)).await;
1437        let url = format!("http://{addr}/testfile.txt");
1438
1439        let result = tokio::task::spawn_blocking(move || {
1440            ensure_file_exists_or_download_http_with_config(
1441                &filepath_clone,
1442                &url,
1443                None,
1444                5,
1445                Some(test_retry_config()),
1446                Some(0),
1447            )
1448        })
1449        .await
1450        .unwrap();
1451
1452        assert!(result.is_err(), "all retries should fail");
1453        assert!(!filepath.exists(), "no corrupt file at final path");
1454        assert_eq!(
1455            count_partial_siblings(&filepath),
1456            0,
1457            "no .partial siblings may leak after exhausted retries",
1458        );
1459    }
1460
1461    #[rstest]
1462    fn test_partial_path_for_is_unique_and_marked() {
1463        let target = Path::new("/tmp/data.parquet");
1464        let stem = target.file_name().unwrap().to_string_lossy().into_owned();
1465        let expected_prefix = format!("{stem}.partial.");
1466
1467        let a = partial_path_for(target);
1468        let b = partial_path_for(target);
1469        assert_ne!(a, b, "partial paths must be unique per call");
1470        for p in [&a, &b] {
1471            let name = p.file_name().unwrap().to_string_lossy().into_owned();
1472            assert!(name.starts_with(&expected_prefix), "got {name}");
1473        }
1474    }
1475
1476    #[tokio::test]
1477    async fn test_unrelated_partial_sibling_not_clobbered() {
1478        let temp_dir = TempDir::new().unwrap();
1479        let filepath = temp_dir.path().join("testfile.txt");
1480        let filepath_clone = filepath.clone();
1481        // A user-owned file whose name happens to start with `<target>.partial`
1482        let bystander = temp_dir.path().join("testfile.txt.partial");
1483        fs::write(&bystander, b"do not touch").unwrap();
1484
1485        let server_content = "downloaded".to_string();
1486        let addr = setup_test_server(Some(server_content.clone()), StatusCode::OK).await;
1487        let url = format!("http://{addr}/testfile.txt");
1488
1489        let result = tokio::task::spawn_blocking(move || {
1490            ensure_file_exists_or_download_http_with_config(
1491                &filepath_clone,
1492                &url,
1493                None,
1494                5,
1495                Some(test_retry_config()),
1496                Some(0),
1497            )
1498        })
1499        .await
1500        .unwrap();
1501
1502        assert!(result.is_ok());
1503        assert_eq!(fs::read_to_string(&filepath).unwrap(), server_content);
1504        assert_eq!(
1505            fs::read_to_string(&bystander).unwrap(),
1506            "do not touch",
1507            "unrelated sibling file must be preserved",
1508        );
1509    }
1510
1511    #[tokio::test]
1512    async fn test_download_removes_stale_partial_downloads() {
1513        let temp_dir = TempDir::new().unwrap();
1514        let filepath = temp_dir.path().join("testfile.txt");
1515        let filepath_clone = filepath.clone();
1516        let stale_partial = temp_dir.path().join("testfile.txt.partial.123.456");
1517        File::create(&stale_partial)
1518            .unwrap()
1519            .set_modified(std::time::SystemTime::now() - Duration::from_secs(7200))
1520            .unwrap();
1521
1522        let server_content = "downloaded".to_string();
1523        let addr = setup_test_server(Some(server_content.clone()), StatusCode::OK).await;
1524        let url = format!("http://{addr}/testfile.txt");
1525
1526        let result = tokio::task::spawn_blocking(move || {
1527            ensure_file_exists_or_download_http_with_config(
1528                &filepath_clone,
1529                &url,
1530                None,
1531                5,
1532                Some(test_retry_config()),
1533                Some(0),
1534            )
1535        })
1536        .await
1537        .unwrap();
1538
1539        result.unwrap();
1540        assert_eq!(fs::read_to_string(&filepath).unwrap(), server_content);
1541        assert!(!stale_partial.exists());
1542    }
1543
1544    fn calculate_sha256_bytes(data: &[u8]) -> String {
1545        let mut ctx = digest::Context::new(&digest::SHA256);
1546        ctx.update(data);
1547        hex::encode(ctx.finish().as_ref())
1548    }
1549
1550    #[rstest]
1551    #[expect(clippy::panic_in_result_fn)]
1552    fn test_calculate_sha256() -> anyhow::Result<()> {
1553        let temp_dir = TempDir::new()?;
1554        let test_file_path = temp_dir.path().join("test_file.txt");
1555        let mut test_file = File::create(&test_file_path)?;
1556        let content = b"Hello, world!";
1557        test_file.write_all(content)?;
1558
1559        let expected_hash = "315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3";
1560        let calculated_hash = calculate_sha256(&test_file_path)?;
1561
1562        assert_eq!(calculated_hash, expected_hash);
1563        Ok(())
1564    }
1565
1566    #[rstest]
1567    #[expect(clippy::panic_in_result_fn)]
1568    fn test_verify_sha256_checksum() -> anyhow::Result<()> {
1569        let temp_dir = TempDir::new()?;
1570        let test_file_path = temp_dir.path().join("test_file.txt");
1571        let mut test_file = File::create(&test_file_path)?;
1572        let content = b"Hello, world!";
1573        test_file.write_all(content)?;
1574
1575        let calculated_checksum = calculate_sha256(&test_file_path)?;
1576
1577        // Create checksums.json containing the checksum
1578        let checksums_path = temp_dir.path().join("checksums.json");
1579        let checksums_data = json!({
1580            "test_file.txt": format!("sha256:{}", calculated_checksum)
1581        });
1582        let checksums_file = File::create(&checksums_path)?;
1583        let writer = BufWriter::new(checksums_file);
1584        to_writer(writer, &checksums_data)?;
1585
1586        let is_valid = verify_sha256_checksum(&test_file_path, &checksums_path)?;
1587        assert!(is_valid, "The checksum should be valid");
1588        Ok(())
1589    }
1590
1591    #[rstest]
1592    fn test_verify_sha256_checksum_rejects_non_string_entry() {
1593        let temp_dir = TempDir::new().unwrap();
1594        let test_file_path = temp_dir.path().join("test_file.txt");
1595        fs::write(&test_file_path, b"Hello, world!").unwrap();
1596        let checksums_path = temp_dir.path().join("checksums.json");
1597        fs::write(&checksums_path, r#"{"test_file.txt": 5}"#).unwrap();
1598
1599        let error = verify_sha256_checksum(&test_file_path, &checksums_path).unwrap_err();
1600
1601        assert_eq!(
1602            error.to_string(),
1603            format!(
1604                "checksum for test_file.txt in {} is not a string",
1605                checksums_path.display()
1606            )
1607        );
1608    }
1609
1610    #[cfg(unix)]
1611    #[rstest]
1612    fn test_verify_sha256_checksum_rejects_non_utf8_file_name() {
1613        use std::os::unix::ffi::OsStrExt;
1614
1615        let temp_dir = TempDir::new().unwrap();
1616        let test_file_path = temp_dir
1617            .path()
1618            .join(std::ffi::OsStr::from_bytes(b"bad\xff.txt"));
1619        let checksums_path = temp_dir.path().join("checksums.json");
1620        fs::write(&checksums_path, "{}").unwrap();
1621
1622        let error = verify_sha256_checksum(&test_file_path, &checksums_path).unwrap_err();
1623
1624        assert_eq!(
1625            error.to_string(),
1626            format!("no UTF-8 file name in {}", test_file_path.display())
1627        );
1628    }
1629
1630    #[rstest]
1631    fn test_update_sha256_checksums_replaces_manifest_without_leftovers() {
1632        let temp_dir = TempDir::new().unwrap();
1633        let checksums_path = temp_dir.path().join("checksums.json");
1634        fs::write(&checksums_path, r#"{"existing.txt": "sha256:aaa"}"#).unwrap();
1635
1636        update_sha256_checksums(&temp_dir.path().join("new.txt"), &checksums_path, "bbb").unwrap();
1637
1638        let manifest: Value =
1639            serde_json::from_str(&fs::read_to_string(&checksums_path).unwrap()).unwrap();
1640        assert_eq!(
1641            manifest,
1642            json!({"existing.txt": "sha256:aaa", "new.txt": "sha256:bbb"})
1643        );
1644        let entries: Vec<_> = fs::read_dir(temp_dir.path())
1645            .unwrap()
1646            .map(|entry| entry.unwrap().file_name())
1647            .collect();
1648        assert_eq!(entries, vec![OsString::from("checksums.json")]);
1649    }
1650
1651    #[rstest]
1652    fn test_update_sha256_checksums_rejects_non_object_manifest() {
1653        let temp_dir = TempDir::new().unwrap();
1654        let checksums_path = temp_dir.path().join("checksums.json");
1655        fs::write(&checksums_path, "[]").unwrap();
1656
1657        let error =
1658            update_sha256_checksums(&temp_dir.path().join("new.txt"), &checksums_path, "bbb")
1659                .unwrap_err();
1660
1661        assert_eq!(
1662            error.to_string(),
1663            format!(
1664                "checksums file {} is not a JSON object",
1665                checksums_path.display()
1666            )
1667        );
1668        assert_eq!(fs::read_to_string(&checksums_path).unwrap(), "[]");
1669    }
1670
1671    #[rstest]
1672    fn test_remove_stale_partials_only_removes_old_download_partials() {
1673        let temp_dir = TempDir::new().unwrap();
1674        let filepath = temp_dir.path().join("testfile.txt");
1675        let two_hours_ago = std::time::SystemTime::now() - Duration::from_secs(7200);
1676
1677        let write_file = |name: &str, stale: bool| {
1678            let path = temp_dir.path().join(name);
1679            let file = File::create(&path).unwrap();
1680
1681            if stale {
1682                file.set_modified(two_hours_ago).unwrap();
1683            }
1684
1685            path
1686        };
1687
1688        let stale_partial = write_file("testfile.txt.partial.123.456", true);
1689        let fresh_partial = write_file("testfile.txt.partial.124.457", false);
1690        let bystander = write_file("testfile.txt.partial", true);
1691        let non_numeric = write_file("testfile.txt.partial.abc.def", true);
1692        let other_target = write_file("other.txt.partial.123.456", true);
1693
1694        remove_stale_partials(&filepath);
1695
1696        assert!(!stale_partial.exists());
1697        assert!(fresh_partial.exists());
1698        assert!(bystander.exists());
1699        assert!(non_numeric.exists());
1700        assert!(other_target.exists());
1701    }
1702}