test: add deterministic crash checkpoints for installs

src/dist/manifestation.rs

@@ -21,6 +21,8 @@ use futures_util::{
 use tokio::task::{JoinHandle, spawn_blocking};
 use tracing::{debug, info, warn};
 
+#[cfg(feature = "test")]
+use crate::test::checkpoint;
 use crate::{
     diskio::{Executor, IO_CHUNK_SIZE, get_executor, unpack_ram},
     dist::{
@@ -291,6 +293,9 @@ impl Manifestation {
             drop(stream);
         }
 
+        #[cfg(feature = "test")]
+        checkpoint(download_cfg.process, "manifestation-update-before-metadata");
+
         // Install new distribution manifest
         let new_manifest_str = new_manifest.clone().stringify()?;
         tx.modify_file(rel_installed_manifest_path)?;

src/test.rs

@@ -21,7 +21,7 @@ use anyhow::Result;
 use sha2::{Digest, Sha256};
 
 use crate::dist::TargetTuple;
-use crate::process::TestProcess;
+use crate::process::{Process, TestProcess};
 
 #[cfg(windows)]
 pub use crate::cli::self_update::{RegistryGuard, RegistryValueId, USER_PATH, get_path};
@@ -36,6 +36,31 @@ pub use dist::DistContext;
 pub(super) mod mock;
 pub use mock::{MockComponentBuilder, MockFile, MockInstallerBuilder};
 
+/// Signal that a selected test checkpoint has been reached, then wait for the
+/// test driver to terminate this process.
+pub(crate) fn checkpoint(process: &Process, name: &str) {
+    if process.var(CHECKPOINT_ENV).as_deref() != Ok(name) {
+        return;
+    }
+
+    let rustup_home = process
+        .rustup_home()
+        .expect("selected test checkpoint requires RUSTUP_HOME");
+    let test_root = rustup_home
+        .parent()
+        .expect("test RUSTUP_HOME must be inside the test root");
+    fs::write(checkpoint_path(test_root, name), name)
+        .expect("failed to write test checkpoint marker");
+
+    loop {
+        std::thread::park();
+    }
+}
+
+fn checkpoint_path(test_root: &Path, name: &str) -> PathBuf {
+    test_root.join(format!("rustup-checkpoint-{name}"))
+}
+
 // Things that can have environment variables applied to them.
 pub trait Env {
     fn env<K, V>(&mut self, key: K, val: V)
@@ -327,3 +352,5 @@ pub static CROSS_ARCH2: &str = "arm-linux-androideabi";
 pub static MULTI_ARCH1: &str = "i686-unknown-linux-gnu";
 #[cfg(not(target_pointer_width = "64"))]
 static MULTI_ARCH1: &str = "x86_64-unknown-linux-gnu";
+
+const CHECKPOINT_ENV: &str = "RUSTUP_TEST_CHECKPOINT";

src/test/clitools.rs

@@ -12,10 +12,11 @@ use std::{
     mem,
     ops::{Deref, DerefMut},
     path::{Path, PathBuf},
-    process::Command,
+    process::{Command, ExitStatus},
     string::FromUtf8Error,
     sync::{Arc, LazyLock, RwLock, RwLockWriteGuard},
-    time::{Instant, SystemTime, UNIX_EPOCH},
+    thread,
+    time::{Duration as StdDuration, Instant, SystemTime, UNIX_EPOCH},
 };
 
 use chrono::{DateTime, Duration};
@@ -33,7 +34,7 @@ use crate::test::this_host_tuple;
 use crate::utils;
 
 use super::{
-    CROSS_ARCH1, CROSS_ARCH2, MULTI_ARCH1,
+    CHECKPOINT_ENV, CROSS_ARCH1, CROSS_ARCH2, MULTI_ARCH1, checkpoint_path,
     dist::{MockDistServer, MockManifestVersion, Release, RlsStatus, change_channel_date},
     mock::MockFile,
 };
@@ -487,7 +488,7 @@ impl Config {
                         && e.raw_os_error() == Some(26)
                     {
                         // This is an ETXTBSY situation
-                        std::thread::sleep(std::time::Duration::from_millis(250));
+                        thread::sleep(StdDuration::from_millis(250));
                     } else {
                         panic!("Unable to run test command: {e:?}");
                     }
@@ -972,6 +973,48 @@ impl CliTestContext {
         Self { config, _test_dir }
     }
 
+    /// Run a rustup command until it reaches `checkpoint`, then terminate it
+    /// without giving destructors an opportunity to run.
+    pub fn kill_at_checkpoint(&self, mut command: Command, checkpoint: &str) -> ExitStatus {
+        let marker = checkpoint_path(&self.config.test_root_dir, checkpoint);
+        if let Err(error) = fs::remove_file(&marker)
+            && error.kind() != io::ErrorKind::NotFound
+        {
+            panic!("failed to remove stale checkpoint marker: {error}");
+        }
+
+        command.env(CHECKPOINT_ENV, checkpoint);
+        let mut child = command
+            .spawn()
+            .expect("failed to start command for checkpoint test");
+
+        let deadline = Instant::now() + StdDuration::from_secs(10);
+        loop {
+            if matches!(fs::read_to_string(&marker), Ok(contents) if contents == checkpoint) {
+                break;
+            }
+            if let Some(status) = child
+                .try_wait()
+                .expect("failed to query checkpoint test command")
+            {
+                panic!("command exited before reaching checkpoint {checkpoint:?}: {status}");
+            }
+            if Instant::now() >= deadline {
+                let _ = child.kill();
+                let _ = child.wait();
+                panic!("timed out waiting for checkpoint {checkpoint:?}");
+            }
+            thread::sleep(StdDuration::from_millis(10));
+        }
+
+        child
+            .kill()
+            .expect("failed to terminate command at checkpoint");
+        child
+            .wait()
+            .expect("failed to reap command after checkpoint")
+    }
+
     /// Move the dist server to the specified scenario and restore it
     /// afterwards.
     pub fn with_dist_dir(&mut self, scenario: Scenario) -> DistDirGuard<'_> {

tests/suite/cli_crash.rs

@@ -0,0 +1,19 @@
+use rustup::test::{CliTestContext, Scenario};
+
+#[tokio::test]
+async fn interrupted_install_can_be_retried() {
+    let cx = CliTestContext::new(Scenario::SimpleV2).await;
+    let command = cx.config.cmd("rustup", ["toolchain", "install", "nightly"]);
+
+    let status = cx.kill_at_checkpoint(command, "manifestation-update-before-metadata");
+    assert!(!status.success());
+
+    cx.config
+        .expect(["rustup", "toolchain", "install", "nightly"])
+        .await
+        .is_ok();
+    cx.config
+        .expect(["rustc", "+nightly", "--version"])
+        .await
+        .is_ok();
+}

tests/suite/mod.rs

@@ -1,3 +1,4 @@
+mod cli_crash;
 mod cli_exact;
 mod cli_inst_interactive;
 mod cli_misc;