fix(ant-devnet): stop cleanly on SIGTERM and use a per-run data dir by default

.github/workflows/ci.yml

@@ -129,6 +129,12 @@ jobs:
           failed=""
           for t in $targets; do
             extra=""
+            if [ "$t" = devnet_lifecycle ] && [ "$RUNNER_OS" = Windows ]; then
+              # It stops ant-devnet with SIGTERM and SIGKILL, so it is Unix
+              # only, and compiles to nothing on Windows.
+              echo "skipping $t on $RUNNER_OS (Unix only)"
+              continue
+            fi
             if [ "$t" = storage_scale ]; then
               # Its gates compare the store scan against timed directory walks, and
               # were measured on Linux; it has only ever run there.
@@ -165,6 +171,18 @@ jobs:
       - name: Unit tests without default features (no logging, no webrtc-direct)
         if: runner.os == 'Linux'
         run: cargo test --lib --no-default-features
+      # The run-directory tests that mount a tmpfs and bind mounts inside run
+      # directories, make directories another user owns, or act as another
+      # user. They need root, which the runner gives through sudo, so
+      # `cargo test` alone skips them.
+      - name: ant-devnet run directories against real mounts and other users
+        if: runner.os == 'Linux'
+        shell: bash
+        run: |
+          set -euo pipefail
+          cargo test --lib --features test-utils with_sudo_ -- --ignored --test-threads=1 2>&1 | tee /tmp/run_dir_sudo.log
+          grep -qE 'test result: ok\. [1-9]' /tmp/run_dir_sudo.log \
+            || { echo 'the run-directory sudo tests ran nothing'; exit 1; }
 
   filesystems:
     name: Storage on ${{ matrix.fs }}

Cargo.toml

@@ -122,12 +122,18 @@ self_encryption = { version = "0.36", optional = true }
 libc = "0.2"
 # Safe wrappers for `openat`/`unlinkat`, so the storage-migration cleanup can delete relative
 # to a directory handle it opened `O_NOFOLLOW` rather than by re-resolving a path it has
-# already checked. Adds nothing to the build: `tempfile`, a direct dependency, already pulls
-# this exact crate and version in with the `fs` feature on.
-rustix = { version = "1", features = ["fs"] }
+# already checked. Adds no crate to the build: `tempfile`, a direct dependency, already pulls
+# this exact crate and version in with the `fs` feature on. `process` is for `geteuid`, which
+# the devnet uses to tell a run directory it created from one another user put in its place.
+rustix = { version = "1", features = ["fs", "process"] }
 
 [target.'cfg(windows)'.dependencies]
 self-replace = "1"
+# The volume serial number and file index of a directory handle, which tells
+# a devnet run directory from a copy put in its place. Already in Cargo.lock
+# (through walkdir), and what it needs, windows-sys 0.48, already ships in the
+# Windows build through `directories`.
+winapi-util = "0.1"
 
 [target.'cfg(target_os = "macos")'.dependencies]
 objc2 = "0.6"

src/bin/ant-devnet/cli.rs

@@ -25,11 +25,17 @@ pub struct Cli {
     #[arg(long)]
     pub base_port: Option<u16>,
 
-    /// Data directory for node state.
+    /// Directory to create this run's own data directory in (default:
+    /// `devnets/` in the ant data root). The run directory is removed on
+    /// shutdown unless `--no-cleanup` is set; this directory, and anything else
+    /// in it, never is. Run directories left here by runs that were killed
+    /// are removed when the next run starts. On Unix only you and root may be
+    /// able to change it, or any directory above it.
     #[arg(long)]
     pub data_dir: Option<PathBuf>,
 
-    /// Keep node data directories on shutdown instead of removing them.
+    /// Keep this run's data directory on shutdown. Later runs leave it alone
+    /// too.
     #[arg(long)]
     pub no_cleanup: bool,
 

src/bin/ant-devnet/main.rs

@@ -42,6 +42,13 @@ use cli::Cli;
 async fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
 
+    // Listen for stop requests before anything starts. Until a handler is
+    // installed, SIGTERM kills the process outright: the devnet is never shut
+    // down, Anvil is orphaned and the data directory is left behind
+    // (ant-node#261).
+    let stop = stop_requested()?;
+    tokio::pin!(stop);
+
     let cli = Cli::parse();
 
     #[cfg(feature = "logging")]
@@ -118,19 +125,54 @@ async fn main() -> color_eyre::Result<()> {
     if let Some(base_port) = cli.webrtc_direct_base_port {
         config.webrtc_direct_base_port = base_port;
     }
+    // A stop request during EVM setup drops the half-finished setup, and with
+    // it any Anvil it already started.
     let ResolvedEvm {
         manifest: evm_info,
-        local_testnet: _local_evm_testnet,
-    } = resolve_evm_info(
-        cli.evm_network.as_deref(),
-        cli.enable_evm,
-        cli.host,
-        &mut config,
-    )
-    .await?;
+        local_testnet,
+    } = tokio::select! {
+        evm = resolve_evm_info(
+            cli.evm_network.as_deref(),
+            cli.enable_evm,
+            cli.host,
+            &mut config,
+        ) => evm?,
+        signal = &mut stop => {
+            ant_node::logging::info!("{signal} during EVM setup; exiting");
+            return Ok(());
+        }
+    };
 
     let mut devnet = Devnet::new(config).await?;
-    devnet.start().await?;
+    // A stop request during startup cancels the start rather than dropping
+    // it, so the start returns with every node it started registered and
+    // the shutdown below stops them all before removing the data directory.
+    let cancel = devnet.cancellation_token();
+    let started = {
+        let start = devnet.start();
+        tokio::pin!(start);
+        tokio::select! {
+            started = &mut start => started,
+            signal = &mut stop => {
+                ant_node::logging::info!("{signal} during startup; stopping the devnet");
+                cancel.cancel();
+                start.await
+            }
+        }
+    };
+    // Stopped or failed part-way: shut down the nodes that did start and
+    // remove the data directory. Anvil stops when `local_testnet` drops on
+    // return.
+    if cancel.is_cancelled() {
+        devnet.shutdown().await?;
+        return Ok(());
+    }
+    if let Err(e) = started {
+        if let Err(cleanup) = devnet.shutdown().await {
+            ant_node::logging::warn!("Cleanup after a failed start also failed: {cleanup}");
+        }
+        return Err(e.into());
+    }
 
     let created_at = chrono::Utc::now().to_rfc3339();
 
@@ -209,13 +251,52 @@ async fn main() -> color_eyre::Result<()> {
         )?;
     }
 
-    ant_node::logging::info!("Devnet running. Press Ctrl+C to stop.");
-    tokio::signal::ctrl_c().await?;
+    ant_node::logging::info!("Devnet running. Press Ctrl+C or send SIGTERM to stop.");
+    let signal = stop.await;
+    ant_node::logging::info!("{signal} received; stopping the devnet");
 
     devnet.shutdown().await?;
+    // Stop Anvil after the nodes, which verify payments against it.
+    drop(local_testnet);
     Ok(())
 }
 
+/// A future that resolves when the process is asked to stop, naming the
+/// request: SIGINT (Ctrl+C), SIGTERM (`kill`, service managers, `ant dev
+/// stop`) or SIGHUP (the terminal closed) on Unix; Ctrl+C or Ctrl+Break on
+/// Windows.
+///
+/// The handlers are installed by this call, not when the future is first
+/// polled, so call it before starting anything.
+fn stop_requested() -> std::io::Result<impl std::future::Future<Output = &'static str>> {
+    #[cfg(unix)]
+    {
+        use tokio::signal::unix::{signal, SignalKind};
+        let mut interrupt = signal(SignalKind::interrupt())?;
+        let mut terminate = signal(SignalKind::terminate())?;
+        let mut hangup = signal(SignalKind::hangup())?;
+        Ok(async move {
+            tokio::select! {
+                _ = interrupt.recv() => "SIGINT",
+                _ = terminate.recv() => "SIGTERM",
+                _ = hangup.recv() => "SIGHUP",
+            }
+        })
+    }
+    #[cfg(windows)]
+    {
+        use tokio::signal::windows::{ctrl_break, ctrl_c};
+        let mut interrupt = ctrl_c()?;
+        let mut brk = ctrl_break()?;
+        Ok(async move {
+            tokio::select! {
+                _ = interrupt.recv() => "Ctrl+C",
+                _ = brk.recv() => "Ctrl+Break",
+            }
+        })
+    }
+}
+
 #[cfg(all(feature = "webrtc-direct", feature = "test-utils"))]
 async fn load_public_file(
     path: Option<&std::path::Path>,
@@ -451,7 +532,7 @@ fn serve_manifest_api(
 /// payloads are tiny and a devnet serves a handful of LAN devices, so a single
 /// thread bounds resource use (no per-connection thread to exhaust). Each
 /// connection gets a read timeout so a slow/idle client can't stall the loop.
-/// The thread is detached and dies when the process exits on Ctrl+C.
+/// The thread is detached and dies when the process exits.
 fn spawn_manifest_server(
     listener: std::net::TcpListener,
     manifest_json: String,