Add ts keys rotate and ts keys deactivate commands

CHANGELOG.md

@@ -67,6 +67,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add `ts keys rotate` and `ts keys deactivate` to drive a deployed service's signing-key admin routes, reading the admin password from `TRUSTED_SERVER_ADMIN_PASSWORD` only. `ts keys deactivate --delete` asks for confirmation first; runs whose stdin or stderr is not a terminal must pass `--yes` (#1430).
 - Add opt-in mobile ad-render tracing through `[integrations.gpt_diagnostics].trace_page_enabled` (default `false`, requiring GPT diagnostics). The same-tab `/_ts/trace` viewer presents bounded, redacted browser-local request, auction and GPT observations with copy, download, share and independent local/server cleanup controls. See the GPT diagnostics guide for deployment and rollout checks.
 - Add the opt-in `[trusted_forwarder]` section with required `auth_header` and secret-store `shared_secret` reference fields for authenticated public-origin forwarding.
 - `[rewrite] include_domains` limits creative asset rewriting to listed hosts. When non-empty, only asset URLs on matching hosts are proxied through `/first-party/proxy`, so only those hosts receive the EC ID on newly rewritten or signed asset URLs. The list is not enforced when a proxy URL is fetched, so previously issued proxy URLs can still forward the EC ID to hosts later removed from it. Other assets keep their original URL, and `/first-party/sign` returns `502` for them so the creative runtime loads them directly. Click-through links are not affected, and `exclude_domains` still wins. Entries must be host names, optionally prefixed with `*.`, dotted-quad IPv4 addresses, or bracketed IPv6 addresses; entries with a scheme, port, path, spaces or non-ASCII characters are rejected at load. Upgrading: deploy the binary first, then push a config that sets `include_domains`. Rolling back: remove `include_domains`, push the resulting config, then roll back the binary; older binaries reject the key because `[rewrite]` denies unknown fields.

crates/trusted-server-cli/src/admin_client.rs

@@ -0,0 +1,204 @@
+//! Shared client for a deployed service's `^/_ts/admin` routes.
+//!
+//! `ts cache purge` and `ts keys` all call basic-auth admin routes on a running service.
+//! They share one implementation of the transport rules here so that the service URL
+//! checks ([`crate::url_guard::service_endpoint`]), the password source, redirect refusal
+//! and 401 guidance cannot drift apart between commands.
+
+use core::error::Error;
+use std::time::Duration;
+
+use reqwest::StatusCode;
+
+use crate::error::CliResult;
+
+/// Environment variable carrying the admin password.
+///
+/// Read from the environment and never accepted as a flag: an argument is visible to every
+/// other process on the host through `ps`, and lands in shell history.
+pub const ADMIN_PASSWORD_ENVIRONMENT_VARIABLE: &str = "TRUSTED_SERVER_ADMIN_PASSWORD";
+
+/// Hint appended to a 401 failure, shared by every admin command.
+pub(crate) const UNAUTHORIZED_HINT: &str =
+    " — check the admin username and $TRUSTED_SERVER_ADMIN_PASSWORD";
+
+const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
+
+/// Shorter than [`REQUEST_TIMEOUT`] so a stalled TCP or TLS handshake fails as a connect
+/// error rather than as an overall timeout that reqwest does not mark as one.
+const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
+
+/// The raw answer from an admin route.
+pub(crate) struct AdminResponse {
+    /// HTTP status the service answered with.
+    pub(crate) status: StatusCode,
+    /// Response body as text.
+    pub(crate) body: String,
+}
+
+/// Read the admin password from [`ADMIN_PASSWORD_ENVIRONMENT_VARIABLE`].
+///
+/// # Errors
+///
+/// Returns an error explaining where the password must come from when it is unset.
+pub(crate) fn admin_password() -> CliResult<String> {
+    std::env::var(ADMIN_PASSWORD_ENVIRONMENT_VARIABLE).map_err(|_| {
+        format!(
+            "set {ADMIN_PASSWORD_ENVIRONMENT_VARIABLE} to the service's admin password \
+             (it is read from the environment, never a flag, so it does not reach `ps` \
+             output or shell history)"
+        )
+    })
+}
+
+/// POST a JSON body to an admin route and return the status and body.
+///
+/// The endpoint is validated and the password read before any connection is made, so an
+/// unsafe URL or a missing credential never reaches the network. Redirects are not
+/// followed, so a login page cannot masquerade as a success.
+///
+/// # Errors
+///
+/// Returns an error when the endpoint is invalid, the password is missing, or the
+/// service cannot be reached or its body read. Non-success statuses are returned as an
+/// [`AdminResponse`] for the caller to interpret.
+pub(crate) fn post_admin_json(
+    service: &str,
+    path: &str,
+    username: &str,
+    body: String,
+) -> CliResult<AdminResponse> {
+    let endpoint = crate::url_guard::service_endpoint(service, path)?;
+    let password = admin_password()?;
+
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .map_err(|error| format!("failed to start the HTTP runtime: {error}"))?;
+
+    // Without this the first HTTPS request panics with "No provider set".
+    crate::tls::install_crypto_provider();
+
+    runtime.block_on(async {
+        let client = reqwest::Client::builder()
+            .timeout(REQUEST_TIMEOUT)
+            // Without it the overall timeout fires first on a stalled handshake, and the
+            // message below would claim the request was sent when nothing left this machine.
+            .connect_timeout(CONNECT_TIMEOUT)
+            .redirect(reqwest::redirect::Policy::none())
+            .build()
+            .map_err(|error| format!("failed to build the HTTP client: {error}"))?;
+        let response = client
+            .post(&endpoint)
+            .basic_auth(username, Some(&password))
+            .header("content-type", "application/json")
+            .body(body)
+            .send()
+            .await
+            .map_err(|error| {
+                let is_connect = error.is_connect();
+                let error = describe_cause(&error.without_url());
+                if is_connect {
+                    format!("could not reach {endpoint}: {error}")
+                } else {
+                    // The request may have been delivered: a key rotation is not idempotent, so
+                    // the operator must check before retrying rather than assume nothing ran.
+                    format!(
+                        "sent the request to {endpoint} but got no response ({error}); the \
+                         service may have applied it, so check its state before retrying"
+                    )
+                }
+            })?;
+        let status = response.status();
+        let body = response.text().await.map_err(|error| {
+            let error = describe_cause(&error.without_url());
+            // A status line means the service handled the request, so a retry could apply a
+            // rotation twice.
+            format!(
+                "{endpoint} answered {status} but its body could not be read ({error}); the \
+                 service may have applied the request, so check its state before retrying"
+            )
+        })?;
+        Ok(AdminResponse { status, body })
+    })
+}
+
+/// Describe an error by its causes rather than its generic top-level text.
+///
+/// reqwest reports every transport failure as `error sending request`; the useful detail
+/// (refused, timed out, reset) lives in the `source()` chain. The outermost message is kept
+/// only when there is no cause, and repeated messages in the chain are dropped.
+fn describe_cause(error: &(dyn Error + 'static)) -> String {
+    let mut messages: Vec<String> = Vec::new();
+    let mut cause = error.source();
+    while let Some(current) = cause {
+        let message = current.to_string();
+        if messages.last() != Some(&message) {
+            messages.push(message);
+        }
+        cause = current.source();
+    }
+    if messages.is_empty() {
+        error.to_string()
+    } else {
+        messages.join(": ")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(Debug, derive_more::Display)]
+    #[display("{message}")]
+    struct ChainError {
+        message: &'static str,
+        source: Option<Box<ChainError>>,
+    }
+
+    impl Error for ChainError {
+        fn source(&self) -> Option<&(dyn Error + 'static)> {
+            self.source
+                .as_deref()
+                .map(|source| source as &(dyn Error + 'static))
+        }
+    }
+
+    fn chain(messages: &[&'static str]) -> ChainError {
+        let mut error: Option<ChainError> = None;
+        for message in messages.iter().rev() {
+            error = Some(ChainError {
+                message,
+                source: error.map(Box::new),
+            });
+        }
+        error.expect("should build a non-empty chain")
+    }
+
+    #[test]
+    fn describe_cause_drops_the_generic_outer_message_and_repeats() {
+        let error = chain(&[
+            "error sending request",
+            "client error (Connect)",
+            "connection refused",
+            "connection refused",
+        ]);
+
+        assert_eq!(
+            describe_cause(&error),
+            "client error (Connect): connection refused",
+            "should list the causes once each without the outer text"
+        );
+    }
+
+    #[test]
+    fn describe_cause_keeps_the_message_when_there_is_no_cause() {
+        let error = chain(&["error decoding response body"]);
+
+        assert_eq!(
+            describe_cause(&error),
+            "error decoding response body",
+            "should fall back to the error's own text"
+        );
+    }
+}

crates/trusted-server-cli/src/commands/cache/mod.rs

@@ -44,12 +44,6 @@ pub struct PurgeArgs {
     pub username: String,
 }
 
-/// Environment variable carrying the admin password.
-///
-/// Read from the environment and never accepted as a flag: an argument is visible to every
-/// other process on the host through `ps`, and lands in shell history.
-pub const ADMIN_PASSWORD_ENVIRONMENT_VARIABLE: &str = "TRUSTED_SERVER_ADMIN_PASSWORD";
-
 /// Run a `ts cache` subcommand.
 ///
 /// # Errors

crates/trusted-server-cli/src/commands/cache/purge.rs

@@ -1,14 +1,12 @@
 //! Calling a deployed service's cache-purge endpoint.
 
-use std::time::Duration;
-
 use serde::Deserialize;
 
-use crate::commands::cache::{ADMIN_PASSWORD_ENVIRONMENT_VARIABLE, PurgeArgs};
+use crate::admin_client::{AdminResponse, UNAUTHORIZED_HINT, post_admin_json};
+use crate::commands::cache::PurgeArgs;
 use crate::error::{CliResult, cli_error};
 
 const PURGE_PATH: &str = "/_ts/admin/cache/purge";
-const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// The purge request body, as the endpoint's parser expects it.
 ///
@@ -30,26 +28,6 @@ fn request_body(args: &PurgeArgs) -> CliResult<String> {
     }
 }
 
-/// Join the service base URL and the purge path without doubling or dropping a slash.
-fn purge_endpoint(service: &str) -> CliResult<String> {
-    let mut url =
-        reqwest::Url::parse(service).map_err(|_| "--service must be an absolute HTTPS URL")?;
-    crate::url_guard::require_credential_safe_transport(&url, "--service")?;
-    if url.host().is_none()
-        || !url.username().is_empty()
-        || url.password().is_some()
-        || url.query().is_some()
-        || url.fragment().is_some()
-        || !url.path().trim_matches('/').is_empty()
-    {
-        return cli_error(
-            "--service must be a base HTTPS URL without credentials, a path, query, or fragment",
-        );
-    }
-    url.set_path(PURGE_PATH);
-    Ok(url.to_string())
-}
-
 /// The acknowledgment returned by the purge endpoint.
 #[derive(Deserialize)]
 struct PurgeAcknowledgment {
@@ -67,45 +45,10 @@ struct PurgeAcknowledgment {
 /// status, redirects, or does not acknowledge the requested purge scope.
 pub fn run_purge(args: &PurgeArgs, out: &mut impl std::io::Write) -> CliResult<()> {
     let body = request_body(args)?;
-    let endpoint = purge_endpoint(&args.service)?;
-
-    let password = std::env::var(ADMIN_PASSWORD_ENVIRONMENT_VARIABLE).map_err(|_| {
-        format!(
-            "set {ADMIN_PASSWORD_ENVIRONMENT_VARIABLE} to the service's admin password \
-             (it is read from the environment, never a flag, so it does not reach `ps` \
-             output or shell history)"
-        )
-    })?;
-
-    let runtime = tokio::runtime::Builder::new_current_thread()
-        .enable_all()
-        .build()
-        .map_err(|error| format!("failed to start the HTTP runtime: {error}"))?;
-
-    // Without this the first HTTPS request panics with "No provider set".
-    crate::tls::install_crypto_provider();
-
-    let (status, response_body) = runtime.block_on(async {
-        let client = reqwest::Client::builder()
-            .timeout(REQUEST_TIMEOUT)
-            .redirect(reqwest::redirect::Policy::none())
-            .build()
-            .map_err(|error| format!("failed to build the HTTP client: {error}"))?;
-        let response = client
-            .post(&endpoint)
-            .basic_auth(&args.username, Some(&password))
-            .header("content-type", "application/json")
-            .body(body)
-            .send()
-            .await
-            .map_err(|error| format!("could not reach {endpoint}: {error}"))?;
-        let status = response.status();
-        let text = response
-            .text()
-            .await
-            .map_err(|error| format!("could not read the purge acknowledgment: {error}"))?;
-        Ok::<_, String>((status, text))
-    })?;
+    let AdminResponse {
+        status,
+        body: response_body,
+    } = post_admin_json(&args.service, PURGE_PATH, &args.username, body)?;
 
     if status.is_success() {
         let acknowledgment: PurgeAcknowledgment = serde_json::from_str(&response_body)
@@ -132,7 +75,7 @@ pub fn run_purge(args: &PurgeArgs, out: &mut impl std::io::Write) -> CliResult<(
     // Named individually, because each one sends the operator somewhere different and a
     // purge is usually run mid-incident.
     let hint = match status.as_u16() {
-        401 => " — check the admin username and $TRUSTED_SERVER_ADMIN_PASSWORD",
+        401 => UNAUTHORIZED_HINT,
         404 => " — this service may predate the purge endpoint",
         501 => " — the template cache is Fastly-backed; this adapter cannot purge",
         _ => "",
@@ -210,59 +153,4 @@ mod tests {
         // Defaulting to --all would make a bare `ts cache purge` flush production.
         assert!(request_body(&args(false, None)).is_err());
     }
-
-    #[test]
-    fn the_endpoint_url_survives_a_trailing_slash() {
-        for service in [
-            "https://edge.example.com",
-            "https://edge.example.com/",
-            "https://edge.example.com///",
-        ] {
-            assert_eq!(
-                purge_endpoint(service).expect("should accept a secure base URL"),
-                "https://edge.example.com/_ts/admin/cache/purge",
-                "{service} must resolve to one well-formed endpoint"
-            );
-        }
-    }
-
-    #[test]
-    fn only_loopback_services_may_use_plaintext_http() {
-        for service in [
-            "http://127.0.0.1:8080",
-            "http://[::1]:8080",
-            "http://localhost:8080",
-        ] {
-            assert!(
-                purge_endpoint(service).is_ok(),
-                "should permit loopback development: {service}"
-            );
-        }
-        for service in [
-            "http://192.0.2.1",
-            "http://[2001:db8::1]",
-            "http://localhost.example.com",
-        ] {
-            assert!(
-                purge_endpoint(service).is_err(),
-                "should refuse remote plaintext transport: {service}"
-            );
-        }
-    }
-
-    #[test]
-    fn service_urls_cannot_redirect_the_purge_path_or_embed_credentials() {
-        for service in [
-            "not a URL",
-            "https://example.com/path",
-            "https://example.com?query=1",
-            "https://example.com#fragment",
-            "https://user:example-password@example.com",
-        ] {
-            assert!(
-                purge_endpoint(service).is_err(),
-                "should require an unambiguous service base URL"
-            );
-        }
-    }
 }