[STACKED on #952] feat(kms): return configured historical root keys

dstack/kms/README.md

@@ -232,3 +232,21 @@ The `SignCert` RPC is used by the dstack app to sign a TLS certificate. In this
 - Verify the CSR signature
 - Query the smart contract to check if the app is authorized
 - If authorized, sign the CSR with the CA root key and return the certificate chain to the app
+
+## Cold backup and recovery
+
+KMS root material is backed up as a complete, offline copy of `core.cert_dir`.
+There is no online backup RPC. Stop the KMS before taking or restoring a copy,
+preserve file modes and ownership, and protect the backup with the same controls
+as the live private keys. Do not copy only one key: the root CA, root K256 key,
+temporary CA, RPC identity, domain, and public certificates form one identity
+set.
+
+A supported recovery restores the complete directory while KMS is stopped,
+verifies that every private-key file remains owner-only (`0600`), and then
+starts KMS with the unchanged configuration. Compare `KMS.GetMeta` before the
+backup and after recovery: `ca_cert` and `k256_pubkey` must match exactly. KMS
+must fail closed when a required key is missing or malformed; never generate a
+new identity to repair a partial restore. Orphaned `*.private-tmp` files from an
+interrupted atomic write are not trust anchors and may be removed only after the
+final key file has been verified.

dstack/kms/kms.toml

@@ -24,6 +24,10 @@ mandatory = false
 
 [core]
 cert_dir = "/etc/kms/certs"
+# Previous root-key pairs for authorized handover, ordered newest to oldest:
+# [[core.historical_keys]]
+# ca_key = "/etc/kms/history/previous/root-ca.key"
+# k256_key = "/etc/kms/history/previous/root-k256.key"
 subject_postfix = ".dstack"
 site_name = ""
 # Whether trusted RPCs require the KMS to first attest itself to its own

dstack/kms/src/config.rs

@@ -23,6 +23,12 @@ const RPC_DOMAIN: &str = "rpc-domain";
 const K256_KEY: &str = "root-k256.key";
 const BOOTSTRAP_INFO: &str = "bootstrap-info.json";
 
+#[derive(Debug, Clone, Deserialize)]
+pub(crate) struct HistoricalKeyConfig {
+    pub ca_key: PathBuf,
+    pub k256_key: PathBuf,
+}
+
 #[derive(Debug, Clone, Deserialize)]
 pub(crate) struct ImageConfig {
     pub verify: bool,
@@ -35,6 +41,10 @@ pub(crate) struct ImageConfig {
 #[derive(Debug, Clone, Deserialize)]
 pub(crate) struct KmsConfig {
     pub cert_dir: PathBuf,
+    /// Previous root-key pairs returned after the current pair during an
+    /// authorized KMS handover. Configuration order is rotation order.
+    #[serde(default)]
+    pub historical_keys: Vec<HistoricalKeyConfig>,
     #[serde(default)]
     pub attestation: AttestationVerifierConfig,
     pub auth_api: AuthApi,

dstack/kms/src/main_service.rs

@@ -32,7 +32,7 @@ use tracing::{info, warn};
 use upgrade_authority::{build_boot_info, ensure_app_id_len, local_kms_boot_info, BootInfo};
 
 use crate::{
-    config::KmsConfig,
+    config::{HistoricalKeyConfig, KmsConfig},
     crypto::{derive_k256_key, sign_message, sign_message_with_timestamp},
 };
 
@@ -56,6 +56,7 @@ pub struct KmsStateInner {
     config: KmsConfig,
     root_ca: CaCert,
     k256_key: SigningKey,
+    historical_keys: Vec<KmsKeys>,
     temp_ca_cert: String,
     temp_ca_key: String,
     verifier: CvmVerifier,
@@ -120,6 +121,24 @@ fn remove_cache(parent_dir: &Path, sub_dir: &str) -> Result<()> {
     Ok(())
 }
 
+fn load_historical_keys(configs: &[HistoricalKeyConfig]) -> Result<Vec<KmsKeys>> {
+    configs
+        .iter()
+        .enumerate()
+        .map(|(index, config)| {
+            let ca_key = fs::read_to_string(&config.ca_key)
+                .with_context(|| format!("Failed to read historical CA key {index}"))?;
+            ra_tls::rcgen::KeyPair::from_pem(&ca_key)
+                .with_context(|| format!("Failed to parse historical CA key {index}"))?;
+            let k256_key = fs::read(&config.k256_key)
+                .with_context(|| format!("Failed to read historical ECDSA key {index}"))?;
+            SigningKey::from_slice(&k256_key)
+                .with_context(|| format!("Failed to parse historical ECDSA key {index}"))?;
+            Ok(KmsKeys { ca_key, k256_key })
+        })
+        .collect()
+}
+
 impl KmsState {
     /// clear cached image and measurement material for the given hashes. Used by
     /// the admin `ClearImageCache` RPC; authorization is enforced by the admin
@@ -139,6 +158,8 @@ impl KmsState {
         let key_bytes = fs::read(config.k256_key()).context("Failed to read ECDSA root key")?;
         let k256_key =
             SigningKey::from_slice(&key_bytes).context("Failed to load ECDSA root key")?;
+        let historical_keys = load_historical_keys(&config.historical_keys)
+            .context("Failed to load historical root keys")?;
         let temp_ca_key =
             fs::read_to_string(config.tmp_ca_key()).context("Faeild to read temp ca key")?;
         let temp_ca_cert =
@@ -163,6 +184,7 @@ impl KmsState {
                 config,
                 root_ca,
                 k256_key,
+                historical_keys,
                 temp_ca_cert,
                 temp_ca_key,
                 verifier,
@@ -485,12 +507,15 @@ impl KmsRpc for RpcHandler {
             self.state.config.sev_snp_key_release,
             self.state.config.aws_nitro_tpm_key_release,
         )?;
+        let mut keys = Vec::with_capacity(1 + self.state.inner.historical_keys.len());
+        keys.push(KmsKeys {
+            ca_key: self.state.inner.root_ca.key.serialize_pem(),
+            k256_key: self.state.inner.k256_key.to_bytes().to_vec(),
+        });
+        keys.extend(self.state.inner.historical_keys.iter().cloned());
         Ok(KmsKeyResponse {
             temp_ca_key: self.state.inner.temp_ca_key.clone(),
-            keys: vec![KmsKeys {
-                ca_key: self.state.inner.root_ca.key.serialize_pem(),
-                k256_key: self.state.inner.k256_key.to_bytes().to_vec(),
-            }],
+            keys,
         })
     }
 
@@ -585,6 +610,68 @@ mod tests {
     use cc_eventlog::RuntimeEvent;
     use sha2::{Digest, Sha256, Sha384};
 
+    fn historical_key_fixture(
+        directory: &Path,
+        name: &str,
+        scalar: u8,
+    ) -> (HistoricalKeyConfig, String, Vec<u8>) {
+        let key_dir = directory.join(name);
+        fs::create_dir_all(&key_dir).unwrap();
+        let ca_key = ra_tls::rcgen::KeyPair::generate_for(&ra_tls::rcgen::PKCS_ECDSA_P256_SHA256)
+            .unwrap()
+            .serialize_pem();
+        let k256_key = SigningKey::from_slice(&[scalar; 32])
+            .unwrap()
+            .to_bytes()
+            .to_vec();
+        let ca_path = key_dir.join("root-ca.key");
+        let k256_path = key_dir.join("root-k256.key");
+        fs::write(&ca_path, &ca_key).unwrap();
+        fs::write(&k256_path, &k256_key).unwrap();
+        (
+            HistoricalKeyConfig {
+                ca_key: ca_path,
+                k256_key: k256_path,
+            },
+            ca_key,
+            k256_key,
+        )
+    }
+
+    #[test]
+    fn historical_root_keys_preserve_configured_rotation_order() {
+        let directory = tempfile::tempdir().unwrap();
+        let (newer, newer_ca, newer_k256) = historical_key_fixture(directory.path(), "newer", 0x21);
+        let (older, older_ca, older_k256) = historical_key_fixture(directory.path(), "older", 0x22);
+        let loaded = load_historical_keys(&[newer, older]).unwrap();
+
+        assert_eq!(loaded.len(), 2);
+        assert_eq!(loaded[0].ca_key, newer_ca);
+        assert_eq!(loaded[0].k256_key, newer_k256);
+        assert_eq!(loaded[1].ca_key, older_ca);
+        assert_eq!(loaded[1].k256_key, older_k256);
+    }
+
+    #[test]
+    fn historical_root_keys_fail_closed_on_missing_or_malformed_material() {
+        let directory = tempfile::tempdir().unwrap();
+        let missing = HistoricalKeyConfig {
+            ca_key: directory.path().join("missing-ca.key"),
+            k256_key: directory.path().join("missing-k256.key"),
+        };
+        assert!(load_historical_keys(&[missing]).is_err());
+
+        let ca_path = directory.path().join("bad-ca.key");
+        let k256_path = directory.path().join("bad-k256.key");
+        fs::write(&ca_path, "not a PEM key").unwrap();
+        fs::write(&k256_path, [0u8; 31]).unwrap();
+        let malformed = HistoricalKeyConfig {
+            ca_key: ca_path,
+            k256_key: k256_path,
+        };
+        assert!(load_historical_keys(&[malformed]).is_err());
+    }
+
     #[test]
     fn remove_cache_only_deletes_the_named_hex_entry() {
         let dir = tempfile::tempdir().unwrap();