From 6d285879d7a2cdc5022b9d467996cb95c34de4fc Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Sat, 15 Aug 2026 08:32:45 -0400 Subject: [PATCH] fix(certificates): protect local TLS certificate lifecycle Signed-off-by: Yordis Prieto --- .github/workflows/common.yml | 19 ++- docker-compose.yml | 55 +++---- docs/installation.md | 4 +- docs/operations.md | 6 + docs/security.md | 19 +++ scripts/certificates/Dockerfile | 7 + .../generate-cluster-certificates.sh | 155 ++++++++++++++++++ 7 files changed, 231 insertions(+), 34 deletions(-) create mode 100644 scripts/certificates/Dockerfile create mode 100755 scripts/certificates/generate-cluster-certificates.sh diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml index 311d2c06fb..d088542fd7 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/common.yml @@ -252,7 +252,24 @@ jobs: NUGET_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | docker compose build + docker compose run --rm cert-gen + docker compose run --rm --entrypoint sh cert-gen -c ' + test -z "$(find /certs/ca -mindepth 1 -maxdepth 1 ! -name ca.crt -print -quit)" + test "$(stat -c %a /certs/ca/ca.crt)" = 644 + for certificate in /certs/node*/node.crt; do + test "$(stat -c %a "$certificate")" = 644 + done + for private_key in /certs/node*/node.key; do + test "$(stat -c %a "$private_key")" = 600 + done + ' + docker compose run --rm --entrypoint sh cert-gen -c 'sha256sum /certs/ca/ca.crt /certs/node*/node.crt /certs/node*/node.key' > /tmp/certificate-fingerprints.before + docker compose run --rm cert-gen + docker compose run --rm --entrypoint sh cert-gen -c 'sha256sum /certs/ca/ca.crt /certs/node*/node.crt /certs/node*/node.key' > /tmp/certificate-fingerprints.after + diff --unified /tmp/certificate-fingerprints.before /tmp/certificate-fingerprints.after docker compose up --detach printf 'Waiting for cluster' - timeout 60 bash -c -- 'until curl --output /dev/null --silent --insecure --fail https://localhost:2113/-/readiness; do printf '.'; sleep 2; done' + for port in 2111 2112 2113; do + timeout 60 bash -c -- "until curl --output /dev/null --silent --fail --cacert certs/ca/ca.crt https://localhost:$port/-/readiness; do printf '.'; sleep 2; done" + done docker compose down diff --git a/docker-compose.yml b/docker-compose.yml index 402b9cd208..0896f11148 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,27 +1,12 @@ services: - volumes-provisioner: - image: hasnat/volumes-provisioner - environment: - PROVISION_DIRECTORIES: "1000:1000:0755:/tmp/certs" - volumes: - - "./certs:/tmp/certs" - network_mode: none - + # Local development only. Production certificates must come from the deployment platform's PKI. cert-gen: - image: eventstore/es-gencert-cli:1.0.2 - entrypoint: bash - user: "1000:1000" - command: > - -c "mkdir -p ./certs && cd /certs - && es-gencert-cli create-ca - && es-gencert-cli create-node -out ./node1 -ip-addresses 127.0.0.1,172.30.240.11 -dns-names localhost - && es-gencert-cli create-node -out ./node2 -ip-addresses 127.0.0.1,172.30.240.12 -dns-names localhost - && es-gencert-cli create-node -out ./node3 -ip-addresses 127.0.0.1,172.30.240.13 -dns-names localhost - && find . -type f -print0 | xargs -0 chmod 666" + build: + context: ./ + dockerfile: scripts/certificates/Dockerfile volumes: - "./certs:/certs" - depends_on: - - volumes-provisioner + network_mode: none esdb-node1: build: @@ -33,8 +18,8 @@ services: environment: - EVENTSTORE_GOSSIP_SEED=172.30.240.12:2113,172.30.240.13:2113 - EVENTSTORE_REPLICATION_IP=172.30.240.11 - - EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node1/node.crt - - EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node1/node.key + - EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt + - EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key - EVENTSTORE_ADVERTISE_HOST_TO_CLIENT_AS=127.0.0.1 - EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2111 ports: @@ -43,10 +28,12 @@ services: clusternetwork: ipv4_address: 172.30.240.11 volumes: - - ./certs:/etc/eventstore/certs + - ./certs/ca:/etc/eventstore/certs/ca:ro + - ./certs/node1:/etc/eventstore/certs/node:ro restart: unless-stopped depends_on: - - cert-gen + cert-gen: + condition: service_completed_successfully esdb-node2: build: @@ -58,8 +45,8 @@ services: environment: - EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.13:2113 - EVENTSTORE_REPLICATION_IP=172.30.240.12 - - EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node2/node.crt - - EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node2/node.key + - EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt + - EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key - EVENTSTORE_ADVERTISE_HOST_TO_CLIENT_AS=127.0.0.1 - EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2112 ports: @@ -68,10 +55,12 @@ services: clusternetwork: ipv4_address: 172.30.240.12 volumes: - - ./certs:/etc/eventstore/certs + - ./certs/ca:/etc/eventstore/certs/ca:ro + - ./certs/node2:/etc/eventstore/certs/node:ro restart: unless-stopped depends_on: - - cert-gen + cert-gen: + condition: service_completed_successfully esdb-node3: build: @@ -83,8 +72,8 @@ services: environment: - EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.12:2113 - EVENTSTORE_REPLICATION_IP=172.30.240.13 - - EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node3/node.crt - - EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node3/node.key + - EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt + - EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key - EVENTSTORE_ADVERTISE_HOST_TO_CLIENT_AS=127.0.0.1 - EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2113 ports: @@ -93,10 +82,12 @@ services: clusternetwork: ipv4_address: 172.30.240.13 volumes: - - ./certs:/etc/eventstore/certs + - ./certs/ca:/etc/eventstore/certs/ca:ro + - ./certs/node3:/etc/eventstore/certs/node:ro restart: unless-stopped depends_on: - - cert-gen + cert-gen: + condition: service_completed_successfully networks: clusternetwork: diff --git a/docs/installation.md b/docs/installation.md index fb5c8561a2..8e4a2a9166 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -67,7 +67,9 @@ container. Before running a durable node or cluster: -- Provide node certificates explicitly. +- Choose an approved PKI or issuer and follow the [production certificate management](security.md#production-certificate-management) guidance. +- Issue a separate node certificate with the required SAN, common-name, and key-usage policy for each node. +- Mount node certificates, private keys, and trusted roots read-only, and automate renewal plus explicit reload or rolling restart. - Decide whether clients use TLS and configure the connection strings accordingly. - Configure authentication methods in [Security](security.md). diff --git a/docs/operations.md b/docs/operations.md index 573203ee18..621cbeac1e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -507,6 +507,10 @@ As of version 23.10.0, it is possible to do a rolling update with new certificat The next step is to replace the outdated certificates with the newly generated certificates. +Replace each node certificate and its private key as one versioned unit. Do not expose a new certificate with an old key, or a new key with an old certificate. Kubernetes Secret and CSI projections can take time to update, so confirm that all mounted files contain the expected version before reloading the node. Certificate files mounted with `subPath` do not receive Secret updates. + +When rotating the CA, establish overlapping trust before changing node certificates. Add both the old and new trusted roots to every node and client. Reload or restart every process, or verify that a client dynamically reloads its trust bundle, before issuing or activating any node certificate from the new CA. Then issue and activate node certificates from the new CA one node at a time. Remove the old root only after all node and client traffic has been verified against the new chain. + If you are using symlinks, then you can update the symlink to point it to the new certificates. #### Linux OS @@ -531,6 +535,8 @@ You can reload the certificate configuration without restarting the node by call `Operations.ReloadConfig` method with the credentials of an `admin` or `ops` user. You can also reload the configuration from the _Operations_ page of the Admin UI. +Automated certificate renewal is not complete until the running node loads the new files. If your certificate controller cannot call `Operations.ReloadConfig` safely, perform a rolling restart one node at a time. Verify successful certificate-loading logs, readiness, and cluster membership before continuing to the next node. + #### Linux OS Linux users can also send the SIGHUP signal to the TrogonEventStore to reload the certificates. diff --git a/docs/security.md b/docs/security.md index 6132a047e1..b9eb538865 100644 --- a/docs/security.md +++ b/docs/security.md @@ -323,6 +323,8 @@ If multiple matching root certificates are found, then the root certificate with Use your platform certificate tooling, public CA, or private CA process to create certificates that match your deployment. The built-in development mode can generate a certificate for one secure node on localhost, but it is not a cluster PKI bootstrap mechanism. +The repository Docker Compose configuration uses a small OpenSSL-based bootstrap helper for disposable local clusters. It creates a private CA for the persisted local certificate set, but does not retain the CA private key in the directory mounted into the nodes. It is not a production PKI. + For a multi-node deployment, provision: - A trusted CA certificate available to every node and client. @@ -337,6 +339,23 @@ compatibility. Keep CA private keys outside the node and client environments. Install only the CA certificate, node certificate, and node private key required by each machine. +#### Production certificate management + +TrogonEventStore does not integrate directly with a certificate authority or secret store. It loads the node certificate, private key, intermediate chain, and trusted roots from the configured files. Use the certificate lifecycle system already operated by your platform to issue and deliver those files. + +For Kubernetes, issue a separate certificate for every StatefulSet pod or stable node identity. Use [cert-manager](https://cert-manager.io/) with an approved issuer, or a Secrets Store CSI provider, and mount the certificate, private key, and trusted roots as read-only volumes. Do not mount certificate files using `subPath`, because Kubernetes does not propagate Secret updates to those mounts. Limit access to the TrogonEventStore service account. + +Common provider patterns include: + +- **Vault PKI:** use a constrained Vault PKI role through cert-manager's Vault issuer or an authenticated Vault Agent. Permit only the DNS names, IP addresses, usages, and lifetimes required by the cluster. +- **AWS:** use [AWS Private CA Connector for Kubernetes](https://docs.aws.amazon.com/privateca/latest/userguide/PcaKubernetes.html) with cert-manager. Request both Server Authentication and Client Authentication usages for node certificates. +- **Azure:** use [Azure Key Vault with the Secrets Store CSI driver](https://learn.microsoft.com/azure/aks/csi-secrets-store-driver). Retrieve a certificate as a secret when the node needs its private key; Key Vault key or certificate objects alone do not provide the complete private-key material to the file-based loader. +- **Google Cloud:** issue node certificates from Certificate Authority Service and deliver the PEM files through a Kubernetes Secret or [Secret Manager CSI](https://cloud.google.com/secret-manager/docs/secret-manager-managed-csi-component). + +Managed load-balancer certificates do not replace node-to-node certificate authentication. The current file-based loader also cannot use a non-exportable private key held only by an HSM or cloud key service. + +Certificate renewal updates the mounted files but does not by itself activate them in the running process. After the files change, reload each node using `Operations.ReloadConfig`, the Admin UI, or `SIGHUP` on Linux. If the platform cannot invoke reload safely, restart nodes one at a time. See [Certificate update upon expiry](operations.md#certificate-update-upon-expiry). + ::: warning Keep certificate private keys readable only by the account running TrogonEventStore. Restrictive permissions reduce the risk of another local account reading the key. diff --git a/scripts/certificates/Dockerfile b/scripts/certificates/Dockerfile new file mode 100644 index 0000000000..13b5bbdb8c --- /dev/null +++ b/scripts/certificates/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 + +RUN apk add --no-cache openssl=3.5.7-r0 + +COPY --chmod=755 scripts/certificates/generate-cluster-certificates.sh /usr/local/bin/generate-cluster-certificates + +ENTRYPOINT ["/usr/local/bin/generate-cluster-certificates"] diff --git a/scripts/certificates/generate-cluster-certificates.sh b/scripts/certificates/generate-cluster-certificates.sh new file mode 100755 index 0000000000..6fa84acb86 --- /dev/null +++ b/scripts/certificates/generate-cluster-certificates.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env sh + +set -eu + +output_directory="${CERT_OUTPUT_DIRECTORY:-/certs}" +ca_days="${CERT_CA_DAYS:-3650}" +node_days="${CERT_NODE_DAYS:-825}" +node_common_name="${CERT_NODE_COMMON_NAME:-trogondb-node}" +output_owner="${CERT_OUTPUT_OWNER:-10000:10000}" + +expected_files=" +$output_directory/ca/ca.crt +$output_directory/node1/node.crt +$output_directory/node1/node.key +$output_directory/node2/node.crt +$output_directory/node2/node.key +$output_directory/node3/node.crt +$output_directory/node3/node.key +" + +validate_node() { + node_name="$1" + node_ip="$2" + node_directory="$output_directory/$node_name" + + openssl verify -CAfile "$output_directory/ca/ca.crt" "$node_directory/node.crt" >/dev/null + openssl verify -purpose sslserver -CAfile "$output_directory/ca/ca.crt" "$node_directory/node.crt" >/dev/null + openssl verify -purpose sslclient -CAfile "$output_directory/ca/ca.crt" "$node_directory/node.crt" >/dev/null + openssl x509 -in "$node_directory/node.crt" -noout -checkend 0 >/dev/null + openssl x509 -in "$node_directory/node.crt" -noout -checkhost localhost >/dev/null + openssl x509 -in "$node_directory/node.crt" -noout -checkhost "esdb-$node_name" >/dev/null + openssl x509 -in "$node_directory/node.crt" -noout -checkip 127.0.0.1 >/dev/null + openssl x509 -in "$node_directory/node.crt" -noout -checkip "$node_ip" >/dev/null + test "$(openssl x509 -in "$node_directory/node.crt" -noout -subject -nameopt RFC2253)" = "subject=CN=$node_common_name" + + certificate_public_key="$(mktemp)" + private_public_key="$(mktemp)" + openssl x509 -in "$node_directory/node.crt" -pubkey -noout >"$certificate_public_key" + openssl pkey -in "$node_directory/node.key" -pubout >"$private_public_key" 2>/dev/null + cmp "$certificate_public_key" "$private_public_key" >/dev/null + rm -f "$certificate_public_key" "$private_public_key" +} + +normalize_output_permissions() { + chown -R "$output_owner" "$output_directory" + chmod 755 "$output_directory" "$output_directory/ca" + chmod 700 "$output_directory"/node* + chmod 600 "$output_directory"/node*/node.key + chmod 644 "$output_directory/ca/ca.crt" "$output_directory"/node*/node.crt +} + +validate_existing_certificates() { + openssl verify -CAfile "$output_directory/ca/ca.crt" "$output_directory/ca/ca.crt" >/dev/null + validate_node node1 172.30.240.11 + validate_node node2 172.30.240.12 + validate_node node3 172.30.240.13 +} + +existing_files=0 +missing_files=0 + +for output_path in "$output_directory" "$output_directory/ca" "$output_directory/node1" "$output_directory/node2" "$output_directory/node3"; do + if [ -L "$output_path" ] || { [ -e "$output_path" ] && [ ! -d "$output_path" ]; }; then + echo "Certificate output path '$output_path' must be a real directory, not a link or another file type." >&2 + exit 1 + fi +done + +if [ -d "$output_directory/ca" ]; then + unexpected_ca_entry="$(find "$output_directory/ca" -mindepth 1 -maxdepth 1 ! -name ca.crt -print -quit)" + if [ -n "$unexpected_ca_entry" ]; then + echo "Unexpected content exists in '$output_directory/ca'. Remove the local certificate directory and regenerate it so only the public ca.crt is exposed to nodes." >&2 + exit 1 + fi +fi + +for expected_file in $expected_files; do + if [ -L "$expected_file" ] || { [ -e "$expected_file" ] && [ ! -f "$expected_file" ]; }; then + echo "Certificate output '$expected_file' must be a regular file, not a link or another file type." >&2 + exit 1 + elif [ -f "$expected_file" ]; then + existing_files=$((existing_files + 1)) + else + missing_files=$((missing_files + 1)) + fi +done + +if [ "$existing_files" -gt 0 ]; then + if [ "$missing_files" -gt 0 ]; then + echo "Certificate output is incomplete. Remove '$output_directory' before regenerating it." >&2 + exit 1 + fi + + normalize_output_permissions + validate_existing_certificates + echo "Using the existing validated cluster certificates in '$output_directory'." + exit 0 +fi + +umask 077 +mkdir -p "$output_directory/ca" "$output_directory/node1" "$output_directory/node2" "$output_directory/node3" +private_directory="$(mktemp -d)" +trap 'rm -rf "$private_directory"' EXIT +ca_key="$private_directory/ca.key" + +openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:3072 -out "$ca_key" 2>/dev/null +openssl req -x509 -new -sha256 \ + -key "$ca_key" \ + -out "$output_directory/ca/ca.crt" \ + -days "$ca_days" \ + -subj "/CN=TrogonEventStore Development CA" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -addext "subjectKeyIdentifier=hash" + +generate_node() { + node_name="$1" + node_ip="$2" + serial_number="$3" + node_directory="$output_directory/$node_name" + extension_file="$private_directory/$node_name.extensions" + request_file="$private_directory/$node_name.csr" + + cat >"$extension_file" </dev/null + openssl x509 -req -sha256 \ + -in "$request_file" \ + -CA "$output_directory/ca/ca.crt" \ + -CAkey "$ca_key" \ + -set_serial "$serial_number" \ + -days "$node_days" \ + -extfile "$extension_file" \ + -extensions node \ + -out "$node_directory/node.crt" +} + +generate_node node1 172.30.240.11 1001 +generate_node node2 172.30.240.12 1002 +generate_node node3 172.30.240.13 1003 + +normalize_output_permissions +validate_existing_certificates +echo "Generated and validated cluster certificates in '$output_directory'."