Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion .github/workflows/common.yml
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,24 @@ jobs:
NUGET_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
docker compose build
docker compose run --rm cert-gen
docker compose run --rm --entrypoint sh cert-gen -c '
test -z "$(find /certs/ca -mindepth 1 -maxdepth 1 ! -name ca.crt -print -quit)"
test "$(stat -c %a /certs/ca/ca.crt)" = 644
for certificate in /certs/node*/node.crt; do
test "$(stat -c %a "$certificate")" = 644
done
for private_key in /certs/node*/node.key; do
test "$(stat -c %a "$private_key")" = 600
done
'
docker compose run --rm --entrypoint sh cert-gen -c 'sha256sum /certs/ca/ca.crt /certs/node*/node.crt /certs/node*/node.key' > /tmp/certificate-fingerprints.before
docker compose run --rm cert-gen
docker compose run --rm --entrypoint sh cert-gen -c 'sha256sum /certs/ca/ca.crt /certs/node*/node.crt /certs/node*/node.key' > /tmp/certificate-fingerprints.after
diff --unified /tmp/certificate-fingerprints.before /tmp/certificate-fingerprints.after
Comment thread
coderabbitai[bot] marked this conversation as resolved.
docker compose up --detach
printf 'Waiting for cluster'
timeout 60 bash -c -- 'until curl --output /dev/null --silent --insecure --fail https://localhost:2113/-/readiness; do printf '.'; sleep 2; done'
for port in 2111 2112 2113; do
timeout 60 bash -c -- "until curl --output /dev/null --silent --fail --cacert certs/ca/ca.crt https://localhost:$port/-/readiness; do printf '.'; sleep 2; done"
done
docker compose down
55 changes: 23 additions & 32 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,27 +1,12 @@
services:
volumes-provisioner:
image: hasnat/volumes-provisioner
environment:
PROVISION_DIRECTORIES: "1000:1000:0755:/tmp/certs"
volumes:
- "./certs:/tmp/certs"
network_mode: none

# Local development only. Production certificates must come from the deployment platform's PKI.
cert-gen:
image: eventstore/es-gencert-cli:1.0.2
entrypoint: bash
user: "1000:1000"
command: >
-c "mkdir -p ./certs && cd /certs
&& es-gencert-cli create-ca
&& es-gencert-cli create-node -out ./node1 -ip-addresses 127.0.0.1,172.30.240.11 -dns-names localhost
&& es-gencert-cli create-node -out ./node2 -ip-addresses 127.0.0.1,172.30.240.12 -dns-names localhost
&& es-gencert-cli create-node -out ./node3 -ip-addresses 127.0.0.1,172.30.240.13 -dns-names localhost
&& find . -type f -print0 | xargs -0 chmod 666"
build:
context: ./
dockerfile: scripts/certificates/Dockerfile
volumes:
- "./certs:/certs"
depends_on:
- volumes-provisioner
network_mode: none

esdb-node1:
build:
Expand All @@ -33,8 +18,8 @@ services:
environment:
- EVENTSTORE_GOSSIP_SEED=172.30.240.12:2113,172.30.240.13:2113
- EVENTSTORE_REPLICATION_IP=172.30.240.11
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node1/node.crt
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node1/node.key
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key
- EVENTSTORE_ADVERTISE_HOST_TO_CLIENT_AS=127.0.0.1
- EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2111
ports:
Expand All @@ -43,10 +28,12 @@ services:
clusternetwork:
ipv4_address: 172.30.240.11
volumes:
- ./certs:/etc/eventstore/certs
- ./certs/ca:/etc/eventstore/certs/ca:ro
- ./certs/node1:/etc/eventstore/certs/node:ro
restart: unless-stopped
depends_on:
- cert-gen
cert-gen:
condition: service_completed_successfully

esdb-node2:
build:
Expand All @@ -58,8 +45,8 @@ services:
environment:
- EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.13:2113
- EVENTSTORE_REPLICATION_IP=172.30.240.12
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node2/node.crt
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node2/node.key
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key
- EVENTSTORE_ADVERTISE_HOST_TO_CLIENT_AS=127.0.0.1
- EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2112
ports:
Expand All @@ -68,10 +55,12 @@ services:
clusternetwork:
ipv4_address: 172.30.240.12
volumes:
- ./certs:/etc/eventstore/certs
- ./certs/ca:/etc/eventstore/certs/ca:ro
- ./certs/node2:/etc/eventstore/certs/node:ro
restart: unless-stopped
depends_on:
- cert-gen
cert-gen:
condition: service_completed_successfully

esdb-node3:
build:
Expand All @@ -83,8 +72,8 @@ services:
environment:
- EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.12:2113
- EVENTSTORE_REPLICATION_IP=172.30.240.13
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node3/node.crt
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node3/node.key
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key
- EVENTSTORE_ADVERTISE_HOST_TO_CLIENT_AS=127.0.0.1
- EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2113
ports:
Expand All @@ -93,10 +82,12 @@ services:
clusternetwork:
ipv4_address: 172.30.240.13
volumes:
- ./certs:/etc/eventstore/certs
- ./certs/ca:/etc/eventstore/certs/ca:ro
- ./certs/node3:/etc/eventstore/certs/node:ro
restart: unless-stopped
depends_on:
- cert-gen
cert-gen:
condition: service_completed_successfully

networks:
clusternetwork:
Expand Down
4 changes: 3 additions & 1 deletion docs/installation.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,9 @@ container.

Before running a durable node or cluster:

- Provide node certificates explicitly.
- Choose an approved PKI or issuer and follow the [production certificate management](security.md#production-certificate-management) guidance.
- Issue a separate node certificate with the required SAN, common-name, and key-usage policy for each node.
- Mount node certificates, private keys, and trusted roots read-only, and automate renewal plus explicit reload or rolling restart.
- Decide whether clients use TLS and configure the connection strings
accordingly.
- Configure authentication methods in [Security](security.md).
Expand Down
6 changes: 6 additions & 0 deletions docs/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,10 @@ As of version 23.10.0, it is possible to do a rolling update with new certificat

The next step is to replace the outdated certificates with the newly generated certificates.

Replace each node certificate and its private key as one versioned unit. Do not expose a new certificate with an old key, or a new key with an old certificate. Kubernetes Secret and CSI projections can take time to update, so confirm that all mounted files contain the expected version before reloading the node. Certificate files mounted with `subPath` do not receive Secret updates.

When rotating the CA, establish overlapping trust before changing node certificates. Add both the old and new trusted roots to every node and client. Reload or restart every process, or verify that a client dynamically reloads its trust bundle, before issuing or activating any node certificate from the new CA. Then issue and activate node certificates from the new CA one node at a time. Remove the old root only after all node and client traffic has been verified against the new chain.

If you are using symlinks, then you can update the symlink to point it to the new certificates.

#### Linux OS
Expand All @@ -531,6 +535,8 @@ You can reload the certificate configuration without restarting the node by call
`Operations.ReloadConfig` method with the credentials of an `admin` or `ops` user. You can also
reload the configuration from the _Operations_ page of the Admin UI.

Automated certificate renewal is not complete until the running node loads the new files. If your certificate controller cannot call `Operations.ReloadConfig` safely, perform a rolling restart one node at a time. Verify successful certificate-loading logs, readiness, and cluster membership before continuing to the next node.

#### Linux OS

Linux users can also send the SIGHUP signal to the TrogonEventStore to reload the certificates.
Expand Down
19 changes: 19 additions & 0 deletions docs/security.md
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,8 @@ If multiple matching root certificates are found, then the root certificate with

Use your platform certificate tooling, public CA, or private CA process to create certificates that match your deployment. The built-in development mode can generate a certificate for one secure node on localhost, but it is not a cluster PKI bootstrap mechanism.

The repository Docker Compose configuration uses a small OpenSSL-based bootstrap helper for disposable local clusters. It creates a private CA for the persisted local certificate set, but does not retain the CA private key in the directory mounted into the nodes. It is not a production PKI.

For a multi-node deployment, provision:

- A trusted CA certificate available to every node and client.
Expand All @@ -337,6 +339,23 @@ compatibility.

Keep CA private keys outside the node and client environments. Install only the CA certificate, node certificate, and node private key required by each machine.

#### Production certificate management

TrogonEventStore does not integrate directly with a certificate authority or secret store. It loads the node certificate, private key, intermediate chain, and trusted roots from the configured files. Use the certificate lifecycle system already operated by your platform to issue and deliver those files.

For Kubernetes, issue a separate certificate for every StatefulSet pod or stable node identity. Use [cert-manager](https://cert-manager.io/) with an approved issuer, or a Secrets Store CSI provider, and mount the certificate, private key, and trusted roots as read-only volumes. Do not mount certificate files using `subPath`, because Kubernetes does not propagate Secret updates to those mounts. Limit access to the TrogonEventStore service account.

Common provider patterns include:

- **Vault PKI:** use a constrained Vault PKI role through cert-manager's Vault issuer or an authenticated Vault Agent. Permit only the DNS names, IP addresses, usages, and lifetimes required by the cluster.
- **AWS:** use [AWS Private CA Connector for Kubernetes](https://docs.aws.amazon.com/privateca/latest/userguide/PcaKubernetes.html) with cert-manager. Request both Server Authentication and Client Authentication usages for node certificates.
- **Azure:** use [Azure Key Vault with the Secrets Store CSI driver](https://learn.microsoft.com/azure/aks/csi-secrets-store-driver). Retrieve a certificate as a secret when the node needs its private key; Key Vault key or certificate objects alone do not provide the complete private-key material to the file-based loader.
- **Google Cloud:** issue node certificates from Certificate Authority Service and deliver the PEM files through a Kubernetes Secret or [Secret Manager CSI](https://cloud.google.com/secret-manager/docs/secret-manager-managed-csi-component).

Managed load-balancer certificates do not replace node-to-node certificate authentication. The current file-based loader also cannot use a non-exportable private key held only by an HSM or cloud key service.

Certificate renewal updates the mounted files but does not by itself activate them in the running process. After the files change, reload each node using `Operations.ReloadConfig`, the Admin UI, or `SIGHUP` on Linux. If the platform cannot invoke reload safely, restart nodes one at a time. See [Certificate update upon expiry](operations.md#certificate-update-upon-expiry).

::: warning
Keep certificate private keys readable only by the account running TrogonEventStore. Restrictive permissions
reduce the risk of another local account reading the key.
Expand Down
7 changes: 7 additions & 0 deletions scripts/certificates/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1

RUN apk add --no-cache openssl=3.5.7-r0

COPY --chmod=755 scripts/certificates/generate-cluster-certificates.sh /usr/local/bin/generate-cluster-certificates

ENTRYPOINT ["/usr/local/bin/generate-cluster-certificates"]
155 changes: 155 additions & 0 deletions scripts/certificates/generate-cluster-certificates.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
#!/usr/bin/env sh

set -eu

output_directory="${CERT_OUTPUT_DIRECTORY:-/certs}"
ca_days="${CERT_CA_DAYS:-3650}"
node_days="${CERT_NODE_DAYS:-825}"
node_common_name="${CERT_NODE_COMMON_NAME:-trogondb-node}"
output_owner="${CERT_OUTPUT_OWNER:-10000:10000}"

expected_files="
$output_directory/ca/ca.crt
$output_directory/node1/node.crt
$output_directory/node1/node.key
$output_directory/node2/node.crt
$output_directory/node2/node.key
$output_directory/node3/node.crt
$output_directory/node3/node.key
"

validate_node() {
node_name="$1"
node_ip="$2"
node_directory="$output_directory/$node_name"

openssl verify -CAfile "$output_directory/ca/ca.crt" "$node_directory/node.crt" >/dev/null
openssl verify -purpose sslserver -CAfile "$output_directory/ca/ca.crt" "$node_directory/node.crt" >/dev/null
openssl verify -purpose sslclient -CAfile "$output_directory/ca/ca.crt" "$node_directory/node.crt" >/dev/null
openssl x509 -in "$node_directory/node.crt" -noout -checkend 0 >/dev/null
openssl x509 -in "$node_directory/node.crt" -noout -checkhost localhost >/dev/null
openssl x509 -in "$node_directory/node.crt" -noout -checkhost "esdb-$node_name" >/dev/null
openssl x509 -in "$node_directory/node.crt" -noout -checkip 127.0.0.1 >/dev/null
openssl x509 -in "$node_directory/node.crt" -noout -checkip "$node_ip" >/dev/null
test "$(openssl x509 -in "$node_directory/node.crt" -noout -subject -nameopt RFC2253)" = "subject=CN=$node_common_name"

certificate_public_key="$(mktemp)"
private_public_key="$(mktemp)"
openssl x509 -in "$node_directory/node.crt" -pubkey -noout >"$certificate_public_key"
openssl pkey -in "$node_directory/node.key" -pubout >"$private_public_key" 2>/dev/null
cmp "$certificate_public_key" "$private_public_key" >/dev/null
rm -f "$certificate_public_key" "$private_public_key"
}

normalize_output_permissions() {
chown -R "$output_owner" "$output_directory"
chmod 755 "$output_directory" "$output_directory/ca"
chmod 700 "$output_directory"/node*
chmod 600 "$output_directory"/node*/node.key
chmod 644 "$output_directory/ca/ca.crt" "$output_directory"/node*/node.crt
}
Comment thread
cursor[bot] marked this conversation as resolved.

validate_existing_certificates() {
openssl verify -CAfile "$output_directory/ca/ca.crt" "$output_directory/ca/ca.crt" >/dev/null
validate_node node1 172.30.240.11
validate_node node2 172.30.240.12
validate_node node3 172.30.240.13
}

existing_files=0
missing_files=0

for output_path in "$output_directory" "$output_directory/ca" "$output_directory/node1" "$output_directory/node2" "$output_directory/node3"; do
if [ -L "$output_path" ] || { [ -e "$output_path" ] && [ ! -d "$output_path" ]; }; then
echo "Certificate output path '$output_path' must be a real directory, not a link or another file type." >&2
exit 1
fi
done

if [ -d "$output_directory/ca" ]; then
unexpected_ca_entry="$(find "$output_directory/ca" -mindepth 1 -maxdepth 1 ! -name ca.crt -print -quit)"
if [ -n "$unexpected_ca_entry" ]; then
echo "Unexpected content exists in '$output_directory/ca'. Remove the local certificate directory and regenerate it so only the public ca.crt is exposed to nodes." >&2
exit 1
fi
fi

for expected_file in $expected_files; do
if [ -L "$expected_file" ] || { [ -e "$expected_file" ] && [ ! -f "$expected_file" ]; }; then
echo "Certificate output '$expected_file' must be a regular file, not a link or another file type." >&2
exit 1
elif [ -f "$expected_file" ]; then
existing_files=$((existing_files + 1))
else
missing_files=$((missing_files + 1))
fi
done

if [ "$existing_files" -gt 0 ]; then
if [ "$missing_files" -gt 0 ]; then
echo "Certificate output is incomplete. Remove '$output_directory' before regenerating it." >&2
exit 1
fi

normalize_output_permissions
validate_existing_certificates
echo "Using the existing validated cluster certificates in '$output_directory'."
exit 0
fi

umask 077
mkdir -p "$output_directory/ca" "$output_directory/node1" "$output_directory/node2" "$output_directory/node3"
private_directory="$(mktemp -d)"
trap 'rm -rf "$private_directory"' EXIT
ca_key="$private_directory/ca.key"

openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:3072 -out "$ca_key" 2>/dev/null
openssl req -x509 -new -sha256 \
-key "$ca_key" \
-out "$output_directory/ca/ca.crt" \
-days "$ca_days" \
-subj "/CN=TrogonEventStore Development CA" \
-addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
-addext "keyUsage=critical,keyCertSign,cRLSign" \
-addext "subjectKeyIdentifier=hash"

generate_node() {
node_name="$1"
node_ip="$2"
serial_number="$3"
node_directory="$output_directory/$node_name"
extension_file="$private_directory/$node_name.extensions"
request_file="$private_directory/$node_name.csr"

cat >"$extension_file" <<EOF
[node]
basicConstraints=critical,CA:FALSE
keyUsage=critical,digitalSignature,keyEncipherment
extendedKeyUsage=serverAuth,clientAuth
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
subjectAltName=DNS:localhost,DNS:esdb-$node_name,IP:127.0.0.1,IP:$node_ip
EOF

openssl req -new -newkey rsa:3072 -nodes -sha256 \
-keyout "$node_directory/node.key" \
-out "$request_file" \
-subj "/CN=$node_common_name" 2>/dev/null
openssl x509 -req -sha256 \
-in "$request_file" \
-CA "$output_directory/ca/ca.crt" \
-CAkey "$ca_key" \
-set_serial "$serial_number" \
-days "$node_days" \
-extfile "$extension_file" \
-extensions node \
-out "$node_directory/node.crt"
}

generate_node node1 172.30.240.11 1001
generate_node node2 172.30.240.12 1002
generate_node node3 172.30.240.13 1003

normalize_output_permissions
validate_existing_certificates
echo "Generated and validated cluster certificates in '$output_directory'."
Loading