From fe878cb96d8386ddc34827b1432adafce92a9d34 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Fri, 18 Sep 2026 15:45:58 +0200 Subject: [PATCH 1/3] Revert "WPB-28645: presence cleanup must not swallow async exceptions (#5535)" This reverts commit da4e4baafd00fe3c5fd5f03d60c806bddca923f3. --- changelog.d/3-bug-fixes/WPB-28645 | 4 ---- services/gundeck/src/Gundeck/Run.hs | 11 ++++------- 2 files changed, 4 insertions(+), 11 deletions(-) delete mode 100644 changelog.d/3-bug-fixes/WPB-28645 diff --git a/changelog.d/3-bug-fixes/WPB-28645 b/changelog.d/3-bug-fixes/WPB-28645 deleted file mode 100644 index 570d00f42f..0000000000 --- a/changelog.d/3-bug-fixes/WPB-28645 +++ /dev/null @@ -1,4 +0,0 @@ -Gundeck's presence cleanup background thread no longer swallows asynchronous -exceptions. On shutdown this removes the spurious "presence cleanup failed" -(AsyncCancelled) error log line and lets the thread terminate promptly instead -of lingering for up to an hour. diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs index 590fe96e1c..6f4b388d64 100644 --- a/services/gundeck/src/Gundeck/Run.hs +++ b/services/gundeck/src/Gundeck/Run.hs @@ -41,8 +41,8 @@ import Cassandra (runClient, shutdown) import Cassandra.Schema (versionCheck) import Control.Error (ExceptT (ExceptT)) import Control.Exception (finally) -import Control.Exception.Safe (catchAny) import Control.Lens ((.~), (^.)) +import Control.Monad.Catch (catchAll) import Control.Monad.Extra import Data.Map qualified as Map import Data.Metrics.AWS (gaugeTokenRemaing) @@ -183,18 +183,15 @@ collectAuthMetrics env = do -- | Hourly janitor replacing the redis key TTL: deletes presence rows older -- than a week (leak guard for abnormally dead pods). Never let a transient DB --- error kill the thread — log and retry next hour. Async exceptions (e.g. --- 'AsyncCancelled' from 'Async.cancel' during shutdown) propagate because --- 'Control.Exception.Safe.catchAny' rethrows asynchronously-delivered --- exceptions and only handles synchronous ones. +-- error kill the thread — log and retry next hour. cleanupPresenceLoop :: Log.Logger -> Gundeck () cleanupPresenceLoop logger = forever $ (PresenceData.cleanup >> threadDelay cleanupInterval) - `catchAny` \e -> do + `catchAll` \e -> do liftIO . Log.err logger $ Log.msg (Log.val "presence cleanup failed") - . Log.field "error" (displayException e) + . Log.field "error" (displayException (e :: SomeException)) threadDelay cleanupInterval cleanupInterval :: Int From dde9842f0829b6b33be35826f0293d53e063dc26 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Fri, 18 Sep 2026 15:46:01 +0200 Subject: [PATCH 2/3] Revert "WPB-28377: migrate gundeck presence from redis to PostGreSQL (#5493)" This reverts commit 9a39b9cb4a87f21de51d3e6bcd3eea5c6008aedf. --- Makefile | 4 +- .../0-release-notes/WPB-28377-remove-redis | 17 -- .../WPB-28377-gundeck-presence-postgres | 1 - charts/databases-ephemeral/requirements.yaml | 7 + .../databases-ephemeral/templates/NOTES.txt | 1 + .../templates/integration-integration.yaml | 27 +- charts/integration/templates/secret.yaml | 6 + charts/reaper/.helmignore | 21 ++ charts/reaper/Chart.yaml | 10 + charts/reaper/README.md | 71 +++++ charts/reaper/scripts/reaper.sh | 89 +++++++ charts/reaper/templates/_helpers.tpl | 26 ++ charts/reaper/templates/configmap.yaml | 10 + charts/reaper/templates/deployment.yaml | 81 ++++++ charts/reaper/templates/rbac.yaml | 38 +++ charts/reaper/values.yaml | 45 ++++ charts/redis-ephemeral/Chart.yaml | 4 + charts/redis-ephemeral/requirements.yaml | 5 + charts/redis-ephemeral/values.yaml | 60 +++++ charts/wire-server/templates/_helpers.tpl | 40 +++ .../templates/cannon/statefulset.yaml | 2 +- .../templates/gundeck/configmap.yaml | 24 +- .../templates/gundeck/deployment.yaml | 49 +++- .../templates/gundeck/redis-ca-secret.yaml | 30 +++ .../wire-server/templates/gundeck/secret.yaml | 13 +- .../templates/gundeck/tests/configmap.yaml | 7 + .../gundeck/tests/gundeck-integration.yaml | 37 +++ .../templates/gundeck/tests/secret.yaml | 6 + charts/wire-server/values.yaml | 40 ++- deploy/dockerephemeral/docker-compose.yaml | 142 ++++++++++ .../docker/redis-master-mode.conf | 1 + .../docker/redis-node-1-cert.pem | 19 ++ .../docker/redis-node-1-key.pem | 28 ++ .../dockerephemeral/docker/redis-node-1.conf | 17 ++ .../docker/redis-node-2-cert.pem | 19 ++ .../docker/redis-node-2-key.pem | 28 ++ .../dockerephemeral/docker/redis-node-2.conf | 17 ++ .../docker/redis-node-3-cert.pem | 19 ++ .../docker/redis-node-3-key.pem | 28 ++ .../dockerephemeral/docker/redis-node-3.conf | 17 ++ .../docker/redis-node-4-cert.pem | 19 ++ .../docker/redis-node-4-key.pem | 28 ++ .../dockerephemeral/docker/redis-node-4.conf | 17 ++ .../docker/redis-node-5-cert.pem | 19 ++ .../docker/redis-node-5-key.pem | 28 ++ .../dockerephemeral/docker/redis-node-5.conf | 17 ++ .../docker/redis-node-6-cert.pem | 19 ++ .../docker/redis-node-6-key.pem | 28 ++ .../dockerephemeral/docker/redis-node-6.conf | 17 ++ docs/src/developer/developer/building.md | 1 + .../src/developer/reference/config-options.md | 94 ++++++- .../install/infrastructure-configuration.md | 5 +- docs/src/how-to/install/troubleshooting.md | 4 +- hack/bin/gen-certs.sh | 13 + hack/helm_vars/certs/values.yaml.gotmpl | 53 ++++ hack/helm_vars/redis-ephemeral/values.yaml | 47 ++++ hack/helm_vars/wire-server/values.yaml.gotmpl | 20 +- hack/helmfile.yaml.gotmpl | 36 +++ libs/wire-api/src/Wire/API/Presence.hs | 7 +- .../Test/Wire/API/Golden/Manual/Presence.hs | 3 + .../20260828093750-gundeck-presence.sql | 12 - .../src/Wire/JobSubsystem/Migrations.hs | 4 +- libs/wire-subsystems/src/Wire/Postgres.hs | 1 - postgres-schema.sql | 30 --- services/brig/src/Brig/Run.hs | 5 - services/gundeck/default.nix | 17 +- services/gundeck/gundeck.cabal | 14 +- services/gundeck/gundeck.integration.yaml | 23 +- services/gundeck/src/Gundeck/Env.hs | 81 +++++- services/gundeck/src/Gundeck/Monad.hs | 73 +++++ services/gundeck/src/Gundeck/Options.hs | 32 ++- services/gundeck/src/Gundeck/Presence.hs | 4 +- services/gundeck/src/Gundeck/Presence/Data.hs | 252 ++++++++---------- services/gundeck/src/Gundeck/Push.hs | 2 +- .../gundeck/src/Gundeck/Push/Websocket.hs | 6 +- services/gundeck/src/Gundeck/Redis.hs | 127 +++++++++ services/gundeck/src/Gundeck/Run.hs | 32 +-- services/gundeck/src/Gundeck/Util/Redis.hs | 61 +++++ services/gundeck/test/integration/API.hs | 67 ++++- services/gundeck/test/integration/Main.hs | 9 +- .../gundeck/test/integration/TestSetup.hs | 8 +- services/gundeck/test/integration/Util.hs | 119 +++++++++ services/gundeck/test/unit/MockGundeck.hs | 1 + services/integration.yaml | 6 + 84 files changed, 2239 insertions(+), 328 deletions(-) delete mode 100644 changelog.d/0-release-notes/WPB-28377-remove-redis delete mode 100644 changelog.d/5-internal/WPB-28377-gundeck-presence-postgres create mode 100644 charts/reaper/.helmignore create mode 100644 charts/reaper/Chart.yaml create mode 100644 charts/reaper/README.md create mode 100755 charts/reaper/scripts/reaper.sh create mode 100644 charts/reaper/templates/_helpers.tpl create mode 100644 charts/reaper/templates/configmap.yaml create mode 100644 charts/reaper/templates/deployment.yaml create mode 100644 charts/reaper/templates/rbac.yaml create mode 100644 charts/reaper/values.yaml create mode 100644 charts/redis-ephemeral/Chart.yaml create mode 100644 charts/redis-ephemeral/requirements.yaml create mode 100644 charts/redis-ephemeral/values.yaml create mode 100644 charts/wire-server/templates/gundeck/redis-ca-secret.yaml create mode 100644 deploy/dockerephemeral/docker/redis-master-mode.conf create mode 100644 deploy/dockerephemeral/docker/redis-node-1-cert.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-1-key.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-1.conf create mode 100644 deploy/dockerephemeral/docker/redis-node-2-cert.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-2-key.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-2.conf create mode 100644 deploy/dockerephemeral/docker/redis-node-3-cert.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-3-key.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-3.conf create mode 100644 deploy/dockerephemeral/docker/redis-node-4-cert.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-4-key.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-4.conf create mode 100644 deploy/dockerephemeral/docker/redis-node-5-cert.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-5-key.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-5.conf create mode 100644 deploy/dockerephemeral/docker/redis-node-6-cert.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-6-key.pem create mode 100644 deploy/dockerephemeral/docker/redis-node-6.conf create mode 100644 hack/helm_vars/redis-ephemeral/values.yaml delete mode 100644 libs/wire-subsystems/postgres-migrations/20260828093750-gundeck-presence.sql create mode 100644 services/gundeck/src/Gundeck/Redis.hs create mode 100644 services/gundeck/src/Gundeck/Util/Redis.hs create mode 100644 services/gundeck/test/integration/Util.hs diff --git a/Makefile b/Makefile index 0e2e771c98..2a3250275e 100644 --- a/Makefile +++ b/Makefile @@ -13,11 +13,11 @@ CHARTS_INTEGRATION := wire-server databases-ephemeral rabbitmq fake-aws ingre # (e.g. move charts/brig to charts/wire-server/brig) # this list could be generated from the folder names under ./charts/ like so: # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts) -CHARTS_RELEASE := wire-server rabbitmq rabbitmq-external databases-ephemeral \ +CHARTS_RELEASE := wire-server redis-ephemeral rabbitmq rabbitmq-external databases-ephemeral \ fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice \ calling-test demo-smtp elasticsearch-curator elasticsearch-external \ elasticsearch-ephemeral minio-external cassandra-external \ -ingress-nginx-controller nginx-ingress-services \ +ingress-nginx-controller nginx-ingress-services reaper \ k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise \ wire-ingress KIND_CLUSTER_NAME := wire-server diff --git a/changelog.d/0-release-notes/WPB-28377-remove-redis b/changelog.d/0-release-notes/WPB-28377-remove-redis deleted file mode 100644 index b7f75b78e8..0000000000 --- a/changelog.d/0-release-notes/WPB-28377-remove-redis +++ /dev/null @@ -1,17 +0,0 @@ -Gundeck no longer uses redis: presence tracking is stored in PostgreSQL. - -Operators must: - -- Remove redis deployments that were only used by gundeck, and the gundeck - `redis:` and `redisAdditionalWrite:` configuration, the `REDIS_USERNAME`, - `REDIS_PASSWORD`, `REDIS_ADDITIONAL_WRITE_USERNAME` and - `REDIS_ADDITIONAL_WRITE_PASSWORD` environment variables, and gundeck redis - TLS secrets (`redisUsername`/`redisPassword`/`redisAdditionalWrite*` secrets - and the redis CA certificates). -- Add the new required configuration `gundeck.config.postgresql` (plus - `postgresqlPool`, and optionally `secrets.pgPassword` for the password file), - following the same format as brig's postgresql settings. -- Restart all cannons after deployment is successful. Presence data does - not carry over, this will make sure all clients reconnect after the - presence data is being written to PostgreSQL. -- Stop deploying `redis-ephemeral` and `reaper`, these have been removed. diff --git a/changelog.d/5-internal/WPB-28377-gundeck-presence-postgres b/changelog.d/5-internal/WPB-28377-gundeck-presence-postgres deleted file mode 100644 index 3862322c3a..0000000000 --- a/changelog.d/5-internal/WPB-28377-gundeck-presence-postgres +++ /dev/null @@ -1 +0,0 @@ -Gundeck presence tracking moved from redis to postgres; redis and the gundeck redis configuration options are gone. (WPB-28377) diff --git a/charts/databases-ephemeral/requirements.yaml b/charts/databases-ephemeral/requirements.yaml index fff1534c38..74dbd7594d 100644 --- a/charts/databases-ephemeral/requirements.yaml +++ b/charts/databases-ephemeral/requirements.yaml @@ -13,6 +13,13 @@ dependencies: # since cassandra-migrations did not yet run; but the cassandra-migrations hook # requires all pods to be in a 'Ready' state before starting (condition for post-install); this is impossible. ##################################################### +- name: redis-ephemeral + version: "0.0.42" + repository: "file://../redis-ephemeral" + tags: + - redis-ephemeral + - databases-ephemeral + - demo - name: elasticsearch-ephemeral version: "0.0.42" repository: "file://../elasticsearch-ephemeral" diff --git a/charts/databases-ephemeral/templates/NOTES.txt b/charts/databases-ephemeral/templates/NOTES.txt index 7f07b9ed7c..2e2ad5b059 100644 --- a/charts/databases-ephemeral/templates/NOTES.txt +++ b/charts/databases-ephemeral/templates/NOTES.txt @@ -2,6 +2,7 @@ You now have an in-memory, non-persistent, non-highly-available set of databases * cassandra-ephemeral * elasticsearch-ephemeral +* redis-ephemeral !! WARNING WARNING !! This is fine for testing and demo purposes, but NOT for a production use case. diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 4841ef372b..9fea9fbde3 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -41,10 +41,6 @@ spec: configMap: name: "gundeck" - - name: "gundeck-secrets" - secret: - secretName: "gundeck" - - name: "cargohold-config" configMap: name: "cargohold" @@ -97,6 +93,9 @@ spec: secret: secretName: {{ .Values.config.elasticsearch.tlsCaSecretRef.name }} + - name: redis-ca + secret: + secretName: {{ .Values.config.redis.tlsCaSecretRef.name }} - name: rabbitmq-ca secret: @@ -238,9 +237,6 @@ spec: - name: gundeck-config mountPath: /etc/wire/gundeck/conf - - name: gundeck-secrets - mountPath: /etc/wire/gundeck/secrets - - name: cargohold-config mountPath: /etc/wire/cargohold/conf @@ -280,6 +276,9 @@ spec: - name: elasticsearch-ca mountPath: /etc/wire/brig/elasticsearch-ca + - name: redis-ca + mountPath: /etc/wire/gundeck/redis-ca + - name: rabbitmq-ca mountPath: /etc/wire/brig/rabbitmq-ca @@ -344,6 +343,20 @@ spec: - name: ENABLE_FEDERATION_V{{$version}} value: "1" {{- end }} + {{- if hasKey .Values.secrets "redisUsername" }} + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: integration + key: redisUsername + {{- end }} + {{- if hasKey .Values.secrets "redisPassword" }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: integration + key: redisPassword + {{- end }} - name: TEST_XML value: /tmp/result.xml {{- if .Values.config.uploadXml }} diff --git a/charts/integration/templates/secret.yaml b/charts/integration/templates/secret.yaml index 34e6698ed2..32f6085176 100644 --- a/charts/integration/templates/secret.yaml +++ b/charts/integration/templates/secret.yaml @@ -16,4 +16,10 @@ data: {{- if hasKey . "uploadXmlAwsSecretAccessKey" }} uploadXmlAwsSecretAccessKey: {{ .uploadXmlAwsSecretAccessKey | b64enc | quote }} {{- end }} + {{- if hasKey . "redisUsername" }} + redisUsername: {{ .redisUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisPassword" }} + redisPassword: {{ .redisPassword | b64enc | quote }} + {{- end }} {{- end }} diff --git a/charts/reaper/.helmignore b/charts/reaper/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/reaper/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/reaper/Chart.yaml b/charts/reaper/Chart.yaml new file mode 100644 index 0000000000..131654fa44 --- /dev/null +++ b/charts/reaper/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +version: 0.0.42 +name: reaper +appVersion: 0.1.0 +description: A helm charts to restart cannons if redis-ephemeal has died +annotations: + # must conform to https://github.com/helm/community/blob/main/hips/hip-0015.md + helm.sh/images: | + - name: kubectl + image: docker.io/alpine/kubectl:1.36.3 diff --git a/charts/reaper/README.md b/charts/reaper/README.md new file mode 100644 index 0000000000..f4b73e4e67 --- /dev/null +++ b/charts/reaper/README.md @@ -0,0 +1,71 @@ +Reaper +------ + +This pod is useful in the following scenario: You run wire-server alongside a single +redis-ephemeral (part of databases-ephemeral). If you have a different setup for redis, +do not use this chart. + +Due to the nature of pods and their ephemerality, there might be situations where a +redis-ephemeral pod is restarted. In such cases, wire clients will have stale +connections (they will have an active websocket connection, but gundeck (responsible for +sending messages) will be unaware of this (as the record of who is connected where is +gone with a redis-ephemeral restart). So these stale clients will not receive any +messages. Here, this reaper will check that the `redis-ephemeral` pod is older than any +other `cannon`; if that is not the case, it kills the `cannon`s forcing clients to +reconnect. + +Image +----- + +The reaper runs `scripts/reaper.sh` through `kubectl`, so `image` must point at a +kubectl image that **contains a POSIX shell** at `/bin/sh`. Distroless kubectl images +do not ship one and the pod will fail to start. The script itself is POSIX sh, so +busybox `ash` is enough, bash not required. + +The image is fully configurable: + +```yaml +image: + registry: docker.io # set to "" for an unqualified repository + repository: alpine/kubectl + tag: 1.36.3 + digest: "" # e.g. "sha256:..."; takes precedence over tag + pullPolicy: IfNotPresent +imagePullSecrets: + - name: my-pull-secret +``` + +RBAC +---- + +The chart creates a namespaced `Role`/`RoleBinding` granting `get`, `list`, `watch` and +`delete` on pods, bound to a `-reaper` ServiceAccount. + +`watch` is required even though the script never watches anything explicitly: +`kubectl delete pod` blocks until the pod is gone and opens a watch to do so. Without it +the reaper deletes the first cannon and then hangs, without crashing. + +Earlier versions bound the ServiceAccount to `cluster-admin` through a fixed-name +`ClusterRoleBinding`, which gave the pod read access to every Secret in the cluster. +`helm upgrade` removes that binding and the old `reaper-role` ServiceAccount. Because +nothing is cluster-scoped any more and all names are release-scoped, several reaper +releases can now coexist in one cluster; previously a second release failed to install +with a `ClusterRoleBinding` ownership conflict. + +Runtime +------- + +The container runs as uid/gid 65534 with a read-only root filesystem and has resource +requests and limits. `nodeSelector`, `tolerations` and `affinity` are honoured. + +`checkIntervalSeconds` (default `15`) controls how long the script waits between checks. +Earlier versions listed pods once per second. + +Logs distinguish a failure to reach the API from "there are no matching pods", and +include the underlying error: + + Failed to list pods: Error from server (Forbidden): ... Skipping this iteration... + No cannon pods found. Doing nothing... + +Both cases previously printed `Failed to list pods. Skipping this iteration...`, so a +reaper that could not list pods at all looked exactly like an idle one. diff --git a/charts/reaper/scripts/reaper.sh b/charts/reaper/scripts/reaper.sh new file mode 100755 index 0000000000..f67049e76a --- /dev/null +++ b/charts/reaper/scripts/reaper.sh @@ -0,0 +1,89 @@ +#!/bin/sh + +# See the readme of the reaper chart. +# +# This is POSIX sh on purpose: the only actively maintained kubectl images that +# ship busybox ash, not bash. + +# we loop forever, and on transient errors sleep and try again. +# setting -e would crash the pod on transient e.g. network errors, which isn't useful. +set -u +# shellcheck disable=SC3040 # busybox ash supports pipefail +set -o pipefail + +USAGE="$0 [INTERVAL_SECONDS]" +NAMESPACE="${1:?$USAGE}" +INTERVAL="${2:-15}" + +echo "Using namespace: $NAMESPACE, check interval: ${INTERVAL}s" + +kill_all_cannons() { + echo "Killing all cannons" + RAW_PODS=$(kubectl -n "$NAMESPACE" get pods 2>&1) || { + echo "Failed to list cannon pods: $RAW_PODS. Skipping this iteration..." + return + } + CANNON_PODS=$(echo "$RAW_PODS" | grep -e "cannon" | awk '{ print $1 }') || CANNON_PODS="" + + # A here-document rather than a pipeline, so the loop runs in the current + # shell and the `exit 1` below actually terminates the script. + while IFS= read -r cannon; do + if [ -n "$cannon" ]; then + echo "Deleting $cannon" + # If a single delete fails, we skip it but keep going. + kubectl -n "$NAMESPACE" delete pod "$cannon" || { + echo "Failed to delete pod $cannon, crash reaper and try again" + exit 1 + } + fi + done <&1) || { + echo "Failed to list pods: $RAW_PODS. Skipping this iteration..." + sleep "$INTERVAL" + continue + } + + # Gather all pods that contain "cannon" or "redis-ephemeral", sorted by creation time + ALL_PODS=$(echo "$RAW_PODS" | grep -e "cannon" -e "redis-ephemeral") || ALL_PODS="" + + # Check if we have any cannon pods at all + if ! echo "$ALL_PODS" | grep -q "cannon"; then + echo "No cannon pods found. Doing nothing..." + sleep "$INTERVAL" + continue + fi + + # Check if we have any redis-ephemeral pods at all + if ! echo "$ALL_PODS" | grep -q "redis-ephemeral"; then + echo "No redis-ephemeral pod found. Doing nothing..." + sleep "$INTERVAL" + continue + fi + + # At this point, we have both cannon and redis-ephemeral pods in ALL_PODS + # Check which is oldest + FIRST_POD=$(echo "$ALL_PODS" | head -n 1 | awk '{ print $1 }') + + if [ -z "$FIRST_POD" ]; then + echo "Could not determine the oldest pod from the list. Doing nothing..." + sleep "$INTERVAL" + continue + fi + + case "$FIRST_POD" in + *redis-ephemeral*) + echo "redis-ephemeral is the oldest pod, all good." + ;; + *) + kill_all_cannons + ;; + esac + + sleep "$INTERVAL" +done diff --git a/charts/reaper/templates/_helpers.tpl b/charts/reaper/templates/_helpers.tpl new file mode 100644 index 0000000000..47fc05fa16 --- /dev/null +++ b/charts/reaper/templates/_helpers.tpl @@ -0,0 +1,26 @@ +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{- define "includeSecurityContext" -}} + {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} +{{- end -}} + +{{/* Fully qualified image reference, digest taking precedence over tag. */}} +{{- define "reaper.image" -}} +{{- $repository := .Values.image.repository -}} +{{- if .Values.image.registry -}} +{{- $repository = printf "%s/%s" .Values.image.registry .Values.image.repository -}} +{{- end -}} +{{- if .Values.image.digest -}} +{{- printf "%s@%s" $repository .Values.image.digest -}} +{{- else -}} +{{- printf "%s:%s" $repository (.Values.image.tag | toString) -}} +{{- end -}} +{{- end -}} + +{{/* Release-scoped name for the ServiceAccount, Role and RoleBinding. */}} +{{- define "reaper.serviceAccountName" -}} +{{- printf "%s-reaper" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/reaper/templates/configmap.yaml b/charts/reaper/templates/configmap.yaml new file mode 100644 index 0000000000..571e81a1f4 --- /dev/null +++ b/charts/reaper/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: reaper-script + labels: + app: reaper +data: + reaper.sh: |- + {{- .Files.Get "scripts/reaper.sh" | nindent 4 }} + diff --git a/charts/reaper/templates/deployment.yaml b/charts/reaper/templates/deployment.yaml new file mode 100644 index 0000000000..9d50439dc5 --- /dev/null +++ b/charts/reaper/templates/deployment.yaml @@ -0,0 +1,81 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reaper + labels: + app: reaper + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: 1 + selector: + matchLabels: + app: reaper + release: {{ .Release.Name }} + template: + metadata: + labels: + app: reaper + release: {{ .Release.Name }} + annotations: + # Ensure changes to the script cause a redeployment upon `helm upgrade` + checksum/configmap: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum }} + spec: + serviceAccountName: {{ include "reaper.serviceAccountName" . }} + automountServiceAccountToken: true + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: reaper + containers: + - name: reaper + image: {{ include "reaper.image" . | quote }} + imagePullPolicy: {{ default "" .Values.image.pullPolicy | quote }} + command: ["/bin/sh", "/app/reaper.sh", "{{ .Release.Namespace }}", "{{ .Values.checkIntervalSeconds }}"] + {{- if eq (include "includeSecurityContext" .) "true" }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 12 }} + {{- end }} + env: + # kubectl writes its discovery cache below $HOME; the root + # filesystem is read-only, so point it at the emptyDir. + - name: HOME + value: /tmp + volumeMounts: + - name: reaper-script + mountPath: /app + readOnly: true + - name: tmp + mountPath: /tmp + resources: +{{ toYaml .Values.resources | indent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: reaper-script + configMap: + name: reaper-script + defaultMode: 0755 + items: + - key: reaper.sh + path: reaper.sh + - name: tmp + emptyDir: {} diff --git a/charts/reaper/templates/rbac.yaml b/charts/reaper/templates/rbac.yaml new file mode 100644 index 0000000000..5e4caafb6f --- /dev/null +++ b/charts/reaper/templates/rbac.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "reaper.serviceAccountName" . }} + labels: + app: reaper + release: {{ .Release.Name }} +--- +# The reaper only ever lists and deletes pods in its own namespace, so a +# namespaced Role is sufficient. +# `watch` is required even though the script never watches anything explicitly. +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "reaper.serviceAccountName" . }} + labels: + app: reaper + release: {{ .Release.Name }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "delete"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "reaper.serviceAccountName" . }} + labels: + app: reaper + release: {{ .Release.Name }} +roleRef: + kind: Role + name: {{ include "reaper.serviceAccountName" . }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ include "reaper.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/charts/reaper/values.yaml b/charts/reaper/values.yaml new file mode 100644 index 0000000000..cf2f56cf9e --- /dev/null +++ b/charts/reaper/values.yaml @@ -0,0 +1,45 @@ +image: + # The reaper executes a shell script through kubectl, so this image must + # contain a POSIX shell at /bin/sh. Distroless kubectl images do not + # ship one and the pod will fail to start with them. + # + # Set `registry` to "" to use an unqualified repository (e.g. when mirroring + # into a registry configured as the daemon default). + registry: docker.io + repository: alpine/kubectl + tag: 1.36.3 + # Optional: pin by digest (e.g. "sha256:abc..."). Takes precedence over `tag`. + digest: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] + +# How long to wait between two checks, in seconds. The condition this chart +# watches for (a redis-ephemeral restart) is rare, so there is no reason to poll +# the API server aggressively. +checkIntervalSeconds: 15 + +resources: + requests: + memory: 32Mi + cpu: 10m + limits: + memory: 64Mi + +nodeSelector: {} +tolerations: [] +affinity: {} + +# Applied as the container securityContext. runAsUser/runAsGroup are set +# explicitly because alpine/kubectl runs as root by default. +podSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + seccompProfile: + type: RuntimeDefault diff --git a/charts/redis-ephemeral/Chart.yaml b/charts/redis-ephemeral/Chart.yaml new file mode 100644 index 0000000000..c907a99957 --- /dev/null +++ b/charts/redis-ephemeral/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Wrapper chart for https://github.com/groundhog2k/helm-charts/tree/redis-1.3.8/charts/redis +name: redis-ephemeral +version: 0.0.42 diff --git a/charts/redis-ephemeral/requirements.yaml b/charts/redis-ephemeral/requirements.yaml new file mode 100644 index 0000000000..0cdad0dfbc --- /dev/null +++ b/charts/redis-ephemeral/requirements.yaml @@ -0,0 +1,5 @@ +dependencies: +- name: redis + version: 1.3.8 + repository: https://groundhog2k.github.io/helm-charts/ + alias: redis-ephemeral diff --git a/charts/redis-ephemeral/values.yaml b/charts/redis-ephemeral/values.yaml new file mode 100644 index 0000000000..aafcf440e4 --- /dev/null +++ b/charts/redis-ephemeral/values.yaml @@ -0,0 +1,60 @@ +redis-ephemeral: + image: + tag: "7.4.6" + + haMode: + enabled: false + + redisConfig: | + # dont write rdb to emptyDir disk for an ephemeral setup + save "" + +# To add a password add the following to redisConfig: +# requirepass my-plaintext-password + + +# How to enable SSL connections: +# +# Add the following lines to redisConfig: +# +# port 0 +# tls-port 6379 +# tls-cert-file /data/ssl/tls.crt +# tls-key-file /data/ssl/tls.key +# tls-ca-cert-file /data/ssl/ca.crt +# tls-auth-clients no +# +# Mount the certificate and adjust probes to use SSL +# +# redis-ephemeral: +# extraRedisSecrets: +# - name: redis-certificate +# mountPath: /data/ssl +# +# livenessProbe: +# enabled: false +# customLivenessProbe: +# exec: +# command: +# - sh +# - -c +# - redis-cli --tls --cacert /data/ssl/ca.crt ping +# +# readinessProbe: +# enabled: false +# customReadinessProbe: +# exec: +# command: +# - sh +# - -c +# - redis-cli --tls --cacert /data/ssl/ca.crt ping +# +# startupProbe: +# enabled: false +# customStartupProbe: +# exec: +# command: +# - sh +# - -c +# - redis-cli --tls --cacert /data/ssl/ca.crt ping +# diff --git a/charts/wire-server/templates/_helpers.tpl b/charts/wire-server/templates/_helpers.tpl index 94aca197dc..5edb025145 100644 --- a/charts/wire-server/templates/_helpers.tpl +++ b/charts/wire-server/templates/_helpers.tpl @@ -106,6 +106,46 @@ {{- end -}} {{- end -}} +{{- define "gundeck.configureRedisCa" -}} +{{ or (hasKey .redis "tlsCa") (hasKey .redis "tlsCaSecretRef") }} +{{- end -}} + +{{- define "gundeck.redisTlsSecretName" -}} +{{- if .redis.tlsCaSecretRef -}} +{{ .redis.tlsCaSecretRef.name }} +{{- else }} +{{- print "gundeck-redis-ca" -}} +{{- end -}} +{{- end -}} + +{{- define "gundeck.redisTlsSecretKey" -}} +{{- if .redis.tlsCaSecretRef -}} +{{ .redis.tlsCaSecretRef.key }} +{{- else }} +{{- print "ca.pem" -}} +{{- end -}} +{{- end -}} + +{{- define "gundeck.configureAdditionalRedisCa" -}} +{{ and (hasKey . "redisAdditionalWrite") (or (hasKey .redis "additionalTlsCa") (hasKey .redis "additionalTlsCaSecretRef")) }} +{{- end -}} + +{{- define "gundeck.additionalRedisTlsSecretName" -}} +{{- if .redis.additionalTlsCaSecretRef -}} +{{ .redis.additionalTlsCaSecretRef.name }} +{{- else }} +{{- print "gundeck-additional-redis-ca" -}} +{{- end -}} +{{- end -}} + +{{- define "gundeck.additionalRedisTlsSecretKey" -}} +{{- if .redis.additionalTlsCaSecretRef -}} +{{ .redis.additionalTlsCaSecretRef.key }} +{{- else }} +{{- print "ca.pem" -}} +{{- end -}} +{{- end -}} + {{/* SPAR */}} {{- define "spar.tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} diff --git a/charts/wire-server/templates/cannon/statefulset.yaml b/charts/wire-server/templates/cannon/statefulset.yaml index f60d658645..00103604bf 100644 --- a/charts/wire-server/templates/cannon/statefulset.yaml +++ b/charts/wire-server/templates/cannon/statefulset.yaml @@ -2,7 +2,7 @@ # Specific pods can be accessed within the cluster at cannon-.cannon. # (the second 'cannon' is the name of the headless service) # Note: In fact, cannon-.cannon can also be used to access the service but assuming -# that we can have multiple namespaces accessing the same cannon cluster, appending `.` +# that we can have multiple namespaces accessing the same redis cluster, appending `.` # makes the service unambiguous apiVersion: apps/v1 kind: StatefulSet diff --git a/charts/wire-server/templates/gundeck/configmap.yaml b/charts/wire-server/templates/gundeck/configmap.yaml index 6aae6aa47d..10be21c34e 100644 --- a/charts/wire-server/templates/gundeck/configmap.yaml +++ b/charts/wire-server/templates/gundeck/configmap.yaml @@ -41,10 +41,26 @@ data: {{- end }} {{- end }} - postgresql: {{ toYaml .postgresql | nindent 6 }} - postgresqlPool: {{ toYaml .postgresqlPool | nindent 6 }} - {{- if hasKey $.Values.gundeck.secrets "pgPassword" }} - postgresqlPassword: /etc/wire/gundeck/secrets/pgPassword + redis: + host: {{ .redis.host }} + port: {{ .redis.port }} + connectionMode: {{ .redis.connectionMode }} + enableTls: {{ .redis.enableTls }} + insecureSkipVerifyTls: {{ .redis.insecureSkipVerifyTls }} + {{- if eq (include "gundeck.configureRedisCa" .) "true" }} + tlsCa: /etc/wire/gundeck/redis-ca/{{ include "gundeck.redisTlsSecretKey" . }} + {{- end }} + + {{- if .redisAdditionalWrite }} + redisAdditionalWrite: + host: {{ .redisAdditionalWrite.host }} + port: {{ .redisAdditionalWrite.port }} + connectionMode: {{ .redisAdditionalWrite.connectionMode }} + enableTls: {{ .redisAdditionalWrite.enableTls }} + insecureSkipVerifyTls: {{ .redisAdditionalWrite.insecureSkipVerifyTls }} + {{- if eq (include "gundeck.configureAdditionalRedisCa" .) "true" }} + tlsCa: /etc/wire/gundeck/additional-redis-ca/{{ include "gundeck.additionalRedisTlsSecretKey" . }} + {{- end }} {{- end }} # Gundeck uses discovery for AWS access key / secrets diff --git a/charts/wire-server/templates/gundeck/deployment.yaml b/charts/wire-server/templates/gundeck/deployment.yaml index ff7a457fc6..bc46a53ec0 100644 --- a/charts/wire-server/templates/gundeck/deployment.yaml +++ b/charts/wire-server/templates/gundeck/deployment.yaml @@ -50,9 +50,16 @@ spec: secret: secretName: {{ (include "gundeck.tlsSecretRef" .Values.gundeck.config | fromYaml).name }} {{- end }} - - name: "gundeck-secrets" + {{- if eq (include "gundeck.configureRedisCa" .Values.gundeck.config) "true" }} + - name: "redis-ca" secret: - secretName: "gundeck" + secretName: {{ include "gundeck.redisTlsSecretName" .Values.gundeck.config }} + {{- end }} + {{- if eq (include "gundeck.configureAdditionalRedisCa" .Values.gundeck.config) "true" }} + - name: "additional-redis-ca" + secret: + secretName: {{ include "gundeck.additionalRedisTlsSecretName" .Values.gundeck.config }} + {{- end }} containers: - name: gundeck image: "{{ .Values.gundeck.image.repository }}:{{ .Values.gundeck.image.tag }}" @@ -68,8 +75,14 @@ spec: - name: "gundeck-cassandra" mountPath: "/etc/wire/gundeck/cassandra" {{- end }} - - name: "gundeck-secrets" - mountPath: "/etc/wire/gundeck/secrets" + {{- if eq (include "gundeck.configureRedisCa" .Values.gundeck.config) "true" }} + - name: "redis-ca" + mountPath: "/etc/wire/gundeck/redis-ca/" + {{- end }} + {{- if eq (include "gundeck.configureAdditionalRedisCa" .Values.gundeck.config) "true" }} + - name: "additional-redis-ca" + mountPath: "/etc/wire/gundeck/additional-redis-ca/" + {{- end }} {{- if and .Values.gundeck.config.rabbitmq .Values.gundeck.config.rabbitmq.tlsCaSecretRef }} - name: "rabbitmq-ca" mountPath: "/etc/wire/gundeck/rabbitmq-ca/" @@ -97,6 +110,34 @@ spec: name: gundeck key: awsSecretKey {{- end }} + {{- if hasKey .Values.gundeck.secrets "redisUsername" }} + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: gundeck + key: redisUsername + {{- end }} + {{- if hasKey .Values.gundeck.secrets "redisPassword" }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck + key: redisPassword + {{- end }} + {{- if hasKey .Values.gundeck.secrets "redisAdditionalWriteUsername" }} + - name: REDIS_ADDITIONAL_WRITE_USERNAME + valueFrom: + secretKeyRef: + name: gundeck + key: redisAdditionalWriteUsername + {{- end }} + {{- if hasKey .Values.gundeck.secrets "redisAdditionalWritePassword" }} + - name: REDIS_ADDITIONAL_WRITE_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck + key: redisAdditionalWritePassword + {{- end }} - name: AWS_REGION value: "{{ .Values.gundeck.config.aws.region }}" {{- with .Values.gundeck.config.proxy }} diff --git a/charts/wire-server/templates/gundeck/redis-ca-secret.yaml b/charts/wire-server/templates/gundeck/redis-ca-secret.yaml new file mode 100644 index 0000000000..a82eab555c --- /dev/null +++ b/charts/wire-server/templates/gundeck/redis-ca-secret.yaml @@ -0,0 +1,30 @@ +--- +{{- if not (empty .Values.gundeck.config.redis.tlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: "gundeck-redis-ca" + labels: + app: gundeck + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.gundeck.config.redis.tlsCa | b64enc | quote }} +{{- end }} +--- +{{- if not (empty .Values.gundeck.config.redis.additionalTlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: "gundeck-additional-redis-ca" + labels: + app: gundeck + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.gundeck.config.redis.additionalTlsCa | b64enc | quote }} +{{- end }} diff --git a/charts/wire-server/templates/gundeck/secret.yaml b/charts/wire-server/templates/gundeck/secret.yaml index a17529f4c7..b1f744ff60 100644 --- a/charts/wire-server/templates/gundeck/secret.yaml +++ b/charts/wire-server/templates/gundeck/secret.yaml @@ -19,8 +19,17 @@ data: {{- if hasKey . "awsSecretKey" }} awsSecretKey: {{ .awsSecretKey | b64enc | quote }} {{- end }} - {{- if hasKey . "pgPassword" }} - pgPassword: {{ .pgPassword | b64enc | quote }} + {{- if hasKey . "redisUsername" }} + redisUsername: {{ .redisUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisPassword" }} + redisPassword: {{ .redisPassword | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisAdditionalWriteUsername" }} + redisAdditionalWriteUsername: {{ .redisAdditionalWriteUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisAdditionalWritePassword" }} + redisAdditionalWritePassword: {{ .redisAdditionalWritePassword | b64enc | quote }} {{- end }} {{- end }} {{- end }} diff --git a/charts/wire-server/templates/gundeck/tests/configmap.yaml b/charts/wire-server/templates/gundeck/tests/configmap.yaml index 28d7677307..c8c23ce518 100644 --- a/charts/wire-server/templates/gundeck/tests/configmap.yaml +++ b/charts/wire-server/templates/gundeck/tests/configmap.yaml @@ -39,3 +39,10 @@ data: host: brig port: 8080 + # a "redis migration" test in gundeck makes use of a second (distinct) redis + redis2: + host: redis-ephemeral-2 + port: 6379 + connectionMode: master + enableTls: false + insecureSkipVerifyTls: false diff --git a/charts/wire-server/templates/gundeck/tests/gundeck-integration.yaml b/charts/wire-server/templates/gundeck/tests/gundeck-integration.yaml index 60b5d26e3d..f1a661b4a5 100644 --- a/charts/wire-server/templates/gundeck/tests/gundeck-integration.yaml +++ b/charts/wire-server/templates/gundeck/tests/gundeck-integration.yaml @@ -18,6 +18,11 @@ spec: secret: secretName: {{ (include "gundeck.tlsSecretRef" .Values.gundeck.config | fromYaml).name }} {{- end }} + {{- if eq (include "gundeck.configureRedisCa" .Values.gundeck.config) "true" }} + - name: "redis-ca" + secret: + secretName: {{ include "gundeck.redisTlsSecretName" .Values.gundeck.config }} + {{- end }} {{- if .Values.gundeck.config.rabbitmq.tlsCaSecretRef }} - name: "rabbitmq-ca" secret: @@ -68,6 +73,10 @@ spec: - name: "gundeck-cassandra" mountPath: "/etc/wire/gundeck/cassandra" {{- end }} + {{- if eq (include "gundeck.configureRedisCa" .Values.gundeck.config) "true" }} + - name: "redis-ca" + mountPath: "/etc/wire/gundeck/redis-ca/" + {{- end }} {{- if .Values.gundeck.config.rabbitmq.tlsCaSecretRef }} - name: "rabbitmq-ca" mountPath: "/etc/wire/gundeck/rabbitmq-ca/" @@ -87,6 +96,34 @@ spec: value: "guest" - name: RABBITMQ_PASSWORD value: "guest" + {{- if hasKey .Values.gundeck.secrets "redisUsername" }} + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: gundeck + key: redisUsername + {{- end }} + {{- if hasKey .Values.gundeck.secrets "redisPassword" }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck + key: redisPassword + {{- end }} + {{- if and (hasKey .Values.gundeck.tests "secrets") (hasKey .Values.gundeck.tests.secrets "redisAdditionalWriteUsername") }} + - name: REDIS_ADDITIONAL_WRITE_USERNAME + valueFrom: + secretKeyRef: + name: gundeck-integration + key: redisAdditionalWriteUsername + {{- end }} + {{- if and (hasKey .Values.gundeck.tests "secrets") (hasKey .Values.gundeck.tests.secrets "redisAdditionalWritePassword") }} + - name: REDIS_ADDITIONAL_WRITE_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck-integration + key: redisAdditionalWritePassword + {{- end }} {{- if .Values.gundeck.tests.config.uploadXml }} - name: UPLOAD_XML_S3_BASE_URL value: {{ .Values.gundeck.tests.config.uploadXml.baseUrl }} diff --git a/charts/wire-server/templates/gundeck/tests/secret.yaml b/charts/wire-server/templates/gundeck/tests/secret.yaml index df7b82695f..60aed14a3a 100644 --- a/charts/wire-server/templates/gundeck/tests/secret.yaml +++ b/charts/wire-server/templates/gundeck/tests/secret.yaml @@ -17,5 +17,11 @@ data: {{- if hasKey . "uploadXmlAwsSecretAccessKey" }} uploadXmlAwsSecretAccessKey: {{ .uploadXmlAwsSecretAccessKey | b64enc | quote }} {{- end }} + {{- if hasKey . "redisAdditionalWriteUsername" }} + redisAdditionalWriteUsername: {{ .redisAdditionalWriteUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisAdditionalWritePassword" }} + redisAdditionalWritePassword: {{ .redisAdditionalWritePassword | b64enc | quote }} + {{- end }} {{- end }} {{- end }} diff --git a/charts/wire-server/values.yaml b/charts/wire-server/values.yaml index 9a98be5d52..d2a7deafdc 100644 --- a/charts/wire-server/values.yaml +++ b/charts/wire-server/values.yaml @@ -744,19 +744,35 @@ gundeck: # tlsCaSecretRef: # name: # key: - # Postgres connection settings for presence tracking. + redis: + host: redis-ephemeral + port: 6379 + connectionMode: "master" # master | cluster + enableTls: false + insecureSkipVerifyTls: false + # To configure custom TLS CA, please provide one of these: + # tlsCa: + # + # Or refer to an existing secret (containing the CA): + # tlsCaSecretRef: + # name: + # key: + + # To enable additional writes during a migration: + # redisAdditionalWrite: + # host: redis-two + # port: 6379 + # connectionMode: master + # enableTls: false + # insecureSkipVerifyTls: false # - # Values are described in https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS - # To set the password via a gundeck secret see `secrets.pgPassword`. - postgresql: - host: postgresql # DNS name without protocol - port: "5432" - user: wire-server - dbname: wire-server - postgresqlPool: - size: 100 - acquisitionTimeout: 10s - idlenessTimeout: 10m + # # To configure custom TLS CA, please provide one of these: + # # tlsCa: + # # + # # Or refer to an existing secret (containing the CA): + # # tlsCaSecretRef: + # # name: + # # key: aws: region: "eu-west-1" proxy: {} diff --git a/deploy/dockerephemeral/docker-compose.yaml b/deploy/dockerephemeral/docker-compose.yaml index a8a9ab66d5..fb2a4801eb 100644 --- a/deploy/dockerephemeral/docker-compose.yaml +++ b/deploy/dockerephemeral/docker-compose.yaml @@ -1,4 +1,10 @@ networks: + redis: + driver: bridge + ipam: + config: + - subnet: 172.20.0.0/24 + coredns: driver: bridge ipam: @@ -72,6 +78,134 @@ services: networks: - demo_wire + redis-master: + container_name: demo_wire_redis + image: redis:7.2-alpine + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6379:6379" + volumes: + - ./docker/redis-master-mode.conf:/usr/local/etc/redis/redis.conf + networks: + - demo_wire + + redis-cluster: + image: "redis:7.2-alpine" + command: + - redis-cli + - --cluster + - create + - 172.20.0.31:6373 + - 172.20.0.32:6374 + - 172.20.0.33:6375 + - 172.20.0.34:6376 + - 172.20.0.35:6377 + - 172.20.0.36:6378 + - --cluster-replicas + - "1" + - --cluster-yes + - -a + - very-secure-redis-cluster-password + - --cacert + - /usr/local/etc/redis/ca.pem + - --tls + volumes: + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.30 + depends_on: + - redis-node-1 + - redis-node-2 + - redis-node-3 + - redis-node-4 + - redis-node-5 + - redis-node-6 + redis-node-1: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6373:6373" + volumes: + - redis-node-1-data:/var/lib/redis + - ./docker/redis-node-1.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-1-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-1-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.31 + redis-node-2: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6374:6374" + volumes: + - redis-node-2-data:/var/lib/redis + - ./docker/redis-node-2.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-2-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-2-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.32 + redis-node-3: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6375:6375" + volumes: + - redis-node-3-data:/var/lib/redis + - ./docker/redis-node-3.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-3-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-3-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.33 + redis-node-4: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6376:6376" + volumes: + - redis-node-4-data:/var/lib/redis + - ./docker/redis-node-4.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-4-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-4-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.34 + redis-node-5: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6377:6377" + volumes: + - redis-node-5-data:/var/lib/redis + - ./docker/redis-node-5.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-5-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-5-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.35 + redis-node-6: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6378:6378" + volumes: + - redis-node-6-data:/var/lib/redis + - ./docker/redis-node-6.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-6-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-6-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.36 + elasticsearch: container_name: demo_wire_elasticsearch image: elasticsearch:6.8.23 @@ -284,3 +418,11 @@ services: # - DNS_SERVER_RECURSION_DENIED_NETWORKS=1.1.1.0/24 #Comma separated list of IP addresses or network addresses to deny recursion. Valid only for `UseSpecifiedNetworkACL` recursion option. This option is obsolete and DNS_SERVER_RECURSION_NETWORK_ACL should be used instead. # - DNS_SERVER_RECURSION_ALLOWED_NETWORKS=127.0.0.1, 192.168.1.0/24 #Comma separated list of IP addresses or network addresses to allow recursion. Valid only for `UseSpecifiedNetworkACL` recursion option. This option is obsolete and DNS_SERVER_RECURSION_NETWORK_ACL should be used instead. # - DNS_SERVER_ENABLE_BLOCKING=false #Sets the DNS server to block domain names using Blocked Zone and Block List Zone. + +volumes: + redis-node-1-data: + redis-node-2-data: + redis-node-3-data: + redis-node-4-data: + redis-node-5-data: + redis-node-6-data: diff --git a/deploy/dockerephemeral/docker/redis-master-mode.conf b/deploy/dockerephemeral/docker/redis-master-mode.conf new file mode 100644 index 0000000000..d71dbc51c9 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-master-mode.conf @@ -0,0 +1 @@ +requirepass very-secure-redis-master-password \ No newline at end of file diff --git a/deploy/dockerephemeral/docker/redis-node-1-cert.pem b/deploy/dockerephemeral/docker/redis-node-1-cert.pem new file mode 100644 index 0000000000..7756f82bbd --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-1-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDTtp3U0VPwBVJNULQF +S4BBlWBNf/8NMidOq23IsTcjkIFWO1XL+HFZoa1AArUSA/TaLBYyz9WmX7eLWvAU +ADM6mfAf2V6whmIs2H9ZRnY89bFWO2hzLWWp1qq3dXK1ywTLpw7DqU4OT0rtYZbp +QHeVY0mKKspF+YJTZzWB1hs8IX9355wXRlYBLPNQ5oHRb4/16J/UUFPIJjpUyHsq +T1LWmVREqisrq9u50FnNPeLXE6SDnHGRkYGQXzQOM/yAI75/QUOOqo5rt3Et52t5 +pkOT45R0PbAC2UpR1usew0zVjRoQfFk9n38tXUSHKw/tW+ZY1xJqEKEiLGfnhhza +t4kjAgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy0xhwSsFAAfMB0GA1UdDgQWBBQsOxsq4X8dS/Ddl9l1 +TWDb8Q5KKzAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAaByeD08xOZCV0ZKsx7lHtiem5/XG01rMcDxNVrVguSD+mqhR +/j8ciTW2CruJ2X8ReTjNrI4X1nWLbh4rsrA56q4xkjkgJIfWQAdKCibXTrHOWfk5 +dcYG1pqVdpD5bvsxAsY95jxqoVJHXHGN8ynC+lV39HbDJQFOdHLAP66NUrphp76a +OZKiuzUS6naeiHWoA9eIANFRz/JoQvyp109gdce5MH0iFwGFqNJU2rwilOpzQVc7 +qldx7MHMnW5UYSTqryTOr8PS+xo24TSdHjIXmnOO3Ov0Pw7iPpGVGj56dAKgEisG +yGOAWYto8UBWKLox1vSSlfdkhAoDXluvE8EwRw== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-1-key.pem b/deploy/dockerephemeral/docker/redis-node-1-key.pem new file mode 100644 index 0000000000..6d8b29bbde --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-1-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDTtp3U0VPwBVJN +ULQFS4BBlWBNf/8NMidOq23IsTcjkIFWO1XL+HFZoa1AArUSA/TaLBYyz9WmX7eL +WvAUADM6mfAf2V6whmIs2H9ZRnY89bFWO2hzLWWp1qq3dXK1ywTLpw7DqU4OT0rt +YZbpQHeVY0mKKspF+YJTZzWB1hs8IX9355wXRlYBLPNQ5oHRb4/16J/UUFPIJjpU +yHsqT1LWmVREqisrq9u50FnNPeLXE6SDnHGRkYGQXzQOM/yAI75/QUOOqo5rt3Et +52t5pkOT45R0PbAC2UpR1usew0zVjRoQfFk9n38tXUSHKw/tW+ZY1xJqEKEiLGfn +hhzat4kjAgMBAAECggEAFqMmoixVxMrU34Z7ETve9WRC/VZrz53mvQ8weG6WfjuD +0NQcWuhwOkzCyR7g/JGmuzNOllVJu3Xtmr15ATJ6R9BQ8B7edJKR6cimaUXS+7ar +pRRKGVKn1a6p517sCoswMpRkzEAMpBQPZ21xZPRrNPJ+WQM1SKEiscdN3dmmZNng +MvroH1dPVbyZ49xkjMQ0NaOtk4rvopzdKKZea2qz41w/vXR9hShnfVDs/q86clmx +5mnEvXcEdguioAfUWz+qQ7dXlWsASKa/gAMjUN9GW9uOn4LclFsVCD2MW+IUJMxe ++JtFM0xiQ3HaK0Fem8+XR8mG3BB5a/06ZHBfcv/lsQKBgQD4WSJjZwMBG80uGidR +ls+VhhFjysxm5qrF34MWziLczi1nAStc/PzVcA7tHapKX5JiKYT6d8Ptngz6FLIo +/72OshmLzctxRprlpihWxMIYOqwb2PLB0//ghuUE81Zbxj1MQ6k9WbTGHBwUbaiv +PSzclhmMubypfLLcmMEnHeZFswKBgQDaPIWmyax3Eft8DzC3Om7X3WMN0NXE96z2 +6hUAon5tqinMuWUWa2cyWzPsdBgFM8mCynoiIu08YFpZQivoB6QSal4x2mLg4R+u +aLm3h9f6NS4/VvpWPL5wMUAqeCCbP/2PVKk///0mtQGixUOxeQftTncQeLtfXOXd +4gDJHjfW0QKBgQDND7xnW42Ngsk+wfWpVt981UDSp4dziA+GZ3I0iG0c6Vlv7fVC +SNrz2h1ZCN+tnZCfYS0eK3oqYBDTBfe+Br0ccE7Ls1fC5svLyBES5FBn9TpbnB2G +kmh7mqbMGak7CktfB5dcww+TbW56J7nbSKYcVgwuuMbhI8gEglUq2XNkJQKBgQDV +VojIzSmdlKSlWCwlUif9OdyVKutuizg4gAhcAH1bMxd9nFbnncLaBTIzGiJJI6EA +DHNsX3xOo1pvGzLUtnN71SOT1IsIjsprstCqS0+ktswo+xvppaP9BQhW++vUGLAE +p5x0hgixCA07U1+jZE+NekEGhx+UT7oeN8rQ0IuBoQKBgQC4PF4WwqashYHkYW2j +4LaMu5kWY/0OI9Vh/h1iOcKPzVUn61aabjsx1wF9rummIdxP03/bs7ZpkwPypcVR +v7XnNbi+hDZFEN6s/+Gl4S6RfAbWXs3sgnhVlctlkzzwG8UHCef4DWMPxFI1JQI8 +X+SdDfpmB/ayQb8TlYvke/s8cQ== +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-1.conf b/deploy/dockerephemeral/docker/redis-node-1.conf new file mode 100644 index 0000000000..aa772f502f --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-1.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6373 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-2-cert.pem b/deploy/dockerephemeral/docker/redis-node-2-cert.pem new file mode 100644 index 0000000000..ea4b4507d6 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-2-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4mm9GWOVh0ttlzXaJ +11/rQQX3vYQ3zyeMdz/KGTKArOF+pxVCxbETlI3ZufO8Ht9zqa7Doh5R86iNtVMR +LoQZVWeXjQsMATwNUZT3lEOezpDE0ZI8d5JyU946Z+7s0VjIMbXOzjTSjTNSi57N +li59/1NTG5CW9EtgnnYoP5SOrYTpK+fzawXD18tD8kq/VBLt8OoG7xn6DIpGsFr9 +h1Ot/yrUejvrHg2KIi3av/cnqA8twzFpkdvGSEarjRuYG6fHGL67dgSpLvzh/v7h +QiJDFFB8fHnUc5ioZXFw88P4Oq7UlzBhnkC8nhUi1X1vWoF9Xz4FXXJ1P4WkZfWB +Vui7AgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy0yhwSsFAAgMB0GA1UdDgQWBBQQK2od431iWKznJEQz +zy5GXgt1DDAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAhvNbLzlY4sS/xmn8alzIYjY/uIc5c0PaUaXc7SSjeoChRfNQ +tE5YLmOo86WYNThtaNmiRLFv3yNBXCcqdVgNdL78EIQlKvPxHwzZXxkKDmOcfIZS +nUa4w+OmKJLsdNjphBGmR94h8WycwoFMThw55vnTJ2+AnCFPsLDfjtHiKB8AsW8u +gtSTtVyu+QyvGTDxEFDgqFgyFjJpVp37bOakRuzuZZ8VUssQbb11YHyhnNGTcL3a +hLXeGVSRA7SyDXxxRs5PmmJVsUOWkgbIjguvZK5APpqaGEYwBYo036DFSgt6DTOu +8YsCTeSOmue0xNlPDiVPSP8HUGfq3tTBKMXbUQ== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-2-key.pem b/deploy/dockerephemeral/docker/redis-node-2-key.pem new file mode 100644 index 0000000000..fba9118998 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-2-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC4mm9GWOVh0ttl +zXaJ11/rQQX3vYQ3zyeMdz/KGTKArOF+pxVCxbETlI3ZufO8Ht9zqa7Doh5R86iN +tVMRLoQZVWeXjQsMATwNUZT3lEOezpDE0ZI8d5JyU946Z+7s0VjIMbXOzjTSjTNS +i57Nli59/1NTG5CW9EtgnnYoP5SOrYTpK+fzawXD18tD8kq/VBLt8OoG7xn6DIpG +sFr9h1Ot/yrUejvrHg2KIi3av/cnqA8twzFpkdvGSEarjRuYG6fHGL67dgSpLvzh +/v7hQiJDFFB8fHnUc5ioZXFw88P4Oq7UlzBhnkC8nhUi1X1vWoF9Xz4FXXJ1P4Wk +ZfWBVui7AgMBAAECggEAONB+8r2lSygkEf7cPqwkfzjx5z9SlAKTf22sGj0LCAMt +G1e8+WHyj74msh3C3+D4kJZmjRs2Da7Z71MhD6arTUi1qzTjc3xlyQuUt2XQMe4N +LCX7xdRfJASf3oXiSMxdcK+r7swUAcEnTH5gD5HrGSgdsvRG2c6x7DiY0OZQiGBk +2rPHQDUKeb7Z0YLWc8nldzlnOe2OeWpFVEraAOzmANnV5FVJZP8RNoiKviZnB77O +qbA6Xtpg4ytVhMaymUmjkjdFuxm5XcMCOIz4W9SZVJ9uSzjZqATzgjsiOWYozB7q +2xb1yOyCVPgf+dZj32D8DvqSrwwRBR3LcNhnj2wUIQKBgQDqL4VNp54Lrf+oZ0ZF +h3s6lL2NquY0xHs91YvoO187VetyUlNjOcGXt8ROhSSAf6qTLQvrreVdjYHr52xr +smCohhQ9QDm3d+Inh3ARgr75O577aPwJHBmo0fnu9h6OkDr8nx05SthW4XenHqoE +iWQ9FnibAFz5KLBSYC7x9wfGaQKBgQDJzI3UC6AqQS8ILbcqHm7ZmnpUUjn7vPUm +lkB3/YtV7ewWJhFzdPdaKHKe2YO9WXQTCF7iPRK3+gWt8uh4DWCrSObBkmSUlF66 +wbRof3lsYiWDPed9OTgoDHRwbMPeYrJ3A0TMrGJQsbedljneaat+DM3kNgjgChfW +JiL0g9c5gwKBgQDBi8zMRT/lv0SQVepKBJLf85ZFw3zHF6wTiq46nPcz/uq8bTXl +yBIr5gEkM/3bBahgQtabTflGvHEoGvgMejxQi5+mj7Ij47zRlqoUjs5vBct7VWUX +0lWSpRe/W0Id6S4XIxnwA9+Qzn8pa7pwTWy+4BeFY2NzuSEgs8WYzOVsIQKBgHbI +IPOfpDc7ByQZRKdWIomTlE3t2JOFNgfwiSIX69w4n66p2bvMLYy0IkO+ZP0fmmNZ +mgAxUsNYN9+cC5oexbgMwUdPlESg0OG9AyQ/ZImXe900ov3ioFtyeVdzrhdIoSPM +mMKg9X3qHdp0gruYF4mqn8akx7SYPE+hQxIKSLVhAoGAJP+TshJj8xAeE1Uroyc/ +yIWThbp0Q/EFaXkpS6aJqBjdcLfh2U+Zo9ZaTn9OBlzXHk9WttzeWuMY9PrINodJ +8DSg5f0PslYxJ5DQuKnDWUeqX3zCnXkgnymlvh78t6wWp+BUAEjI8qH5IgKVwKd+ +VJbPX4mzhAl/0kIablU6SqM= +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-2.conf b/deploy/dockerephemeral/docker/redis-node-2.conf new file mode 100644 index 0000000000..de7687558b --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-2.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6374 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-3-cert.pem b/deploy/dockerephemeral/docker/redis-node-3-cert.pem new file mode 100644 index 0000000000..e550d0e30f --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-3-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCd72omHFn1mEFw/GBp +gkPM5BkF7giGx7GOLyijCoi4NLNVKJn6mOJt9vX2PbBYedy1OcskObLbEwqUwcZr +7fVim34xrE4AmdJqBWTkcMFnhbjzYIynfvejej/05kWlzp3JuhTpi7i2W+nnZjqb +S6UHgeTwF/iENA1oysuq0jC4oaVGNa2ZCoz3W+uAEbpUYNjN7/uQeEwRyZjSEJUY +KyG69Wrl9KnzBX0mkltq8rJiCqaG+qOZwP+XH7TxjYM1SlAxLHrnjDQHWyZXJzPY +fikRk2Zf8nDobA5thXVR/2PicDxUs1VyGYSg/vK1EMwOIHIZdxalo0x75vFjBJ9T +l+HFAgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy0zhwSsFAAhMB0GA1UdDgQWBBQyljx2OR3L7yZLVax4 +MLTDhj4xPjAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAUv3JN0ip/LWmtWHyqzPuq9tbVFs2M5waRO2ZZtEp6Pzudr9x +JKrmtz7IlnwK2E3eqw1Hh3kZYiM5XT2GzqFjPn+Na32i3IsR/S1Y4ZDq6T1WOjht +u+3EjrUvpTXAcLfaO60gJ7DrfC4PsuNuaRr23BiF3lIb7A693hnESg3EnUqGvAvA +ikR/Cv48kAvxpFlXZfnGApFEP49svj676emodRUlk4aCOjIniPByLF318Dl+MwzW +KbnjynzjnOqfcXeD67axFqIBAhZPBDWIDOLNo/ASAROkPntycBGFPUL+Wgdq75vs +8WnftwfCzYtKcASNVSeoSFtJhVy2cAqHK1bd/g== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-3-key.pem b/deploy/dockerephemeral/docker/redis-node-3-key.pem new file mode 100644 index 0000000000..d7be5cf147 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-3-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCd72omHFn1mEFw +/GBpgkPM5BkF7giGx7GOLyijCoi4NLNVKJn6mOJt9vX2PbBYedy1OcskObLbEwqU +wcZr7fVim34xrE4AmdJqBWTkcMFnhbjzYIynfvejej/05kWlzp3JuhTpi7i2W+nn +ZjqbS6UHgeTwF/iENA1oysuq0jC4oaVGNa2ZCoz3W+uAEbpUYNjN7/uQeEwRyZjS +EJUYKyG69Wrl9KnzBX0mkltq8rJiCqaG+qOZwP+XH7TxjYM1SlAxLHrnjDQHWyZX +JzPYfikRk2Zf8nDobA5thXVR/2PicDxUs1VyGYSg/vK1EMwOIHIZdxalo0x75vFj +BJ9Tl+HFAgMBAAECggEABYejI9UiS+MaMiaOtE2x/16NMb6f4Hg600umFJoDJ3qm +PM5rIHHHRn+7JPVhU00RA+y+HB/uZJVKGDigsJloWhzaUkrs1ZXiiYEe2JDKH3cj +KVexamabrRxUA53RxSMdizlPZM4A7axSMvP1YV1IrfadBCW9Ydj2DzvqiFShDWst +asKPAa6MAU63zfZZaBQvicswd1nJUvc8ZNp1p0JiVcwWPWVTYH9d2c+0WZLlfCHm +GxUurHwyVc6b7T4OSrsiDaQN0kdLJDAYowp+T94JDBCH3m4e/NF9W6gkoO2UGXTH +6A9HVDI3FwUBzXdT9rL/Wmp4kKXB4xO2TU/yeZoYAQKBgQDK2Q2vG+BucY2aJxGw +7HNeXov2lLma2Vn4TRr+cyzcXH7Jmc8J/h9RMU7AEfg3CQwMbXE60P561/1q1e0Z +fD55x9ka3FZ2dG+a5CDzjkqnUgnLYOK1bxx5UUq+Sf6IeNjGPikejRPcPBmvFVuu +NvoPU0HwWLm67BnantJIpUFvRQKBgQDHUaPa6SIMGAWHasI6EvZMGgBAy5iJa4s3 +o+DuESF+6lD989ZnOltsPFeYhwbIzm14EzhK/y4MVR46gXLMZ9FwlGCGdXE7LWiN +VKCm9kRcxcH9Sak70LkZ9yv08Nl45f9vTOzBcKzu6bgZ2LOeSJ0oTmiVEb98pL7N +w6XxD2iQgQKBgAVVPYncBsOAksN5wXpQTRwvCij6cgLDMh1YEZyc9JH6kI7GT24o +0zP0QujD0C3KPBnbir2MHxSltxDm/OvNm2riOS/+mPtWRlThKIiethG+E2nYaz1v +5WS/IWLtWRbHbpOPsM8P0HTa06YJvrZO1bYvby1dd8yVRny77jVgut6tAoGAHpMK +ZHkgjORebMBWnNvtxgyy/z1735CMoXNU/I/KKJK+68WsnNcZ0QeMlEwaIVFw/1tL +Zk2wfZnM8kKLHonKWc+Y4uc/AEnd4NgbcKEUKXr4X+cdu5wv2KjOqFsNsPru7N7K +7n1fOaLGZ8iS/PO8j8M/TaaUTgVjc2LQoKKxcoECgYAtPzq1Y0yc22M+m1m6nK/W +L7rsUI0zDs0VZcJ5mrJg8nahOM/f+BsFYN5oAHYxuXPUyynZyD2nPtdsES75DGOH +PEqr9DhgSig4JmHS/6SEBnWql+zyNdn1/FaYOkKRHiY7jNhjTayiDObJrXg0g4OT +BmzY39BABb52ogQbjWslow== +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-3.conf b/deploy/dockerephemeral/docker/redis-node-3.conf new file mode 100644 index 0000000000..7f406d7232 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-3.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6375 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-4-cert.pem b/deploy/dockerephemeral/docker/redis-node-4-cert.pem new file mode 100644 index 0000000000..185f8f9701 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-4-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKxOmFo5c4ae38qC3z +I89R1F6xjaMyR6jjd6k5qsW7eU/y8+trgY4HV/jbzD3CDOjMkj70la5EiV+GTA0i +GeJH/BkKjqEPsIy/vAPux9xt2ZpIO9ieO2BF75ojrcM7tAbeOLQNAgYA7zAyIpQk +J2P8IyOYSJ31ujLJCR7d0zudAbXJXfAAyPUWqUrmmRHIY7hRi1tUv74JARqnU2tH +ZhFgGyBCaLROK69S/Wy+xPKo5w9Ol5L9eIccrK2/JwNpfsFAxJqXawNm1l1M9gGk +2MpQXzZeTg/hlusqCtPieOPUQKoEDXAgYArQy8iYkLuZzOtg2WwcPOhtfsgVRLNE +wXihAgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy00hwSsFAAiMB0GA1UdDgQWBBQrI/peejY55qjXOc6W +XUU+/q6R+TAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAdf1N+gPpnkEHzDAMnK4kUCHq2ymLBBWJVAPDcmmtcMjEiEVC +/9BU+hcdqgLXxonEqiA4kEs9Mkj8AcUk0Dzl5Gfk2haZO6yzVEp97zwto+3Tgzya +0l6bvRv4OSfdVeSTYx8T48h23O8FBD/Gp9l5sFOZgc1TCWrb7ReJQS+XThAksIdW +DLvwbOU1I2qRL3ZbT49FAhmVcrMkHJjzkugXDoGG3Rgdzx/HePUjXWdWC1L+7/Kn +U/7w72ymW1mC5PbjoW9zzkVKesj++mhzSb5+sXa/is3hUJ17zy4Bqc71Mb2q8tqM +G/uMrdwfPeoad3qRVPRsK8QlVnJ0eIpiUDk+Ow== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-4-key.pem b/deploy/dockerephemeral/docker/redis-node-4-key.pem new file mode 100644 index 0000000000..355661d6a9 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-4-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDKxOmFo5c4ae38 +qC3zI89R1F6xjaMyR6jjd6k5qsW7eU/y8+trgY4HV/jbzD3CDOjMkj70la5EiV+G +TA0iGeJH/BkKjqEPsIy/vAPux9xt2ZpIO9ieO2BF75ojrcM7tAbeOLQNAgYA7zAy +IpQkJ2P8IyOYSJ31ujLJCR7d0zudAbXJXfAAyPUWqUrmmRHIY7hRi1tUv74JARqn +U2tHZhFgGyBCaLROK69S/Wy+xPKo5w9Ol5L9eIccrK2/JwNpfsFAxJqXawNm1l1M +9gGk2MpQXzZeTg/hlusqCtPieOPUQKoEDXAgYArQy8iYkLuZzOtg2WwcPOhtfsgV +RLNEwXihAgMBAAECggEABRxG5XEc0dVro9tKQy9DHaUcWN3Av/bp5QfCSluJPcMe +Nnma1JwQjBNVyJZidRZVtLg34Xq3SG9s6qnWh+Y+m4FZUTiMiyRwO7HdqII9hkA+ +gPUPLdfBwql6CU2rFsFgDfBAa3aCV7ovjQftk2axwKxTDJbB8mxFtObnsgANp9SU +c+MTlNTs1IQ4ev4u1i9ntR8SlFMcYQUA2AxvOiEDu7b4x/Ph9TEGuR6wLxdImRq/ +7hXcPtGAJKYgZLzAwCrZrjGjHILSskTxdii+Tr52Aq75SA3tLYGkJfSxHTJjFe0u +1k4Ot4uSEjRf4DIwohbSFFbK/ZXG2uscn36OphtbUQKBgQDwQY263RPJ/M5mKvME +15DK1JW3DOLWCBiV0XzwXsS+QpE8pKs2YLeyrY7sV/w1tdnfNdfINCknuzC4tG7Y +I+QzCQGhyKrP2nj4K3SsKUcFk6OWxgiPF5CRmlWySJ+H6+yITKcSJt/ZjUvvGQyQ +TV+IQ8s4RbKII9Pvifai6SLJ2QKBgQDYDn4bqIfZKR0I46//AycGXAUl55Yfgeog +8CR5MatNz26crrmDzjnDgsRbKUxK+UZLl/zEXY5Npn06sOG1G0bO/t7wQqcPsXZt +rZTx58lKvW7LQhEBAz48y9QeK3WUvT1E3JMJ6rt+6IfHvbvCLIu9DwyGJ7Zc7N+6 +k5GduC9gCQKBgQC4Zdfd3+hcUwgnKjezM7ARvO/buqwvEa+s7UgzRMlELdtC7C/s +YHcdUFAt3anZn2VFCBJBuqcLs4RFf1bD1WhEM1lpTparSUcnUlMN//Beu14HTp8r +FC8FUasMVuj6bXzxb8ObDvMoCmaJcHRQHNKBx2amHfhUvQrhAsalasIkoQKBgAFo +XsP5XiE5FlpXeW8U6y0sblAn6R99bjQWvHYZr78LCfJ1ZPoJ3vB6KqNZaojWhPG7 +JMd2wJWa7xfxzRar/dMdcABqvsHoaxgd2GmXFAWrpEwouwmhpscooNItgE+eyAZp +1X9sCxqxkyjnAJEsTyDFN1Ssb5C9blu92GYJrC1ZAoGASChIICMp0HWrDSRxRCen +Fddf993aEI4e46NTWY54u2p0Ga62XUcaw5eND9QX6craD8nd7mMhwvdvQ5vuORBk +m+dqt0oU5cloVp0srHDA861CO8topJFaNGWdF4wDgLU8YKRzd6hNX8X0/CCRl1vd +z/YmtxfgU56SaqExe0X65eA= +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-4.conf b/deploy/dockerephemeral/docker/redis-node-4.conf new file mode 100644 index 0000000000..55b360f9f9 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-4.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6376 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-5-cert.pem b/deploy/dockerephemeral/docker/redis-node-5-cert.pem new file mode 100644 index 0000000000..e1221b9df7 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-5-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCSXrnTzrNfHudXjC0A +h0CiFRe3yp4j6cN3Hfv4snS6tPWXM4MH9Dka8zMLZvzRVQZK3PxDh/R/DQYBZhpy +LEvT7wYCDsS+F+tie2sPjzSAbdM5dolD8fGwACOqobI4vPz0QrwDqHde/OdVWAZl +h5Pzw5rDUu84CdfPSWRN1pomCFWG7gVkpuFzIcBfz+smPodyw3BfU8969q6tFACE +pjGPF/RufmHoIaHe2q/c+3HBY06ro0oTqTtRe36v4Jp2HLE/jE8wc+YggTmHE670 +uEXIR9N3fF3AbPVnhimEwcQ5fpJtMonUvfj5Z/4KfKo/0Yrh0wljeRiz/tZgTwwb +h5ATAgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy01hwSsFAAjMB0GA1UdDgQWBBRkbb1LScfQthztQJ3l +R+QFCKjXUTAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAn8TOFqomU30SmIDIHYBKMRGq3bVDLkDDC2yy6LCCwwG2rpoO +UtnUMig2w3iNQ6nvqR4LJB1ha0hLK5FP3iX/JcqZiO0NaucOTe7aJlt9taCADgAw +4vRW/pDuxtq7H1hc2pOue6i05UtGqy2E12jYowQc8a/5hylfEO3b5t5Z7xoQzyAZ +1ov7sYatBinwhqyDI5qNvZCuyT7SMx7H10T7cPrEec4uq55AJ0ReXnAAy1MLhpGd +nW5FX3F4gnyJcK2xL/V+ScL4NTzA8qWT+qOK33KxU1qrGripAkFaF6Z110nuIDiP +Z2tneIovCKKChgFsmZjy2spRpDw6R3Am6rXjpA== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-5-key.pem b/deploy/dockerephemeral/docker/redis-node-5-key.pem new file mode 100644 index 0000000000..467778629c --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-5-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCSXrnTzrNfHudX +jC0Ah0CiFRe3yp4j6cN3Hfv4snS6tPWXM4MH9Dka8zMLZvzRVQZK3PxDh/R/DQYB +ZhpyLEvT7wYCDsS+F+tie2sPjzSAbdM5dolD8fGwACOqobI4vPz0QrwDqHde/OdV +WAZlh5Pzw5rDUu84CdfPSWRN1pomCFWG7gVkpuFzIcBfz+smPodyw3BfU8969q6t +FACEpjGPF/RufmHoIaHe2q/c+3HBY06ro0oTqTtRe36v4Jp2HLE/jE8wc+YggTmH +E670uEXIR9N3fF3AbPVnhimEwcQ5fpJtMonUvfj5Z/4KfKo/0Yrh0wljeRiz/tZg +Twwbh5ATAgMBAAECggEAFEPciJsaseBueUGQItLsZkRzVzVMtdOHW4uhjOpFnRVc +LLCrbe4opeGRaf0P+HpEIm38LewPNDP9ETPYv4FV3PmVTwhKbGNAFLovtXocnmzA +4jjWLRESEaMYombmwlJFghq8kJPCNeWKsIHnyNDU8YVd0mM+JE0V6GjUjq5YA0x4 +co87wiNxAtjdNuAmI8elOqH3YhCwCQjYO1NeEJwIhWz5tgb2J9Rvn85LOh65cU17 +FaxiTBNSrMW44yk+uhEyj8IDbcZax0s8gLbCSLIj/MnuSbm74VkGKXme0U3mJSmn +dY2tpO3DnNZ+qvakpUk50e/LXYFofsJH9cs1BlZ7AQKBgQDJufGOp07KcIG4N3ei +YxH1IRZ8vThOHksbKVnzQLRcJcEY6SHrL3DgxS+kO3IfjkKZGw5vZgV4/jfTfWQP +eDXwIl0t/YVCEDECppfAIN7fyvIVI14quRogbIrn0jn5ijhVzPI8SWvi/viFbFvn +2O/8KUaHudv9yQ6zKItZ1zHAkwKBgQC5wBLKYdeQT5EfvfXT+rHoioUyywFxTpOF +em14JfNwKLdhqEVB99MzGEdRs6HNz88YbhKQpuEQjwJkbBXZUpAXyYPLDN5uQtV7 +Xw1MY7d8O7U5qNevos+Yti8rrv4w8Cb8ppOX0DJ2SD7J4OQjuyiRYx6sE+tQH6p+ +6N2Gt9YigQKBgQCqpnt7s3uK9Aw42+t/2xFo7lnIooYMR8I/swaeKsGpJmMpAKep +/pMeApHf/E359e3O+b2HbaX5ig2OAwhvscDnaRqsekiN74aWeHntlaEVbujGCwpx +V++LOGd13zkeKdiodN0DNRVojUuOC3HgO3whNIWu8gLxuXGPDCB+mvZCswKBgH+I +vh4QgZYG22iE37U0ylQUT5HpSktGnQknXuQAgp1+hzJY+3xosKzDPax9/lk2FkX6 +xWpl+d+JoSXcBEBbbK24YXHXmxzvbG4xfAr36DI3OJ2nLLfdvFVouQhwNPza1pnf +sTSp8Qu/XMT1UQ6rYRY5jQSvBIDVzRUnw3nM3QyBAoGAXs5Mg1jcQme6X56e0Db0 +zDCcEJuYL+nWXSkClsQCaDwafi4PQVP/V351Qruw0n98grD5vacz1HdXosvCaACJ +8P8e4sFJmSGu8SQt4zbReq8DHNTWZyPC8muurnMSKtfg3XulY8SFsoog7dlMzGGY +IMDiEb5jIb6DFcpNxjigXsM= +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-5.conf b/deploy/dockerephemeral/docker/redis-node-5.conf new file mode 100644 index 0000000000..ba2cfde9c6 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-5.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6377 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-6-cert.pem b/deploy/dockerephemeral/docker/redis-node-6-cert.pem new file mode 100644 index 0000000000..c176eae043 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-6-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCmN9ktdBsuxTOPFUsU +qAjMnQSyBz/BpYDGMagy9e7PbtniVGTHHOvGgoq5VvPdtiVTerwefNAQaL3nLLvg +24hOEWBlQuBgK0gW48NPZJAbzYvNdF2jOzIzsDu8edEz4TcI8oKvw2WS5HQGl213 +06f2tMN1Ng0O07WoW8cxOYISsKVT9EyQJX4M/Oq5/nzHkXvS97ayFT0OvVdIRzPU +A6VsSyr/X1LgVmZEGfWcdv+cxJGBiXRsiWdW+Y+n6qvRBC2WpTEhCXomtbbDtuSH +e+8EXk9eKSc5QYFNCDWEMk25JuEQpXIMfdiHbMmK+9BgdRTUh8Pm94yD3hkMO6z5 +N5e3AgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy02hwSsFAAkMB0GA1UdDgQWBBRyt96xEM6o5VkG9JV5 +vLVxnBELSjAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAmxFmsjenSgrrI1sE7DJahX1CaNVGodx4CwVc2etEq5PBWC6r +DpfCcYDW+Hg64Ac+NiPaLxFaG/8aM7JSePbAa71AQN+2hJpsV3/ANvSUaJfbHSFx +xfTRr8m5l33IV7ynjvZCPXWK4Gc5o7/shPKObHjwb03DLJjW0rvD5SYIjfCLjlOk +na2ufQnrmEP0XO77EvP4G/sHBjUaXrthsYTISO3lBTnGoKWNj8YwTFtXILC3O1to +sKWKYe5A6FB6xathUVBfS+Drp0PIYdAU9N3adymv4tZf52ofMsbJNkDqY3JaWmcO +dYHuYTeYg6ZiVhzZeasd3V+wc/CKAD8U5UfD5A== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-6-key.pem b/deploy/dockerephemeral/docker/redis-node-6-key.pem new file mode 100644 index 0000000000..0bc3f36618 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-6-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCmN9ktdBsuxTOP +FUsUqAjMnQSyBz/BpYDGMagy9e7PbtniVGTHHOvGgoq5VvPdtiVTerwefNAQaL3n +LLvg24hOEWBlQuBgK0gW48NPZJAbzYvNdF2jOzIzsDu8edEz4TcI8oKvw2WS5HQG +l21306f2tMN1Ng0O07WoW8cxOYISsKVT9EyQJX4M/Oq5/nzHkXvS97ayFT0OvVdI +RzPUA6VsSyr/X1LgVmZEGfWcdv+cxJGBiXRsiWdW+Y+n6qvRBC2WpTEhCXomtbbD +tuSHe+8EXk9eKSc5QYFNCDWEMk25JuEQpXIMfdiHbMmK+9BgdRTUh8Pm94yD3hkM +O6z5N5e3AgMBAAECggEAQ088YB6zX0Y2McvyooPFRG6VVy5+UAGgWyICtdhHg7Kl +AvUf9k2s4K8+U/11NaQsC1kZUtNCQlLYDARedJkR4mNBAOCLEgaU48gJ8F2NyeR7 +p5Bm1tIC61GDbzh5UiPycGocJ+bdfBWNMpohlzObwdjDifSAZy+uUWYRDMr39G7x +9SH6aLL7ZHBg0Oc4dw6K4GQMrU7sdomQcSqNyi5sn6PN8FsuO1wMp8C8V+U6y5sb +36Y1rz90ZOFqmOBnG/IdPR8tFbdql1Yy31tzy/I4thK+1v4QN6JVLPvyw4H0RzFe +j347k5IsNehRdwltplhckeAUzWGGNiTx0zhQPAchuQKBgQDmyGB055GCtRoEIpqN +ANNa8PxTp2sCH+/J7KZma6gSJ9WY73xtGSVXX/Ubz4l8FHiGoA0CQCElARJ9zff/ +tAiNXqvcQeBPVC23CMJL3hxeHLNs0ipoD8qvdQpGit3DAZMjdjtt5jd48CulEmfP +/rVmeHKChZaPPR1EgrMnIytaCwKBgQC4YWygHnDjW9zekpsDRMKkvK5QMIey9ygB +LqXlXw6GANhVDGSr7zOHBtF1aBc6FA1FKlVRXz3Fag4pPZLd2HbEaKnzfCNPH5PL +UTX8fukftrzY03bvpYcr+/YabPO8H5hkeUqHyH9EyIgdj5hOhKEVj9kJkqENt3el +GvohkgdwhQKBgG0itPqTx6wYGIV8F7o2eby32Zt1wJTwpWTIFKi6oHB1hf0cw6qU +CaSYLEFKk6mpxJVlesFlskbdivETRgQWDzVLX9p5DKp3FGdKLRfToXaf+/mqKYOs +dB0lLAbQBK8DP6G1d8Uw6Wq3qOwXGCC0QvSCYSR4KAr0y7JqXG5Vo1qhAoGATLCh +GNxwgfDEpoL+HNbtys18B3iYCLVKm2tGr2fhR5V0ZbOY7/a3TPNmDdp0xsBuYJVi +FU1zCPi62SZ2PvX5OGp8Pf0lRpTQyWGG/fXfi0RbuigCsVz9IytSyt0EZ/wQS8Iz +YNThMr/h9cGzTP1Xbvt8/8FQYb8s8ayN24a8t20CgYBDyjVifJHw6iVl3vu/O+R9 ++AdSe5bEGGDuIKZRDJbEj2ScgD3Nwqdst7X5wC+rcUuyJNW22GihyLiC/+OCaJPl +9fyaRpWWjEUkzpvR+3GhzzykDnemw1z39AJrg3ewSaBdbw9Bvq0ebrGaDF+uCReY +V+yVEYFsBaK0JrbkIffXbA== +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-6.conf b/deploy/dockerephemeral/docker/redis-node-6.conf new file mode 100644 index 0000000000..2989c5550e --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-6.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6378 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/docs/src/developer/developer/building.md b/docs/src/developer/developer/building.md index 4cbdba8382..bcbca56502 100644 --- a/docs/src/developer/developer/building.md +++ b/docs/src/developer/developer/building.md @@ -156,6 +156,7 @@ These services require most of the deployment dependencies as seen in the archit - Required internal dependencies: - cassandra (with the correct schema) - elasticsearch (with the correct schema) + - redis - Required external dependencies are the following configured AWS services (or “fake” replacements providing the same API): - SES - SQS diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index 9154c97ea7..062f891f03 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -1983,6 +1983,96 @@ elasticsearch-index: insecureSkipVerifyTls: true ``` +## Configure Redis authentication + +If the redis used needs authentication with either username and password or just +password (legacy auth), it can be configured like this: + +```yaml +gundeck: + secrets: + redisUsername: + redisPassword: +``` + +**NOTE**: When using redis < 6, the `redisUsername` must not be set at all (not +even set to `null` or empty string, the key must be absent from the config). +When using redis >= 6 and using legacy auth, the `redisUsername` must either be +not set at all or set to `"default"`. + +While doing migrations to another redis instance, the credentials for the +addtional redis can be set as follows: + +```yaml +gundeck: + secrets: + redisAdditionalWriteUsername: # Do not set this at all when using legacy auth + redisAdditionalWritePassword: +``` + +**NOTE**: `redisAddtiionalWriteUsername` follows same restrictions as +`redisUsername` when using legacy auth. + +## Configure TLS for Redis + +If the redis instance requires TLS, it can be configured like this: + +```yaml +gundeck: + config: + redis: + enableTls: true +``` + +In case a custom CA certificate is required it can be provided like this: + +```yaml +gundeck: + config: + redis: + tlsCa: +``` + +There is another way to provide this, in case there already exists a kubernetes +secret containing the CA certificate(s): + +```yaml +gundeck: + config: + redis: + tlsCaSecretRef: + name: + key: +``` + +For configuring `redisAdditionalWrite` in gundeck (this is required during a +migration from one redis instance to another), the settings need to be like +this: + +```yaml +gundeck: + config: + redisAdditionalWrite: + enableTls: true + # One or none of these: + # tlsCa: + # tlsCaSecretRef: +``` + +**WARNING:** Please do this only if you know what you’re doing. + +In case it is not possible to verify TLS certificate of the redis +server, it can be turned off without tuning off TLS like this: + +```yaml +gundeck: + config: + redis: + insecureSkipVerifyTls: true + redisAdditionalWrite: + insecureSkipVerifyTls: true +``` + ## Configure RabbitMQ RabbitMQ authentication must be configured on brig, galley and background-worker. For example: @@ -2015,7 +2105,7 @@ server, verification can be turned off by settings `insecureSkipVerifyTls` to ## Configure PostgreSQL -`brig`, `galley`, `gundeck`, and `background-worker` require a PostgreSQL database. The configured user needs to +`brig`, `galley`, and `background-worker` require a PostgreSQL database. The configured user needs to be able to write data and change the schema (e.g. create and alter tables.) The internal configuration YAML file format and the Helm charts for `brig` and @@ -2072,7 +2162,7 @@ The `port` needs to be a number provided as string. Besides the password file (`postgresqlPassword`), the fields correspond to [libpq-connect parameters](https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS). -The `postgresqlPassword` file is read by `brig`, `galley`, `gundeck`, and `background-worker`. Its content is +The `postgresqlPassword` file is read by `brig`, `galley`, and `background-worker`. Its content is used as `password` field. ### Using PostgreSQL for storing Cassandra-backed data diff --git a/docs/src/how-to/install/infrastructure-configuration.md b/docs/src/how-to/install/infrastructure-configuration.md index 06ef7a218d..34e1eb2c19 100644 --- a/docs/src/how-to/install/infrastructure-configuration.md +++ b/docs/src/how-to/install/infrastructure-configuration.md @@ -27,6 +27,7 @@ gundeck: - "10.0.0.0/8" - "elasticsearch-external" - "cassandra-external" + - "redis-ephemeral" - "fake-aws-sqs" - "fake-aws-dynamodb" - "fake-aws-sns" @@ -414,8 +415,8 @@ cassandra cannot reliably be installed on kubernetes. Some people have tried, e.g. [this project](https://github.com/instaclustr/cassandra-operator) though at the time of writing (Nov 2018), this does not yet work as advertised. We -recommend therefore to install cassandra, (possibly also elasticsearch) -separately, i.e. outside of kubernetes (using 3 nodes each). +recommend therefore to install cassandra, (possibly also elasticsearch +and redis) separately, i.e. outside of kubernetes (using 3 nodes each). For further higher-availability: diff --git a/docs/src/how-to/install/troubleshooting.md b/docs/src/how-to/install/troubleshooting.md index 3703500a9f..ecb0ffb636 100644 --- a/docs/src/how-to/install/troubleshooting.md +++ b/docs/src/how-to/install/troubleshooting.md @@ -252,7 +252,7 @@ These are some steps you can take to debug what is going on when the installatio As an example, we’ll take a case where we try installing `wire-server` with `helm`, but it fails due to `cassandra` being broken in some way. -This guide, while focusing on a `cassandra` related issue, will also provide general steps to debug problems that could be related to other components like `rabbitmq`, etc. +This guide, while focusing on a `cassandra` related issue, will also provide general steps to debug problems that could be related to other components like `rabbitmq`, `redis`, etc. Our first step is to identify and isolate which component is causing the issue. @@ -266,6 +266,7 @@ fake-aws-sns-76fb45cf4f-t6mg6 2/2 Running 0 75m fake-aws-sqs-6495cd7c98-w8f8w 2/2 Running 0 75m rabbitmq-external-0 0/1 Pending 0 78m reaper-84cfbf746d-wk8nc 1/1 Running 0 75m +redis-ephemeral-master-0 1/1 Running 0 76m ``` We then run the `wire-server` helm installation command: @@ -293,6 +294,7 @@ fake-aws-sns-76fb45cf4f-t6mg6 2/2 Running 0 95m fake-aws-sqs-6495cd7c98-w8f8w 2/2 Running 0 95m rabbitmq-external-0 0/1 Pending 0 98m reaper-84cfbf746d-wk8nc 1/1 Running 0 95m +redis-ephemeral-master-0 1/1 Running 0 96m ``` (You can also do `d kubectl get pods -o wide` to get more details though that’s not necessary here) diff --git a/hack/bin/gen-certs.sh b/hack/bin/gen-certs.sh index d4840f8af6..f995a238aa 100755 --- a/hack/bin/gen-certs.sh +++ b/hack/bin/gen-certs.sh @@ -81,6 +81,19 @@ install_certs "$TEMP/es" "$ROOT_DIR/deploy/dockerephemeral/docker" \ install_certs "$TEMP/es" "$ROOT_DIR/hack/helm_vars/certs" \ elasticsearch-ca elasticsearch-ca-key +# redis +mkdir -p "$TEMP/redis" +gen_ca "$TEMP/redis" redis.ca.example.com +REDIS="$ROOT_DIR/deploy/dockerephemeral/docker" +cp "$TEMP/redis/ca.pem" "$REDIS/redis-ca.pem" +for redis_node in $(seq 1 6); do + gen_cert "$TEMP/redis" "DNS:redis-${redis_node}, IP:172.20.0.3${redis_node}" + chmod 0644 "$TEMP/redis/key.pem" + install_certs "$TEMP/redis" "$REDIS" "" "" \ + "redis-node-${redis_node}-cert" \ + "redis-node-${redis_node}-key" +done + # rabbitmq RABBITMQ="$ROOT_DIR/deploy/dockerephemeral/rabbitmq-config/certificates" gen_ca "$RABBITMQ" rabbitmq.ca.example.com diff --git a/hack/helm_vars/certs/values.yaml.gotmpl b/hack/helm_vars/certs/values.yaml.gotmpl index 7cd8a63365..307d50fa48 100644 --- a/hack/helm_vars/certs/values.yaml.gotmpl +++ b/hack/helm_vars/certs/values.yaml.gotmpl @@ -16,6 +16,59 @@ resources: ca: secretName: elasticsearch-ca + # redis CA and certificate + - apiVersion: cert-manager.io/v1 + kind: Issuer + metadata: + name: redis-ca-issuer + namespace: '{{ .Release.Namespace }}' + spec: + selfSigned: {} + - apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: redis-ca + namespace: '{{ .Release.Namespace }}' + spec: + secretName: redis-ca-certificate + isCA: true + duration: 2160h # 90d + renewBefore: 360h # 15d + commonName: redis.example.com + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + issuerRef: + name: redis-ca-issuer + kind: Issuer + - apiVersion: cert-manager.io/v1 + kind: Issuer + metadata: + name: redis-issuer + namespace: '{{ .Release.Namespace }}' + spec: + ca: + secretName: redis-ca-certificate + - apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: redis + namespace: '{{ .Release.Namespace }}' + spec: + secretName: redis-certificate + isCA: false + duration: 2160h # 90d + renewBefore: 360h # 15d + commonName: redis-ephemeral + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + issuerRef: + name: redis-issuer + kind: Issuer + # RabbitMQ CA and certificate - apiVersion: cert-manager.io/v1 kind: Issuer diff --git a/hack/helm_vars/redis-ephemeral/values.yaml b/hack/helm_vars/redis-ephemeral/values.yaml new file mode 100644 index 0000000000..996dc30e45 --- /dev/null +++ b/hack/helm_vars/redis-ephemeral/values.yaml @@ -0,0 +1,47 @@ +redis-ephemeral: + image: + registry: public.ecr.aws + repository: docker/library/redis + + redisConfig: | + requirepass very-secure-redis-master-password + + # ephemeral + save "" + + port 0 + tls-port 6379 + tls-cert-file /data/ssl/tls.crt + tls-key-file /data/ssl/tls.key + tls-ca-cert-file /data/ssl/ca.crt + tls-auth-clients no + + extraRedisSecrets: + - name: redis-certificate + mountPath: /data/ssl + + livenessProbe: + enabled: false + customLivenessProbe: + exec: + command: + - sh + - -c + - redis-cli --tls --cacert /data/ssl/ca.crt ping + readinessProbe: + enabled: false + customReadinessProbe: + exec: + command: + - sh + - -c + - redis-cli --tls --cacert /data/ssl/ca.crt ping + startupProbe: + enabled: false + customStartupProbe: + exec: + command: + - sh + - -c + - redis-cli --tls --cacert /data/ssl/ca.crt ping + diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index 43373b1cf2..f05d42a98e 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -492,11 +492,13 @@ gundeck: tlsCaSecretRef: name: "rabbitmq-certificate" key: "ca.crt" - postgresql: - host: "postgresql" - port: "5432" - user: wire-server - dbname: wire-server + redis: + host: redis-ephemeral + connectionMode: master + enableTls: true + tlsCaSecretRef: + name: "redis-certificate" + key: "ca.crt" aws: account: "123456789012" region: eu-west-1 @@ -512,7 +514,7 @@ gundeck: secrets: awsKeyId: dummykey awsSecretKey: dummysecret - pgPassword: posty-the-gres + redisPassword: very-secure-redis-master-password rabbitmq: username: {{ .Values.rabbitmqUsername }} password: {{ .Values.rabbitmqPassword }} @@ -527,6 +529,7 @@ gundeck: uploadXmlAwsAccessKeyId: {{ .Values.uploadXml.awsAccessKeyId }} uploadXmlAwsSecretAccessKey: {{ .Values.uploadXml.awsSecretAccessKey }} {{- end }} + redisAdditionalWritePassword: very-secure-redis-master-password-2 nginz: replicaCount: 1 @@ -726,6 +729,10 @@ integration: tlsCaSecretRef: name: {{ .Values.elasticsearch.caSecretName }} key: "ca.crt" + redis: + tlsCaSecretRef: + name: "redis-certificate" + key: "ca.crt" rabbitmq: tlsCaSecretRef: name: "rabbitmq-certificate" @@ -739,6 +746,7 @@ integration: uploadXmlAwsAccessKeyId: {{ .Values.uploadXml.awsAccessKeyId }} uploadXmlAwsSecretAccessKey: {{ .Values.uploadXml.awsSecretAccessKey }} {{- end }} + redisPassword: very-secure-redis-master-password tls: caNamespace: wire-federation-v0 diff --git a/hack/helmfile.yaml.gotmpl b/hack/helmfile.yaml.gotmpl index f28ed995af..09c825f9c3 100644 --- a/hack/helmfile.yaml.gotmpl +++ b/hack/helmfile.yaml.gotmpl @@ -114,6 +114,15 @@ releases: values: - './helm_vars/certs/values.yaml.gotmpl' + - name: 'redis-ephemeral' + namespace: '{{ .Values.namespace1 }}' + chart: '../.local/charts/redis-ephemeral' + values: + - './helm_vars/wire-image-mirror.yaml' + - './helm_vars/redis-ephemeral/values.yaml' + needs: + - certs + - name: 'cassandra-ephemeral' namespace: '{{ .Values.namespace1 }}' chart: '../.local/charts/cassandra-ephemeral' @@ -138,6 +147,15 @@ releases: name: elasticsearch kind: Issuer + - name: 'redis-ephemeral' + namespace: '{{ .Values.namespace2 }}' + chart: '../.local/charts/redis-ephemeral' + values: + - './helm_vars/wire-image-mirror.yaml' + - './helm_vars/redis-ephemeral/values.yaml' + needs: + - certs + - name: 'cassandra-ephemeral' namespace: '{{ .Values.namespace2 }}' chart: '../.local/charts/cassandra-ephemeral' @@ -198,6 +216,22 @@ releases: values: - './helm_vars/opensearch/values.yaml.gotmpl' + # Required for testing redis migration + - name: redis-ephemeral-2 + namespace: '{{ .Values.namespace1 }}' + chart: '../.local/charts/redis-ephemeral' + values: + - redis-ephemeral: + image: + registry: public.ecr.aws + repository: docker/library/redis + + redisConfig: | + requirepass very-secure-redis-master-password-2 + + # ephemeral + save "" + - name: 'certs' namespace: '{{ .Values.namespace2 }}' chart: bedag/raw @@ -324,6 +358,7 @@ releases: value: true needs: - 'cassandra-ephemeral' + - 'redis-ephemeral' - 'postgresql' - name: 'wire-server' @@ -341,6 +376,7 @@ releases: value: {{ .Values.federationDomain2 }} needs: - 'cassandra-ephemeral' + - 'redis-ephemeral' - 'postgresql' - name: wire-server-enterprise diff --git a/libs/wire-api/src/Wire/API/Presence.hs b/libs/wire-api/src/Wire/API/Presence.hs index 9ec538f064..427e0a0f8a 100644 --- a/libs/wire-api/src/Wire/API/Presence.hs +++ b/libs/wire-api/src/Wire/API/Presence.hs @@ -23,6 +23,7 @@ import Data.Aeson.Types qualified as A import Data.Attoparsec.ByteString (takeByteString) import Data.ByteString.Char8 qualified as Bytes import Data.ByteString.Conversion +import Data.ByteString.Lazy qualified as Lazy import Data.Id import Data.Misc (Milliseconds) import Data.OpenApi qualified as S @@ -34,6 +35,7 @@ import Imports import Network.URI qualified as Net import Servant.API (ToHttpApiData (toUrlPiece)) +-- FUTUREWORK: use Network.URI and toss this newtype. servant should have all these instances for us these days. newtype URI = URI { fromURI :: Net.URI } @@ -75,7 +77,9 @@ data Presence = Presence -- operating the team settings pages without the need for -- end-to-end crypto. clientId :: !(Maybe ClientId), - createdAt :: !Milliseconds + createdAt :: !Milliseconds, + -- | REFACTOR: temp. addition to ease migration + __field :: !Lazy.ByteString } deriving (Eq, Ord, Show) deriving (A.FromJSON, A.ToJSON, S.ToSchema) via (Schema Presence) @@ -90,6 +94,7 @@ instance ToSchema Presence where <*> clientId .= optField "client_id" (maybeWithDefault A.Null schema) -- keep null for backwards compat <*> createdAt .= (fromMaybe 0 <$> (optField "created_at" schema)) ) + <&> ($ ("" :: Lazy.ByteString)) uriSchema :: ValueSchema NamedSwaggerDoc URI uriSchema = mkSchema desc uriFromJSON (Just . uriToJSON) diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Manual/Presence.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Manual/Presence.hs index adb5f582d4..97005af0a0 100644 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Manual/Presence.hs +++ b/libs/wire-api/test/golden/Test/Wire/API/Golden/Manual/Presence.hs @@ -35,6 +35,7 @@ testObject_Presence_1 = (fromJust $ parse "http://example.com/") Nothing 0 + "" testObject_Presence_2 :: Presence testObject_Presence_2 = @@ -44,6 +45,7 @@ testObject_Presence_2 = (fromJust $ parse "http://example.com/3") (Just (ClientId 1)) 12323 + "" -- __field always has to be "", see ToSchema instance. testObject_Presence_3 :: Presence testObject_Presence_3 = @@ -53,3 +55,4 @@ testObject_Presence_3 = (fromJust $ parse "http://example.com/3") (Just (ClientId 1)) 0 + "" -- __field always has to be "", see ToSchema instance. diff --git a/libs/wire-subsystems/postgres-migrations/20260828093750-gundeck-presence.sql b/libs/wire-subsystems/postgres-migrations/20260828093750-gundeck-presence.sql deleted file mode 100644 index eb84c4ed7f..0000000000 --- a/libs/wire-subsystems/postgres-migrations/20260828093750-gundeck-presence.sql +++ /dev/null @@ -1,12 +0,0 @@ --- WPB-28377: gundeck presence (replaces redis presence hashes) -CREATE TABLE IF NOT EXISTS presence ( - user_id uuid NOT NULL, - conn_id text NOT NULL, - resource text NOT NULL, - client_id text, - created_at timestamptz NOT NULL, - PRIMARY KEY (user_id, conn_id) -); - --- index for cleanup deletes like `DELETE ... WHERE created_at < now() - interval '7 days'` -CREATE INDEX presence_created_at_idx ON presence (created_at); diff --git a/libs/wire-subsystems/src/Wire/JobSubsystem/Migrations.hs b/libs/wire-subsystems/src/Wire/JobSubsystem/Migrations.hs index 2410cb98c6..920782cc85 100644 --- a/libs/wire-subsystems/src/Wire/JobSubsystem/Migrations.hs +++ b/libs/wire-subsystems/src/Wire/JobSubsystem/Migrations.hs @@ -19,13 +19,11 @@ -- with this program. If not, see . module Wire.JobSubsystem.Migrations - ( defaultSchemaName, - mkArbiterConnectionString, + ( mkArbiterConnectionString, runJobMigrations, ) where -import Arbiter.Core (defaultSchemaName) import Arbiter.Migrations qualified as ArbiterMigrations import Control.Exception (bracket, bracket_, throwIO) import Data.Hashable qualified as Hashable diff --git a/libs/wire-subsystems/src/Wire/Postgres.hs b/libs/wire-subsystems/src/Wire/Postgres.hs index a0210c21cf..f91f3637ef 100644 --- a/libs/wire-subsystems/src/Wire/Postgres.hs +++ b/libs/wire-subsystems/src/Wire/Postgres.hs @@ -43,7 +43,6 @@ module Wire.Postgres runTransaction, runTransactionWithRetry, runPipeline, - useWithResetAndRetry, parseCount, PGConstraints, diff --git a/postgres-schema.sql b/postgres-schema.sql index 2d3df5fb27..de4a57a0f2 100644 --- a/postgres-schema.sql +++ b/postgres-schema.sql @@ -1511,21 +1511,6 @@ CREATE TABLE public.mls_history_client ( ALTER TABLE public.mls_history_client OWNER TO "wire-server"; --- --- Name: presence; Type: TABLE; Schema: public; Owner: wire-server --- - -CREATE TABLE public.presence ( - user_id uuid NOT NULL, - conn_id text NOT NULL, - resource text NOT NULL, - client_id text, - created_at timestamp with time zone NOT NULL -); - - -ALTER TABLE public.presence OWNER TO "wire-server"; - -- -- Name: remote_conversation_local_member; Type: TABLE; Schema: public; Owner: wire-server -- @@ -1982,14 +1967,6 @@ ALTER TABLE ONLY public.mls_history_client ADD CONSTRAINT mls_history_client_pkey PRIMARY KEY (group_id, id); --- --- Name: presence presence_pkey; Type: CONSTRAINT; Schema: public; Owner: wire-server --- - -ALTER TABLE ONLY public.presence - ADD CONSTRAINT presence_pkey PRIMARY KEY (user_id, conn_id); - - -- -- Name: remote_conversation_local_member remote_conversation_local_member_pkey; Type: CONSTRAINT; Schema: public; Owner: wire-server -- @@ -2462,13 +2439,6 @@ CREATE INDEX idx_meetings_recurrence_eff_end ON public.meetings USING btree (GRE CREATE INDEX idx_meetings_start_time ON public.meetings USING btree (start_time); --- --- Name: presence_created_at_idx; Type: INDEX; Schema: public; Owner: wire-server --- - -CREATE INDEX presence_created_at_idx ON public.presence USING btree (created_at); - - -- -- Name: user_group_member_user_id_idx; Type: INDEX; Schema: public; Owner: wire-server -- diff --git a/services/brig/src/Brig/Run.hs b/services/brig/src/Brig/Run.hs index 20365d1ebb..fd170bdbaf 100644 --- a/services/brig/src/Brig/Run.hs +++ b/services/brig/src/Brig/Run.hs @@ -69,7 +69,6 @@ import Wire.API.Routes.Version import Wire.API.Routes.Version.Wai import Wire.API.User (AccountStatus (PendingInvitation)) import Wire.DeleteQueue -import Wire.JobSubsystem.Migrations (defaultSchemaName, mkArbiterConnectionString, runJobMigrations) import Wire.OpenTelemetry (withTracer) import Wire.PostgresMigrations import Wire.Sem.Paging qualified as P @@ -118,10 +117,6 @@ migratePostgres opts resetFirst = do pool <- (.rawPool) <$> initPostgresPool opts.postgresqlPool opts.postgresql opts.postgresqlPassword when resetFirst $ resetSchema pool logger runAllMigrations pool logger - -- Also create the arbiter job schema, so that this command yields the full - -- database schema (e.g. for `make postgres-schema`). - arbiterConnStr <- mkArbiterConnectionString opts.postgresql opts.postgresqlPassword - runJobMigrations arbiterConnStr defaultSchemaName flush logger mkApp :: Opts -> IO (Wai.Application, Env) diff --git a/services/gundeck/default.nix b/services/gundeck/default.nix index 5de25d7fff..2e4f8b69d5 100644 --- a/services/gundeck/default.nix +++ b/services/gundeck/default.nix @@ -22,14 +22,14 @@ , conduit , containers , criterion +, crypton-x509-store , data-timeout , errors , exceptions , extended , extra , foldl -, hasql -, hasql-th +, hedis , hs-opentelemetry-instrumentation-wai , hs-opentelemetry-sdk , HsOpenSSL @@ -37,6 +37,7 @@ , http-client-tls , http-types , imports +, kan-extensions , lens , lens-aeson , lib @@ -45,6 +46,7 @@ , MonadRandom , mtl , multiset +, network , network-uri , optparse-applicative , prometheus-client @@ -77,7 +79,6 @@ , unliftio , unordered-containers , uuid -, vector , wai , wai-extra , wai-middleware-gunzip @@ -85,7 +86,6 @@ , websockets , wire-api , wire-otel -, wire-subsystems , yaml }: mkDerivation { @@ -110,14 +110,14 @@ mkDerivation { bytestring-conversion cassandra-util containers + crypton-x509-store data-timeout errors exceptions extended extra foldl - hasql - hasql-th + hedis hs-opentelemetry-instrumentation-wai hs-opentelemetry-sdk http-client @@ -148,14 +148,12 @@ mkDerivation { unliftio unordered-containers uuid - vector wai wai-extra wai-middleware-gunzip wai-utilities wire-api wire-otel - wire-subsystems yaml ]; executableHaskellDepends = [ @@ -175,8 +173,10 @@ mkDerivation { http-client http-client-tls imports + kan-extensions lens lens-aeson + network network-uri optparse-applicative random @@ -191,6 +191,7 @@ mkDerivation { tinylog types-common uuid + wai-utilities websockets wire-api yaml diff --git a/services/gundeck/gundeck.cabal b/services/gundeck/gundeck.cabal index 9ad88362bc..06bf1b5024 100644 --- a/services/gundeck/gundeck.cabal +++ b/services/gundeck/gundeck.cabal @@ -39,6 +39,7 @@ library Gundeck.Push.Native.Types Gundeck.Push.Websocket Gundeck.React + Gundeck.Redis Gundeck.Run Gundeck.Schema.Run Gundeck.Schema.V1 @@ -57,6 +58,7 @@ library Gundeck.ThreadBudget.Internal Gundeck.Util Gundeck.Util.DelayQueue + Gundeck.Util.Redis other-modules: Paths_gundeck hs-source-dirs: src @@ -124,14 +126,14 @@ library , bytestring-conversion >=0.2 , cassandra-util >=0.16.2 , containers >=0.5 + , crypton-x509-store , data-timeout , errors >=2.0 , exceptions >=0.4 , extended , extra >=1.1 , foldl - , hasql - , hasql-th + , hedis >=0.14.0 , hs-opentelemetry-instrumentation-wai , hs-opentelemetry-sdk , http-client >=0.7 @@ -162,14 +164,12 @@ library , unliftio >=0.2 , unordered-containers >=0.2 , uuid >=1.3 - , vector , wai >=3.2 , wai-extra >=3.0 , wai-middleware-gunzip >=0.0.2 , wai-utilities >=0.16 , wire-api , wire-otel - , wire-subsystems , yaml >=0.8 default-language: GHC2021 @@ -244,6 +244,7 @@ executable gundeck-integration Metrics Paths_gundeck TestSetup + Util hs-source-dirs: test/integration default-extensions: @@ -296,7 +297,7 @@ executable gundeck-integration build-depends: , aeson , async - , base >=4 && <5 + , base >=4 && <5 , base16-bytestring >=0.1 , bilge , bytestring @@ -309,8 +310,10 @@ executable gundeck-integration , http-client , http-client-tls , imports + , kan-extensions , lens , lens-aeson + , network , network-uri , optparse-applicative , random @@ -324,6 +327,7 @@ executable gundeck-integration , tinylog , types-common , uuid + , wai-utilities >=0.16 , websockets >=0.8 , wire-api , yaml diff --git a/services/gundeck/gundeck.integration.yaml b/services/gundeck/gundeck.integration.yaml index bb19ab89eb..00c8057479 100644 --- a/services/gundeck/gundeck.integration.yaml +++ b/services/gundeck/gundeck.integration.yaml @@ -13,17 +13,18 @@ cassandra: keyspace: gundeck_test # filterNodesByDatacentre: datacenter1 -postgresql: - host: 127.0.0.1 - port: "5432" - user: wire-server - dbname: backendA - password: posty-the-gres - -postgresqlPool: - size: 20 - acquisitionTimeout: 10s - idlenessTimeout: 10m +redis: + host: 172.20.0.31 + port: 6373 + connectionMode: cluster # master | cluster + enableTls: true + tlsCa: ../../deploy/dockerephemeral/docker/redis-ca.pem + insecureSkipVerifyTls: false + +# redisAdditionalWrite: +# host: 127.0.0.1 +# port: 6379 +# connectionMode: master aws: queueName: integration-gundeck-events diff --git a/services/gundeck/src/Gundeck/Env.hs b/services/gundeck/src/Gundeck/Env.hs index a7f233fb2c..39f6f98bda 100644 --- a/services/gundeck/src/Gundeck/Env.hs +++ b/services/gundeck/src/Gundeck/Env.hs @@ -23,19 +23,30 @@ import Bilge hiding (host, port) import Cassandra (ClientState) import Cassandra.Util (initCassandraForService) import Control.AutoUpdate +import Control.Concurrent.Async (Async) import Control.Lens (makeLenses, (^.)) +import Control.Retry (capDelay, exponentialBackoff) +import Data.ByteString.Char8 qualified as BSChar8 import Data.Id import Data.Misc (Milliseconds (..)) +import Data.Text qualified as Text +import Data.Time.Clock import Data.Time.Clock.POSIX +import Data.X509.CertificateStore as CertStore +import Database.Redis qualified as Redis import Gundeck.Aws qualified as Aws -import Gundeck.Options +import Gundeck.Options as Opt hiding (host, port) +import Gundeck.Options qualified as O +import Gundeck.Redis qualified as Redis import Gundeck.ThreadBudget -import Hasql.Pool.Extended qualified as HasqlPoolExt import Imports import Network.AMQP (Channel) import Network.AMQP.Extended qualified as Q import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.TLS (tlsManagerSettings) +import Network.TLS as TLS +import Network.TLS.Extra qualified as TLS +import System.Logger qualified as Log import System.Logger.Extended qualified as Logger data Env = Env @@ -44,7 +55,8 @@ data Env = Env _applog :: !Logger.Logger, _manager :: !Manager, _cstate :: !ClientState, - _hasqlPool :: !HasqlPoolExt.Pool, + _rstate :: !Redis.RobustConnection, + _rstateAdditionalWrite :: !(Maybe Redis.RobustConnection), _awsEnv :: !Aws.Env, _time :: !(IO Milliseconds), _threadBudgetState :: !(Maybe ThreadBudgetState), @@ -53,7 +65,7 @@ data Env = Env makeLenses ''Env -createEnv :: Opts -> IO Env +createEnv :: Opts -> IO ([Async ()], Env) createEnv o = do l <- Logger.mkLogger (o ^. logLevel) (o ^. logNetStrings) (o ^. logFormat) n <- @@ -64,7 +76,17 @@ createEnv o = do managerResponseTimeout = responseTimeoutMicro 5000000 } - pgPool <- HasqlPoolExt.initPostgresPool (o ^. postgresqlPool) (o ^. postgresql) (o ^. postgresqlPassword) + redisUsername <- BSChar8.pack <$$> lookupEnv "REDIS_USERNAME" + redisPassword <- BSChar8.pack <$$> lookupEnv "REDIS_PASSWORD" + (rThread, r) <- createRedisPool l (o ^. redis) redisUsername redisPassword "main-redis" + + (rAdditionalThreads, rAdditional) <- case o ^. redisAdditionalWrite of + Nothing -> pure ([], Nothing) + Just additionalRedis -> do + additionalRedisUsername <- BSChar8.pack <$$> lookupEnv "REDIS_ADDITIONAL_WRITE_USERNAME" + addtionalRedisPassword <- BSChar8.pack <$$> lookupEnv "REDIS_ADDITIONAL_WRITE_PASSWORD" + (rAddThread, rAdd) <- createRedisPool l additionalRedis additionalRedisUsername addtionalRedisPassword "additional-write-redis" + pure ([rAddThread], Just rAdd) p <- initCassandraForService @@ -82,8 +104,55 @@ createEnv o = do } mtbs <- mkThreadBudgetState `mapM` (o ^. settings . maxConcurrentNativePushes) rabbitMqChannelMVar <- Q.mkRabbitMqChannelMVar l (Just "gundeck") (o ^. rabbitmq) - pure $! Env (RequestId defRequestId) o l n p pgPool a io mtbs rabbitMqChannelMVar + pure $! (rThread : rAdditionalThreads,) $! Env (RequestId defRequestId) o l n p r rAdditional a io mtbs rabbitMqChannelMVar reqIdMsg :: RequestId -> Logger.Msg -> Logger.Msg reqIdMsg = ("request" Logger..=) . unRequestId {-# INLINE reqIdMsg #-} + +createRedisPool :: Logger.Logger -> RedisEndpoint -> Maybe ByteString -> Maybe ByteString -> ByteString -> IO (Async (), Redis.RobustConnection) +createRedisPool l ep username password identifier = do + customCertStore <- case ep._tlsCa of + Nothing -> pure Nothing + Just caPath -> CertStore.readCertificateStore caPath + let defClientParams = defaultParamsClient (Text.unpack ep._host) "" + tlsParams = + guard ep._enableTls + $> defClientParams + { clientHooks = + if ep._insecureSkipVerifyTls + then defClientParams.clientHooks {onServerCertificate = \_ _ _ _ -> pure []} + else defClientParams.clientHooks, + clientShared = + case customCertStore of + Nothing -> defClientParams.clientShared + Just sharedCAStore -> defClientParams.clientShared {sharedCAStore}, + clientSupported = + defClientParams.clientSupported + { supportedVersions = [TLS.TLS13, TLS.TLS12], + supportedCiphers = TLS.ciphersuite_strong + } + } + let redisConnInfo = + Redis.defaultConnectInfo + { Redis.connectAddr = Redis.ConnectAddrHostPort (Text.unpack ep._host) (fromIntegral ep._port), + Redis.connectUsername = username, + Redis.connectAuth = password, + Redis.connectTimeout = Just (secondsToNominalDiffTime 5), + Redis.connectMaxConnections = 100, + Redis.connectTLSParams = tlsParams + } + + Log.info l $ + Log.msg (Log.val $ "starting connection to " <> identifier <> "...") + . Log.field "connectionMode" (show $ ep ^. O.connectionMode) + . Log.field "connInfo" (safeShowConnInfo redisConnInfo) + let connectWithRetry = Redis.connectRobust l (capDelay 1000000 (exponentialBackoff 50000)) + r <- case ep ^. O.connectionMode of + Master -> connectWithRetry $ Redis.checkedConnect redisConnInfo + Cluster -> connectWithRetry $ Redis.checkedConnectCluster redisConnInfo + Log.info l $ Log.msg (Log.val $ "Established connection to " <> identifier <> ".") + pure r + +safeShowConnInfo :: Redis.ConnectInfo -> String +safeShowConnInfo connInfo = show $ connInfo {Redis.connectAuth = "[REDACTED]" <$ Redis.connectAuth connInfo} diff --git a/services/gundeck/src/Gundeck/Monad.hs b/services/gundeck/src/Gundeck/Monad.hs index db3eda9618..832bff5d89 100644 --- a/services/gundeck/src/Gundeck/Monad.hs +++ b/services/gundeck/src/Gundeck/Monad.hs @@ -33,6 +33,10 @@ module Gundeck.Monad runGundeck, posixTime, getRabbitMqChan, + + -- * Select which redis to target + runWithDefaultRedis, + runWithAdditionalRedis, msToUTCSecs, ) where @@ -49,7 +53,9 @@ import Data.Time (UTCTime) import Data.Time.Clock.POSIX (posixSecondsToUTCTime) import Data.UUID as UUID import Data.UUID.V4 as UUID +import Database.Redis qualified as Redis import Gundeck.Env +import Gundeck.Redis qualified as Redis import Imports import Network.AMQP import Network.HTTP.Types @@ -61,6 +67,7 @@ import System.Logger (Logger) import System.Logger qualified as Logger import System.Logger.Class qualified as Log import System.Timeout +import UnliftIO (async) -- | TODO: 'Client' already has an 'Env'. Why do we need two? How does this even work? We should -- probably explain this here. @@ -84,6 +91,72 @@ newtype Gundeck a = Gundeck instance MonadMonitor Gundeck where doIO = liftIO +-- | 'Gundeck' doesn't have an instance for 'MonadRedis' because it contains two +-- connections to two redis instances. When using 'WithDefaultRedis', any redis +-- operation will only target the default redis instance (configured under +-- 'redis:' in the gundeck config). To write to both redises use +-- 'WithAdditionalRedis'. +newtype WithDefaultRedis a = WithDefaultRedis {runWithDefaultRedis :: Gundeck a} + deriving newtype + ( Functor, + Applicative, + Monad, + MonadIO, + MonadThrow, + MonadCatch, + MonadMask, + MonadReader Env, + MonadClient, + MonadUnliftIO, + Log.MonadLogger + ) + +instance Redis.MonadRedis WithDefaultRedis where + liftRedis action = do + defaultConn <- view rstate + Redis.runRobust defaultConn action + +instance Redis.RedisCtx WithDefaultRedis (Either Redis.Reply) where + returnDecode :: (Redis.RedisResult a) => Redis.Reply -> WithDefaultRedis (Either Redis.Reply a) + returnDecode = Redis.liftRedis . Redis.returnDecode + +-- | 'Gundeck' doesn't have an instance for 'MonadRedis' because it contains two +-- connections to two redis instances. When using 'WithAdditionalRedis', any +-- redis operation will target both redis instances (configured under 'redis:' +-- and 'redisAddtionalWrite:' in the gundeck config). To write to only the +-- default redis use 'WithDefaultRedis'. +newtype WithAdditionalRedis a = WithAdditionalRedis {runWithAdditionalRedis :: Gundeck a} + deriving newtype + ( Functor, + Applicative, + Monad, + MonadIO, + MonadThrow, + MonadCatch, + MonadMask, + MonadReader Env, + MonadClient, + MonadUnliftIO, + Log.MonadLogger + ) + +instance Redis.MonadRedis WithAdditionalRedis where + liftRedis action = do + defaultConn <- view rstate + ret <- Redis.runRobust defaultConn action + + mAdditionalRedisConn <- view rstateAdditionalWrite + for_ mAdditionalRedisConn $ \additionalRedisConn -> + -- We just fire and forget this call, as there is not much we can do if + -- this fails. + async $ Redis.runRobust additionalRedisConn action + + pure ret + +instance Redis.RedisCtx WithAdditionalRedis (Either Redis.Reply) where + returnDecode :: (Redis.RedisResult a) => Redis.Reply -> WithAdditionalRedis (Either Redis.Reply a) + returnDecode = Redis.liftRedis . Redis.returnDecode + instance Log.MonadLogger Gundeck where log l m = do e <- ask diff --git a/services/gundeck/src/Gundeck/Options.hs b/services/gundeck/src/Gundeck/Options.hs index 5222248da2..d70bbc4f91 100644 --- a/services/gundeck/src/Gundeck/Options.hs +++ b/services/gundeck/src/Gundeck/Options.hs @@ -24,7 +24,6 @@ import Control.Lens hiding (Level) import Data.Aeson.TH import Data.Yaml (FromJSON) import Gundeck.Aws.Arn -import Hasql.Pool.Extended (PoolConfig) import Imports import Network.AMQP.Extended import System.Logger.Extended (Level, LogFormat) @@ -103,6 +102,30 @@ deriveFromJSON toOptionFieldName ''MaxConcurrentNativePushes makeLenses ''MaxConcurrentNativePushes +data RedisConnectionMode + = Master + | Cluster + deriving (Show, Generic) + +deriveJSON defaultOptions {constructorTagModifier = map toLower} ''RedisConnectionMode + +data RedisEndpoint = RedisEndpoint + { _host :: !Text, + _port :: !Word16, + _connectionMode :: !RedisConnectionMode, + _enableTls :: !Bool, + -- | When not specified, use system CA bundle + _tlsCa :: !(Maybe FilePath), + -- | When 'True', uses TLS but does not verify hostname or CA or validity of + -- the cert. Not recommended to set to 'True'. + _insecureSkipVerifyTls :: !Bool + } + deriving (Show, Generic) + +deriveFromJSON toOptionFieldName ''RedisEndpoint + +makeLenses ''RedisEndpoint + makeLenses ''Settings deriveFromJSON toOptionFieldName ''Settings @@ -112,11 +135,8 @@ data Opts = Opts _gundeck :: !Endpoint, _brig :: !Endpoint, _cassandra :: !CassandraOpts, - -- | Postgresql settings, the key values must be in libpq format. - -- https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS - _postgresql :: !(Map Text Text), - _postgresqlPassword :: !(Maybe FilePathSecrets), - _postgresqlPool :: !PoolConfig, + _redis :: !RedisEndpoint, + _redisAdditionalWrite :: !(Maybe RedisEndpoint), _aws :: !AWSOpts, _rabbitmq :: !AmqpEndpoint, _discoUrl :: !(Maybe Text), diff --git a/services/gundeck/src/Gundeck/Presence.hs b/services/gundeck/src/Gundeck/Presence.hs index 6c6b757ef5..aa8fb77809 100644 --- a/services/gundeck/src/Gundeck/Presence.hs +++ b/services/gundeck/src/Gundeck/Presence.hs @@ -33,10 +33,10 @@ import Wire.API.CannonId import Wire.API.Presence listH :: UserId -> Gundeck [Presence] -listH = Data.list +listH = runWithDefaultRedis . Data.list listAllH :: CommaSeparatedList UserId -> Gundeck [Presence] -listAllH uids = concat <$> Data.listAll (fromCommaSeparatedList uids) +listAllH uids = concat <$> runWithDefaultRedis (Data.listAll (fromCommaSeparatedList uids)) addH :: Presence -> Gundeck (Headers '[Header "Location" URI] NoContent) addH p = do diff --git a/services/gundeck/src/Gundeck/Presence/Data.hs b/services/gundeck/src/Gundeck/Presence/Data.hs index 622dba9329..6173ace303 100644 --- a/services/gundeck/src/Gundeck/Presence/Data.hs +++ b/services/gundeck/src/Gundeck/Presence/Data.hs @@ -20,156 +20,128 @@ module Gundeck.Presence.Data list, listAll, deleteAll, - cleanup, ) where -import Control.Lens (view) -import Control.Monad.Catch (throwM) -import Data.ByteString.Conversion (fromByteString, toByteString') +import Control.Monad.Catch +import Data.Aeson as Aeson +import Data.ByteString qualified as Strict +import Data.ByteString.Builder (byteString) +import Data.ByteString.Char8 qualified as StrictChars +import Data.ByteString.Conversion hiding (fromList) +import Data.ByteString.Lazy qualified as Lazy import Data.Id -import Data.Map.Strict qualified as Map -import Data.Misc (Milliseconds (..)) -import Data.Text (pack, unpack) -import Data.Text.Encoding (decodeUtf8, encodeUtf8) -import Data.Time (UTCTime) -import Data.Time.Clock.POSIX (posixSecondsToUTCTime, utcTimeToPOSIXSeconds) -import Data.UUID (UUID) -import Data.Vector qualified as Vector -import Gundeck.Env (hasqlPool) -import Gundeck.Monad -import Hasql.Session (Session, statement) -import Hasql.Statement (Statement) -import Hasql.TH +import Data.List.NonEmpty qualified as NonEmpty +import Data.Misc (Milliseconds) +import Database.Redis +import Gundeck.Monad (Gundeck, posixTime, runWithAdditionalRedis) +import Gundeck.Util.Redis import Imports -import System.Logger.Class qualified as Log +import System.Logger.Class (MonadLogger) import Wire.API.Presence -import Wire.Postgres qualified as Postgres --- | Register (or refresh) a presence. The server-side timestamp is stamped --- here, the 'Presence'\'s own 'createdAt' value is ignored (as in the redis --- implementation before). +-- Note [Migration] --------------------------------------------------------- +-- +-- Previous redis schema: user:=@= +-- New redis schema: user:= = +-- +-- The previous redis schema encodes cannon's ID in the subkey. The migration +-- proceeds as follows: +-- +-- 1. When adding new entries, we only use the connection as subkey. +-- 2. When listing entries (which does not use the subkey fortunately) we +-- store the original field name in the `Presence` record property `__field`. +-- 3. When deleting entries, we use this `Presence`'s `__field` value. +-- 4. Eventually `__field` can be removed from the `Presence` type and the +-- connection can be used directly instead. +-- + add :: Presence -> Gundeck () add p = do - nowMs <- posixTime - runPool $ - statement - (toUUID (userId p), connIdText (connId p), uriText (resource p), clientToText <$> clientId p, msToUtc (fromIntegral (ms nowMs))) - upsertPresence - --- | Read all presences of a single user. -list :: UserId -> Gundeck [Presence] -list u = fromMaybe [] . listToMaybe <$> listAll [u] - --- | Read all presences of the given users, one list per user (input order, --- empty list for users without presences). Single round trip. -listAll :: [UserId] -> Gundeck [[Presence]] -listAll [] = pure [] -listAll uu = do - rows <- runPool $ statement (Vector.fromList (toUUID <$> uu)) selectByUsers - presencesByUser <- - foldM - ( \acc (u, c, r, cl, t) -> case readPresenceRow u c r cl t of - Just p -> pure $! Map.insertWith (<>) (userId p) [p] acc - Nothing -> do - Log.warn $ - Log.msg (Log.val "ignoring unreadable presence row") - . Log.field "user_id" (show u) - . Log.field "conn_id" (show c) - pure acc - ) - Map.empty - (Vector.toList rows) - pure [Map.findWithDefault [] u presencesByUser | u <- uu] - --- | Compare-and-delete: only delete the stored presence if it is not newer --- than the given one (a newer re-registration with the same conn id must not --- be deleted by a stale disconnect). -deleteAll :: [Presence] -> Gundeck () -deleteAll [] = pure () -deleteAll pp = - runPool . statement params $ deleteMany + now <- posixTime + let k = toKey (userId p) + let v = toField (connId p) + let d = Lazy.toStrict $ Aeson.encode $ PresenceData p.resource p.clientId now + runWithAdditionalRedis . retry x3 $ do + void . fromTxResult <=< (liftRedis . multiExec) $ do + void $ hset k (NonEmpty.singleton (v, d)) + -- nb. All presences of a user are expired 'maxIdleTime' after the + -- last presence was registered. A client who keeps a presence + -- (i.e. websocket) connected for longer than 'maxIdleTime' will be + -- silently dropped and receives no more notifications. + expire k maxIdleTime where - params = - ( Vector.fromList (toUUID . userId <$> pp), - Vector.fromList (connIdText . connId <$> pp), - Vector.fromList (msToUtc . fromIntegral . ms . createdAt <$> pp) - ) - --- | Delete presences older than a week. Normal disconnects delete their --- presence rows; this only guards against leaks from abnormally dead pods --- (replaces the redis key TTL). -cleanup :: Gundeck () -cleanup = runPool $ statement () deleteStale - --- Helpers ------------------------------------------------------------------- + maxIdleTime = 7 * 24 * 60 * 60 -- 7 days in seconds --- | Millis <-> UTC. Exact (milliseconds nest inside timestamptz's microseconds); --- do NOT reuse 'Gundeck.Monad.msToUTCSecs', it truncates to whole seconds. -msToUtc :: Int64 -> UTCTime -msToUtc p = posixSecondsToUTCTime (fromRational (fromIntegral p / 1000 :: Rational)) - -utcToMs :: UTCTime -> Int64 -utcToMs = floor . (* 1000) . utcTimeToPOSIXSeconds - -newtype PresenceDbError = PresenceDbError Text deriving (Show) - -instance Exception PresenceDbError - -runPool :: Session a -> Gundeck a -runPool sess = do - pool <- view hasqlPool - liftIO (Postgres.useWithResetAndRetry pool sess) >>= either (throwM . PresenceDbError . pack . show) pure - -connIdText :: ConnId -> Text -connIdText = decodeUtf8 . fromConnId +deleteAll :: (MonadMask m, MonadIO m, RedisCtx m (Either Reply), MonadLogger m) => [Presence] -> m () +deleteAll [] = pure () +deleteAll pp = for_ pp $ \p -> do + let k = toKey (userId p) + let f = Lazy.toStrict $ __field p + void . retry x3 $ do + void . liftRedis $ watch (pure k) + value <- either (throwM . RedisSimpleError) id <$> hget k f + void . liftRedis . multiExec $ do + case value of + Nothing -> pure $ pure () + Just v -> do + let p' = readPresence (userId p) (f, v) + if Just p == p' + then void <$> hdel k (pure f) + else pure $ pure () + +list :: (MonadRedis m, MonadThrow m) => UserId -> m [Presence] +list u = do + ePresenses <- liftRedis $ list' u + case ePresenses of + Left r -> throwM $ RedisSimpleError r + Right ps -> pure ps + +list' :: (RedisCtx m f, Functor f) => UserId -> m (f [Presence]) +list' u = mapMaybe (readPresence u) <$$> hgetall (toKey u) + +-- FUTUREWORK: Make this not fail if it fails only for a few users. +listAll :: (MonadRedis m, MonadThrow m) => [UserId] -> m [[Presence]] +listAll [] = pure [] +listAll uu = mapM list uu -uriText :: URI -> Text -uriText = decodeUtf8 . toByteString' +-- Helpers ------------------------------------------------------------------- -readPresenceRow :: UUID -> Text -> Text -> Maybe Text -> UTCTime -> Maybe Presence -readPresenceRow u c r cl t = do - uri <- parse (unpack r) - cid <- traverse parseClient cl - pure (Presence (Id u) (ConnId (encodeUtf8 c)) uri cid (Ms (fromIntegral (utcToMs t)))) - where - parseClient = fromByteString . encodeUtf8 - -upsertPresence :: Statement (UUID, Text, Text, Maybe Text, UTCTime) () -upsertPresence = - [resultlessStatement| - INSERT INTO presence (user_id, conn_id, resource, client_id, created_at) - VALUES ($1 :: uuid, $2 :: text, $3 :: text, $4 :: text?, $5 :: timestamptz) - ON CONFLICT (user_id, conn_id) DO UPDATE - SET resource = EXCLUDED.resource, - client_id = EXCLUDED.client_id, - created_at = EXCLUDED.created_at - |] - -selectByUsers :: Statement (Vector.Vector UUID) (Vector.Vector (UUID, Text, Text, Maybe Text, UTCTime)) -selectByUsers = - [vectorStatement| - SELECT user_id :: uuid, conn_id :: text, resource :: text, client_id :: text?, created_at :: timestamptz - FROM presence - WHERE user_id = ANY ($1 :: uuid[]) - |] - --- | Compare-and-delete, in one round trip: only delete each stored presence --- if it is not newer than the given one (a newer re-registration with the --- same conn id must not be deleted by a stale disconnect). -deleteMany :: Statement (Vector.Vector UUID, Vector.Vector Text, Vector.Vector UTCTime) () -deleteMany = - [resultlessStatement| - DELETE FROM presence p - USING unnest($1 :: uuid[], $2 :: text[], $3 :: timestamptz[]) AS d (user_id, conn_id, created_at) - WHERE p.user_id = d.user_id - AND p.conn_id = d.conn_id - AND p.created_at <= d.created_at - |] - -deleteStale :: Statement () () -deleteStale = - [resultlessStatement| - DELETE FROM presence - WHERE created_at < now() - interval '7 days' - |] +data PresenceData = PresenceData !URI !(Maybe ClientId) !Milliseconds + deriving (Eq) + +instance ToJSON PresenceData where + toJSON (PresenceData r c t) = + object + [ "r" .= r, + "c" .= c, + "t" .= t + ] + +instance FromJSON PresenceData where + parseJSON = withObject "PresenceData" $ \o -> + PresenceData + <$> o + .: "r" + <*> o + .:? "c" + <*> o + .:? "t" + .!= 0 + +toKey :: UserId -> ByteString +toKey u = Lazy.toStrict $ runBuilder (byteString "user:" <> builder u) + +toField :: ConnId -> ByteString +toField (ConnId con) = con + +fromField :: ByteString -> ConnId +fromField = ConnId . StrictChars.takeWhile (/= '@') + +readPresence :: UserId -> (ByteString, ByteString) -> Maybe Presence +readPresence u (f, b) = do + PresenceData uri clt tme <- + if "http" `Strict.isPrefixOf` b + then PresenceData <$> fromByteString b <*> pure Nothing <*> pure 0 + else decodeStrict' b + pure (Presence u (fromField f) uri clt tme (Lazy.fromStrict f)) diff --git a/services/gundeck/src/Gundeck/Push.hs b/services/gundeck/src/Gundeck/Push.hs index 77149b6efe..a6cdf75906 100644 --- a/services/gundeck/src/Gundeck/Push.hs +++ b/services/gundeck/src/Gundeck/Push.hs @@ -122,7 +122,7 @@ instance MonadPushAll Gundeck where mpaNotificationTTL = view (options . settings . notificationTTL) mpaCellsEventQueue = view (options . settings . cellsEventQueue) mpaMkNotificationId = mkNotificationId - mpaListAllPresences = Presence.listAll + mpaListAllPresences = runWithDefaultRedis . Presence.listAll mpaBulkPush = Web.bulkPush mpaStreamAdd = Data.add mpaPushNative = pushNative diff --git a/services/gundeck/src/Gundeck/Push/Websocket.hs b/services/gundeck/src/Gundeck/Push/Websocket.hs index 721dadd8ea..562bcb1073 100644 --- a/services/gundeck/src/Gundeck/Push/Websocket.hs +++ b/services/gundeck/src/Gundeck/Push/Websocket.hs @@ -64,7 +64,7 @@ class (Monad m, MonadThrow m, Log.MonadLogger m) => MonadBulkPush m where instance MonadBulkPush Gundeck where mbpBulkSend = bulkSend - mbpDeleteAllPresences = Presence.deleteAll + mbpDeleteAllPresences = runWithAdditionalRedis . Presence.deleteAll mbpPosixTime = posixTime mbpMapConcurrently = mapConcurrently mbpMonitorBadCannons = monitorBadCannons @@ -315,7 +315,7 @@ push :: push notif (toList -> tgts) originUser originConn conns = do pp <- handleAny noPresences listPresences (ok, gone) <- foldM onResult ([], []) =<< send notif pp - Presence.deleteAll gone + runWithAdditionalRedis $ Presence.deleteAll gone pure ok where listPresences = @@ -324,7 +324,7 @@ push notif (toList -> tgts) originUser originConn conns = do . concat . filterByClient . zip tgts - <$> Presence.listAll (view targetUser <$> tgts) + <$> runWithDefaultRedis (Presence.listAll (view targetUser <$> tgts)) noPresences exn = do Log.err $ Log.field "error" (displayException exn) diff --git a/services/gundeck/src/Gundeck/Redis.hs b/services/gundeck/src/Gundeck/Redis.hs new file mode 100644 index 0000000000..e9bf1affaf --- /dev/null +++ b/services/gundeck/src/Gundeck/Redis.hs @@ -0,0 +1,127 @@ +{-# LANGUAGE NumDecimals #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Gundeck.Redis + ( RobustConnection, + connectRobust, + runRobust, + PingException, + ) +where + +import Control.Concurrent.Async (Async, async) +import Control.Monad.Catch qualified as Catch +import Control.Retry +import Database.Redis +import Database.Redis.Connection (ClusterDownError) +import Imports +import System.Logger qualified as Log +import System.Logger.Class (MonadLogger) +import System.Logger.Class qualified as LogClass +import System.Logger.Extended +import UnliftIO.Exception + +-- | Connection to Redis which allows reconnecting. +type RobustConnection = MVar Connection + +-- | Connection to Redis which can be reestablished on connection errors. +-- +-- Reconnecting even when Redis IPs change as long as the DNS name remains +-- constant. The server type (cluster or not) and the connection information of +-- the initial connection are used when reconnecting. +-- +-- Throws 'ConnectError', 'ConnectTimeout', 'ConnectionLostException', +-- 'PingException', or 'IOException' if retry policy is finite. +connectRobust :: + Logger -> + -- | e. g., @exponentialBackoff 50000@ + RetryPolicy -> + -- | action returning a fresh initial 'Connection', e. g., @(checkedConnect connInfo)@ or @(checkedConnectCluster connInfo)@ + IO Connection -> + IO (Async (), RobustConnection) +connectRobust l retryStrategy connectLowLevel = do + robustConnection <- newEmptyMVar @IO @Connection + thread <- + async $ safeForever l $ do + Log.info l $ Log.msg (Log.val "connecting to Redis") + conn <- retry connectLowLevel + Log.info l $ Log.msg (Log.val "successfully connected to Redis") + putMVar robustConnection conn + catch + ( forever $ do + _ <- runRedis conn ping + threadDelay 1e6 + ) + $ \(_ :: SomeException) -> void $ takeMVar robustConnection + pure (thread, robustConnection) + where + retry = + recovering -- retry connecting, e. g., with exponential back-off + retryStrategy + [ const $ Catch.Handler (\(e :: ClusterDownError) -> logEx (Log.err l) e "Redis cluster down" >> pure True), + const $ Catch.Handler (\(e :: ConnectError) -> logEx (Log.err l) e "Redis not in cluster mode" >> pure True), + const $ Catch.Handler (\(e :: ConnectTimeout) -> logEx (Log.err l) e "timeout when connecting to Redis" >> pure True), + const $ Catch.Handler (\(e :: ConnectionLostException) -> logEx (Log.err l) e "Redis connection lost during request" >> pure True), + const $ Catch.Handler (\(e :: PingException) -> logEx (Log.err l) e "pinging Redis failed" >> pure True), + const $ Catch.Handler (\(e :: IOException) -> logEx (Log.err l) e "network error when connecting to Redis" >> pure True) + ] + . const -- ignore RetryStatus + logEx :: (Exception e) => ((Msg -> Msg) -> IO ()) -> e -> ByteString -> IO () + logEx lLevel e description = lLevel $ Log.msg (Log.val description) . Log.field "error" (displayException e) + +-- | Run a 'Redis' action through a 'RobustConnection'. +-- +-- Blocks on connection errors as long as the connection is not reestablished. +-- Without externally enforcing timeouts, this may lead to leaking threads. +runRobust :: (MonadUnliftIO m, MonadLogger m, Catch.MonadMask m) => RobustConnection -> Redis a -> m a +runRobust mvar action = retry $ do + robustConnection <- readMVar mvar + liftIO $ runRedis robustConnection action + where + retryStrategy = capDelay 1000000 (exponentialBackoff 50000) + retry = + recovering -- retry connecting, e. g., with exponential back-off + retryStrategy + [ logAndHandle $ Catch.Handler (\(_ :: ConnectionLostException) -> pure True), + logAndHandle $ Catch.Handler (\(_ :: IOException) -> pure True) + ] + . const -- ignore RetryStatus + logAndHandle (Handler handler) _ = + Handler $ \e -> do + LogClass.err $ Log.msg (Log.val "Redis connection failed") . Log.field "error" (displayException e) + handler e + +data PingException = PingException Reply deriving (Show) + +instance Exception PingException + +safeForever :: + forall m. + (MonadUnliftIO m) => + Logger -> + m () -> + m () +safeForever l action = + forever $ + action `catchAny` \e -> do + Log.err l $ Log.msg (Log.val "Uncaught exception while connecting to redis") . Log.field "error" (displayException e) + threadDelay 1e6 -- pause to keep worst-case noise in logs manageable diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs index 6f4b388d64..89e4c9f8ef 100644 --- a/services/gundeck/src/Gundeck/Run.hs +++ b/services/gundeck/src/Gundeck/Run.hs @@ -42,25 +42,23 @@ import Cassandra.Schema (versionCheck) import Control.Error (ExceptT (ExceptT)) import Control.Exception (finally) import Control.Lens ((.~), (^.)) -import Control.Monad.Catch (catchAll) import Control.Monad.Extra import Data.Map qualified as Map import Data.Metrics.AWS (gaugeTokenRemaing) import Data.Metrics.Servant qualified as Metrics import Data.Proxy (Proxy (Proxy)) import Data.Text (unpack) +import Database.Redis qualified as Redis import Gundeck.API.Internal as Internal (InternalAPI, servantSitemap) import Gundeck.API.Public as Public (servantSitemap) import Gundeck.Aws qualified as Aws import Gundeck.Env import Gundeck.Env qualified as Env import Gundeck.Monad -import Gundeck.Options -import Gundeck.Presence.Data qualified as PresenceData +import Gundeck.Options hiding (host, port) import Gundeck.React import Gundeck.Schema.Run (lastSchemaVersion) import Gundeck.ThreadBudget -import Hasql.Pool.Extended (Pool (rawPool)) import Imports import Network.AMQP import Network.AMQP.Types @@ -83,13 +81,11 @@ import Wire.API.Routes.Public.Gundeck (GundeckAPI) import Wire.API.Routes.Version import Wire.API.Routes.Version.Wai import Wire.OpenTelemetry -import Wire.PostgresMigrations qualified as PostgresMigrations run :: Opts -> IO () run opts = withTracer \tracer -> do - env <- createEnv opts + (rThreads, env) <- createEnv opts let logger = env ^. applog - PostgresMigrations.runAllMigrations (env ^. hasqlPool).rawPool logger runDirect env setUpRabbitMqExchangesAndQueues @@ -97,10 +93,10 @@ run opts = withTracer \tracer -> do versionCheck lastSchemaVersion let s = newSettings $ defaultServer (unpack . host $ opts ^. gundeck) (port $ opts ^. gundeck) logger let throttleMillis = fromMaybe defSqsThrottleMillis $ opts ^. (settings . sqsThrottleMillis) + lst <- Async.async $ Aws.execute (env ^. awsEnv) (Aws.listen throttleMillis (runDirect env . onEvent)) wtbs <- forM (env ^. threadBudgetState) $ \tbs -> Async.async $ runDirect env $ watchThreadBudgetState tbs 10 wCollectAuth <- Async.async (collectAuthMetrics (Aws._awsEnv (Env._awsEnv env))) - pcleanup <- Async.async $ runDirect env $ cleanupPresenceLoop logger app <- middleware env <*> pure (mkApp env) inSpan tracer "gundeck" defaultSpanArguments {kind = Otel.Server} (runSettingsWithShutdown s app Nothing) `finally` do @@ -108,8 +104,10 @@ run opts = withTracer \tracer -> do shutdown (env ^. cstate) Async.cancel lst Async.cancel wCollectAuth - Async.cancel pcleanup forM_ wtbs Async.cancel + forM_ rThreads Async.cancel + Redis.disconnect =<< takeMVar (env ^. rstate) + whenJust (env ^. rstateAdditionalWrite) $ (=<<) Redis.disconnect . takeMVar Log.close (env ^. applog) where setUpRabbitMqExchangesAndQueues :: Gundeck () @@ -180,19 +178,3 @@ collectAuthMetrics env = do mbRemaining <- readAuthExpiration env gaugeTokenRemaing mbRemaining threadDelay 1_000_000 - --- | Hourly janitor replacing the redis key TTL: deletes presence rows older --- than a week (leak guard for abnormally dead pods). Never let a transient DB --- error kill the thread — log and retry next hour. -cleanupPresenceLoop :: Log.Logger -> Gundeck () -cleanupPresenceLoop logger = - forever $ - (PresenceData.cleanup >> threadDelay cleanupInterval) - `catchAll` \e -> do - liftIO . Log.err logger $ - Log.msg (Log.val "presence cleanup failed") - . Log.field "error" (displayException (e :: SomeException)) - threadDelay cleanupInterval - -cleanupInterval :: Int -cleanupInterval = 3_600_000_000 -- one hour, in microseconds diff --git a/services/gundeck/src/Gundeck/Util/Redis.hs b/services/gundeck/src/Gundeck/Util/Redis.hs new file mode 100644 index 0000000000..d125d04bac --- /dev/null +++ b/services/gundeck/src/Gundeck/Util/Redis.hs @@ -0,0 +1,61 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Gundeck.Util.Redis where + +import Control.Monad.Catch +import Control.Retry +import Data.ByteString qualified as BS +import Database.Redis +import Imports +import System.Logger.Class (MonadLogger) +import System.Logger.Class qualified as Log +import System.Logger.Message + +retry :: (MonadIO m, MonadMask m, MonadLogger m) => RetryPolicyM m -> m a -> m a +retry x = recovering x handlers . const + +x3 :: RetryPolicy +x3 = limitRetries 3 <> exponentialBackoff 100000 + +handlers :: (MonadLogger m) => [a -> Handler m Bool] +handlers = + [ const . Handler $ \case + RedisSimpleError (Error err) -> pure $ "READONLY" `BS.isPrefixOf` err + RedisTxError err -> pure $ "READONLY" `isPrefixOf` err + err -> do + Log.warn $ + Log.msg (Log.val "Redis error; not retrying.") + ~~ "redis.errMsg" .= show err + pure False + ] + +-- Error ------------------------------------------------------------------- + +data RedisError + = RedisSimpleError Reply + | RedisTxAborted + | RedisTxError String + deriving (Show) + +instance Exception RedisError + +fromTxResult :: (MonadThrow m) => TxResult a -> m a +fromTxResult = \case + TxSuccess a -> pure a + TxAborted -> throwM RedisTxAborted + TxError e -> throwM $ RedisTxError e diff --git a/services/gundeck/test/integration/API.hs b/services/gundeck/test/integration/API.hs index 6346d07a43..27de5b8602 100644 --- a/services/gundeck/test/integration/API.hs +++ b/services/gundeck/test/integration/API.hs @@ -28,6 +28,7 @@ import Bilge hiding (head) import Bilge.Assert import Control.Arrow ((&&&)) import Control.Concurrent.Async (Async, async, concurrently_, wait) +import Control.Concurrent.Async qualified as Async import Control.Lens (view, (%~), (.~), (?~), (^.), (^?), _2) import Control.Retry (constantDelay, limitRetries, recoverAll, retrying) import Data.Aeson @@ -46,6 +47,8 @@ import Data.Set qualified as Set import Data.Text.Encoding qualified as T import Data.UUID qualified as UUID import Data.UUID.V4 +import Gundeck.Options +import Gundeck.Options qualified as O import Imports import Network.HTTP.Client qualified as Http import Network.URI (parseURI) @@ -56,6 +59,7 @@ import System.Timeout (timeout) import Test.Tasty import Test.Tasty.HUnit import TestSetup +import Util (runRedisProxy, withEnvOverrides, withSettingsOverrides) import Wire.API.Event.Gundeck import Wire.API.Internal.Notification import Wire.API.Presence @@ -72,7 +76,8 @@ tests s = test s "Remove stale presence" removeStalePresence, test s "Single user push" singleUserPush, test s "Single user push with large message" singleUserPushLargeMessage, - test s "Send a push, ensure origin does not receive it" sendSingleUserNoPiggyback + test s "Send a push, ensure origin does not receive it" sendSingleUserNoPiggyback, + test s "Store notifications even when redis is down" storeNotificationsEvenWhenRedisIsDown ], testGroup "Notifications" @@ -103,6 +108,10 @@ tests s = test s "control pings with payload produce pongs with the same payload" testControlPingPongWithData, test s "data non-pings are ignored" testNoPingNoPong ], + testGroup + "Redis migration" + [ test s "redis migration should work" testRedisMigration + ], -- TODO: The following tests require (at the moment), the usage real AWS -- services so they are kept in a separate group to simplify testing testGroup @@ -126,8 +135,8 @@ replacePresence = do con <- randomConnId let localhost8080 = URI . fromJust $ parseURI "http://localhost:8080" let localhost8081 = URI . fromJust $ parseURI "http://localhost:8081" - let pres1 = Presence uid (ConnId "dummy_dev") localhost8080 Nothing 0 - let pres2 = Presence uid (ConnId "dummy_dev") localhost8081 Nothing 0 + let pres1 = Presence uid (ConnId "dummy_dev") localhost8080 Nothing 0 "" + let pres2 = Presence uid (ConnId "dummy_dev") localhost8081 Nothing 0 "" void $ connectUser ca uid con setPresence gu pres1 !!! const 201 === statusCode sendPush (push uid [uid]) @@ -260,6 +269,28 @@ sendMultipleUsers = do pevent = KeyMap.fromList ["foo" .= (42 :: Int)] push u us = newPush (Just u) (toRecipients us) pload & pushOriginConnection ?~ ConnId "dev" +storeNotificationsEvenWhenRedisIsDown :: TestM () +storeNotificationsEvenWhenRedisIsDown = do + ally <- randomId + origRedisEndpoint <- view $ tsOpts . redis + let proxyPort = 10112 + redisProxyServer <- liftIO . async $ runRedisProxy (origRedisEndpoint ^. O.host) (origRedisEndpoint ^. O.port) proxyPort + withSettingsOverrides + ( \gundeckSettings -> + gundeckSettings + & redis . Gundeck.Options.host .~ "localhost" + & redis . Gundeck.Options.port .~ proxyPort + ) + $ do + let pload = textPayload "hello" + push = buildPush ally [(ally, RecipientClientsAll)] pload + gu <- view tsGundeck + liftIO $ Async.cancel redisProxyServer + post (runGundeckR gu . path "i/push/v2" . json [push]) !!! const 200 === statusCode + + ns <- listNotifications ally Nothing + liftIO $ assertEqual ("Expected 1 notification, got: " <> show ns) 1 (length ns) + ----------------------------------------------------------------------------- -- Notifications @@ -698,6 +729,36 @@ testLongPushToken = do tkn4 <- randomToken clt gcmToken {tSize = 5000} registerPushTokenRequest uid tkn4 !!! const 413 === statusCode +-- * Redis Migration + +testRedisMigration :: TestM () +testRedisMigration = do + uid <- randomUser + con <- randomConnId + cannonURI <- Wire.API.Presence.parse "http://cannon.example" + let presence = Presence uid con cannonURI Nothing 1 "" + redis2 <- view tsRedis2 + + withSettingsOverrides (redisAdditionalWrite ?~ redis2) $ do + g <- view tsGundeck + setPresence g presence + !!! const 201 + === statusCode + retrievedPresence <- + map resource . decodePresence <$> (getPresence g (toByteString' uid) lookupEnv "REDIS_ADDITIONAL_WRITE_USERNAME" + password <- ("REDIS_PASSWORD",) <$$> lookupEnv "REDIS_ADDITIONAL_WRITE_PASSWORD" + pure $ catMaybes [username, password] + + withEnvOverrides redis2CredsAsRedis1Creds $ withSettingsOverrides (redis .~ redis2) $ do + g <- view tsGundeck + retrievedPresence <- + map resource . decodePresence <$> (getPresence g (toByteString' uid) UserId -> Int -> TestM () diff --git a/services/gundeck/test/integration/Main.hs b/services/gundeck/test/integration/Main.hs index 05a385e40b..767f28a4ae 100644 --- a/services/gundeck/test/integration/Main.hs +++ b/services/gundeck/test/integration/Main.hs @@ -30,7 +30,7 @@ import Data.Proxy import Data.Tagged import Data.Text.Encoding (encodeUtf8) import Data.Yaml (decodeFileEither) -import Gundeck.Options +import Gundeck.Options hiding (host, port) import Imports hiding (local) import Metrics qualified import Network.HTTP.Client (responseTimeoutMicro) @@ -52,7 +52,8 @@ data IntegrationConfig = IntegrationConfig { gundeck :: Endpoint, cannon :: Endpoint, cannon2 :: Endpoint, - brig :: Endpoint + brig :: Endpoint, + redis2 :: RedisEndpoint } deriving (Show, Generic) @@ -113,6 +114,6 @@ main = withOpenSSL $ runTests go b = BrigR $ mkRequest iConf.brig lg <- Logger.new Logger.defSettings db <- defInitCassandra (gConf ^. cassandra) lg - pure $ TestSetup m g c c2 b db lg - mkRequest (Endpoint h p) = Bilge.host (encodeUtf8 h) . Bilge.port p + pure $ TestSetup m g c c2 b db lg gConf (redis2 iConf) releaseOpts _ = pure () + mkRequest (Endpoint h p) = Bilge.host (encodeUtf8 h) . Bilge.port p diff --git a/services/gundeck/test/integration/TestSetup.hs b/services/gundeck/test/integration/TestSetup.hs index 70e8cd77dc..ea49d1b322 100644 --- a/services/gundeck/test/integration/TestSetup.hs +++ b/services/gundeck/test/integration/TestSetup.hs @@ -28,6 +28,8 @@ module TestSetup tsBrig, tsCass, tsLogger, + tsOpts, + tsRedis2, TestM (..), TestSetup (..), BrigR (..), @@ -40,6 +42,8 @@ import Bilge (HttpT (..), Manager, MonadHttp, Request, runHttpT) import Cassandra qualified as Cql import Control.Lens (makeLenses, (^.)) import Control.Monad.Catch (MonadCatch, MonadMask, MonadThrow) +import Gundeck.Options (RedisEndpoint) +import Gundeck.Options qualified as Gundeck import Imports import System.Logger qualified as Log import Test.Tasty (TestName, TestTree) @@ -75,7 +79,9 @@ data TestSetup = TestSetup _tsCannon2 :: CannonR, _tsBrig :: BrigR, _tsCass :: Cql.ClientState, - _tsLogger :: Log.Logger + _tsLogger :: Log.Logger, + _tsOpts :: Gundeck.Opts, + _tsRedis2 :: RedisEndpoint } makeLenses ''TestSetup diff --git a/services/gundeck/test/integration/Util.hs b/services/gundeck/test/integration/Util.hs new file mode 100644 index 0000000000..d6790424b2 --- /dev/null +++ b/services/gundeck/test/integration/Util.hs @@ -0,0 +1,119 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2025 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Util where + +import Bilge qualified +import Control.Concurrent (forkFinally) +import Control.Concurrent.Async (race_) +import Control.Exception qualified as E +import Control.Lens +import Control.Monad.Catch +import Control.Monad.Codensity +import Data.ByteString qualified as S +import Data.Text qualified as Text +import Gundeck.Env (createEnv) +import Gundeck.Options +import Gundeck.Run (mkApp) +import Imports +import Network.Socket hiding (openSocket) +import Network.Socket.ByteString (recv, sendAll) +import Network.Wai.Utilities.MockServer (withMockServer) +import TestSetup + +withSettingsOverrides :: (Opts -> Opts) -> TestM a -> TestM a +withSettingsOverrides f action = do + ts <- ask + let opts = f (view tsOpts ts) + (_rThreads, env) <- liftIO $ createEnv opts + liftIO . lowerCodensity $ do + let app = mkApp env + p <- withMockServer app + liftIO $ Bilge.runHttpT (ts ^. tsManager) $ runReaderT (runTestM action) $ ts & tsGundeck .~ GundeckR (mkRequest p) + where + mkRequest p = Bilge.host "127.0.0.1" . Bilge.port p + +withEnvOverrides :: forall m a. (MonadIO m, MonadMask m) => [(String, String)] -> m a -> m a +withEnvOverrides envOverrides action = do + bracket (setEnvVars envOverrides) (resetEnvVars) $ const action + where + setEnvVars :: [(String, String)] -> m [(String, Maybe String)] + setEnvVars newVars = liftIO $ do + oldVars <- mapM (\(k, _) -> (k,) <$> lookupEnv k) newVars + mapM_ (uncurry setEnv) newVars + pure oldVars + + resetEnvVars :: [(String, Maybe String)] -> m () + resetEnvVars = + mapM_ (\(k, mV) -> maybe (unsetEnv k) (setEnv k) mV) + +runRedisProxy :: Text -> Word16 -> Word16 -> IO () +runRedisProxy redisHost redisPort proxyPort = do + (servAddr : _) <- getAddrInfo Nothing (Just $ Text.unpack redisHost) (Just $ show redisPort) + runTCPServer Nothing (show proxyPort) $ \client -> do + server <- getServerSocket servAddr + client <~~> server + where + getServerSocket servAddr = do + server <- socket (addrFamily servAddr) Stream defaultProtocol + connect server (addrAddress servAddr) >> pure server + p1 <~~> p2 = finally (race_ (p1 `mapData` p2) (p2 `mapData` p1)) (close p1 >> close p2) + mapData f t = do + content <- recv f 4096 + unless (S.null content) $ sendAll t content >> mapData f t + +-- Forked from network-run, added logic to cleanup clients when server is closed + +-- | Running a TCP server with an accepted socket and its peer name. +runTCPServer :: Maybe HostName -> ServiceName -> (Socket -> IO a) -> IO b +runTCPServer mhost port' server = withSocketsDo $ do + addr <- resolve Stream mhost port' True + clientThreads <- newTVarIO [] + E.bracket (open addr) (cleanupClients clientThreads) (loop clientThreads) + where + open addr = E.bracketOnError (openServerSocket addr) close $ \sock -> do + listen sock 1024 + pure sock + loop clientThreads sock = forever $ do + E.bracketOnError (accept sock) (close . fst) $ + \(conn, _peer) -> do + thread <- forkFinally (server conn) (const $ gracefulClose conn 5000) + atomically $ modifyTVar clientThreads (thread :) + cleanupClients :: TVar [ThreadId] -> Socket -> IO () + cleanupClients clientThreads sock = do + close sock + mapM_ killThread =<< readTVarIO clientThreads + +resolve :: SocketType -> Maybe HostName -> ServiceName -> Bool -> IO AddrInfo +resolve socketType mhost port' passive = + head <$> getAddrInfo (Just hints) mhost (Just port') + where + hints = + defaultHints + { addrSocketType = socketType, + addrFlags = [AI_PASSIVE | passive] + } + +openServerSocket :: AddrInfo -> IO Socket +openServerSocket addr = E.bracketOnError (openSocket addr) close $ \sock -> do + setSocketOption sock ReuseAddr 1 + withFdSocket sock $ setCloseOnExecIfNeeded + bind sock $ addrAddress addr + pure sock + +openSocket :: AddrInfo -> IO Socket +openSocket addr = socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr) diff --git a/services/gundeck/test/unit/MockGundeck.hs b/services/gundeck/test/unit/MockGundeck.hs index 647e30f376..6e4f27df53 100644 --- a/services/gundeck/test/unit/MockGundeck.hs +++ b/services/gundeck/test/unit/MockGundeck.hs @@ -770,6 +770,7 @@ fakePresence userId clientId_ = Presence {..} connId = fakeConnId clientId_ resource = URI . fromJust $ URI.parseURI "http://127.0.0.1:8080" createdAt = 0 + __field = mempty -- | See also: 'fakePresence'. fakeConnId :: ClientId -> ConnId diff --git a/services/integration.yaml b/services/integration.yaml index acb0e595b2..2da7e194e1 100644 --- a/services/integration.yaml +++ b/services/integration.yaml @@ -128,6 +128,12 @@ backendTwo: originDomain: b.example.com +redis2: + host: 127.0.0.1 + port: 6379 + connectionMode: master + enableTls: false + insecureSkipVerifyTls: false dynamicBackends: dynamic-backend-1: From aeb8588fc030e7ced8f697b12d6d5e69863355c6 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Fri, 18 Sep 2026 15:46:17 +0200 Subject: [PATCH 3/3] Make gundeck config compatible with Redis and Postgres --- charts/wire-server/templates/gundeck/configmap.yaml | 8 ++++++++ charts/wire-server/templates/gundeck/secret.yaml | 3 +++ 2 files changed, 11 insertions(+) diff --git a/charts/wire-server/templates/gundeck/configmap.yaml b/charts/wire-server/templates/gundeck/configmap.yaml index 10be21c34e..7e84404367 100644 --- a/charts/wire-server/templates/gundeck/configmap.yaml +++ b/charts/wire-server/templates/gundeck/configmap.yaml @@ -63,6 +63,14 @@ data: {{- end }} {{- end }} + {{- if .postgresql }} + postgresql: {{ toYaml .postgresql | nindent 6 }} + postgresqlPool: {{ toYaml .postgresqlPool | nindent 6 }} + {{- if hasKey $.Values.gundeck.secrets "pgPassword" }} + postgresqlPassword: /etc/wire/gundeck/secrets/pgPassword + {{- end }} + {{- end }} + # Gundeck uses discovery for AWS access key / secrets # For more details, check amazonka's documentation at: # https://hackage.haskell.org/package/amazonka-1.4.5/docs/Network-AWS.html#t:Credentials diff --git a/charts/wire-server/templates/gundeck/secret.yaml b/charts/wire-server/templates/gundeck/secret.yaml index b1f744ff60..000255ba0a 100644 --- a/charts/wire-server/templates/gundeck/secret.yaml +++ b/charts/wire-server/templates/gundeck/secret.yaml @@ -31,5 +31,8 @@ data: {{- if hasKey . "redisAdditionalWritePassword" }} redisAdditionalWritePassword: {{ .redisAdditionalWritePassword | b64enc | quote }} {{- end }} + {{- if hasKey . "pgPassword" }} + pgPassword: {{ .pgPassword | b64enc | quote }} + {{- end }} {{- end }} {{- end }}