diff --git a/Makefile b/Makefile index 0e2e771c983..2a3250275e6 100644 --- a/Makefile +++ b/Makefile @@ -13,11 +13,11 @@ CHARTS_INTEGRATION := wire-server databases-ephemeral rabbitmq fake-aws ingre # (e.g. move charts/brig to charts/wire-server/brig) # this list could be generated from the folder names under ./charts/ like so: # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts) -CHARTS_RELEASE := wire-server rabbitmq rabbitmq-external databases-ephemeral \ +CHARTS_RELEASE := wire-server redis-ephemeral rabbitmq rabbitmq-external databases-ephemeral \ fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice \ calling-test demo-smtp elasticsearch-curator elasticsearch-external \ elasticsearch-ephemeral minio-external cassandra-external \ -ingress-nginx-controller nginx-ingress-services \ +ingress-nginx-controller nginx-ingress-services reaper \ k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise \ wire-ingress KIND_CLUSTER_NAME := wire-server diff --git a/changelog.d/0-release-notes/WPB-28377-remove-redis b/changelog.d/0-release-notes/WPB-28377-remove-redis deleted file mode 100644 index b7f75b78e85..00000000000 --- a/changelog.d/0-release-notes/WPB-28377-remove-redis +++ /dev/null @@ -1,17 +0,0 @@ -Gundeck no longer uses redis: presence tracking is stored in PostgreSQL. - -Operators must: - -- Remove redis deployments that were only used by gundeck, and the gundeck - `redis:` and `redisAdditionalWrite:` configuration, the `REDIS_USERNAME`, - `REDIS_PASSWORD`, `REDIS_ADDITIONAL_WRITE_USERNAME` and - `REDIS_ADDITIONAL_WRITE_PASSWORD` environment variables, and gundeck redis - TLS secrets (`redisUsername`/`redisPassword`/`redisAdditionalWrite*` secrets - and the redis CA certificates). -- Add the new required configuration `gundeck.config.postgresql` (plus - `postgresqlPool`, and optionally `secrets.pgPassword` for the password file), - following the same format as brig's postgresql settings. -- Restart all cannons after deployment is successful. Presence data does - not carry over, this will make sure all clients reconnect after the - presence data is being written to PostgreSQL. -- Stop deploying `redis-ephemeral` and `reaper`, these have been removed. diff --git a/changelog.d/3-bug-fixes/WPB-28645 b/changelog.d/3-bug-fixes/WPB-28645 deleted file mode 100644 index 570d00f42f0..00000000000 --- a/changelog.d/3-bug-fixes/WPB-28645 +++ /dev/null @@ -1,4 +0,0 @@ -Gundeck's presence cleanup background thread no longer swallows asynchronous -exceptions. On shutdown this removes the spurious "presence cleanup failed" -(AsyncCancelled) error log line and lets the thread terminate promptly instead -of lingering for up to an hour. diff --git a/changelog.d/5-internal/WPB-28377-gundeck-presence-postgres b/changelog.d/5-internal/WPB-28377-gundeck-presence-postgres deleted file mode 100644 index 3862322c3a1..00000000000 --- a/changelog.d/5-internal/WPB-28377-gundeck-presence-postgres +++ /dev/null @@ -1 +0,0 @@ -Gundeck presence tracking moved from redis to postgres; redis and the gundeck redis configuration options are gone. (WPB-28377) diff --git a/charts/databases-ephemeral/requirements.yaml b/charts/databases-ephemeral/requirements.yaml index fff1534c387..74dbd7594d8 100644 --- a/charts/databases-ephemeral/requirements.yaml +++ b/charts/databases-ephemeral/requirements.yaml @@ -13,6 +13,13 @@ dependencies: # since cassandra-migrations did not yet run; but the cassandra-migrations hook # requires all pods to be in a 'Ready' state before starting (condition for post-install); this is impossible. ##################################################### +- name: redis-ephemeral + version: "0.0.42" + repository: "file://../redis-ephemeral" + tags: + - redis-ephemeral + - databases-ephemeral + - demo - name: elasticsearch-ephemeral version: "0.0.42" repository: "file://../elasticsearch-ephemeral" diff --git a/charts/databases-ephemeral/templates/NOTES.txt b/charts/databases-ephemeral/templates/NOTES.txt index 7f07b9ed7ca..2e2ad5b0592 100644 --- a/charts/databases-ephemeral/templates/NOTES.txt +++ b/charts/databases-ephemeral/templates/NOTES.txt @@ -2,6 +2,7 @@ You now have an in-memory, non-persistent, non-highly-available set of databases * cassandra-ephemeral * elasticsearch-ephemeral +* redis-ephemeral !! WARNING WARNING !! This is fine for testing and demo purposes, but NOT for a production use case. diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 4841ef372b7..9fea9fbde3e 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -41,10 +41,6 @@ spec: configMap: name: "gundeck" - - name: "gundeck-secrets" - secret: - secretName: "gundeck" - - name: "cargohold-config" configMap: name: "cargohold" @@ -97,6 +93,9 @@ spec: secret: secretName: {{ .Values.config.elasticsearch.tlsCaSecretRef.name }} + - name: redis-ca + secret: + secretName: {{ .Values.config.redis.tlsCaSecretRef.name }} - name: rabbitmq-ca secret: @@ -238,9 +237,6 @@ spec: - name: gundeck-config mountPath: /etc/wire/gundeck/conf - - name: gundeck-secrets - mountPath: /etc/wire/gundeck/secrets - - name: cargohold-config mountPath: /etc/wire/cargohold/conf @@ -280,6 +276,9 @@ spec: - name: elasticsearch-ca mountPath: /etc/wire/brig/elasticsearch-ca + - name: redis-ca + mountPath: /etc/wire/gundeck/redis-ca + - name: rabbitmq-ca mountPath: /etc/wire/brig/rabbitmq-ca @@ -344,6 +343,20 @@ spec: - name: ENABLE_FEDERATION_V{{$version}} value: "1" {{- end }} + {{- if hasKey .Values.secrets "redisUsername" }} + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: integration + key: redisUsername + {{- end }} + {{- if hasKey .Values.secrets "redisPassword" }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: integration + key: redisPassword + {{- end }} - name: TEST_XML value: /tmp/result.xml {{- if .Values.config.uploadXml }} diff --git a/charts/integration/templates/secret.yaml b/charts/integration/templates/secret.yaml index 34e6698ed2f..32f6085176e 100644 --- a/charts/integration/templates/secret.yaml +++ b/charts/integration/templates/secret.yaml @@ -16,4 +16,10 @@ data: {{- if hasKey . "uploadXmlAwsSecretAccessKey" }} uploadXmlAwsSecretAccessKey: {{ .uploadXmlAwsSecretAccessKey | b64enc | quote }} {{- end }} + {{- if hasKey . "redisUsername" }} + redisUsername: {{ .redisUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisPassword" }} + redisPassword: {{ .redisPassword | b64enc | quote }} + {{- end }} {{- end }} diff --git a/charts/reaper/.helmignore b/charts/reaper/.helmignore new file mode 100644 index 00000000000..f0c13194444 --- /dev/null +++ b/charts/reaper/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/reaper/Chart.yaml b/charts/reaper/Chart.yaml new file mode 100644 index 00000000000..131654fa443 --- /dev/null +++ b/charts/reaper/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +version: 0.0.42 +name: reaper +appVersion: 0.1.0 +description: A helm charts to restart cannons if redis-ephemeal has died +annotations: + # must conform to https://github.com/helm/community/blob/main/hips/hip-0015.md + helm.sh/images: | + - name: kubectl + image: docker.io/alpine/kubectl:1.36.3 diff --git a/charts/reaper/README.md b/charts/reaper/README.md new file mode 100644 index 00000000000..f4b73e4e670 --- /dev/null +++ b/charts/reaper/README.md @@ -0,0 +1,71 @@ +Reaper +------ + +This pod is useful in the following scenario: You run wire-server alongside a single +redis-ephemeral (part of databases-ephemeral). If you have a different setup for redis, +do not use this chart. + +Due to the nature of pods and their ephemerality, there might be situations where a +redis-ephemeral pod is restarted. In such cases, wire clients will have stale +connections (they will have an active websocket connection, but gundeck (responsible for +sending messages) will be unaware of this (as the record of who is connected where is +gone with a redis-ephemeral restart). So these stale clients will not receive any +messages. Here, this reaper will check that the `redis-ephemeral` pod is older than any +other `cannon`; if that is not the case, it kills the `cannon`s forcing clients to +reconnect. + +Image +----- + +The reaper runs `scripts/reaper.sh` through `kubectl`, so `image` must point at a +kubectl image that **contains a POSIX shell** at `/bin/sh`. Distroless kubectl images +do not ship one and the pod will fail to start. The script itself is POSIX sh, so +busybox `ash` is enough, bash not required. + +The image is fully configurable: + +```yaml +image: + registry: docker.io # set to "" for an unqualified repository + repository: alpine/kubectl + tag: 1.36.3 + digest: "" # e.g. "sha256:..."; takes precedence over tag + pullPolicy: IfNotPresent +imagePullSecrets: + - name: my-pull-secret +``` + +RBAC +---- + +The chart creates a namespaced `Role`/`RoleBinding` granting `get`, `list`, `watch` and +`delete` on pods, bound to a `-reaper` ServiceAccount. + +`watch` is required even though the script never watches anything explicitly: +`kubectl delete pod` blocks until the pod is gone and opens a watch to do so. Without it +the reaper deletes the first cannon and then hangs, without crashing. + +Earlier versions bound the ServiceAccount to `cluster-admin` through a fixed-name +`ClusterRoleBinding`, which gave the pod read access to every Secret in the cluster. +`helm upgrade` removes that binding and the old `reaper-role` ServiceAccount. Because +nothing is cluster-scoped any more and all names are release-scoped, several reaper +releases can now coexist in one cluster; previously a second release failed to install +with a `ClusterRoleBinding` ownership conflict. + +Runtime +------- + +The container runs as uid/gid 65534 with a read-only root filesystem and has resource +requests and limits. `nodeSelector`, `tolerations` and `affinity` are honoured. + +`checkIntervalSeconds` (default `15`) controls how long the script waits between checks. +Earlier versions listed pods once per second. + +Logs distinguish a failure to reach the API from "there are no matching pods", and +include the underlying error: + + Failed to list pods: Error from server (Forbidden): ... Skipping this iteration... + No cannon pods found. Doing nothing... + +Both cases previously printed `Failed to list pods. Skipping this iteration...`, so a +reaper that could not list pods at all looked exactly like an idle one. diff --git a/charts/reaper/scripts/reaper.sh b/charts/reaper/scripts/reaper.sh new file mode 100755 index 00000000000..f67049e76a6 --- /dev/null +++ b/charts/reaper/scripts/reaper.sh @@ -0,0 +1,89 @@ +#!/bin/sh + +# See the readme of the reaper chart. +# +# This is POSIX sh on purpose: the only actively maintained kubectl images that +# ship busybox ash, not bash. + +# we loop forever, and on transient errors sleep and try again. +# setting -e would crash the pod on transient e.g. network errors, which isn't useful. +set -u +# shellcheck disable=SC3040 # busybox ash supports pipefail +set -o pipefail + +USAGE="$0 [INTERVAL_SECONDS]" +NAMESPACE="${1:?$USAGE}" +INTERVAL="${2:-15}" + +echo "Using namespace: $NAMESPACE, check interval: ${INTERVAL}s" + +kill_all_cannons() { + echo "Killing all cannons" + RAW_PODS=$(kubectl -n "$NAMESPACE" get pods 2>&1) || { + echo "Failed to list cannon pods: $RAW_PODS. Skipping this iteration..." + return + } + CANNON_PODS=$(echo "$RAW_PODS" | grep -e "cannon" | awk '{ print $1 }') || CANNON_PODS="" + + # A here-document rather than a pipeline, so the loop runs in the current + # shell and the `exit 1` below actually terminates the script. + while IFS= read -r cannon; do + if [ -n "$cannon" ]; then + echo "Deleting $cannon" + # If a single delete fails, we skip it but keep going. + kubectl -n "$NAMESPACE" delete pod "$cannon" || { + echo "Failed to delete pod $cannon, crash reaper and try again" + exit 1 + } + fi + done <&1) || { + echo "Failed to list pods: $RAW_PODS. Skipping this iteration..." + sleep "$INTERVAL" + continue + } + + # Gather all pods that contain "cannon" or "redis-ephemeral", sorted by creation time + ALL_PODS=$(echo "$RAW_PODS" | grep -e "cannon" -e "redis-ephemeral") || ALL_PODS="" + + # Check if we have any cannon pods at all + if ! echo "$ALL_PODS" | grep -q "cannon"; then + echo "No cannon pods found. Doing nothing..." + sleep "$INTERVAL" + continue + fi + + # Check if we have any redis-ephemeral pods at all + if ! echo "$ALL_PODS" | grep -q "redis-ephemeral"; then + echo "No redis-ephemeral pod found. Doing nothing..." + sleep "$INTERVAL" + continue + fi + + # At this point, we have both cannon and redis-ephemeral pods in ALL_PODS + # Check which is oldest + FIRST_POD=$(echo "$ALL_PODS" | head -n 1 | awk '{ print $1 }') + + if [ -z "$FIRST_POD" ]; then + echo "Could not determine the oldest pod from the list. Doing nothing..." + sleep "$INTERVAL" + continue + fi + + case "$FIRST_POD" in + *redis-ephemeral*) + echo "redis-ephemeral is the oldest pod, all good." + ;; + *) + kill_all_cannons + ;; + esac + + sleep "$INTERVAL" +done diff --git a/charts/reaper/templates/_helpers.tpl b/charts/reaper/templates/_helpers.tpl new file mode 100644 index 00000000000..47fc05fa161 --- /dev/null +++ b/charts/reaper/templates/_helpers.tpl @@ -0,0 +1,26 @@ +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{- define "includeSecurityContext" -}} + {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} +{{- end -}} + +{{/* Fully qualified image reference, digest taking precedence over tag. */}} +{{- define "reaper.image" -}} +{{- $repository := .Values.image.repository -}} +{{- if .Values.image.registry -}} +{{- $repository = printf "%s/%s" .Values.image.registry .Values.image.repository -}} +{{- end -}} +{{- if .Values.image.digest -}} +{{- printf "%s@%s" $repository .Values.image.digest -}} +{{- else -}} +{{- printf "%s:%s" $repository (.Values.image.tag | toString) -}} +{{- end -}} +{{- end -}} + +{{/* Release-scoped name for the ServiceAccount, Role and RoleBinding. */}} +{{- define "reaper.serviceAccountName" -}} +{{- printf "%s-reaper" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/reaper/templates/configmap.yaml b/charts/reaper/templates/configmap.yaml new file mode 100644 index 00000000000..571e81a1f4a --- /dev/null +++ b/charts/reaper/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: reaper-script + labels: + app: reaper +data: + reaper.sh: |- + {{- .Files.Get "scripts/reaper.sh" | nindent 4 }} + diff --git a/charts/reaper/templates/deployment.yaml b/charts/reaper/templates/deployment.yaml new file mode 100644 index 00000000000..9d50439dc5c --- /dev/null +++ b/charts/reaper/templates/deployment.yaml @@ -0,0 +1,81 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reaper + labels: + app: reaper + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: 1 + selector: + matchLabels: + app: reaper + release: {{ .Release.Name }} + template: + metadata: + labels: + app: reaper + release: {{ .Release.Name }} + annotations: + # Ensure changes to the script cause a redeployment upon `helm upgrade` + checksum/configmap: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum }} + spec: + serviceAccountName: {{ include "reaper.serviceAccountName" . }} + automountServiceAccountToken: true + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: reaper + containers: + - name: reaper + image: {{ include "reaper.image" . | quote }} + imagePullPolicy: {{ default "" .Values.image.pullPolicy | quote }} + command: ["/bin/sh", "/app/reaper.sh", "{{ .Release.Namespace }}", "{{ .Values.checkIntervalSeconds }}"] + {{- if eq (include "includeSecurityContext" .) "true" }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 12 }} + {{- end }} + env: + # kubectl writes its discovery cache below $HOME; the root + # filesystem is read-only, so point it at the emptyDir. + - name: HOME + value: /tmp + volumeMounts: + - name: reaper-script + mountPath: /app + readOnly: true + - name: tmp + mountPath: /tmp + resources: +{{ toYaml .Values.resources | indent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: reaper-script + configMap: + name: reaper-script + defaultMode: 0755 + items: + - key: reaper.sh + path: reaper.sh + - name: tmp + emptyDir: {} diff --git a/charts/reaper/templates/rbac.yaml b/charts/reaper/templates/rbac.yaml new file mode 100644 index 00000000000..5e4caafb6f2 --- /dev/null +++ b/charts/reaper/templates/rbac.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "reaper.serviceAccountName" . }} + labels: + app: reaper + release: {{ .Release.Name }} +--- +# The reaper only ever lists and deletes pods in its own namespace, so a +# namespaced Role is sufficient. +# `watch` is required even though the script never watches anything explicitly. +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "reaper.serviceAccountName" . }} + labels: + app: reaper + release: {{ .Release.Name }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "delete"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "reaper.serviceAccountName" . }} + labels: + app: reaper + release: {{ .Release.Name }} +roleRef: + kind: Role + name: {{ include "reaper.serviceAccountName" . }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ include "reaper.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/charts/reaper/values.yaml b/charts/reaper/values.yaml new file mode 100644 index 00000000000..cf2f56cf9e4 --- /dev/null +++ b/charts/reaper/values.yaml @@ -0,0 +1,45 @@ +image: + # The reaper executes a shell script through kubectl, so this image must + # contain a POSIX shell at /bin/sh. Distroless kubectl images do not + # ship one and the pod will fail to start with them. + # + # Set `registry` to "" to use an unqualified repository (e.g. when mirroring + # into a registry configured as the daemon default). + registry: docker.io + repository: alpine/kubectl + tag: 1.36.3 + # Optional: pin by digest (e.g. "sha256:abc..."). Takes precedence over `tag`. + digest: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] + +# How long to wait between two checks, in seconds. The condition this chart +# watches for (a redis-ephemeral restart) is rare, so there is no reason to poll +# the API server aggressively. +checkIntervalSeconds: 15 + +resources: + requests: + memory: 32Mi + cpu: 10m + limits: + memory: 64Mi + +nodeSelector: {} +tolerations: [] +affinity: {} + +# Applied as the container securityContext. runAsUser/runAsGroup are set +# explicitly because alpine/kubectl runs as root by default. +podSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + seccompProfile: + type: RuntimeDefault diff --git a/charts/redis-ephemeral/Chart.yaml b/charts/redis-ephemeral/Chart.yaml new file mode 100644 index 00000000000..c907a999576 --- /dev/null +++ b/charts/redis-ephemeral/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Wrapper chart for https://github.com/groundhog2k/helm-charts/tree/redis-1.3.8/charts/redis +name: redis-ephemeral +version: 0.0.42 diff --git a/charts/redis-ephemeral/requirements.yaml b/charts/redis-ephemeral/requirements.yaml new file mode 100644 index 00000000000..0cdad0dfbc4 --- /dev/null +++ b/charts/redis-ephemeral/requirements.yaml @@ -0,0 +1,5 @@ +dependencies: +- name: redis + version: 1.3.8 + repository: https://groundhog2k.github.io/helm-charts/ + alias: redis-ephemeral diff --git a/charts/redis-ephemeral/values.yaml b/charts/redis-ephemeral/values.yaml new file mode 100644 index 00000000000..aafcf440e40 --- /dev/null +++ b/charts/redis-ephemeral/values.yaml @@ -0,0 +1,60 @@ +redis-ephemeral: + image: + tag: "7.4.6" + + haMode: + enabled: false + + redisConfig: | + # dont write rdb to emptyDir disk for an ephemeral setup + save "" + +# To add a password add the following to redisConfig: +# requirepass my-plaintext-password + + +# How to enable SSL connections: +# +# Add the following lines to redisConfig: +# +# port 0 +# tls-port 6379 +# tls-cert-file /data/ssl/tls.crt +# tls-key-file /data/ssl/tls.key +# tls-ca-cert-file /data/ssl/ca.crt +# tls-auth-clients no +# +# Mount the certificate and adjust probes to use SSL +# +# redis-ephemeral: +# extraRedisSecrets: +# - name: redis-certificate +# mountPath: /data/ssl +# +# livenessProbe: +# enabled: false +# customLivenessProbe: +# exec: +# command: +# - sh +# - -c +# - redis-cli --tls --cacert /data/ssl/ca.crt ping +# +# readinessProbe: +# enabled: false +# customReadinessProbe: +# exec: +# command: +# - sh +# - -c +# - redis-cli --tls --cacert /data/ssl/ca.crt ping +# +# startupProbe: +# enabled: false +# customStartupProbe: +# exec: +# command: +# - sh +# - -c +# - redis-cli --tls --cacert /data/ssl/ca.crt ping +# diff --git a/charts/wire-server/templates/_helpers.tpl b/charts/wire-server/templates/_helpers.tpl index 94aca197dc7..5edb0251456 100644 --- a/charts/wire-server/templates/_helpers.tpl +++ b/charts/wire-server/templates/_helpers.tpl @@ -106,6 +106,46 @@ {{- end -}} {{- end -}} +{{- define "gundeck.configureRedisCa" -}} +{{ or (hasKey .redis "tlsCa") (hasKey .redis "tlsCaSecretRef") }} +{{- end -}} + +{{- define "gundeck.redisTlsSecretName" -}} +{{- if .redis.tlsCaSecretRef -}} +{{ .redis.tlsCaSecretRef.name }} +{{- else }} +{{- print "gundeck-redis-ca" -}} +{{- end -}} +{{- end -}} + +{{- define "gundeck.redisTlsSecretKey" -}} +{{- if .redis.tlsCaSecretRef -}} +{{ .redis.tlsCaSecretRef.key }} +{{- else }} +{{- print "ca.pem" -}} +{{- end -}} +{{- end -}} + +{{- define "gundeck.configureAdditionalRedisCa" -}} +{{ and (hasKey . "redisAdditionalWrite") (or (hasKey .redis "additionalTlsCa") (hasKey .redis "additionalTlsCaSecretRef")) }} +{{- end -}} + +{{- define "gundeck.additionalRedisTlsSecretName" -}} +{{- if .redis.additionalTlsCaSecretRef -}} +{{ .redis.additionalTlsCaSecretRef.name }} +{{- else }} +{{- print "gundeck-additional-redis-ca" -}} +{{- end -}} +{{- end -}} + +{{- define "gundeck.additionalRedisTlsSecretKey" -}} +{{- if .redis.additionalTlsCaSecretRef -}} +{{ .redis.additionalTlsCaSecretRef.key }} +{{- else }} +{{- print "ca.pem" -}} +{{- end -}} +{{- end -}} + {{/* SPAR */}} {{- define "spar.tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} diff --git a/charts/wire-server/templates/cannon/statefulset.yaml b/charts/wire-server/templates/cannon/statefulset.yaml index f60d6586453..00103604bf8 100644 --- a/charts/wire-server/templates/cannon/statefulset.yaml +++ b/charts/wire-server/templates/cannon/statefulset.yaml @@ -2,7 +2,7 @@ # Specific pods can be accessed within the cluster at cannon-.cannon. # (the second 'cannon' is the name of the headless service) # Note: In fact, cannon-.cannon can also be used to access the service but assuming -# that we can have multiple namespaces accessing the same cannon cluster, appending `.` +# that we can have multiple namespaces accessing the same redis cluster, appending `.` # makes the service unambiguous apiVersion: apps/v1 kind: StatefulSet diff --git a/charts/wire-server/templates/gundeck/configmap.yaml b/charts/wire-server/templates/gundeck/configmap.yaml index 6aae6aa47d8..7e844043675 100644 --- a/charts/wire-server/templates/gundeck/configmap.yaml +++ b/charts/wire-server/templates/gundeck/configmap.yaml @@ -41,11 +41,35 @@ data: {{- end }} {{- end }} + redis: + host: {{ .redis.host }} + port: {{ .redis.port }} + connectionMode: {{ .redis.connectionMode }} + enableTls: {{ .redis.enableTls }} + insecureSkipVerifyTls: {{ .redis.insecureSkipVerifyTls }} + {{- if eq (include "gundeck.configureRedisCa" .) "true" }} + tlsCa: /etc/wire/gundeck/redis-ca/{{ include "gundeck.redisTlsSecretKey" . }} + {{- end }} + + {{- if .redisAdditionalWrite }} + redisAdditionalWrite: + host: {{ .redisAdditionalWrite.host }} + port: {{ .redisAdditionalWrite.port }} + connectionMode: {{ .redisAdditionalWrite.connectionMode }} + enableTls: {{ .redisAdditionalWrite.enableTls }} + insecureSkipVerifyTls: {{ .redisAdditionalWrite.insecureSkipVerifyTls }} + {{- if eq (include "gundeck.configureAdditionalRedisCa" .) "true" }} + tlsCa: /etc/wire/gundeck/additional-redis-ca/{{ include "gundeck.additionalRedisTlsSecretKey" . }} + {{- end }} + {{- end }} + + {{- if .postgresql }} postgresql: {{ toYaml .postgresql | nindent 6 }} postgresqlPool: {{ toYaml .postgresqlPool | nindent 6 }} {{- if hasKey $.Values.gundeck.secrets "pgPassword" }} postgresqlPassword: /etc/wire/gundeck/secrets/pgPassword {{- end }} + {{- end }} # Gundeck uses discovery for AWS access key / secrets # For more details, check amazonka's documentation at: diff --git a/charts/wire-server/templates/gundeck/deployment.yaml b/charts/wire-server/templates/gundeck/deployment.yaml index ff7a457fc6e..bc46a53ec0d 100644 --- a/charts/wire-server/templates/gundeck/deployment.yaml +++ b/charts/wire-server/templates/gundeck/deployment.yaml @@ -50,9 +50,16 @@ spec: secret: secretName: {{ (include "gundeck.tlsSecretRef" .Values.gundeck.config | fromYaml).name }} {{- end }} - - name: "gundeck-secrets" + {{- if eq (include "gundeck.configureRedisCa" .Values.gundeck.config) "true" }} + - name: "redis-ca" secret: - secretName: "gundeck" + secretName: {{ include "gundeck.redisTlsSecretName" .Values.gundeck.config }} + {{- end }} + {{- if eq (include "gundeck.configureAdditionalRedisCa" .Values.gundeck.config) "true" }} + - name: "additional-redis-ca" + secret: + secretName: {{ include "gundeck.additionalRedisTlsSecretName" .Values.gundeck.config }} + {{- end }} containers: - name: gundeck image: "{{ .Values.gundeck.image.repository }}:{{ .Values.gundeck.image.tag }}" @@ -68,8 +75,14 @@ spec: - name: "gundeck-cassandra" mountPath: "/etc/wire/gundeck/cassandra" {{- end }} - - name: "gundeck-secrets" - mountPath: "/etc/wire/gundeck/secrets" + {{- if eq (include "gundeck.configureRedisCa" .Values.gundeck.config) "true" }} + - name: "redis-ca" + mountPath: "/etc/wire/gundeck/redis-ca/" + {{- end }} + {{- if eq (include "gundeck.configureAdditionalRedisCa" .Values.gundeck.config) "true" }} + - name: "additional-redis-ca" + mountPath: "/etc/wire/gundeck/additional-redis-ca/" + {{- end }} {{- if and .Values.gundeck.config.rabbitmq .Values.gundeck.config.rabbitmq.tlsCaSecretRef }} - name: "rabbitmq-ca" mountPath: "/etc/wire/gundeck/rabbitmq-ca/" @@ -97,6 +110,34 @@ spec: name: gundeck key: awsSecretKey {{- end }} + {{- if hasKey .Values.gundeck.secrets "redisUsername" }} + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: gundeck + key: redisUsername + {{- end }} + {{- if hasKey .Values.gundeck.secrets "redisPassword" }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck + key: redisPassword + {{- end }} + {{- if hasKey .Values.gundeck.secrets "redisAdditionalWriteUsername" }} + - name: REDIS_ADDITIONAL_WRITE_USERNAME + valueFrom: + secretKeyRef: + name: gundeck + key: redisAdditionalWriteUsername + {{- end }} + {{- if hasKey .Values.gundeck.secrets "redisAdditionalWritePassword" }} + - name: REDIS_ADDITIONAL_WRITE_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck + key: redisAdditionalWritePassword + {{- end }} - name: AWS_REGION value: "{{ .Values.gundeck.config.aws.region }}" {{- with .Values.gundeck.config.proxy }} diff --git a/charts/wire-server/templates/gundeck/redis-ca-secret.yaml b/charts/wire-server/templates/gundeck/redis-ca-secret.yaml new file mode 100644 index 00000000000..a82eab555cb --- /dev/null +++ b/charts/wire-server/templates/gundeck/redis-ca-secret.yaml @@ -0,0 +1,30 @@ +--- +{{- if not (empty .Values.gundeck.config.redis.tlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: "gundeck-redis-ca" + labels: + app: gundeck + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.gundeck.config.redis.tlsCa | b64enc | quote }} +{{- end }} +--- +{{- if not (empty .Values.gundeck.config.redis.additionalTlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: "gundeck-additional-redis-ca" + labels: + app: gundeck + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.gundeck.config.redis.additionalTlsCa | b64enc | quote }} +{{- end }} diff --git a/charts/wire-server/templates/gundeck/secret.yaml b/charts/wire-server/templates/gundeck/secret.yaml index a17529f4c77..000255ba0a6 100644 --- a/charts/wire-server/templates/gundeck/secret.yaml +++ b/charts/wire-server/templates/gundeck/secret.yaml @@ -19,6 +19,18 @@ data: {{- if hasKey . "awsSecretKey" }} awsSecretKey: {{ .awsSecretKey | b64enc | quote }} {{- end }} + {{- if hasKey . "redisUsername" }} + redisUsername: {{ .redisUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisPassword" }} + redisPassword: {{ .redisPassword | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisAdditionalWriteUsername" }} + redisAdditionalWriteUsername: {{ .redisAdditionalWriteUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisAdditionalWritePassword" }} + redisAdditionalWritePassword: {{ .redisAdditionalWritePassword | b64enc | quote }} + {{- end }} {{- if hasKey . "pgPassword" }} pgPassword: {{ .pgPassword | b64enc | quote }} {{- end }} diff --git a/charts/wire-server/templates/gundeck/tests/configmap.yaml b/charts/wire-server/templates/gundeck/tests/configmap.yaml index 28d76773076..c8c23ce5185 100644 --- a/charts/wire-server/templates/gundeck/tests/configmap.yaml +++ b/charts/wire-server/templates/gundeck/tests/configmap.yaml @@ -39,3 +39,10 @@ data: host: brig port: 8080 + # a "redis migration" test in gundeck makes use of a second (distinct) redis + redis2: + host: redis-ephemeral-2 + port: 6379 + connectionMode: master + enableTls: false + insecureSkipVerifyTls: false diff --git a/charts/wire-server/templates/gundeck/tests/gundeck-integration.yaml b/charts/wire-server/templates/gundeck/tests/gundeck-integration.yaml index 60b5d26e3d5..f1a661b4a58 100644 --- a/charts/wire-server/templates/gundeck/tests/gundeck-integration.yaml +++ b/charts/wire-server/templates/gundeck/tests/gundeck-integration.yaml @@ -18,6 +18,11 @@ spec: secret: secretName: {{ (include "gundeck.tlsSecretRef" .Values.gundeck.config | fromYaml).name }} {{- end }} + {{- if eq (include "gundeck.configureRedisCa" .Values.gundeck.config) "true" }} + - name: "redis-ca" + secret: + secretName: {{ include "gundeck.redisTlsSecretName" .Values.gundeck.config }} + {{- end }} {{- if .Values.gundeck.config.rabbitmq.tlsCaSecretRef }} - name: "rabbitmq-ca" secret: @@ -68,6 +73,10 @@ spec: - name: "gundeck-cassandra" mountPath: "/etc/wire/gundeck/cassandra" {{- end }} + {{- if eq (include "gundeck.configureRedisCa" .Values.gundeck.config) "true" }} + - name: "redis-ca" + mountPath: "/etc/wire/gundeck/redis-ca/" + {{- end }} {{- if .Values.gundeck.config.rabbitmq.tlsCaSecretRef }} - name: "rabbitmq-ca" mountPath: "/etc/wire/gundeck/rabbitmq-ca/" @@ -87,6 +96,34 @@ spec: value: "guest" - name: RABBITMQ_PASSWORD value: "guest" + {{- if hasKey .Values.gundeck.secrets "redisUsername" }} + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: gundeck + key: redisUsername + {{- end }} + {{- if hasKey .Values.gundeck.secrets "redisPassword" }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck + key: redisPassword + {{- end }} + {{- if and (hasKey .Values.gundeck.tests "secrets") (hasKey .Values.gundeck.tests.secrets "redisAdditionalWriteUsername") }} + - name: REDIS_ADDITIONAL_WRITE_USERNAME + valueFrom: + secretKeyRef: + name: gundeck-integration + key: redisAdditionalWriteUsername + {{- end }} + {{- if and (hasKey .Values.gundeck.tests "secrets") (hasKey .Values.gundeck.tests.secrets "redisAdditionalWritePassword") }} + - name: REDIS_ADDITIONAL_WRITE_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck-integration + key: redisAdditionalWritePassword + {{- end }} {{- if .Values.gundeck.tests.config.uploadXml }} - name: UPLOAD_XML_S3_BASE_URL value: {{ .Values.gundeck.tests.config.uploadXml.baseUrl }} diff --git a/charts/wire-server/templates/gundeck/tests/secret.yaml b/charts/wire-server/templates/gundeck/tests/secret.yaml index df7b82695fe..60aed14a3a4 100644 --- a/charts/wire-server/templates/gundeck/tests/secret.yaml +++ b/charts/wire-server/templates/gundeck/tests/secret.yaml @@ -17,5 +17,11 @@ data: {{- if hasKey . "uploadXmlAwsSecretAccessKey" }} uploadXmlAwsSecretAccessKey: {{ .uploadXmlAwsSecretAccessKey | b64enc | quote }} {{- end }} + {{- if hasKey . "redisAdditionalWriteUsername" }} + redisAdditionalWriteUsername: {{ .redisAdditionalWriteUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisAdditionalWritePassword" }} + redisAdditionalWritePassword: {{ .redisAdditionalWritePassword | b64enc | quote }} + {{- end }} {{- end }} {{- end }} diff --git a/charts/wire-server/values.yaml b/charts/wire-server/values.yaml index 9a98be5d528..d2a7deafdc5 100644 --- a/charts/wire-server/values.yaml +++ b/charts/wire-server/values.yaml @@ -744,19 +744,35 @@ gundeck: # tlsCaSecretRef: # name: # key: - # Postgres connection settings for presence tracking. + redis: + host: redis-ephemeral + port: 6379 + connectionMode: "master" # master | cluster + enableTls: false + insecureSkipVerifyTls: false + # To configure custom TLS CA, please provide one of these: + # tlsCa: + # + # Or refer to an existing secret (containing the CA): + # tlsCaSecretRef: + # name: + # key: + + # To enable additional writes during a migration: + # redisAdditionalWrite: + # host: redis-two + # port: 6379 + # connectionMode: master + # enableTls: false + # insecureSkipVerifyTls: false # - # Values are described in https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS - # To set the password via a gundeck secret see `secrets.pgPassword`. - postgresql: - host: postgresql # DNS name without protocol - port: "5432" - user: wire-server - dbname: wire-server - postgresqlPool: - size: 100 - acquisitionTimeout: 10s - idlenessTimeout: 10m + # # To configure custom TLS CA, please provide one of these: + # # tlsCa: + # # + # # Or refer to an existing secret (containing the CA): + # # tlsCaSecretRef: + # # name: + # # key: aws: region: "eu-west-1" proxy: {} diff --git a/deploy/dockerephemeral/docker-compose.yaml b/deploy/dockerephemeral/docker-compose.yaml index a8a9ab66d5c..fb2a4801ebc 100644 --- a/deploy/dockerephemeral/docker-compose.yaml +++ b/deploy/dockerephemeral/docker-compose.yaml @@ -1,4 +1,10 @@ networks: + redis: + driver: bridge + ipam: + config: + - subnet: 172.20.0.0/24 + coredns: driver: bridge ipam: @@ -72,6 +78,134 @@ services: networks: - demo_wire + redis-master: + container_name: demo_wire_redis + image: redis:7.2-alpine + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6379:6379" + volumes: + - ./docker/redis-master-mode.conf:/usr/local/etc/redis/redis.conf + networks: + - demo_wire + + redis-cluster: + image: "redis:7.2-alpine" + command: + - redis-cli + - --cluster + - create + - 172.20.0.31:6373 + - 172.20.0.32:6374 + - 172.20.0.33:6375 + - 172.20.0.34:6376 + - 172.20.0.35:6377 + - 172.20.0.36:6378 + - --cluster-replicas + - "1" + - --cluster-yes + - -a + - very-secure-redis-cluster-password + - --cacert + - /usr/local/etc/redis/ca.pem + - --tls + volumes: + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.30 + depends_on: + - redis-node-1 + - redis-node-2 + - redis-node-3 + - redis-node-4 + - redis-node-5 + - redis-node-6 + redis-node-1: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6373:6373" + volumes: + - redis-node-1-data:/var/lib/redis + - ./docker/redis-node-1.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-1-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-1-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.31 + redis-node-2: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6374:6374" + volumes: + - redis-node-2-data:/var/lib/redis + - ./docker/redis-node-2.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-2-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-2-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.32 + redis-node-3: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6375:6375" + volumes: + - redis-node-3-data:/var/lib/redis + - ./docker/redis-node-3.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-3-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-3-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.33 + redis-node-4: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6376:6376" + volumes: + - redis-node-4-data:/var/lib/redis + - ./docker/redis-node-4.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-4-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-4-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.34 + redis-node-5: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6377:6377" + volumes: + - redis-node-5-data:/var/lib/redis + - ./docker/redis-node-5.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-5-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-5-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.35 + redis-node-6: + image: "redis:7.2-alpine" + command: redis-server /usr/local/etc/redis/redis.conf + ports: + - "127.0.0.1:6378:6378" + volumes: + - redis-node-6-data:/var/lib/redis + - ./docker/redis-node-6.conf:/usr/local/etc/redis/redis.conf + - ./docker/redis-node-6-cert.pem:/usr/local/etc/redis/cert.pem + - ./docker/redis-node-6-key.pem:/usr/local/etc/redis/key.pem + - ./docker/redis-ca.pem:/usr/local/etc/redis/ca.pem + networks: + redis: + ipv4_address: 172.20.0.36 + elasticsearch: container_name: demo_wire_elasticsearch image: elasticsearch:6.8.23 @@ -284,3 +418,11 @@ services: # - DNS_SERVER_RECURSION_DENIED_NETWORKS=1.1.1.0/24 #Comma separated list of IP addresses or network addresses to deny recursion. Valid only for `UseSpecifiedNetworkACL` recursion option. This option is obsolete and DNS_SERVER_RECURSION_NETWORK_ACL should be used instead. # - DNS_SERVER_RECURSION_ALLOWED_NETWORKS=127.0.0.1, 192.168.1.0/24 #Comma separated list of IP addresses or network addresses to allow recursion. Valid only for `UseSpecifiedNetworkACL` recursion option. This option is obsolete and DNS_SERVER_RECURSION_NETWORK_ACL should be used instead. # - DNS_SERVER_ENABLE_BLOCKING=false #Sets the DNS server to block domain names using Blocked Zone and Block List Zone. + +volumes: + redis-node-1-data: + redis-node-2-data: + redis-node-3-data: + redis-node-4-data: + redis-node-5-data: + redis-node-6-data: diff --git a/deploy/dockerephemeral/docker/redis-master-mode.conf b/deploy/dockerephemeral/docker/redis-master-mode.conf new file mode 100644 index 00000000000..d71dbc51c97 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-master-mode.conf @@ -0,0 +1 @@ +requirepass very-secure-redis-master-password \ No newline at end of file diff --git a/deploy/dockerephemeral/docker/redis-node-1-cert.pem b/deploy/dockerephemeral/docker/redis-node-1-cert.pem new file mode 100644 index 00000000000..7756f82bbd0 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-1-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDTtp3U0VPwBVJNULQF +S4BBlWBNf/8NMidOq23IsTcjkIFWO1XL+HFZoa1AArUSA/TaLBYyz9WmX7eLWvAU +ADM6mfAf2V6whmIs2H9ZRnY89bFWO2hzLWWp1qq3dXK1ywTLpw7DqU4OT0rtYZbp +QHeVY0mKKspF+YJTZzWB1hs8IX9355wXRlYBLPNQ5oHRb4/16J/UUFPIJjpUyHsq +T1LWmVREqisrq9u50FnNPeLXE6SDnHGRkYGQXzQOM/yAI75/QUOOqo5rt3Et52t5 +pkOT45R0PbAC2UpR1usew0zVjRoQfFk9n38tXUSHKw/tW+ZY1xJqEKEiLGfnhhza +t4kjAgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy0xhwSsFAAfMB0GA1UdDgQWBBQsOxsq4X8dS/Ddl9l1 +TWDb8Q5KKzAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAaByeD08xOZCV0ZKsx7lHtiem5/XG01rMcDxNVrVguSD+mqhR +/j8ciTW2CruJ2X8ReTjNrI4X1nWLbh4rsrA56q4xkjkgJIfWQAdKCibXTrHOWfk5 +dcYG1pqVdpD5bvsxAsY95jxqoVJHXHGN8ynC+lV39HbDJQFOdHLAP66NUrphp76a +OZKiuzUS6naeiHWoA9eIANFRz/JoQvyp109gdce5MH0iFwGFqNJU2rwilOpzQVc7 +qldx7MHMnW5UYSTqryTOr8PS+xo24TSdHjIXmnOO3Ov0Pw7iPpGVGj56dAKgEisG +yGOAWYto8UBWKLox1vSSlfdkhAoDXluvE8EwRw== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-1-key.pem b/deploy/dockerephemeral/docker/redis-node-1-key.pem new file mode 100644 index 00000000000..6d8b29bbdee --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-1-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDTtp3U0VPwBVJN +ULQFS4BBlWBNf/8NMidOq23IsTcjkIFWO1XL+HFZoa1AArUSA/TaLBYyz9WmX7eL +WvAUADM6mfAf2V6whmIs2H9ZRnY89bFWO2hzLWWp1qq3dXK1ywTLpw7DqU4OT0rt +YZbpQHeVY0mKKspF+YJTZzWB1hs8IX9355wXRlYBLPNQ5oHRb4/16J/UUFPIJjpU +yHsqT1LWmVREqisrq9u50FnNPeLXE6SDnHGRkYGQXzQOM/yAI75/QUOOqo5rt3Et +52t5pkOT45R0PbAC2UpR1usew0zVjRoQfFk9n38tXUSHKw/tW+ZY1xJqEKEiLGfn +hhzat4kjAgMBAAECggEAFqMmoixVxMrU34Z7ETve9WRC/VZrz53mvQ8weG6WfjuD +0NQcWuhwOkzCyR7g/JGmuzNOllVJu3Xtmr15ATJ6R9BQ8B7edJKR6cimaUXS+7ar +pRRKGVKn1a6p517sCoswMpRkzEAMpBQPZ21xZPRrNPJ+WQM1SKEiscdN3dmmZNng +MvroH1dPVbyZ49xkjMQ0NaOtk4rvopzdKKZea2qz41w/vXR9hShnfVDs/q86clmx +5mnEvXcEdguioAfUWz+qQ7dXlWsASKa/gAMjUN9GW9uOn4LclFsVCD2MW+IUJMxe ++JtFM0xiQ3HaK0Fem8+XR8mG3BB5a/06ZHBfcv/lsQKBgQD4WSJjZwMBG80uGidR +ls+VhhFjysxm5qrF34MWziLczi1nAStc/PzVcA7tHapKX5JiKYT6d8Ptngz6FLIo +/72OshmLzctxRprlpihWxMIYOqwb2PLB0//ghuUE81Zbxj1MQ6k9WbTGHBwUbaiv +PSzclhmMubypfLLcmMEnHeZFswKBgQDaPIWmyax3Eft8DzC3Om7X3WMN0NXE96z2 +6hUAon5tqinMuWUWa2cyWzPsdBgFM8mCynoiIu08YFpZQivoB6QSal4x2mLg4R+u +aLm3h9f6NS4/VvpWPL5wMUAqeCCbP/2PVKk///0mtQGixUOxeQftTncQeLtfXOXd +4gDJHjfW0QKBgQDND7xnW42Ngsk+wfWpVt981UDSp4dziA+GZ3I0iG0c6Vlv7fVC +SNrz2h1ZCN+tnZCfYS0eK3oqYBDTBfe+Br0ccE7Ls1fC5svLyBES5FBn9TpbnB2G +kmh7mqbMGak7CktfB5dcww+TbW56J7nbSKYcVgwuuMbhI8gEglUq2XNkJQKBgQDV +VojIzSmdlKSlWCwlUif9OdyVKutuizg4gAhcAH1bMxd9nFbnncLaBTIzGiJJI6EA +DHNsX3xOo1pvGzLUtnN71SOT1IsIjsprstCqS0+ktswo+xvppaP9BQhW++vUGLAE +p5x0hgixCA07U1+jZE+NekEGhx+UT7oeN8rQ0IuBoQKBgQC4PF4WwqashYHkYW2j +4LaMu5kWY/0OI9Vh/h1iOcKPzVUn61aabjsx1wF9rummIdxP03/bs7ZpkwPypcVR +v7XnNbi+hDZFEN6s/+Gl4S6RfAbWXs3sgnhVlctlkzzwG8UHCef4DWMPxFI1JQI8 +X+SdDfpmB/ayQb8TlYvke/s8cQ== +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-1.conf b/deploy/dockerephemeral/docker/redis-node-1.conf new file mode 100644 index 00000000000..aa772f502fe --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-1.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6373 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-2-cert.pem b/deploy/dockerephemeral/docker/redis-node-2-cert.pem new file mode 100644 index 00000000000..ea4b4507d6b --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-2-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4mm9GWOVh0ttlzXaJ +11/rQQX3vYQ3zyeMdz/KGTKArOF+pxVCxbETlI3ZufO8Ht9zqa7Doh5R86iNtVMR +LoQZVWeXjQsMATwNUZT3lEOezpDE0ZI8d5JyU946Z+7s0VjIMbXOzjTSjTNSi57N +li59/1NTG5CW9EtgnnYoP5SOrYTpK+fzawXD18tD8kq/VBLt8OoG7xn6DIpGsFr9 +h1Ot/yrUejvrHg2KIi3av/cnqA8twzFpkdvGSEarjRuYG6fHGL67dgSpLvzh/v7h +QiJDFFB8fHnUc5ioZXFw88P4Oq7UlzBhnkC8nhUi1X1vWoF9Xz4FXXJ1P4WkZfWB +Vui7AgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy0yhwSsFAAgMB0GA1UdDgQWBBQQK2od431iWKznJEQz +zy5GXgt1DDAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAhvNbLzlY4sS/xmn8alzIYjY/uIc5c0PaUaXc7SSjeoChRfNQ +tE5YLmOo86WYNThtaNmiRLFv3yNBXCcqdVgNdL78EIQlKvPxHwzZXxkKDmOcfIZS +nUa4w+OmKJLsdNjphBGmR94h8WycwoFMThw55vnTJ2+AnCFPsLDfjtHiKB8AsW8u +gtSTtVyu+QyvGTDxEFDgqFgyFjJpVp37bOakRuzuZZ8VUssQbb11YHyhnNGTcL3a +hLXeGVSRA7SyDXxxRs5PmmJVsUOWkgbIjguvZK5APpqaGEYwBYo036DFSgt6DTOu +8YsCTeSOmue0xNlPDiVPSP8HUGfq3tTBKMXbUQ== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-2-key.pem b/deploy/dockerephemeral/docker/redis-node-2-key.pem new file mode 100644 index 00000000000..fba9118998e --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-2-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC4mm9GWOVh0ttl +zXaJ11/rQQX3vYQ3zyeMdz/KGTKArOF+pxVCxbETlI3ZufO8Ht9zqa7Doh5R86iN +tVMRLoQZVWeXjQsMATwNUZT3lEOezpDE0ZI8d5JyU946Z+7s0VjIMbXOzjTSjTNS +i57Nli59/1NTG5CW9EtgnnYoP5SOrYTpK+fzawXD18tD8kq/VBLt8OoG7xn6DIpG +sFr9h1Ot/yrUejvrHg2KIi3av/cnqA8twzFpkdvGSEarjRuYG6fHGL67dgSpLvzh +/v7hQiJDFFB8fHnUc5ioZXFw88P4Oq7UlzBhnkC8nhUi1X1vWoF9Xz4FXXJ1P4Wk +ZfWBVui7AgMBAAECggEAONB+8r2lSygkEf7cPqwkfzjx5z9SlAKTf22sGj0LCAMt +G1e8+WHyj74msh3C3+D4kJZmjRs2Da7Z71MhD6arTUi1qzTjc3xlyQuUt2XQMe4N +LCX7xdRfJASf3oXiSMxdcK+r7swUAcEnTH5gD5HrGSgdsvRG2c6x7DiY0OZQiGBk +2rPHQDUKeb7Z0YLWc8nldzlnOe2OeWpFVEraAOzmANnV5FVJZP8RNoiKviZnB77O +qbA6Xtpg4ytVhMaymUmjkjdFuxm5XcMCOIz4W9SZVJ9uSzjZqATzgjsiOWYozB7q +2xb1yOyCVPgf+dZj32D8DvqSrwwRBR3LcNhnj2wUIQKBgQDqL4VNp54Lrf+oZ0ZF +h3s6lL2NquY0xHs91YvoO187VetyUlNjOcGXt8ROhSSAf6qTLQvrreVdjYHr52xr +smCohhQ9QDm3d+Inh3ARgr75O577aPwJHBmo0fnu9h6OkDr8nx05SthW4XenHqoE +iWQ9FnibAFz5KLBSYC7x9wfGaQKBgQDJzI3UC6AqQS8ILbcqHm7ZmnpUUjn7vPUm +lkB3/YtV7ewWJhFzdPdaKHKe2YO9WXQTCF7iPRK3+gWt8uh4DWCrSObBkmSUlF66 +wbRof3lsYiWDPed9OTgoDHRwbMPeYrJ3A0TMrGJQsbedljneaat+DM3kNgjgChfW +JiL0g9c5gwKBgQDBi8zMRT/lv0SQVepKBJLf85ZFw3zHF6wTiq46nPcz/uq8bTXl +yBIr5gEkM/3bBahgQtabTflGvHEoGvgMejxQi5+mj7Ij47zRlqoUjs5vBct7VWUX +0lWSpRe/W0Id6S4XIxnwA9+Qzn8pa7pwTWy+4BeFY2NzuSEgs8WYzOVsIQKBgHbI +IPOfpDc7ByQZRKdWIomTlE3t2JOFNgfwiSIX69w4n66p2bvMLYy0IkO+ZP0fmmNZ +mgAxUsNYN9+cC5oexbgMwUdPlESg0OG9AyQ/ZImXe900ov3ioFtyeVdzrhdIoSPM +mMKg9X3qHdp0gruYF4mqn8akx7SYPE+hQxIKSLVhAoGAJP+TshJj8xAeE1Uroyc/ +yIWThbp0Q/EFaXkpS6aJqBjdcLfh2U+Zo9ZaTn9OBlzXHk9WttzeWuMY9PrINodJ +8DSg5f0PslYxJ5DQuKnDWUeqX3zCnXkgnymlvh78t6wWp+BUAEjI8qH5IgKVwKd+ +VJbPX4mzhAl/0kIablU6SqM= +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-2.conf b/deploy/dockerephemeral/docker/redis-node-2.conf new file mode 100644 index 00000000000..de7687558b3 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-2.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6374 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-3-cert.pem b/deploy/dockerephemeral/docker/redis-node-3-cert.pem new file mode 100644 index 00000000000..e550d0e30f9 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-3-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCd72omHFn1mEFw/GBp +gkPM5BkF7giGx7GOLyijCoi4NLNVKJn6mOJt9vX2PbBYedy1OcskObLbEwqUwcZr +7fVim34xrE4AmdJqBWTkcMFnhbjzYIynfvejej/05kWlzp3JuhTpi7i2W+nnZjqb +S6UHgeTwF/iENA1oysuq0jC4oaVGNa2ZCoz3W+uAEbpUYNjN7/uQeEwRyZjSEJUY +KyG69Wrl9KnzBX0mkltq8rJiCqaG+qOZwP+XH7TxjYM1SlAxLHrnjDQHWyZXJzPY +fikRk2Zf8nDobA5thXVR/2PicDxUs1VyGYSg/vK1EMwOIHIZdxalo0x75vFjBJ9T +l+HFAgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy0zhwSsFAAhMB0GA1UdDgQWBBQyljx2OR3L7yZLVax4 +MLTDhj4xPjAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAUv3JN0ip/LWmtWHyqzPuq9tbVFs2M5waRO2ZZtEp6Pzudr9x +JKrmtz7IlnwK2E3eqw1Hh3kZYiM5XT2GzqFjPn+Na32i3IsR/S1Y4ZDq6T1WOjht +u+3EjrUvpTXAcLfaO60gJ7DrfC4PsuNuaRr23BiF3lIb7A693hnESg3EnUqGvAvA +ikR/Cv48kAvxpFlXZfnGApFEP49svj676emodRUlk4aCOjIniPByLF318Dl+MwzW +KbnjynzjnOqfcXeD67axFqIBAhZPBDWIDOLNo/ASAROkPntycBGFPUL+Wgdq75vs +8WnftwfCzYtKcASNVSeoSFtJhVy2cAqHK1bd/g== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-3-key.pem b/deploy/dockerephemeral/docker/redis-node-3-key.pem new file mode 100644 index 00000000000..d7be5cf147d --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-3-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCd72omHFn1mEFw +/GBpgkPM5BkF7giGx7GOLyijCoi4NLNVKJn6mOJt9vX2PbBYedy1OcskObLbEwqU +wcZr7fVim34xrE4AmdJqBWTkcMFnhbjzYIynfvejej/05kWlzp3JuhTpi7i2W+nn +ZjqbS6UHgeTwF/iENA1oysuq0jC4oaVGNa2ZCoz3W+uAEbpUYNjN7/uQeEwRyZjS +EJUYKyG69Wrl9KnzBX0mkltq8rJiCqaG+qOZwP+XH7TxjYM1SlAxLHrnjDQHWyZX +JzPYfikRk2Zf8nDobA5thXVR/2PicDxUs1VyGYSg/vK1EMwOIHIZdxalo0x75vFj +BJ9Tl+HFAgMBAAECggEABYejI9UiS+MaMiaOtE2x/16NMb6f4Hg600umFJoDJ3qm +PM5rIHHHRn+7JPVhU00RA+y+HB/uZJVKGDigsJloWhzaUkrs1ZXiiYEe2JDKH3cj +KVexamabrRxUA53RxSMdizlPZM4A7axSMvP1YV1IrfadBCW9Ydj2DzvqiFShDWst +asKPAa6MAU63zfZZaBQvicswd1nJUvc8ZNp1p0JiVcwWPWVTYH9d2c+0WZLlfCHm +GxUurHwyVc6b7T4OSrsiDaQN0kdLJDAYowp+T94JDBCH3m4e/NF9W6gkoO2UGXTH +6A9HVDI3FwUBzXdT9rL/Wmp4kKXB4xO2TU/yeZoYAQKBgQDK2Q2vG+BucY2aJxGw +7HNeXov2lLma2Vn4TRr+cyzcXH7Jmc8J/h9RMU7AEfg3CQwMbXE60P561/1q1e0Z +fD55x9ka3FZ2dG+a5CDzjkqnUgnLYOK1bxx5UUq+Sf6IeNjGPikejRPcPBmvFVuu +NvoPU0HwWLm67BnantJIpUFvRQKBgQDHUaPa6SIMGAWHasI6EvZMGgBAy5iJa4s3 +o+DuESF+6lD989ZnOltsPFeYhwbIzm14EzhK/y4MVR46gXLMZ9FwlGCGdXE7LWiN +VKCm9kRcxcH9Sak70LkZ9yv08Nl45f9vTOzBcKzu6bgZ2LOeSJ0oTmiVEb98pL7N +w6XxD2iQgQKBgAVVPYncBsOAksN5wXpQTRwvCij6cgLDMh1YEZyc9JH6kI7GT24o +0zP0QujD0C3KPBnbir2MHxSltxDm/OvNm2riOS/+mPtWRlThKIiethG+E2nYaz1v +5WS/IWLtWRbHbpOPsM8P0HTa06YJvrZO1bYvby1dd8yVRny77jVgut6tAoGAHpMK +ZHkgjORebMBWnNvtxgyy/z1735CMoXNU/I/KKJK+68WsnNcZ0QeMlEwaIVFw/1tL +Zk2wfZnM8kKLHonKWc+Y4uc/AEnd4NgbcKEUKXr4X+cdu5wv2KjOqFsNsPru7N7K +7n1fOaLGZ8iS/PO8j8M/TaaUTgVjc2LQoKKxcoECgYAtPzq1Y0yc22M+m1m6nK/W +L7rsUI0zDs0VZcJ5mrJg8nahOM/f+BsFYN5oAHYxuXPUyynZyD2nPtdsES75DGOH +PEqr9DhgSig4JmHS/6SEBnWql+zyNdn1/FaYOkKRHiY7jNhjTayiDObJrXg0g4OT +BmzY39BABb52ogQbjWslow== +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-3.conf b/deploy/dockerephemeral/docker/redis-node-3.conf new file mode 100644 index 00000000000..7f406d72324 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-3.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6375 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-4-cert.pem b/deploy/dockerephemeral/docker/redis-node-4-cert.pem new file mode 100644 index 00000000000..185f8f97014 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-4-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKxOmFo5c4ae38qC3z +I89R1F6xjaMyR6jjd6k5qsW7eU/y8+trgY4HV/jbzD3CDOjMkj70la5EiV+GTA0i +GeJH/BkKjqEPsIy/vAPux9xt2ZpIO9ieO2BF75ojrcM7tAbeOLQNAgYA7zAyIpQk +J2P8IyOYSJ31ujLJCR7d0zudAbXJXfAAyPUWqUrmmRHIY7hRi1tUv74JARqnU2tH +ZhFgGyBCaLROK69S/Wy+xPKo5w9Ol5L9eIccrK2/JwNpfsFAxJqXawNm1l1M9gGk +2MpQXzZeTg/hlusqCtPieOPUQKoEDXAgYArQy8iYkLuZzOtg2WwcPOhtfsgVRLNE +wXihAgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy00hwSsFAAiMB0GA1UdDgQWBBQrI/peejY55qjXOc6W +XUU+/q6R+TAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAdf1N+gPpnkEHzDAMnK4kUCHq2ymLBBWJVAPDcmmtcMjEiEVC +/9BU+hcdqgLXxonEqiA4kEs9Mkj8AcUk0Dzl5Gfk2haZO6yzVEp97zwto+3Tgzya +0l6bvRv4OSfdVeSTYx8T48h23O8FBD/Gp9l5sFOZgc1TCWrb7ReJQS+XThAksIdW +DLvwbOU1I2qRL3ZbT49FAhmVcrMkHJjzkugXDoGG3Rgdzx/HePUjXWdWC1L+7/Kn +U/7w72ymW1mC5PbjoW9zzkVKesj++mhzSb5+sXa/is3hUJ17zy4Bqc71Mb2q8tqM +G/uMrdwfPeoad3qRVPRsK8QlVnJ0eIpiUDk+Ow== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-4-key.pem b/deploy/dockerephemeral/docker/redis-node-4-key.pem new file mode 100644 index 00000000000..355661d6a99 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-4-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDKxOmFo5c4ae38 +qC3zI89R1F6xjaMyR6jjd6k5qsW7eU/y8+trgY4HV/jbzD3CDOjMkj70la5EiV+G +TA0iGeJH/BkKjqEPsIy/vAPux9xt2ZpIO9ieO2BF75ojrcM7tAbeOLQNAgYA7zAy +IpQkJ2P8IyOYSJ31ujLJCR7d0zudAbXJXfAAyPUWqUrmmRHIY7hRi1tUv74JARqn +U2tHZhFgGyBCaLROK69S/Wy+xPKo5w9Ol5L9eIccrK2/JwNpfsFAxJqXawNm1l1M +9gGk2MpQXzZeTg/hlusqCtPieOPUQKoEDXAgYArQy8iYkLuZzOtg2WwcPOhtfsgV +RLNEwXihAgMBAAECggEABRxG5XEc0dVro9tKQy9DHaUcWN3Av/bp5QfCSluJPcMe +Nnma1JwQjBNVyJZidRZVtLg34Xq3SG9s6qnWh+Y+m4FZUTiMiyRwO7HdqII9hkA+ +gPUPLdfBwql6CU2rFsFgDfBAa3aCV7ovjQftk2axwKxTDJbB8mxFtObnsgANp9SU +c+MTlNTs1IQ4ev4u1i9ntR8SlFMcYQUA2AxvOiEDu7b4x/Ph9TEGuR6wLxdImRq/ +7hXcPtGAJKYgZLzAwCrZrjGjHILSskTxdii+Tr52Aq75SA3tLYGkJfSxHTJjFe0u +1k4Ot4uSEjRf4DIwohbSFFbK/ZXG2uscn36OphtbUQKBgQDwQY263RPJ/M5mKvME +15DK1JW3DOLWCBiV0XzwXsS+QpE8pKs2YLeyrY7sV/w1tdnfNdfINCknuzC4tG7Y +I+QzCQGhyKrP2nj4K3SsKUcFk6OWxgiPF5CRmlWySJ+H6+yITKcSJt/ZjUvvGQyQ +TV+IQ8s4RbKII9Pvifai6SLJ2QKBgQDYDn4bqIfZKR0I46//AycGXAUl55Yfgeog +8CR5MatNz26crrmDzjnDgsRbKUxK+UZLl/zEXY5Npn06sOG1G0bO/t7wQqcPsXZt +rZTx58lKvW7LQhEBAz48y9QeK3WUvT1E3JMJ6rt+6IfHvbvCLIu9DwyGJ7Zc7N+6 +k5GduC9gCQKBgQC4Zdfd3+hcUwgnKjezM7ARvO/buqwvEa+s7UgzRMlELdtC7C/s +YHcdUFAt3anZn2VFCBJBuqcLs4RFf1bD1WhEM1lpTparSUcnUlMN//Beu14HTp8r +FC8FUasMVuj6bXzxb8ObDvMoCmaJcHRQHNKBx2amHfhUvQrhAsalasIkoQKBgAFo +XsP5XiE5FlpXeW8U6y0sblAn6R99bjQWvHYZr78LCfJ1ZPoJ3vB6KqNZaojWhPG7 +JMd2wJWa7xfxzRar/dMdcABqvsHoaxgd2GmXFAWrpEwouwmhpscooNItgE+eyAZp +1X9sCxqxkyjnAJEsTyDFN1Ssb5C9blu92GYJrC1ZAoGASChIICMp0HWrDSRxRCen +Fddf993aEI4e46NTWY54u2p0Ga62XUcaw5eND9QX6craD8nd7mMhwvdvQ5vuORBk +m+dqt0oU5cloVp0srHDA861CO8topJFaNGWdF4wDgLU8YKRzd6hNX8X0/CCRl1vd +z/YmtxfgU56SaqExe0X65eA= +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-4.conf b/deploy/dockerephemeral/docker/redis-node-4.conf new file mode 100644 index 00000000000..55b360f9f90 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-4.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6376 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-5-cert.pem b/deploy/dockerephemeral/docker/redis-node-5-cert.pem new file mode 100644 index 00000000000..e1221b9df77 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-5-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCSXrnTzrNfHudXjC0A +h0CiFRe3yp4j6cN3Hfv4snS6tPWXM4MH9Dka8zMLZvzRVQZK3PxDh/R/DQYBZhpy +LEvT7wYCDsS+F+tie2sPjzSAbdM5dolD8fGwACOqobI4vPz0QrwDqHde/OdVWAZl +h5Pzw5rDUu84CdfPSWRN1pomCFWG7gVkpuFzIcBfz+smPodyw3BfU8969q6tFACE +pjGPF/RufmHoIaHe2q/c+3HBY06ro0oTqTtRe36v4Jp2HLE/jE8wc+YggTmHE670 +uEXIR9N3fF3AbPVnhimEwcQ5fpJtMonUvfj5Z/4KfKo/0Yrh0wljeRiz/tZgTwwb +h5ATAgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy01hwSsFAAjMB0GA1UdDgQWBBRkbb1LScfQthztQJ3l +R+QFCKjXUTAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAn8TOFqomU30SmIDIHYBKMRGq3bVDLkDDC2yy6LCCwwG2rpoO +UtnUMig2w3iNQ6nvqR4LJB1ha0hLK5FP3iX/JcqZiO0NaucOTe7aJlt9taCADgAw +4vRW/pDuxtq7H1hc2pOue6i05UtGqy2E12jYowQc8a/5hylfEO3b5t5Z7xoQzyAZ +1ov7sYatBinwhqyDI5qNvZCuyT7SMx7H10T7cPrEec4uq55AJ0ReXnAAy1MLhpGd +nW5FX3F4gnyJcK2xL/V+ScL4NTzA8qWT+qOK33KxU1qrGripAkFaF6Z110nuIDiP +Z2tneIovCKKChgFsmZjy2spRpDw6R3Am6rXjpA== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-5-key.pem b/deploy/dockerephemeral/docker/redis-node-5-key.pem new file mode 100644 index 00000000000..467778629cd --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-5-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCSXrnTzrNfHudX +jC0Ah0CiFRe3yp4j6cN3Hfv4snS6tPWXM4MH9Dka8zMLZvzRVQZK3PxDh/R/DQYB +ZhpyLEvT7wYCDsS+F+tie2sPjzSAbdM5dolD8fGwACOqobI4vPz0QrwDqHde/OdV +WAZlh5Pzw5rDUu84CdfPSWRN1pomCFWG7gVkpuFzIcBfz+smPodyw3BfU8969q6t +FACEpjGPF/RufmHoIaHe2q/c+3HBY06ro0oTqTtRe36v4Jp2HLE/jE8wc+YggTmH +E670uEXIR9N3fF3AbPVnhimEwcQ5fpJtMonUvfj5Z/4KfKo/0Yrh0wljeRiz/tZg +Twwbh5ATAgMBAAECggEAFEPciJsaseBueUGQItLsZkRzVzVMtdOHW4uhjOpFnRVc +LLCrbe4opeGRaf0P+HpEIm38LewPNDP9ETPYv4FV3PmVTwhKbGNAFLovtXocnmzA +4jjWLRESEaMYombmwlJFghq8kJPCNeWKsIHnyNDU8YVd0mM+JE0V6GjUjq5YA0x4 +co87wiNxAtjdNuAmI8elOqH3YhCwCQjYO1NeEJwIhWz5tgb2J9Rvn85LOh65cU17 +FaxiTBNSrMW44yk+uhEyj8IDbcZax0s8gLbCSLIj/MnuSbm74VkGKXme0U3mJSmn +dY2tpO3DnNZ+qvakpUk50e/LXYFofsJH9cs1BlZ7AQKBgQDJufGOp07KcIG4N3ei +YxH1IRZ8vThOHksbKVnzQLRcJcEY6SHrL3DgxS+kO3IfjkKZGw5vZgV4/jfTfWQP +eDXwIl0t/YVCEDECppfAIN7fyvIVI14quRogbIrn0jn5ijhVzPI8SWvi/viFbFvn +2O/8KUaHudv9yQ6zKItZ1zHAkwKBgQC5wBLKYdeQT5EfvfXT+rHoioUyywFxTpOF +em14JfNwKLdhqEVB99MzGEdRs6HNz88YbhKQpuEQjwJkbBXZUpAXyYPLDN5uQtV7 +Xw1MY7d8O7U5qNevos+Yti8rrv4w8Cb8ppOX0DJ2SD7J4OQjuyiRYx6sE+tQH6p+ +6N2Gt9YigQKBgQCqpnt7s3uK9Aw42+t/2xFo7lnIooYMR8I/swaeKsGpJmMpAKep +/pMeApHf/E359e3O+b2HbaX5ig2OAwhvscDnaRqsekiN74aWeHntlaEVbujGCwpx +V++LOGd13zkeKdiodN0DNRVojUuOC3HgO3whNIWu8gLxuXGPDCB+mvZCswKBgH+I +vh4QgZYG22iE37U0ylQUT5HpSktGnQknXuQAgp1+hzJY+3xosKzDPax9/lk2FkX6 +xWpl+d+JoSXcBEBbbK24YXHXmxzvbG4xfAr36DI3OJ2nLLfdvFVouQhwNPza1pnf +sTSp8Qu/XMT1UQ6rYRY5jQSvBIDVzRUnw3nM3QyBAoGAXs5Mg1jcQme6X56e0Db0 +zDCcEJuYL+nWXSkClsQCaDwafi4PQVP/V351Qruw0n98grD5vacz1HdXosvCaACJ +8P8e4sFJmSGu8SQt4zbReq8DHNTWZyPC8muurnMSKtfg3XulY8SFsoog7dlMzGGY +IMDiEb5jIb6DFcpNxjigXsM= +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-5.conf b/deploy/dockerephemeral/docker/redis-node-5.conf new file mode 100644 index 00000000000..ba2cfde9c65 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-5.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6377 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-6-cert.pem b/deploy/dockerephemeral/docker/redis-node-6-cert.pem new file mode 100644 index 00000000000..c176eae043d --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-6-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGDCCAgCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRyZWRp +cy5jYS5leGFtcGxlLmNvbTAeFw0yNDA5MDMxMjAzMzlaFw0zNDA5MDExMjAzMzla +MAAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCmN9ktdBsuxTOPFUsU +qAjMnQSyBz/BpYDGMagy9e7PbtniVGTHHOvGgoq5VvPdtiVTerwefNAQaL3nLLvg +24hOEWBlQuBgK0gW48NPZJAbzYvNdF2jOzIzsDu8edEz4TcI8oKvw2WS5HQGl213 +06f2tMN1Ng0O07WoW8cxOYISsKVT9EyQJX4M/Oq5/nzHkXvS97ayFT0OvVdIRzPU +A6VsSyr/X1LgVmZEGfWcdv+cxJGBiXRsiWdW+Y+n6qvRBC2WpTEhCXomtbbDtuSH +e+8EXk9eKSc5QYFNCDWEMk25JuEQpXIMfdiHbMmK+9BgdRTUh8Pm94yD3hkMO6z5 +N5e3AgMBAAGjfjB8MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAbBgNV +HREBAf8EETAPggdyZWRpcy02hwSsFAAkMB0GA1UdDgQWBBRyt96xEM6o5VkG9JV5 +vLVxnBELSjAfBgNVHSMEGDAWgBSNcgRaq4sddGR0qWD3eVT168cULDANBgkqhkiG +9w0BAQsFAAOCAQEAmxFmsjenSgrrI1sE7DJahX1CaNVGodx4CwVc2etEq5PBWC6r +DpfCcYDW+Hg64Ac+NiPaLxFaG/8aM7JSePbAa71AQN+2hJpsV3/ANvSUaJfbHSFx +xfTRr8m5l33IV7ynjvZCPXWK4Gc5o7/shPKObHjwb03DLJjW0rvD5SYIjfCLjlOk +na2ufQnrmEP0XO77EvP4G/sHBjUaXrthsYTISO3lBTnGoKWNj8YwTFtXILC3O1to +sKWKYe5A6FB6xathUVBfS+Drp0PIYdAU9N3adymv4tZf52ofMsbJNkDqY3JaWmcO +dYHuYTeYg6ZiVhzZeasd3V+wc/CKAD8U5UfD5A== +-----END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/redis-node-6-key.pem b/deploy/dockerephemeral/docker/redis-node-6-key.pem new file mode 100644 index 00000000000..0bc3f366189 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-6-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCmN9ktdBsuxTOP +FUsUqAjMnQSyBz/BpYDGMagy9e7PbtniVGTHHOvGgoq5VvPdtiVTerwefNAQaL3n +LLvg24hOEWBlQuBgK0gW48NPZJAbzYvNdF2jOzIzsDu8edEz4TcI8oKvw2WS5HQG +l21306f2tMN1Ng0O07WoW8cxOYISsKVT9EyQJX4M/Oq5/nzHkXvS97ayFT0OvVdI +RzPUA6VsSyr/X1LgVmZEGfWcdv+cxJGBiXRsiWdW+Y+n6qvRBC2WpTEhCXomtbbD +tuSHe+8EXk9eKSc5QYFNCDWEMk25JuEQpXIMfdiHbMmK+9BgdRTUh8Pm94yD3hkM +O6z5N5e3AgMBAAECggEAQ088YB6zX0Y2McvyooPFRG6VVy5+UAGgWyICtdhHg7Kl +AvUf9k2s4K8+U/11NaQsC1kZUtNCQlLYDARedJkR4mNBAOCLEgaU48gJ8F2NyeR7 +p5Bm1tIC61GDbzh5UiPycGocJ+bdfBWNMpohlzObwdjDifSAZy+uUWYRDMr39G7x +9SH6aLL7ZHBg0Oc4dw6K4GQMrU7sdomQcSqNyi5sn6PN8FsuO1wMp8C8V+U6y5sb +36Y1rz90ZOFqmOBnG/IdPR8tFbdql1Yy31tzy/I4thK+1v4QN6JVLPvyw4H0RzFe +j347k5IsNehRdwltplhckeAUzWGGNiTx0zhQPAchuQKBgQDmyGB055GCtRoEIpqN +ANNa8PxTp2sCH+/J7KZma6gSJ9WY73xtGSVXX/Ubz4l8FHiGoA0CQCElARJ9zff/ +tAiNXqvcQeBPVC23CMJL3hxeHLNs0ipoD8qvdQpGit3DAZMjdjtt5jd48CulEmfP +/rVmeHKChZaPPR1EgrMnIytaCwKBgQC4YWygHnDjW9zekpsDRMKkvK5QMIey9ygB +LqXlXw6GANhVDGSr7zOHBtF1aBc6FA1FKlVRXz3Fag4pPZLd2HbEaKnzfCNPH5PL +UTX8fukftrzY03bvpYcr+/YabPO8H5hkeUqHyH9EyIgdj5hOhKEVj9kJkqENt3el +GvohkgdwhQKBgG0itPqTx6wYGIV8F7o2eby32Zt1wJTwpWTIFKi6oHB1hf0cw6qU +CaSYLEFKk6mpxJVlesFlskbdivETRgQWDzVLX9p5DKp3FGdKLRfToXaf+/mqKYOs +dB0lLAbQBK8DP6G1d8Uw6Wq3qOwXGCC0QvSCYSR4KAr0y7JqXG5Vo1qhAoGATLCh +GNxwgfDEpoL+HNbtys18B3iYCLVKm2tGr2fhR5V0ZbOY7/a3TPNmDdp0xsBuYJVi +FU1zCPi62SZ2PvX5OGp8Pf0lRpTQyWGG/fXfi0RbuigCsVz9IytSyt0EZ/wQS8Iz +YNThMr/h9cGzTP1Xbvt8/8FQYb8s8ayN24a8t20CgYBDyjVifJHw6iVl3vu/O+R9 ++AdSe5bEGGDuIKZRDJbEj2ScgD3Nwqdst7X5wC+rcUuyJNW22GihyLiC/+OCaJPl +9fyaRpWWjEUkzpvR+3GhzzykDnemw1z39AJrg3ewSaBdbw9Bvq0ebrGaDF+uCReY +V+yVEYFsBaK0JrbkIffXbA== +-----END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/redis-node-6.conf b/deploy/dockerephemeral/docker/redis-node-6.conf new file mode 100644 index 00000000000..2989c5550ea --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-node-6.conf @@ -0,0 +1,17 @@ +port 0 +tls-port 6378 +tls-cert-file /usr/local/etc/redis/cert.pem +tls-key-file /usr/local/etc/redis/key.pem +tls-ca-cert-file /usr/local/etc/redis/ca.pem +tls-auth-clients no +tls-cluster yes +tls-replication yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +appendonly yes + +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/docs/src/developer/developer/building.md b/docs/src/developer/developer/building.md index 4cbdba83822..bcbca56502c 100644 --- a/docs/src/developer/developer/building.md +++ b/docs/src/developer/developer/building.md @@ -156,6 +156,7 @@ These services require most of the deployment dependencies as seen in the archit - Required internal dependencies: - cassandra (with the correct schema) - elasticsearch (with the correct schema) + - redis - Required external dependencies are the following configured AWS services (or “fake” replacements providing the same API): - SES - SQS diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index 9154c97ea73..062f891f03c 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -1983,6 +1983,96 @@ elasticsearch-index: insecureSkipVerifyTls: true ``` +## Configure Redis authentication + +If the redis used needs authentication with either username and password or just +password (legacy auth), it can be configured like this: + +```yaml +gundeck: + secrets: + redisUsername: + redisPassword: +``` + +**NOTE**: When using redis < 6, the `redisUsername` must not be set at all (not +even set to `null` or empty string, the key must be absent from the config). +When using redis >= 6 and using legacy auth, the `redisUsername` must either be +not set at all or set to `"default"`. + +While doing migrations to another redis instance, the credentials for the +addtional redis can be set as follows: + +```yaml +gundeck: + secrets: + redisAdditionalWriteUsername: # Do not set this at all when using legacy auth + redisAdditionalWritePassword: +``` + +**NOTE**: `redisAddtiionalWriteUsername` follows same restrictions as +`redisUsername` when using legacy auth. + +## Configure TLS for Redis + +If the redis instance requires TLS, it can be configured like this: + +```yaml +gundeck: + config: + redis: + enableTls: true +``` + +In case a custom CA certificate is required it can be provided like this: + +```yaml +gundeck: + config: + redis: + tlsCa: +``` + +There is another way to provide this, in case there already exists a kubernetes +secret containing the CA certificate(s): + +```yaml +gundeck: + config: + redis: + tlsCaSecretRef: + name: + key: +``` + +For configuring `redisAdditionalWrite` in gundeck (this is required during a +migration from one redis instance to another), the settings need to be like +this: + +```yaml +gundeck: + config: + redisAdditionalWrite: + enableTls: true + # One or none of these: + # tlsCa: + # tlsCaSecretRef: +``` + +**WARNING:** Please do this only if you know what you’re doing. + +In case it is not possible to verify TLS certificate of the redis +server, it can be turned off without tuning off TLS like this: + +```yaml +gundeck: + config: + redis: + insecureSkipVerifyTls: true + redisAdditionalWrite: + insecureSkipVerifyTls: true +``` + ## Configure RabbitMQ RabbitMQ authentication must be configured on brig, galley and background-worker. For example: @@ -2015,7 +2105,7 @@ server, verification can be turned off by settings `insecureSkipVerifyTls` to ## Configure PostgreSQL -`brig`, `galley`, `gundeck`, and `background-worker` require a PostgreSQL database. The configured user needs to +`brig`, `galley`, and `background-worker` require a PostgreSQL database. The configured user needs to be able to write data and change the schema (e.g. create and alter tables.) The internal configuration YAML file format and the Helm charts for `brig` and @@ -2072,7 +2162,7 @@ The `port` needs to be a number provided as string. Besides the password file (`postgresqlPassword`), the fields correspond to [libpq-connect parameters](https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS). -The `postgresqlPassword` file is read by `brig`, `galley`, `gundeck`, and `background-worker`. Its content is +The `postgresqlPassword` file is read by `brig`, `galley`, and `background-worker`. Its content is used as `password` field. ### Using PostgreSQL for storing Cassandra-backed data diff --git a/docs/src/how-to/install/infrastructure-configuration.md b/docs/src/how-to/install/infrastructure-configuration.md index 06ef7a218d1..34e1eb2c19d 100644 --- a/docs/src/how-to/install/infrastructure-configuration.md +++ b/docs/src/how-to/install/infrastructure-configuration.md @@ -27,6 +27,7 @@ gundeck: - "10.0.0.0/8" - "elasticsearch-external" - "cassandra-external" + - "redis-ephemeral" - "fake-aws-sqs" - "fake-aws-dynamodb" - "fake-aws-sns" @@ -414,8 +415,8 @@ cassandra cannot reliably be installed on kubernetes. Some people have tried, e.g. [this project](https://github.com/instaclustr/cassandra-operator) though at the time of writing (Nov 2018), this does not yet work as advertised. We -recommend therefore to install cassandra, (possibly also elasticsearch) -separately, i.e. outside of kubernetes (using 3 nodes each). +recommend therefore to install cassandra, (possibly also elasticsearch +and redis) separately, i.e. outside of kubernetes (using 3 nodes each). For further higher-availability: diff --git a/docs/src/how-to/install/troubleshooting.md b/docs/src/how-to/install/troubleshooting.md index 3703500a9fd..ecb0ffb636a 100644 --- a/docs/src/how-to/install/troubleshooting.md +++ b/docs/src/how-to/install/troubleshooting.md @@ -252,7 +252,7 @@ These are some steps you can take to debug what is going on when the installatio As an example, we’ll take a case where we try installing `wire-server` with `helm`, but it fails due to `cassandra` being broken in some way. -This guide, while focusing on a `cassandra` related issue, will also provide general steps to debug problems that could be related to other components like `rabbitmq`, etc. +This guide, while focusing on a `cassandra` related issue, will also provide general steps to debug problems that could be related to other components like `rabbitmq`, `redis`, etc. Our first step is to identify and isolate which component is causing the issue. @@ -266,6 +266,7 @@ fake-aws-sns-76fb45cf4f-t6mg6 2/2 Running 0 75m fake-aws-sqs-6495cd7c98-w8f8w 2/2 Running 0 75m rabbitmq-external-0 0/1 Pending 0 78m reaper-84cfbf746d-wk8nc 1/1 Running 0 75m +redis-ephemeral-master-0 1/1 Running 0 76m ``` We then run the `wire-server` helm installation command: @@ -293,6 +294,7 @@ fake-aws-sns-76fb45cf4f-t6mg6 2/2 Running 0 95m fake-aws-sqs-6495cd7c98-w8f8w 2/2 Running 0 95m rabbitmq-external-0 0/1 Pending 0 98m reaper-84cfbf746d-wk8nc 1/1 Running 0 95m +redis-ephemeral-master-0 1/1 Running 0 96m ``` (You can also do `d kubectl get pods -o wide` to get more details though that’s not necessary here) diff --git a/hack/bin/gen-certs.sh b/hack/bin/gen-certs.sh index d4840f8af6d..f995a238aaa 100755 --- a/hack/bin/gen-certs.sh +++ b/hack/bin/gen-certs.sh @@ -81,6 +81,19 @@ install_certs "$TEMP/es" "$ROOT_DIR/deploy/dockerephemeral/docker" \ install_certs "$TEMP/es" "$ROOT_DIR/hack/helm_vars/certs" \ elasticsearch-ca elasticsearch-ca-key +# redis +mkdir -p "$TEMP/redis" +gen_ca "$TEMP/redis" redis.ca.example.com +REDIS="$ROOT_DIR/deploy/dockerephemeral/docker" +cp "$TEMP/redis/ca.pem" "$REDIS/redis-ca.pem" +for redis_node in $(seq 1 6); do + gen_cert "$TEMP/redis" "DNS:redis-${redis_node}, IP:172.20.0.3${redis_node}" + chmod 0644 "$TEMP/redis/key.pem" + install_certs "$TEMP/redis" "$REDIS" "" "" \ + "redis-node-${redis_node}-cert" \ + "redis-node-${redis_node}-key" +done + # rabbitmq RABBITMQ="$ROOT_DIR/deploy/dockerephemeral/rabbitmq-config/certificates" gen_ca "$RABBITMQ" rabbitmq.ca.example.com diff --git a/hack/helm_vars/certs/values.yaml.gotmpl b/hack/helm_vars/certs/values.yaml.gotmpl index 7cd8a633653..307d50fa48a 100644 --- a/hack/helm_vars/certs/values.yaml.gotmpl +++ b/hack/helm_vars/certs/values.yaml.gotmpl @@ -16,6 +16,59 @@ resources: ca: secretName: elasticsearch-ca + # redis CA and certificate + - apiVersion: cert-manager.io/v1 + kind: Issuer + metadata: + name: redis-ca-issuer + namespace: '{{ .Release.Namespace }}' + spec: + selfSigned: {} + - apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: redis-ca + namespace: '{{ .Release.Namespace }}' + spec: + secretName: redis-ca-certificate + isCA: true + duration: 2160h # 90d + renewBefore: 360h # 15d + commonName: redis.example.com + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + issuerRef: + name: redis-ca-issuer + kind: Issuer + - apiVersion: cert-manager.io/v1 + kind: Issuer + metadata: + name: redis-issuer + namespace: '{{ .Release.Namespace }}' + spec: + ca: + secretName: redis-ca-certificate + - apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: redis + namespace: '{{ .Release.Namespace }}' + spec: + secretName: redis-certificate + isCA: false + duration: 2160h # 90d + renewBefore: 360h # 15d + commonName: redis-ephemeral + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + issuerRef: + name: redis-issuer + kind: Issuer + # RabbitMQ CA and certificate - apiVersion: cert-manager.io/v1 kind: Issuer diff --git a/hack/helm_vars/redis-ephemeral/values.yaml b/hack/helm_vars/redis-ephemeral/values.yaml new file mode 100644 index 00000000000..996dc30e45c --- /dev/null +++ b/hack/helm_vars/redis-ephemeral/values.yaml @@ -0,0 +1,47 @@ +redis-ephemeral: + image: + registry: public.ecr.aws + repository: docker/library/redis + + redisConfig: | + requirepass very-secure-redis-master-password + + # ephemeral + save "" + + port 0 + tls-port 6379 + tls-cert-file /data/ssl/tls.crt + tls-key-file /data/ssl/tls.key + tls-ca-cert-file /data/ssl/ca.crt + tls-auth-clients no + + extraRedisSecrets: + - name: redis-certificate + mountPath: /data/ssl + + livenessProbe: + enabled: false + customLivenessProbe: + exec: + command: + - sh + - -c + - redis-cli --tls --cacert /data/ssl/ca.crt ping + readinessProbe: + enabled: false + customReadinessProbe: + exec: + command: + - sh + - -c + - redis-cli --tls --cacert /data/ssl/ca.crt ping + startupProbe: + enabled: false + customStartupProbe: + exec: + command: + - sh + - -c + - redis-cli --tls --cacert /data/ssl/ca.crt ping + diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index 43373b1cf2e..f05d42a98e4 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -492,11 +492,13 @@ gundeck: tlsCaSecretRef: name: "rabbitmq-certificate" key: "ca.crt" - postgresql: - host: "postgresql" - port: "5432" - user: wire-server - dbname: wire-server + redis: + host: redis-ephemeral + connectionMode: master + enableTls: true + tlsCaSecretRef: + name: "redis-certificate" + key: "ca.crt" aws: account: "123456789012" region: eu-west-1 @@ -512,7 +514,7 @@ gundeck: secrets: awsKeyId: dummykey awsSecretKey: dummysecret - pgPassword: posty-the-gres + redisPassword: very-secure-redis-master-password rabbitmq: username: {{ .Values.rabbitmqUsername }} password: {{ .Values.rabbitmqPassword }} @@ -527,6 +529,7 @@ gundeck: uploadXmlAwsAccessKeyId: {{ .Values.uploadXml.awsAccessKeyId }} uploadXmlAwsSecretAccessKey: {{ .Values.uploadXml.awsSecretAccessKey }} {{- end }} + redisAdditionalWritePassword: very-secure-redis-master-password-2 nginz: replicaCount: 1 @@ -726,6 +729,10 @@ integration: tlsCaSecretRef: name: {{ .Values.elasticsearch.caSecretName }} key: "ca.crt" + redis: + tlsCaSecretRef: + name: "redis-certificate" + key: "ca.crt" rabbitmq: tlsCaSecretRef: name: "rabbitmq-certificate" @@ -739,6 +746,7 @@ integration: uploadXmlAwsAccessKeyId: {{ .Values.uploadXml.awsAccessKeyId }} uploadXmlAwsSecretAccessKey: {{ .Values.uploadXml.awsSecretAccessKey }} {{- end }} + redisPassword: very-secure-redis-master-password tls: caNamespace: wire-federation-v0 diff --git a/hack/helmfile.yaml.gotmpl b/hack/helmfile.yaml.gotmpl index f28ed995af5..09c825f9c31 100644 --- a/hack/helmfile.yaml.gotmpl +++ b/hack/helmfile.yaml.gotmpl @@ -114,6 +114,15 @@ releases: values: - './helm_vars/certs/values.yaml.gotmpl' + - name: 'redis-ephemeral' + namespace: '{{ .Values.namespace1 }}' + chart: '../.local/charts/redis-ephemeral' + values: + - './helm_vars/wire-image-mirror.yaml' + - './helm_vars/redis-ephemeral/values.yaml' + needs: + - certs + - name: 'cassandra-ephemeral' namespace: '{{ .Values.namespace1 }}' chart: '../.local/charts/cassandra-ephemeral' @@ -138,6 +147,15 @@ releases: name: elasticsearch kind: Issuer + - name: 'redis-ephemeral' + namespace: '{{ .Values.namespace2 }}' + chart: '../.local/charts/redis-ephemeral' + values: + - './helm_vars/wire-image-mirror.yaml' + - './helm_vars/redis-ephemeral/values.yaml' + needs: + - certs + - name: 'cassandra-ephemeral' namespace: '{{ .Values.namespace2 }}' chart: '../.local/charts/cassandra-ephemeral' @@ -198,6 +216,22 @@ releases: values: - './helm_vars/opensearch/values.yaml.gotmpl' + # Required for testing redis migration + - name: redis-ephemeral-2 + namespace: '{{ .Values.namespace1 }}' + chart: '../.local/charts/redis-ephemeral' + values: + - redis-ephemeral: + image: + registry: public.ecr.aws + repository: docker/library/redis + + redisConfig: | + requirepass very-secure-redis-master-password-2 + + # ephemeral + save "" + - name: 'certs' namespace: '{{ .Values.namespace2 }}' chart: bedag/raw @@ -324,6 +358,7 @@ releases: value: true needs: - 'cassandra-ephemeral' + - 'redis-ephemeral' - 'postgresql' - name: 'wire-server' @@ -341,6 +376,7 @@ releases: value: {{ .Values.federationDomain2 }} needs: - 'cassandra-ephemeral' + - 'redis-ephemeral' - 'postgresql' - name: wire-server-enterprise diff --git a/libs/wire-api/src/Wire/API/Presence.hs b/libs/wire-api/src/Wire/API/Presence.hs index 9ec538f064f..427e0a0f8af 100644 --- a/libs/wire-api/src/Wire/API/Presence.hs +++ b/libs/wire-api/src/Wire/API/Presence.hs @@ -23,6 +23,7 @@ import Data.Aeson.Types qualified as A import Data.Attoparsec.ByteString (takeByteString) import Data.ByteString.Char8 qualified as Bytes import Data.ByteString.Conversion +import Data.ByteString.Lazy qualified as Lazy import Data.Id import Data.Misc (Milliseconds) import Data.OpenApi qualified as S @@ -34,6 +35,7 @@ import Imports import Network.URI qualified as Net import Servant.API (ToHttpApiData (toUrlPiece)) +-- FUTUREWORK: use Network.URI and toss this newtype. servant should have all these instances for us these days. newtype URI = URI { fromURI :: Net.URI } @@ -75,7 +77,9 @@ data Presence = Presence -- operating the team settings pages without the need for -- end-to-end crypto. clientId :: !(Maybe ClientId), - createdAt :: !Milliseconds + createdAt :: !Milliseconds, + -- | REFACTOR: temp. addition to ease migration + __field :: !Lazy.ByteString } deriving (Eq, Ord, Show) deriving (A.FromJSON, A.ToJSON, S.ToSchema) via (Schema Presence) @@ -90,6 +94,7 @@ instance ToSchema Presence where <*> clientId .= optField "client_id" (maybeWithDefault A.Null schema) -- keep null for backwards compat <*> createdAt .= (fromMaybe 0 <$> (optField "created_at" schema)) ) + <&> ($ ("" :: Lazy.ByteString)) uriSchema :: ValueSchema NamedSwaggerDoc URI uriSchema = mkSchema desc uriFromJSON (Just . uriToJSON) diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Manual/Presence.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Manual/Presence.hs index adb5f582d44..97005af0a0d 100644 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Manual/Presence.hs +++ b/libs/wire-api/test/golden/Test/Wire/API/Golden/Manual/Presence.hs @@ -35,6 +35,7 @@ testObject_Presence_1 = (fromJust $ parse "http://example.com/") Nothing 0 + "" testObject_Presence_2 :: Presence testObject_Presence_2 = @@ -44,6 +45,7 @@ testObject_Presence_2 = (fromJust $ parse "http://example.com/3") (Just (ClientId 1)) 12323 + "" -- __field always has to be "", see ToSchema instance. testObject_Presence_3 :: Presence testObject_Presence_3 = @@ -53,3 +55,4 @@ testObject_Presence_3 = (fromJust $ parse "http://example.com/3") (Just (ClientId 1)) 0 + "" -- __field always has to be "", see ToSchema instance. diff --git a/libs/wire-subsystems/postgres-migrations/20260828093750-gundeck-presence.sql b/libs/wire-subsystems/postgres-migrations/20260828093750-gundeck-presence.sql deleted file mode 100644 index eb84c4ed7f1..00000000000 --- a/libs/wire-subsystems/postgres-migrations/20260828093750-gundeck-presence.sql +++ /dev/null @@ -1,12 +0,0 @@ --- WPB-28377: gundeck presence (replaces redis presence hashes) -CREATE TABLE IF NOT EXISTS presence ( - user_id uuid NOT NULL, - conn_id text NOT NULL, - resource text NOT NULL, - client_id text, - created_at timestamptz NOT NULL, - PRIMARY KEY (user_id, conn_id) -); - --- index for cleanup deletes like `DELETE ... WHERE created_at < now() - interval '7 days'` -CREATE INDEX presence_created_at_idx ON presence (created_at); diff --git a/libs/wire-subsystems/src/Wire/JobSubsystem/Migrations.hs b/libs/wire-subsystems/src/Wire/JobSubsystem/Migrations.hs index 2410cb98c65..920782cc856 100644 --- a/libs/wire-subsystems/src/Wire/JobSubsystem/Migrations.hs +++ b/libs/wire-subsystems/src/Wire/JobSubsystem/Migrations.hs @@ -19,13 +19,11 @@ -- with this program. If not, see . module Wire.JobSubsystem.Migrations - ( defaultSchemaName, - mkArbiterConnectionString, + ( mkArbiterConnectionString, runJobMigrations, ) where -import Arbiter.Core (defaultSchemaName) import Arbiter.Migrations qualified as ArbiterMigrations import Control.Exception (bracket, bracket_, throwIO) import Data.Hashable qualified as Hashable diff --git a/libs/wire-subsystems/src/Wire/Postgres.hs b/libs/wire-subsystems/src/Wire/Postgres.hs index a0210c21cf8..f91f3637efe 100644 --- a/libs/wire-subsystems/src/Wire/Postgres.hs +++ b/libs/wire-subsystems/src/Wire/Postgres.hs @@ -43,7 +43,6 @@ module Wire.Postgres runTransaction, runTransactionWithRetry, runPipeline, - useWithResetAndRetry, parseCount, PGConstraints, diff --git a/postgres-schema.sql b/postgres-schema.sql index 2d3df5fb27f..de4a57a0f2e 100644 --- a/postgres-schema.sql +++ b/postgres-schema.sql @@ -1511,21 +1511,6 @@ CREATE TABLE public.mls_history_client ( ALTER TABLE public.mls_history_client OWNER TO "wire-server"; --- --- Name: presence; Type: TABLE; Schema: public; Owner: wire-server --- - -CREATE TABLE public.presence ( - user_id uuid NOT NULL, - conn_id text NOT NULL, - resource text NOT NULL, - client_id text, - created_at timestamp with time zone NOT NULL -); - - -ALTER TABLE public.presence OWNER TO "wire-server"; - -- -- Name: remote_conversation_local_member; Type: TABLE; Schema: public; Owner: wire-server -- @@ -1982,14 +1967,6 @@ ALTER TABLE ONLY public.mls_history_client ADD CONSTRAINT mls_history_client_pkey PRIMARY KEY (group_id, id); --- --- Name: presence presence_pkey; Type: CONSTRAINT; Schema: public; Owner: wire-server --- - -ALTER TABLE ONLY public.presence - ADD CONSTRAINT presence_pkey PRIMARY KEY (user_id, conn_id); - - -- -- Name: remote_conversation_local_member remote_conversation_local_member_pkey; Type: CONSTRAINT; Schema: public; Owner: wire-server -- @@ -2462,13 +2439,6 @@ CREATE INDEX idx_meetings_recurrence_eff_end ON public.meetings USING btree (GRE CREATE INDEX idx_meetings_start_time ON public.meetings USING btree (start_time); --- --- Name: presence_created_at_idx; Type: INDEX; Schema: public; Owner: wire-server --- - -CREATE INDEX presence_created_at_idx ON public.presence USING btree (created_at); - - -- -- Name: user_group_member_user_id_idx; Type: INDEX; Schema: public; Owner: wire-server -- diff --git a/services/brig/src/Brig/Run.hs b/services/brig/src/Brig/Run.hs index 20365d1ebb9..fd170bdbaff 100644 --- a/services/brig/src/Brig/Run.hs +++ b/services/brig/src/Brig/Run.hs @@ -69,7 +69,6 @@ import Wire.API.Routes.Version import Wire.API.Routes.Version.Wai import Wire.API.User (AccountStatus (PendingInvitation)) import Wire.DeleteQueue -import Wire.JobSubsystem.Migrations (defaultSchemaName, mkArbiterConnectionString, runJobMigrations) import Wire.OpenTelemetry (withTracer) import Wire.PostgresMigrations import Wire.Sem.Paging qualified as P @@ -118,10 +117,6 @@ migratePostgres opts resetFirst = do pool <- (.rawPool) <$> initPostgresPool opts.postgresqlPool opts.postgresql opts.postgresqlPassword when resetFirst $ resetSchema pool logger runAllMigrations pool logger - -- Also create the arbiter job schema, so that this command yields the full - -- database schema (e.g. for `make postgres-schema`). - arbiterConnStr <- mkArbiterConnectionString opts.postgresql opts.postgresqlPassword - runJobMigrations arbiterConnStr defaultSchemaName flush logger mkApp :: Opts -> IO (Wai.Application, Env) diff --git a/services/gundeck/default.nix b/services/gundeck/default.nix index 5de25d7fff5..2e4f8b69d5f 100644 --- a/services/gundeck/default.nix +++ b/services/gundeck/default.nix @@ -22,14 +22,14 @@ , conduit , containers , criterion +, crypton-x509-store , data-timeout , errors , exceptions , extended , extra , foldl -, hasql -, hasql-th +, hedis , hs-opentelemetry-instrumentation-wai , hs-opentelemetry-sdk , HsOpenSSL @@ -37,6 +37,7 @@ , http-client-tls , http-types , imports +, kan-extensions , lens , lens-aeson , lib @@ -45,6 +46,7 @@ , MonadRandom , mtl , multiset +, network , network-uri , optparse-applicative , prometheus-client @@ -77,7 +79,6 @@ , unliftio , unordered-containers , uuid -, vector , wai , wai-extra , wai-middleware-gunzip @@ -85,7 +86,6 @@ , websockets , wire-api , wire-otel -, wire-subsystems , yaml }: mkDerivation { @@ -110,14 +110,14 @@ mkDerivation { bytestring-conversion cassandra-util containers + crypton-x509-store data-timeout errors exceptions extended extra foldl - hasql - hasql-th + hedis hs-opentelemetry-instrumentation-wai hs-opentelemetry-sdk http-client @@ -148,14 +148,12 @@ mkDerivation { unliftio unordered-containers uuid - vector wai wai-extra wai-middleware-gunzip wai-utilities wire-api wire-otel - wire-subsystems yaml ]; executableHaskellDepends = [ @@ -175,8 +173,10 @@ mkDerivation { http-client http-client-tls imports + kan-extensions lens lens-aeson + network network-uri optparse-applicative random @@ -191,6 +191,7 @@ mkDerivation { tinylog types-common uuid + wai-utilities websockets wire-api yaml diff --git a/services/gundeck/gundeck.cabal b/services/gundeck/gundeck.cabal index 9ad88362bc4..06bf1b5024f 100644 --- a/services/gundeck/gundeck.cabal +++ b/services/gundeck/gundeck.cabal @@ -39,6 +39,7 @@ library Gundeck.Push.Native.Types Gundeck.Push.Websocket Gundeck.React + Gundeck.Redis Gundeck.Run Gundeck.Schema.Run Gundeck.Schema.V1 @@ -57,6 +58,7 @@ library Gundeck.ThreadBudget.Internal Gundeck.Util Gundeck.Util.DelayQueue + Gundeck.Util.Redis other-modules: Paths_gundeck hs-source-dirs: src @@ -124,14 +126,14 @@ library , bytestring-conversion >=0.2 , cassandra-util >=0.16.2 , containers >=0.5 + , crypton-x509-store , data-timeout , errors >=2.0 , exceptions >=0.4 , extended , extra >=1.1 , foldl - , hasql - , hasql-th + , hedis >=0.14.0 , hs-opentelemetry-instrumentation-wai , hs-opentelemetry-sdk , http-client >=0.7 @@ -162,14 +164,12 @@ library , unliftio >=0.2 , unordered-containers >=0.2 , uuid >=1.3 - , vector , wai >=3.2 , wai-extra >=3.0 , wai-middleware-gunzip >=0.0.2 , wai-utilities >=0.16 , wire-api , wire-otel - , wire-subsystems , yaml >=0.8 default-language: GHC2021 @@ -244,6 +244,7 @@ executable gundeck-integration Metrics Paths_gundeck TestSetup + Util hs-source-dirs: test/integration default-extensions: @@ -296,7 +297,7 @@ executable gundeck-integration build-depends: , aeson , async - , base >=4 && <5 + , base >=4 && <5 , base16-bytestring >=0.1 , bilge , bytestring @@ -309,8 +310,10 @@ executable gundeck-integration , http-client , http-client-tls , imports + , kan-extensions , lens , lens-aeson + , network , network-uri , optparse-applicative , random @@ -324,6 +327,7 @@ executable gundeck-integration , tinylog , types-common , uuid + , wai-utilities >=0.16 , websockets >=0.8 , wire-api , yaml diff --git a/services/gundeck/gundeck.integration.yaml b/services/gundeck/gundeck.integration.yaml index bb19ab89eb7..00c80574794 100644 --- a/services/gundeck/gundeck.integration.yaml +++ b/services/gundeck/gundeck.integration.yaml @@ -13,17 +13,18 @@ cassandra: keyspace: gundeck_test # filterNodesByDatacentre: datacenter1 -postgresql: - host: 127.0.0.1 - port: "5432" - user: wire-server - dbname: backendA - password: posty-the-gres - -postgresqlPool: - size: 20 - acquisitionTimeout: 10s - idlenessTimeout: 10m +redis: + host: 172.20.0.31 + port: 6373 + connectionMode: cluster # master | cluster + enableTls: true + tlsCa: ../../deploy/dockerephemeral/docker/redis-ca.pem + insecureSkipVerifyTls: false + +# redisAdditionalWrite: +# host: 127.0.0.1 +# port: 6379 +# connectionMode: master aws: queueName: integration-gundeck-events diff --git a/services/gundeck/src/Gundeck/Env.hs b/services/gundeck/src/Gundeck/Env.hs index a7f233fb2c0..39f6f98bda7 100644 --- a/services/gundeck/src/Gundeck/Env.hs +++ b/services/gundeck/src/Gundeck/Env.hs @@ -23,19 +23,30 @@ import Bilge hiding (host, port) import Cassandra (ClientState) import Cassandra.Util (initCassandraForService) import Control.AutoUpdate +import Control.Concurrent.Async (Async) import Control.Lens (makeLenses, (^.)) +import Control.Retry (capDelay, exponentialBackoff) +import Data.ByteString.Char8 qualified as BSChar8 import Data.Id import Data.Misc (Milliseconds (..)) +import Data.Text qualified as Text +import Data.Time.Clock import Data.Time.Clock.POSIX +import Data.X509.CertificateStore as CertStore +import Database.Redis qualified as Redis import Gundeck.Aws qualified as Aws -import Gundeck.Options +import Gundeck.Options as Opt hiding (host, port) +import Gundeck.Options qualified as O +import Gundeck.Redis qualified as Redis import Gundeck.ThreadBudget -import Hasql.Pool.Extended qualified as HasqlPoolExt import Imports import Network.AMQP (Channel) import Network.AMQP.Extended qualified as Q import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.TLS (tlsManagerSettings) +import Network.TLS as TLS +import Network.TLS.Extra qualified as TLS +import System.Logger qualified as Log import System.Logger.Extended qualified as Logger data Env = Env @@ -44,7 +55,8 @@ data Env = Env _applog :: !Logger.Logger, _manager :: !Manager, _cstate :: !ClientState, - _hasqlPool :: !HasqlPoolExt.Pool, + _rstate :: !Redis.RobustConnection, + _rstateAdditionalWrite :: !(Maybe Redis.RobustConnection), _awsEnv :: !Aws.Env, _time :: !(IO Milliseconds), _threadBudgetState :: !(Maybe ThreadBudgetState), @@ -53,7 +65,7 @@ data Env = Env makeLenses ''Env -createEnv :: Opts -> IO Env +createEnv :: Opts -> IO ([Async ()], Env) createEnv o = do l <- Logger.mkLogger (o ^. logLevel) (o ^. logNetStrings) (o ^. logFormat) n <- @@ -64,7 +76,17 @@ createEnv o = do managerResponseTimeout = responseTimeoutMicro 5000000 } - pgPool <- HasqlPoolExt.initPostgresPool (o ^. postgresqlPool) (o ^. postgresql) (o ^. postgresqlPassword) + redisUsername <- BSChar8.pack <$$> lookupEnv "REDIS_USERNAME" + redisPassword <- BSChar8.pack <$$> lookupEnv "REDIS_PASSWORD" + (rThread, r) <- createRedisPool l (o ^. redis) redisUsername redisPassword "main-redis" + + (rAdditionalThreads, rAdditional) <- case o ^. redisAdditionalWrite of + Nothing -> pure ([], Nothing) + Just additionalRedis -> do + additionalRedisUsername <- BSChar8.pack <$$> lookupEnv "REDIS_ADDITIONAL_WRITE_USERNAME" + addtionalRedisPassword <- BSChar8.pack <$$> lookupEnv "REDIS_ADDITIONAL_WRITE_PASSWORD" + (rAddThread, rAdd) <- createRedisPool l additionalRedis additionalRedisUsername addtionalRedisPassword "additional-write-redis" + pure ([rAddThread], Just rAdd) p <- initCassandraForService @@ -82,8 +104,55 @@ createEnv o = do } mtbs <- mkThreadBudgetState `mapM` (o ^. settings . maxConcurrentNativePushes) rabbitMqChannelMVar <- Q.mkRabbitMqChannelMVar l (Just "gundeck") (o ^. rabbitmq) - pure $! Env (RequestId defRequestId) o l n p pgPool a io mtbs rabbitMqChannelMVar + pure $! (rThread : rAdditionalThreads,) $! Env (RequestId defRequestId) o l n p r rAdditional a io mtbs rabbitMqChannelMVar reqIdMsg :: RequestId -> Logger.Msg -> Logger.Msg reqIdMsg = ("request" Logger..=) . unRequestId {-# INLINE reqIdMsg #-} + +createRedisPool :: Logger.Logger -> RedisEndpoint -> Maybe ByteString -> Maybe ByteString -> ByteString -> IO (Async (), Redis.RobustConnection) +createRedisPool l ep username password identifier = do + customCertStore <- case ep._tlsCa of + Nothing -> pure Nothing + Just caPath -> CertStore.readCertificateStore caPath + let defClientParams = defaultParamsClient (Text.unpack ep._host) "" + tlsParams = + guard ep._enableTls + $> defClientParams + { clientHooks = + if ep._insecureSkipVerifyTls + then defClientParams.clientHooks {onServerCertificate = \_ _ _ _ -> pure []} + else defClientParams.clientHooks, + clientShared = + case customCertStore of + Nothing -> defClientParams.clientShared + Just sharedCAStore -> defClientParams.clientShared {sharedCAStore}, + clientSupported = + defClientParams.clientSupported + { supportedVersions = [TLS.TLS13, TLS.TLS12], + supportedCiphers = TLS.ciphersuite_strong + } + } + let redisConnInfo = + Redis.defaultConnectInfo + { Redis.connectAddr = Redis.ConnectAddrHostPort (Text.unpack ep._host) (fromIntegral ep._port), + Redis.connectUsername = username, + Redis.connectAuth = password, + Redis.connectTimeout = Just (secondsToNominalDiffTime 5), + Redis.connectMaxConnections = 100, + Redis.connectTLSParams = tlsParams + } + + Log.info l $ + Log.msg (Log.val $ "starting connection to " <> identifier <> "...") + . Log.field "connectionMode" (show $ ep ^. O.connectionMode) + . Log.field "connInfo" (safeShowConnInfo redisConnInfo) + let connectWithRetry = Redis.connectRobust l (capDelay 1000000 (exponentialBackoff 50000)) + r <- case ep ^. O.connectionMode of + Master -> connectWithRetry $ Redis.checkedConnect redisConnInfo + Cluster -> connectWithRetry $ Redis.checkedConnectCluster redisConnInfo + Log.info l $ Log.msg (Log.val $ "Established connection to " <> identifier <> ".") + pure r + +safeShowConnInfo :: Redis.ConnectInfo -> String +safeShowConnInfo connInfo = show $ connInfo {Redis.connectAuth = "[REDACTED]" <$ Redis.connectAuth connInfo} diff --git a/services/gundeck/src/Gundeck/Monad.hs b/services/gundeck/src/Gundeck/Monad.hs index db3eda96184..832bff5d890 100644 --- a/services/gundeck/src/Gundeck/Monad.hs +++ b/services/gundeck/src/Gundeck/Monad.hs @@ -33,6 +33,10 @@ module Gundeck.Monad runGundeck, posixTime, getRabbitMqChan, + + -- * Select which redis to target + runWithDefaultRedis, + runWithAdditionalRedis, msToUTCSecs, ) where @@ -49,7 +53,9 @@ import Data.Time (UTCTime) import Data.Time.Clock.POSIX (posixSecondsToUTCTime) import Data.UUID as UUID import Data.UUID.V4 as UUID +import Database.Redis qualified as Redis import Gundeck.Env +import Gundeck.Redis qualified as Redis import Imports import Network.AMQP import Network.HTTP.Types @@ -61,6 +67,7 @@ import System.Logger (Logger) import System.Logger qualified as Logger import System.Logger.Class qualified as Log import System.Timeout +import UnliftIO (async) -- | TODO: 'Client' already has an 'Env'. Why do we need two? How does this even work? We should -- probably explain this here. @@ -84,6 +91,72 @@ newtype Gundeck a = Gundeck instance MonadMonitor Gundeck where doIO = liftIO +-- | 'Gundeck' doesn't have an instance for 'MonadRedis' because it contains two +-- connections to two redis instances. When using 'WithDefaultRedis', any redis +-- operation will only target the default redis instance (configured under +-- 'redis:' in the gundeck config). To write to both redises use +-- 'WithAdditionalRedis'. +newtype WithDefaultRedis a = WithDefaultRedis {runWithDefaultRedis :: Gundeck a} + deriving newtype + ( Functor, + Applicative, + Monad, + MonadIO, + MonadThrow, + MonadCatch, + MonadMask, + MonadReader Env, + MonadClient, + MonadUnliftIO, + Log.MonadLogger + ) + +instance Redis.MonadRedis WithDefaultRedis where + liftRedis action = do + defaultConn <- view rstate + Redis.runRobust defaultConn action + +instance Redis.RedisCtx WithDefaultRedis (Either Redis.Reply) where + returnDecode :: (Redis.RedisResult a) => Redis.Reply -> WithDefaultRedis (Either Redis.Reply a) + returnDecode = Redis.liftRedis . Redis.returnDecode + +-- | 'Gundeck' doesn't have an instance for 'MonadRedis' because it contains two +-- connections to two redis instances. When using 'WithAdditionalRedis', any +-- redis operation will target both redis instances (configured under 'redis:' +-- and 'redisAddtionalWrite:' in the gundeck config). To write to only the +-- default redis use 'WithDefaultRedis'. +newtype WithAdditionalRedis a = WithAdditionalRedis {runWithAdditionalRedis :: Gundeck a} + deriving newtype + ( Functor, + Applicative, + Monad, + MonadIO, + MonadThrow, + MonadCatch, + MonadMask, + MonadReader Env, + MonadClient, + MonadUnliftIO, + Log.MonadLogger + ) + +instance Redis.MonadRedis WithAdditionalRedis where + liftRedis action = do + defaultConn <- view rstate + ret <- Redis.runRobust defaultConn action + + mAdditionalRedisConn <- view rstateAdditionalWrite + for_ mAdditionalRedisConn $ \additionalRedisConn -> + -- We just fire and forget this call, as there is not much we can do if + -- this fails. + async $ Redis.runRobust additionalRedisConn action + + pure ret + +instance Redis.RedisCtx WithAdditionalRedis (Either Redis.Reply) where + returnDecode :: (Redis.RedisResult a) => Redis.Reply -> WithAdditionalRedis (Either Redis.Reply a) + returnDecode = Redis.liftRedis . Redis.returnDecode + instance Log.MonadLogger Gundeck where log l m = do e <- ask diff --git a/services/gundeck/src/Gundeck/Options.hs b/services/gundeck/src/Gundeck/Options.hs index 5222248da27..d70bbc4f91d 100644 --- a/services/gundeck/src/Gundeck/Options.hs +++ b/services/gundeck/src/Gundeck/Options.hs @@ -24,7 +24,6 @@ import Control.Lens hiding (Level) import Data.Aeson.TH import Data.Yaml (FromJSON) import Gundeck.Aws.Arn -import Hasql.Pool.Extended (PoolConfig) import Imports import Network.AMQP.Extended import System.Logger.Extended (Level, LogFormat) @@ -103,6 +102,30 @@ deriveFromJSON toOptionFieldName ''MaxConcurrentNativePushes makeLenses ''MaxConcurrentNativePushes +data RedisConnectionMode + = Master + | Cluster + deriving (Show, Generic) + +deriveJSON defaultOptions {constructorTagModifier = map toLower} ''RedisConnectionMode + +data RedisEndpoint = RedisEndpoint + { _host :: !Text, + _port :: !Word16, + _connectionMode :: !RedisConnectionMode, + _enableTls :: !Bool, + -- | When not specified, use system CA bundle + _tlsCa :: !(Maybe FilePath), + -- | When 'True', uses TLS but does not verify hostname or CA or validity of + -- the cert. Not recommended to set to 'True'. + _insecureSkipVerifyTls :: !Bool + } + deriving (Show, Generic) + +deriveFromJSON toOptionFieldName ''RedisEndpoint + +makeLenses ''RedisEndpoint + makeLenses ''Settings deriveFromJSON toOptionFieldName ''Settings @@ -112,11 +135,8 @@ data Opts = Opts _gundeck :: !Endpoint, _brig :: !Endpoint, _cassandra :: !CassandraOpts, - -- | Postgresql settings, the key values must be in libpq format. - -- https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS - _postgresql :: !(Map Text Text), - _postgresqlPassword :: !(Maybe FilePathSecrets), - _postgresqlPool :: !PoolConfig, + _redis :: !RedisEndpoint, + _redisAdditionalWrite :: !(Maybe RedisEndpoint), _aws :: !AWSOpts, _rabbitmq :: !AmqpEndpoint, _discoUrl :: !(Maybe Text), diff --git a/services/gundeck/src/Gundeck/Presence.hs b/services/gundeck/src/Gundeck/Presence.hs index 6c6b757ef59..aa8fb778095 100644 --- a/services/gundeck/src/Gundeck/Presence.hs +++ b/services/gundeck/src/Gundeck/Presence.hs @@ -33,10 +33,10 @@ import Wire.API.CannonId import Wire.API.Presence listH :: UserId -> Gundeck [Presence] -listH = Data.list +listH = runWithDefaultRedis . Data.list listAllH :: CommaSeparatedList UserId -> Gundeck [Presence] -listAllH uids = concat <$> Data.listAll (fromCommaSeparatedList uids) +listAllH uids = concat <$> runWithDefaultRedis (Data.listAll (fromCommaSeparatedList uids)) addH :: Presence -> Gundeck (Headers '[Header "Location" URI] NoContent) addH p = do diff --git a/services/gundeck/src/Gundeck/Presence/Data.hs b/services/gundeck/src/Gundeck/Presence/Data.hs index 622dba9329b..6173ace303d 100644 --- a/services/gundeck/src/Gundeck/Presence/Data.hs +++ b/services/gundeck/src/Gundeck/Presence/Data.hs @@ -20,156 +20,128 @@ module Gundeck.Presence.Data list, listAll, deleteAll, - cleanup, ) where -import Control.Lens (view) -import Control.Monad.Catch (throwM) -import Data.ByteString.Conversion (fromByteString, toByteString') +import Control.Monad.Catch +import Data.Aeson as Aeson +import Data.ByteString qualified as Strict +import Data.ByteString.Builder (byteString) +import Data.ByteString.Char8 qualified as StrictChars +import Data.ByteString.Conversion hiding (fromList) +import Data.ByteString.Lazy qualified as Lazy import Data.Id -import Data.Map.Strict qualified as Map -import Data.Misc (Milliseconds (..)) -import Data.Text (pack, unpack) -import Data.Text.Encoding (decodeUtf8, encodeUtf8) -import Data.Time (UTCTime) -import Data.Time.Clock.POSIX (posixSecondsToUTCTime, utcTimeToPOSIXSeconds) -import Data.UUID (UUID) -import Data.Vector qualified as Vector -import Gundeck.Env (hasqlPool) -import Gundeck.Monad -import Hasql.Session (Session, statement) -import Hasql.Statement (Statement) -import Hasql.TH +import Data.List.NonEmpty qualified as NonEmpty +import Data.Misc (Milliseconds) +import Database.Redis +import Gundeck.Monad (Gundeck, posixTime, runWithAdditionalRedis) +import Gundeck.Util.Redis import Imports -import System.Logger.Class qualified as Log +import System.Logger.Class (MonadLogger) import Wire.API.Presence -import Wire.Postgres qualified as Postgres --- | Register (or refresh) a presence. The server-side timestamp is stamped --- here, the 'Presence'\'s own 'createdAt' value is ignored (as in the redis --- implementation before). +-- Note [Migration] --------------------------------------------------------- +-- +-- Previous redis schema: user:=@= +-- New redis schema: user:= = +-- +-- The previous redis schema encodes cannon's ID in the subkey. The migration +-- proceeds as follows: +-- +-- 1. When adding new entries, we only use the connection as subkey. +-- 2. When listing entries (which does not use the subkey fortunately) we +-- store the original field name in the `Presence` record property `__field`. +-- 3. When deleting entries, we use this `Presence`'s `__field` value. +-- 4. Eventually `__field` can be removed from the `Presence` type and the +-- connection can be used directly instead. +-- + add :: Presence -> Gundeck () add p = do - nowMs <- posixTime - runPool $ - statement - (toUUID (userId p), connIdText (connId p), uriText (resource p), clientToText <$> clientId p, msToUtc (fromIntegral (ms nowMs))) - upsertPresence - --- | Read all presences of a single user. -list :: UserId -> Gundeck [Presence] -list u = fromMaybe [] . listToMaybe <$> listAll [u] - --- | Read all presences of the given users, one list per user (input order, --- empty list for users without presences). Single round trip. -listAll :: [UserId] -> Gundeck [[Presence]] -listAll [] = pure [] -listAll uu = do - rows <- runPool $ statement (Vector.fromList (toUUID <$> uu)) selectByUsers - presencesByUser <- - foldM - ( \acc (u, c, r, cl, t) -> case readPresenceRow u c r cl t of - Just p -> pure $! Map.insertWith (<>) (userId p) [p] acc - Nothing -> do - Log.warn $ - Log.msg (Log.val "ignoring unreadable presence row") - . Log.field "user_id" (show u) - . Log.field "conn_id" (show c) - pure acc - ) - Map.empty - (Vector.toList rows) - pure [Map.findWithDefault [] u presencesByUser | u <- uu] - --- | Compare-and-delete: only delete the stored presence if it is not newer --- than the given one (a newer re-registration with the same conn id must not --- be deleted by a stale disconnect). -deleteAll :: [Presence] -> Gundeck () -deleteAll [] = pure () -deleteAll pp = - runPool . statement params $ deleteMany + now <- posixTime + let k = toKey (userId p) + let v = toField (connId p) + let d = Lazy.toStrict $ Aeson.encode $ PresenceData p.resource p.clientId now + runWithAdditionalRedis . retry x3 $ do + void . fromTxResult <=< (liftRedis . multiExec) $ do + void $ hset k (NonEmpty.singleton (v, d)) + -- nb. All presences of a user are expired 'maxIdleTime' after the + -- last presence was registered. A client who keeps a presence + -- (i.e. websocket) connected for longer than 'maxIdleTime' will be + -- silently dropped and receives no more notifications. + expire k maxIdleTime where - params = - ( Vector.fromList (toUUID . userId <$> pp), - Vector.fromList (connIdText . connId <$> pp), - Vector.fromList (msToUtc . fromIntegral . ms . createdAt <$> pp) - ) - --- | Delete presences older than a week. Normal disconnects delete their --- presence rows; this only guards against leaks from abnormally dead pods --- (replaces the redis key TTL). -cleanup :: Gundeck () -cleanup = runPool $ statement () deleteStale - --- Helpers ------------------------------------------------------------------- + maxIdleTime = 7 * 24 * 60 * 60 -- 7 days in seconds --- | Millis <-> UTC. Exact (milliseconds nest inside timestamptz's microseconds); --- do NOT reuse 'Gundeck.Monad.msToUTCSecs', it truncates to whole seconds. -msToUtc :: Int64 -> UTCTime -msToUtc p = posixSecondsToUTCTime (fromRational (fromIntegral p / 1000 :: Rational)) - -utcToMs :: UTCTime -> Int64 -utcToMs = floor . (* 1000) . utcTimeToPOSIXSeconds - -newtype PresenceDbError = PresenceDbError Text deriving (Show) - -instance Exception PresenceDbError - -runPool :: Session a -> Gundeck a -runPool sess = do - pool <- view hasqlPool - liftIO (Postgres.useWithResetAndRetry pool sess) >>= either (throwM . PresenceDbError . pack . show) pure - -connIdText :: ConnId -> Text -connIdText = decodeUtf8 . fromConnId +deleteAll :: (MonadMask m, MonadIO m, RedisCtx m (Either Reply), MonadLogger m) => [Presence] -> m () +deleteAll [] = pure () +deleteAll pp = for_ pp $ \p -> do + let k = toKey (userId p) + let f = Lazy.toStrict $ __field p + void . retry x3 $ do + void . liftRedis $ watch (pure k) + value <- either (throwM . RedisSimpleError) id <$> hget k f + void . liftRedis . multiExec $ do + case value of + Nothing -> pure $ pure () + Just v -> do + let p' = readPresence (userId p) (f, v) + if Just p == p' + then void <$> hdel k (pure f) + else pure $ pure () + +list :: (MonadRedis m, MonadThrow m) => UserId -> m [Presence] +list u = do + ePresenses <- liftRedis $ list' u + case ePresenses of + Left r -> throwM $ RedisSimpleError r + Right ps -> pure ps + +list' :: (RedisCtx m f, Functor f) => UserId -> m (f [Presence]) +list' u = mapMaybe (readPresence u) <$$> hgetall (toKey u) + +-- FUTUREWORK: Make this not fail if it fails only for a few users. +listAll :: (MonadRedis m, MonadThrow m) => [UserId] -> m [[Presence]] +listAll [] = pure [] +listAll uu = mapM list uu -uriText :: URI -> Text -uriText = decodeUtf8 . toByteString' +-- Helpers ------------------------------------------------------------------- -readPresenceRow :: UUID -> Text -> Text -> Maybe Text -> UTCTime -> Maybe Presence -readPresenceRow u c r cl t = do - uri <- parse (unpack r) - cid <- traverse parseClient cl - pure (Presence (Id u) (ConnId (encodeUtf8 c)) uri cid (Ms (fromIntegral (utcToMs t)))) - where - parseClient = fromByteString . encodeUtf8 - -upsertPresence :: Statement (UUID, Text, Text, Maybe Text, UTCTime) () -upsertPresence = - [resultlessStatement| - INSERT INTO presence (user_id, conn_id, resource, client_id, created_at) - VALUES ($1 :: uuid, $2 :: text, $3 :: text, $4 :: text?, $5 :: timestamptz) - ON CONFLICT (user_id, conn_id) DO UPDATE - SET resource = EXCLUDED.resource, - client_id = EXCLUDED.client_id, - created_at = EXCLUDED.created_at - |] - -selectByUsers :: Statement (Vector.Vector UUID) (Vector.Vector (UUID, Text, Text, Maybe Text, UTCTime)) -selectByUsers = - [vectorStatement| - SELECT user_id :: uuid, conn_id :: text, resource :: text, client_id :: text?, created_at :: timestamptz - FROM presence - WHERE user_id = ANY ($1 :: uuid[]) - |] - --- | Compare-and-delete, in one round trip: only delete each stored presence --- if it is not newer than the given one (a newer re-registration with the --- same conn id must not be deleted by a stale disconnect). -deleteMany :: Statement (Vector.Vector UUID, Vector.Vector Text, Vector.Vector UTCTime) () -deleteMany = - [resultlessStatement| - DELETE FROM presence p - USING unnest($1 :: uuid[], $2 :: text[], $3 :: timestamptz[]) AS d (user_id, conn_id, created_at) - WHERE p.user_id = d.user_id - AND p.conn_id = d.conn_id - AND p.created_at <= d.created_at - |] - -deleteStale :: Statement () () -deleteStale = - [resultlessStatement| - DELETE FROM presence - WHERE created_at < now() - interval '7 days' - |] +data PresenceData = PresenceData !URI !(Maybe ClientId) !Milliseconds + deriving (Eq) + +instance ToJSON PresenceData where + toJSON (PresenceData r c t) = + object + [ "r" .= r, + "c" .= c, + "t" .= t + ] + +instance FromJSON PresenceData where + parseJSON = withObject "PresenceData" $ \o -> + PresenceData + <$> o + .: "r" + <*> o + .:? "c" + <*> o + .:? "t" + .!= 0 + +toKey :: UserId -> ByteString +toKey u = Lazy.toStrict $ runBuilder (byteString "user:" <> builder u) + +toField :: ConnId -> ByteString +toField (ConnId con) = con + +fromField :: ByteString -> ConnId +fromField = ConnId . StrictChars.takeWhile (/= '@') + +readPresence :: UserId -> (ByteString, ByteString) -> Maybe Presence +readPresence u (f, b) = do + PresenceData uri clt tme <- + if "http" `Strict.isPrefixOf` b + then PresenceData <$> fromByteString b <*> pure Nothing <*> pure 0 + else decodeStrict' b + pure (Presence u (fromField f) uri clt tme (Lazy.fromStrict f)) diff --git a/services/gundeck/src/Gundeck/Push.hs b/services/gundeck/src/Gundeck/Push.hs index 77149b6efef..a6cdf759062 100644 --- a/services/gundeck/src/Gundeck/Push.hs +++ b/services/gundeck/src/Gundeck/Push.hs @@ -122,7 +122,7 @@ instance MonadPushAll Gundeck where mpaNotificationTTL = view (options . settings . notificationTTL) mpaCellsEventQueue = view (options . settings . cellsEventQueue) mpaMkNotificationId = mkNotificationId - mpaListAllPresences = Presence.listAll + mpaListAllPresences = runWithDefaultRedis . Presence.listAll mpaBulkPush = Web.bulkPush mpaStreamAdd = Data.add mpaPushNative = pushNative diff --git a/services/gundeck/src/Gundeck/Push/Websocket.hs b/services/gundeck/src/Gundeck/Push/Websocket.hs index 721dadd8eaf..562bcb10730 100644 --- a/services/gundeck/src/Gundeck/Push/Websocket.hs +++ b/services/gundeck/src/Gundeck/Push/Websocket.hs @@ -64,7 +64,7 @@ class (Monad m, MonadThrow m, Log.MonadLogger m) => MonadBulkPush m where instance MonadBulkPush Gundeck where mbpBulkSend = bulkSend - mbpDeleteAllPresences = Presence.deleteAll + mbpDeleteAllPresences = runWithAdditionalRedis . Presence.deleteAll mbpPosixTime = posixTime mbpMapConcurrently = mapConcurrently mbpMonitorBadCannons = monitorBadCannons @@ -315,7 +315,7 @@ push :: push notif (toList -> tgts) originUser originConn conns = do pp <- handleAny noPresences listPresences (ok, gone) <- foldM onResult ([], []) =<< send notif pp - Presence.deleteAll gone + runWithAdditionalRedis $ Presence.deleteAll gone pure ok where listPresences = @@ -324,7 +324,7 @@ push notif (toList -> tgts) originUser originConn conns = do . concat . filterByClient . zip tgts - <$> Presence.listAll (view targetUser <$> tgts) + <$> runWithDefaultRedis (Presence.listAll (view targetUser <$> tgts)) noPresences exn = do Log.err $ Log.field "error" (displayException exn) diff --git a/services/gundeck/src/Gundeck/Redis.hs b/services/gundeck/src/Gundeck/Redis.hs new file mode 100644 index 00000000000..e9bf1affafe --- /dev/null +++ b/services/gundeck/src/Gundeck/Redis.hs @@ -0,0 +1,127 @@ +{-# LANGUAGE NumDecimals #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Gundeck.Redis + ( RobustConnection, + connectRobust, + runRobust, + PingException, + ) +where + +import Control.Concurrent.Async (Async, async) +import Control.Monad.Catch qualified as Catch +import Control.Retry +import Database.Redis +import Database.Redis.Connection (ClusterDownError) +import Imports +import System.Logger qualified as Log +import System.Logger.Class (MonadLogger) +import System.Logger.Class qualified as LogClass +import System.Logger.Extended +import UnliftIO.Exception + +-- | Connection to Redis which allows reconnecting. +type RobustConnection = MVar Connection + +-- | Connection to Redis which can be reestablished on connection errors. +-- +-- Reconnecting even when Redis IPs change as long as the DNS name remains +-- constant. The server type (cluster or not) and the connection information of +-- the initial connection are used when reconnecting. +-- +-- Throws 'ConnectError', 'ConnectTimeout', 'ConnectionLostException', +-- 'PingException', or 'IOException' if retry policy is finite. +connectRobust :: + Logger -> + -- | e. g., @exponentialBackoff 50000@ + RetryPolicy -> + -- | action returning a fresh initial 'Connection', e. g., @(checkedConnect connInfo)@ or @(checkedConnectCluster connInfo)@ + IO Connection -> + IO (Async (), RobustConnection) +connectRobust l retryStrategy connectLowLevel = do + robustConnection <- newEmptyMVar @IO @Connection + thread <- + async $ safeForever l $ do + Log.info l $ Log.msg (Log.val "connecting to Redis") + conn <- retry connectLowLevel + Log.info l $ Log.msg (Log.val "successfully connected to Redis") + putMVar robustConnection conn + catch + ( forever $ do + _ <- runRedis conn ping + threadDelay 1e6 + ) + $ \(_ :: SomeException) -> void $ takeMVar robustConnection + pure (thread, robustConnection) + where + retry = + recovering -- retry connecting, e. g., with exponential back-off + retryStrategy + [ const $ Catch.Handler (\(e :: ClusterDownError) -> logEx (Log.err l) e "Redis cluster down" >> pure True), + const $ Catch.Handler (\(e :: ConnectError) -> logEx (Log.err l) e "Redis not in cluster mode" >> pure True), + const $ Catch.Handler (\(e :: ConnectTimeout) -> logEx (Log.err l) e "timeout when connecting to Redis" >> pure True), + const $ Catch.Handler (\(e :: ConnectionLostException) -> logEx (Log.err l) e "Redis connection lost during request" >> pure True), + const $ Catch.Handler (\(e :: PingException) -> logEx (Log.err l) e "pinging Redis failed" >> pure True), + const $ Catch.Handler (\(e :: IOException) -> logEx (Log.err l) e "network error when connecting to Redis" >> pure True) + ] + . const -- ignore RetryStatus + logEx :: (Exception e) => ((Msg -> Msg) -> IO ()) -> e -> ByteString -> IO () + logEx lLevel e description = lLevel $ Log.msg (Log.val description) . Log.field "error" (displayException e) + +-- | Run a 'Redis' action through a 'RobustConnection'. +-- +-- Blocks on connection errors as long as the connection is not reestablished. +-- Without externally enforcing timeouts, this may lead to leaking threads. +runRobust :: (MonadUnliftIO m, MonadLogger m, Catch.MonadMask m) => RobustConnection -> Redis a -> m a +runRobust mvar action = retry $ do + robustConnection <- readMVar mvar + liftIO $ runRedis robustConnection action + where + retryStrategy = capDelay 1000000 (exponentialBackoff 50000) + retry = + recovering -- retry connecting, e. g., with exponential back-off + retryStrategy + [ logAndHandle $ Catch.Handler (\(_ :: ConnectionLostException) -> pure True), + logAndHandle $ Catch.Handler (\(_ :: IOException) -> pure True) + ] + . const -- ignore RetryStatus + logAndHandle (Handler handler) _ = + Handler $ \e -> do + LogClass.err $ Log.msg (Log.val "Redis connection failed") . Log.field "error" (displayException e) + handler e + +data PingException = PingException Reply deriving (Show) + +instance Exception PingException + +safeForever :: + forall m. + (MonadUnliftIO m) => + Logger -> + m () -> + m () +safeForever l action = + forever $ + action `catchAny` \e -> do + Log.err l $ Log.msg (Log.val "Uncaught exception while connecting to redis") . Log.field "error" (displayException e) + threadDelay 1e6 -- pause to keep worst-case noise in logs manageable diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs index 590fe96e1ce..89e4c9f8ef2 100644 --- a/services/gundeck/src/Gundeck/Run.hs +++ b/services/gundeck/src/Gundeck/Run.hs @@ -41,7 +41,6 @@ import Cassandra (runClient, shutdown) import Cassandra.Schema (versionCheck) import Control.Error (ExceptT (ExceptT)) import Control.Exception (finally) -import Control.Exception.Safe (catchAny) import Control.Lens ((.~), (^.)) import Control.Monad.Extra import Data.Map qualified as Map @@ -49,18 +48,17 @@ import Data.Metrics.AWS (gaugeTokenRemaing) import Data.Metrics.Servant qualified as Metrics import Data.Proxy (Proxy (Proxy)) import Data.Text (unpack) +import Database.Redis qualified as Redis import Gundeck.API.Internal as Internal (InternalAPI, servantSitemap) import Gundeck.API.Public as Public (servantSitemap) import Gundeck.Aws qualified as Aws import Gundeck.Env import Gundeck.Env qualified as Env import Gundeck.Monad -import Gundeck.Options -import Gundeck.Presence.Data qualified as PresenceData +import Gundeck.Options hiding (host, port) import Gundeck.React import Gundeck.Schema.Run (lastSchemaVersion) import Gundeck.ThreadBudget -import Hasql.Pool.Extended (Pool (rawPool)) import Imports import Network.AMQP import Network.AMQP.Types @@ -83,13 +81,11 @@ import Wire.API.Routes.Public.Gundeck (GundeckAPI) import Wire.API.Routes.Version import Wire.API.Routes.Version.Wai import Wire.OpenTelemetry -import Wire.PostgresMigrations qualified as PostgresMigrations run :: Opts -> IO () run opts = withTracer \tracer -> do - env <- createEnv opts + (rThreads, env) <- createEnv opts let logger = env ^. applog - PostgresMigrations.runAllMigrations (env ^. hasqlPool).rawPool logger runDirect env setUpRabbitMqExchangesAndQueues @@ -97,10 +93,10 @@ run opts = withTracer \tracer -> do versionCheck lastSchemaVersion let s = newSettings $ defaultServer (unpack . host $ opts ^. gundeck) (port $ opts ^. gundeck) logger let throttleMillis = fromMaybe defSqsThrottleMillis $ opts ^. (settings . sqsThrottleMillis) + lst <- Async.async $ Aws.execute (env ^. awsEnv) (Aws.listen throttleMillis (runDirect env . onEvent)) wtbs <- forM (env ^. threadBudgetState) $ \tbs -> Async.async $ runDirect env $ watchThreadBudgetState tbs 10 wCollectAuth <- Async.async (collectAuthMetrics (Aws._awsEnv (Env._awsEnv env))) - pcleanup <- Async.async $ runDirect env $ cleanupPresenceLoop logger app <- middleware env <*> pure (mkApp env) inSpan tracer "gundeck" defaultSpanArguments {kind = Otel.Server} (runSettingsWithShutdown s app Nothing) `finally` do @@ -108,8 +104,10 @@ run opts = withTracer \tracer -> do shutdown (env ^. cstate) Async.cancel lst Async.cancel wCollectAuth - Async.cancel pcleanup forM_ wtbs Async.cancel + forM_ rThreads Async.cancel + Redis.disconnect =<< takeMVar (env ^. rstate) + whenJust (env ^. rstateAdditionalWrite) $ (=<<) Redis.disconnect . takeMVar Log.close (env ^. applog) where setUpRabbitMqExchangesAndQueues :: Gundeck () @@ -180,22 +178,3 @@ collectAuthMetrics env = do mbRemaining <- readAuthExpiration env gaugeTokenRemaing mbRemaining threadDelay 1_000_000 - --- | Hourly janitor replacing the redis key TTL: deletes presence rows older --- than a week (leak guard for abnormally dead pods). Never let a transient DB --- error kill the thread — log and retry next hour. Async exceptions (e.g. --- 'AsyncCancelled' from 'Async.cancel' during shutdown) propagate because --- 'Control.Exception.Safe.catchAny' rethrows asynchronously-delivered --- exceptions and only handles synchronous ones. -cleanupPresenceLoop :: Log.Logger -> Gundeck () -cleanupPresenceLoop logger = - forever $ - (PresenceData.cleanup >> threadDelay cleanupInterval) - `catchAny` \e -> do - liftIO . Log.err logger $ - Log.msg (Log.val "presence cleanup failed") - . Log.field "error" (displayException e) - threadDelay cleanupInterval - -cleanupInterval :: Int -cleanupInterval = 3_600_000_000 -- one hour, in microseconds diff --git a/services/gundeck/src/Gundeck/Util/Redis.hs b/services/gundeck/src/Gundeck/Util/Redis.hs new file mode 100644 index 00000000000..d125d04baca --- /dev/null +++ b/services/gundeck/src/Gundeck/Util/Redis.hs @@ -0,0 +1,61 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Gundeck.Util.Redis where + +import Control.Monad.Catch +import Control.Retry +import Data.ByteString qualified as BS +import Database.Redis +import Imports +import System.Logger.Class (MonadLogger) +import System.Logger.Class qualified as Log +import System.Logger.Message + +retry :: (MonadIO m, MonadMask m, MonadLogger m) => RetryPolicyM m -> m a -> m a +retry x = recovering x handlers . const + +x3 :: RetryPolicy +x3 = limitRetries 3 <> exponentialBackoff 100000 + +handlers :: (MonadLogger m) => [a -> Handler m Bool] +handlers = + [ const . Handler $ \case + RedisSimpleError (Error err) -> pure $ "READONLY" `BS.isPrefixOf` err + RedisTxError err -> pure $ "READONLY" `isPrefixOf` err + err -> do + Log.warn $ + Log.msg (Log.val "Redis error; not retrying.") + ~~ "redis.errMsg" .= show err + pure False + ] + +-- Error ------------------------------------------------------------------- + +data RedisError + = RedisSimpleError Reply + | RedisTxAborted + | RedisTxError String + deriving (Show) + +instance Exception RedisError + +fromTxResult :: (MonadThrow m) => TxResult a -> m a +fromTxResult = \case + TxSuccess a -> pure a + TxAborted -> throwM RedisTxAborted + TxError e -> throwM $ RedisTxError e diff --git a/services/gundeck/test/integration/API.hs b/services/gundeck/test/integration/API.hs index 6346d07a438..27de5b8602d 100644 --- a/services/gundeck/test/integration/API.hs +++ b/services/gundeck/test/integration/API.hs @@ -28,6 +28,7 @@ import Bilge hiding (head) import Bilge.Assert import Control.Arrow ((&&&)) import Control.Concurrent.Async (Async, async, concurrently_, wait) +import Control.Concurrent.Async qualified as Async import Control.Lens (view, (%~), (.~), (?~), (^.), (^?), _2) import Control.Retry (constantDelay, limitRetries, recoverAll, retrying) import Data.Aeson @@ -46,6 +47,8 @@ import Data.Set qualified as Set import Data.Text.Encoding qualified as T import Data.UUID qualified as UUID import Data.UUID.V4 +import Gundeck.Options +import Gundeck.Options qualified as O import Imports import Network.HTTP.Client qualified as Http import Network.URI (parseURI) @@ -56,6 +59,7 @@ import System.Timeout (timeout) import Test.Tasty import Test.Tasty.HUnit import TestSetup +import Util (runRedisProxy, withEnvOverrides, withSettingsOverrides) import Wire.API.Event.Gundeck import Wire.API.Internal.Notification import Wire.API.Presence @@ -72,7 +76,8 @@ tests s = test s "Remove stale presence" removeStalePresence, test s "Single user push" singleUserPush, test s "Single user push with large message" singleUserPushLargeMessage, - test s "Send a push, ensure origin does not receive it" sendSingleUserNoPiggyback + test s "Send a push, ensure origin does not receive it" sendSingleUserNoPiggyback, + test s "Store notifications even when redis is down" storeNotificationsEvenWhenRedisIsDown ], testGroup "Notifications" @@ -103,6 +108,10 @@ tests s = test s "control pings with payload produce pongs with the same payload" testControlPingPongWithData, test s "data non-pings are ignored" testNoPingNoPong ], + testGroup + "Redis migration" + [ test s "redis migration should work" testRedisMigration + ], -- TODO: The following tests require (at the moment), the usage real AWS -- services so they are kept in a separate group to simplify testing testGroup @@ -126,8 +135,8 @@ replacePresence = do con <- randomConnId let localhost8080 = URI . fromJust $ parseURI "http://localhost:8080" let localhost8081 = URI . fromJust $ parseURI "http://localhost:8081" - let pres1 = Presence uid (ConnId "dummy_dev") localhost8080 Nothing 0 - let pres2 = Presence uid (ConnId "dummy_dev") localhost8081 Nothing 0 + let pres1 = Presence uid (ConnId "dummy_dev") localhost8080 Nothing 0 "" + let pres2 = Presence uid (ConnId "dummy_dev") localhost8081 Nothing 0 "" void $ connectUser ca uid con setPresence gu pres1 !!! const 201 === statusCode sendPush (push uid [uid]) @@ -260,6 +269,28 @@ sendMultipleUsers = do pevent = KeyMap.fromList ["foo" .= (42 :: Int)] push u us = newPush (Just u) (toRecipients us) pload & pushOriginConnection ?~ ConnId "dev" +storeNotificationsEvenWhenRedisIsDown :: TestM () +storeNotificationsEvenWhenRedisIsDown = do + ally <- randomId + origRedisEndpoint <- view $ tsOpts . redis + let proxyPort = 10112 + redisProxyServer <- liftIO . async $ runRedisProxy (origRedisEndpoint ^. O.host) (origRedisEndpoint ^. O.port) proxyPort + withSettingsOverrides + ( \gundeckSettings -> + gundeckSettings + & redis . Gundeck.Options.host .~ "localhost" + & redis . Gundeck.Options.port .~ proxyPort + ) + $ do + let pload = textPayload "hello" + push = buildPush ally [(ally, RecipientClientsAll)] pload + gu <- view tsGundeck + liftIO $ Async.cancel redisProxyServer + post (runGundeckR gu . path "i/push/v2" . json [push]) !!! const 200 === statusCode + + ns <- listNotifications ally Nothing + liftIO $ assertEqual ("Expected 1 notification, got: " <> show ns) 1 (length ns) + ----------------------------------------------------------------------------- -- Notifications @@ -698,6 +729,36 @@ testLongPushToken = do tkn4 <- randomToken clt gcmToken {tSize = 5000} registerPushTokenRequest uid tkn4 !!! const 413 === statusCode +-- * Redis Migration + +testRedisMigration :: TestM () +testRedisMigration = do + uid <- randomUser + con <- randomConnId + cannonURI <- Wire.API.Presence.parse "http://cannon.example" + let presence = Presence uid con cannonURI Nothing 1 "" + redis2 <- view tsRedis2 + + withSettingsOverrides (redisAdditionalWrite ?~ redis2) $ do + g <- view tsGundeck + setPresence g presence + !!! const 201 + === statusCode + retrievedPresence <- + map resource . decodePresence <$> (getPresence g (toByteString' uid) lookupEnv "REDIS_ADDITIONAL_WRITE_USERNAME" + password <- ("REDIS_PASSWORD",) <$$> lookupEnv "REDIS_ADDITIONAL_WRITE_PASSWORD" + pure $ catMaybes [username, password] + + withEnvOverrides redis2CredsAsRedis1Creds $ withSettingsOverrides (redis .~ redis2) $ do + g <- view tsGundeck + retrievedPresence <- + map resource . decodePresence <$> (getPresence g (toByteString' uid) UserId -> Int -> TestM () diff --git a/services/gundeck/test/integration/Main.hs b/services/gundeck/test/integration/Main.hs index 05a385e40b4..767f28a4ae4 100644 --- a/services/gundeck/test/integration/Main.hs +++ b/services/gundeck/test/integration/Main.hs @@ -30,7 +30,7 @@ import Data.Proxy import Data.Tagged import Data.Text.Encoding (encodeUtf8) import Data.Yaml (decodeFileEither) -import Gundeck.Options +import Gundeck.Options hiding (host, port) import Imports hiding (local) import Metrics qualified import Network.HTTP.Client (responseTimeoutMicro) @@ -52,7 +52,8 @@ data IntegrationConfig = IntegrationConfig { gundeck :: Endpoint, cannon :: Endpoint, cannon2 :: Endpoint, - brig :: Endpoint + brig :: Endpoint, + redis2 :: RedisEndpoint } deriving (Show, Generic) @@ -113,6 +114,6 @@ main = withOpenSSL $ runTests go b = BrigR $ mkRequest iConf.brig lg <- Logger.new Logger.defSettings db <- defInitCassandra (gConf ^. cassandra) lg - pure $ TestSetup m g c c2 b db lg - mkRequest (Endpoint h p) = Bilge.host (encodeUtf8 h) . Bilge.port p + pure $ TestSetup m g c c2 b db lg gConf (redis2 iConf) releaseOpts _ = pure () + mkRequest (Endpoint h p) = Bilge.host (encodeUtf8 h) . Bilge.port p diff --git a/services/gundeck/test/integration/TestSetup.hs b/services/gundeck/test/integration/TestSetup.hs index 70e8cd77dca..ea49d1b3222 100644 --- a/services/gundeck/test/integration/TestSetup.hs +++ b/services/gundeck/test/integration/TestSetup.hs @@ -28,6 +28,8 @@ module TestSetup tsBrig, tsCass, tsLogger, + tsOpts, + tsRedis2, TestM (..), TestSetup (..), BrigR (..), @@ -40,6 +42,8 @@ import Bilge (HttpT (..), Manager, MonadHttp, Request, runHttpT) import Cassandra qualified as Cql import Control.Lens (makeLenses, (^.)) import Control.Monad.Catch (MonadCatch, MonadMask, MonadThrow) +import Gundeck.Options (RedisEndpoint) +import Gundeck.Options qualified as Gundeck import Imports import System.Logger qualified as Log import Test.Tasty (TestName, TestTree) @@ -75,7 +79,9 @@ data TestSetup = TestSetup _tsCannon2 :: CannonR, _tsBrig :: BrigR, _tsCass :: Cql.ClientState, - _tsLogger :: Log.Logger + _tsLogger :: Log.Logger, + _tsOpts :: Gundeck.Opts, + _tsRedis2 :: RedisEndpoint } makeLenses ''TestSetup diff --git a/services/gundeck/test/integration/Util.hs b/services/gundeck/test/integration/Util.hs new file mode 100644 index 00000000000..d6790424b2f --- /dev/null +++ b/services/gundeck/test/integration/Util.hs @@ -0,0 +1,119 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2025 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Util where + +import Bilge qualified +import Control.Concurrent (forkFinally) +import Control.Concurrent.Async (race_) +import Control.Exception qualified as E +import Control.Lens +import Control.Monad.Catch +import Control.Monad.Codensity +import Data.ByteString qualified as S +import Data.Text qualified as Text +import Gundeck.Env (createEnv) +import Gundeck.Options +import Gundeck.Run (mkApp) +import Imports +import Network.Socket hiding (openSocket) +import Network.Socket.ByteString (recv, sendAll) +import Network.Wai.Utilities.MockServer (withMockServer) +import TestSetup + +withSettingsOverrides :: (Opts -> Opts) -> TestM a -> TestM a +withSettingsOverrides f action = do + ts <- ask + let opts = f (view tsOpts ts) + (_rThreads, env) <- liftIO $ createEnv opts + liftIO . lowerCodensity $ do + let app = mkApp env + p <- withMockServer app + liftIO $ Bilge.runHttpT (ts ^. tsManager) $ runReaderT (runTestM action) $ ts & tsGundeck .~ GundeckR (mkRequest p) + where + mkRequest p = Bilge.host "127.0.0.1" . Bilge.port p + +withEnvOverrides :: forall m a. (MonadIO m, MonadMask m) => [(String, String)] -> m a -> m a +withEnvOverrides envOverrides action = do + bracket (setEnvVars envOverrides) (resetEnvVars) $ const action + where + setEnvVars :: [(String, String)] -> m [(String, Maybe String)] + setEnvVars newVars = liftIO $ do + oldVars <- mapM (\(k, _) -> (k,) <$> lookupEnv k) newVars + mapM_ (uncurry setEnv) newVars + pure oldVars + + resetEnvVars :: [(String, Maybe String)] -> m () + resetEnvVars = + mapM_ (\(k, mV) -> maybe (unsetEnv k) (setEnv k) mV) + +runRedisProxy :: Text -> Word16 -> Word16 -> IO () +runRedisProxy redisHost redisPort proxyPort = do + (servAddr : _) <- getAddrInfo Nothing (Just $ Text.unpack redisHost) (Just $ show redisPort) + runTCPServer Nothing (show proxyPort) $ \client -> do + server <- getServerSocket servAddr + client <~~> server + where + getServerSocket servAddr = do + server <- socket (addrFamily servAddr) Stream defaultProtocol + connect server (addrAddress servAddr) >> pure server + p1 <~~> p2 = finally (race_ (p1 `mapData` p2) (p2 `mapData` p1)) (close p1 >> close p2) + mapData f t = do + content <- recv f 4096 + unless (S.null content) $ sendAll t content >> mapData f t + +-- Forked from network-run, added logic to cleanup clients when server is closed + +-- | Running a TCP server with an accepted socket and its peer name. +runTCPServer :: Maybe HostName -> ServiceName -> (Socket -> IO a) -> IO b +runTCPServer mhost port' server = withSocketsDo $ do + addr <- resolve Stream mhost port' True + clientThreads <- newTVarIO [] + E.bracket (open addr) (cleanupClients clientThreads) (loop clientThreads) + where + open addr = E.bracketOnError (openServerSocket addr) close $ \sock -> do + listen sock 1024 + pure sock + loop clientThreads sock = forever $ do + E.bracketOnError (accept sock) (close . fst) $ + \(conn, _peer) -> do + thread <- forkFinally (server conn) (const $ gracefulClose conn 5000) + atomically $ modifyTVar clientThreads (thread :) + cleanupClients :: TVar [ThreadId] -> Socket -> IO () + cleanupClients clientThreads sock = do + close sock + mapM_ killThread =<< readTVarIO clientThreads + +resolve :: SocketType -> Maybe HostName -> ServiceName -> Bool -> IO AddrInfo +resolve socketType mhost port' passive = + head <$> getAddrInfo (Just hints) mhost (Just port') + where + hints = + defaultHints + { addrSocketType = socketType, + addrFlags = [AI_PASSIVE | passive] + } + +openServerSocket :: AddrInfo -> IO Socket +openServerSocket addr = E.bracketOnError (openSocket addr) close $ \sock -> do + setSocketOption sock ReuseAddr 1 + withFdSocket sock $ setCloseOnExecIfNeeded + bind sock $ addrAddress addr + pure sock + +openSocket :: AddrInfo -> IO Socket +openSocket addr = socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr) diff --git a/services/gundeck/test/unit/MockGundeck.hs b/services/gundeck/test/unit/MockGundeck.hs index 647e30f376d..6e4f27df53d 100644 --- a/services/gundeck/test/unit/MockGundeck.hs +++ b/services/gundeck/test/unit/MockGundeck.hs @@ -770,6 +770,7 @@ fakePresence userId clientId_ = Presence {..} connId = fakeConnId clientId_ resource = URI . fromJust $ URI.parseURI "http://127.0.0.1:8080" createdAt = 0 + __field = mempty -- | See also: 'fakePresence'. fakeConnId :: ClientId -> ConnId diff --git a/services/integration.yaml b/services/integration.yaml index acb0e595b23..2da7e194e1f 100644 --- a/services/integration.yaml +++ b/services/integration.yaml @@ -128,6 +128,12 @@ backendTwo: originDomain: b.example.com +redis2: + host: 127.0.0.1 + port: 6379 + connectionMode: master + enableTls: false + insecureSkipVerifyTls: false dynamicBackends: dynamic-backend-1: