revert redis retirement

Makefile

@@ -13,11 +13,11 @@ CHARTS_INTEGRATION    := wire-server databases-ephemeral rabbitmq fake-aws ingre
 # (e.g. move charts/brig to charts/wire-server/brig)
 # this list could be generated from the folder names under ./charts/ like so:
 # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts)
-CHARTS_RELEASE := wire-server rabbitmq rabbitmq-external databases-ephemeral	\
+CHARTS_RELEASE := wire-server redis-ephemeral rabbitmq rabbitmq-external databases-ephemeral	\
 fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\
-ingress-nginx-controller nginx-ingress-services \
+ingress-nginx-controller nginx-ingress-services reaper \
 k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise \
 wire-ingress
 KIND_CLUSTER_NAME     := wire-server

charts/databases-ephemeral/requirements.yaml

@@ -13,6 +13,13 @@ dependencies:
 # since cassandra-migrations did not yet run; but the cassandra-migrations hook
 # requires all pods to be in a 'Ready' state before starting (condition for post-install); this is impossible.
 #####################################################
+- name: redis-ephemeral
+  version: "0.0.42"
+  repository: "file://../redis-ephemeral"
+  tags:
+    - redis-ephemeral
+    - databases-ephemeral
+    - demo
 - name: elasticsearch-ephemeral
   version: "0.0.42"
   repository: "file://../elasticsearch-ephemeral"

charts/databases-ephemeral/templates/NOTES.txt

@@ -2,6 +2,7 @@ You now have an in-memory, non-persistent, non-highly-available set of databases
 
 * cassandra-ephemeral
 * elasticsearch-ephemeral
+* redis-ephemeral
 
 !! WARNING WARNING !!
 This is fine for testing and demo purposes, but NOT for a production use case.

charts/integration/templates/integration-integration.yaml

@@ -41,10 +41,6 @@ spec:
       configMap:
         name: "gundeck"
 
-    - name: "gundeck-secrets"
-      secret:
-        secretName: "gundeck"
-
     - name: "cargohold-config"
       configMap:
         name: "cargohold"
@@ -97,6 +93,9 @@ spec:
       secret:
         secretName: {{ .Values.config.elasticsearch.tlsCaSecretRef.name }}
 
+    - name: redis-ca
+      secret:
+        secretName: {{ .Values.config.redis.tlsCaSecretRef.name }}
 
     - name: rabbitmq-ca
       secret:
@@ -238,9 +237,6 @@ spec:
     - name: gundeck-config
       mountPath: /etc/wire/gundeck/conf
 
-    - name: gundeck-secrets
-      mountPath: /etc/wire/gundeck/secrets
-
     - name: cargohold-config
       mountPath: /etc/wire/cargohold/conf
 
@@ -280,6 +276,9 @@ spec:
     - name: elasticsearch-ca
       mountPath: /etc/wire/brig/elasticsearch-ca
 
+    - name: redis-ca
+      mountPath: /etc/wire/gundeck/redis-ca
+
     - name: rabbitmq-ca
       mountPath: /etc/wire/brig/rabbitmq-ca
 
@@ -344,6 +343,20 @@ spec:
     - name: ENABLE_FEDERATION_V{{$version}}
       value: "1"
     {{- end }}
+    {{- if hasKey .Values.secrets "redisUsername" }}
+    - name: REDIS_USERNAME
+      valueFrom:
+        secretKeyRef:
+          name: integration
+          key: redisUsername
+    {{- end }}
+    {{- if hasKey .Values.secrets "redisPassword" }}
+    - name: REDIS_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: integration
+          key: redisPassword
+    {{- end }}
     - name: TEST_XML
       value: /tmp/result.xml
     {{- if .Values.config.uploadXml }}

charts/integration/templates/secret.yaml

@@ -16,4 +16,10 @@ data:
   {{- if hasKey . "uploadXmlAwsSecretAccessKey" }}
   uploadXmlAwsSecretAccessKey: {{ .uploadXmlAwsSecretAccessKey | b64enc | quote }}
   {{- end }}
+  {{- if hasKey . "redisUsername" }}
+  redisUsername: {{ .redisUsername | b64enc | quote }}
+  {{- end }}
+  {{- if hasKey . "redisPassword" }}
+  redisPassword: {{ .redisPassword | b64enc | quote }}
+  {{- end }}
   {{- end }}

charts/reaper/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

charts/reaper/Chart.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+version: 0.0.42
+name: reaper
+appVersion: 0.1.0
+description: A helm charts to restart cannons if redis-ephemeal has died
+annotations:
+  # must conform to https://github.com/helm/community/blob/main/hips/hip-0015.md
+  helm.sh/images: |
+    - name: kubectl
+      image: docker.io/alpine/kubectl:1.36.3

charts/reaper/README.md

@@ -0,0 +1,71 @@
+Reaper
+------
+
+This pod is useful in the following scenario: You run wire-server alongside a single
+redis-ephemeral (part of databases-ephemeral). If you have a different setup for redis,
+do not use this chart.
+
+Due to the nature of pods and their ephemerality, there might be situations where a
+redis-ephemeral pod is restarted. In such cases, wire clients will have stale
+connections (they will have an active websocket connection, but gundeck (responsible for
+sending messages) will be unaware of this (as the record of who is connected where is
+gone with a redis-ephemeral restart). So these stale clients will not receive any
+messages. Here, this reaper will check that the `redis-ephemeral` pod is older than any
+other `cannon`; if that is not the case, it kills the `cannon`s forcing clients to
+reconnect.
+
+Image
+-----
+
+The reaper runs `scripts/reaper.sh` through `kubectl`, so `image` must point at a
+kubectl image that **contains a POSIX shell** at `/bin/sh`. Distroless kubectl images
+do not ship one and the pod will fail to start. The script itself is POSIX sh, so
+busybox `ash` is enough, bash not required.
+
+The image is fully configurable:
+
+```yaml
+image:
+  registry: docker.io      # set to "" for an unqualified repository
+  repository: alpine/kubectl
+  tag: 1.36.3
+  digest: ""               # e.g. "sha256:..."; takes precedence over tag
+  pullPolicy: IfNotPresent
+imagePullSecrets:
+  - name: my-pull-secret
+```
+
+RBAC
+----
+
+The chart creates a namespaced `Role`/`RoleBinding` granting `get`, `list`, `watch` and
+`delete` on pods, bound to a `<release>-reaper` ServiceAccount.
+
+`watch` is required even though the script never watches anything explicitly:
+`kubectl delete pod` blocks until the pod is gone and opens a watch to do so. Without it
+the reaper deletes the first cannon and then hangs, without crashing.
+
+Earlier versions bound the ServiceAccount to `cluster-admin` through a fixed-name
+`ClusterRoleBinding`, which gave the pod read access to every Secret in the cluster.
+`helm upgrade` removes that binding and the old `reaper-role` ServiceAccount. Because
+nothing is cluster-scoped any more and all names are release-scoped, several reaper
+releases can now coexist in one cluster; previously a second release failed to install
+with a `ClusterRoleBinding` ownership conflict.
+
+Runtime
+-------
+
+The container runs as uid/gid 65534 with a read-only root filesystem and has resource
+requests and limits. `nodeSelector`, `tolerations` and `affinity` are honoured.
+
+`checkIntervalSeconds` (default `15`) controls how long the script waits between checks.
+Earlier versions listed pods once per second.
+
+Logs distinguish a failure to reach the API from "there are no matching pods", and
+include the underlying error:
+
+    Failed to list pods: Error from server (Forbidden): ... Skipping this iteration...
+    No cannon pods found. Doing nothing...
+
+Both cases previously printed `Failed to list pods. Skipping this iteration...`, so a
+reaper that could not list pods at all looked exactly like an idle one.

charts/reaper/scripts/reaper.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+
+# See the readme of the reaper chart.
+#
+# This is POSIX sh on purpose: the only actively maintained kubectl images that
+# ship busybox ash, not bash.
+
+# we loop forever, and on transient errors sleep and try again.
+# setting -e would crash the pod on transient e.g. network errors, which isn't useful.
+set -u
+# shellcheck disable=SC3040 # busybox ash supports pipefail
+set -o pipefail
+
+USAGE="$0 <NAMESPACE> [INTERVAL_SECONDS]"
+NAMESPACE="${1:?$USAGE}"
+INTERVAL="${2:-15}"
+
+echo "Using namespace: $NAMESPACE, check interval: ${INTERVAL}s"
+
+kill_all_cannons() {
+  echo "Killing all cannons"
+  RAW_PODS=$(kubectl -n "$NAMESPACE" get pods 2>&1) || {
+    echo "Failed to list cannon pods: $RAW_PODS. Skipping this iteration..."
+    return
+  }
+  CANNON_PODS=$(echo "$RAW_PODS" | grep -e "cannon" | awk '{ print $1 }') || CANNON_PODS=""
+
+  # A here-document rather than a pipeline, so the loop runs in the current
+  # shell and the `exit 1` below actually terminates the script.
+  while IFS= read -r cannon; do
+    if [ -n "$cannon" ]; then
+      echo "Deleting $cannon"
+      # If a single delete fails, we skip it but keep going.
+      kubectl -n "$NAMESPACE" delete pod "$cannon" || {
+        echo "Failed to delete pod $cannon, crash reaper and try again"
+        exit 1
+      }
+    fi
+  done <<EOF
+$CANNON_PODS
+EOF
+}
+
+while true; do
+  # List first, filter second. Folding both into one pipeline made an API failure.
+  RAW_PODS=$(kubectl -n "$NAMESPACE" get pods --sort-by=.metadata.creationTimestamp 2>&1) || {
+    echo "Failed to list pods: $RAW_PODS. Skipping this iteration..."
+    sleep "$INTERVAL"
+    continue
+  }
+
+  # Gather all pods that contain "cannon" or "redis-ephemeral", sorted by creation time
+  ALL_PODS=$(echo "$RAW_PODS" | grep -e "cannon" -e "redis-ephemeral") || ALL_PODS=""
+
+  # Check if we have any cannon pods at all
+  if ! echo "$ALL_PODS" | grep -q "cannon"; then
+    echo "No cannon pods found. Doing nothing..."
+    sleep "$INTERVAL"
+    continue
+  fi
+
+  # Check if we have any redis-ephemeral pods at all
+  if ! echo "$ALL_PODS" | grep -q "redis-ephemeral"; then
+    echo "No redis-ephemeral pod found. Doing nothing..."
+    sleep "$INTERVAL"
+    continue
+  fi
+
+  # At this point, we have both cannon and redis-ephemeral pods in ALL_PODS
+  # Check which is oldest
+  FIRST_POD=$(echo "$ALL_PODS" | head -n 1 | awk '{ print $1 }')
+
+  if [ -z "$FIRST_POD" ]; then
+    echo "Could not determine the oldest pod from the list. Doing nothing..."
+    sleep "$INTERVAL"
+    continue
+  fi
+
+  case "$FIRST_POD" in
+    *redis-ephemeral*)
+      echo "redis-ephemeral is the oldest pod, all good."
+      ;;
+    *)
+      kill_all_cannons
+      ;;
+  esac
+
+  sleep "$INTERVAL"
+done

charts/reaper/templates/_helpers.tpl

@@ -0,0 +1,26 @@
+{{/* Allow KubeVersion to be overridden. */}}
+{{- define "kubeVersion" -}}
+  {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}}
+{{- end -}}
+
+{{- define "includeSecurityContext" -}}
+  {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}}
+{{- end -}}
+
+{{/* Fully qualified image reference, digest taking precedence over tag. */}}
+{{- define "reaper.image" -}}
+{{- $repository := .Values.image.repository -}}
+{{- if .Values.image.registry -}}
+{{- $repository = printf "%s/%s" .Values.image.registry .Values.image.repository -}}
+{{- end -}}
+{{- if .Values.image.digest -}}
+{{- printf "%s@%s" $repository .Values.image.digest -}}
+{{- else -}}
+{{- printf "%s:%s" $repository (.Values.image.tag | toString) -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Release-scoped name for the ServiceAccount, Role and RoleBinding. */}}
+{{- define "reaper.serviceAccountName" -}}
+{{- printf "%s-reaper" .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

charts/reaper/templates/configmap.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: reaper-script
+  labels:
+    app: reaper
+data:
+  reaper.sh: |-
+    {{- .Files.Get "scripts/reaper.sh" | nindent 4 }}
+