diff --git a/optional-modifications/patches/resolve-upstreams/README.md b/optional-modifications/patches/resolve-upstreams/README.md new file mode 100644 index 00000000000..b7ac453cfac --- /dev/null +++ b/optional-modifications/patches/resolve-upstreams/README.md @@ -0,0 +1,46 @@ +# resolve-upstreams + +nginx caches the addresses of `relay` and `web` until it restarts. +A recreated container can return on a different address. +nginx keeps using the old one and answers every request with 502. +Ingest breaks silently, because `relay` serves `/api/store/`, `/api//` and `/api/0/relays/`. +This patch makes nginx re-resolve both names. + +`restart: true` on nginx's `depends_on` already restarts nginx after a Compose operation recreates `web` or `relay`. +Apply this patch only if you also need to survive address changes that no Compose operation causes, such as a crash restart or a Docker daemon restart. + +## Apply + +Run these from the repository root, keeping the order and the `&&`. +Then run `./install.sh`. + +```bash +patch -p0 < optional-modifications/patches/resolve-upstreams/docker-compose.yml.patch && \ +patch -p0 < optional-modifications/patches/resolve-upstreams/nginx-resolver.conf.template.patch && \ +patch -p0 < optional-modifications/patches/resolve-upstreams/nginx.conf.patch +``` + +A rejected hunk stops the sequence before anything depends on it. + +## Notes + +- `valid=10s` sets the recovery time. + It costs 6 DNS queries per minute per upstream. + Worker count and traffic do not affect that. +- An unresolvable upstream no longer stops nginx from starting. + nginx serves 502s instead. + The healthcheck misses this, because it requests `/` and `curl` without `-f` exits 0 on a 502. +- nginx refuses to start if its `/etc/resolv.conf` lists no `nameserver`. + An unpatched install also breaks in that case, with a different message. +- Do not bind-mount over `/etc/nginx/conf.d`. + The entrypoint renders the resolver snippet there. +- Tested on Docker. + The patch reads the resolver address from the container, so it should suit Podman, but nobody has + tested that. + +## Background + +- [Report of nginx routing to a stale relay address](https://github.com/getsentry/self-hosted/issues/3894) +- [The `depends_on` policy that covers Compose operations](https://github.com/getsentry/self-hosted/pull/3914) +- [First attempt at re-resolution, with the discussion that led to this patch](https://github.com/getsentry/self-hosted/pull/4295) +- [Second attempt, which changed the default configuration](https://github.com/getsentry/self-hosted/pull/4492) diff --git a/optional-modifications/patches/resolve-upstreams/docker-compose.yml.patch b/optional-modifications/patches/resolve-upstreams/docker-compose.yml.patch new file mode 100644 index 00000000000..3a2e15d0e44 --- /dev/null +++ b/optional-modifications/patches/resolve-upstreams/docker-compose.yml.patch @@ -0,0 +1,17 @@ +--- docker-compose.yml 2026-08-31 09:58:28.827732343 +0200 ++++ docker-compose.resolve-upstreams.yml 2026-08-31 10:21:46.281494103 +0200 +@@ -606,8 +606,14 @@ + read_only: true + source: ./nginx.conf + target: /etc/nginx/nginx.conf ++ - type: bind ++ read_only: true ++ source: ./nginx-resolver.conf.template ++ target: /etc/nginx/templates/nginx-resolver.conf.template + - sentry-nginx-cache:/var/cache/nginx + - sentry-nginx-www:/var/www ++ environment: ++ NGINX_ENTRYPOINT_LOCAL_RESOLVERS: 1 + healthcheck: + <<: *healthcheck_defaults + test: diff --git a/optional-modifications/patches/resolve-upstreams/nginx-resolver.conf.template.patch b/optional-modifications/patches/resolve-upstreams/nginx-resolver.conf.template.patch new file mode 100644 index 00000000000..0a079a4ec59 --- /dev/null +++ b/optional-modifications/patches/resolve-upstreams/nginx-resolver.conf.template.patch @@ -0,0 +1,4 @@ +--- /dev/null 2026-08-27 11:01:02.392055778 +0200 ++++ nginx-resolver.conf.template 2026-08-31 09:59:17.962753633 +0200 +@@ -0,0 +1 @@ ++resolver ${NGINX_LOCAL_RESOLVERS} ipv6=off valid=10s; diff --git a/optional-modifications/patches/resolve-upstreams/nginx.conf.patch b/optional-modifications/patches/resolve-upstreams/nginx.conf.patch new file mode 100644 index 00000000000..11899408270 --- /dev/null +++ b/optional-modifications/patches/resolve-upstreams/nginx.conf.patch @@ -0,0 +1,22 @@ +--- nginx.conf ++++ nginx.resolve-upstreams.conf +@@ -65,13 +65,17 @@ + proxy_read_timeout 90s; + proxy_send_timeout 5s; + ++ include /etc/nginx/conf.d/nginx-resolver.conf; # see optional-modifications/patches/resolve-upstreams ++ + upstream relay { +- server relay:3000; ++ zone relay 512k; ++ server relay:3000 resolve; + keepalive 2; + } + + upstream sentry { +- server web:9000; ++ zone sentry 512k; ++ server web:9000 resolve; + keepalive 2; + } +