From 1291da94cfcf06e79577186b0adf716af6e3c3fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 13:48:38 +0200
Subject: [PATCH 01/14] Add support for setting custom https port in
 node_exporter config

---
 .../defaults/main.yml                         |  4 +++
 .../handlers/main.yml                         |  6 +++++
 .../prometheus_node_exporter/tasks/main.yml   | 26 +++++++++++++++++++
 .../templates/nginx-prometheus.j2             |  8 +++---
 4 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/ansible/roles/prometheus_node_exporter/defaults/main.yml b/ansible/roles/prometheus_node_exporter/defaults/main.yml
index 9066e94c..0cdc0011 100644
--- a/ansible/roles/prometheus_node_exporter/defaults/main.yml
+++ b/ansible/roles/prometheus_node_exporter/defaults/main.yml
@@ -18,3 +18,7 @@ node_exporter_options: ''
 node_exporter_state: started
 node_exporter_enabled: true
 node_exporter_restart: on-failure
+http_port: 9001
+https_port: 443
+use_https: false
+monitoring_server_ip: "{{ lookup('dig', 'monitoring.ooni.org') }}"
diff --git a/ansible/roles/prometheus_node_exporter/handlers/main.yml b/ansible/roles/prometheus_node_exporter/handlers/main.yml
index 4ec66003..092c7285 100644
--- a/ansible/roles/prometheus_node_exporter/handlers/main.yml
+++ b/ansible/roles/prometheus_node_exporter/handlers/main.yml
@@ -18,3 +18,9 @@
   service:
     name: node_exporter
     state: restarted
+
+- name: reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded
diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml
index 7512d5b9..35e32438 100644
--- a/ansible/roles/prometheus_node_exporter/tasks/main.yml
+++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml
@@ -23,4 +23,30 @@
     - config
   when: use_nginx
 
+- name: Allow traffic on the node exporter http_port from the monitoring host only
+  become: true
+  tags: [prometheus_node_exporter, monitoring, node_exporter, nftables]
+  ansible.builtin.blockinfile:
+    path: "/etc/ooni/nftables/tcp/{{ http_port }}.nft"
+    create: yes
+    mode: "0644"
+    block: |
+      add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport {{ http_port }} counter accept comment "node exporter metrics"
+  when: use_nginx and not use_https
+  notify:
+    - reload nftables
+
+- name: Allow traffic on the node exporter https_port from the monitoring host only
+  become: true
+  tags: [prometheus_node_exporter, monitoring, node_exporter, nftables]
+  ansible.builtin.blockinfile:
+    path: "/etc/ooni/nftables/tcp/{{ http_port }}.nft"
+    create: yes
+    mode: "0644"
+    block: |
+      add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport {{ https_port }} counter accept comment "node exporter metrics"
+  when: use_nginx and use_https
+  notify:
+    - reload nftables
+
 - include_tasks: install.yml
diff --git a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2
index 6fb7c5ca..5af3c130 100644
--- a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2
+++ b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2
@@ -2,7 +2,7 @@
 
 server {
     {% if use_https %}
-    listen 443 ssl http2;
+    listen {{ https_port }} ssl http2;
 
     server_name {{ inventory_hostname }};
     include /etc/nginx/ssl_intermediate.conf;
@@ -11,9 +11,9 @@ server {
     ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem;
     ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem;
     {% else %}
-    listen {{http_port}};
+    listen {{ http_port }};
 
-    server_name {{inventory_hostname}};
+    server_name {{ inventory_hostname }};
     {% endif %}
 
     {% for config in prometheus_nginx_proxy_config %}
@@ -30,4 +30,4 @@ server {
     }
 
     {% endfor %}
-}
\ No newline at end of file
+}

From f57b9e91a2003c78a72197c8858faba3f2c613dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 13:49:09 +0200
Subject: [PATCH 02/14] Add oohelpderd role

---
 ansible/roles/oohelperd/defaults/main.yml     | 10 ++
 ansible/roles/oohelperd/handlers/main.yml     |  4 +
 ansible/roles/oohelperd/tasks/main.yml        | 93 +++++++++++++++++++
 .../oohelperd/templates/nginx-oohelperd.j2    | 73 +++++++++++++++
 .../roles/oohelperd/templates/oohelperd.env   |  5 +
 .../oohelperd/templates/oohelperd.service     | 30 ++++++
 6 files changed, 215 insertions(+)
 create mode 100644 ansible/roles/oohelperd/defaults/main.yml
 create mode 100644 ansible/roles/oohelperd/handlers/main.yml
 create mode 100644 ansible/roles/oohelperd/tasks/main.yml
 create mode 100644 ansible/roles/oohelperd/templates/nginx-oohelperd.j2
 create mode 100644 ansible/roles/oohelperd/templates/oohelperd.env
 create mode 100644 ansible/roles/oohelperd/templates/oohelperd.service

diff --git a/ansible/roles/oohelperd/defaults/main.yml b/ansible/roles/oohelperd/defaults/main.yml
new file mode 100644
index 00000000..1c4f7a86
--- /dev/null
+++ b/ansible/roles/oohelperd/defaults/main.yml
@@ -0,0 +1,10 @@
+# The Web Connectivity test helper (oohelperd) is built and released as part of
+# https://github.com/ooni/probe-cli
+oohelperd_version: v3.30.0
+oohelperd_base_url: "https://github.com/ooni/probe-cli/releases/download/{{ oohelperd_version }}"
+# sha256 of {{ oohelperd_base_url }}/oohelperd-linux-amd64
+oohelperd_checksum: "fff38b9366fecae400cb3f616ffa2a37dba211c33ee4e1c1b343bff1c384d48e"
+
+# A test helper request measures the target before it can answer, so it needs
+# way more than the nginx default of 60s
+oohelperd_nginx_proxy_timeout: 900
diff --git a/ansible/roles/oohelperd/handlers/main.yml b/ansible/roles/oohelperd/handlers/main.yml
new file mode 100644
index 00000000..71ffc659
--- /dev/null
+++ b/ansible/roles/oohelperd/handlers/main.yml
@@ -0,0 +1,4 @@
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded
diff --git a/ansible/roles/oohelperd/tasks/main.yml b/ansible/roles/oohelperd/tasks/main.yml
new file mode 100644
index 00000000..c20d5e63
--- /dev/null
+++ b/ansible/roles/oohelperd/tasks/main.yml
@@ -0,0 +1,93 @@
+---
+- name: Flush all handlers now
+  ansible.builtin.meta: flush_handlers
+
+- name: Create the oohelperd user
+  tags: [oohelperd]
+  ansible.builtin.user:
+    name: oohelperd
+    shell: /usr/sbin/nologin
+    create_home: no
+    system: yes
+
+- name: Download the oohelperd binary
+  tags: [oohelperd]
+  ansible.builtin.get_url:
+    url: "{{ oohelperd_base_url }}/oohelperd-linux-amd64"
+    dest: /usr/local/bin/oohelperd
+    checksum: "sha256:{{ oohelperd_checksum }}"
+    mode: "0755"
+    owner: root
+  register: oohelperd_binary
+
+- name: Create the ooni configuration directory
+  tags: [oohelperd, config]
+  ansible.builtin.file:
+    path: /etc/ooni
+    state: directory
+    owner: root
+    mode: "0755"
+
+# Read by systemd as root, hence not readable by the oohelperd user itself
+- name: Write the oohelperd environment file
+  tags: [oohelperd, secrets]
+  no_log: true
+  ansible.builtin.template:
+    src: oohelperd.env
+    dest: /etc/ooni/oohelperd.env
+    owner: root
+    group: root
+    mode: "0600"
+  register: oohelperd_env
+
+- name: Create the oohelperd.service file
+  tags: [oohelperd]
+  ansible.builtin.template:
+    src: oohelperd.service
+    dest: /etc/systemd/system/oohelperd.service
+    mode: "0644"
+    owner: root
+  register: oohelperd_unit
+
+- name: reload systemd
+  tags: [oohelperd]
+  ansible.builtin.systemd_service:
+    daemon_reload: yes
+  when: oohelperd_unit.changed
+
+- name: Start oohelperd
+  tags: [oohelperd]
+  ansible.builtin.systemd_service:
+    name: oohelperd.service
+    state: started
+    enabled: yes
+
+- name: Restart oohelperd if its binary, unit file or environment changed
+  tags: [oohelperd]
+  ansible.builtin.systemd_service:
+    name: oohelperd.service
+    state: restarted
+  when: oohelperd_binary.changed or oohelperd_unit.changed or oohelperd_env.changed
+
+# Expose the test helper to the probes
+- name: Add the oohelperd nginx vhost
+  tags: [oohelperd, nginx, config]
+  ansible.builtin.template:
+    src: nginx-oohelperd.j2
+    dest: /etc/nginx/sites-enabled/02-oohelperd
+    mode: "0644"
+    owner: root
+  notify:
+    - reload nginx
+
+- name: Verify oohelperd is answering requests
+  tags: [oohelperd]
+  # GET / is oohelperd's health check, it answers "Hello OONItarian!"
+  ansible.builtin.uri:
+    url: http://127.0.0.1:8080/
+    return_content: true
+  register: oohelperd_check
+  when: not ansible_check_mode
+  until: "'OONItarian' in oohelperd_check.content"
+  retries: 5
+  delay: 2
diff --git a/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2
new file mode 100644
index 00000000..0a97ae10
--- /dev/null
+++ b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2
@@ -0,0 +1,73 @@
+# Generated by ansible
+# roles/oohelperd/templates/nginx-oohelperd.j2
+
+# The probes send the same measurement request to the test helper over and
+# over, so responses are cached, keyed by the request body since they are POSTs
+proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M
+                 max_size=5g inactive=24h use_temp_path=off;
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name {{ inventory_hostname }};
+
+    # This server block is more specific than the catch-all one installed by
+    # the dehydrated role, so it must serve the ACME challenge itself
+    location ^~ /.well-known/acme-challenge {
+        alias /var/lib/dehydrated/acme-challenges;
+    }
+
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    # nginx >= 1.25.1, which is what nginx.org ships
+    http2 on;
+
+    server_name {{ inventory_hostname }};
+    include /etc/nginx/ssl_modern.conf;
+
+    ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem;
+    ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem;
+    ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem;
+
+    # Test helper application metrics. oohelperd protects them with basic auth
+    # using the password in /etc/ooni/oohelperd.env (user "prom"). Kept out of
+    # the cache so that prometheus always scrapes fresh values.
+    location = /metrics {
+        proxy_pass http://127.0.0.1:8080;
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    # Local test helper
+    location / {
+        proxy_pass http://127.0.0.1:8080;
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_read_timeout {{ oohelperd_nginx_proxy_timeout }};
+
+        proxy_cache thcache;
+        proxy_cache_min_uses 1;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 30;
+        proxy_cache_lock_age 30;
+        proxy_cache_use_stale error timeout invalid_header updating;
+        # Cache POST without headers set by the test helper!
+        proxy_cache_methods POST;
+        proxy_cache_key "$request_uri|$request_body";
+        proxy_cache_valid 200 10m;
+        proxy_cache_valid any 0;
+        add_header X-Cache-Status $upstream_cache_status;
+    }
+}
diff --git a/ansible/roles/oohelperd/templates/oohelperd.env b/ansible/roles/oohelperd/templates/oohelperd.env
new file mode 100644
index 00000000..80167997
--- /dev/null
+++ b/ansible/roles/oohelperd/templates/oohelperd.env
@@ -0,0 +1,5 @@
+# Generated by ansible
+# roles/oohelperd/templates/oohelperd.env
+
+# Password for the "prom" user on oohelperd's own /metrics endpoint
+PROMETHEUS_METRICS_PASSWORD={{ prometheus_metrics_password }}
diff --git a/ansible/roles/oohelperd/templates/oohelperd.service b/ansible/roles/oohelperd/templates/oohelperd.service
new file mode 100644
index 00000000..b09d034d
--- /dev/null
+++ b/ansible/roles/oohelperd/templates/oohelperd.service
@@ -0,0 +1,30 @@
+# Generated by ansible
+# roles/oohelperd/templates/oohelperd.service
+
+[Unit]
+Description=OONI Web Connectivity test helper
+Documentation=https://github.com/ooni/probe-cli
+After=network.target
+StartLimitIntervalSec=60
+StartLimitBurst=3
+
+[Service]
+Type=simple
+# nginx terminates TLS in front of it, so it only listens on loopback
+ExecStart=/usr/local/bin/oohelperd -api-endpoint 127.0.0.1:8080 -pprof-endpoint 127.0.0.1:6061
+
+# Holds PROMETHEUS_METRICS_PASSWORD, used to protect /metrics
+EnvironmentFile=/etc/ooni/oohelperd.env
+Restart=on-failure
+RestartSec=5
+User=oohelperd
+Group=oohelperd
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+PrivateTmp=yes
+# Every measurement opens several connections towards the target
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

From 8b4b4ff39932f805f895ae796f6b905effecf888 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 14:35:02 +0200
Subject: [PATCH 03/14] Update deploy script

---
 ansible/deploy-wc-test-helper.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ansible/deploy-wc-test-helper.yml b/ansible/deploy-wc-test-helper.yml
index 1b0ce980..b3cee6ff 100644
--- a/ansible/deploy-wc-test-helper.yml
+++ b/ansible/deploy-wc-test-helper.yml
@@ -1,5 +1,5 @@
 ---
-- name: Deploy test helpers
+- name: Deploy the Web Connectivity test helper
   hosts:
     - wc.th.dev.ooni.io
   become: true
@@ -12,3 +12,7 @@
     - role: nginx
     - role: dehydrated
     - role: prometheus_node_exporter
+      vars:
+        use_https: true
+        https_port: 9001
+    - role: oohelperd

From 9c0998800e14bf660c36267c611f883a4e1f89ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 16:35:38 +0200
Subject: [PATCH 04/14] Add support for deploying prod test helpers

---
 tf/environments/prod/main.tf | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf
index 801ee5f9..142e0c1f 100644
--- a/tf/environments/prod/main.tf
+++ b/tf/environments/prod/main.tf
@@ -442,6 +442,27 @@ module "ooni_th_droplet" {
   dns_zone_ooni_io = local.dns_zone_ooni_io
 }
 
+resource "digitalocean_ssh_key" "oonidevops" {
+  name       = "oonidevops"
+  public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"]
+}
+
+
+module "ooni_test_helpers_wc" {
+  source = "../../modules/ooni_th_binary_droplet"
+
+  stage    = local.environment
+  instance_location = "fra1"
+  name     = "ooniwcth"
+  hostname = "wcth${count.index}.fra1"
+
+  ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint]
+
+  dns_zone_ooni_io = local.dns_zone_ooni_io
+  count = 3
+}
+
+
 module "ooniapi_reverseproxy_deployer" {
   source = "../../modules/ooniapi_service_deployer"
 

From babb26efddb4bf078d5b3ed652fce9117597c338 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 16:36:22 +0200
Subject: [PATCH 05/14] Remove duplcate key definition

---
 tf/environments/prod/main.tf | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf
index 142e0c1f..4185f7a5 100644
--- a/tf/environments/prod/main.tf
+++ b/tf/environments/prod/main.tf
@@ -442,12 +442,6 @@ module "ooni_th_droplet" {
   dns_zone_ooni_io = local.dns_zone_ooni_io
 }
 
-resource "digitalocean_ssh_key" "oonidevops" {
-  name       = "oonidevops"
-  public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"]
-}
-
-
 module "ooni_test_helpers_wc" {
   source = "../../modules/ooni_th_binary_droplet"
 

From ed2e4bb9f3d1185b16021218b2df4990779509a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 16:36:43 +0200
Subject: [PATCH 06/14] terraform fmt

---
 tf/environments/prod/main.tf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf
index 4185f7a5..2553f6de 100644
--- a/tf/environments/prod/main.tf
+++ b/tf/environments/prod/main.tf
@@ -445,15 +445,15 @@ module "ooni_th_droplet" {
 module "ooni_test_helpers_wc" {
   source = "../../modules/ooni_th_binary_droplet"
 
-  stage    = local.environment
+  stage             = local.environment
   instance_location = "fra1"
-  name     = "ooniwcth"
-  hostname = "wcth${count.index}.fra1"
+  name              = "ooniwcth"
+  hostname          = "wcth${count.index}.fra1"
 
   ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint]
 
   dns_zone_ooni_io = local.dns_zone_ooni_io
-  count = 3
+  count            = 3
 }
 
 

From 790faf54c844947f9d51fd0cc202207fa7679ba9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 16:40:29 +0200
Subject: [PATCH 07/14] Add wcth[0-2] to inventory

---
 ansible/inventory | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ansible/inventory b/ansible/inventory
index bcc6e3d1..0abceec9 100644
--- a/ansible/inventory
+++ b/ansible/inventory
@@ -37,6 +37,9 @@ ams-ps.ooni.nu
 
 [do_fra]
 wc.th.dev.ooni.io
+wcth0.fra1.dev.ooni.io
+wcth1.fra1.dev.ooni.io
+wcth2.fra1.dev.ooni.io
 
 [aws-proxy]
 clickhouseproxy.dev.ooni.io

From 244830d2bc251af6b691d3507bac0e53439bdefc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 16:41:17 +0200
Subject: [PATCH 08/14] Fix inventory lines

---
 ansible/inventory | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ansible/inventory b/ansible/inventory
index 0abceec9..754130cd 100644
--- a/ansible/inventory
+++ b/ansible/inventory
@@ -37,9 +37,9 @@ ams-ps.ooni.nu
 
 [do_fra]
 wc.th.dev.ooni.io
-wcth0.fra1.dev.ooni.io
-wcth1.fra1.dev.ooni.io
-wcth2.fra1.dev.ooni.io
+wcth0.fra1.prod.ooni.io
+wcth1.fra1.prod.ooni.io
+wcth2.fra1.prod.ooni.io
 
 [aws-proxy]
 clickhouseproxy.dev.ooni.io

From e7615283c538a8b62b0ed1697368ab42129147e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 16:46:45 +0200
Subject: [PATCH 09/14] Add prod test helpers to deployment

---
 ansible/deploy-wc-test-helper.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ansible/deploy-wc-test-helper.yml b/ansible/deploy-wc-test-helper.yml
index b3cee6ff..9b415ddd 100644
--- a/ansible/deploy-wc-test-helper.yml
+++ b/ansible/deploy-wc-test-helper.yml
@@ -2,6 +2,9 @@
 - name: Deploy the Web Connectivity test helper
   hosts:
     - wc.th.dev.ooni.io
+    - wcth0.fra1.prod.ooni.io
+    - wcth1.fra1.prod.ooni.io
+    - wcth2.fra1.prod.ooni.io
   become: true
   vars:
     ssl_domains:

From 0414ed11c63d84c6f44fbec033735cecc0d34ba5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 16:54:14 +0200
Subject: [PATCH 10/14] Put flush all handlers sooner in the dehydrated role

---
 ansible/roles/dehydrated/tasks/main.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ansible/roles/dehydrated/tasks/main.yml b/ansible/roles/dehydrated/tasks/main.yml
index 860ffcc3..4dbd6c1a 100644
--- a/ansible/roles/dehydrated/tasks/main.yml
+++ b/ansible/roles/dehydrated/tasks/main.yml
@@ -92,4 +92,7 @@
     # crashing the playbook
     # 
     # See: https://github.com/ooni/devops/pull/235#discussion_r2053664605
-    - reload nginx and restart dehydrated
\ No newline at end of file
+    - reload nginx and restart dehydrated
+
+- name: Flush all handlers
+  meta: flush_handlers

From b477e78f33007326f684d96f660b7580ac921e20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 16:58:16 +0200
Subject: [PATCH 11/14] Set instance name in terraform config

---
 tf/environments/prod/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf
index 2553f6de..f167fe37 100644
--- a/tf/environments/prod/main.tf
+++ b/tf/environments/prod/main.tf
@@ -447,7 +447,7 @@ module "ooni_test_helpers_wc" {
 
   stage             = local.environment
   instance_location = "fra1"
-  name              = "ooniwcth"
+  name              = "ooniwcth${count.index}-fra1"
   hostname          = "wcth${count.index}.fra1"
 
   ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint]

From 1435e1d113263fb5747d4265c80a7376880660fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 17:08:31 +0200
Subject: [PATCH 12/14] More debugging of dehydrated role

---
 ansible/roles/dehydrated/handlers/main.yml | 12 +++++-----
 ansible/roles/dehydrated/tasks/main.yml    | 26 +++++++++++++---------
 2 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/ansible/roles/dehydrated/handlers/main.yml b/ansible/roles/dehydrated/handlers/main.yml
index a539854a..71d72a81 100644
--- a/ansible/roles/dehydrated/handlers/main.yml
+++ b/ansible/roles/dehydrated/handlers/main.yml
@@ -3,6 +3,11 @@
     name: nginx
     state: reloaded
 
+- name: restart nginx
+  service:
+    name: nginx
+    state: restarted
+
 - name: reload nftables
   service: 
     name: nftables
@@ -13,10 +18,3 @@
     name: dehydrated
     state: restarted
     enabled: yes
-
-- name: reload nginx and restart dehydrated
-  service: 
-    name: nginx
-    state: reloaded
-  notify:
-    restart dehydrated
\ No newline at end of file
diff --git a/ansible/roles/dehydrated/tasks/main.yml b/ansible/roles/dehydrated/tasks/main.yml
index 4dbd6c1a..56af5a5d 100644
--- a/ansible/roles/dehydrated/tasks/main.yml
+++ b/ansible/roles/dehydrated/tasks/main.yml
@@ -82,17 +82,21 @@
     state: started
     enabled: yes
   notify:
-    # creates:
-    # /var/lib/dehydrated/certs/<name>/chain.pem cert.pem privkey.pem fullchain.pem
-
-    # Note that we need to restart dehydrated ensuring that nginx reloads before dehydrated restarts.
-    # When we first run dehydrated with the tasks above it creates an nginx rule that is required
-    # to pass the ACME challenge. 
-    # If nginx doesn't picks this rule before dehydrated runs again, the ACME challenge will fail
-    # crashing the playbook
-    # 
-    # See: https://github.com/ooni/devops/pull/235#discussion_r2053664605
-    - reload nginx and restart dehydrated
+    - restart nginx
 
 - name: Flush all handlers
+  tags: [dehydrated, certs]
   meta: flush_handlers
+
+- name: Ensure nginx is running
+  tags: [dehydrated, certs]
+  ansible.builtin.systemd_service:
+    state: started
+    name: nginx
+
+- name: Force the restarting of the dehydrated service
+  tags: [dehydrated, certs]
+  ansible.builtin.systemd_service:
+    state: restarted
+    daemon_reload: true
+    name: dehydrated

From b8e59566034b9b18ab30494dae5375da9c18a831 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 17:15:19 +0200
Subject: [PATCH 13/14] Add new test helpers to monitoring

---
 ansible/roles/prometheus/templates/prometheus.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml
index 2790f27e..1e1d2028 100755
--- a/ansible/roles/prometheus/templates/prometheus.yml
+++ b/ansible/roles/prometheus/templates/prometheus.yml
@@ -208,6 +208,9 @@ scrape_configs:
         - 0.do.th.prod.ooni.io
         - 1.do.th.prod.ooni.io
         - 2.do.th.prod.ooni.io
+        - wcth0.fra1.prod.ooni.io
+        - wcth1.fra1.prod.ooni.io
+        - wcth2.fra1.prod.ooni.io
   - job_name: 'ooni-web'
     scrape_interval: 5m
     scheme: https

From 8009f9db94a13b3da4882be4e308266e6e9927a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 19 Aug 2026 17:16:05 +0200
Subject: [PATCH 14/14] Add test helpers to monitoring

---
 ansible/roles/prometheus/vars/main.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml
index 1da7f053..1e819869 100644
--- a/ansible/roles/prometheus/vars/main.yml
+++ b/ansible/roles/prometheus/vars/main.yml
@@ -25,6 +25,9 @@ blackbox_jobs:
       - "https://4.th.ooni.org/"
       - "https://5.th.ooni.org/"
       - "https://6.th.ooni.org/"
+      - "https://wcth0.fra1.prod.ooni.io/"
+      - "https://wcth1.fra1.prod.ooni.io/"
+      - "https://wcth2.fra1.prod.ooni.io/"
       - "https://d33d1gs9kpq1c5.cloudfront.net/status"
 
   - name: "ooni collector"