Web Connectivity test helper deployment

ansible/deploy-wc-test-helper.yml

@@ -1,7 +1,10 @@
 ---
-- name: Deploy test helpers
+- name: Deploy the Web Connectivity test helper
   hosts:
     - wc.th.dev.ooni.io
+    - wcth0.fra1.prod.ooni.io
+    - wcth1.fra1.prod.ooni.io
+    - wcth2.fra1.prod.ooni.io
   become: true
   vars:
     ssl_domains:
@@ -12,3 +15,7 @@
     - role: nginx
     - role: dehydrated
     - role: prometheus_node_exporter
+      vars:
+        use_https: true
+        https_port: 9001
+    - role: oohelperd

ansible/inventory

@@ -37,6 +37,9 @@ ams-ps.ooni.nu
 
 [do_fra]
 wc.th.dev.ooni.io
+wcth0.fra1.prod.ooni.io
+wcth1.fra1.prod.ooni.io
+wcth2.fra1.prod.ooni.io
 
 [aws-proxy]
 clickhouseproxy.dev.ooni.io

ansible/roles/dehydrated/handlers/main.yml

@@ -3,6 +3,11 @@
     name: nginx
     state: reloaded
 
+- name: restart nginx
+  service:
+    name: nginx
+    state: restarted
+
 - name: reload nftables
   service: 
     name: nftables
@@ -13,10 +18,3 @@
     name: dehydrated
     state: restarted
     enabled: yes
-
-- name: reload nginx and restart dehydrated
-  service: 
-    name: nginx
-    state: reloaded
-  notify:
-    restart dehydrated

ansible/roles/dehydrated/tasks/main.yml

@@ -82,14 +82,21 @@
     state: started
     enabled: yes
   notify:
-    # creates:
-    # /var/lib/dehydrated/certs/<name>/chain.pem cert.pem privkey.pem fullchain.pem
+    - restart nginx
 
-    # Note that we need to restart dehydrated ensuring that nginx reloads before dehydrated restarts.
-    # When we first run dehydrated with the tasks above it creates an nginx rule that is required
-    # to pass the ACME challenge. 
-    # If nginx doesn't picks this rule before dehydrated runs again, the ACME challenge will fail
-    # crashing the playbook
-    # 
-    # See: https://github.com/ooni/devops/pull/235#discussion_r2053664605
-    - reload nginx and restart dehydrated
+- name: Flush all handlers
+  tags: [dehydrated, certs]
+  meta: flush_handlers
+
+- name: Ensure nginx is running
+  tags: [dehydrated, certs]
+  ansible.builtin.systemd_service:
+    state: started
+    name: nginx
+
+- name: Force the restarting of the dehydrated service
+  tags: [dehydrated, certs]
+  ansible.builtin.systemd_service:
+    state: restarted
+    daemon_reload: true
+    name: dehydrated

ansible/roles/oohelperd/defaults/main.yml

@@ -0,0 +1,10 @@
+# The Web Connectivity test helper (oohelperd) is built and released as part of
+# https://github.com/ooni/probe-cli
+oohelperd_version: v3.30.0
+oohelperd_base_url: "https://github.com/ooni/probe-cli/releases/download/{{ oohelperd_version }}"
+# sha256 of {{ oohelperd_base_url }}/oohelperd-linux-amd64
+oohelperd_checksum: "fff38b9366fecae400cb3f616ffa2a37dba211c33ee4e1c1b343bff1c384d48e"
+
+# A test helper request measures the target before it can answer, so it needs
+# way more than the nginx default of 60s
+oohelperd_nginx_proxy_timeout: 900

ansible/roles/oohelperd/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded

ansible/roles/oohelperd/tasks/main.yml

@@ -0,0 +1,93 @@
+---
+- name: Flush all handlers now
+  ansible.builtin.meta: flush_handlers
+
+- name: Create the oohelperd user
+  tags: [oohelperd]
+  ansible.builtin.user:
+    name: oohelperd
+    shell: /usr/sbin/nologin
+    create_home: no
+    system: yes
+
+- name: Download the oohelperd binary
+  tags: [oohelperd]
+  ansible.builtin.get_url:
+    url: "{{ oohelperd_base_url }}/oohelperd-linux-amd64"
+    dest: /usr/local/bin/oohelperd
+    checksum: "sha256:{{ oohelperd_checksum }}"
+    mode: "0755"
+    owner: root
+  register: oohelperd_binary
+
+- name: Create the ooni configuration directory
+  tags: [oohelperd, config]
+  ansible.builtin.file:
+    path: /etc/ooni
+    state: directory
+    owner: root
+    mode: "0755"
+
+# Read by systemd as root, hence not readable by the oohelperd user itself
+- name: Write the oohelperd environment file
+  tags: [oohelperd, secrets]
+  no_log: true
+  ansible.builtin.template:
+    src: oohelperd.env
+    dest: /etc/ooni/oohelperd.env
+    owner: root
+    group: root
+    mode: "0600"
+  register: oohelperd_env
+
+- name: Create the oohelperd.service file
+  tags: [oohelperd]
+  ansible.builtin.template:
+    src: oohelperd.service
+    dest: /etc/systemd/system/oohelperd.service
+    mode: "0644"
+    owner: root
+  register: oohelperd_unit
+
+- name: reload systemd
+  tags: [oohelperd]
+  ansible.builtin.systemd_service:
+    daemon_reload: yes
+  when: oohelperd_unit.changed
+
+- name: Start oohelperd
+  tags: [oohelperd]
+  ansible.builtin.systemd_service:
+    name: oohelperd.service
+    state: started
+    enabled: yes
+
+- name: Restart oohelperd if its binary, unit file or environment changed
+  tags: [oohelperd]
+  ansible.builtin.systemd_service:
+    name: oohelperd.service
+    state: restarted
+  when: oohelperd_binary.changed or oohelperd_unit.changed or oohelperd_env.changed
+
+# Expose the test helper to the probes
+- name: Add the oohelperd nginx vhost
+  tags: [oohelperd, nginx, config]
+  ansible.builtin.template:
+    src: nginx-oohelperd.j2
+    dest: /etc/nginx/sites-enabled/02-oohelperd
+    mode: "0644"
+    owner: root
+  notify:
+    - reload nginx
+
+- name: Verify oohelperd is answering requests
+  tags: [oohelperd]
+  # GET / is oohelperd's health check, it answers "Hello OONItarian!"
+  ansible.builtin.uri:
+    url: http://127.0.0.1:8080/
+    return_content: true
+  register: oohelperd_check
+  when: not ansible_check_mode
+  until: "'OONItarian' in oohelperd_check.content"
+  retries: 5
+  delay: 2

ansible/roles/oohelperd/templates/nginx-oohelperd.j2

@@ -0,0 +1,73 @@
+# Generated by ansible
+# roles/oohelperd/templates/nginx-oohelperd.j2
+
+# The probes send the same measurement request to the test helper over and
+# over, so responses are cached, keyed by the request body since they are POSTs
+proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M
+                 max_size=5g inactive=24h use_temp_path=off;
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name {{ inventory_hostname }};
+
+    # This server block is more specific than the catch-all one installed by
+    # the dehydrated role, so it must serve the ACME challenge itself
+    location ^~ /.well-known/acme-challenge {
+        alias /var/lib/dehydrated/acme-challenges;
+    }
+
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    # nginx >= 1.25.1, which is what nginx.org ships
+    http2 on;
+
+    server_name {{ inventory_hostname }};
+    include /etc/nginx/ssl_modern.conf;
+
+    ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem;
+    ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem;
+    ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem;
+
+    # Test helper application metrics. oohelperd protects them with basic auth
+    # using the password in /etc/ooni/oohelperd.env (user "prom"). Kept out of
+    # the cache so that prometheus always scrapes fresh values.
+    location = /metrics {
+        proxy_pass http://127.0.0.1:8080;
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    # Local test helper
+    location / {
+        proxy_pass http://127.0.0.1:8080;
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_read_timeout {{ oohelperd_nginx_proxy_timeout }};
+
+        proxy_cache thcache;
+        proxy_cache_min_uses 1;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 30;
+        proxy_cache_lock_age 30;
+        proxy_cache_use_stale error timeout invalid_header updating;
+        # Cache POST without headers set by the test helper!
+        proxy_cache_methods POST;
+        proxy_cache_key "$request_uri|$request_body";
+        proxy_cache_valid 200 10m;
+        proxy_cache_valid any 0;
+        add_header X-Cache-Status $upstream_cache_status;
+    }
+}

ansible/roles/oohelperd/templates/oohelperd.env

@@ -0,0 +1,5 @@
+# Generated by ansible
+# roles/oohelperd/templates/oohelperd.env
+
+# Password for the "prom" user on oohelperd's own /metrics endpoint
+PROMETHEUS_METRICS_PASSWORD={{ prometheus_metrics_password }}

ansible/roles/oohelperd/templates/oohelperd.service

@@ -0,0 +1,30 @@
+# Generated by ansible
+# roles/oohelperd/templates/oohelperd.service
+
+[Unit]
+Description=OONI Web Connectivity test helper
+Documentation=https://github.com/ooni/probe-cli
+After=network.target
+StartLimitIntervalSec=60
+StartLimitBurst=3
+
+[Service]
+Type=simple
+# nginx terminates TLS in front of it, so it only listens on loopback
+ExecStart=/usr/local/bin/oohelperd -api-endpoint 127.0.0.1:8080 -pprof-endpoint 127.0.0.1:6061
+
+# Holds PROMETHEUS_METRICS_PASSWORD, used to protect /metrics
+EnvironmentFile=/etc/ooni/oohelperd.env
+Restart=on-failure
+RestartSec=5
+User=oohelperd
+Group=oohelperd
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+PrivateTmp=yes
+# Every measurement opens several connections towards the target
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/prometheus/templates/prometheus.yml

@@ -208,6 +208,9 @@ scrape_configs:
         - 0.do.th.prod.ooni.io
         - 1.do.th.prod.ooni.io
         - 2.do.th.prod.ooni.io
+        - wcth0.fra1.prod.ooni.io
+        - wcth1.fra1.prod.ooni.io
+        - wcth2.fra1.prod.ooni.io
   - job_name: 'ooni-web'
     scrape_interval: 5m
     scheme: https

ansible/roles/prometheus/vars/main.yml

@@ -25,6 +25,9 @@ blackbox_jobs:
       - "https://4.th.ooni.org/"
       - "https://5.th.ooni.org/"
       - "https://6.th.ooni.org/"
+      - "https://wcth0.fra1.prod.ooni.io/"
+      - "https://wcth1.fra1.prod.ooni.io/"
+      - "https://wcth2.fra1.prod.ooni.io/"
       - "https://d33d1gs9kpq1c5.cloudfront.net/status"
 
   - name: "ooni collector"

ansible/roles/prometheus_node_exporter/defaults/main.yml

@@ -18,3 +18,7 @@ node_exporter_options: ''
 node_exporter_state: started
 node_exporter_enabled: true
 node_exporter_restart: on-failure
+http_port: 9001
+https_port: 443
+use_https: false
+monitoring_server_ip: "{{ lookup('dig', 'monitoring.ooni.org') }}"

ansible/roles/prometheus_node_exporter/handlers/main.yml

@@ -18,3 +18,9 @@
   service:
     name: node_exporter
     state: restarted
+
+- name: reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded

ansible/roles/prometheus_node_exporter/tasks/main.yml

@@ -23,4 +23,30 @@
     - config
   when: use_nginx
 
+- name: Allow traffic on the node exporter http_port from the monitoring host only
+  become: true
+  tags: [prometheus_node_exporter, monitoring, node_exporter, nftables]
+  ansible.builtin.blockinfile:
+    path: "/etc/ooni/nftables/tcp/{{ http_port }}.nft"
+    create: yes
+    mode: "0644"
+    block: |
+      add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport {{ http_port }} counter accept comment "node exporter metrics"
+  when: use_nginx and not use_https
+  notify:
+    - reload nftables
+
+- name: Allow traffic on the node exporter https_port from the monitoring host only
+  become: true
+  tags: [prometheus_node_exporter, monitoring, node_exporter, nftables]
+  ansible.builtin.blockinfile:
+    path: "/etc/ooni/nftables/tcp/{{ http_port }}.nft"
+    create: yes
+    mode: "0644"
+    block: |
+      add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport {{ https_port }} counter accept comment "node exporter metrics"
+  when: use_nginx and use_https
+  notify:
+    - reload nftables
+
 - include_tasks: install.yml