diff --git a/ansible/deploy-wc-test-helper.yml b/ansible/deploy-wc-test-helper.yml index 1b0ce980..9b415ddd 100644 --- a/ansible/deploy-wc-test-helper.yml +++ b/ansible/deploy-wc-test-helper.yml @@ -1,7 +1,10 @@ --- -- name: Deploy test helpers +- name: Deploy the Web Connectivity test helper hosts: - wc.th.dev.ooni.io + - wcth0.fra1.prod.ooni.io + - wcth1.fra1.prod.ooni.io + - wcth2.fra1.prod.ooni.io become: true vars: ssl_domains: @@ -12,3 +15,7 @@ - role: nginx - role: dehydrated - role: prometheus_node_exporter + vars: + use_https: true + https_port: 9001 + - role: oohelperd diff --git a/ansible/inventory b/ansible/inventory index bcc6e3d1..754130cd 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -37,6 +37,9 @@ ams-ps.ooni.nu [do_fra] wc.th.dev.ooni.io +wcth0.fra1.prod.ooni.io +wcth1.fra1.prod.ooni.io +wcth2.fra1.prod.ooni.io [aws-proxy] clickhouseproxy.dev.ooni.io diff --git a/ansible/roles/dehydrated/handlers/main.yml b/ansible/roles/dehydrated/handlers/main.yml index a539854a..71d72a81 100644 --- a/ansible/roles/dehydrated/handlers/main.yml +++ b/ansible/roles/dehydrated/handlers/main.yml @@ -3,6 +3,11 @@ name: nginx state: reloaded +- name: restart nginx + service: + name: nginx + state: restarted + - name: reload nftables service: name: nftables @@ -13,10 +18,3 @@ name: dehydrated state: restarted enabled: yes - -- name: reload nginx and restart dehydrated - service: - name: nginx - state: reloaded - notify: - restart dehydrated \ No newline at end of file diff --git a/ansible/roles/dehydrated/tasks/main.yml b/ansible/roles/dehydrated/tasks/main.yml index 860ffcc3..56af5a5d 100644 --- a/ansible/roles/dehydrated/tasks/main.yml +++ b/ansible/roles/dehydrated/tasks/main.yml @@ -82,14 +82,21 @@ state: started enabled: yes notify: - # creates: - # /var/lib/dehydrated/certs//chain.pem cert.pem privkey.pem fullchain.pem + - restart nginx - # Note that we need to restart dehydrated ensuring that nginx reloads before dehydrated restarts. - # When we first run dehydrated with the tasks above it creates an nginx rule that is required - # to pass the ACME challenge. - # If nginx doesn't picks this rule before dehydrated runs again, the ACME challenge will fail - # crashing the playbook - # - # See: https://github.com/ooni/devops/pull/235#discussion_r2053664605 - - reload nginx and restart dehydrated \ No newline at end of file +- name: Flush all handlers + tags: [dehydrated, certs] + meta: flush_handlers + +- name: Ensure nginx is running + tags: [dehydrated, certs] + ansible.builtin.systemd_service: + state: started + name: nginx + +- name: Force the restarting of the dehydrated service + tags: [dehydrated, certs] + ansible.builtin.systemd_service: + state: restarted + daemon_reload: true + name: dehydrated diff --git a/ansible/roles/oohelperd/defaults/main.yml b/ansible/roles/oohelperd/defaults/main.yml new file mode 100644 index 00000000..1c4f7a86 --- /dev/null +++ b/ansible/roles/oohelperd/defaults/main.yml @@ -0,0 +1,10 @@ +# The Web Connectivity test helper (oohelperd) is built and released as part of +# https://github.com/ooni/probe-cli +oohelperd_version: v3.30.0 +oohelperd_base_url: "https://github.com/ooni/probe-cli/releases/download/{{ oohelperd_version }}" +# sha256 of {{ oohelperd_base_url }}/oohelperd-linux-amd64 +oohelperd_checksum: "fff38b9366fecae400cb3f616ffa2a37dba211c33ee4e1c1b343bff1c384d48e" + +# A test helper request measures the target before it can answer, so it needs +# way more than the nginx default of 60s +oohelperd_nginx_proxy_timeout: 900 diff --git a/ansible/roles/oohelperd/handlers/main.yml b/ansible/roles/oohelperd/handlers/main.yml new file mode 100644 index 00000000..71ffc659 --- /dev/null +++ b/ansible/roles/oohelperd/handlers/main.yml @@ -0,0 +1,4 @@ +- name: reload nginx + service: + name: nginx + state: reloaded diff --git a/ansible/roles/oohelperd/tasks/main.yml b/ansible/roles/oohelperd/tasks/main.yml new file mode 100644 index 00000000..c20d5e63 --- /dev/null +++ b/ansible/roles/oohelperd/tasks/main.yml @@ -0,0 +1,93 @@ +--- +- name: Flush all handlers now + ansible.builtin.meta: flush_handlers + +- name: Create the oohelperd user + tags: [oohelperd] + ansible.builtin.user: + name: oohelperd + shell: /usr/sbin/nologin + create_home: no + system: yes + +- name: Download the oohelperd binary + tags: [oohelperd] + ansible.builtin.get_url: + url: "{{ oohelperd_base_url }}/oohelperd-linux-amd64" + dest: /usr/local/bin/oohelperd + checksum: "sha256:{{ oohelperd_checksum }}" + mode: "0755" + owner: root + register: oohelperd_binary + +- name: Create the ooni configuration directory + tags: [oohelperd, config] + ansible.builtin.file: + path: /etc/ooni + state: directory + owner: root + mode: "0755" + +# Read by systemd as root, hence not readable by the oohelperd user itself +- name: Write the oohelperd environment file + tags: [oohelperd, secrets] + no_log: true + ansible.builtin.template: + src: oohelperd.env + dest: /etc/ooni/oohelperd.env + owner: root + group: root + mode: "0600" + register: oohelperd_env + +- name: Create the oohelperd.service file + tags: [oohelperd] + ansible.builtin.template: + src: oohelperd.service + dest: /etc/systemd/system/oohelperd.service + mode: "0644" + owner: root + register: oohelperd_unit + +- name: reload systemd + tags: [oohelperd] + ansible.builtin.systemd_service: + daemon_reload: yes + when: oohelperd_unit.changed + +- name: Start oohelperd + tags: [oohelperd] + ansible.builtin.systemd_service: + name: oohelperd.service + state: started + enabled: yes + +- name: Restart oohelperd if its binary, unit file or environment changed + tags: [oohelperd] + ansible.builtin.systemd_service: + name: oohelperd.service + state: restarted + when: oohelperd_binary.changed or oohelperd_unit.changed or oohelperd_env.changed + +# Expose the test helper to the probes +- name: Add the oohelperd nginx vhost + tags: [oohelperd, nginx, config] + ansible.builtin.template: + src: nginx-oohelperd.j2 + dest: /etc/nginx/sites-enabled/02-oohelperd + mode: "0644" + owner: root + notify: + - reload nginx + +- name: Verify oohelperd is answering requests + tags: [oohelperd] + # GET / is oohelperd's health check, it answers "Hello OONItarian!" + ansible.builtin.uri: + url: http://127.0.0.1:8080/ + return_content: true + register: oohelperd_check + when: not ansible_check_mode + until: "'OONItarian' in oohelperd_check.content" + retries: 5 + delay: 2 diff --git a/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 new file mode 100644 index 00000000..0a97ae10 --- /dev/null +++ b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 @@ -0,0 +1,73 @@ +# Generated by ansible +# roles/oohelperd/templates/nginx-oohelperd.j2 + +# The probes send the same measurement request to the test helper over and +# over, so responses are cached, keyed by the request body since they are POSTs +proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M + max_size=5g inactive=24h use_temp_path=off; + +server { + listen 80; + listen [::]:80; + server_name {{ inventory_hostname }}; + + # This server block is more specific than the catch-all one installed by + # the dehydrated role, so it must serve the ACME challenge itself + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + # nginx >= 1.25.1, which is what nginx.org ships + http2 on; + + server_name {{ inventory_hostname }}; + include /etc/nginx/ssl_modern.conf; + + ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; + + # Test helper application metrics. oohelperd protects them with basic auth + # using the password in /etc/ooni/oohelperd.env (user "prom"). Kept out of + # the cache so that prometheus always scrapes fresh values. + location = /metrics { + proxy_pass http://127.0.0.1:8080; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Local test helper + location / { + proxy_pass http://127.0.0.1:8080; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout {{ oohelperd_nginx_proxy_timeout }}; + + proxy_cache thcache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + # Cache POST without headers set by the test helper! + proxy_cache_methods POST; + proxy_cache_key "$request_uri|$request_body"; + proxy_cache_valid 200 10m; + proxy_cache_valid any 0; + add_header X-Cache-Status $upstream_cache_status; + } +} diff --git a/ansible/roles/oohelperd/templates/oohelperd.env b/ansible/roles/oohelperd/templates/oohelperd.env new file mode 100644 index 00000000..80167997 --- /dev/null +++ b/ansible/roles/oohelperd/templates/oohelperd.env @@ -0,0 +1,5 @@ +# Generated by ansible +# roles/oohelperd/templates/oohelperd.env + +# Password for the "prom" user on oohelperd's own /metrics endpoint +PROMETHEUS_METRICS_PASSWORD={{ prometheus_metrics_password }} diff --git a/ansible/roles/oohelperd/templates/oohelperd.service b/ansible/roles/oohelperd/templates/oohelperd.service new file mode 100644 index 00000000..b09d034d --- /dev/null +++ b/ansible/roles/oohelperd/templates/oohelperd.service @@ -0,0 +1,30 @@ +# Generated by ansible +# roles/oohelperd/templates/oohelperd.service + +[Unit] +Description=OONI Web Connectivity test helper +Documentation=https://github.com/ooni/probe-cli +After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +Type=simple +# nginx terminates TLS in front of it, so it only listens on loopback +ExecStart=/usr/local/bin/oohelperd -api-endpoint 127.0.0.1:8080 -pprof-endpoint 127.0.0.1:6061 + +# Holds PROMETHEUS_METRICS_PASSWORD, used to protect /metrics +EnvironmentFile=/etc/ooni/oohelperd.env +Restart=on-failure +RestartSec=5 +User=oohelperd +Group=oohelperd +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +PrivateTmp=yes +# Every measurement opens several connections towards the target +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index 2790f27e..1e1d2028 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -208,6 +208,9 @@ scrape_configs: - 0.do.th.prod.ooni.io - 1.do.th.prod.ooni.io - 2.do.th.prod.ooni.io + - wcth0.fra1.prod.ooni.io + - wcth1.fra1.prod.ooni.io + - wcth2.fra1.prod.ooni.io - job_name: 'ooni-web' scrape_interval: 5m scheme: https diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index 1da7f053..1e819869 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -25,6 +25,9 @@ blackbox_jobs: - "https://4.th.ooni.org/" - "https://5.th.ooni.org/" - "https://6.th.ooni.org/" + - "https://wcth0.fra1.prod.ooni.io/" + - "https://wcth1.fra1.prod.ooni.io/" + - "https://wcth2.fra1.prod.ooni.io/" - "https://d33d1gs9kpq1c5.cloudfront.net/status" - name: "ooni collector" diff --git a/ansible/roles/prometheus_node_exporter/defaults/main.yml b/ansible/roles/prometheus_node_exporter/defaults/main.yml index 9066e94c..0cdc0011 100644 --- a/ansible/roles/prometheus_node_exporter/defaults/main.yml +++ b/ansible/roles/prometheus_node_exporter/defaults/main.yml @@ -18,3 +18,7 @@ node_exporter_options: '' node_exporter_state: started node_exporter_enabled: true node_exporter_restart: on-failure +http_port: 9001 +https_port: 443 +use_https: false +monitoring_server_ip: "{{ lookup('dig', 'monitoring.ooni.org') }}" diff --git a/ansible/roles/prometheus_node_exporter/handlers/main.yml b/ansible/roles/prometheus_node_exporter/handlers/main.yml index 4ec66003..092c7285 100644 --- a/ansible/roles/prometheus_node_exporter/handlers/main.yml +++ b/ansible/roles/prometheus_node_exporter/handlers/main.yml @@ -18,3 +18,9 @@ service: name: node_exporter state: restarted + +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index 7512d5b9..35e32438 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -23,4 +23,30 @@ - config when: use_nginx +- name: Allow traffic on the node exporter http_port from the monitoring host only + become: true + tags: [prometheus_node_exporter, monitoring, node_exporter, nftables] + ansible.builtin.blockinfile: + path: "/etc/ooni/nftables/tcp/{{ http_port }}.nft" + create: yes + mode: "0644" + block: | + add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport {{ http_port }} counter accept comment "node exporter metrics" + when: use_nginx and not use_https + notify: + - reload nftables + +- name: Allow traffic on the node exporter https_port from the monitoring host only + become: true + tags: [prometheus_node_exporter, monitoring, node_exporter, nftables] + ansible.builtin.blockinfile: + path: "/etc/ooni/nftables/tcp/{{ http_port }}.nft" + create: yes + mode: "0644" + block: | + add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport {{ https_port }} counter accept comment "node exporter metrics" + when: use_nginx and use_https + notify: + - reload nftables + - include_tasks: install.yml diff --git a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 index 6fb7c5ca..5af3c130 100644 --- a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 +++ b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 @@ -2,7 +2,7 @@ server { {% if use_https %} - listen 443 ssl http2; + listen {{ https_port }} ssl http2; server_name {{ inventory_hostname }}; include /etc/nginx/ssl_intermediate.conf; @@ -11,9 +11,9 @@ server { ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; {% else %} - listen {{http_port}}; + listen {{ http_port }}; - server_name {{inventory_hostname}}; + server_name {{ inventory_hostname }}; {% endif %} {% for config in prometheus_nginx_proxy_config %} @@ -30,4 +30,4 @@ server { } {% endfor %} -} \ No newline at end of file +} diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 801ee5f9..f167fe37 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -442,6 +442,21 @@ module "ooni_th_droplet" { dns_zone_ooni_io = local.dns_zone_ooni_io } +module "ooni_test_helpers_wc" { + source = "../../modules/ooni_th_binary_droplet" + + stage = local.environment + instance_location = "fra1" + name = "ooniwcth${count.index}-fra1" + hostname = "wcth${count.index}.fra1" + + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] + + dns_zone_ooni_io = local.dns_zone_ooni_io + count = 3 +} + + module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer"