Test helpers machine

ansible/deploy-echo-test-helper.yml

@@ -0,0 +1,23 @@
+---
+- name: Deploy test helpers
+  hosts:
+    - echo.th.dev.ooni.io
+    - echo.th.prod.ooni.io
+  become: true
+  roles:
+    - role: bootstrap
+    - role: prometheus_node_exporter
+      vars:
+        node_exporter_port: 9100
+        node_exporter_host: "0.0.0.0"
+        prometheus_nginx_proxy_config:
+          - location: /metrics/node_exporter
+            proxy_pass: http://127.0.0.1:9100/metrics
+        use_https: false
+        http_port: 8080 # if we leave port 80, it's taken by nginx
+    - role: test_helpers
+      vars:
+        helper: ooechohelpd
+        # sha256 of https://github.com/ooni/ooniprobe-rs/releases/download/v0.1.6/ooechohelpd
+        test_helpers_checksum: 10511a8b918c2244eba9e17c32f3cec9efbc5ffa51d6ad7f59fe648cd0986615
+        port: 80

ansible/deploy-json-test-helper.yml

@@ -0,0 +1,23 @@
+---
+- name: Deploy test helpers
+  hosts:
+    - json.th.dev.ooni.io
+    - json.th.prod.ooni.io
+  become: true
+  roles:
+    - role: bootstrap
+    - role: prometheus_node_exporter
+      vars:
+        node_exporter_port: 9100
+        node_exporter_host: "0.0.0.0"
+        prometheus_nginx_proxy_config:
+          - location: /metrics/node_exporter
+            proxy_pass: http://127.0.0.1:9100/metrics
+        use_https: false
+        http_port: 8080 # if we leave port 80, it's taken by nginx
+    - role: test_helpers
+      vars:
+        helper: oojsonhelpd
+        # sha256 of https://github.com/ooni/ooniprobe-rs/releases/download/v0.1.6/oojsonhelpd
+        test_helpers_checksum: 3d977e1418ea398f520dff3de951e36010a294dc3d1b564c961542cc3729a224
+        port: 80

ansible/inventory

@@ -48,6 +48,10 @@ openvpn2.htz-fsn.prod.ooni.nu
 [aws-backend]
 fastpath.dev.ooni.io
 fastpath.prod.ooni.io
+json.th.dev.ooni.io
+echo.th.dev.ooni.io
+json.th.prod.ooni.io
+echo.th.prod.ooni.io
 fastpath2.prod.ooni.io
 reuploaderfastpath.prod.ooni.io
 anonc.dev.ooni.io

ansible/requirements/ansible-galaxy.yml

@@ -1,6 +1,6 @@
 roles: 
   - src: nginxinc.nginx
-    version: 0.24.3
+    version: 0.26.0
   - src: geerlingguy.certbot
     version: 5.2.0
   - src: https://github.com/idealista/clickhouse_role

ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2

@@ -11,9 +11,9 @@ server {
     ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem;
     ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem;
     {% else %}
-    listen 80;
+    listen {{http_port}};
 
-    server_name {{ inventory_hostname }};
+    server_name {{inventory_hostname}};
     {% endif %}
 
     {% for config in prometheus_nginx_proxy_config %}

ansible/roles/ssh_users/tasks/main.yml

@@ -85,8 +85,8 @@
     path: /etc/sudoers.d/adm
     state: absent
 
-- name: reload sshd
+- name: reload ssh
   tags: ssh_users
   ansible.builtin.systemd_service:
-    name: sshd
+    name: ssh
     state: reloaded

ansible/roles/test_helpers/defaults/main.yml

@@ -0,0 +1,5 @@
+test_helpers_version: v0.1.6
+test_helpers_base_url: "https://github.com/ooni/ooniprobe-rs/releases/download/{{ test_helpers_version }}"
+
+# monitoring server IP
+monitoring_server_ip: "{{ lookup('dig', 'monitoring.ooni.org') }}"

ansible/roles/test_helpers/handlers/main.yml

@@ -0,0 +1,10 @@
+- name: reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded
+
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded

ansible/roles/test_helpers/tasks/main.yml

@@ -0,0 +1,76 @@
+---
+
+# For prometheus scrape requests.
+- name: Allow traffic on port 9100 from the monitoring host only
+  become: true
+  tags: prometheus-proxy
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/9100.nft
+    create: yes
+    block: |
+      add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport 9100 counter accept comment "node exporter"
+  notify:
+   - reload nftables
+
+- name: Allow traffic on the {{ helper }} test helper port
+  become: true
+  tags: test-helpers
+  blockinfile:
+    path: "/etc/ooni/nftables/tcp/{{ port }}.nft"
+    create: yes
+    block: |
+      add rule inet filter input tcp dport {{ port }} counter accept comment "{{ helper }} test helper"
+  notify:
+   - reload nftables
+
+# Create test helpers user
+- name: Create the testhelpers user
+  ansible.builtin.user:
+    name: "testhelpers"
+    shell: "/bin/bash"
+    create_home: no
+    system: yes
+  become: yes
+
+# Install test helpers
+- name: Download the {{ helper }} test helper binary
+  ansible.builtin.get_url:
+    url: "{{ test_helpers_base_url }}/{{ helper }}"
+    dest: "/usr/local/bin/{{ helper }}"
+    checksum: "sha256:{{ test_helpers_checksum }}"
+    mode: '0755'
+    force: yes
+  become: yes
+  register: helper_binary
+
+# Create systemd units
+
+- name: Create <helper>.service file
+  tags: test-helpers
+  ansible.builtin.template:
+    src: templates/{{helper}}.service
+    dest: /etc/systemd/system/{{helper}}.service
+    mode: '0755'
+    owner: root
+  register: helper_unit
+
+- name: reload systemd
+  tags: test-helpers
+  ansible.builtin.systemd_service:
+    daemon_reload: yes
+  when: helper_unit.changed
+
+- name: Start helper
+  tags: test-helpers
+  systemd:
+    name: "{{helper}}.service"
+    state: started
+    enabled: yes
+
+- name: Restart helper if its binary or unit file changed
+  tags: test-helpers
+  ansible.builtin.systemd_service:
+    name: "{{ helper }}.service"
+    state: restarted
+  become: yes
+  when: helper_binary.changed or helper_unit.changed

ansible/roles/test_helpers/templates/ooechohelpd.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Test helper that will start an echo session on request
+After=network.target
+StartLimitIntervalSec=60
+StartLimitBurst=3
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/ooechohelpd --port {{port}}
+Restart=on-failure
+RestartSec=5
+User=testhelpers
+Group=testhelpers
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+PrivateTmp=yes
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/test_helpers/templates/oojsonhelpd.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Test helper that will respond with a json showing the headers it received
+After=network.target
+StartLimitIntervalSec=60
+StartLimitBurst=3
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/oojsonhelpd --port {{port}}
+Restart=on-failure
+RestartSec=5
+User=testhelpers
+Group=testhelpers
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+PrivateTmp=yes
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/test_helpers/vars/main.yml

@@ -0,0 +1,4 @@
+
+# choices: oojsonhelpd, ooechohelpd
+helper: "oojsonhelpd"
+port: "80"

tf/environments/prod/main.tf

@@ -1033,6 +1033,45 @@ module "fastpath_builder" {
   codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket
 }
 
+
+#### Test Helpers Machines
+#
+
+# Registers the same oonidevops keypair used for the EC2 instances (see
+# module.adm_iam_roles) as a DigitalOcean account key, so it can be installed
+# on droplets via their ssh_keys argument too.
+resource "digitalocean_ssh_key" "oonidevops" {
+  name       = "oonidevops"
+  public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"]
+}
+
+module "ooni_test_helpers_json" {
+  source = "../../modules/ooni_th_binary_droplet"
+
+  stage    = local.environment
+  name     = "oonijsonth"
+  hostname = "json.th"
+
+  ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint]
+
+  dns_zone_ooni_io = local.dns_zone_ooni_io
+}
+
+# Echo test helper requires a dedicated machine bc it's a tcp server,
+# not an HTTP server. It's impossible to reroute using nginx
+module "ooni_test_helpers_echo" {
+  source = "../../modules/ooni_th_binary_droplet"
+
+  stage    = local.environment
+  name     = "ooniechoth"
+  hostname = "echo.th"
+
+  ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint]
+
+  dns_zone_ooni_io = local.dns_zone_ooni_io
+}
+
+
 module "reuploader_builder" {
   source      = "../../modules/ooni_docker_build"
   trigger_tag = ""

tf/modules/ooni_th_binary_droplet/main.tf

@@ -0,0 +1,31 @@
+terraform {
+  required_providers {
+    digitalocean = {
+      source  = "digitalocean/digitalocean"
+      version = "~> 2.0"
+    }
+  }
+}
+
+resource "digitalocean_droplet" "ooni_th" {
+  image    = "debian-13-x64"
+  name     = "${var.name}-${var.stage}"
+  region   = var.instance_location
+  size     = var.instance_size
+  ipv6     = true
+  ssh_keys = var.ssh_keys
+
+  lifecycle {
+    create_before_destroy = true
+    ignore_changes        = all
+  }
+}
+
+resource "aws_route53_record" "ooni_th" {
+  zone_id = var.dns_zone_ooni_io
+  name    = "${var.hostname}.${var.stage}.ooni.io"
+  type    = "A"
+  ttl     = 60
+
+  records = [digitalocean_droplet.ooni_th.ipv4_address]
+}

tf/modules/ooni_th_binary_droplet/outputs.tf

@@ -0,0 +1,7 @@
+output "droplet_ipv4_address" {
+  value = digitalocean_droplet.ooni_th.ipv4_address
+}
+
+output "fqdn" {
+  value = aws_route53_record.ooni_th.fqdn
+}

tf/modules/ooni_th_binary_droplet/variables.tf

@@ -0,0 +1,33 @@
+variable "stage" {
+  type = string
+}
+
+variable "name" {
+  description = "Name of the droplet (will be suffixed with -<stage>)"
+  type        = string
+}
+
+variable "hostname" {
+  description = "DNS label for the helper, e.g. \"json.th\" or \"echo.th\". The record is created as <hostname>.<stage>.ooni.io"
+  type        = string
+}
+
+variable "instance_location" {
+  type    = string
+  default = "fra1"
+}
+
+variable "instance_size" {
+  type    = string
+  default = "s-1vcpu-1gb"
+}
+
+variable "ssh_keys" {
+  description = "Fingerprints of the DigitalOcean account SSH keys to grant root access to"
+  type        = list(string)
+}
+
+variable "dns_zone_ooni_io" {
+  description = "id of the DNS zone for ooni_io"
+  type        = string
+}