diff --git a/ansible/deploy-echo-test-helper.yml b/ansible/deploy-echo-test-helper.yml new file mode 100644 index 00000000..2e875c1e --- /dev/null +++ b/ansible/deploy-echo-test-helper.yml @@ -0,0 +1,23 @@ +--- +- name: Deploy test helpers + hosts: + - echo.th.dev.ooni.io + - echo.th.prod.ooni.io + become: true + roles: + - role: bootstrap + - role: prometheus_node_exporter + vars: + node_exporter_port: 9100 + node_exporter_host: "0.0.0.0" + prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:9100/metrics + use_https: false + http_port: 8080 # if we leave port 80, it's taken by nginx + - role: test_helpers + vars: + helper: ooechohelpd + # sha256 of https://github.com/ooni/ooniprobe-rs/releases/download/v0.1.6/ooechohelpd + test_helpers_checksum: 10511a8b918c2244eba9e17c32f3cec9efbc5ffa51d6ad7f59fe648cd0986615 + port: 80 diff --git a/ansible/deploy-json-test-helper.yml b/ansible/deploy-json-test-helper.yml new file mode 100644 index 00000000..e591018d --- /dev/null +++ b/ansible/deploy-json-test-helper.yml @@ -0,0 +1,23 @@ +--- +- name: Deploy test helpers + hosts: + - json.th.dev.ooni.io + - json.th.prod.ooni.io + become: true + roles: + - role: bootstrap + - role: prometheus_node_exporter + vars: + node_exporter_port: 9100 + node_exporter_host: "0.0.0.0" + prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:9100/metrics + use_https: false + http_port: 8080 # if we leave port 80, it's taken by nginx + - role: test_helpers + vars: + helper: oojsonhelpd + # sha256 of https://github.com/ooni/ooniprobe-rs/releases/download/v0.1.6/oojsonhelpd + test_helpers_checksum: 3d977e1418ea398f520dff3de951e36010a294dc3d1b564c961542cc3729a224 + port: 80 diff --git a/ansible/inventory b/ansible/inventory index 8b2a9211..5008d568 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -48,6 +48,10 @@ openvpn2.htz-fsn.prod.ooni.nu [aws-backend] fastpath.dev.ooni.io fastpath.prod.ooni.io +json.th.dev.ooni.io +echo.th.dev.ooni.io +json.th.prod.ooni.io +echo.th.prod.ooni.io fastpath2.prod.ooni.io reuploaderfastpath.prod.ooni.io anonc.dev.ooni.io diff --git a/ansible/requirements/ansible-galaxy.yml b/ansible/requirements/ansible-galaxy.yml index e31076d3..5172cbec 100644 --- a/ansible/requirements/ansible-galaxy.yml +++ b/ansible/requirements/ansible-galaxy.yml @@ -1,6 +1,6 @@ roles: - src: nginxinc.nginx - version: 0.24.3 + version: 0.26.0 - src: geerlingguy.certbot version: 5.2.0 - src: https://github.com/idealista/clickhouse_role diff --git a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 index 6e7de3d5..6fb7c5ca 100644 --- a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 +++ b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 @@ -11,9 +11,9 @@ server { ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; {% else %} - listen 80; + listen {{http_port}}; - server_name {{ inventory_hostname }}; + server_name {{inventory_hostname}}; {% endif %} {% for config in prometheus_nginx_proxy_config %} diff --git a/ansible/roles/ssh_users/tasks/main.yml b/ansible/roles/ssh_users/tasks/main.yml index 3615a9ba..14a5bd6e 100644 --- a/ansible/roles/ssh_users/tasks/main.yml +++ b/ansible/roles/ssh_users/tasks/main.yml @@ -85,8 +85,8 @@ path: /etc/sudoers.d/adm state: absent -- name: reload sshd +- name: reload ssh tags: ssh_users ansible.builtin.systemd_service: - name: sshd + name: ssh state: reloaded diff --git a/ansible/roles/test_helpers/defaults/main.yml b/ansible/roles/test_helpers/defaults/main.yml new file mode 100644 index 00000000..d3d76986 --- /dev/null +++ b/ansible/roles/test_helpers/defaults/main.yml @@ -0,0 +1,5 @@ +test_helpers_version: v0.1.6 +test_helpers_base_url: "https://github.com/ooni/ooniprobe-rs/releases/download/{{ test_helpers_version }}" + +# monitoring server IP +monitoring_server_ip: "{{ lookup('dig', 'monitoring.ooni.org') }}" diff --git a/ansible/roles/test_helpers/handlers/main.yml b/ansible/roles/test_helpers/handlers/main.yml new file mode 100644 index 00000000..1d820eff --- /dev/null +++ b/ansible/roles/test_helpers/handlers/main.yml @@ -0,0 +1,10 @@ +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded + +- name: reload nginx + service: + name: nginx + state: reloaded diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml new file mode 100644 index 00000000..02b57055 --- /dev/null +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -0,0 +1,76 @@ +--- + +# For prometheus scrape requests. +- name: Allow traffic on port 9100 from the monitoring host only + become: true + tags: prometheus-proxy + blockinfile: + path: /etc/ooni/nftables/tcp/9100.nft + create: yes + block: | + add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport 9100 counter accept comment "node exporter" + notify: + - reload nftables + +- name: Allow traffic on the {{ helper }} test helper port + become: true + tags: test-helpers + blockinfile: + path: "/etc/ooni/nftables/tcp/{{ port }}.nft" + create: yes + block: | + add rule inet filter input tcp dport {{ port }} counter accept comment "{{ helper }} test helper" + notify: + - reload nftables + +# Create test helpers user +- name: Create the testhelpers user + ansible.builtin.user: + name: "testhelpers" + shell: "/bin/bash" + create_home: no + system: yes + become: yes + +# Install test helpers +- name: Download the {{ helper }} test helper binary + ansible.builtin.get_url: + url: "{{ test_helpers_base_url }}/{{ helper }}" + dest: "/usr/local/bin/{{ helper }}" + checksum: "sha256:{{ test_helpers_checksum }}" + mode: '0755' + force: yes + become: yes + register: helper_binary + +# Create systemd units + +- name: Create .service file + tags: test-helpers + ansible.builtin.template: + src: templates/{{helper}}.service + dest: /etc/systemd/system/{{helper}}.service + mode: '0755' + owner: root + register: helper_unit + +- name: reload systemd + tags: test-helpers + ansible.builtin.systemd_service: + daemon_reload: yes + when: helper_unit.changed + +- name: Start helper + tags: test-helpers + systemd: + name: "{{helper}}.service" + state: started + enabled: yes + +- name: Restart helper if its binary or unit file changed + tags: test-helpers + ansible.builtin.systemd_service: + name: "{{ helper }}.service" + state: restarted + become: yes + when: helper_binary.changed or helper_unit.changed diff --git a/ansible/roles/test_helpers/templates/ooechohelpd.service b/ansible/roles/test_helpers/templates/ooechohelpd.service new file mode 100644 index 00000000..2ba60f1c --- /dev/null +++ b/ansible/roles/test_helpers/templates/ooechohelpd.service @@ -0,0 +1,22 @@ +[Unit] +Description=Test helper that will start an echo session on request +After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +Type=simple +ExecStart=/usr/local/bin/ooechohelpd --port {{port}} +Restart=on-failure +RestartSec=5 +User=testhelpers +Group=testhelpers +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +PrivateTmp=yes +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/test_helpers/templates/oojsonhelpd.service b/ansible/roles/test_helpers/templates/oojsonhelpd.service new file mode 100644 index 00000000..de3c4367 --- /dev/null +++ b/ansible/roles/test_helpers/templates/oojsonhelpd.service @@ -0,0 +1,22 @@ +[Unit] +Description=Test helper that will respond with a json showing the headers it received +After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +Type=simple +ExecStart=/usr/local/bin/oojsonhelpd --port {{port}} +Restart=on-failure +RestartSec=5 +User=testhelpers +Group=testhelpers +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +PrivateTmp=yes +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/test_helpers/vars/main.yml b/ansible/roles/test_helpers/vars/main.yml new file mode 100644 index 00000000..3d224a6d --- /dev/null +++ b/ansible/roles/test_helpers/vars/main.yml @@ -0,0 +1,4 @@ + +# choices: oojsonhelpd, ooechohelpd +helper: "oojsonhelpd" +port: "80" diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 3aa8c3c5..801ee5f9 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1033,6 +1033,45 @@ module "fastpath_builder" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket } + +#### Test Helpers Machines +# + +# Registers the same oonidevops keypair used for the EC2 instances (see +# module.adm_iam_roles) as a DigitalOcean account key, so it can be installed +# on droplets via their ssh_keys argument too. +resource "digitalocean_ssh_key" "oonidevops" { + name = "oonidevops" + public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] +} + +module "ooni_test_helpers_json" { + source = "../../modules/ooni_th_binary_droplet" + + stage = local.environment + name = "oonijsonth" + hostname = "json.th" + + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] + + dns_zone_ooni_io = local.dns_zone_ooni_io +} + +# Echo test helper requires a dedicated machine bc it's a tcp server, +# not an HTTP server. It's impossible to reroute using nginx +module "ooni_test_helpers_echo" { + source = "../../modules/ooni_th_binary_droplet" + + stage = local.environment + name = "ooniechoth" + hostname = "echo.th" + + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] + + dns_zone_ooni_io = local.dns_zone_ooni_io +} + + module "reuploader_builder" { source = "../../modules/ooni_docker_build" trigger_tag = "" diff --git a/tf/modules/ooni_th_binary_droplet/main.tf b/tf/modules/ooni_th_binary_droplet/main.tf new file mode 100644 index 00000000..f0d3c1f6 --- /dev/null +++ b/tf/modules/ooni_th_binary_droplet/main.tf @@ -0,0 +1,31 @@ +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + } +} + +resource "digitalocean_droplet" "ooni_th" { + image = "debian-13-x64" + name = "${var.name}-${var.stage}" + region = var.instance_location + size = var.instance_size + ipv6 = true + ssh_keys = var.ssh_keys + + lifecycle { + create_before_destroy = true + ignore_changes = all + } +} + +resource "aws_route53_record" "ooni_th" { + zone_id = var.dns_zone_ooni_io + name = "${var.hostname}.${var.stage}.ooni.io" + type = "A" + ttl = 60 + + records = [digitalocean_droplet.ooni_th.ipv4_address] +} diff --git a/tf/modules/ooni_th_binary_droplet/outputs.tf b/tf/modules/ooni_th_binary_droplet/outputs.tf new file mode 100644 index 00000000..e091c016 --- /dev/null +++ b/tf/modules/ooni_th_binary_droplet/outputs.tf @@ -0,0 +1,7 @@ +output "droplet_ipv4_address" { + value = digitalocean_droplet.ooni_th.ipv4_address +} + +output "fqdn" { + value = aws_route53_record.ooni_th.fqdn +} diff --git a/tf/modules/ooni_th_binary_droplet/variables.tf b/tf/modules/ooni_th_binary_droplet/variables.tf new file mode 100644 index 00000000..b257b344 --- /dev/null +++ b/tf/modules/ooni_th_binary_droplet/variables.tf @@ -0,0 +1,33 @@ +variable "stage" { + type = string +} + +variable "name" { + description = "Name of the droplet (will be suffixed with -)" + type = string +} + +variable "hostname" { + description = "DNS label for the helper, e.g. \"json.th\" or \"echo.th\". The record is created as ..ooni.io" + type = string +} + +variable "instance_location" { + type = string + default = "fra1" +} + +variable "instance_size" { + type = string + default = "s-1vcpu-1gb" +} + +variable "ssh_keys" { + description = "Fingerprints of the DigitalOcean account SSH keys to grant root access to" + type = list(string) +} + +variable "dns_zone_ooni_io" { + description = "id of the DNS zone for ooni_io" + type = string +}