From 69334dab64f9099e80023f58ca97dc74e9ef9c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 22 Aug 2025 16:18:15 +0200 Subject: [PATCH 001/201] Add role to deploy test helpers repo --- ansible/deploy-test-helpers.yml | 17 +++ ansible/inventory | 3 +- ansible/roles/test_helpers/defaults/main.yml | 1 + ansible/roles/test_helpers/tasks/main.yml | 108 ++++++++++++++++++ .../test_helpers/templates/jsonth.service | 20 ++++ tf/environments/dev/main.tf | 101 ++++++++++++++++ 6 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 ansible/deploy-test-helpers.yml create mode 100644 ansible/roles/test_helpers/defaults/main.yml create mode 100644 ansible/roles/test_helpers/tasks/main.yml create mode 100644 ansible/roles/test_helpers/templates/jsonth.service diff --git a/ansible/deploy-test-helpers.yml b/ansible/deploy-test-helpers.yml new file mode 100644 index 00000000..bccfddf2 --- /dev/null +++ b/ansible/deploy-test-helpers.yml @@ -0,0 +1,17 @@ +--- +- name: Deploy test helpers + hosts: + - test-helpers.dev.ooni.io + - test-helpers.prod.ooni.io + become: true + roles: + - role: bootstrap + - role: nginx + - role: prometheus_node_exporter + vars: + node_exporter_port: 9100 + node_exporter_host: "0.0.0.0" + prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:9100/metrics + - role: test_helpers \ No newline at end of file diff --git a/ansible/inventory b/ansible/inventory index 3a23e524..eede599b 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -45,4 +45,5 @@ openvpn2.htz-fsn.prod.ooni.nu [aws-backend] fastpath.dev.ooni.io -# fastpath.prod.ooni.io \ No newline at end of file +# fastpath.prod.ooni.io +test-helpers.dev.ooni.io \ No newline at end of file diff --git a/ansible/roles/test_helpers/defaults/main.yml b/ansible/roles/test_helpers/defaults/main.yml new file mode 100644 index 00000000..5b636ac5 --- /dev/null +++ b/ansible/roles/test_helpers/defaults/main.yml @@ -0,0 +1 @@ +test_helpers_url: https://github.com/ooni/ooniprobe-rs/releases/download/0.1.0-dfe5/ooniprobe-helpers@0.1.0-dfe5.tar.gz diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml new file mode 100644 index 00000000..47dc176b --- /dev/null +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -0,0 +1,108 @@ +--- + +# For prometheus scrape requests +- name: Allow traffic on port 9100 + become: true + tags: prometheus-proxy + blockinfile: + path: /etc/ooni/nftables/tcp/9100.nft + create: yes + block: | + add rule inet filter input tcp dport 9100 counter accept comment "node exporter" + notify: + - reload nftables + + +# For incoming test helper traffic +- name: Allow traffic on port 8000 (echo) + become: true + tags: test-helpers + blockinfile: + path: /etc/ooni/nftables/tcp/8000.nft + create: yes + block: | + add rule inet filter input tcp dport 8000 counter accept comment "echo" + notify: + - reload nftables + +- name: Allow traffic on port 8001 (json) + become: true + tags: test-helpers + blockinfile: + path: /etc/ooni/nftables/tcp/8001.nft + create: yes + block: | + add rule inet filter input tcp dport 8001 counter accept comment "json" + notify: + - reload nftables + +# Create test helpers user +- name: Create the testhelpers user + ansible.builtin.user: + name: "testhelpers" + shell: "/bin/bash" + create_home: no + system: yes + become: yes + +# Install test helpers +- name: Donwload binaries for test helpers + ansible.builtin.get_url: + url: "{{test_helpers_url}}" + dest: "/tmp/test-helpers.tar.gz" + mode: '0600' + become: true + +- name: Extract tar content + ansible.builtin.unarchive: + src: "/tmp/test-helpers.tar.gz" + dest: "/tmp/test-helpers" + remote_src: yes + become: yes + +- name: Make jsonth accessible system wide + ansible.builtin.copy: + src: "/tmp/test-helpers/jsonth" + dest: "/usr/local/bin/" + mode: '0755' + become: yes + +- name: Make echo accessible system wide + ansible.builtin.copy: + src: "/tmp/test-helpers/echo" + dest: "/usr/local/bin/" + mode: '0755' + become: yes + +- name: Clean up temporary files + ansible.builtin.file: + path: "/tmp/test-helpers" + state: absent + become: yes + +- name: Remove downloaded tarball + ansible.builtin.file: + path: "/tmp/test-helpers.tar.gz" + state: absent + become: yes + +# Create systemd units +- name: Create jsonth.service file + tags: test-helpers + ansible.builtin.template: + src: templates/jsonth.service + dest: /etc/systemd/system/jsonth.service + mode: '0755' + owner: root + +- name: reload systemd + tags: test-helpers + shell: systemctl daemon-reload + +- name: Start jupyter + tags: test-helpers + systemd: + name: jsonth.service + state: started + enabled: yes + \ No newline at end of file diff --git a/ansible/roles/test_helpers/templates/jsonth.service b/ansible/roles/test_helpers/templates/jsonth.service new file mode 100644 index 00000000..ebd3c660 --- /dev/null +++ b/ansible/roles/test_helpers/templates/jsonth.service @@ -0,0 +1,20 @@ +[Unit] +Description=Test helper that will respond with a json showing the headers it received +After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +Type=simple +ExecStart=/usr/local/bin/jsonth +Restart=on-failure +RestartSec=5 +User=testhelpers +Group=testhelpers +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +PrivateTmp=yes + +[Install] +WantedBy=multi-user.target diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 03fbd04b..c41670c2 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -683,6 +683,107 @@ module "fastpath_builder" { ecs_cluster_name = module.ooniapi_cluster.cluster_name } + +#### Test Helpers Machine + +module "ooni_test_helpers" { + source = "../../modules/ec2" + + stage = local.environment + + vpc_id = module.network.vpc_id + subnet_id = module.network.vpc_subnet_public[0].id + private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + dns_zone_ooni_io = local.dns_zone_ooni_io + + key_name = module.adm_iam_roles.oonidevops_key_name + instance_type = "t3a.small" + + name = "oonitesthelpers" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 80, # Echo test helper + to_port = 80, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 8000, # Echo test helper + to_port = 8000, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 8001, # Json test helper + to_port = 8001, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 9100, # Prometheus monitoring + to_port = 9100, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"], + }] + + sg_prefix = "oonitesthelpers" + tg_prefix = "tshp" + + disk_size = 20 + + tags = merge( + local.tags, + { Name = "ooni-tier0-testhelpers" } + ) +} + +resource "aws_route53_record" "testhelpers_alias" { + zone_id = local.dns_zone_ooni_io + name = "test-helpers.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooni_test_helpers.aws_instance_public_dns + ] +} + +resource "aws_route53_record" "testhelpers_echo_alias" { + zone_id = local.dns_zone_ooni_io + name = "echo-th.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooni_test_helpers.aws_instance_public_dns + ] +} + +resource "aws_route53_record" "testhelpers_json_alias" { + zone_id = local.dns_zone_ooni_io + name = "json-th.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooni_test_helpers.aws_instance_public_dns + ] +} + + #### OONI Run service module "ooniapi_oonirun_deployer" { From cc15c5ecb3219ac0fda00b0f334682cf0b33edd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 25 Aug 2025 11:56:26 +0200 Subject: [PATCH 002/201] Change test helpers url --- ansible/roles/test_helpers/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/test_helpers/defaults/main.yml b/ansible/roles/test_helpers/defaults/main.yml index 5b636ac5..2cef1999 100644 --- a/ansible/roles/test_helpers/defaults/main.yml +++ b/ansible/roles/test_helpers/defaults/main.yml @@ -1 +1 @@ -test_helpers_url: https://github.com/ooni/ooniprobe-rs/releases/download/0.1.0-dfe5/ooniprobe-helpers@0.1.0-dfe5.tar.gz +test_helpers_url: https://github.com/ooni/test-helpers/releases/download/0.1.0-dbca/test-helpers@0.1.0-dbca.tar.gz From 0904feae8d5c6a34cb8eece536befdabf24f5ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 25 Aug 2025 13:41:53 +0200 Subject: [PATCH 003/201] Set up echo and jsonth services --- ansible/roles/test_helpers/handlers/main.yml | 17 ++++++++++ ansible/roles/test_helpers/tasks/main.yml | 31 ++++++++++++++++++- .../roles/test_helpers/templates/echo.service | 20 ++++++++++++ .../test_helpers/templates/jsonth.service | 2 +- 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/test_helpers/handlers/main.yml create mode 100644 ansible/roles/test_helpers/templates/echo.service diff --git a/ansible/roles/test_helpers/handlers/main.yml b/ansible/roles/test_helpers/handlers/main.yml new file mode 100644 index 00000000..c9efe5d9 --- /dev/null +++ b/ansible/roles/test_helpers/handlers/main.yml @@ -0,0 +1,17 @@ +- name: restart echo + tags: test-helpers + ansible.builtin.systemd_service: + name: echo + state: restarted + +- name: restart jsonth + tags: test-helpers + ansible.builtin.systemd_service: + name: jsonth + state: restarted + +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded \ No newline at end of file diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index 47dc176b..e646144e 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -53,6 +53,13 @@ mode: '0600' become: true +- name: Create test helpers temp dir + ansible.builtin.file: + path: "/tmp/test-helpers" + state: directory + mode: "0700" + become: yes + - name: Extract tar content ansible.builtin.unarchive: src: "/tmp/test-helpers.tar.gz" @@ -65,6 +72,7 @@ src: "/tmp/test-helpers/jsonth" dest: "/usr/local/bin/" mode: '0755' + remote_src: yes become: yes - name: Make echo accessible system wide @@ -72,6 +80,7 @@ src: "/tmp/test-helpers/echo" dest: "/usr/local/bin/" mode: '0755' + remote_src: yes become: yes - name: Clean up temporary files @@ -87,6 +96,7 @@ become: yes # Create systemd units + - name: Create jsonth.service file tags: test-helpers ansible.builtin.template: @@ -94,15 +104,34 @@ dest: /etc/systemd/system/jsonth.service mode: '0755' owner: root + notify: + - restart jsonth + +- name: Create echo.service file + tags: test-helpers + ansible.builtin.template: + src: templates/echo.service + dest: /etc/systemd/system/echo.service + mode: '0755' + owner: root + notify: + - restart echo - name: reload systemd tags: test-helpers shell: systemctl daemon-reload -- name: Start jupyter +- name: Start json tags: test-helpers systemd: name: jsonth.service state: started enabled: yes + +- name: Start echo + tags: test-helpers + systemd: + name: echo.service + state: started + enabled: yes \ No newline at end of file diff --git a/ansible/roles/test_helpers/templates/echo.service b/ansible/roles/test_helpers/templates/echo.service new file mode 100644 index 00000000..13388f7b --- /dev/null +++ b/ansible/roles/test_helpers/templates/echo.service @@ -0,0 +1,20 @@ +[Unit] +Description=Test helper that will start an echo session on request +After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +Type=simple +ExecStart=/usr/local/bin/echo --port 8000 +Restart=on-failure +RestartSec=5 +User=testhelpers +Group=testhelpers +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +PrivateTmp=yes + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/test_helpers/templates/jsonth.service b/ansible/roles/test_helpers/templates/jsonth.service index ebd3c660..fb908501 100644 --- a/ansible/roles/test_helpers/templates/jsonth.service +++ b/ansible/roles/test_helpers/templates/jsonth.service @@ -6,7 +6,7 @@ StartLimitBurst=3 [Service] Type=simple -ExecStart=/usr/local/bin/jsonth +ExecStart=/usr/local/bin/jsonth --port 8001 Restart=on-failure RestartSec=5 User=testhelpers From 620d5a1c94584ede7334140965ff3c9f7642fff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 25 Aug 2025 13:42:35 +0200 Subject: [PATCH 004/201] Fix bad comment --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index c41670c2..239ec335 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -706,7 +706,7 @@ module "ooni_test_helpers" { protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { - from_port = 80, # Echo test helper + from_port = 80, # dehydrated to_port = 80, protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], From b1ee5cc9eb5be1657571baa2c85426554708816a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 28 Aug 2025 12:19:56 +0200 Subject: [PATCH 005/201] fix certificate deadlock issue --- tf/environments/dev/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 239ec335..189fa7f6 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1084,6 +1084,10 @@ resource "aws_acm_certificate" "ooniapi_frontend" { tags = local.tags subject_alternative_names = keys(local.ooniapi_frontend_alternative_domains) + + lifecycle { + create_before_destroy = true + } } resource "aws_route53_record" "ooniapi_frontend_cert_validation" { From 3beab5dfc3127384352cf364231d32777e4c2f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 28 Aug 2025 13:01:52 +0200 Subject: [PATCH 006/201] Routing test helper traffic with nginx --- ansible/roles/test_helpers/handlers/main.yml | 7 ++++- ansible/roles/test_helpers/tasks/main.yml | 28 +++++++++++++------- tf/environments/dev/main.tf | 4 +-- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/ansible/roles/test_helpers/handlers/main.yml b/ansible/roles/test_helpers/handlers/main.yml index c9efe5d9..30593401 100644 --- a/ansible/roles/test_helpers/handlers/main.yml +++ b/ansible/roles/test_helpers/handlers/main.yml @@ -14,4 +14,9 @@ tags: nftables ansible.builtin.systemd_service: name: nftables - state: reloaded \ No newline at end of file + state: reloaded + +- name: reload nginx + service: + name: nginx + state: reloaded diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index e646144e..636bd65a 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -10,7 +10,7 @@ block: | add rule inet filter input tcp dport 9100 counter accept comment "node exporter" notify: - - reload nftables + - reload nftables # For incoming test helper traffic @@ -23,7 +23,7 @@ block: | add rule inet filter input tcp dport 8000 counter accept comment "echo" notify: - - reload nftables + - reload nftables - name: Allow traffic on port 8001 (json) become: true @@ -34,7 +34,7 @@ block: | add rule inet filter input tcp dport 8001 counter accept comment "json" notify: - - reload nftables + - reload nftables # Create test helpers user - name: Create the testhelpers user @@ -99,22 +99,22 @@ - name: Create jsonth.service file tags: test-helpers - ansible.builtin.template: + ansible.builtin.template: src: templates/jsonth.service dest: /etc/systemd/system/jsonth.service mode: '0755' owner: root - notify: + notify: - restart jsonth - + - name: Create echo.service file tags: test-helpers - ansible.builtin.template: + ansible.builtin.template: src: templates/echo.service dest: /etc/systemd/system/echo.service mode: '0755' owner: root - notify: + notify: - restart echo - name: reload systemd @@ -134,4 +134,14 @@ name: echo.service state: started enabled: yes - \ No newline at end of file + +# Nginx routing +- name: Copy nginx config + tags: test-helpers + ansible.builtin.template: + src: templates/test-helpers.conf + dest: /etc/nginx/sites-enabled/02-test-helpers.conf + mode: '744' + owner: nginx + notify: + - reload nginx \ No newline at end of file diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 189fa7f6..d4584f89 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -763,7 +763,7 @@ resource "aws_route53_record" "testhelpers_alias" { resource "aws_route53_record" "testhelpers_echo_alias" { zone_id = local.dns_zone_ooni_io - name = "echo-th.${local.environment}.ooni.io" + name = "42.th.${local.environment}.ooni.io" type = "CNAME" ttl = 300 @@ -774,7 +774,7 @@ resource "aws_route53_record" "testhelpers_echo_alias" { resource "aws_route53_record" "testhelpers_json_alias" { zone_id = local.dns_zone_ooni_io - name = "json-th.${local.environment}.ooni.io" + name = "43.th.${local.environment}.ooni.io" type = "CNAME" ttl = 300 From d0a9298a9ebbf2592c5be4b212a67323176547b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 28 Aug 2025 13:08:55 +0200 Subject: [PATCH 007/201] nginx config for test helpers --- .../test_helpers/templates/test-helpers.conf | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 ansible/roles/test_helpers/templates/test-helpers.conf diff --git a/ansible/roles/test_helpers/templates/test-helpers.conf b/ansible/roles/test_helpers/templates/test-helpers.conf new file mode 100644 index 00000000..a8199377 --- /dev/null +++ b/ansible/roles/test_helpers/templates/test-helpers.conf @@ -0,0 +1,17 @@ +# nginx configuration for routing test helpers depending on their host name + +server { + listen 80; + server_name 42.th.dev.ooni.io; # echo + location / { + proxy_pass http://127.0.0.1:8001; + } +} + +server { + listen 80; + server_name 43.th.dev.ooni.io; # jsonth + location / { + proxy_pass http://127.0.0.1:8000; + } +} From 4c5fcc6ad7b3a9b3e97e223d94aeaca43edd00af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 28 Aug 2025 14:11:38 +0200 Subject: [PATCH 008/201] Creating echo machine --- ...elpers.yml => deploy-echo-test-helper.yml} | 7 +- ansible/deploy-json-test-helper.yml | 20 +++++ ansible/roles/test_helpers/tasks/main.yml | 23 +---- ansible/roles/test_helpers/vars/main.yml | 3 + tf/environments/dev/main.tf | 87 +++++++++++++++---- 5 files changed, 103 insertions(+), 37 deletions(-) rename ansible/{deploy-test-helpers.yml => deploy-echo-test-helper.yml} (77%) create mode 100644 ansible/deploy-json-test-helper.yml create mode 100644 ansible/roles/test_helpers/vars/main.yml diff --git a/ansible/deploy-test-helpers.yml b/ansible/deploy-echo-test-helper.yml similarity index 77% rename from ansible/deploy-test-helpers.yml rename to ansible/deploy-echo-test-helper.yml index bccfddf2..d325b3dc 100644 --- a/ansible/deploy-test-helpers.yml +++ b/ansible/deploy-echo-test-helper.yml @@ -11,7 +11,10 @@ vars: node_exporter_port: 9100 node_exporter_host: "0.0.0.0" - prometheus_nginx_proxy_config: + prometheus_nginx_proxy_config: - location: /metrics/node_exporter proxy_pass: http://127.0.0.1:9100/metrics - - role: test_helpers \ No newline at end of file + - role: test_helpers + vars: + services: + - jsonth diff --git a/ansible/deploy-json-test-helper.yml b/ansible/deploy-json-test-helper.yml new file mode 100644 index 00000000..d325b3dc --- /dev/null +++ b/ansible/deploy-json-test-helper.yml @@ -0,0 +1,20 @@ +--- +- name: Deploy test helpers + hosts: + - test-helpers.dev.ooni.io + - test-helpers.prod.ooni.io + become: true + roles: + - role: bootstrap + - role: nginx + - role: prometheus_node_exporter + vars: + node_exporter_port: 9100 + node_exporter_host: "0.0.0.0" + prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:9100/metrics + - role: test_helpers + vars: + services: + - jsonth diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index 636bd65a..1b9be3fa 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -121,27 +121,10 @@ tags: test-helpers shell: systemctl daemon-reload -- name: Start json +- name: Start service tags: test-helpers systemd: - name: jsonth.service + name: "{{item}}.service" state: started enabled: yes - -- name: Start echo - tags: test-helpers - systemd: - name: echo.service - state: started - enabled: yes - -# Nginx routing -- name: Copy nginx config - tags: test-helpers - ansible.builtin.template: - src: templates/test-helpers.conf - dest: /etc/nginx/sites-enabled/02-test-helpers.conf - mode: '744' - owner: nginx - notify: - - reload nginx \ No newline at end of file + loop: "{{helpers}}" diff --git a/ansible/roles/test_helpers/vars/main.yml b/ansible/roles/test_helpers/vars/main.yml new file mode 100644 index 00000000..bb106ad6 --- /dev/null +++ b/ansible/roles/test_helpers/vars/main.yml @@ -0,0 +1,3 @@ + +# choices: jsonth, echo +helpers: "jsonth" \ No newline at end of file diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index d4584f89..b1d3e3a8 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -591,7 +591,7 @@ resource "aws_route53_record" "monitoring_proxy_alias" { } -### Fastpath +### Fastpath module "ooni_fastpath" { source = "../../modules/ec2" @@ -684,8 +684,9 @@ module "fastpath_builder" { } -#### Test Helpers Machine +#### Test Helpers Machines +# jsonth and other http helpers module "ooni_test_helpers" { source = "../../modules/ec2" @@ -697,7 +698,7 @@ module "ooni_test_helpers" { dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name - instance_type = "t3a.small" + instance_type = "t3.micro" name = "oonitesthelpers" ingress_rules = [{ @@ -707,17 +708,12 @@ module "ooni_test_helpers" { cidr_blocks = ["0.0.0.0/0"], }, { from_port = 80, # dehydrated - to_port = 80, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 8000, # Echo test helper - to_port = 8000, + to_port = 80, protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { - from_port = 8001, # Json test helper - to_port = 8001, + from_port = 8000, # Json test helper + to_port = 8000, protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { @@ -750,6 +746,67 @@ module "ooni_test_helpers" { ) } +# Echo test helper, requires a dedicated machine bc it's a tcp server, +# not an HTTP server, so it's harder to reroute using nginx +module "ooni_test_helpers_echo" { + source = "../../modules/ec2" + + stage = local.environment + + vpc_id = module.network.vpc_id + subnet_id = module.network.vpc_subnet_public[0].id + private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + dns_zone_ooni_io = local.dns_zone_ooni_io + + key_name = module.adm_iam_roles.oonidevops_key_name + instance_type = "t3.micro" + + name = "ooniechoth" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 80, # dehydrated + to_port = 80, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 8000, # Echo test helper + to_port = 8000, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 9100, # Prometheus monitoring + to_port = 9100, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"], + }] + + sg_prefix = "ooniechoth" + tg_prefix = "echo" + + disk_size = 20 + + tags = merge( + local.tags, + { Name = "ooni-tier0-echoth" } + ) +} + resource "aws_route53_record" "testhelpers_alias" { zone_id = local.dns_zone_ooni_io name = "test-helpers.${local.environment}.ooni.io" @@ -763,7 +820,7 @@ resource "aws_route53_record" "testhelpers_alias" { resource "aws_route53_record" "testhelpers_echo_alias" { zone_id = local.dns_zone_ooni_io - name = "42.th.${local.environment}.ooni.io" + name = "42.th.${local.environment}.ooni.io" # json and others type = "CNAME" ttl = 300 @@ -774,12 +831,12 @@ resource "aws_route53_record" "testhelpers_echo_alias" { resource "aws_route53_record" "testhelpers_json_alias" { zone_id = local.dns_zone_ooni_io - name = "43.th.${local.environment}.ooni.io" + name = "43.th.${local.environment}.ooni.io" # echo type = "CNAME" ttl = 300 records = [ - module.ooni_test_helpers.aws_instance_public_dns + module.ooni_test_helpers_echo.aws_instance_public_dns ] } @@ -992,7 +1049,7 @@ module "ooniapi_oonimeasurements" { task_environment = { # it has to be a json-compliant array - OTHER_COLLECTORS = jsonencode(["http://fastpath.${local.environment}.ooni.io:8475"]) + OTHER_COLLECTORS = jsonencode(["http://fastpath.${local.environment}.ooni.io:8475"]) BASE_URL = "https://api.${local.environment}.ooni.io" S3_BUCKET_NAME = "ooni-data-eu-fra-test" } From 61d705eb3b54e48d3c1d7b270491a0fb840ea47e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 28 Aug 2025 16:10:33 +0200 Subject: [PATCH 009/201] Creating host for each th --- ansible/roles/test_helpers/tasks/main.yml | 23 ++++-------- .../roles/test_helpers/templates/echo.service | 2 +- .../test_helpers/templates/jsonth.service | 2 +- .../test_helpers/templates/test-helpers.conf | 17 --------- ansible/roles/test_helpers/vars/main.yml | 3 +- tf/environments/dev/main.tf | 35 ++++--------------- 6 files changed, 17 insertions(+), 65 deletions(-) delete mode 100644 ansible/roles/test_helpers/templates/test-helpers.conf diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index 1b9be3fa..252a51f6 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -97,34 +97,23 @@ # Create systemd units -- name: Create jsonth.service file +- name: Create .service file tags: test-helpers ansible.builtin.template: - src: templates/jsonth.service - dest: /etc/systemd/system/jsonth.service + src: templates/{{helper}}.service + dest: /etc/systemd/system/{{helper}}.service mode: '0755' owner: root notify: - - restart jsonth - -- name: Create echo.service file - tags: test-helpers - ansible.builtin.template: - src: templates/echo.service - dest: /etc/systemd/system/echo.service - mode: '0755' - owner: root - notify: - - restart echo + - "restart {{helper}}" - name: reload systemd tags: test-helpers shell: systemctl daemon-reload -- name: Start service +- name: Start helper tags: test-helpers systemd: - name: "{{item}}.service" + name: "{{helper}}.service" state: started enabled: yes - loop: "{{helpers}}" diff --git a/ansible/roles/test_helpers/templates/echo.service b/ansible/roles/test_helpers/templates/echo.service index 13388f7b..cffdf252 100644 --- a/ansible/roles/test_helpers/templates/echo.service +++ b/ansible/roles/test_helpers/templates/echo.service @@ -6,7 +6,7 @@ StartLimitBurst=3 [Service] Type=simple -ExecStart=/usr/local/bin/echo --port 8000 +ExecStart=/usr/local/bin/echo --port {{port}} Restart=on-failure RestartSec=5 User=testhelpers diff --git a/ansible/roles/test_helpers/templates/jsonth.service b/ansible/roles/test_helpers/templates/jsonth.service index fb908501..eb4b4bf7 100644 --- a/ansible/roles/test_helpers/templates/jsonth.service +++ b/ansible/roles/test_helpers/templates/jsonth.service @@ -6,7 +6,7 @@ StartLimitBurst=3 [Service] Type=simple -ExecStart=/usr/local/bin/jsonth --port 8001 +ExecStart=/usr/local/bin/jsonth --port {{port}} Restart=on-failure RestartSec=5 User=testhelpers diff --git a/ansible/roles/test_helpers/templates/test-helpers.conf b/ansible/roles/test_helpers/templates/test-helpers.conf deleted file mode 100644 index a8199377..00000000 --- a/ansible/roles/test_helpers/templates/test-helpers.conf +++ /dev/null @@ -1,17 +0,0 @@ -# nginx configuration for routing test helpers depending on their host name - -server { - listen 80; - server_name 42.th.dev.ooni.io; # echo - location / { - proxy_pass http://127.0.0.1:8001; - } -} - -server { - listen 80; - server_name 43.th.dev.ooni.io; # jsonth - location / { - proxy_pass http://127.0.0.1:8000; - } -} diff --git a/ansible/roles/test_helpers/vars/main.yml b/ansible/roles/test_helpers/vars/main.yml index bb106ad6..671bb91e 100644 --- a/ansible/roles/test_helpers/vars/main.yml +++ b/ansible/roles/test_helpers/vars/main.yml @@ -1,3 +1,4 @@ # choices: jsonth, echo -helpers: "jsonth" \ No newline at end of file +helper: "jsonth" +port: "80" \ No newline at end of file diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index b1d3e3a8..623b813c 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -687,7 +687,7 @@ module "fastpath_builder" { #### Test Helpers Machines # jsonth and other http helpers -module "ooni_test_helpers" { +module "ooni_test_helpers_json" { source = "../../modules/ec2" stage = local.environment @@ -700,23 +700,18 @@ module "ooni_test_helpers" { key_name = module.adm_iam_roles.oonidevops_key_name instance_type = "t3.micro" - name = "oonitesthelpers" + name = "oonijsonth" ingress_rules = [{ from_port = 22, to_port = 22, protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { - from_port = 80, # dehydrated + from_port = 80, # jsonth to_port = 80, protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { - from_port = 8000, # Json test helper - to_port = 8000, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { from_port = 9100, # Prometheus monitoring to_port = 9100, protocol = "tcp" @@ -768,16 +763,11 @@ module "ooni_test_helpers_echo" { protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { - from_port = 80, # dehydrated + from_port = 80, # echo to_port = 80, protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { - from_port = 8000, # Echo test helper - to_port = 8000, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { from_port = 9100, # Prometheus monitoring to_port = 9100, protocol = "tcp" @@ -807,9 +797,9 @@ module "ooni_test_helpers_echo" { ) } -resource "aws_route53_record" "testhelpers_alias" { +resource "aws_route53_record" "testhelpers_json_alias" { zone_id = local.dns_zone_ooni_io - name = "test-helpers.${local.environment}.ooni.io" + name = "json.th.${local.environment}.ooni.io" # json type = "CNAME" ttl = 300 @@ -820,18 +810,7 @@ resource "aws_route53_record" "testhelpers_alias" { resource "aws_route53_record" "testhelpers_echo_alias" { zone_id = local.dns_zone_ooni_io - name = "42.th.${local.environment}.ooni.io" # json and others - type = "CNAME" - ttl = 300 - - records = [ - module.ooni_test_helpers.aws_instance_public_dns - ] -} - -resource "aws_route53_record" "testhelpers_json_alias" { - zone_id = local.dns_zone_ooni_io - name = "43.th.${local.environment}.ooni.io" # echo + name = "echo.th.${local.environment}.ooni.io" # echo type = "CNAME" ttl = 300 From 91bfe43d409be839ff52ac4c75d5c2c576f07739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 28 Aug 2025 16:12:43 +0200 Subject: [PATCH 010/201] Fix bad module name --- tf/environments/dev/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 623b813c..0ee6ff45 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -799,18 +799,18 @@ module "ooni_test_helpers_echo" { resource "aws_route53_record" "testhelpers_json_alias" { zone_id = local.dns_zone_ooni_io - name = "json.th.${local.environment}.ooni.io" # json + name = "json.th.${local.environment}.ooni.io" type = "CNAME" ttl = 300 records = [ - module.ooni_test_helpers.aws_instance_public_dns + module.ooni_test_helpers_json.aws_instance_public_dns ] } resource "aws_route53_record" "testhelpers_echo_alias" { zone_id = local.dns_zone_ooni_io - name = "echo.th.${local.environment}.ooni.io" # echo + name = "echo.th.${local.environment}.ooni.io" type = "CNAME" ttl = 300 From 1f08b62062faeb51e556838d9e7909d1482ef226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 28 Aug 2025 16:46:38 +0200 Subject: [PATCH 011/201] Set up ansible for each test helper --- ansible/deploy-echo-test-helper.yml | 10 +++++----- ansible/deploy-json-test-helper.yml | 9 ++++----- ansible/inventory | 3 ++- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ansible/deploy-echo-test-helper.yml b/ansible/deploy-echo-test-helper.yml index d325b3dc..101c2980 100644 --- a/ansible/deploy-echo-test-helper.yml +++ b/ansible/deploy-echo-test-helper.yml @@ -1,12 +1,11 @@ --- - name: Deploy test helpers hosts: - - test-helpers.dev.ooni.io - - test-helpers.prod.ooni.io + - echo.th.dev.ooni.io + - echo.th.prod.ooni.io become: true roles: - role: bootstrap - - role: nginx - role: prometheus_node_exporter vars: node_exporter_port: 9100 @@ -16,5 +15,6 @@ proxy_pass: http://127.0.0.1:9100/metrics - role: test_helpers vars: - services: - - jsonth + helper: echo + port: 80 + diff --git a/ansible/deploy-json-test-helper.yml b/ansible/deploy-json-test-helper.yml index d325b3dc..48dfcefa 100644 --- a/ansible/deploy-json-test-helper.yml +++ b/ansible/deploy-json-test-helper.yml @@ -1,12 +1,11 @@ --- - name: Deploy test helpers hosts: - - test-helpers.dev.ooni.io - - test-helpers.prod.ooni.io + - json.th.dev.ooni.io + - json.th.prod.ooni.io become: true roles: - role: bootstrap - - role: nginx - role: prometheus_node_exporter vars: node_exporter_port: 9100 @@ -16,5 +15,5 @@ proxy_pass: http://127.0.0.1:9100/metrics - role: test_helpers vars: - services: - - jsonth + helper: jsonth + port: 80 diff --git a/ansible/inventory b/ansible/inventory index eede599b..5b39bd18 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -46,4 +46,5 @@ openvpn2.htz-fsn.prod.ooni.nu [aws-backend] fastpath.dev.ooni.io # fastpath.prod.ooni.io -test-helpers.dev.ooni.io \ No newline at end of file +json.th.dev.ooni.io +echo.th.dev.ooni.io \ No newline at end of file From b77339300bed4d4aa1ca40129d912d9ee2095d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 29 Aug 2025 10:19:37 +0200 Subject: [PATCH 012/201] Run prometheus without dehydrated when no https is required --- ansible/deploy-json-test-helper.yml | 1 + ansible/roles/prometheus_node_exporter/tasks/main.yml | 1 + tf/environments/dev/main.tf | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ansible/deploy-json-test-helper.yml b/ansible/deploy-json-test-helper.yml index 48dfcefa..ea35bd98 100644 --- a/ansible/deploy-json-test-helper.yml +++ b/ansible/deploy-json-test-helper.yml @@ -13,6 +13,7 @@ prometheus_nginx_proxy_config: - location: /metrics/node_exporter proxy_pass: http://127.0.0.1:9100/metrics + use_https: false - role: test_helpers vars: helper: jsonth diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index c79a618e..9a4510b7 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -12,6 +12,7 @@ vars: ssl_domains: - "{{ inventory_hostname }}" + when: use_https - include_tasks: install.yml diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 0ee6ff45..9cd05cc7 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -730,14 +730,14 @@ module "ooni_test_helpers_json" { ipv6_cidr_blocks = ["::/0"], }] - sg_prefix = "oonitesthelpers" + sg_prefix = "oonijsonth" tg_prefix = "tshp" disk_size = 20 tags = merge( local.tags, - { Name = "ooni-tier0-testhelpers" } + { Name = "ooni-tier0-jsonth" } ) } From ed6f3a204a72f8504bf6c9301efd98b63fcac6f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 29 Aug 2025 10:57:31 +0200 Subject: [PATCH 013/201] removing unused listen 80 entry in prometheus config --- .../prometheus_node_exporter/templates/nginx-prometheus.j2 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 index 6e7de3d5..5d9c6471 100644 --- a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 +++ b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 @@ -10,10 +10,6 @@ server { ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; - {% else %} - listen 80; - - server_name {{ inventory_hostname }}; {% endif %} {% for config in prometheus_nginx_proxy_config %} From 94b24cd12b1b87e036bc0930689e0f79566f9225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 29 Aug 2025 11:22:49 +0200 Subject: [PATCH 014/201] Allow services to run on port 80 --- ansible/roles/test_helpers/templates/echo.service | 2 ++ ansible/roles/test_helpers/templates/jsonth.service | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ansible/roles/test_helpers/templates/echo.service b/ansible/roles/test_helpers/templates/echo.service index cffdf252..996e6aa1 100644 --- a/ansible/roles/test_helpers/templates/echo.service +++ b/ansible/roles/test_helpers/templates/echo.service @@ -15,6 +15,8 @@ ProtectSystem=full ProtectHome=yes NoNewPrivileges=yes PrivateTmp=yes +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE [Install] WantedBy=multi-user.target diff --git a/ansible/roles/test_helpers/templates/jsonth.service b/ansible/roles/test_helpers/templates/jsonth.service index eb4b4bf7..54392c17 100644 --- a/ansible/roles/test_helpers/templates/jsonth.service +++ b/ansible/roles/test_helpers/templates/jsonth.service @@ -15,6 +15,8 @@ ProtectSystem=full ProtectHome=yes NoNewPrivileges=yes PrivateTmp=yes +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE [Install] WantedBy=multi-user.target From 393803e2ee6cb27ccebcf6cd0248371c36b6e692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 29 Aug 2025 12:10:45 +0200 Subject: [PATCH 015/201] Add port parameters to use a different port for test helpers metrics --- ansible/deploy-echo-test-helper.yml | 3 ++- ansible/deploy-json-test-helper.yml | 1 + .../prometheus_node_exporter/templates/nginx-prometheus.j2 | 6 +++++- ansible/roles/prometheus_node_exporter/vars/main.yml | 4 +++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ansible/deploy-echo-test-helper.yml b/ansible/deploy-echo-test-helper.yml index 101c2980..fe6434a1 100644 --- a/ansible/deploy-echo-test-helper.yml +++ b/ansible/deploy-echo-test-helper.yml @@ -13,8 +13,9 @@ prometheus_nginx_proxy_config: - location: /metrics/node_exporter proxy_pass: http://127.0.0.1:9100/metrics + use_https: false + http_port: 8080 # if we leave port 80, it's taken by nginx - role: test_helpers vars: helper: echo port: 80 - diff --git a/ansible/deploy-json-test-helper.yml b/ansible/deploy-json-test-helper.yml index ea35bd98..ba08b1fc 100644 --- a/ansible/deploy-json-test-helper.yml +++ b/ansible/deploy-json-test-helper.yml @@ -14,6 +14,7 @@ - location: /metrics/node_exporter proxy_pass: http://127.0.0.1:9100/metrics use_https: false + http_port: 8080 # if we leave port 80, it's taken by nginx - role: test_helpers vars: helper: jsonth diff --git a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 index 5d9c6471..a0019f86 100644 --- a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 +++ b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 @@ -2,7 +2,7 @@ server { {% if use_https %} - listen 443 ssl http2; + listen {{https_port}} ssl http2; server_name {{ inventory_hostname }}; include /etc/nginx/ssl_intermediate.conf; @@ -10,6 +10,10 @@ server { ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; + {% else %} + listen {{http_port}}; + + server_name {{inventory_hostname}}; {% endif %} {% for config in prometheus_nginx_proxy_config %} diff --git a/ansible/roles/prometheus_node_exporter/vars/main.yml b/ansible/roles/prometheus_node_exporter/vars/main.yml index 1cf0521e..567f660b 100644 --- a/ansible/roles/prometheus_node_exporter/vars/main.yml +++ b/ansible/roles/prometheus_node_exporter/vars/main.yml @@ -1 +1,3 @@ -use_https: true \ No newline at end of file +use_https: true +http_port: 80 +https_port: 443 \ No newline at end of file From f5f5819bd9e14a1f55673facd2ca1b3c764855d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 1 Sep 2025 11:01:14 +0200 Subject: [PATCH 016/201] Remove unused firewall rules --- ansible/roles/test_helpers/tasks/main.yml | 24 ----------------------- 1 file changed, 24 deletions(-) diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index 252a51f6..bd728c14 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -12,30 +12,6 @@ notify: - reload nftables - -# For incoming test helper traffic -- name: Allow traffic on port 8000 (echo) - become: true - tags: test-helpers - blockinfile: - path: /etc/ooni/nftables/tcp/8000.nft - create: yes - block: | - add rule inet filter input tcp dport 8000 counter accept comment "echo" - notify: - - reload nftables - -- name: Allow traffic on port 8001 (json) - become: true - tags: test-helpers - blockinfile: - path: /etc/ooni/nftables/tcp/8001.nft - create: yes - block: | - add rule inet filter input tcp dport 8001 counter accept comment "json" - notify: - - reload nftables - # Create test helpers user - name: Create the testhelpers user ansible.builtin.user: From 901639d95700528b6aaa2c852dc4dee0921bd2fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 1 Sep 2025 11:07:20 +0200 Subject: [PATCH 017/201] Updated comment --- tf/environments/dev/main.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 9cd05cc7..9815be3c 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -686,7 +686,6 @@ module "fastpath_builder" { #### Test Helpers Machines -# jsonth and other http helpers module "ooni_test_helpers_json" { source = "../../modules/ec2" @@ -742,7 +741,7 @@ module "ooni_test_helpers_json" { } # Echo test helper, requires a dedicated machine bc it's a tcp server, -# not an HTTP server, so it's harder to reroute using nginx +# not an HTTP server. It's impossible to reroute using nginx module "ooni_test_helpers_echo" { source = "../../modules/ec2" From e45f17300c35ba72c0eb044a60647ba5d4806602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 4 Sep 2025 11:12:59 +0200 Subject: [PATCH 018/201] Add checksum for test helpers tar download --- ansible/roles/test_helpers/defaults/main.yml | 4 +++- ansible/roles/test_helpers/tasks/main.yml | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ansible/roles/test_helpers/defaults/main.yml b/ansible/roles/test_helpers/defaults/main.yml index 2cef1999..a431bc19 100644 --- a/ansible/roles/test_helpers/defaults/main.yml +++ b/ansible/roles/test_helpers/defaults/main.yml @@ -1 +1,3 @@ -test_helpers_url: https://github.com/ooni/test-helpers/releases/download/0.1.0-dbca/test-helpers@0.1.0-dbca.tar.gz +test_helpers_url: https://github.com/ooni/test-helpers/releases/download/0.1.0-1ac1/test-helpers@0.1.0-1ac1.tar.gz +# remember to remove the "sha256:" prefix from github +checksum: 9a7387050412d747df8d0479c004357edfc4cd7825ce7e1c83141e1e0838715c diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index bd728c14..f35613cc 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -29,6 +29,17 @@ mode: '0600' become: true +- name: Get checksum of downloaded file + ansible.builtin.stat: + path: "/tmp/test-helpers.tar.gz" + checksum_algorithm: sha256 + register: file_stat + +- name: Verify checksum + ansible.builtin.fail: + msg: "Checksum failed! Expected: {{checksum}} but got: {{file_stat.stat.checksum}}" + when: file_stat.stat.checksum != checksum + - name: Create test helpers temp dir ansible.builtin.file: path: "/tmp/test-helpers" From 9f86d7f3a8a3af83cb8416930af39bc56d20b73a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 17 Apr 2026 15:10:07 +0200 Subject: [PATCH 019/201] Apply stricter constraints on oonimeasurements user queries --- ansible/group_vars/clickhouse/vars.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 7b48ebe1..efcf2bc6 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -239,6 +239,14 @@ clickhouse_custom_users: - "max_memory_usage = 501001000" # 60 seconds - "max_execution_time = 30" + # 500 GB + - "max_bytes_to_read = 501001001000" + # 5 B + - "max_rows_to_read = 5001001000" + # 5s + - "timeout_before_checking_execution_speed = 5" + # 10 M + - "max_result_rows = 11001000" profile: - readonly quota: "oonimeasurements" From d9addcad900f21731fff1f801c9970fc74b75186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 17 Apr 2026 17:16:20 +0200 Subject: [PATCH 020/201] Reduce max rows to 50k --- ansible/group_vars/clickhouse/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index efcf2bc6..96c1ce2f 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -245,8 +245,8 @@ clickhouse_custom_users: - "max_rows_to_read = 5001001000" # 5s - "timeout_before_checking_execution_speed = 5" - # 10 M - - "max_result_rows = 11001000" + # 50k + - "max_result_rows = 51000" profile: - readonly quota: "oonimeasurements" From d442c904625067734d572004a5721abe6d9e668b Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 27 Apr 2026 16:49:59 +0200 Subject: [PATCH 021/201] raise oonimeasuremenets max memory --- ansible/group_vars/clickhouse/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 7b48ebe1..30f3ca0c 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -235,8 +235,8 @@ clickhouse_custom_users: networks: - "IP '0.0.0.0/0'" settings: - # 500 MB - - "max_memory_usage = 501001000" + # 1 GB + - "max_memory_usage = 1001001000" # 60 seconds - "max_execution_time = 30" profile: From b7a42466a6c03fbb401286ff24babdf1d1fdbf58 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 18:12:44 +0200 Subject: [PATCH 022/201] enable clickhouse_role_manage_settings_profiles and clickhouse_role_manage_quotas https://github.com/idealista/clickhouse_role/blob/main/molecule/default/group_vars/clickhouse_group.yml actually use sha256 password type clickhouse role disregards password_type and only looks at key password_sha256_hex ... fix quotas keys --- ansible/group_vars/clickhouse/vars.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 63738884..efeb752d 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -165,6 +165,7 @@ clickhouse_distributed_ddl: cleanup_delay_period: 60 max_tasks_in_queue: 1000 +clickhouse_role_manage_settings_profiles: True clickhouse_default_profiles: default: readonly: 2 @@ -226,12 +227,12 @@ clickhouse_default_users: profile: write quota: default -clickhouse_role_manage_users: true +clickhouse_role_manage_users: True clickhouse_custom_users: - user: name: oonimeasurements password_type: sha256_password - password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') }}" + password_sha256_hex: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') | hash('sha256') }}" networks: - "IP '0.0.0.0/0'" settings: @@ -252,19 +253,19 @@ clickhouse_custom_users: quota: "oonimeasurements" databases: [ooni] -# TODO: this quota was created by hand since it wasn't working in the idealista playbook -clickhouse_role_manage_quotas: false +clickhouse_role_manage_quotas: True clickhouse_custom_quotas: # quota over a 10 minute window - quota: name: oonimeasurements - settings: - - "INTERVAL 10 minute MAX queries = 12000, MAX errors = 1000, MAX execution_time = 1000" - to: - - oonimeasurements + duration: 600 + queries: 12000 + errors: 1000 + result_rows: 0 + read_rows: 0 + execution_time: 1000 -clickhouse_role_manage_grants: true -clickhouse_role_manage_roles: true +clickhouse_role_manage_grants: True clickhouse_custom_grants: - on: databases: [ooni] @@ -276,6 +277,7 @@ clickhouse_custom_grant_roles: - roles: [oonimeasurements] to: [oonimeasurements] +clickhouse_role_manage_roles: True clickhouse_custom_roles: - role: name: oonimeasurements From 798b0c22e456c44f97b8d4c874f1040345123af9 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 30 Apr 2026 10:44:10 +0200 Subject: [PATCH 023/201] fix user password misconfiguration the difference between the _xml and sql managed user settings is poorly documented and fails open. --- ansible/group_vars/clickhouse/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index efeb752d..30390f1a 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -231,8 +231,8 @@ clickhouse_role_manage_users: True clickhouse_custom_users: - user: name: oonimeasurements - password_type: sha256_password - password_sha256_hex: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') | hash('sha256') }}" + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') }}" networks: - "IP '0.0.0.0/0'" settings: From 3773e152ae4c99f9af606de8a2b2b9339d46d6db Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 30 Apr 2026 10:53:24 +0200 Subject: [PATCH 024/201] sql managed users does NOT read var password_sha256_hex nor hash password if type is sha256_hash --- ansible/group_vars/clickhouse/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 30390f1a..0080d366 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -232,7 +232,7 @@ clickhouse_custom_users: - user: name: oonimeasurements password_type: sha256_hash - password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') }}" + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') | hash('sha256') }}" networks: - "IP '0.0.0.0/0'" settings: From b7baff6a71c245bd2db830ae6b68e52f68297121 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 1 May 2026 11:56:39 +0200 Subject: [PATCH 025/201] add clickhouse users for each ooniapi service, fastpath, and testlists --- ansible/group_vars/clickhouse/vars.yml | 115 +++++++++++++++++- .../host_vars/fastpath.dev.ooni.io/vars.yml | 2 +- .../host_vars/fastpath.prod.ooni.io/vars.yml | 2 +- .../host_vars/fastpath2.prod.ooni.io/vars.yml | 2 +- .../testlist-ec2.dev.ooni.io/vars.yml | 1 + .../testlist-ec2.prod.ooni.io/vars.yml | 2 +- ansible/roles/fastpath/defaults/main.yml | 2 +- tf/environments/dev/main.tf | 28 +++-- 8 files changed, 137 insertions(+), 17 deletions(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 0080d366..21fe072a 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -229,6 +229,28 @@ clickhouse_default_users: clickhouse_role_manage_users: True clickhouse_custom_users: + - user: + name: fastpath + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: "fastpath" + databases: [ooni] + + - user: + name: oonifindings + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonifindings_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - readonly + quota: "oonifindings" + databases: [ooni] + - user: name: oonimeasurements password_type: sha256_hash @@ -250,12 +272,54 @@ clickhouse_custom_users: - "max_result_rows = 51000" profile: - readonly - quota: "oonimeasurements" + quota: oonimeasurements + databases: [ooni, oonitest] + + - user: + name: ooniprobe + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_ooniprobe_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: ooniprobe + databases: [ooni] + + - user: + name: oonirun + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonirun_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: oonirun + databases: [ooni] + + - user: + name: oonitestlists + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: oonitestlists databases: [ooni] clickhouse_role_manage_quotas: True clickhouse_custom_quotas: # quota over a 10 minute window + - quota: + name: oonifindings + duration: 600 + queries: 12000 + errors: 1000 + result_rows: 0 + read_rows: 0 + execution_time: 1000 + - quota: name: oonimeasurements duration: 600 @@ -265,13 +329,60 @@ clickhouse_custom_quotas: read_rows: 0 execution_time: 1000 + # no limits set + - quota: + name: ooniprobe + duration: 0 + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 + + # no limits set + - quota: + name: oonirun + duration: 0 + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 + + - quota: + name: oonitestlists + duration: 600 + queries: 12000 + errors: 1000 + result_rows: 0 + read_rows: 0 + execution_time: 1000 + clickhouse_role_manage_grants: True clickhouse_custom_grants: - on: databases: [ooni] tables: ["*"] privileges: [SELECT] - to: [oonimeasurements] + to: [oonifindings, ooniprobe, oonimeasurements, oonirun, fastpath] + +- on: + databases: [ooni] + tables: [url_priorities] + privileges: [INSERT] + to: [oonitestlists] + +- on: + databases: [ooni] + tables: [faulty_measurements] + privileges: [INSERT] + to: [ooniprobe] + +- on: + databases: [ooni] + tables: [fastpath, obs_web, obs_openvpn, jsonl, new_jsonl] + privileges: [INSERT] + to: [fastpath] clickhouse_custom_grant_roles: - roles: [oonimeasurements] diff --git a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml index fbd3d273..e60bcb31 100644 --- a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml @@ -1,5 +1,5 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_dev') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" bucket_name: "ooni-data-eu-fra-test" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "3" diff --git a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml index e7210f1f..ffcb68ef 100644 --- a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml @@ -1,5 +1,5 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "1" diff --git a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml index 28e22d69..fa79213e 100644 --- a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml @@ -1,5 +1,5 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@data3.htz-fsn.prod.ooni.nu/ooni" +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@data3.htz-fsn.prod.ooni.nu/ooni" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "4" diff --git a/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml b/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml index 7ab99273..b6497100 100644 --- a/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml +++ b/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml @@ -1,3 +1,4 @@ jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_dev') }}" github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_dev') }}" log_level: "debug" +clickhouse_url: "clickhouse://oonitestlists:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/ooni" diff --git a/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml b/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml index 59e4bb4c..fb061292 100644 --- a/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml @@ -1,4 +1,4 @@ jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_prod') }}" github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_prod') }}" log_level: "info" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" +clickhouse_url: "clickhouse://oonitestlists:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" diff --git a/ansible/roles/fastpath/defaults/main.yml b/ansible/roles/fastpath/defaults/main.yml index 1a15aea5..2a40e928 100644 --- a/ansible/roles/fastpath/defaults/main.yml +++ b/ansible/roles/fastpath/defaults/main.yml @@ -5,4 +5,4 @@ fastpath_user: fastpath fastpath_home: "/opt/{{ fastpath_user }}" # Fastpath settings -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" \ No newline at end of file +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index c569935b..6b955ffe 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -237,16 +237,24 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } -data "aws_ssm_parameter" "clickhouse_readonly_url" { - name = "/oonidevops/secrets/clickhouse_readonly_url" +data "aws_ssm_parameter" "clickhouse_oonifindings_url" { + name = "/oonidevops/secrets/clickhouse_oonifindings_url" } -data "aws_ssm_parameter" "clickhouse_readonly_test_url" { - name = "/oonidevops/secrets/clickhouse_readonly_test_url" +data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" { + name = "/oonidevops/secrets/clickhouse_oonimeasurements_url" } -data "aws_ssm_parameter" "clickhouse_write_url" { - name = "/oonidevops/secrets/clickhouse_write_url" +data "aws_ssm_parameter" "clickhouse_oonimeasurements_test_url" { + name = "/oonidevops/secrets/clickhouse_oonimeasurements_test_url" +} + +data "aws_ssm_parameter" "clickhouse_ooniprobe_url" { + name = "/oonidevops/secrets/clickhouse_ooniprobe_url" +} + +data "aws_ssm_parameter" "clickhouse_oonirun_url" { + name = "/oonidevops/secrets/clickhouse_oonirun_url" } data "aws_ssm_parameter" "account_id_hashing_key" { @@ -592,7 +600,7 @@ module "ooniapi_ooniprobe" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.arn ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn } @@ -973,7 +981,7 @@ module "ooniapi_oonirun" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.arn } ooniapi_service_security_groups = [ @@ -1024,7 +1032,7 @@ module "ooniapi_oonifindings" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonifindings_url.arn } ooniapi_service_security_groups = [ @@ -1145,7 +1153,7 @@ module "ooniapi_oonimeasurements" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_test_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.arn ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.arn } From ea6e8832463413627d13124165dda60eb896c55b Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 1 May 2026 15:27:47 +0200 Subject: [PATCH 026/201] add clickhouse secrets, clickhouse_url to prod environment note: why was oonifindings missing CLICKHOUSE_URL ? --- tf/environments/prod/main.tf | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 4c343293..16b79de9 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -239,16 +239,24 @@ data "aws_ssm_parameter" "oonipg_url" { name = "/oonidevops/secrets/ooni-tier0-postgres/postgresql_write_url" } -data "aws_ssm_parameter" "clickhouse_readonly_url" { - name = "/oonidevops/secrets/clickhouse_readonly_url" +data "aws_ssm_parameter" "clickhouse_oonifindings_url" { + name = "/oonidevops/secrets/clickhouse_oonifindings_url" } data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" { name = "/oonidevops/secrets/clickhouse_oonimeasurements_url" } -data "aws_ssm_parameter" "clickhouse_write_url" { - name = "/oonidevops/secrets/clickhouse_write_url" +data "aws_ssm_parameter" "clickhouse_oonimeasurements_test_url" { + name = "/oonidevops/secrets/clickhouse_oonimeasurements_test_url" +} + +data "aws_ssm_parameter" "clickhouse_ooniprobe_url" { + name = "/oonidevops/secrets/clickhouse_ooniprobe_url" +} + +data "aws_ssm_parameter" "clickhouse_oonirun_url" { + name = "/oonidevops/secrets/clickhouse_oonirun_url" } data "aws_ssm_parameter" "account_id_hashing_key" { @@ -890,7 +898,7 @@ module "ooniapi_ooniprobe" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.arn ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn } @@ -1088,7 +1096,7 @@ module "ooniapi_oonirun" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.arn } ooniapi_service_security_groups = [ @@ -1140,6 +1148,7 @@ module "ooniapi_oonifindings" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonifindings_url.arn } ooniapi_service_security_groups = [ From 7520f888bde383b714ee5928074a3658d52bb4d0 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 1 May 2026 15:43:58 +0200 Subject: [PATCH 027/201] remove clickhouse settings for oonifindings: uses psgql --- ansible/group_vars/clickhouse/vars.yml | 23 +---------------------- tf/environments/dev/main.tf | 5 ----- tf/environments/prod/main.tf | 5 ----- 3 files changed, 1 insertion(+), 32 deletions(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 21fe072a..236ce652 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -240,17 +240,6 @@ clickhouse_custom_users: quota: "fastpath" databases: [ooni] - - user: - name: oonifindings - password_type: sha256_hash - password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonifindings_password', profile='oonidevops_user_prod') | hash('sha256') }}" - networks: - - "IP '0.0.0.0/0'" - profile: - - readonly - quota: "oonifindings" - databases: [ooni] - - user: name: oonimeasurements password_type: sha256_hash @@ -310,16 +299,6 @@ clickhouse_custom_users: clickhouse_role_manage_quotas: True clickhouse_custom_quotas: - # quota over a 10 minute window - - quota: - name: oonifindings - duration: 600 - queries: 12000 - errors: 1000 - result_rows: 0 - read_rows: 0 - execution_time: 1000 - - quota: name: oonimeasurements duration: 600 @@ -364,7 +343,7 @@ clickhouse_custom_grants: databases: [ooni] tables: ["*"] privileges: [SELECT] - to: [oonifindings, ooniprobe, oonimeasurements, oonirun, fastpath] + to: [ooniprobe, oonimeasurements, oonirun, fastpath] - on: databases: [ooni] diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 6b955ffe..09590756 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -237,10 +237,6 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } -data "aws_ssm_parameter" "clickhouse_oonifindings_url" { - name = "/oonidevops/secrets/clickhouse_oonifindings_url" -} - data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" { name = "/oonidevops/secrets/clickhouse_oonimeasurements_url" } @@ -1032,7 +1028,6 @@ module "ooniapi_oonifindings" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonifindings_url.arn } ooniapi_service_security_groups = [ diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 16b79de9..7c26100e 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -239,10 +239,6 @@ data "aws_ssm_parameter" "oonipg_url" { name = "/oonidevops/secrets/ooni-tier0-postgres/postgresql_write_url" } -data "aws_ssm_parameter" "clickhouse_oonifindings_url" { - name = "/oonidevops/secrets/clickhouse_oonifindings_url" -} - data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" { name = "/oonidevops/secrets/clickhouse_oonimeasurements_url" } @@ -1148,7 +1144,6 @@ module "ooniapi_oonifindings" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonifindings_url.arn } ooniapi_service_security_groups = [ From 065b07613aa99de0383efb32c463eca3edc302a6 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 8 Jun 2026 11:58:55 +0200 Subject: [PATCH 028/201] Add reuploader builder task to dev --- tf/environments/dev/main.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 84de383f..9b4511cb 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -936,6 +936,21 @@ module "fastpath_builder" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket } +module "reuploader_builder" { + source = "../../modules/ooni_docker_build" + trigger_tag = "" + + service_name = "reuploader" + repo = "ooni/backend" + branch_name = "add_fastpath_reuploader" + environment = local.environment + buildspec_path = "reuploader/buildspec.yml" + trigger_path = "fastpath/**" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket +} + #### OONI Run service module "ooniapi_oonirun_deployer" { From ccda78dd1c82f6766c8746ba09bd46ab74cfd747 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 8 Jun 2026 12:30:49 +0200 Subject: [PATCH 029/201] fix trigger path --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 9b4511cb..82b678c1 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -945,7 +945,7 @@ module "reuploader_builder" { branch_name = "add_fastpath_reuploader" environment = local.environment buildspec_path = "reuploader/buildspec.yml" - trigger_path = "fastpath/**" + trigger_path = "reuploader/**" codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket From 01ab56876484898e49182b50199bf4c27735b3ab Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 8 Jun 2026 14:22:37 +0200 Subject: [PATCH 030/201] Extend ooniapi_service to provide scheduled run --- tf/modules/ooniapi_service/main.tf | 68 ++++++++++++++++++++++++- tf/modules/ooniapi_service/variables.tf | 22 ++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/tf/modules/ooniapi_service/main.tf b/tf/modules/ooniapi_service/main.tf index 84e9f38f..068d1651 100644 --- a/tf/modules/ooniapi_service/main.tf +++ b/tf/modules/ooniapi_service/main.tf @@ -36,6 +36,72 @@ resource "aws_iam_role_policy" "ooniapi_service_task" { policy = templatefile("${path.module}/templates/profile_policy.json", {}) } +resource "aws_iam_role" "events_run_task" { + count = var.run_on_schedule ? 1 : 0 + name = "${local.name}-events-run-task-role" + + assume_role_policy = < Date: Tue, 9 Jun 2026 14:18:25 +0200 Subject: [PATCH 031/201] add scheduled_service module --- tf/modules/scheduled_service/main.tf | 156 ++++++++++++++++++ tf/modules/scheduled_service/outputs.tf | 7 + .../templates/profile_policy.json | 61 +++++++ tf/modules/scheduled_service/variables.tf | 79 +++++++++ 4 files changed, 303 insertions(+) create mode 100644 tf/modules/scheduled_service/main.tf create mode 100644 tf/modules/scheduled_service/outputs.tf create mode 100644 tf/modules/scheduled_service/templates/profile_policy.json create mode 100644 tf/modules/scheduled_service/variables.tf diff --git a/tf/modules/scheduled_service/main.tf b/tf/modules/scheduled_service/main.tf new file mode 100644 index 00000000..19e4a3b1 --- /dev/null +++ b/tf/modules/scheduled_service/main.tf @@ -0,0 +1,156 @@ +locals { + name = "scheduled-service-${var.service_name}" + # We construct a stripped name that is without the "ooni" substring and all + # vocals are stripped. + stripped_name = replace(replace(var.service_name, "ooni", ""), "[aeiou]", "") + # Short prefix should be less than 5 characters + short_prefix = "O${substr(local.stripped_name, 0, 3)}" +} + +resource "aws_iam_role" "scheduled_service_task" { + name = "${local.name}-task-role" + + tags = var.tags + + assume_role_policy = < Date: Tue, 9 Jun 2026 14:47:14 +0200 Subject: [PATCH 032/201] add reuploader scheduled service (hourly) --- tf/environments/dev/main.tf | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 82b678c1..ff4089f2 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -951,6 +951,44 @@ module "reuploader_builder" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket } +module "reuploader" { + source = "../../modules/scheduled_service" + + task_memory = 256 + + vpc_id = module.network.vpc_id + + service_name = "reuploader" + default_docker_image_url = "ooni/reuploader:latest" + schedule_expression = "cron(0 * * * ? 2000-2199)" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_secrets = { + AWS_ACCESS_KEY_ID = data.aws_ssm_parameter.s3_user_access_id + AWS_SECRET_ACCESS_KEY = data.aws_ssm_parameter.s3_user_secret_key + #ROLE_ARN = + #ROLE_DURATION_SECONDS = "3600" + AWS_REGION = var.aws_region + # required + BUCKET_NAME = "ooniprobe-failed-reports-eu-central-1-1d24426a" + # PREFIX # s3 path prefix + # fastpath API endpoint; use the last (fallback) fastpath instance in set + FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + tags = merge( + local.tags, + { Name = "ooni-tier0-reuploader" } + ) +} + #### OONI Run service module "ooniapi_oonirun_deployer" { From d245b73db9805c7204042d4f34a9e4fb66be9198 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 9 Jun 2026 16:56:51 +0200 Subject: [PATCH 033/201] set failed reports bucket --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index ff4089f2..92fd1f27 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -973,7 +973,7 @@ module "reuploader" { #ROLE_DURATION_SECONDS = "3600" AWS_REGION = var.aws_region # required - BUCKET_NAME = "ooniprobe-failed-reports-eu-central-1-1d24426a" + BUCKET_NAME = aws_s3_bucket.ooniprobe_failed_reports.bucket # PREFIX # s3 path prefix # fastpath API endpoint; use the last (fallback) fastpath instance in set FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" From 4dd954ed217ea3961c29a5e90851589fcc29ddc1 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 09:42:21 +0200 Subject: [PATCH 034/201] reuploader: set DRY_RUN=true --- tf/environments/dev/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 92fd1f27..689d1f6f 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -974,6 +974,7 @@ module "reuploader" { AWS_REGION = var.aws_region # required BUCKET_NAME = aws_s3_bucket.ooniprobe_failed_reports.bucket + DRY_RUN = true # PREFIX # s3 path prefix # fastpath API endpoint; use the last (fallback) fastpath instance in set FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" From b6e2ba72c0ec9a784d66bddd207472c4e9630b9a Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 09:43:25 +0200 Subject: [PATCH 035/201] reuploader: set BATCH_SIZE=10 --- tf/environments/dev/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 689d1f6f..21dbed28 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -972,6 +972,7 @@ module "reuploader" { #ROLE_ARN = #ROLE_DURATION_SECONDS = "3600" AWS_REGION = var.aws_region + BATCH_SIZE = 10 # required BUCKET_NAME = aws_s3_bucket.ooniprobe_failed_reports.bucket DRY_RUN = true From 0b94e88d3dddb3c3dff62c05243b8ec39a33c424 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 09:43:49 +0200 Subject: [PATCH 036/201] reuploader: set AWS_SECRET_ACCESS_KEY from module --- tf/environments/dev/main.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 21dbed28..7c610dc9 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -967,8 +967,9 @@ module "reuploader" { ecs_cluster_id = module.ooniapi_cluster.cluster_id task_secrets = { - AWS_ACCESS_KEY_ID = data.aws_ssm_parameter.s3_user_access_id - AWS_SECRET_ACCESS_KEY = data.aws_ssm_parameter.s3_user_secret_key + AWS_SECRET_ACCESS_KEY = module.ooniapi_user.aws_secret_access_key_arn + AWS_ACCESS_KEY_ID = module.ooniapi_user.aws_access_key_id_arn + #ROLE_ARN = #ROLE_DURATION_SECONDS = "3600" AWS_REGION = var.aws_region From 22f35d53bfb4f65584a8f135ddeb8bc59c7cb70d Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 10:13:40 +0200 Subject: [PATCH 037/201] reuploader: set scheduled_task_cluster --- tf/environments/dev/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 7c610dc9..94bf5d5a 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -964,6 +964,7 @@ module "reuploader" { stage = local.environment dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name + scheduled_task_cluster = module.ooniapi_cluster.cluster_name ecs_cluster_id = module.ooniapi_cluster.cluster_id task_secrets = { From 3a0b89536a5cf3817443c863f3957de3aee34e2e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 10:14:07 +0200 Subject: [PATCH 038/201] reuploader: remove unused outputs --- tf/modules/scheduled_service/outputs.tf | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tf/modules/scheduled_service/outputs.tf b/tf/modules/scheduled_service/outputs.tf index 85f5994d..e69de29b 100644 --- a/tf/modules/scheduled_service/outputs.tf +++ b/tf/modules/scheduled_service/outputs.tf @@ -1,7 +0,0 @@ -output "ecs_service_name" { - value = aws_ecs_service.ooniapi_service.name -} - -output "alb_target_group_id" { - value = aws_alb_target_group.ooniapi_service.id -} From d171d28fb6248821df8e84a9b1062b4d9036a475 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 10:14:19 +0200 Subject: [PATCH 039/201] reuploader: add first_run to create container definition --- tf/environments/dev/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 94bf5d5a..4cdced80 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -958,6 +958,7 @@ module "reuploader" { vpc_id = module.network.vpc_id + first_run = true service_name = "reuploader" default_docker_image_url = "ooni/reuploader:latest" schedule_expression = "cron(0 * * * ? 2000-2199)" From c2ffbf1275cca3a5b938c2f14fa3621b0a99466a Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 10:16:43 +0200 Subject: [PATCH 040/201] reuploader: pin to tagged container --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 4cdced80..bd42fd48 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -960,7 +960,7 @@ module "reuploader" { first_run = true service_name = "reuploader" - default_docker_image_url = "ooni/reuploader:latest" + default_docker_image_url = "ooni/reuploader:20260611-f9cf0ff7" schedule_expression = "cron(0 * * * ? 2000-2199)" stage = local.environment dns_zone_ooni_io = local.dns_zone_ooni_io From d55b50225000bded876d7b7bf03b57b8a1a885da Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 12:48:24 +0200 Subject: [PATCH 041/201] remove redundant count --- tf/modules/scheduled_service/main.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tf/modules/scheduled_service/main.tf b/tf/modules/scheduled_service/main.tf index 19e4a3b1..9c18fbdc 100644 --- a/tf/modules/scheduled_service/main.tf +++ b/tf/modules/scheduled_service/main.tf @@ -37,7 +37,6 @@ resource "aws_iam_role_policy" "scheduled_service_task" { } resource "aws_iam_role" "events_run_task" { - count = 1 name = "${local.name}-events-run-task-role" assume_role_policy = < Date: Thu, 11 Jun 2026 13:08:55 +0200 Subject: [PATCH 042/201] singleton requires no index --- tf/modules/scheduled_service/main.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tf/modules/scheduled_service/main.tf b/tf/modules/scheduled_service/main.tf index 9c18fbdc..b269742b 100644 --- a/tf/modules/scheduled_service/main.tf +++ b/tf/modules/scheduled_service/main.tf @@ -55,7 +55,7 @@ EOF resource "aws_iam_role_policy" "events_run_task_policy" { name = "${local.name}-events-run-task-policy" - role = aws_iam_role.events_run_task[0].id + role = aws_iam_role.events_run_task.id policy = jsonencode({ Version = "2012-10-17" @@ -82,10 +82,10 @@ resource "aws_cloudwatch_event_rule" "scheduled_run" { } resource "aws_cloudwatch_event_target" "run_ecs_task" { - rule = aws_cloudwatch_event_rule.scheduled_run[0].name - arn = data.aws_ecs_cluster.target[0].arn + rule = aws_cloudwatch_event_rule.scheduled_run.name + arn = data.aws_ecs_cluster.target.arn - role_arn = aws_iam_role.events_run_task[0].arn + role_arn = aws_iam_role.events_run_task.arn ecs_target { task_definition_arn = aws_ecs_task_definition.scheduled_service.arn @@ -119,7 +119,7 @@ resource "aws_ecs_task_definition" "scheduled_service" { memory = var.memory_hard_limit essential = true, image = try( - data.aws_ecs_container_definition.scheduled_service_current[0].image, + data.aws_ecs_container_definition.scheduled_service_current.image, var.default_docker_image_url ), name = local.name, From 27f9c59c62cf8c3eeca8dec37bd12238e604facb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 13:43:12 +0200 Subject: [PATCH 043/201] FIXME: try to add events:PutRule et al to profile I see AccessDeniedException; but this change doesn't fix it --- tf/modules/scheduled_service/main.tf | 7 +++++-- .../scheduled_service/templates/profile_policy.json | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tf/modules/scheduled_service/main.tf b/tf/modules/scheduled_service/main.tf index b269742b..b09cf7d1 100644 --- a/tf/modules/scheduled_service/main.tf +++ b/tf/modules/scheduled_service/main.tf @@ -67,7 +67,10 @@ resource "aws_iam_role_policy" "events_run_task_policy" { "iam:PassRole", "ecs:StartTask", "ecs:DescribeClusters", - "ecs:DescribeTasks" + "ecs:DescribeTasks", + "events:TagResource", + "events:PutRule", + "events:PutTargets", ] Resource = "*" } @@ -119,7 +122,7 @@ resource "aws_ecs_task_definition" "scheduled_service" { memory = var.memory_hard_limit essential = true, image = try( - data.aws_ecs_container_definition.scheduled_service_current.image, + data.aws_ecs_container_definition.scheduled_service_current[0].image, var.default_docker_image_url ), name = local.name, diff --git a/tf/modules/scheduled_service/templates/profile_policy.json b/tf/modules/scheduled_service/templates/profile_policy.json index 3a772893..b0a059d2 100644 --- a/tf/modules/scheduled_service/templates/profile_policy.json +++ b/tf/modules/scheduled_service/templates/profile_policy.json @@ -56,6 +56,16 @@ "elasticloadbalancing:RegisterTargets" ], "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "events:TagResource", + "events:PutRule", + "events:PutTargets" + ], + "Resource": "*" } + ] } From 093c98f73383cb077da4abc8ee5ca14961ecb40b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 11 Jun 2026 14:25:12 +0200 Subject: [PATCH 044/201] Add permission to the ooni_devops role to modify events --- tf/modules/adm_iam_roles/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tf/modules/adm_iam_roles/main.tf b/tf/modules/adm_iam_roles/main.tf index aa5c525f..10fb15f9 100644 --- a/tf/modules/adm_iam_roles/main.tf +++ b/tf/modules/adm_iam_roles/main.tf @@ -53,7 +53,8 @@ resource "aws_iam_policy" "oonidevops" { "secretsmanager:*", "cloudhsm:*", "athena:*", - "glue:*" + "glue:*", + "events:*" ], "Resource": "*" } From 94f3638eedc2d3dad9e9cbf7b3f452312c81f398 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 15:16:01 +0200 Subject: [PATCH 045/201] unmix environment from secrets --- tf/environments/dev/main.tf | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index f17b1a05..dcd73ac6 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -968,15 +968,8 @@ module "reuploader" { scheduled_task_cluster = module.ooniapi_cluster.cluster_name ecs_cluster_id = module.ooniapi_cluster.cluster_id - task_secrets = { - AWS_SECRET_ACCESS_KEY = module.ooniapi_user.aws_secret_access_key_arn - AWS_ACCESS_KEY_ID = module.ooniapi_user.aws_access_key_id_arn - - #ROLE_ARN = - #ROLE_DURATION_SECONDS = "3600" - AWS_REGION = var.aws_region + task_environment = { BATCH_SIZE = 10 - # required BUCKET_NAME = aws_s3_bucket.ooniprobe_failed_reports.bucket DRY_RUN = true # PREFIX # s3 path prefix @@ -984,6 +977,12 @@ module "reuploader" { FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" } + task_secrets = { + AWS_SECRET_ACCESS_KEY = module.ooniapi_user.aws_secret_access_key_arn + AWS_ACCESS_KEY_ID = module.ooniapi_user.aws_access_key_id_arn + AWS_REGION = var.aws_region + } + ooniapi_service_security_groups = [ module.ooniapi_cluster.web_security_group_id ] From 92fb38aee75614bd0f1fd8a356c4613fdebb05d6 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 15:42:38 +0200 Subject: [PATCH 046/201] use bucket from https://github.com/ooni/devops/issues/398 --- tf/environments/dev/main.tf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index dcd73ac6..d69c3683 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -961,7 +961,7 @@ module "reuploader" { first_run = true service_name = "reuploader" default_docker_image_url = "ooni/reuploader:20260611-f9cf0ff7" - schedule_expression = "cron(0 * * * ? 2000-2199)" + schedule_expression = "cron(42 * * * ? 2000-2199)" stage = local.environment dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name @@ -970,10 +970,8 @@ module "reuploader" { task_environment = { BATCH_SIZE = 10 - BUCKET_NAME = aws_s3_bucket.ooniprobe_failed_reports.bucket + BUCKET_NAME = "ooniprobe-failed-reports-eu-central-1-1d24426a" DRY_RUN = true - # PREFIX # s3 path prefix - # fastpath API endpoint; use the last (fallback) fastpath instance in set FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" } From bafa1755bf0f2eaa9fe8e0ffa31407290a029eae Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 16:18:25 +0200 Subject: [PATCH 047/201] update reuploader, fix env --- tf/environments/dev/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index d69c3683..d922aa2e 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -960,8 +960,8 @@ module "reuploader" { first_run = true service_name = "reuploader" - default_docker_image_url = "ooni/reuploader:20260611-f9cf0ff7" - schedule_expression = "cron(42 * * * ? 2000-2199)" + default_docker_image_url = "ooni/reuploader:20260611-840e1b63" + schedule_expression = "cron(0 * * * ? 2000-2199)" stage = local.environment dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name From ed341e5b72dc2b52dcae1749cf332331ec40ebb2 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 16:49:50 +0200 Subject: [PATCH 048/201] add AWS_REGION to task_environment --- tf/environments/dev/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index d922aa2e..c9a70776 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -961,7 +961,7 @@ module "reuploader" { first_run = true service_name = "reuploader" default_docker_image_url = "ooni/reuploader:20260611-840e1b63" - schedule_expression = "cron(0 * * * ? 2000-2199)" + schedule_expression = "cron(0/5 * * * ? 2000-2199)" stage = local.environment dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name @@ -969,6 +969,7 @@ module "reuploader" { ecs_cluster_id = module.ooniapi_cluster.cluster_id task_environment = { + AWS_REGION = var.aws_region BATCH_SIZE = 10 BUCKET_NAME = "ooniprobe-failed-reports-eu-central-1-1d24426a" DRY_RUN = true @@ -978,7 +979,6 @@ module "reuploader" { task_secrets = { AWS_SECRET_ACCESS_KEY = module.ooniapi_user.aws_secret_access_key_arn AWS_ACCESS_KEY_ID = module.ooniapi_user.aws_access_key_id_arn - AWS_REGION = var.aws_region } ooniapi_service_security_groups = [ From bb5a163bdbce9a4349d2cf075c38fcaa3647344d Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 11 Jun 2026 16:58:42 +0200 Subject: [PATCH 049/201] set S3_BUCKET_NAME env --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index c9a70776..436b1d6d 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -971,7 +971,7 @@ module "reuploader" { task_environment = { AWS_REGION = var.aws_region BATCH_SIZE = 10 - BUCKET_NAME = "ooniprobe-failed-reports-eu-central-1-1d24426a" + S3_BUCKET_NAME = "ooniprobe-failed-reports-eu-central-1-1d24426a" DRY_RUN = true FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" } From bea658faea7608bbc3b4d8979f548c2e64331c0e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 16 Jun 2026 17:04:56 +0200 Subject: [PATCH 050/201] test reading primary failed reports bucket --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 436b1d6d..3f3f5252 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -971,7 +971,7 @@ module "reuploader" { task_environment = { AWS_REGION = var.aws_region BATCH_SIZE = 10 - S3_BUCKET_NAME = "ooniprobe-failed-reports-eu-central-1-1d24426a" + S3_BUCKET_NAME = aws_s3_bucket.ooniprobe_failed_reports.bucket DRY_RUN = true FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" } From a5fdbc106cfe65692fc0d309ffd14ee83fb4294f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 16 Jun 2026 17:49:04 +0200 Subject: [PATCH 051/201] add iam_role_policy for reuploader task --- tf/environments/dev/main.tf | 25 +++++++++++++++++++++++++ tf/modules/scheduled_service/outputs.tf | 14 ++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 3f3f5252..d5547783 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -991,6 +991,31 @@ module "reuploader" { ) } +# For reuploader accessing the failed reports s3 bucket +resource "aws_iam_role_policy" "reuploader_role" { + name = "${local.name}-task-role" + role = module.reuploader.task_role_name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "" + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = "${aws_s3_bucket.ooniprobe_failed_reports.arn}/*" + }, + { + Sid = "" + Effect = "Allow" + Action = ["s3:ListBucket"] + Resource = aws_s3_bucket.ooniprobe_failed_reports.arn + } + ] + }) +} + + #### OONI Run service module "ooniapi_oonirun_deployer" { diff --git a/tf/modules/scheduled_service/outputs.tf b/tf/modules/scheduled_service/outputs.tf index e69de29b..a4970d33 100644 --- a/tf/modules/scheduled_service/outputs.tf +++ b/tf/modules/scheduled_service/outputs.tf @@ -0,0 +1,14 @@ +output "task_role_name" { +description = "IAM role name used for scheduled task" +value = aws_iam_role.events_run_task.name +} + +output "task_role_id" { +description = "IAM role ID for the scheduled task" +value = aws_iam_role.events_run_task.id +} + +output "task_role_arn" { +description = "IAM role ARN for the scheduled task" +value = aws_iam_role.events_run_task.arn +} From e9ebb3f85bb95ec50bf9efe69adcadcf988aa4f6 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 16 Jun 2026 18:26:45 +0200 Subject: [PATCH 052/201] remove task secrets from reuploader; container uses ecs task role --- tf/environments/dev/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index d5547783..555dd81e 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -977,8 +977,6 @@ module "reuploader" { } task_secrets = { - AWS_SECRET_ACCESS_KEY = module.ooniapi_user.aws_secret_access_key_arn - AWS_ACCESS_KEY_ID = module.ooniapi_user.aws_access_key_id_arn } ooniapi_service_security_groups = [ From 1e6802e22bf6bdc5dd677e06869d75c0c54fdca3 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 17 Jun 2026 12:18:25 +0200 Subject: [PATCH 053/201] output scheduled_service_task.arn as task_role_arn --- tf/environments/dev/main.tf | 2 +- tf/modules/scheduled_service/outputs.tf | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 555dd81e..46347e81 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -960,7 +960,7 @@ module "reuploader" { first_run = true service_name = "reuploader" - default_docker_image_url = "ooni/reuploader:20260611-840e1b63" + default_docker_image_url = "ooni/reuploader:20260617-8b35a38f" schedule_expression = "cron(0/5 * * * ? 2000-2199)" stage = local.environment dns_zone_ooni_io = local.dns_zone_ooni_io diff --git a/tf/modules/scheduled_service/outputs.tf b/tf/modules/scheduled_service/outputs.tf index a4970d33..c22e9033 100644 --- a/tf/modules/scheduled_service/outputs.tf +++ b/tf/modules/scheduled_service/outputs.tf @@ -1,14 +1,14 @@ output "task_role_name" { description = "IAM role name used for scheduled task" -value = aws_iam_role.events_run_task.name +value = aws_iam_role.scheduled_service_task.name } output "task_role_id" { description = "IAM role ID for the scheduled task" -value = aws_iam_role.events_run_task.id +value = aws_iam_role.scheduled_service_task.id } output "task_role_arn" { description = "IAM role ARN for the scheduled task" -value = aws_iam_role.events_run_task.arn +value = aws_iam_role.scheduled_service_task.arn } From 8a603fadb28ecf5f7a774de077b5c1c452d11b64 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 17 Jun 2026 12:33:37 +0200 Subject: [PATCH 054/201] set task_role_arn = scheduled_service_task.arn --- tf/modules/scheduled_service/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/modules/scheduled_service/main.tf b/tf/modules/scheduled_service/main.tf index b09cf7d1..7dc94d24 100644 --- a/tf/modules/scheduled_service/main.tf +++ b/tf/modules/scheduled_service/main.tf @@ -148,6 +148,7 @@ resource "aws_ecs_task_definition" "scheduled_service" { } } ]) + task_role_arn = aws_iam_role.scheduled_service_task.arn execution_role_arn = aws_iam_role.scheduled_service_task.arn tags = var.tags track_latest = true From 20b309aa76fd76ef67eaa0a7b088321b543a9024 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 17 Jun 2026 16:15:32 +0200 Subject: [PATCH 055/201] WIP: use the bucket from ticket https://github.com/ooni/devops/issues/398 --- tf/environments/dev/main.tf | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 46347e81..9f60b731 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -354,6 +354,10 @@ resource "aws_s3_bucket" "ooniprobe_failed_reports" { bucket = "ooniprobe-failed-reports-${var.aws_region}" } +data "aws_s3_bucket" "ooniprobe_failed_reports_2026_04_10" { + bucket = "ooniprobe-failed-reports-eu-central-1-1d24426a" +} + resource "aws_s3_bucket" "ooniapi_codepipeline_bucket" { bucket = "codepipeline-ooniapi-${var.aws_region}-${random_id.artifact_id.hex}" } @@ -971,7 +975,7 @@ module "reuploader" { task_environment = { AWS_REGION = var.aws_region BATCH_SIZE = 10 - S3_BUCKET_NAME = aws_s3_bucket.ooniprobe_failed_reports.bucket + S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket DRY_RUN = true FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" } @@ -1001,13 +1005,13 @@ resource "aws_iam_role_policy" "reuploader_role" { Sid = "" Effect = "Allow" Action = ["s3:GetObject"] - Resource = "${aws_s3_bucket.ooniprobe_failed_reports.arn}/*" + Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" }, { Sid = "" Effect = "Allow" Action = ["s3:ListBucket"] - Resource = aws_s3_bucket.ooniprobe_failed_reports.arn + Resource = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn } ] }) From 73d4580898402621ec1470ff1f2ea112eb12b813 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 23 Jun 2026 12:06:16 +0200 Subject: [PATCH 056/201] move reuploader scheduled service to prod environment --- tf/environments/dev/main.tf | 78 ----------------------------------- tf/environments/prod/main.tf | 79 ++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 78 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 9f60b731..b081b14b 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -940,84 +940,6 @@ module "fastpath_builder" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket } -module "reuploader_builder" { - source = "../../modules/ooni_docker_build" - trigger_tag = "" - - service_name = "reuploader" - repo = "ooni/backend" - branch_name = "add_fastpath_reuploader" - environment = local.environment - buildspec_path = "reuploader/buildspec.yml" - trigger_path = "reuploader/**" - codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn - - codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket -} - -module "reuploader" { - source = "../../modules/scheduled_service" - - task_memory = 256 - - vpc_id = module.network.vpc_id - - first_run = true - service_name = "reuploader" - default_docker_image_url = "ooni/reuploader:20260617-8b35a38f" - schedule_expression = "cron(0/5 * * * ? 2000-2199)" - stage = local.environment - dns_zone_ooni_io = local.dns_zone_ooni_io - key_name = module.adm_iam_roles.oonidevops_key_name - scheduled_task_cluster = module.ooniapi_cluster.cluster_name - ecs_cluster_id = module.ooniapi_cluster.cluster_id - - task_environment = { - AWS_REGION = var.aws_region - BATCH_SIZE = 10 - S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket - DRY_RUN = true - FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" - } - - task_secrets = { - } - - ooniapi_service_security_groups = [ - module.ooniapi_cluster.web_security_group_id - ] - - tags = merge( - local.tags, - { Name = "ooni-tier0-reuploader" } - ) -} - -# For reuploader accessing the failed reports s3 bucket -resource "aws_iam_role_policy" "reuploader_role" { - name = "${local.name}-task-role" - role = module.reuploader.task_role_name - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Sid = "" - Effect = "Allow" - Action = ["s3:GetObject"] - Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" - }, - { - Sid = "" - Effect = "Allow" - Action = ["s3:ListBucket"] - Resource = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn - } - ] - }) -} - - #### OONI Run service module "ooniapi_oonirun_deployer" { diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 2fb42ce7..8ba689b3 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1021,6 +1021,85 @@ module "fastpath_builder" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket } +module "reuploader_builder" { + source = "../../modules/ooni_docker_build" + trigger_tag = "" + + service_name = "reuploader" + repo = "ooni/backend" + branch_name = "add_fastpath_reuploader" + environment = local.environment + buildspec_path = "reuploader/buildspec.yml" + trigger_path = "reuploader/**" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket +} + +module "reuploader" { + source = "../../modules/scheduled_service" + + task_memory = 256 + + vpc_id = module.network.vpc_id + + first_run = true + service_name = "reuploader" + default_docker_image_url = "ooni/reuploader:20260617-8b35a38f" + schedule_expression = "cron(30 * * * ? 2000-2199)" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + scheduled_task_cluster = module.ooniapi_cluster.cluster_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_environment = { + AWS_REGION = var.aws_region + BATCH_SIZE = 10 + S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket + DRY_RUN = true + FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" + } + + task_secrets = { + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + tags = merge( + local.tags, + { Name = "ooni-tier0-reuploader" } + ) +} + +# For reuploader accessing the failed reports s3 bucket +resource "aws_iam_role_policy" "reuploader_role" { + name = "${local.name}-task-role" + role = module.reuploader.task_role_name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "" + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" + }, + { + Sid = "" + Effect = "Allow" + Action = ["s3:ListBucket"] + Resource = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn + } + ] + }) +} + + + #### OONI Run service From 6b941c73f3202f7857f3036f08db046865432535 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 23 Jun 2026 12:08:54 +0200 Subject: [PATCH 057/201] use reuploader fastpath instance --- tf/environments/prod/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 8ba689b3..1d7759d2 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1058,7 +1058,7 @@ module "reuploader" { BATCH_SIZE = 10 S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket DRY_RUN = true - FASTPATH_API = "http://${local.fastpath_hosts[length(local.fastpath_hosts) - 1]}:8472" + FASTPATH_API = "http://${module.ooni_reuploader_fastpath.aws_instance_private_ip}:8472" } task_secrets = { From 9fd3ddc1906c3f555d0fd111fd3faa6a093db45f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 23 Jun 2026 12:10:44 +0200 Subject: [PATCH 058/201] moved ref to failed_reports_2026_04_10 to prod --- tf/environments/dev/main.tf | 4 ---- tf/environments/prod/main.tf | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index b081b14b..5d1f9e64 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -354,10 +354,6 @@ resource "aws_s3_bucket" "ooniprobe_failed_reports" { bucket = "ooniprobe-failed-reports-${var.aws_region}" } -data "aws_s3_bucket" "ooniprobe_failed_reports_2026_04_10" { - bucket = "ooniprobe-failed-reports-eu-central-1-1d24426a" -} - resource "aws_s3_bucket" "ooniapi_codepipeline_bucket" { bucket = "codepipeline-ooniapi-${var.aws_region}-${random_id.artifact_id.hex}" } diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 1d7759d2..251134e0 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -293,6 +293,10 @@ resource "random_id" "artifact_id" { byte_length = 4 } +data "aws_s3_bucket" "ooniprobe_failed_reports_2026_04_10" { + bucket = "ooniprobe-failed-reports-eu-central-1-1d24426a" +} + resource "aws_s3_bucket" "ooniprobe_failed_reports" { bucket = "ooniprobe-failed-reports-${var.aws_region}-${random_id.artifact_id.hex}" } From d58bca75b0d7c4ec196dbd1a289f4e080997efdd Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 23 Jun 2026 13:57:23 +0200 Subject: [PATCH 059/201] set LOG_LEVEL to DEBUG --- tf/environments/prod/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 251134e0..05d233c4 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1063,6 +1063,7 @@ module "reuploader" { S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket DRY_RUN = true FASTPATH_API = "http://${module.ooni_reuploader_fastpath.aws_instance_private_ip}:8472" + LOG_LEVEL = "DEBUG" } task_secrets = { From 7e362e72a3cbc183e9091d55fe5136c21a70c905 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 23 Jun 2026 14:57:36 +0200 Subject: [PATCH 060/201] add reuploaderfastpath to clickhouse nft rules --- ansible/group_vars/clickhouse/vars.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 7de27f93..d7a8a175 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -21,6 +21,8 @@ nftables_clickhouse_allow: ip: "{{ lookup('dig', 'fastpath.prod.ooni.io/A') }}" - fqdn: fastpath2.prod.ooni.io ip: "{{ lookup('dig', 'fastpath2.prod.ooni.io/A') }}" + - fqdn: reuploaderfastpath.prod.ooni.io + ip: "{{ lookup('dig', 'reuploaderfastpath.prod.ooni.io/A') }}" nftables_zookeeper_allow: - fqdn: data1.htz-fsn.prod.ooni.nu From f4d0868f6b400f76b11654795925e178970c1c3a Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 23 Jun 2026 16:18:56 +0200 Subject: [PATCH 061/201] revert changes to ooniapi_service --- tf/modules/ooniapi_service/main.tf | 68 +------------------------ tf/modules/ooniapi_service/variables.tf | 22 -------- 2 files changed, 1 insertion(+), 89 deletions(-) diff --git a/tf/modules/ooniapi_service/main.tf b/tf/modules/ooniapi_service/main.tf index 068d1651..84e9f38f 100644 --- a/tf/modules/ooniapi_service/main.tf +++ b/tf/modules/ooniapi_service/main.tf @@ -36,72 +36,6 @@ resource "aws_iam_role_policy" "ooniapi_service_task" { policy = templatefile("${path.module}/templates/profile_policy.json", {}) } -resource "aws_iam_role" "events_run_task" { - count = var.run_on_schedule ? 1 : 0 - name = "${local.name}-events-run-task-role" - - assume_role_policy = < Date: Tue, 23 Jun 2026 16:37:35 +0200 Subject: [PATCH 062/201] fastpath: switch to ooni.io domains --- ansible/host_vars/fastpath.prod.ooni.io/vars.yml | 2 +- ansible/host_vars/fastpath2.prod.ooni.io/vars.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml index b9692f7a..bd108a07 100644 --- a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml @@ -1,5 +1,5 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@data1.htz-fsn.prod.ooni.nu/ooni" +clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "1" diff --git a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml index 17a5af5a..b4deab06 100644 --- a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml @@ -1,5 +1,5 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@data1.htz-fsn.prod.ooni.nu/ooni" +clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "4" From 406f35022f7d0210d1db51f341be4651ec910fff Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 23 Jun 2026 16:40:09 +0200 Subject: [PATCH 063/201] also update reuploaderfastpath clickhouse_url --- ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml b/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml index f1507236..bdb7fd63 100644 --- a/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml @@ -1,5 +1,5 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@data1.htz-fsn.prod.ooni.nu/ooni" +clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "5" From 3fc321f0d25408dce2d71da82eed8c1c37984f61 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 29 Jun 2026 11:00:29 +0200 Subject: [PATCH 064/201] change BATCH_SIZE; allow deleteobject on failed-reports BATCH_SIZE was refactored to count individual measurements. deleteobject removes failed-reports after they are resubmitted. --- tf/environments/prod/main.tf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 05d233c4..388e4a7b 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1059,9 +1059,8 @@ module "reuploader" { task_environment = { AWS_REGION = var.aws_region - BATCH_SIZE = 10 + BATCH_SIZE = 150000 S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket - DRY_RUN = true FASTPATH_API = "http://${module.ooni_reuploader_fastpath.aws_instance_private_ip}:8472" LOG_LEVEL = "DEBUG" } @@ -1098,6 +1097,12 @@ resource "aws_iam_role_policy" "reuploader_role" { Effect = "Allow" Action = ["s3:ListBucket"] Resource = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn + }, + { + Sid = "" + Effect = "Allow" + Action = ["s3:DeleteObject"] + Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" } ] }) From 67d0763d685e67114405e09fbc42d49fed98b5c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 10 Jul 2026 14:47:09 +0200 Subject: [PATCH 065/201] Add new notebook user --- ansible/group_vars/clickhouse/vars.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index f6555811..41e59872 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -303,6 +303,17 @@ clickhouse_custom_users: quota: oonitestlists databases: [ooni] + - user: + name: notebook + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_notebook_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: notebook + databases: [ooni] + clickhouse_role_manage_quotas: True clickhouse_custom_quotas: - quota: @@ -343,13 +354,23 @@ clickhouse_custom_quotas: read_rows: 0 execution_time: 1000 + # no limits set + - quota: + name: notebook + duration: 0 + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 + clickhouse_role_manage_grants: True clickhouse_custom_grants: - on: databases: [ooni] tables: ["*"] privileges: [SELECT] - to: [ooniprobe, oonimeasurements, oonirun, fastpath] + to: [ooniprobe, oonimeasurements, oonirun, fastpath, notebook] - on: databases: [ooni] From 989170dc92de91950c61dd6af89a3948be9d8cc3 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 10 Jul 2026 14:56:53 +0200 Subject: [PATCH 066/201] remove willshersystems.sshd apply the same configuration settings, and drop willshersystems.sshd from galaxy requirements --- ansible/requirements/ansible-galaxy.yml | 4 +--- ansible/roles/ssh_users/tasks/main.yml | 16 ++++++++-------- ansible/roles/ssh_users/templates/allow_users | 0 ansible/roles/ssh_users/templates/sshd_config | 9 +++++++++ 4 files changed, 18 insertions(+), 11 deletions(-) create mode 100644 ansible/roles/ssh_users/templates/allow_users create mode 100644 ansible/roles/ssh_users/templates/sshd_config diff --git a/ansible/requirements/ansible-galaxy.yml b/ansible/requirements/ansible-galaxy.yml index 997d7a83..730768b8 100644 --- a/ansible/requirements/ansible-galaxy.yml +++ b/ansible/requirements/ansible-galaxy.yml @@ -1,6 +1,4 @@ roles: - - src: willshersystems.sshd - version: v0.25.0 - src: nginxinc.nginx version: 0.24.3 - src: geerlingguy.certbot @@ -18,4 +16,4 @@ roles: version: 7.4.7 collections: - name: community.docker # manages containers - version: 4.6.1 \ No newline at end of file + version: 4.6.1 diff --git a/ansible/roles/ssh_users/tasks/main.yml b/ansible/roles/ssh_users/tasks/main.yml index 6ab0e805..3615a9ba 100644 --- a/ansible/roles/ssh_users/tasks/main.yml +++ b/ansible/roles/ssh_users/tasks/main.yml @@ -52,16 +52,16 @@ mode: 0400 with_items: "{{ admin_usernames | union(non_admin_usernames) }}" -- name: configure sshd +- name: configure sshd_config tags: ssh_users - include_role: - name: willshersystems.sshd - vars: - sshd_skip_defaults: false - sshd: - AllowUsers: "{{ admin_usernames | union(non_admin_usernames) | sort | join(' ') }}" + template: + src: sshd_config + dest: /etc/ssh/sshd_config + owner: root + group: root + mode: 0440 -- name: Enesure sudoers dir exists +- name: Ensure sudoers dir exists tags: ssh_users ansible.builtin.file: path: /etc/sudoers.d diff --git a/ansible/roles/ssh_users/templates/allow_users b/ansible/roles/ssh_users/templates/allow_users new file mode 100644 index 00000000..e69de29b diff --git a/ansible/roles/ssh_users/templates/sshd_config b/ansible/roles/ssh_users/templates/sshd_config new file mode 100644 index 00000000..a8b1ea5a --- /dev/null +++ b/ansible/roles/ssh_users/templates/sshd_config @@ -0,0 +1,9 @@ +Include /etc/ssh/sshd_config.d/*.conf +AllowUsers {{ admin_usernames | union(non_admin_usernames) | sort | join(' ') }} +PasswordAuthentication No +KbdInteractiveAuthentication no +UsePAM yes +X11Forwarding yes +PrintMotd no +AcceptEnv LANG LC_* +Subsystem sftp /usr/lib/openssh/sftp-server From 3cb3fbad48464b7980215b4fa7eb4d16fbda37b8 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 10 Jul 2026 15:41:34 +0200 Subject: [PATCH 067/201] Set sshd Ciphers and KexAlgorithms --- ansible/roles/ssh_users/templates/sshd_config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/roles/ssh_users/templates/sshd_config b/ansible/roles/ssh_users/templates/sshd_config index a8b1ea5a..6f7b1273 100644 --- a/ansible/roles/ssh_users/templates/sshd_config +++ b/ansible/roles/ssh_users/templates/sshd_config @@ -7,3 +7,6 @@ X11Forwarding yes PrintMotd no AcceptEnv LANG LC_* Subsystem sftp /usr/lib/openssh/sftp-server +Ciphers chacha20-poly1305@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr +KexAlgorithms mlkem768x25519-sha256,curve25519-sha256 +MACs hmac-sha2-512,hmac-sha2-256 From 421f2cb56ce764145fdca236286dd0eeb3ec6ceb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 10 Jul 2026 17:50:18 +0200 Subject: [PATCH 068/201] remove tailscale from requirements/ansible-galaxy.yml --- ansible/requirements/ansible-galaxy.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/requirements/ansible-galaxy.yml b/ansible/requirements/ansible-galaxy.yml index 730768b8..e31076d3 100644 --- a/ansible/requirements/ansible-galaxy.yml +++ b/ansible/requirements/ansible-galaxy.yml @@ -3,8 +3,6 @@ roles: version: 0.24.3 - src: geerlingguy.certbot version: 5.2.0 - - src: artis3n.tailscale - version: v4.5.0 - src: https://github.com/idealista/clickhouse_role scm: git version: 3.5.1 From c0d9888a81ce36d29d0d709c9234255cec0cc486 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 10 Jul 2026 17:51:58 +0200 Subject: [PATCH 069/201] remove unused role tailnet --- ansible/roles/tailnet/tasks/main.yml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 ansible/roles/tailnet/tasks/main.yml diff --git a/ansible/roles/tailnet/tasks/main.yml b/ansible/roles/tailnet/tasks/main.yml deleted file mode 100644 index 86bc4b3d..00000000 --- a/ansible/roles/tailnet/tasks/main.yml +++ /dev/null @@ -1,4 +0,0 @@ -- ansible.builtin.include_role: - name: artis3n.tailscale - tags: - - tailnet From f4207d68f66f498bddcde2769251b0f629c8f113 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 13 Jul 2026 11:58:12 +0200 Subject: [PATCH 070/201] run reuploader daily, limit the batch size --- tf/environments/prod/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 388e4a7b..cde4e0b7 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1050,7 +1050,7 @@ module "reuploader" { first_run = true service_name = "reuploader" default_docker_image_url = "ooni/reuploader:20260617-8b35a38f" - schedule_expression = "cron(30 * * * ? 2000-2199)" + schedule_expression = "cron(30 0 * * ? 2000-2199)" stage = local.environment dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name @@ -1059,7 +1059,7 @@ module "reuploader" { task_environment = { AWS_REGION = var.aws_region - BATCH_SIZE = 150000 + BATCH_SIZE = 50000 S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket FASTPATH_API = "http://${module.ooni_reuploader_fastpath.aws_instance_private_ip}:8472" LOG_LEVEL = "DEBUG" From da0f139df6ab33f405f0509e466eb07070f29531 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 13 Jul 2026 10:21:49 +0000 Subject: [PATCH 071/201] chore(terraform): auto-format on merge to main --- tf/environments/prod/main.tf | 44 +++++++++++------------ tf/modules/scheduled_service/main.tf | 10 +++--- tf/modules/scheduled_service/outputs.tf | 12 +++---- tf/modules/scheduled_service/variables.tf | 6 ++-- 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 4a09269b..deb6869c 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1062,11 +1062,11 @@ module "reuploader" { ecs_cluster_id = module.ooniapi_cluster.cluster_id task_environment = { - AWS_REGION = var.aws_region - BATCH_SIZE = 50000 - S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket - FASTPATH_API = "http://${module.ooni_reuploader_fastpath.aws_instance_private_ip}:8472" - LOG_LEVEL = "DEBUG" + AWS_REGION = var.aws_region + BATCH_SIZE = 50000 + S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket + FASTPATH_API = "http://${module.ooni_reuploader_fastpath.aws_instance_private_ip}:8472" + LOG_LEVEL = "DEBUG" } task_secrets = { @@ -1091,24 +1091,24 @@ resource "aws_iam_role_policy" "reuploader_role" { Version = "2012-10-17" Statement = [ { - Sid = "" - Effect = "Allow" - Action = ["s3:GetObject"] + Sid = "" + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" + }, + { + Sid = "" + Effect = "Allow" + Action = ["s3:ListBucket"] + Resource = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn + }, + { + Sid = "" + Effect = "Allow" + Action = ["s3:DeleteObject"] Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" - }, - { - Sid = "" - Effect = "Allow" - Action = ["s3:ListBucket"] - Resource = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn - }, - { - Sid = "" - Effect = "Allow" - Action = ["s3:DeleteObject"] - Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" - } - ] + } + ] }) } diff --git a/tf/modules/scheduled_service/main.tf b/tf/modules/scheduled_service/main.tf index 7dc94d24..63f745c1 100644 --- a/tf/modules/scheduled_service/main.tf +++ b/tf/modules/scheduled_service/main.tf @@ -37,7 +37,7 @@ resource "aws_iam_role_policy" "scheduled_service_task" { } resource "aws_iam_role" "events_run_task" { - name = "${local.name}-events-run-task-role" + name = "${local.name}-events-run-task-role" assume_role_policy = < Date: Mon, 13 Jul 2026 12:22:26 +0200 Subject: [PATCH 072/201] change reuploader branch_name to master https://github.com/ooni/backend/pull/1219 is merged --- tf/environments/prod/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 4a09269b..34bfb1ab 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1035,7 +1035,7 @@ module "reuploader_builder" { service_name = "reuploader" repo = "ooni/backend" - branch_name = "add_fastpath_reuploader" + branch_name = "master" environment = local.environment buildspec_path = "reuploader/buildspec.yml" trigger_path = "reuploader/**" From 759d72d882b3b3d78848a769f682168ca49ab4e6 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 13 Jul 2026 13:14:52 +0200 Subject: [PATCH 073/201] remove group_vars related to tailscale --- ansible/group_vars/prod/vars.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ansible/group_vars/prod/vars.yml b/ansible/group_vars/prod/vars.yml index afec7630..79566b91 100644 --- a/ansible/group_vars/prod/vars.yml +++ b/ansible/group_vars/prod/vars.yml @@ -1,7 +1,3 @@ prometheus_metrics_password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}" -tailscale_authkey: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/tailscale_authkey_devops', profile='oonidevops_user_prod') }}" -tailscale_tags: - - "devops-prod" -tailscale_oauth_ephemeral: false admin_usernames: [ art, mehul ] non_admin_usernames: [ ] From e73e2ddbe4dbbc3bf4bb290876150731b820ee26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 15 Jul 2026 11:27:05 +0200 Subject: [PATCH 074/201] Use notebook.ooni.org domain --- ansible/deploy-notebook.yml | 2 +- ansible/inventory | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/deploy-notebook.yml b/ansible/deploy-notebook.yml index a9a528ee..b8c594f7 100644 --- a/ansible/deploy-notebook.yml +++ b/ansible/deploy-notebook.yml @@ -1,6 +1,6 @@ --- - name: Deploy notebook host - hosts: notebook1.htz-fsn.prod.ooni.nu + hosts: notebook.ooni.org become: true tags: - notebook diff --git a/ansible/inventory b/ansible/inventory index 9a878bf1..0838e742 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -5,7 +5,7 @@ ghs_ams ## Role tags [clickhouse] -notebook1.htz-fsn.prod.ooni.nu +notebook.ooni.org data1.htz-fsn.prod.ooni.nu data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu From 3c4538411acba54953b70c85e5a2c84e325c7642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 15 Jul 2026 11:40:49 +0200 Subject: [PATCH 075/201] Replace old .nu domain with .org domain for notebook --- ansible/deploy-clickhouse.yml | 2 +- ansible/group_vars/clickhouse/vars.yml | 10 +++++----- ...otebook1.htz-fsn.prod.ooni.nu => notebook.ooni.org} | 0 ansible/inventory | 2 +- ansible/roles/prometheus/templates/prometheus.yml | 4 ++-- docs/Runbooks.md | 10 +++++----- 6 files changed, 14 insertions(+), 14 deletions(-) rename ansible/host_vars/{notebook1.htz-fsn.prod.ooni.nu => notebook.ooni.org} (100%) diff --git a/ansible/deploy-clickhouse.yml b/ansible/deploy-clickhouse.yml index 50261e34..f082fa01 100644 --- a/ansible/deploy-clickhouse.yml +++ b/ansible/deploy-clickhouse.yml @@ -1,7 +1,7 @@ --- - name: Deploy oonidata clickhouse hosts hosts: - - notebook1.htz-fsn.prod.ooni.nu + - notebook.ooni.org - data1.htz-fsn.prod.ooni.nu - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 20ccf46c..ae552161 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -5,7 +5,7 @@ nftables_clickhouse_allow: ip: 23.88.74.249 - fqdn: data3.htz-fsn.prod.ooni.nu ip: 168.119.7.188 - - fqdn: notebook1.htz-fsn.prod.ooni.nu + - fqdn: notebook.ooni.org ip: 138.201.19.39 - fqdn: backend-hel.ooni.org ip: 65.108.192.151 @@ -31,7 +31,7 @@ nftables_zookeeper_allow: ip: 23.88.74.249 - fqdn: data3.htz-fsn.prod.ooni.nu ip: 168.119.7.188 - - fqdn: notebook1.htz-fsn.prod.ooni.nu + - fqdn: notebook.ooni.org ip: 138.201.19.39 clickhouse_version: 24.8.6.70 @@ -109,9 +109,9 @@ clickhouse_keeper: port: 9234 - keeper_server: - server: notebook1.htz-fsn.prod.ooni.nu + server: notebook.ooni.org id: 4 - hostname: notebook1.htz-fsn.prod.ooni.nu + hostname: notebook.ooni.org port: 9234 clickhouse_zookeeper: @@ -125,7 +125,7 @@ clickhouse_zookeeper: host: clickhouse3.prod.ooni.io port: 9181 - node: - host: notebook1.htz-fsn.prod.ooni.nu + host: notebook.ooni.org port: 9181 clickhouse_remote_servers: diff --git a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu b/ansible/host_vars/notebook.ooni.org similarity index 100% rename from ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu rename to ansible/host_vars/notebook.ooni.org diff --git a/ansible/inventory b/ansible/inventory index 0838e742..8b2a9211 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -17,7 +17,7 @@ data1.htz-fsn.prod.ooni.nu [htz_fsn] monitoring.ooni.org -notebook1.htz-fsn.prod.ooni.nu +notebook.ooni.org data1.htz-fsn.prod.ooni.nu data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index ed675d0d..2790f27e 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -83,7 +83,7 @@ scrape_configs: - https://data1.htz-fsn.prod.ooni.nu/metrics/node_exporter - https://data2.htz-fsn.prod.ooni.nu/metrics/node_exporter - https://data3.htz-fsn.prod.ooni.nu/metrics/node_exporter - - https://notebook1.htz-fsn.prod.ooni.nu/metrics/node_exporter + - https://notebook.ooni.org/metrics/node_exporter - http://0.do.th.prod.ooni.io:9001/metrics - http://1.do.th.prod.ooni.io:9001/metrics - http://2.do.th.prod.ooni.io:9001/metrics @@ -154,7 +154,7 @@ scrape_configs: - data1.htz-fsn.prod.ooni.nu - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu - - notebook1.htz-fsn.prod.ooni.nu + - notebook.ooni.org - job_name: 'raw-netdata' scrape_interval: 5s diff --git a/docs/Runbooks.md b/docs/Runbooks.md index 766c53e0..c1b14ffa 100644 --- a/docs/Runbooks.md +++ b/docs/Runbooks.md @@ -1191,12 +1191,12 @@ In order to add new users to the jupyterlab notebook server hosted `notebook.ooni.org` the steps are: 1. Open a PR adding an entry to this list: - https://github.com/ooni/devops/blob/main/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu (we need ssh key, email and username) + https://github.com/ooni/devops/blob/main/ansible/host_vars/notebook.ooni.org (we need ssh key, email and username) 2. An OONI core team member will run: ``` -./play deploy-bootstrap.yml -l notebook1.htz-fsn.prod.ooni.nu --diff -i inventory +./play deploy-bootstrap.yml -l notebook.ooni.org --diff -i inventory ``` -3. An OONI core team member logs into `notebook1.htz-fsn.prod.ooni.nu` and runs: +3. An OONI core team member logs into `notebook.ooni.org` and runs: ``` sudo passwd NEWUSERNAME ``` @@ -1243,11 +1243,11 @@ The notebooks server uses the system’s actual user accounts to authenticate ac ### Procedure 0. Ask the person requesting a new account for the following details: **name**, **username** (used to log in to the server), and **SSH key**. -1. Create a new user entry in the [notebook server’s host vars](https://github.com/ooni/devops/blob/main/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu). Remember to also add the username to the `non_admin_usernames` variable +1. Create a new user entry in the [notebook server’s host vars](https://github.com/ooni/devops/blob/main/ansible/host_vars/notebook.ooni.org). Remember to also add the username to the `non_admin_usernames` variable 2. In the `ansible` directory, run: ``` - ./play -i inventory deploy-bootstrap.yml -l notebook1.htz-fsn.prod.ooni.nu --diff + ./play -i inventory deploy-bootstrap.yml -l notebook.ooni.org --diff ``` 3. Log in to `notebook.ooni.org` via SSH. 4. Set the default password for the new user using: From bb61817e4b3538640370c58304da6fae4dfef9db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 15 Jul 2026 13:14:01 +0200 Subject: [PATCH 076/201] Remove breaking algorithm from ssh steps This was breaking the notebook server, probably due to and old implementation of ssh. This was the specific log line: ``` notebook1.htz-fsn.prod.ooni.nu sshd[3316254]: Unsupported KEX algorithm "mlkem768x25519-sha256" ``` --- ansible/roles/ssh_users/templates/sshd_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/ssh_users/templates/sshd_config b/ansible/roles/ssh_users/templates/sshd_config index 6f7b1273..a2488721 100644 --- a/ansible/roles/ssh_users/templates/sshd_config +++ b/ansible/roles/ssh_users/templates/sshd_config @@ -8,5 +8,5 @@ PrintMotd no AcceptEnv LANG LC_* Subsystem sftp /usr/lib/openssh/sftp-server Ciphers chacha20-poly1305@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr -KexAlgorithms mlkem768x25519-sha256,curve25519-sha256 +KexAlgorithms curve25519-sha256 MACs hmac-sha2-512,hmac-sha2-256 From 812a3a4c2eac269ccab56cbe55207e9804cdc69e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 15 Jul 2026 13:17:23 +0200 Subject: [PATCH 077/201] remove mlkem768x25519-sha256 from KexAlgorithms some of our servers are too old to support this algorithm and sshd does not handle this gracefully. --- ansible/roles/ssh_users/templates/sshd_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/ssh_users/templates/sshd_config b/ansible/roles/ssh_users/templates/sshd_config index 6f7b1273..a2488721 100644 --- a/ansible/roles/ssh_users/templates/sshd_config +++ b/ansible/roles/ssh_users/templates/sshd_config @@ -8,5 +8,5 @@ PrintMotd no AcceptEnv LANG LC_* Subsystem sftp /usr/lib/openssh/sftp-server Ciphers chacha20-poly1305@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr -KexAlgorithms mlkem768x25519-sha256,curve25519-sha256 +KexAlgorithms curve25519-sha256 MACs hmac-sha2-512,hmac-sha2-256 From 390587a5b11be7a8b9f22ffbac178f0f562ba813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 15 Jul 2026 15:59:33 +0200 Subject: [PATCH 078/201] Remove redundant hostname for notebook server --- ansible/deploy-notebook.yml | 1 - ansible/roles/notebook/templates/nginx-jupyterhub.j2 | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/deploy-notebook.yml b/ansible/deploy-notebook.yml index b8c594f7..318f62e4 100644 --- a/ansible/deploy-notebook.yml +++ b/ansible/deploy-notebook.yml @@ -8,6 +8,5 @@ notebook_domain: "notebook.ooni.org" ssl_domains: - "{{ inventory_hostname }}" - - "notebook.ooni.org" roles: - notebook diff --git a/ansible/roles/notebook/templates/nginx-jupyterhub.j2 b/ansible/roles/notebook/templates/nginx-jupyterhub.j2 index 79c954a4..92eb9eba 100644 --- a/ansible/roles/notebook/templates/nginx-jupyterhub.j2 +++ b/ansible/roles/notebook/templates/nginx-jupyterhub.j2 @@ -5,6 +5,7 @@ map $http_upgrade $connection_upgrade { '' close; } +# Comment this block when running dehydrated for the first time server { listen 80; server_name {{ notebook_domain }}; From ddfef8707e1bd9456c8e184c8e6b68eb7e71ec50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 12:11:46 +0200 Subject: [PATCH 079/201] Use old server name for notebooks --- ansible/group_vars/clickhouse/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index ae552161..ca7c434e 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -109,7 +109,7 @@ clickhouse_keeper: port: 9234 - keeper_server: - server: notebook.ooni.org + server: notebook1.htz-fsn.prod.ooni.nu id: 4 hostname: notebook.ooni.org port: 9234 From 4dee89e20fbfcf30aaa0bb750f7a02f88be6417e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 12:36:11 +0200 Subject: [PATCH 080/201] Roll back hostname for notebook server --- ansible/group_vars/clickhouse/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index ca7c434e..090e6032 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -111,7 +111,7 @@ clickhouse_keeper: - keeper_server: server: notebook1.htz-fsn.prod.ooni.nu id: 4 - hostname: notebook.ooni.org + hostname: notebook1.htz-fsn.prod.ooni.nu port: 9234 clickhouse_zookeeper: From 03a5090d617618be8dfbc2d7ced90164aa6b3f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 12:20:00 +0200 Subject: [PATCH 081/201] Change service count for ooniprobe We see a significant slowdown in measurement submission, we suspect it might be related to the anonymous credentials verification being slow. We will add more instances to see if horizontal scaling helps --- tf/environments/prod/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index deb6869c..c23a7378 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -922,7 +922,7 @@ module "ooniapi_ooniprobe" { ] use_autoscaling = false - service_desired_count = 2 + service_desired_count = 4 # max_desired_count = 8 # autoscale_policies = [ # { From ee69d6d280435709ec81bc64d788e722705332e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 13:25:15 +0200 Subject: [PATCH 082/201] Fix broken ooniprobe clickhouse url The dedicated ooniprobe db user has a broken password. The password itself contains a '/' charcter that breaks the parsing of the clickhouse url. This was breaking the deployment of new tasks --- tf/environments/prod/main.tf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index c23a7378..3d2c1dc6 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -239,6 +239,10 @@ data "aws_ssm_parameter" "oonipg_url" { name = "/oonidevops/secrets/ooni-tier0-postgres/postgresql_write_url" } +data "aws_ssm_parameter" "clickhouse_write_url" { + name = "/oonidevops/secrets/clickhouse_write_url" +} + data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" { name = "/oonidevops/secrets/clickhouse_oonimeasurements_url" } @@ -900,7 +904,7 @@ module "ooniapi_ooniprobe" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.arn ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn } From 7952ba3af493ed9f5a6bee9f3f0d00f30f3af991 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 22 Jul 2026 13:53:23 +0200 Subject: [PATCH 083/201] Delete deb-ci.ooni.org dns record --- tf/environments/prod/dns_records.tf | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tf/environments/prod/dns_records.tf b/tf/environments/prod/dns_records.tf index 1bbc99a5..061686de 100644 --- a/tf/environments/prod/dns_records.tf +++ b/tf/environments/prod/dns_records.tf @@ -38,14 +38,6 @@ resource "aws_route53_record" "deb-ooni-org-_CNAME_" { zone_id = local.dns_root_zone_ooni_org } -resource "aws_route53_record" "deb-ci-ooni-org-_A_" { - name = "deb-ci.ooni.org" - records = ["188.166.93.143"] - ttl = "1799" - type = "A" - zone_id = local.dns_root_zone_ooni_org -} - resource "aws_route53_record" "docs-ooni-org-_CNAME_" { name = "docs.ooni.org" records = ["cname.vercel-dns.com"] From 4fa4ab380ba5bbbaf38f71f1e0b3d6b43bb5cc82 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Jul 2026 14:03:23 +0200 Subject: [PATCH 084/201] add grants to oonitest db, #462 --- ansible/group_vars/clickhouse/vars.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 090e6032..5920e7b5 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -246,7 +246,7 @@ clickhouse_custom_users: profile: - write quota: "fastpath" - databases: [ooni] + databases: [ooni, oonitest] - user: name: oonimeasurements @@ -281,7 +281,7 @@ clickhouse_custom_users: profile: - write quota: ooniprobe - databases: [ooni] + databases: [ooni, oonitest] - user: name: oonirun @@ -292,7 +292,7 @@ clickhouse_custom_users: profile: - write quota: oonirun - databases: [ooni] + databases: [ooni, oonitest] - user: name: oonitestlists @@ -303,7 +303,7 @@ clickhouse_custom_users: profile: - write quota: oonitestlists - databases: [ooni] + databases: [ooni, oonitest] - user: name: notebook @@ -369,25 +369,25 @@ clickhouse_custom_quotas: clickhouse_role_manage_grants: True clickhouse_custom_grants: - on: - databases: [ooni] + databases: [ooni, oonitest] tables: ["*"] privileges: [SELECT] to: [ooniprobe, oonimeasurements, oonirun, fastpath, notebook] - on: - databases: [ooni] + databases: [ooni, oonitest] tables: [url_priorities] privileges: [INSERT] to: [oonitestlists] - on: - databases: [ooni] + databases: [ooni, oonitest] tables: [faulty_measurements] privileges: [INSERT] to: [ooniprobe] - on: - databases: [ooni] + databases: [ooni, oonitest] tables: [fastpath, obs_web, obs_openvpn, jsonl, new_jsonl] privileges: [INSERT] to: [fastpath] From 6c49d0888074f79764f642308d2f519491b76136 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Jul 2026 14:43:54 +0200 Subject: [PATCH 085/201] remove notebook as keeper, re-add clickhouse2 as keeper applying the last commit nuked notebook.ooni.org's clickhouse_keeper section of config. I think this might have to do with the rename of the node. --- ansible/group_vars/clickhouse/vars.yml | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 5920e7b5..83d700b5 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -96,11 +96,11 @@ clickhouse_keeper: hostname: clickhouse1.prod.ooni.io port: 9234 - #- keeper_server: - # server: data2.htz-fsn.prod.ooni.nu - # id: 2 - # hostname: clickhouse2.prod.ooni.io - # port: 9234 + - keeper_server: + server: data2.htz-fsn.prod.ooni.nu + id: 2 + hostname: clickhouse2.prod.ooni.io + port: 9234 - keeper_server: server: data3.htz-fsn.prod.ooni.nu @@ -108,24 +108,15 @@ clickhouse_keeper: hostname: clickhouse3.prod.ooni.io port: 9234 - - keeper_server: - server: notebook1.htz-fsn.prod.ooni.nu - id: 4 - hostname: notebook1.htz-fsn.prod.ooni.nu - port: 9234 - clickhouse_zookeeper: - node: host: clickhouse1.prod.ooni.io port: 9181 -# - node: -# host: clickhouse2.prod.ooni.io -# port: 9181 - node: - host: clickhouse3.prod.ooni.io + host: clickhouse2.prod.ooni.io port: 9181 - node: - host: notebook.ooni.org + host: clickhouse3.prod.ooni.io port: 9181 clickhouse_remote_servers: From ae4e729aa681e5b5bffd9a6faffe78cbeecced36 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Jul 2026 09:37:19 +0200 Subject: [PATCH 086/201] prometheus_node_exporter: don't override host_vars ssl_domains only set ssl_domains to inventory_hostname if ssl_domains is not defined --- ansible/roles/prometheus_node_exporter/tasks/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index 21726b00..ff9573ff 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -11,8 +11,7 @@ - oonidata - dehydrated vars: - ssl_domains: - - "{{ inventory_hostname }}" + ssl_domains: "{{ ssl_domains | default([inventory_hostname]) }}" when: use_https - name: create ooni configuration directory From b6e7a361ac5de785ba25a2637ef703afa0cf7176 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Jul 2026 11:19:17 +0200 Subject: [PATCH 087/201] add per-role tags to roles in deploy-clickhouse playbook this will allow deploying each role by tag, so that the clickhouse role does not need to be run in order to update prometheus_node_exporter --- ansible/deploy-clickhouse.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/deploy-clickhouse.yml b/ansible/deploy-clickhouse.yml index f082fa01..630d1736 100644 --- a/ansible/deploy-clickhouse.yml +++ b/ansible/deploy-clickhouse.yml @@ -6,8 +6,8 @@ - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu become: true - tags: - - clickhouse roles: - - prometheus_node_exporter - - oonidata_clickhouse + - role: prometheus_node_exporter + tags: [node_exporter] + - role: oonidata_clickhouse + tags: [clickhouse] From 9efb956532c1689799fec0eaa969cbf0595e75fd Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Jul 2026 11:35:16 +0200 Subject: [PATCH 088/201] do not self-reference ssl_domains in expression --- ansible/roles/prometheus_node_exporter/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index ff9573ff..fb9e0e6f 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -11,7 +11,7 @@ - oonidata - dehydrated vars: - ssl_domains: "{{ ssl_domains | default([inventory_hostname]) }}" + ssl_domains: "{{ hostvars[inventory_hostname].ssl_domains | default([inventory_hostname]) }}" when: use_https - name: create ooni configuration directory From fdd7c172111579a77a78bfa0c610afcf7d97b61f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Jul 2026 11:51:37 +0200 Subject: [PATCH 089/201] override clickhouse_role_manage_grants on notebook --- ansible/host_vars/notebook.ooni.org | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/host_vars/notebook.ooni.org b/ansible/host_vars/notebook.ooni.org index 1090f8cf..2b68cbfc 100644 --- a/ansible/host_vars/notebook.ooni.org +++ b/ansible/host_vars/notebook.ooni.org @@ -145,6 +145,7 @@ clickhouse_listen_hosts: # this only applies to SQL managed users clickhouse_role_manage_users: false +clickhouse_role_manage_grants: false # these are handled via config files clickhouse_default_users: - user: From f3fb7b4ff10751f92af033cb102e9c68af5c555a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 31 Jul 2026 16:05:18 +0200 Subject: [PATCH 090/201] Point oonimeasurements in dev to prod db --- tf/environments/dev/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 85474480..17641442 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -563,7 +563,7 @@ module "ooniapi_ooniprobe_deployer" { service_name = "ooniprobe" repo = "ooni/backend" - branch_name = "master" + branch_name = "add_ooniprobe_private_api" environment = local.environment trigger_path = "ooniapi/services/ooniprobe/**" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" @@ -1148,7 +1148,7 @@ module "ooniapi_oonimeasurements" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_url.arn ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.arn } From ebfce3776046440497baa9e0502fbd297b629d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 31 Jul 2026 16:29:57 +0200 Subject: [PATCH 091/201] Remove anonc ec2 instance --- tf/environments/dev/main.tf | 71 ------------------------------------- 1 file changed, 71 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 17641442..6db16def 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1391,77 +1391,6 @@ module "ooni_monitoring" { tags = local.tags } -### Anonymous credentials testing instance -module "ooni_anonc" { - source = "../../modules/ec2" - - stage = local.environment - - vpc_id = module.network.vpc_id - subnet_id = module.network.vpc_subnet_public[0].id - private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block - dns_zone_ooni_io = local.dns_zone_ooni_io - - key_name = module.adm_iam_roles.oonidevops_key_name - instance_type = "t3a.small" - - name = "anonc" - ingress_rules = [{ - from_port = 22, - to_port = 22, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 80, # for dehydrated challenge - to_port = 80, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 443, # for the POC hosting - to_port = 443, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 9100, # for node exporter metrics - to_port = 9100, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"], - }] - - egress_rules = [{ - from_port = 0, - to_port = 0, - protocol = "-1", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 0, - to_port = 0, - protocol = "-1", - ipv6_cidr_blocks = ["::/0"], - }] - - sg_prefix = "oonianonc" - tg_prefix = "anon" - - disk_size = 20 - - tags = merge( - local.tags, - { Name = "ooni-tier0-anonc" } - ) -} - -resource "aws_route53_record" "anonc_alias" { - zone_id = local.dns_zone_ooni_io - name = "anonc.${local.environment}.ooni.io" - type = "CNAME" - ttl = 300 - - records = [ - module.ooni_anonc.aws_instance_public_dns - ] -} - # Jump host for accessing postgres module "ooni_jumphost" { source = "../../modules/ec2" From 61fdd14add370d9becc7c8842ce0fad7ccfb49d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 31 Jul 2026 16:32:07 +0200 Subject: [PATCH 092/201] Point dev env to labeling branch of oonimeasurements --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 6db16def..9c11113b 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1117,7 +1117,7 @@ module "ooniapi_oonimeasurements_deployer" { service_name = "oonimeasurements" repo = "ooni/backend" - branch_name = "master" + branch_name = "labeling" environment = local.environment trigger_path = "ooniapi/services/oonimeasurements/**" buildspec_path = "ooniapi/services/oonimeasurements/buildspec.yml" From ce371e49b0c10d04f1e75ae8c66a5d323d5dc188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 3 Aug 2026 16:35:44 +0200 Subject: [PATCH 093/201] Create new domain name for airflow --- tf/environments/prod/main.tf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 9a26a919..13240002 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1752,3 +1752,15 @@ resource "aws_route53_record" "jumphost_alias" { module.ooni_jumphost.aws_instance_public_dns ] } + +resource "aws_route53_record" "detector_panel_alias" { + zone_id = local.dns_zone_ooni_io + name = "detector-panel.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + # Airflow host + "142.132.254.225" + ] +} From 34d6b74fbf2ee7a45e4f2e322e47a68fe8e49ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 3 Aug 2026 16:43:49 +0200 Subject: [PATCH 094/201] Add nginx rules to direct traffic to panel --- ansible/deploy-airflow.yml | 2 ++ .../templates/nginx-airflow.j2 | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/ansible/deploy-airflow.yml b/ansible/deploy-airflow.yml index b070fce2..57a81498 100644 --- a/ansible/deploy-airflow.yml +++ b/ansible/deploy-airflow.yml @@ -11,6 +11,8 @@ vars: airflow_public_fqdn: "airflow.prod.ooni.io" tls_cert_dir: /var/lib/dehydrated/certs + event_detector_panel_domain: detector-panel.prod.ooni.io ssl_domains: - "data1.htz-fsn.prod.ooni.nu" - "airflow.prod.ooni.io" + - "detector-panel.prod.ooni.io" diff --git a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 index 14b8dae6..c41ec401 100644 --- a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 +++ b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 @@ -50,3 +50,28 @@ server { proxy_buffering off; } } + +server { + listen 443 ssl http2; + include /etc/nginx/ssl_intermediate.conf; + ssl_certificate {{ tls_cert_dir }}/{{ event_detector_panel_domain }}/fullchain.pem; + ssl_certificate_key {{ tls_cert_dir }}/{{ event_detector_panel_domain }}/privkey.pem; + ssl_trusted_certificate {{ tls_cert_dir }}/{{ event_detector_panel_domain }}/chain.pem; + server_name {{ event_detector_panel_domain }}; + access_log /var/log/nginx/{{ event_detector_panel_domain }}.access.log; + error_log /var/log/nginx/{{ event_detector_panel_domain }}.log warn; + + location / { + proxy_pass http://127.0.0.1:8501; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + client_max_body_size 100M; + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header X-Scheme $scheme; + proxy_buffering off; + } +} From 60dd0190540967d1ff92c0fc3ca3a65c37c9daa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 3 Aug 2026 16:55:12 +0200 Subject: [PATCH 095/201] Add basic http auth --- ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 index c41ec401..231f2dec 100644 --- a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 +++ b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 @@ -62,6 +62,8 @@ server { error_log /var/log/nginx/{{ event_detector_panel_domain }}.log warn; location / { + auth_basic "Administrator's Area"; + auth_basic_user_file /etc/ooni/detector_panel_passwd; proxy_pass http://127.0.0.1:8501; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; From 73416aef33b89bb9ca3080a7f71c3cc245540bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 3 Aug 2026 17:04:26 +0200 Subject: [PATCH 096/201] Add password file for detector panel --- ansible/deploy-airflow.yml | 1 + ansible/roles/oonidata_airflow/tasks/main.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/ansible/deploy-airflow.yml b/ansible/deploy-airflow.yml index 57a81498..6db66069 100644 --- a/ansible/deploy-airflow.yml +++ b/ansible/deploy-airflow.yml @@ -16,3 +16,4 @@ - "data1.htz-fsn.prod.ooni.nu" - "airflow.prod.ooni.io" - "detector-panel.prod.ooni.io" + detector_panel_passwd: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/detector_panel_passwd', profile='oonidevops_user_prod') }}" diff --git a/ansible/roles/oonidata_airflow/tasks/main.yml b/ansible/roles/oonidata_airflow/tasks/main.yml index a466a7c3..0a02751f 100644 --- a/ansible/roles/oonidata_airflow/tasks/main.yml +++ b/ansible/roles/oonidata_airflow/tasks/main.yml @@ -97,3 +97,15 @@ tags: - oonidata - config + +# Event detector panel +- name: Setup event detector password file and ensure permissions are set + community.general.htpasswd: + path: "/etc/ooni/detector_panel_passwd" + name: prom + password: "{{ detector_panel_passwd }}" + owner: root + group: "nginx" + mode: 0640 + tags: + - detector_panel From c098480660c14477f47505cbad5bc6b2b4a2dbc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 4 Aug 2026 10:54:09 +0200 Subject: [PATCH 097/201] Add detector panel service unit --- ansible/roles/oonidata_airflow/handlers/main.yml | 10 ++++++++++ ansible/roles/oonidata_airflow/tasks/main.yml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/ansible/roles/oonidata_airflow/handlers/main.yml b/ansible/roles/oonidata_airflow/handlers/main.yml index f6dda47d..e7434941 100644 --- a/ansible/roles/oonidata_airflow/handlers/main.yml +++ b/ansible/roles/oonidata_airflow/handlers/main.yml @@ -2,3 +2,13 @@ ansible.builtin.systemd_service: name: nginx state: reloaded + +- name: reload systemd + ansible.builtin.systemd: + daemon_reload: true + +- name: restart detector-panel + ansible.builtin.systemd: + name: detector-panel.service + state: restarted + enabled: true diff --git a/ansible/roles/oonidata_airflow/tasks/main.yml b/ansible/roles/oonidata_airflow/tasks/main.yml index 0a02751f..809dc1b0 100644 --- a/ansible/roles/oonidata_airflow/tasks/main.yml +++ b/ansible/roles/oonidata_airflow/tasks/main.yml @@ -99,6 +99,16 @@ - config # Event detector panel +- name: Deploy detector-panel systemd unit + ansible.builtin.template: + src: templates/detector-panel.service + dest: /etc/systemd/system/detector-panel.service + owner: root + mode: '0644' + notify: + - reload systemd + - restart detector-panel + - name: Setup event detector password file and ensure permissions are set community.general.htpasswd: path: "/etc/ooni/detector_panel_passwd" From bfc3625d65cc41f03b2c28291ecd8bf7d8a7aeec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 4 Aug 2026 11:04:24 +0200 Subject: [PATCH 098/201] Install oonipipeline with the analysis profile --- ansible/roles/oonidata_airflow/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/oonidata_airflow/tasks/main.yml b/ansible/roles/oonidata_airflow/tasks/main.yml index 809dc1b0..b8fc5632 100644 --- a/ansible/roles/oonidata_airflow/tasks/main.yml +++ b/ansible/roles/oonidata_airflow/tasks/main.yml @@ -46,7 +46,7 @@ - name: "Install pipeline with pip" ansible.builtin.shell: - cmd: "{{ miniconda_install_dir }}/bin/pip install --upgrade /opt/airflow/oonidata/oonipipeline/" + cmd: "{{ miniconda_install_dir }}/bin/pip install --upgrade '/opt/airflow/oonidata/oonipipeline/[analysis]'" chdir: "{{ miniconda_install_dir }}" become_user: miniconda tags: @@ -108,6 +108,8 @@ notify: - reload systemd - restart detector-panel + tags: + - detector_panel - name: Setup event detector password file and ensure permissions are set community.general.htpasswd: From 5bcc21c71d0cbc3e6163b564d1e8cce177e6c571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 4 Aug 2026 12:37:06 +0200 Subject: [PATCH 099/201] Add event-detector panel systemd unit --- .../templates/detector-panel.service | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ansible/roles/oonidata_airflow/templates/detector-panel.service diff --git a/ansible/roles/oonidata_airflow/templates/detector-panel.service b/ansible/roles/oonidata_airflow/templates/detector-panel.service new file mode 100644 index 00000000..18480675 --- /dev/null +++ b/ansible/roles/oonidata_airflow/templates/detector-panel.service @@ -0,0 +1,14 @@ +[Unit] +Description=OONI Pipeline Events Panel +After=network.target + +[Service] +Type=simple +ExecStart=/opt/miniconda/bin/python -m oonipipeline.main events-panel --port 8501 +Restart=on-failure +RestartSec=5 +User=oonipipeline +WorkingDirectory=/opt/miniconda + +[Install] +WantedBy=multi-user.target From f654db19b0092f596a5bbc2d288112f8a7aab279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 4 Aug 2026 13:11:54 +0200 Subject: [PATCH 100/201] Fix wrong register type for detector domain --- tf/environments/prod/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 13240002..3aa8c3c5 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1756,7 +1756,7 @@ resource "aws_route53_record" "jumphost_alias" { resource "aws_route53_record" "detector_panel_alias" { zone_id = local.dns_zone_ooni_io name = "detector-panel.${local.environment}.ooni.io" - type = "CNAME" + type = "A" ttl = 300 records = [ From 49730e06985f5157d15b3f6e6979817e4f437a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 4 Aug 2026 14:13:47 +0200 Subject: [PATCH 101/201] Use inventory hostname for the certificate file --- ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 index 231f2dec..aad1855a 100644 --- a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 +++ b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 @@ -54,9 +54,9 @@ server { server { listen 443 ssl http2; include /etc/nginx/ssl_intermediate.conf; - ssl_certificate {{ tls_cert_dir }}/{{ event_detector_panel_domain }}/fullchain.pem; - ssl_certificate_key {{ tls_cert_dir }}/{{ event_detector_panel_domain }}/privkey.pem; - ssl_trusted_certificate {{ tls_cert_dir }}/{{ event_detector_panel_domain }}/chain.pem; + ssl_certificate /var/lib/dehydrated/certs/{{inventory_hostname}}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{inventory_hostname}}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{inventory_hostname}}/chain.pem; server_name {{ event_detector_panel_domain }}; access_log /var/log/nginx/{{ event_detector_panel_domain }}.access.log; error_log /var/log/nginx/{{ event_detector_panel_domain }}.log warn; From f005f023b6b8eaa1f9f5f2a8e221a3d1a5ade60e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 4 Aug 2026 14:22:55 +0200 Subject: [PATCH 102/201] Set username to admin --- ansible/roles/oonidata_airflow/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/oonidata_airflow/tasks/main.yml b/ansible/roles/oonidata_airflow/tasks/main.yml index b8fc5632..d83cf64b 100644 --- a/ansible/roles/oonidata_airflow/tasks/main.yml +++ b/ansible/roles/oonidata_airflow/tasks/main.yml @@ -114,7 +114,7 @@ - name: Setup event detector password file and ensure permissions are set community.general.htpasswd: path: "/etc/ooni/detector_panel_passwd" - name: prom + name: admin password: "{{ detector_panel_passwd }}" owner: root group: "nginx" From d3065fece7a45f05860a6ab0a0d3e9481d66f0af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 4 Aug 2026 14:24:22 +0200 Subject: [PATCH 103/201] Notify nginx on detector panel changes --- ansible/roles/oonidata_airflow/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/oonidata_airflow/tasks/main.yml b/ansible/roles/oonidata_airflow/tasks/main.yml index d83cf64b..09d8eeed 100644 --- a/ansible/roles/oonidata_airflow/tasks/main.yml +++ b/ansible/roles/oonidata_airflow/tasks/main.yml @@ -119,5 +119,7 @@ owner: root group: "nginx" mode: 0640 + notify: + - reload nginx tags: - detector_panel From 668ffce39f3410842819de038250d2e15965fa40 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 5 Aug 2026 11:12:28 +0200 Subject: [PATCH 104/201] dehydrated: place each domain on a separate line this will create a certificate for each domain instead of a single certificate with altnames --- ansible/roles/dehydrated/templates/domains.txt.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/dehydrated/templates/domains.txt.j2 b/ansible/roles/dehydrated/templates/domains.txt.j2 index 5850d203..4b4a6c0e 100644 --- a/ansible/roles/dehydrated/templates/domains.txt.j2 +++ b/ansible/roles/dehydrated/templates/domains.txt.j2 @@ -1 +1,3 @@ -{% for d in ssl_domains %}{{ d }} {% endfor %} +{% for d in ssl_domains %} +{{ d }} +{% endfor %} From 48bc0f6aac1dbd47206ef22e895e5598b40364be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 6 Aug 2026 10:30:21 +0200 Subject: [PATCH 105/201] Use the right certificate for detector panel --- ansible/deploy-airflow.yml | 2 +- ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/deploy-airflow.yml b/ansible/deploy-airflow.yml index 6db66069..c6468a95 100644 --- a/ansible/deploy-airflow.yml +++ b/ansible/deploy-airflow.yml @@ -15,5 +15,5 @@ ssl_domains: - "data1.htz-fsn.prod.ooni.nu" - "airflow.prod.ooni.io" - - "detector-panel.prod.ooni.io" + - "{{event_detector_panel_domain}}" detector_panel_passwd: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/detector_panel_passwd', profile='oonidevops_user_prod') }}" diff --git a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 index aad1855a..6d201f36 100644 --- a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 +++ b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 @@ -54,9 +54,9 @@ server { server { listen 443 ssl http2; include /etc/nginx/ssl_intermediate.conf; - ssl_certificate /var/lib/dehydrated/certs/{{inventory_hostname}}/fullchain.pem; - ssl_certificate_key /var/lib/dehydrated/certs/{{inventory_hostname}}/privkey.pem; - ssl_trusted_certificate /var/lib/dehydrated/certs/{{inventory_hostname}}/chain.pem; + ssl_certificate /var/lib/dehydrated/certs/{{event_detector_panel_domain}}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{event_detector_panel_domain}}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{event_detector_panel_domain}}/chain.pem; server_name {{ event_detector_panel_domain }}; access_log /var/log/nginx/{{ event_detector_panel_domain }}.access.log; error_log /var/log/nginx/{{ event_detector_panel_domain }}.log warn; From 335152ad70c5d6031127116f4ace8f65ee4b5c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 11 Aug 2026 15:56:03 +0200 Subject: [PATCH 106/201] Add test helpers machines; build th from scratch since we don't have binaries --- ansible/inventory | 2 + ansible/roles/test_helpers/defaults/main.yml | 11 +- ansible/roles/test_helpers/tasks/main.yml | 83 ++++++----- tf/environments/prod/main.tf | 136 +++++++++++++++++++ 4 files changed, 194 insertions(+), 38 deletions(-) diff --git a/ansible/inventory b/ansible/inventory index 8fec5c71..5008d568 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -50,6 +50,8 @@ fastpath.dev.ooni.io fastpath.prod.ooni.io json.th.dev.ooni.io echo.th.dev.ooni.io +json.th.prod.ooni.io +echo.th.prod.ooni.io fastpath2.prod.ooni.io reuploaderfastpath.prod.ooni.io anonc.dev.ooni.io diff --git a/ansible/roles/test_helpers/defaults/main.yml b/ansible/roles/test_helpers/defaults/main.yml index a431bc19..f85d5bac 100644 --- a/ansible/roles/test_helpers/defaults/main.yml +++ b/ansible/roles/test_helpers/defaults/main.yml @@ -1,3 +1,8 @@ -test_helpers_url: https://github.com/ooni/test-helpers/releases/download/0.1.0-1ac1/test-helpers@0.1.0-1ac1.tar.gz -# remember to remove the "sha256:" prefix from github -checksum: 9a7387050412d747df8d0479c004357edfc4cd7825ce7e1c83141e1e0838715c +# TODO(test-helpers-source-build): the test helpers moved into the +# test-helpers crate of https://github.com/ooni/ooniprobe-rs and there is no +# release workflow producing precompiled binaries for it yet. Until that +# workflow exists we build from source below. Once precompiled binaries are +# published again, switch this role back to downloading+checksumming a +# release tarball (see git history of this file for the previous approach). +test_helpers_repo: https://github.com/ooni/ooniprobe-rs.git +test_helpers_version: main diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index f35613cc..335e4e05 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -22,41 +22,60 @@ become: yes # Install test helpers -- name: Donwload binaries for test helpers - ansible.builtin.get_url: - url: "{{test_helpers_url}}" - dest: "/tmp/test-helpers.tar.gz" - mode: '0600' +# +# TODO(test-helpers-source-build): building from source here until +# ooni/ooniprobe-rs has a release workflow that publishes precompiled +# test-helpers binaries. Switch back to downloading+checksumming a release +# tarball once that exists (see git history of this role for the previous +# get_url/checksum-based approach). + +- name: Install build dependencies + ansible.builtin.apt: + name: + - git + - curl + - build-essential + - pkg-config + - libssl-dev + state: present + update_cache: yes become: true -- name: Get checksum of downloaded file +- name: Check if rustup toolchain is already installed ansible.builtin.stat: - path: "/tmp/test-helpers.tar.gz" - checksum_algorithm: sha256 - register: file_stat + path: /usr/local/cargo/bin/cargo + register: cargo_installed -- name: Verify checksum - ansible.builtin.fail: - msg: "Checksum failed! Expected: {{checksum}} but got: {{file_stat.stat.checksum}}" - when: file_stat.stat.checksum != checksum +- name: Install rustup toolchain + ansible.builtin.shell: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal + environment: + RUSTUP_HOME: /usr/local/rustup + CARGO_HOME: /usr/local/cargo + when: not cargo_installed.stat.exists + become: true -- name: Create test helpers temp dir - ansible.builtin.file: - path: "/tmp/test-helpers" - state: directory - mode: "0700" - become: yes +- name: Clone ooniprobe-rs repo + ansible.builtin.git: + repo: "{{ test_helpers_repo }}" + dest: "/tmp/ooniprobe-rs" + version: "{{ test_helpers_version }}" + force: yes + become: true -- name: Extract tar content - ansible.builtin.unarchive: - src: "/tmp/test-helpers.tar.gz" - dest: "/tmp/test-helpers" - remote_src: yes - become: yes +- name: Build the test-helpers crate in release mode + ansible.builtin.command: /usr/local/cargo/bin/cargo build --release --package test-helpers + args: + chdir: "/tmp/ooniprobe-rs" + environment: + RUSTUP_HOME: /usr/local/rustup + CARGO_HOME: /usr/local/cargo + PATH: "/usr/local/cargo/bin:{{ ansible_env.PATH }}" + become: true - name: Make jsonth accessible system wide ansible.builtin.copy: - src: "/tmp/test-helpers/jsonth" + src: "/tmp/ooniprobe-rs/target/release/jsonth" dest: "/usr/local/bin/" mode: '0755' remote_src: yes @@ -64,21 +83,15 @@ - name: Make echo accessible system wide ansible.builtin.copy: - src: "/tmp/test-helpers/echo" + src: "/tmp/ooniprobe-rs/target/release/echo" dest: "/usr/local/bin/" mode: '0755' remote_src: yes become: yes -- name: Clean up temporary files - ansible.builtin.file: - path: "/tmp/test-helpers" - state: absent - become: yes - -- name: Remove downloaded tarball +- name: Clean up build directory ansible.builtin.file: - path: "/tmp/test-helpers.tar.gz" + path: "/tmp/ooniprobe-rs" state: absent become: yes diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 3aa8c3c5..147f99ed 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1033,6 +1033,142 @@ module "fastpath_builder" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket } + +#### Test Helpers Machines + +module "ooni_test_helpers_json" { + source = "../../modules/ec2" + + stage = local.environment + + vpc_id = module.network.vpc_id + subnet_id = module.network.vpc_subnet_public[0].id + private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + dns_zone_ooni_io = local.dns_zone_ooni_io + + key_name = module.adm_iam_roles.oonidevops_key_name + instance_type = "t3.micro" + + name = "oonijsonth" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 80, # jsonth + to_port = 80, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 9100, # Prometheus monitoring + to_port = 9100, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"], + }] + + sg_prefix = "oonijsonth" + tg_prefix = "tshp" + + disk_size = 20 + + tags = merge( + local.tags, + { Name = "ooni-tier0-jsonth" } + ) +} + +# Echo test helper, requires a dedicated machine bc it's a tcp server, +# not an HTTP server. It's impossible to reroute using nginx +module "ooni_test_helpers_echo" { + source = "../../modules/ec2" + + stage = local.environment + + vpc_id = module.network.vpc_id + subnet_id = module.network.vpc_subnet_public[0].id + private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + dns_zone_ooni_io = local.dns_zone_ooni_io + + key_name = module.adm_iam_roles.oonidevops_key_name + instance_type = "t3.micro" + + name = "ooniechoth" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 80, # echo + to_port = 80, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 9100, # Prometheus monitoring + to_port = 9100, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"], + }] + + sg_prefix = "ooniechoth" + tg_prefix = "echo" + + disk_size = 20 + + tags = merge( + local.tags, + { Name = "ooni-tier0-echoth" } + ) +} + +resource "aws_route53_record" "testhelpers_json_alias" { + zone_id = local.dns_zone_ooni_io + name = "json.th.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooni_test_helpers_json.aws_instance_public_dns + ] +} + +resource "aws_route53_record" "testhelpers_echo_alias" { + zone_id = local.dns_zone_ooni_io + name = "echo.th.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooni_test_helpers_echo.aws_instance_public_dns + ] +} + + module "reuploader_builder" { source = "../../modules/ooni_docker_build" trigger_tag = "" From 61c8f79393c328d292db977c4f3fd45238ae9054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 11 Aug 2026 16:56:03 +0200 Subject: [PATCH 107/201] Fix edge case where service won't be properly restarted if a new version is copied --- .../templates/nginx-prometheus.j2 | 2 +- ansible/roles/test_helpers/tasks/main.yml | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 index a0019f86..6fb7c5ca 100644 --- a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 +++ b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 @@ -2,7 +2,7 @@ server { {% if use_https %} - listen {{https_port}} ssl http2; + listen 443 ssl http2; server_name {{ inventory_hostname }}; include /etc/nginx/ssl_intermediate.conf; diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index 335e4e05..217946b6 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -73,6 +73,9 @@ PATH: "/usr/local/cargo/bin:{{ ansible_env.PATH }}" become: true +# Both binaries are built as part of the workspace build above and copied to +# every host regardless of which helper it runs, but only one systemd unit +# ({{helper}}.service) is actually installed per host (see below). - name: Make jsonth accessible system wide ansible.builtin.copy: src: "/tmp/ooniprobe-rs/target/release/jsonth" @@ -80,6 +83,7 @@ mode: '0755' remote_src: yes become: yes + register: jsonth_binary - name: Make echo accessible system wide ansible.builtin.copy: @@ -88,6 +92,7 @@ mode: '0755' remote_src: yes become: yes + register: echo_binary - name: Clean up build directory ansible.builtin.file: @@ -117,3 +122,14 @@ name: "{{helper}}.service" state: started enabled: yes + +# Only the helper actually deployed on this host (the one matching {{helper}}) +# has a unit above; restart it here if its own binary changed, so a source +# rebuild actually takes effect and not just on unit-file changes. +- name: Restart helper if its binary changed + tags: test-helpers + ansible.builtin.systemd_service: + name: "{{ helper }}.service" + state: restarted + become: yes + when: (helper == 'jsonth' and jsonth_binary.changed) or (helper == 'echo' and echo_binary.changed) From 42cafb5418f523c87ab5cd3ece3d591bfaff1753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 12 Aug 2026 09:39:57 +0200 Subject: [PATCH 108/201] point oonimeasurements to master --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 9c11113b..6db16def 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1117,7 +1117,7 @@ module "ooniapi_oonimeasurements_deployer" { service_name = "oonimeasurements" repo = "ooni/backend" - branch_name = "labeling" + branch_name = "master" environment = local.environment trigger_path = "ooniapi/services/oonimeasurements/**" buildspec_path = "ooniapi/services/oonimeasurements/buildspec.yml" From 824a2bf573ba65f0acb935d33656d617ea24e87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 12 Aug 2026 10:44:14 +0200 Subject: [PATCH 109/201] Point oonimeasurements to the temp branch --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index b6978b19..7189f4f0 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1398,7 +1398,7 @@ module "testlists_builder" { service_name = "testlists" repo = "ooni/backend" - branch_name = "master" + branch_name = "fix_1238_testlists_worktree" environment = local.environment buildspec_path = "ooniapi/services/testlists/buildspec.yml" trigger_path = "ooniapi/services/testlists/**" From 7d85944fbd713958709774bf7c53cdc228a2a100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 12 Aug 2026 11:54:39 +0200 Subject: [PATCH 110/201] remove test helpers from dev env; point ooniprobe to dev branch --- tf/environments/dev/main.tf | 138 +----------------------------------- 1 file changed, 1 insertion(+), 137 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 7189f4f0..305df23a 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -563,7 +563,7 @@ module "ooniapi_ooniprobe_deployer" { service_name = "ooniprobe" repo = "ooni/backend" - branch_name = "add_ooniprobe_private_api" + branch_name = "1237-check-in-to-th" environment = local.environment trigger_path = "ooniapi/services/ooniprobe/**" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" @@ -940,142 +940,6 @@ module "fastpath_builder" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket } - -#### Test Helpers Machines - -module "ooni_test_helpers_json" { - source = "../../modules/ec2" - - stage = local.environment - - vpc_id = module.network.vpc_id - subnet_id = module.network.vpc_subnet_public[0].id - private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block - dns_zone_ooni_io = local.dns_zone_ooni_io - - key_name = module.adm_iam_roles.oonidevops_key_name - instance_type = "t3.micro" - - name = "oonijsonth" - ingress_rules = [{ - from_port = 22, - to_port = 22, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 80, # jsonth - to_port = 80, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 9100, # Prometheus monitoring - to_port = 9100, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] - }] - - egress_rules = [{ - from_port = 0, - to_port = 0, - protocol = "-1", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 0, - to_port = 0, - protocol = "-1", - ipv6_cidr_blocks = ["::/0"], - }] - - sg_prefix = "oonijsonth" - tg_prefix = "tshp" - - disk_size = 20 - - tags = merge( - local.tags, - { Name = "ooni-tier0-jsonth" } - ) -} - -# Echo test helper, requires a dedicated machine bc it's a tcp server, -# not an HTTP server. It's impossible to reroute using nginx -module "ooni_test_helpers_echo" { - source = "../../modules/ec2" - - stage = local.environment - - vpc_id = module.network.vpc_id - subnet_id = module.network.vpc_subnet_public[0].id - private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block - dns_zone_ooni_io = local.dns_zone_ooni_io - - key_name = module.adm_iam_roles.oonidevops_key_name - instance_type = "t3.micro" - - name = "ooniechoth" - ingress_rules = [{ - from_port = 22, - to_port = 22, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 80, # echo - to_port = 80, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 9100, # Prometheus monitoring - to_port = 9100, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] - }] - - egress_rules = [{ - from_port = 0, - to_port = 0, - protocol = "-1", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 0, - to_port = 0, - protocol = "-1", - ipv6_cidr_blocks = ["::/0"], - }] - - sg_prefix = "ooniechoth" - tg_prefix = "echo" - - disk_size = 20 - - tags = merge( - local.tags, - { Name = "ooni-tier0-echoth" } - ) -} - -resource "aws_route53_record" "testhelpers_json_alias" { - zone_id = local.dns_zone_ooni_io - name = "json.th.${local.environment}.ooni.io" - type = "CNAME" - ttl = 300 - - records = [ - module.ooni_test_helpers_json.aws_instance_public_dns - ] -} - -resource "aws_route53_record" "testhelpers_echo_alias" { - zone_id = local.dns_zone_ooni_io - name = "echo.th.${local.environment}.ooni.io" - type = "CNAME" - ttl = 300 - - records = [ - module.ooni_test_helpers_echo.aws_instance_public_dns - ] -} - - #### OONI Run service module "ooniapi_oonirun_deployer" { From 41a344cb67209fa56355b55f5c6fad03aacfbfac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 12 Aug 2026 12:55:05 +0200 Subject: [PATCH 111/201] Pull binaries from ooniprobe-rs repo --- ansible/deploy-echo-test-helper.yml | 3 +- ansible/roles/test_helpers/defaults/main.yml | 10 +-- ansible/roles/test_helpers/handlers/main.yml | 4 +- ansible/roles/test_helpers/tasks/main.yml | 86 ++----------------- .../{echo.service => echoth.service} | 2 +- ansible/roles/test_helpers/vars/main.yml | 10 ++- 6 files changed, 22 insertions(+), 93 deletions(-) rename ansible/roles/test_helpers/templates/{echo.service => echoth.service} (89%) diff --git a/ansible/deploy-echo-test-helper.yml b/ansible/deploy-echo-test-helper.yml index fe6434a1..2f3965ae 100644 --- a/ansible/deploy-echo-test-helper.yml +++ b/ansible/deploy-echo-test-helper.yml @@ -17,5 +17,6 @@ http_port: 8080 # if we leave port 80, it's taken by nginx - role: test_helpers vars: - helper: echo + helper: echoth + test_helpers_binary_source: echo port: 80 diff --git a/ansible/roles/test_helpers/defaults/main.yml b/ansible/roles/test_helpers/defaults/main.yml index f85d5bac..ea3d89cd 100644 --- a/ansible/roles/test_helpers/defaults/main.yml +++ b/ansible/roles/test_helpers/defaults/main.yml @@ -1,8 +1,2 @@ -# TODO(test-helpers-source-build): the test helpers moved into the -# test-helpers crate of https://github.com/ooni/ooniprobe-rs and there is no -# release workflow producing precompiled binaries for it yet. Until that -# workflow exists we build from source below. Once precompiled binaries are -# published again, switch this role back to downloading+checksumming a -# release tarball (see git history of this file for the previous approach). -test_helpers_repo: https://github.com/ooni/ooniprobe-rs.git -test_helpers_version: main +test_helpers_version: v0.1.6-beta +test_helpers_base_url: "https://github.com/ooni/ooniprobe-rs/releases/download/{{ test_helpers_version }}" diff --git a/ansible/roles/test_helpers/handlers/main.yml b/ansible/roles/test_helpers/handlers/main.yml index 30593401..a5505f71 100644 --- a/ansible/roles/test_helpers/handlers/main.yml +++ b/ansible/roles/test_helpers/handlers/main.yml @@ -1,7 +1,7 @@ -- name: restart echo +- name: restart echoth tags: test-helpers ansible.builtin.systemd_service: - name: echo + name: echoth state: restarted - name: restart jsonth diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index 217946b6..21d67991 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -22,83 +22,14 @@ become: yes # Install test helpers -# -# TODO(test-helpers-source-build): building from source here until -# ooni/ooniprobe-rs has a release workflow that publishes precompiled -# test-helpers binaries. Switch back to downloading+checksumming a release -# tarball once that exists (see git history of this role for the previous -# get_url/checksum-based approach). - -- name: Install build dependencies - ansible.builtin.apt: - name: - - git - - curl - - build-essential - - pkg-config - - libssl-dev - state: present - update_cache: yes - become: true - -- name: Check if rustup toolchain is already installed - ansible.builtin.stat: - path: /usr/local/cargo/bin/cargo - register: cargo_installed - -- name: Install rustup toolchain - ansible.builtin.shell: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal - environment: - RUSTUP_HOME: /usr/local/rustup - CARGO_HOME: /usr/local/cargo - when: not cargo_installed.stat.exists - become: true - -- name: Clone ooniprobe-rs repo - ansible.builtin.git: - repo: "{{ test_helpers_repo }}" - dest: "/tmp/ooniprobe-rs" - version: "{{ test_helpers_version }}" - force: yes - become: true - -- name: Build the test-helpers crate in release mode - ansible.builtin.command: /usr/local/cargo/bin/cargo build --release --package test-helpers - args: - chdir: "/tmp/ooniprobe-rs" - environment: - RUSTUP_HOME: /usr/local/rustup - CARGO_HOME: /usr/local/cargo - PATH: "/usr/local/cargo/bin:{{ ansible_env.PATH }}" - become: true - -# Both binaries are built as part of the workspace build above and copied to -# every host regardless of which helper it runs, but only one systemd unit -# ({{helper}}.service) is actually installed per host (see below). -- name: Make jsonth accessible system wide - ansible.builtin.copy: - src: "/tmp/ooniprobe-rs/target/release/jsonth" - dest: "/usr/local/bin/" - mode: '0755' - remote_src: yes - become: yes - register: jsonth_binary - -- name: Make echo accessible system wide - ansible.builtin.copy: - src: "/tmp/ooniprobe-rs/target/release/echo" - dest: "/usr/local/bin/" +- name: Download the {{ helper }} test helper binary + ansible.builtin.get_url: + url: "{{ test_helpers_base_url }}/{{ test_helpers_binary_source }}" + dest: "/usr/local/bin/{{ helper }}" mode: '0755' - remote_src: yes - become: yes - register: echo_binary - -- name: Clean up build directory - ansible.builtin.file: - path: "/tmp/ooniprobe-rs" - state: absent + force: yes become: yes + register: helper_binary # Create systemd units @@ -123,13 +54,10 @@ state: started enabled: yes -# Only the helper actually deployed on this host (the one matching {{helper}}) -# has a unit above; restart it here if its own binary changed, so a source -# rebuild actually takes effect and not just on unit-file changes. - name: Restart helper if its binary changed tags: test-helpers ansible.builtin.systemd_service: name: "{{ helper }}.service" state: restarted become: yes - when: (helper == 'jsonth' and jsonth_binary.changed) or (helper == 'echo' and echo_binary.changed) + when: helper_binary.changed diff --git a/ansible/roles/test_helpers/templates/echo.service b/ansible/roles/test_helpers/templates/echoth.service similarity index 89% rename from ansible/roles/test_helpers/templates/echo.service rename to ansible/roles/test_helpers/templates/echoth.service index 996e6aa1..2c0b9f57 100644 --- a/ansible/roles/test_helpers/templates/echo.service +++ b/ansible/roles/test_helpers/templates/echoth.service @@ -6,7 +6,7 @@ StartLimitBurst=3 [Service] Type=simple -ExecStart=/usr/local/bin/echo --port {{port}} +ExecStart=/usr/local/bin/echoth --port {{port}} Restart=on-failure RestartSec=5 User=testhelpers diff --git a/ansible/roles/test_helpers/vars/main.yml b/ansible/roles/test_helpers/vars/main.yml index 671bb91e..abb5927b 100644 --- a/ansible/roles/test_helpers/vars/main.yml +++ b/ansible/roles/test_helpers/vars/main.yml @@ -1,4 +1,10 @@ -# choices: jsonth, echo +# choices: jsonth, echoth helper: "jsonth" -port: "80" \ No newline at end of file +port: "80" + +# Name of the released binary asset to download for this helper. Defaults to +# the helper name. +# The echo test helper's release asset is published as "echo" +# but we rename it locallly to avoid shadowing the `echo` command +test_helpers_binary_source: "{{ helper }}" From 2958a9e5836e73fca5be4d0a3841eb8936f46b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 12 Aug 2026 13:52:20 +0200 Subject: [PATCH 112/201] Point oonimeasurements to test db --- tf/environments/dev/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 6db16def..3c62a013 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -931,7 +931,7 @@ module "fastpath_builder" { service_name = "fastpath" repo = "ooni/backend" - branch_name = "fix-fastpath" + branch_name = "master" environment = local.environment buildspec_path = "fastpath/buildspec.yml" trigger_path = "fastpath/**" @@ -1148,7 +1148,7 @@ module "ooniapi_oonimeasurements" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.arn ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.arn } @@ -1262,7 +1262,7 @@ module "testlists_builder" { service_name = "testlists" repo = "ooni/backend" - branch_name = "master" + branch_name = "fix_1238_testlists_worktree" environment = local.environment buildspec_path = "ooniapi/services/testlists/buildspec.yml" trigger_path = "ooniapi/services/testlists/**" From a3547cae91010ef3a4d4e204848b2ac71b468b7e Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 13 Aug 2026 12:46:24 +0530 Subject: [PATCH 113/201] chore: change deployer branch to enable webconnectivity_0.5 --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 3c62a013..5f089f5c 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -563,7 +563,7 @@ module "ooniapi_ooniprobe_deployer" { service_name = "ooniprobe" repo = "ooni/backend" - branch_name = "add_ooniprobe_private_api" + branch_name = "feat/enable-webconnectivity-lte" environment = local.environment trigger_path = "ooniapi/services/ooniprobe/**" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" From abf93a299577405480ca933b90a70120c2f9ab76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 13 Aug 2026 12:30:52 +0200 Subject: [PATCH 114/201] Get monitoring ip by digging it --- ansible/roles/test_helpers/defaults/main.yml | 3 + ansible/roles/test_helpers/tasks/main.yml | 17 ++- tf/environments/prod/main.tf | 137 +++---------------- 3 files changed, 36 insertions(+), 121 deletions(-) diff --git a/ansible/roles/test_helpers/defaults/main.yml b/ansible/roles/test_helpers/defaults/main.yml index ea3d89cd..5c1e9ae1 100644 --- a/ansible/roles/test_helpers/defaults/main.yml +++ b/ansible/roles/test_helpers/defaults/main.yml @@ -1,2 +1,5 @@ test_helpers_version: v0.1.6-beta test_helpers_base_url: "https://github.com/ooni/ooniprobe-rs/releases/download/{{ test_helpers_version }}" + +# monitoring server IP +monitoring_server_ip: "{{ lookup('dig', 'monitoring.ooni.org') }}" diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index 21d67991..4a8f8ea2 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -1,14 +1,25 @@ --- -# For prometheus scrape requests -- name: Allow traffic on port 9100 +# For prometheus scrape requests. +- name: Allow traffic on port 9100 from the monitoring host only become: true tags: prometheus-proxy blockinfile: path: /etc/ooni/nftables/tcp/9100.nft create: yes block: | - add rule inet filter input tcp dport 9100 counter accept comment "node exporter" + add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport 9100 counter accept comment "node exporter" + notify: + - reload nftables + +- name: Allow traffic on the {{ helper }} test helper port + become: true + tags: test-helpers + blockinfile: + path: "/etc/ooni/nftables/tcp/{{ port }}.nft" + create: yes + block: | + add rule inet filter input tcp dport {{ port }} counter accept comment "{{ helper }} test helper" notify: - reload nftables diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 147f99ed..46c32791 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1035,137 +1035,38 @@ module "fastpath_builder" { #### Test Helpers Machines +# module "ooni_test_helpers_json" { - source = "../../modules/ec2" - - stage = local.environment - - vpc_id = module.network.vpc_id - subnet_id = module.network.vpc_subnet_public[0].id - private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block - dns_zone_ooni_io = local.dns_zone_ooni_io + source = "../../modules/ooni_th_binary_droplet" - key_name = module.adm_iam_roles.oonidevops_key_name - instance_type = "t3.micro" - - name = "oonijsonth" - ingress_rules = [{ - from_port = 22, - to_port = 22, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 80, # jsonth - to_port = 80, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 9100, # Prometheus monitoring - to_port = 9100, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] - }] + stage = local.environment + name = "oonijsonth" + hostname = "json.th" - egress_rules = [{ - from_port = 0, - to_port = 0, - protocol = "-1", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 0, - to_port = 0, - protocol = "-1", - ipv6_cidr_blocks = ["::/0"], - }] - - sg_prefix = "oonijsonth" - tg_prefix = "tshp" - - disk_size = 20 + ssh_keys = [ + "3d:81:99:17:b5:d1:20:a5:fe:2b:14:96:67:93:d6:34", + "f6:4b:8b:e2:0e:d2:97:c5:45:5c:07:a6:fe:54:60:0e" + ] - tags = merge( - local.tags, - { Name = "ooni-tier0-jsonth" } - ) + dns_zone_ooni_io = local.dns_zone_ooni_io } -# Echo test helper, requires a dedicated machine bc it's a tcp server, +# Echo test helper requires a dedicated machine bc it's a tcp server, # not an HTTP server. It's impossible to reroute using nginx module "ooni_test_helpers_echo" { - source = "../../modules/ec2" + source = "../../modules/ooni_th_binary_droplet" - stage = local.environment - - vpc_id = module.network.vpc_id - subnet_id = module.network.vpc_subnet_public[0].id - private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block - dns_zone_ooni_io = local.dns_zone_ooni_io + stage = local.environment + name = "ooniechoth" + hostname = "echo.th" - key_name = module.adm_iam_roles.oonidevops_key_name - instance_type = "t3.micro" - - name = "ooniechoth" - ingress_rules = [{ - from_port = 22, - to_port = 22, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 80, # echo - to_port = 80, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 9100, # Prometheus monitoring - to_port = 9100, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] - }] - - egress_rules = [{ - from_port = 0, - to_port = 0, - protocol = "-1", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 0, - to_port = 0, - protocol = "-1", - ipv6_cidr_blocks = ["::/0"], - }] - - sg_prefix = "ooniechoth" - tg_prefix = "echo" - - disk_size = 20 - - tags = merge( - local.tags, - { Name = "ooni-tier0-echoth" } - ) -} - -resource "aws_route53_record" "testhelpers_json_alias" { - zone_id = local.dns_zone_ooni_io - name = "json.th.${local.environment}.ooni.io" - type = "CNAME" - ttl = 300 - - records = [ - module.ooni_test_helpers_json.aws_instance_public_dns + ssh_keys = [ + "3d:81:99:17:b5:d1:20:a5:fe:2b:14:96:67:93:d6:34", + "f6:4b:8b:e2:0e:d2:97:c5:45:5c:07:a6:fe:54:60:0e" ] -} -resource "aws_route53_record" "testhelpers_echo_alias" { - zone_id = local.dns_zone_ooni_io - name = "echo.th.${local.environment}.ooni.io" - type = "CNAME" - ttl = 300 - - records = [ - module.ooni_test_helpers_echo.aws_instance_public_dns - ] + dns_zone_ooni_io = local.dns_zone_ooni_io } From e4dc5745cd1cea1f5f89f0ce9bca6ad2eb11eb7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 13 Aug 2026 21:47:53 +0200 Subject: [PATCH 115/201] Pull out nginx and dehydrated roles Install nginx directly without needing using third-party role --- ansible/deploy-clickhouse-proxy.yml | 5 +- ansible/deploy-monitoring-proxy.yml | 4 +- ansible/deploy-notebook.yml | 2 + ansible/deploy-ooni-backend.yml | 1 + ansible/roles/dehydrated/meta/main.yml | 5 -- ansible/roles/nginx/defaults/main.yml | 9 +++ ansible/roles/nginx/tasks/main.yml | 75 ++++++++++++++++++- ansible/roles/notebook/tasks/jupyterhub.yml | 11 --- .../prometheus_node_exporter/tasks/main.yml | 16 ---- 9 files changed, 91 insertions(+), 37 deletions(-) delete mode 100644 ansible/roles/dehydrated/meta/main.yml diff --git a/ansible/deploy-clickhouse-proxy.yml b/ansible/deploy-clickhouse-proxy.yml index 69be5ebb..bf69e389 100644 --- a/ansible/deploy-clickhouse-proxy.yml +++ b/ansible/deploy-clickhouse-proxy.yml @@ -6,9 +6,10 @@ become: true roles: - role: bootstrap + - role: nginx - role: dehydrated - vars: - ssl_domains: + vars: + ssl_domains: - "{{ inventory_hostname }}" tls_cert_dir: /var/lib/dehydrated/certs - role: clickhouse_proxy diff --git a/ansible/deploy-monitoring-proxy.yml b/ansible/deploy-monitoring-proxy.yml index edeb3bf2..945021f9 100644 --- a/ansible/deploy-monitoring-proxy.yml +++ b/ansible/deploy-monitoring-proxy.yml @@ -6,13 +6,13 @@ become: true roles: - role: bootstrap + - role: nginx + tags: nginx - role: dehydrated vars: ssl_domains: - "{{ inventory_hostname }}" tls_cert_dir: /var/lib/dehydrated/certs - - role: nginx - tags: nginx - role: monitoring_proxy vars: monitoring_proxy_public_fqdn: "{{ inventory_hostname }}" diff --git a/ansible/deploy-notebook.yml b/ansible/deploy-notebook.yml index 318f62e4..5f4306ec 100644 --- a/ansible/deploy-notebook.yml +++ b/ansible/deploy-notebook.yml @@ -9,4 +9,6 @@ ssl_domains: - "{{ inventory_hostname }}" roles: + - nginx + - dehydrated - notebook diff --git a/ansible/deploy-ooni-backend.yml b/ansible/deploy-ooni-backend.yml index 7e056905..3306f780 100644 --- a/ansible/deploy-ooni-backend.yml +++ b/ansible/deploy-ooni-backend.yml @@ -8,6 +8,7 @@ admin_group_name: adm - role: base-backend - role: nftables + - role: nginx - role: dehydrated tags: dehydrated expand: yes diff --git a/ansible/roles/dehydrated/meta/main.yml b/ansible/roles/dehydrated/meta/main.yml deleted file mode 100644 index 0e72e865..00000000 --- a/ansible/roles/dehydrated/meta/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -dependencies: - - nginx -... - diff --git a/ansible/roles/nginx/defaults/main.yml b/ansible/roles/nginx/defaults/main.yml index 4c0ac11a..41a890b4 100644 --- a/ansible/roles/nginx/defaults/main.yml +++ b/ansible/roles/nginx/defaults/main.yml @@ -1 +1,10 @@ nginx_user: nginx + +# "nginxinc_role" (default, unchanged): install via the nginxinc.nginx galaxy +# role, as today. +# "official_repo": install directly from nginx.org's own apt repo, following +# https://nginx.org/en/linux_packages.html#Debian, with a pinned signing-key +# fingerprint check. Opt in per-playbook; the nginxinc_role default is left +# alone for existing callers since switching apt repos changes what nginx +# package/version they get on their next run. +nginx_install_method: nginxinc_role diff --git a/ansible/roles/nginx/tasks/main.yml b/ansible/roles/nginx/tasks/main.yml index 9af2a9b4..3b31e7a3 100644 --- a/ansible/roles/nginx/tasks/main.yml +++ b/ansible/roles/nginx/tasks/main.yml @@ -13,9 +13,82 @@ - nginx - nftables -- name: install nginx +- name: install nginx via the nginxinc.nginx galaxy role include_role: name: nginxinc.nginx + tags: + - nginx + - packages + when: nginx_install_method == 'nginxinc_role' + +# Follows https://nginx.org/en/linux_packages.html#Debian directly instead +# of a third-party role. +- name: Install nginx from nginx.org's official apt repository + when: nginx_install_method == 'official_repo' + tags: + - nginx + - packages + block: + - name: Install nginx's prerequisites + ansible.builtin.apt: + name: + - curl + - gnupg2 + - ca-certificates + state: present + update_cache: true + + - name: Download nginx's official signing key + ansible.builtin.get_url: + url: https://nginx.org/keys/nginx_signing.key + dest: /tmp/nginx_signing.key + mode: "0644" + + - name: Dearmor nginx's signing key into the apt keyring + ansible.builtin.command: gpg --dearmor --yes --output /usr/share/keyrings/nginx-archive-keyring.gpg /tmp/nginx_signing.key + args: + creates: /usr/share/keyrings/nginx-archive-keyring.gpg + + - name: Read the fingerprint of the downloaded key + ansible.builtin.command: gpg --with-colons --show-keys /usr/share/keyrings/nginx-archive-keyring.gpg + register: nginx_key_check + changed_when: false + + # Pin against nginx.org's gpg key fingerprint: + # 573B FD6B 3D8F BC64 1079 A6AB ABF5 BD82 7BD9 BF62, documented at + # (https://nginx.org/en/linux_packages.html) + - name: Verify the downloaded key's fingerprint matches nginx.org's published one + ansible.builtin.assert: + that: + - nginx_key_check.stdout is + search('^fpr:::::::::573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62:$', multiline=true) + fail_msg: >- + nginx-archive-keyring.gpg's fingerprint does not match the + expected 573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62 -- refusing to + trust it. + success_msg: "nginx-archive-keyring.gpg's fingerprint matches the expected value." + + - name: Add the nginx.org apt repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://nginx.org/packages/debian {{ ansible_distribution_release }} nginx" + filename: nginx + state: present + + - name: Pin nginx.org packages above Debian's own nginx package + ansible.builtin.copy: + dest: /etc/apt/preferences.d/99nginx + content: | + Package: * + Pin: origin nginx.org + Pin: release o=nginx + Pin-Priority: 900 + mode: "0644" + + - name: Install nginx + ansible.builtin.apt: + name: nginx + state: present + update_cache: true # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4 # Guide https://wiki.mozilla.org/Security/Server_Side_TLS#Pre-defined_DHE_groups diff --git a/ansible/roles/notebook/tasks/jupyterhub.yml b/ansible/roles/notebook/tasks/jupyterhub.yml index ea9e0e04..de4a271b 100644 --- a/ansible/roles/notebook/tasks/jupyterhub.yml +++ b/ansible/roles/notebook/tasks/jupyterhub.yml @@ -106,14 +106,3 @@ - jupyterhub - config -- ansible.builtin.include_role: - name: nginx - tags: - - oonidata - - nginx - -- ansible.builtin.include_role: - name: dehydrated - tags: - - oonidata - - dehydrated diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index fb9e0e6f..e647a7cd 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -1,19 +1,3 @@ -- ansible.builtin.include_role: - name: nginx - tags: - - nginx - - node_exporter - when: use_nginx - -- ansible.builtin.include_role: - name: dehydrated - tags: - - oonidata - - dehydrated - vars: - ssl_domains: "{{ hostvars[inventory_hostname].ssl_domains | default([inventory_hostname]) }}" - when: use_https - - name: create ooni configuration directory ansible.builtin.file: path: "/etc/ooni/" From d32df969807c1193b5b0725eeb4310926818bdb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 13 Aug 2026 22:01:01 +0200 Subject: [PATCH 116/201] Apply tags and suggest a tagging convention --- ansible/README.md | 35 +++++++++++++++++++ ansible/roles/dehydrated/tasks/main.yml | 18 +++++----- ansible/roles/nginx/tasks/main.yml | 5 +++ .../tasks/install.yml | 30 +++++++++++++++- .../prometheus_node_exporter/tasks/main.yml | 2 ++ 5 files changed, 80 insertions(+), 10 deletions(-) diff --git a/ansible/README.md b/ansible/README.md index 8507cd29..0f49c25e 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -191,6 +191,41 @@ When deploying files or updating files already existing on the hosts it can be u This helps track down how files on the host were modified and why. ::: +#### Tagging convention + +Tasks are tagged along two independent axes, so that `--tags`/`--skip-tags` can be used to run (or skip) a +meaningful slice of a playbook without guessing what else might be silently pulled in or left out: + +* **Component tag** — matches the role name (`nginx`, `dehydrated`, `nftables`, `prometheus_node_exporter`, + `docker`, `ooniapi_gateway`, ...). Selects everything belonging to that role. +* **Phase tag** — a small, fixed vocabulary describing *what kind* of operation the task performs, independent of + which role it lives in: + + | Tag | Meaning | + |--------------|----------------------------------------------------------------------------------------------------------------------| + | `packages` | apt/package installs — slow, safe to skip when nothing package-level changed | + | `config` | template/config file rendering — fast, safe to run often | + | `certs` | dehydrated cert issuance/renewal — rate-limited by Let's Encrypt, must be independently skippable (e.g. before DNS is cut over to a new host) | + | `network` | nftables rules and docker network setup | + | `service` | service enable/start/restart/reload operations | + | `monitoring` | prometheus/node_exporter wiring and health checks | + | `secrets` | sudoers rules and deploy-credential/password setup | + + Most tasks should carry exactly one component tag and one phase tag. + + Examples: + ``` + ./play -i inventory deploy-monitoring-proxy.yml -l monitoringproxy.prod.ooni.io --tags config + ./play -i inventory deploy-ooni-backend.yml -l backend-hel.ooni.org --skip-tags certs + ./play -i inventory deploy-clickhouse-proxy.yml -l clickhouseproxy.prod.ooni.io --tags monitoring + ``` + +:::note +This taxonomy is applied incrementally as roles are touched, not retrofitted across the whole codebase in one go +— currently `nginx`, `dehydrated`, and `prometheus_node_exporter` follow it. Don't assume every task elsewhere +already does. +::: + ### Platform specific known bugs On macOS you might run into this issue: https://github.com/ansible/ansible/issues/76322 diff --git a/ansible/roles/dehydrated/tasks/main.yml b/ansible/roles/dehydrated/tasks/main.yml index ecf09a5d..860ffcc3 100644 --- a/ansible/roles/dehydrated/tasks/main.yml +++ b/ansible/roles/dehydrated/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Installs packages - tags: dehydrated + tags: [dehydrated, packages] apt: install_recommends: false cache_valid_time: 86400 @@ -23,7 +23,7 @@ # HOOK="/etc/dehydrated/hook.sh" - name: Add ACME dedicated sites-enabled file - tags: dehydrated + tags: [dehydrated, certs] ansible.builtin.template: src: templates/letsencrypt-http # the server block matches all SSL FQDNs and must be @@ -33,7 +33,7 @@ owner: root - name: Add canary file to ensure /.well-known/acme-challenge is reachable by let's encrypt - tags: dehydrated + tags: [dehydrated, certs] ansible.builtin.copy: content: | Generated by ansible using ansible/roles/dehydrated/tasks/main.yml. @@ -44,23 +44,23 @@ owner: root - name: reload nftables service - tags: dehydrated + tags: [dehydrated, network] shell: systemctl reload nftables.service - name: Configure domains {{ ssl_domains }} # https://github.com/dehydrated-io/dehydrated/blob/master/docs/domains_txt.md - tags: dehydrated + tags: [dehydrated, certs] template: src: templates/domains.txt.j2 dest: /etc/dehydrated/domains.txt - name: Register account if needed - tags: dehydrated + tags: [dehydrated, certs] ansible.builtin.shell: cmd: "test -d /var/lib/dehydrated/accounts || dehydrated --register --accept-terms" - name: Install dehydrated.service - tags: dehydrated + tags: [dehydrated, certs] template: src: templates/dehydrated.service dest: /etc/systemd/system/dehydrated.service @@ -68,7 +68,7 @@ owner: root - name: Install dehydrated.timer - tags: dehydrated + tags: [dehydrated, certs] template: src: templates/dehydrated.timer dest: /etc/systemd/system/dehydrated.timer @@ -76,7 +76,7 @@ owner: root - name: Ensure timer runs - tags: dehydrated + tags: [dehydrated, certs] systemd: name: dehydrated.timer state: started diff --git a/ansible/roles/nginx/tasks/main.yml b/ansible/roles/nginx/tasks/main.yml index 3b31e7a3..dbb2c3fc 100644 --- a/ansible/roles/nginx/tasks/main.yml +++ b/ansible/roles/nginx/tasks/main.yml @@ -12,6 +12,7 @@ tags: - nginx - nftables + - network - name: install nginx via the nginxinc.nginx galaxy role include_role: @@ -101,6 +102,7 @@ - ssl_modern.conf tags: - nginx + - config - name: remove `default` vhost file: path={{item}} state=absent @@ -111,6 +113,7 @@ - /etc/nginx/sites-enabled/default tags: - nginx + - config - name: Create nginx sites directory ansible.builtin.file: @@ -118,9 +121,11 @@ state: directory tags: - nginx + - config - name: set nginx.conf template: src=nginx.conf dest=/etc/nginx/nginx.conf mode=0444 notify: reload nginx tags: - nginx + - config diff --git a/ansible/roles/prometheus_node_exporter/tasks/install.yml b/ansible/roles/prometheus_node_exporter/tasks/install.yml index 47ed566e..1ff74425 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/install.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/install.yml @@ -4,6 +4,9 @@ failed_when: false changed_when: false register: node_exporter_version_check + tags: + - prometheus_node_exporter + - packages - name: Download and unarchive node_exporter into temporary location. unarchive: @@ -15,6 +18,9 @@ node_exporter_version_check.stdout is not defined or node_exporter_version not in node_exporter_version_check.stdout register: node_exporter_download_check + tags: + - prometheus_node_exporter + - packages - name: Move node_exporter binary into place. copy: @@ -26,11 +32,17 @@ when: > node_exporter_download_check is changed or node_exporter_version_check.stdout | length == 0 + tags: + - prometheus_node_exporter + - packages - name: Create node_exporter group. group: name: "{{ prometheus_group }}" state: present + tags: + - prometheus_node_exporter + - config - name: Create node_exporter user. user: @@ -38,6 +50,9 @@ shell: /sbin/nologin group: "{{ prometheus_group }}" state: present + tags: + - prometheus_node_exporter + - config - name: Add a user to a password file and ensure permissions are set community.general.htpasswd: @@ -48,9 +63,10 @@ group: "{{ 'nginx' if use_nginx else prometheus_group }}" mode: 0640 tags: + - prometheus_node_exporter - monitoring - node_exporter - - config + - secrets - name: Copy the node_exporter systemd unit file. template: @@ -58,18 +74,27 @@ dest: /etc/systemd/system/node_exporter.service mode: 0644 register: node_exporter_service + tags: + - prometheus_node_exporter + - config - name: Reload systemd daemon if unit file is changed. systemd: daemon_reload: true notify: restart node_exporter when: node_exporter_service is changed + tags: + - prometheus_node_exporter + - service - name: Ensure node_exporter is running and enabled at boot. service: name: node_exporter state: "{{ node_exporter_state }}" enabled: "{{ node_exporter_enabled }}" + tags: + - prometheus_node_exporter + - service - name: Verify node_exporter is responding to requests. uri: @@ -77,3 +102,6 @@ return_content: true register: metrics_output failed_when: "'Metrics' not in metrics_output.content" + tags: + - prometheus_node_exporter + - monitoring diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index e647a7cd..7512d5b9 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -4,6 +4,7 @@ state: directory owner: root tags: + - prometheus_node_exporter - monitoring - node_exporter - config @@ -16,6 +17,7 @@ notify: - Restart nginx tags: + - prometheus_node_exporter - monitoring - node_exporter - config From cd7d168da44a4b40e5c99ef7c7e53eeb41d914cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 14 Aug 2026 10:42:52 +0200 Subject: [PATCH 117/201] Add oonidevops ssh key to digitalocean hosts --- tf/environments/prod/main.tf | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 46c32791..801ee5f9 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -1037,6 +1037,14 @@ module "fastpath_builder" { #### Test Helpers Machines # +# Registers the same oonidevops keypair used for the EC2 instances (see +# module.adm_iam_roles) as a DigitalOcean account key, so it can be installed +# on droplets via their ssh_keys argument too. +resource "digitalocean_ssh_key" "oonidevops" { + name = "oonidevops" + public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] +} + module "ooni_test_helpers_json" { source = "../../modules/ooni_th_binary_droplet" @@ -1044,10 +1052,7 @@ module "ooni_test_helpers_json" { name = "oonijsonth" hostname = "json.th" - ssh_keys = [ - "3d:81:99:17:b5:d1:20:a5:fe:2b:14:96:67:93:d6:34", - "f6:4b:8b:e2:0e:d2:97:c5:45:5c:07:a6:fe:54:60:0e" - ] + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] dns_zone_ooni_io = local.dns_zone_ooni_io } @@ -1061,10 +1066,7 @@ module "ooni_test_helpers_echo" { name = "ooniechoth" hostname = "echo.th" - ssh_keys = [ - "3d:81:99:17:b5:d1:20:a5:fe:2b:14:96:67:93:d6:34", - "f6:4b:8b:e2:0e:d2:97:c5:45:5c:07:a6:fe:54:60:0e" - ] + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] dns_zone_ooni_io = local.dns_zone_ooni_io } From b8b458b1f9a67f6f5e508619727fd766c99e59ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 14 Aug 2026 12:14:28 +0200 Subject: [PATCH 118/201] Fix small issues on playbooks for debian --- ansible/requirements/ansible-galaxy.yml | 2 +- ansible/roles/ssh_users/tasks/main.yml | 4 ++-- ansible/roles/test_helpers/handlers/main.yml | 12 ------------ ansible/roles/test_helpers/tasks/main.yml | 7 ++++--- 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/ansible/requirements/ansible-galaxy.yml b/ansible/requirements/ansible-galaxy.yml index e31076d3..5172cbec 100644 --- a/ansible/requirements/ansible-galaxy.yml +++ b/ansible/requirements/ansible-galaxy.yml @@ -1,6 +1,6 @@ roles: - src: nginxinc.nginx - version: 0.24.3 + version: 0.26.0 - src: geerlingguy.certbot version: 5.2.0 - src: https://github.com/idealista/clickhouse_role diff --git a/ansible/roles/ssh_users/tasks/main.yml b/ansible/roles/ssh_users/tasks/main.yml index 3615a9ba..14a5bd6e 100644 --- a/ansible/roles/ssh_users/tasks/main.yml +++ b/ansible/roles/ssh_users/tasks/main.yml @@ -85,8 +85,8 @@ path: /etc/sudoers.d/adm state: absent -- name: reload sshd +- name: reload ssh tags: ssh_users ansible.builtin.systemd_service: - name: sshd + name: ssh state: reloaded diff --git a/ansible/roles/test_helpers/handlers/main.yml b/ansible/roles/test_helpers/handlers/main.yml index a5505f71..1d820eff 100644 --- a/ansible/roles/test_helpers/handlers/main.yml +++ b/ansible/roles/test_helpers/handlers/main.yml @@ -1,15 +1,3 @@ -- name: restart echoth - tags: test-helpers - ansible.builtin.systemd_service: - name: echoth - state: restarted - -- name: restart jsonth - tags: test-helpers - ansible.builtin.systemd_service: - name: jsonth - state: restarted - - name: reload nftables tags: nftables ansible.builtin.systemd_service: diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index 4a8f8ea2..bec319cc 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -51,12 +51,13 @@ dest: /etc/systemd/system/{{helper}}.service mode: '0755' owner: root - notify: - - "restart {{helper}}" + register: helper_unit - name: reload systemd tags: test-helpers - shell: systemctl daemon-reload + ansible.builtin.systemd_service: + daemon_reload: yes + when: helper_unit.changed - name: Start helper tags: test-helpers From a40258fd576a5ff2b16368ec0e74e1714213263b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 14 Aug 2026 12:15:20 +0200 Subject: [PATCH 119/201] Add simpler test helper module --- tf/modules/ooni_th_binary_droplet/main.tf | 31 +++++++++++++++++ tf/modules/ooni_th_binary_droplet/outputs.tf | 7 ++++ .../ooni_th_binary_droplet/variables.tf | 33 +++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 tf/modules/ooni_th_binary_droplet/main.tf create mode 100644 tf/modules/ooni_th_binary_droplet/outputs.tf create mode 100644 tf/modules/ooni_th_binary_droplet/variables.tf diff --git a/tf/modules/ooni_th_binary_droplet/main.tf b/tf/modules/ooni_th_binary_droplet/main.tf new file mode 100644 index 00000000..f0d3c1f6 --- /dev/null +++ b/tf/modules/ooni_th_binary_droplet/main.tf @@ -0,0 +1,31 @@ +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + } +} + +resource "digitalocean_droplet" "ooni_th" { + image = "debian-13-x64" + name = "${var.name}-${var.stage}" + region = var.instance_location + size = var.instance_size + ipv6 = true + ssh_keys = var.ssh_keys + + lifecycle { + create_before_destroy = true + ignore_changes = all + } +} + +resource "aws_route53_record" "ooni_th" { + zone_id = var.dns_zone_ooni_io + name = "${var.hostname}.${var.stage}.ooni.io" + type = "A" + ttl = 60 + + records = [digitalocean_droplet.ooni_th.ipv4_address] +} diff --git a/tf/modules/ooni_th_binary_droplet/outputs.tf b/tf/modules/ooni_th_binary_droplet/outputs.tf new file mode 100644 index 00000000..e091c016 --- /dev/null +++ b/tf/modules/ooni_th_binary_droplet/outputs.tf @@ -0,0 +1,7 @@ +output "droplet_ipv4_address" { + value = digitalocean_droplet.ooni_th.ipv4_address +} + +output "fqdn" { + value = aws_route53_record.ooni_th.fqdn +} diff --git a/tf/modules/ooni_th_binary_droplet/variables.tf b/tf/modules/ooni_th_binary_droplet/variables.tf new file mode 100644 index 00000000..b257b344 --- /dev/null +++ b/tf/modules/ooni_th_binary_droplet/variables.tf @@ -0,0 +1,33 @@ +variable "stage" { + type = string +} + +variable "name" { + description = "Name of the droplet (will be suffixed with -)" + type = string +} + +variable "hostname" { + description = "DNS label for the helper, e.g. \"json.th\" or \"echo.th\". The record is created as ..ooni.io" + type = string +} + +variable "instance_location" { + type = string + default = "fra1" +} + +variable "instance_size" { + type = string + default = "s-1vcpu-1gb" +} + +variable "ssh_keys" { + description = "Fingerprints of the DigitalOcean account SSH keys to grant root access to" + type = list(string) +} + +variable "dns_zone_ooni_io" { + description = "id of the DNS zone for ooni_io" + type = string +} From f88a07f25103de1d9aff1c884afe9734afcca4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 14 Aug 2026 12:16:17 +0200 Subject: [PATCH 120/201] Fix test helper restart --- ansible/roles/test_helpers/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index bec319cc..3b463873 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -66,10 +66,10 @@ state: started enabled: yes -- name: Restart helper if its binary changed +- name: Restart helper if its binary or unit file changed tags: test-helpers ansible.builtin.systemd_service: name: "{{ helper }}.service" state: restarted become: yes - when: helper_binary.changed + when: helper_binary.changed or helper_unit.changed From 01214d646adffb1b9108930a1cb21bbc9363bd17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 14 Aug 2026 13:20:48 +0200 Subject: [PATCH 121/201] rename binaries; do checksum on download --- ansible/deploy-echo-test-helper.yml | 5 +++-- ansible/deploy-json-test-helper.yml | 4 +++- ansible/roles/test_helpers/defaults/main.yml | 2 +- ansible/roles/test_helpers/tasks/main.yml | 3 ++- .../templates/{echoth.service => ooechohelpd.service} | 2 +- .../templates/{jsonth.service => oojsonhelpd.service} | 2 +- ansible/roles/test_helpers/vars/main.yml | 10 ++-------- 7 files changed, 13 insertions(+), 15 deletions(-) rename ansible/roles/test_helpers/templates/{echoth.service => ooechohelpd.service} (88%) rename ansible/roles/test_helpers/templates/{jsonth.service => oojsonhelpd.service} (89%) diff --git a/ansible/deploy-echo-test-helper.yml b/ansible/deploy-echo-test-helper.yml index 2f3965ae..2e875c1e 100644 --- a/ansible/deploy-echo-test-helper.yml +++ b/ansible/deploy-echo-test-helper.yml @@ -17,6 +17,7 @@ http_port: 8080 # if we leave port 80, it's taken by nginx - role: test_helpers vars: - helper: echoth - test_helpers_binary_source: echo + helper: ooechohelpd + # sha256 of https://github.com/ooni/ooniprobe-rs/releases/download/v0.1.6/ooechohelpd + test_helpers_checksum: 10511a8b918c2244eba9e17c32f3cec9efbc5ffa51d6ad7f59fe648cd0986615 port: 80 diff --git a/ansible/deploy-json-test-helper.yml b/ansible/deploy-json-test-helper.yml index ba08b1fc..e591018d 100644 --- a/ansible/deploy-json-test-helper.yml +++ b/ansible/deploy-json-test-helper.yml @@ -17,5 +17,7 @@ http_port: 8080 # if we leave port 80, it's taken by nginx - role: test_helpers vars: - helper: jsonth + helper: oojsonhelpd + # sha256 of https://github.com/ooni/ooniprobe-rs/releases/download/v0.1.6/oojsonhelpd + test_helpers_checksum: 3d977e1418ea398f520dff3de951e36010a294dc3d1b564c961542cc3729a224 port: 80 diff --git a/ansible/roles/test_helpers/defaults/main.yml b/ansible/roles/test_helpers/defaults/main.yml index 5c1e9ae1..d3d76986 100644 --- a/ansible/roles/test_helpers/defaults/main.yml +++ b/ansible/roles/test_helpers/defaults/main.yml @@ -1,4 +1,4 @@ -test_helpers_version: v0.1.6-beta +test_helpers_version: v0.1.6 test_helpers_base_url: "https://github.com/ooni/ooniprobe-rs/releases/download/{{ test_helpers_version }}" # monitoring server IP diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml index 3b463873..02b57055 100644 --- a/ansible/roles/test_helpers/tasks/main.yml +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -35,8 +35,9 @@ # Install test helpers - name: Download the {{ helper }} test helper binary ansible.builtin.get_url: - url: "{{ test_helpers_base_url }}/{{ test_helpers_binary_source }}" + url: "{{ test_helpers_base_url }}/{{ helper }}" dest: "/usr/local/bin/{{ helper }}" + checksum: "sha256:{{ test_helpers_checksum }}" mode: '0755' force: yes become: yes diff --git a/ansible/roles/test_helpers/templates/echoth.service b/ansible/roles/test_helpers/templates/ooechohelpd.service similarity index 88% rename from ansible/roles/test_helpers/templates/echoth.service rename to ansible/roles/test_helpers/templates/ooechohelpd.service index 2c0b9f57..2ba60f1c 100644 --- a/ansible/roles/test_helpers/templates/echoth.service +++ b/ansible/roles/test_helpers/templates/ooechohelpd.service @@ -6,7 +6,7 @@ StartLimitBurst=3 [Service] Type=simple -ExecStart=/usr/local/bin/echoth --port {{port}} +ExecStart=/usr/local/bin/ooechohelpd --port {{port}} Restart=on-failure RestartSec=5 User=testhelpers diff --git a/ansible/roles/test_helpers/templates/jsonth.service b/ansible/roles/test_helpers/templates/oojsonhelpd.service similarity index 89% rename from ansible/roles/test_helpers/templates/jsonth.service rename to ansible/roles/test_helpers/templates/oojsonhelpd.service index 54392c17..de3c4367 100644 --- a/ansible/roles/test_helpers/templates/jsonth.service +++ b/ansible/roles/test_helpers/templates/oojsonhelpd.service @@ -6,7 +6,7 @@ StartLimitBurst=3 [Service] Type=simple -ExecStart=/usr/local/bin/jsonth --port {{port}} +ExecStart=/usr/local/bin/oojsonhelpd --port {{port}} Restart=on-failure RestartSec=5 User=testhelpers diff --git a/ansible/roles/test_helpers/vars/main.yml b/ansible/roles/test_helpers/vars/main.yml index abb5927b..3d224a6d 100644 --- a/ansible/roles/test_helpers/vars/main.yml +++ b/ansible/roles/test_helpers/vars/main.yml @@ -1,10 +1,4 @@ -# choices: jsonth, echoth -helper: "jsonth" +# choices: oojsonhelpd, ooechohelpd +helper: "oojsonhelpd" port: "80" - -# Name of the released binary asset to download for this helper. Defaults to -# the helper name. -# The echo test helper's release asset is published as "echo" -# but we rename it locallly to avoid shadowing the `echo` command -test_helpers_binary_source: "{{ helper }}" From a88a72558cd34a5c0b2eba8b44b044b3d0ae4fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 14 Aug 2026 13:28:46 +0200 Subject: [PATCH 122/201] Update github actions script --- .github/workflows/check_ansible.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_ansible.yml b/.github/workflows/check_ansible.yml index 396a8afc..c6c14408 100644 --- a/.github/workflows/check_ansible.yml +++ b/.github/workflows/check_ansible.yml @@ -58,7 +58,7 @@ jobs: #- name: Setup tmate session # uses: mxschmitt/action-tmate@v3 - - uses: actions/github-script@v6 + - uses: actions/github-script@v8 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | From 2da3b0c1fac18acc40c93d2738a6316afddb0dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 14 Aug 2026 14:41:43 +0200 Subject: [PATCH 123/201] Bump github actions script to v9 --- .github/workflows/check_ansible.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_ansible.yml b/.github/workflows/check_ansible.yml index c6c14408..92d924d4 100644 --- a/.github/workflows/check_ansible.yml +++ b/.github/workflows/check_ansible.yml @@ -58,7 +58,7 @@ jobs: #- name: Setup tmate session # uses: mxschmitt/action-tmate@v3 - - uses: actions/github-script@v8 + - uses: actions/github-script@v9 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | From 40c0cfcdd078f926ce2189c85e048bc63bfa7bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 14 Aug 2026 14:46:32 +0200 Subject: [PATCH 124/201] Comment out the ansible comment step --- .github/workflows/check_ansible.yml | 60 ++++++++++++++--------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/check_ansible.yml b/.github/workflows/check_ansible.yml index 92d924d4..1c32eb4d 100644 --- a/.github/workflows/check_ansible.yml +++ b/.github/workflows/check_ansible.yml @@ -58,41 +58,41 @@ jobs: #- name: Setup tmate session # uses: mxschmitt/action-tmate@v3 - - uses: actions/github-script@v9 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const commentTitle = "Ansible Run Output"; - const ansiblePlaybookOutput = `${{ steps.playbook.outputs.ansible_playbook}}`; - const parts = ansiblePlaybookOutput.split(/PLAY RECAP \*+/); - const ansiblePlaybookRecap = parts.length > 1 ? parts[1].trim() : ''; + #- uses: actions/github-script@v9 + # with: + # github-token: ${{ secrets.GITHUB_TOKEN }} + # script: | + # const commentTitle = "Ansible Run Output"; + # const ansiblePlaybookOutput = `${{ steps.playbook.outputs.ansible_playbook}}`; + # const parts = ansiblePlaybookOutput.split(/PLAY RECAP \*+/); + # const ansiblePlaybookRecap = parts.length > 1 ? parts[1].trim() : ''; - const commentBody = ` - #### Ansible Playbook Recap 🔍 + # const commentBody = ` + # #### Ansible Playbook Recap 🔍 - \`\`\`\n - ${ansiblePlaybookRecap} - \`\`\` + # \`\`\`\n + # ${ansiblePlaybookRecap} + # \`\`\` - #### Ansible playbook output 📖\`${{ steps.playbook.outcome }}\` + # #### Ansible playbook output 📖\`${{ steps.playbook.outcome }}\` -
Show Execution + #
Show Execution - \`\`\`\n - ${ansiblePlaybookOutput} - \`\`\` + # \`\`\`\n + # ${ansiblePlaybookOutput} + # \`\`\` -
+ #
- | | | - |-------------------|------------------------------------| - | Pusher | @${{ github.actor }} | - | Action | ${{ github.event_name }} | - | Working Directory | ${{ env.tf_actions_working_dir }} | - | Workflow | ${{ github.workflow }} | - | Last updated | ${(new Date()).toUTCString()} | - `; + # | | | + # |-------------------|------------------------------------| + # | Pusher | @${{ github.actor }} | + # | Action | ${{ github.event_name }} | + # | Working Directory | ${{ env.tf_actions_working_dir }} | + # | Workflow | ${{ github.workflow }} | + # | Last updated | ${(new Date()).toUTCString()} | + # `; - // Call the script to write the comment - const script = require('./scripts/ghactions/comment-on-pr.js'); - await script({github, context, core, commentTitle, commentBody}); + # // Call the script to write the comment + # const script = require('./scripts/ghactions/comment-on-pr.js'); + # await script({github, context, core, commentTitle, commentBody}); From 930d7057b7cefa596279111a742bf5736922b697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 14 Aug 2026 17:10:55 +0200 Subject: [PATCH 125/201] Add web_connectivity test helper dev instance --- tf/environments/dev/main.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 5f089f5c..2ccca183 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -409,6 +409,24 @@ module "ooni_th_droplet" { dns_zone_ooni_io = local.dns_zone_ooni_io } +resource "digitalocean_ssh_key" "oonidevops" { + name = "oonidevops" + public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] +} + + +module "ooni_test_helpers_wc" { + source = "../../modules/ooni_th_binary_droplet" + + stage = local.environment + name = "ooniwcth" + hostname = "wc.th" + + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] + + dns_zone_ooni_io = local.dns_zone_ooni_io +} + ### OONI Services Clusters module "ooniapi_cluster" { From c9e58ab3b2719330057f094b4d7eee65052b7499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 14 Aug 2026 17:19:07 +0200 Subject: [PATCH 126/201] Add deploy-wc-test-helper playbook --- ansible/deploy-wc-test-helper.yml | 14 ++++++++++++++ ansible/inventory | 3 +++ 2 files changed, 17 insertions(+) create mode 100644 ansible/deploy-wc-test-helper.yml diff --git a/ansible/deploy-wc-test-helper.yml b/ansible/deploy-wc-test-helper.yml new file mode 100644 index 00000000..1b0ce980 --- /dev/null +++ b/ansible/deploy-wc-test-helper.yml @@ -0,0 +1,14 @@ +--- +- name: Deploy test helpers + hosts: + - wc.th.dev.ooni.io + become: true + vars: + ssl_domains: + - "{{ inventory_hostname }}" + nginx_install_method: official_repo + roles: + - role: bootstrap + - role: nginx + - role: dehydrated + - role: prometheus_node_exporter diff --git a/ansible/inventory b/ansible/inventory index 5008d568..bcc6e3d1 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -35,6 +35,9 @@ ams-ps.ooni.nu #mia-echoth.ooni.nu #mia-httpth.ooni.nu +[do_fra] +wc.th.dev.ooni.io + [aws-proxy] clickhouseproxy.dev.ooni.io clickhouseproxy.prod.ooni.io From 806abc04ca3fc92e7715c172ecc082b5e974ad58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 14 Aug 2026 17:50:55 +0200 Subject: [PATCH 127/201] Fix resolved check command --- ansible/roles/bootstrap/handlers/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/bootstrap/handlers/main.yml b/ansible/roles/bootstrap/handlers/main.yml index a9c712a4..2bdb126d 100644 --- a/ansible/roles/bootstrap/handlers/main.yml +++ b/ansible/roles/bootstrap/handlers/main.yml @@ -9,7 +9,7 @@ state: restarted - name: Test systemd-resolved - ansible.builtin.shell: resolvectl query go.dnscheck.tools --cache=no + ansible.builtin.shell: resolvectl query dnscheck.tools --cache=no - name: Restart systemd-journald ansible.builtin.systemd_service: From a557e8879ad5b6080d3d177a79039d54a968cd4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 14 Aug 2026 17:51:58 +0200 Subject: [PATCH 128/201] Fix updating of locales --- ansible/roles/bootstrap/tasks/main.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index 2b65c611..5b5da161 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -26,6 +26,7 @@ - git - htop - iotop + - locales - lsof - lvm2 - man-db @@ -44,6 +45,24 @@ update_cache: yes install_recommends: no +- name: Generate en_US.UTF-8 locale + community.general.locale_gen: + name: en_US.UTF-8 + state: present + +- name: Set system locale + ansible.builtin.lineinfile: + path: /etc/default/locale + create: yes + regexp: "^{{ item.key }}=" + line: "{{ item.key }}={{ item.value }}" + owner: root + group: root + mode: "0644" + loop: + - { key: "LANG", value: "en_US.UTF-8" } + - { key: "LC_ALL", value: "en_US.UTF-8" } + - name: Set timezone community.general.timezone: name: Etc/UTC From 7ceafd72949ec1a50f7a50e1198c2d17b2a707ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 14 Aug 2026 17:52:04 +0200 Subject: [PATCH 129/201] Enable post quantum sshd algorithms --- ansible/roles/ssh_users/templates/sshd_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/ssh_users/templates/sshd_config b/ansible/roles/ssh_users/templates/sshd_config index a2488721..108ab997 100644 --- a/ansible/roles/ssh_users/templates/sshd_config +++ b/ansible/roles/ssh_users/templates/sshd_config @@ -8,5 +8,5 @@ PrintMotd no AcceptEnv LANG LC_* Subsystem sftp /usr/lib/openssh/sftp-server Ciphers chacha20-poly1305@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr -KexAlgorithms curve25519-sha256 +KexAlgorithms sntrup761x25519-sha512@openssh.com,mlkem768x25519-sha256,curve25519-sha256 MACs hmac-sha2-512,hmac-sha2-256 From 2d06d40006835a96a2b37755ee5236df6aabcdbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 14 Aug 2026 17:54:25 +0200 Subject: [PATCH 130/201] Exclude mlkem768x25519-sha256 since it's not in bookworm --- ansible/roles/ssh_users/tasks/main.yml | 1 + ansible/roles/ssh_users/templates/sshd_config | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/ssh_users/tasks/main.yml b/ansible/roles/ssh_users/tasks/main.yml index 14a5bd6e..77330df0 100644 --- a/ansible/roles/ssh_users/tasks/main.yml +++ b/ansible/roles/ssh_users/tasks/main.yml @@ -60,6 +60,7 @@ owner: root group: root mode: 0440 + validate: 'sshd -t -f %s' - name: Ensure sudoers dir exists tags: ssh_users diff --git a/ansible/roles/ssh_users/templates/sshd_config b/ansible/roles/ssh_users/templates/sshd_config index 108ab997..04a795a9 100644 --- a/ansible/roles/ssh_users/templates/sshd_config +++ b/ansible/roles/ssh_users/templates/sshd_config @@ -8,5 +8,6 @@ PrintMotd no AcceptEnv LANG LC_* Subsystem sftp /usr/lib/openssh/sftp-server Ciphers chacha20-poly1305@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr -KexAlgorithms sntrup761x25519-sha512@openssh.com,mlkem768x25519-sha256,curve25519-sha256 +# mlkem768x25519-sha256 is excluded, but we should enable it once we have all hosts on trixie +KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256 MACs hmac-sha2-512,hmac-sha2-256 From 41b5b5576766e16898e950199d76eec110c96624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 14 Aug 2026 18:13:28 +0200 Subject: [PATCH 131/201] Enable TLSv1.3 --- ansible/roles/nginx/files/ssl_modern.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/nginx/files/ssl_modern.conf b/ansible/roles/nginx/files/ssl_modern.conf index 9ad7c11d..538d3097 100644 --- a/ansible/roles/nginx/files/ssl_modern.conf +++ b/ansible/roles/nginx/files/ssl_modern.conf @@ -1,4 +1,4 @@ # Oldest compatible clients: Firefox 27, Chrome 30, IE 11 on Windows 7, Edge, Opera 17, Safari 9, Android 5.0, and Java 8 -ssl_protocols TLSv1.2; +ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; # NB: technically, it does not require ssl_dhparam as it has no DHE, only ECDHE. From 3d7e2fa6ba9c3a9195530d83ee3b5668ad65dd68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 19 Aug 2026 11:39:58 +0200 Subject: [PATCH 132/201] Point echo and json test helpers to DO hosts --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 5f089f5c..c2d478c8 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -563,7 +563,7 @@ module "ooniapi_ooniprobe_deployer" { service_name = "ooniprobe" repo = "ooni/backend" - branch_name = "feat/enable-webconnectivity-lte" + branch_name = "1237-check-in-to-th" environment = local.environment trigger_path = "ooniapi/services/ooniprobe/**" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" From 1291da94cfcf06e79577186b0adf716af6e3c3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 13:48:38 +0200 Subject: [PATCH 133/201] Add support for setting custom https port in node_exporter config --- .../defaults/main.yml | 4 +++ .../handlers/main.yml | 6 +++++ .../prometheus_node_exporter/tasks/main.yml | 26 +++++++++++++++++++ .../templates/nginx-prometheus.j2 | 8 +++--- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ansible/roles/prometheus_node_exporter/defaults/main.yml b/ansible/roles/prometheus_node_exporter/defaults/main.yml index 9066e94c..0cdc0011 100644 --- a/ansible/roles/prometheus_node_exporter/defaults/main.yml +++ b/ansible/roles/prometheus_node_exporter/defaults/main.yml @@ -18,3 +18,7 @@ node_exporter_options: '' node_exporter_state: started node_exporter_enabled: true node_exporter_restart: on-failure +http_port: 9001 +https_port: 443 +use_https: false +monitoring_server_ip: "{{ lookup('dig', 'monitoring.ooni.org') }}" diff --git a/ansible/roles/prometheus_node_exporter/handlers/main.yml b/ansible/roles/prometheus_node_exporter/handlers/main.yml index 4ec66003..092c7285 100644 --- a/ansible/roles/prometheus_node_exporter/handlers/main.yml +++ b/ansible/roles/prometheus_node_exporter/handlers/main.yml @@ -18,3 +18,9 @@ service: name: node_exporter state: restarted + +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index 7512d5b9..35e32438 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -23,4 +23,30 @@ - config when: use_nginx +- name: Allow traffic on the node exporter http_port from the monitoring host only + become: true + tags: [prometheus_node_exporter, monitoring, node_exporter, nftables] + ansible.builtin.blockinfile: + path: "/etc/ooni/nftables/tcp/{{ http_port }}.nft" + create: yes + mode: "0644" + block: | + add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport {{ http_port }} counter accept comment "node exporter metrics" + when: use_nginx and not use_https + notify: + - reload nftables + +- name: Allow traffic on the node exporter https_port from the monitoring host only + become: true + tags: [prometheus_node_exporter, monitoring, node_exporter, nftables] + ansible.builtin.blockinfile: + path: "/etc/ooni/nftables/tcp/{{ http_port }}.nft" + create: yes + mode: "0644" + block: | + add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport {{ https_port }} counter accept comment "node exporter metrics" + when: use_nginx and use_https + notify: + - reload nftables + - include_tasks: install.yml diff --git a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 index 6fb7c5ca..5af3c130 100644 --- a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 +++ b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 @@ -2,7 +2,7 @@ server { {% if use_https %} - listen 443 ssl http2; + listen {{ https_port }} ssl http2; server_name {{ inventory_hostname }}; include /etc/nginx/ssl_intermediate.conf; @@ -11,9 +11,9 @@ server { ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; {% else %} - listen {{http_port}}; + listen {{ http_port }}; - server_name {{inventory_hostname}}; + server_name {{ inventory_hostname }}; {% endif %} {% for config in prometheus_nginx_proxy_config %} @@ -30,4 +30,4 @@ server { } {% endfor %} -} \ No newline at end of file +} From f57b9e91a2003c78a72197c8858faba3f2c613dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 13:49:09 +0200 Subject: [PATCH 134/201] Add oohelpderd role --- ansible/roles/oohelperd/defaults/main.yml | 10 ++ ansible/roles/oohelperd/handlers/main.yml | 4 + ansible/roles/oohelperd/tasks/main.yml | 93 +++++++++++++++++++ .../oohelperd/templates/nginx-oohelperd.j2 | 73 +++++++++++++++ .../roles/oohelperd/templates/oohelperd.env | 5 + .../oohelperd/templates/oohelperd.service | 30 ++++++ 6 files changed, 215 insertions(+) create mode 100644 ansible/roles/oohelperd/defaults/main.yml create mode 100644 ansible/roles/oohelperd/handlers/main.yml create mode 100644 ansible/roles/oohelperd/tasks/main.yml create mode 100644 ansible/roles/oohelperd/templates/nginx-oohelperd.j2 create mode 100644 ansible/roles/oohelperd/templates/oohelperd.env create mode 100644 ansible/roles/oohelperd/templates/oohelperd.service diff --git a/ansible/roles/oohelperd/defaults/main.yml b/ansible/roles/oohelperd/defaults/main.yml new file mode 100644 index 00000000..1c4f7a86 --- /dev/null +++ b/ansible/roles/oohelperd/defaults/main.yml @@ -0,0 +1,10 @@ +# The Web Connectivity test helper (oohelperd) is built and released as part of +# https://github.com/ooni/probe-cli +oohelperd_version: v3.30.0 +oohelperd_base_url: "https://github.com/ooni/probe-cli/releases/download/{{ oohelperd_version }}" +# sha256 of {{ oohelperd_base_url }}/oohelperd-linux-amd64 +oohelperd_checksum: "fff38b9366fecae400cb3f616ffa2a37dba211c33ee4e1c1b343bff1c384d48e" + +# A test helper request measures the target before it can answer, so it needs +# way more than the nginx default of 60s +oohelperd_nginx_proxy_timeout: 900 diff --git a/ansible/roles/oohelperd/handlers/main.yml b/ansible/roles/oohelperd/handlers/main.yml new file mode 100644 index 00000000..71ffc659 --- /dev/null +++ b/ansible/roles/oohelperd/handlers/main.yml @@ -0,0 +1,4 @@ +- name: reload nginx + service: + name: nginx + state: reloaded diff --git a/ansible/roles/oohelperd/tasks/main.yml b/ansible/roles/oohelperd/tasks/main.yml new file mode 100644 index 00000000..c20d5e63 --- /dev/null +++ b/ansible/roles/oohelperd/tasks/main.yml @@ -0,0 +1,93 @@ +--- +- name: Flush all handlers now + ansible.builtin.meta: flush_handlers + +- name: Create the oohelperd user + tags: [oohelperd] + ansible.builtin.user: + name: oohelperd + shell: /usr/sbin/nologin + create_home: no + system: yes + +- name: Download the oohelperd binary + tags: [oohelperd] + ansible.builtin.get_url: + url: "{{ oohelperd_base_url }}/oohelperd-linux-amd64" + dest: /usr/local/bin/oohelperd + checksum: "sha256:{{ oohelperd_checksum }}" + mode: "0755" + owner: root + register: oohelperd_binary + +- name: Create the ooni configuration directory + tags: [oohelperd, config] + ansible.builtin.file: + path: /etc/ooni + state: directory + owner: root + mode: "0755" + +# Read by systemd as root, hence not readable by the oohelperd user itself +- name: Write the oohelperd environment file + tags: [oohelperd, secrets] + no_log: true + ansible.builtin.template: + src: oohelperd.env + dest: /etc/ooni/oohelperd.env + owner: root + group: root + mode: "0600" + register: oohelperd_env + +- name: Create the oohelperd.service file + tags: [oohelperd] + ansible.builtin.template: + src: oohelperd.service + dest: /etc/systemd/system/oohelperd.service + mode: "0644" + owner: root + register: oohelperd_unit + +- name: reload systemd + tags: [oohelperd] + ansible.builtin.systemd_service: + daemon_reload: yes + when: oohelperd_unit.changed + +- name: Start oohelperd + tags: [oohelperd] + ansible.builtin.systemd_service: + name: oohelperd.service + state: started + enabled: yes + +- name: Restart oohelperd if its binary, unit file or environment changed + tags: [oohelperd] + ansible.builtin.systemd_service: + name: oohelperd.service + state: restarted + when: oohelperd_binary.changed or oohelperd_unit.changed or oohelperd_env.changed + +# Expose the test helper to the probes +- name: Add the oohelperd nginx vhost + tags: [oohelperd, nginx, config] + ansible.builtin.template: + src: nginx-oohelperd.j2 + dest: /etc/nginx/sites-enabled/02-oohelperd + mode: "0644" + owner: root + notify: + - reload nginx + +- name: Verify oohelperd is answering requests + tags: [oohelperd] + # GET / is oohelperd's health check, it answers "Hello OONItarian!" + ansible.builtin.uri: + url: http://127.0.0.1:8080/ + return_content: true + register: oohelperd_check + when: not ansible_check_mode + until: "'OONItarian' in oohelperd_check.content" + retries: 5 + delay: 2 diff --git a/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 new file mode 100644 index 00000000..0a97ae10 --- /dev/null +++ b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 @@ -0,0 +1,73 @@ +# Generated by ansible +# roles/oohelperd/templates/nginx-oohelperd.j2 + +# The probes send the same measurement request to the test helper over and +# over, so responses are cached, keyed by the request body since they are POSTs +proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M + max_size=5g inactive=24h use_temp_path=off; + +server { + listen 80; + listen [::]:80; + server_name {{ inventory_hostname }}; + + # This server block is more specific than the catch-all one installed by + # the dehydrated role, so it must serve the ACME challenge itself + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + # nginx >= 1.25.1, which is what nginx.org ships + http2 on; + + server_name {{ inventory_hostname }}; + include /etc/nginx/ssl_modern.conf; + + ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; + + # Test helper application metrics. oohelperd protects them with basic auth + # using the password in /etc/ooni/oohelperd.env (user "prom"). Kept out of + # the cache so that prometheus always scrapes fresh values. + location = /metrics { + proxy_pass http://127.0.0.1:8080; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Local test helper + location / { + proxy_pass http://127.0.0.1:8080; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout {{ oohelperd_nginx_proxy_timeout }}; + + proxy_cache thcache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + # Cache POST without headers set by the test helper! + proxy_cache_methods POST; + proxy_cache_key "$request_uri|$request_body"; + proxy_cache_valid 200 10m; + proxy_cache_valid any 0; + add_header X-Cache-Status $upstream_cache_status; + } +} diff --git a/ansible/roles/oohelperd/templates/oohelperd.env b/ansible/roles/oohelperd/templates/oohelperd.env new file mode 100644 index 00000000..80167997 --- /dev/null +++ b/ansible/roles/oohelperd/templates/oohelperd.env @@ -0,0 +1,5 @@ +# Generated by ansible +# roles/oohelperd/templates/oohelperd.env + +# Password for the "prom" user on oohelperd's own /metrics endpoint +PROMETHEUS_METRICS_PASSWORD={{ prometheus_metrics_password }} diff --git a/ansible/roles/oohelperd/templates/oohelperd.service b/ansible/roles/oohelperd/templates/oohelperd.service new file mode 100644 index 00000000..b09d034d --- /dev/null +++ b/ansible/roles/oohelperd/templates/oohelperd.service @@ -0,0 +1,30 @@ +# Generated by ansible +# roles/oohelperd/templates/oohelperd.service + +[Unit] +Description=OONI Web Connectivity test helper +Documentation=https://github.com/ooni/probe-cli +After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +Type=simple +# nginx terminates TLS in front of it, so it only listens on loopback +ExecStart=/usr/local/bin/oohelperd -api-endpoint 127.0.0.1:8080 -pprof-endpoint 127.0.0.1:6061 + +# Holds PROMETHEUS_METRICS_PASSWORD, used to protect /metrics +EnvironmentFile=/etc/ooni/oohelperd.env +Restart=on-failure +RestartSec=5 +User=oohelperd +Group=oohelperd +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +PrivateTmp=yes +# Every measurement opens several connections towards the target +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target From 8b4b4ff39932f805f895ae796f6b905effecf888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 14:35:02 +0200 Subject: [PATCH 135/201] Update deploy script --- ansible/deploy-wc-test-helper.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ansible/deploy-wc-test-helper.yml b/ansible/deploy-wc-test-helper.yml index 1b0ce980..b3cee6ff 100644 --- a/ansible/deploy-wc-test-helper.yml +++ b/ansible/deploy-wc-test-helper.yml @@ -1,5 +1,5 @@ --- -- name: Deploy test helpers +- name: Deploy the Web Connectivity test helper hosts: - wc.th.dev.ooni.io become: true @@ -12,3 +12,7 @@ - role: nginx - role: dehydrated - role: prometheus_node_exporter + vars: + use_https: true + https_port: 9001 + - role: oohelperd From 9c0998800e14bf660c36267c611f883a4e1f89ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 16:35:38 +0200 Subject: [PATCH 136/201] Add support for deploying prod test helpers --- tf/environments/prod/main.tf | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 801ee5f9..142e0c1f 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -442,6 +442,27 @@ module "ooni_th_droplet" { dns_zone_ooni_io = local.dns_zone_ooni_io } +resource "digitalocean_ssh_key" "oonidevops" { + name = "oonidevops" + public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] +} + + +module "ooni_test_helpers_wc" { + source = "../../modules/ooni_th_binary_droplet" + + stage = local.environment + instance_location = "fra1" + name = "ooniwcth" + hostname = "wcth${count.index}.fra1" + + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] + + dns_zone_ooni_io = local.dns_zone_ooni_io + count = 3 +} + + module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" From babb26efddb4bf078d5b3ed652fce9117597c338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 16:36:22 +0200 Subject: [PATCH 137/201] Remove duplcate key definition --- tf/environments/prod/main.tf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 142e0c1f..4185f7a5 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -442,12 +442,6 @@ module "ooni_th_droplet" { dns_zone_ooni_io = local.dns_zone_ooni_io } -resource "digitalocean_ssh_key" "oonidevops" { - name = "oonidevops" - public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] -} - - module "ooni_test_helpers_wc" { source = "../../modules/ooni_th_binary_droplet" From ed2e4bb9f3d1185b16021218b2df4990779509a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 16:36:43 +0200 Subject: [PATCH 138/201] terraform fmt --- tf/environments/prod/main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 4185f7a5..2553f6de 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -445,15 +445,15 @@ module "ooni_th_droplet" { module "ooni_test_helpers_wc" { source = "../../modules/ooni_th_binary_droplet" - stage = local.environment + stage = local.environment instance_location = "fra1" - name = "ooniwcth" - hostname = "wcth${count.index}.fra1" + name = "ooniwcth" + hostname = "wcth${count.index}.fra1" ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] dns_zone_ooni_io = local.dns_zone_ooni_io - count = 3 + count = 3 } From 790faf54c844947f9d51fd0cc202207fa7679ba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 16:40:29 +0200 Subject: [PATCH 139/201] Add wcth[0-2] to inventory --- ansible/inventory | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/inventory b/ansible/inventory index bcc6e3d1..0abceec9 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -37,6 +37,9 @@ ams-ps.ooni.nu [do_fra] wc.th.dev.ooni.io +wcth0.fra1.dev.ooni.io +wcth1.fra1.dev.ooni.io +wcth2.fra1.dev.ooni.io [aws-proxy] clickhouseproxy.dev.ooni.io From 244830d2bc251af6b691d3507bac0e53439bdefc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 16:41:17 +0200 Subject: [PATCH 140/201] Fix inventory lines --- ansible/inventory | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/inventory b/ansible/inventory index 0abceec9..754130cd 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -37,9 +37,9 @@ ams-ps.ooni.nu [do_fra] wc.th.dev.ooni.io -wcth0.fra1.dev.ooni.io -wcth1.fra1.dev.ooni.io -wcth2.fra1.dev.ooni.io +wcth0.fra1.prod.ooni.io +wcth1.fra1.prod.ooni.io +wcth2.fra1.prod.ooni.io [aws-proxy] clickhouseproxy.dev.ooni.io From e7615283c538a8b62b0ed1697368ab42129147e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 16:46:45 +0200 Subject: [PATCH 141/201] Add prod test helpers to deployment --- ansible/deploy-wc-test-helper.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/deploy-wc-test-helper.yml b/ansible/deploy-wc-test-helper.yml index b3cee6ff..9b415ddd 100644 --- a/ansible/deploy-wc-test-helper.yml +++ b/ansible/deploy-wc-test-helper.yml @@ -2,6 +2,9 @@ - name: Deploy the Web Connectivity test helper hosts: - wc.th.dev.ooni.io + - wcth0.fra1.prod.ooni.io + - wcth1.fra1.prod.ooni.io + - wcth2.fra1.prod.ooni.io become: true vars: ssl_domains: From 0414ed11c63d84c6f44fbec033735cecc0d34ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 16:54:14 +0200 Subject: [PATCH 142/201] Put flush all handlers sooner in the dehydrated role --- ansible/roles/dehydrated/tasks/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/roles/dehydrated/tasks/main.yml b/ansible/roles/dehydrated/tasks/main.yml index 860ffcc3..4dbd6c1a 100644 --- a/ansible/roles/dehydrated/tasks/main.yml +++ b/ansible/roles/dehydrated/tasks/main.yml @@ -92,4 +92,7 @@ # crashing the playbook # # See: https://github.com/ooni/devops/pull/235#discussion_r2053664605 - - reload nginx and restart dehydrated \ No newline at end of file + - reload nginx and restart dehydrated + +- name: Flush all handlers + meta: flush_handlers From 40668834b0335c7fca73e5aef404d000e0065488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 19 Aug 2026 16:55:22 +0200 Subject: [PATCH 143/201] Remove monitoring of old test helpers --- ansible/roles/prometheus/vars/main.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index 1da7f053..26da57a2 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -1,8 +1,6 @@ dom0_hosts: - ams-slack-1.ooni.org - doams1-countly.ooni.nu - - mia-echoth.ooni.nu - - mia-httpth.ooni.nu blackbox_jobs: # TODO add these records to the ALB config @@ -48,18 +46,6 @@ blackbox_jobs: # cloudfront - "https://d3kr4emv7f56qa.cloudfront.net/bouncer/net-tests" - # IP addresses are used for test-helpers in monitoring configuration for some - # historical reason hopefully remembered by @hellais. - - name: "ooni tcp echo" - module: "ooni_tcp_echo_ok" - targets: - - "{{ lookup('dig', 'c.echo.th.ooni.io/A') }}:80" - - - name: "ooni http return json headers" - module: "ooni_http_return_json_headers_ok" - targets: - - "http://{{ lookup('dig', 'a.http.th.ooni.io/A') }}:80" - - name: "ooni explorer homepage" module: "http_2xx" targets: From b477e78f33007326f684d96f660b7580ac921e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 16:58:16 +0200 Subject: [PATCH 144/201] Set instance name in terraform config --- tf/environments/prod/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 2553f6de..f167fe37 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -447,7 +447,7 @@ module "ooni_test_helpers_wc" { stage = local.environment instance_location = "fra1" - name = "ooniwcth" + name = "ooniwcth${count.index}-fra1" hostname = "wcth${count.index}.fra1" ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] From 1435e1d113263fb5747d4265c80a7376880660fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 17:08:31 +0200 Subject: [PATCH 145/201] More debugging of dehydrated role --- ansible/roles/dehydrated/handlers/main.yml | 12 +++++----- ansible/roles/dehydrated/tasks/main.yml | 26 +++++++++++++--------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/ansible/roles/dehydrated/handlers/main.yml b/ansible/roles/dehydrated/handlers/main.yml index a539854a..71d72a81 100644 --- a/ansible/roles/dehydrated/handlers/main.yml +++ b/ansible/roles/dehydrated/handlers/main.yml @@ -3,6 +3,11 @@ name: nginx state: reloaded +- name: restart nginx + service: + name: nginx + state: restarted + - name: reload nftables service: name: nftables @@ -13,10 +18,3 @@ name: dehydrated state: restarted enabled: yes - -- name: reload nginx and restart dehydrated - service: - name: nginx - state: reloaded - notify: - restart dehydrated \ No newline at end of file diff --git a/ansible/roles/dehydrated/tasks/main.yml b/ansible/roles/dehydrated/tasks/main.yml index 4dbd6c1a..56af5a5d 100644 --- a/ansible/roles/dehydrated/tasks/main.yml +++ b/ansible/roles/dehydrated/tasks/main.yml @@ -82,17 +82,21 @@ state: started enabled: yes notify: - # creates: - # /var/lib/dehydrated/certs//chain.pem cert.pem privkey.pem fullchain.pem - - # Note that we need to restart dehydrated ensuring that nginx reloads before dehydrated restarts. - # When we first run dehydrated with the tasks above it creates an nginx rule that is required - # to pass the ACME challenge. - # If nginx doesn't picks this rule before dehydrated runs again, the ACME challenge will fail - # crashing the playbook - # - # See: https://github.com/ooni/devops/pull/235#discussion_r2053664605 - - reload nginx and restart dehydrated + - restart nginx - name: Flush all handlers + tags: [dehydrated, certs] meta: flush_handlers + +- name: Ensure nginx is running + tags: [dehydrated, certs] + ansible.builtin.systemd_service: + state: started + name: nginx + +- name: Force the restarting of the dehydrated service + tags: [dehydrated, certs] + ansible.builtin.systemd_service: + state: restarted + daemon_reload: true + name: dehydrated From b8e59566034b9b18ab30494dae5375da9c18a831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 17:15:19 +0200 Subject: [PATCH 146/201] Add new test helpers to monitoring --- ansible/roles/prometheus/templates/prometheus.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index 2790f27e..1e1d2028 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -208,6 +208,9 @@ scrape_configs: - 0.do.th.prod.ooni.io - 1.do.th.prod.ooni.io - 2.do.th.prod.ooni.io + - wcth0.fra1.prod.ooni.io + - wcth1.fra1.prod.ooni.io + - wcth2.fra1.prod.ooni.io - job_name: 'ooni-web' scrape_interval: 5m scheme: https From 8009f9db94a13b3da4882be4e308266e6e9927a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 17:16:05 +0200 Subject: [PATCH 147/201] Add test helpers to monitoring --- ansible/roles/prometheus/vars/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index 1da7f053..1e819869 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -25,6 +25,9 @@ blackbox_jobs: - "https://4.th.ooni.org/" - "https://5.th.ooni.org/" - "https://6.th.ooni.org/" + - "https://wcth0.fra1.prod.ooni.io/" + - "https://wcth1.fra1.prod.ooni.io/" + - "https://wcth2.fra1.prod.ooni.io/" - "https://d33d1gs9kpq1c5.cloudfront.net/status" - name: "ooni collector" From 085b0f869dbac69518404d89f8137597c86cddde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 19 Aug 2026 17:57:06 +0200 Subject: [PATCH 148/201] Fix pointers for notebook server --- ansible/roles/prometheus/templates/prometheus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index 1e1d2028..eb854621 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -83,7 +83,7 @@ scrape_configs: - https://data1.htz-fsn.prod.ooni.nu/metrics/node_exporter - https://data2.htz-fsn.prod.ooni.nu/metrics/node_exporter - https://data3.htz-fsn.prod.ooni.nu/metrics/node_exporter - - https://notebook.ooni.org/metrics/node_exporter + - https://notebook1.htz-fsn.prod.ooni.nu/metrics/node_exporter - http://0.do.th.prod.ooni.io:9001/metrics - http://1.do.th.prod.ooni.io:9001/metrics - http://2.do.th.prod.ooni.io:9001/metrics @@ -154,7 +154,7 @@ scrape_configs: - data1.htz-fsn.prod.ooni.nu - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu - - notebook.ooni.org + - notebook1.htz-fsn.prod.ooni.nu - job_name: 'raw-netdata' scrape_interval: 5s From f289d464803a068c0b5868b657732f7da17f5569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 20 Aug 2026 11:35:52 +0200 Subject: [PATCH 149/201] Add new test helpers checks --- ansible/roles/prometheus/files/alert_ooni.yml | 6 ++++++ ansible/roles/prometheus/vars/main.yml | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/ansible/roles/prometheus/files/alert_ooni.yml b/ansible/roles/prometheus/files/alert_ooni.yml index e43397ed..a84b2732 100644 --- a/ansible/roles/prometheus/files/alert_ooni.yml +++ b/ansible/roles/prometheus/files/alert_ooni.yml @@ -27,4 +27,10 @@ groups: summary: 'Mirror {{ $labels.instance }} lags by {{ $value | humanizeDuration }}' description: '{{ if eq $labels.instance "ooni.torproject.org:443" }}Run `make update-site`.{{ end }}' + - alert: TestHelperDown + expr: up{domain!=""} != 1 + for: 5m + annotations: + summary: 'test helper down: {{ $labels.instance }} ({{ $labels.domain }})' + ... diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index 26da57a2..c0cdea9f 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -51,6 +51,26 @@ blackbox_jobs: targets: - "https://explorer.ooni.org/" + # IP addresses are used for these test-helpers since that's what is handed + # out to probes by check-in. Keep them in sync with + # `generate_test_helpers_conf()` in ooni/backend + # ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py + - name: "ooni tcp echo" + module: "ooni_tcp_echo_ok" + ignore_instance_down: true # covered by the more specific TestHelperDown alert + labels: + domain: "echo.th.prod.ooni.io" + targets: + - "134.209.237.204:80" # echo.th.prod.ooni.io, tcp-echo test helper + + - name: "ooni http return json headers" + module: "ooni_http_return_json_headers_ok" + ignore_instance_down: true # covered by the more specific TestHelperDown alert + labels: + domain: "json.th.prod.ooni.io" + targets: + - "http://206.81.31.205:80" # json.th.prod.ooni.io + # API # - name: "ooni API measurements" From 49bc1d9d4e0bce61e15a2f44f601791435a94a47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 20 Aug 2026 12:10:10 +0200 Subject: [PATCH 150/201] Lookup ip of test helper instead of hardcoding it --- ansible/roles/prometheus/vars/main.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index d50e0c8a..e9db75b1 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -54,17 +54,14 @@ blackbox_jobs: targets: - "https://explorer.ooni.org/" - # IP addresses are used for these test-helpers since that's what is handed - # out to probes by check-in. Keep them in sync with - # `generate_test_helpers_conf()` in ooni/backend - # ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py + # We use IP monitoring since this is what is served to users through check-in. - name: "ooni tcp echo" module: "ooni_tcp_echo_ok" ignore_instance_down: true # covered by the more specific TestHelperDown alert labels: domain: "echo.th.prod.ooni.io" targets: - - "134.209.237.204:80" # echo.th.prod.ooni.io, tcp-echo test helper + - "{{ lookup('dig', 'echo.th.prod.ooni.io/A') }}:80" - name: "ooni http return json headers" module: "ooni_http_return_json_headers_ok" @@ -72,7 +69,7 @@ blackbox_jobs: labels: domain: "json.th.prod.ooni.io" targets: - - "http://206.81.31.205:80" # json.th.prod.ooni.io + - "http://{{ lookup('dig', 'json.th.prod.ooni.io/A') }}:80" # API # From 179451912102a82dc1f6b6763fcd1fdcbe8612db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 20 Aug 2026 13:34:12 +0200 Subject: [PATCH 151/201] Add public fqdn for test helper addresses --- tf/environments/prod/main.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index f167fe37..b6980c4b 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -456,6 +456,15 @@ module "ooni_test_helpers_wc" { count = 3 } +resource "aws_route53_record" "ooni_wc_th" { + zone_id = local.dns_root_zone_ooni_org + name = "wcth${count.index}.fra1.ooni.org" + type = "A" + ttl = 60 + + count = 3 + records = [module.ooni_test_helpers_wc[count.index].droplet_ipv4_address] +} module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" From c864901b2a0644b2d948ce35435de0680fe470ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 20 Aug 2026 13:35:54 +0200 Subject: [PATCH 152/201] Pass ssl_domains from host_vars --- ansible/deploy-wc-test-helper.yml | 2 -- ansible/host_vars/wcth0.fra1.prod.ooni.io | 3 +++ ansible/host_vars/wcth1.fra1.prod.ooni.io | 3 +++ ansible/host_vars/wcth2.fra1.prod.ooni.io | 3 +++ 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 ansible/host_vars/wcth0.fra1.prod.ooni.io create mode 100644 ansible/host_vars/wcth1.fra1.prod.ooni.io create mode 100644 ansible/host_vars/wcth2.fra1.prod.ooni.io diff --git a/ansible/deploy-wc-test-helper.yml b/ansible/deploy-wc-test-helper.yml index 9b415ddd..9b65e354 100644 --- a/ansible/deploy-wc-test-helper.yml +++ b/ansible/deploy-wc-test-helper.yml @@ -7,8 +7,6 @@ - wcth2.fra1.prod.ooni.io become: true vars: - ssl_domains: - - "{{ inventory_hostname }}" nginx_install_method: official_repo roles: - role: bootstrap diff --git a/ansible/host_vars/wcth0.fra1.prod.ooni.io b/ansible/host_vars/wcth0.fra1.prod.ooni.io new file mode 100644 index 00000000..42dae12d --- /dev/null +++ b/ansible/host_vars/wcth0.fra1.prod.ooni.io @@ -0,0 +1,3 @@ +ssl_domains: +- wcth0.fra1.prod.ooni.io +- wcth0.fra1.ooni.org diff --git a/ansible/host_vars/wcth1.fra1.prod.ooni.io b/ansible/host_vars/wcth1.fra1.prod.ooni.io new file mode 100644 index 00000000..ea1b7d98 --- /dev/null +++ b/ansible/host_vars/wcth1.fra1.prod.ooni.io @@ -0,0 +1,3 @@ +ssl_domains: +- wcth1.fra1.prod.ooni.io +- wcth1.fra1.ooni.org diff --git a/ansible/host_vars/wcth2.fra1.prod.ooni.io b/ansible/host_vars/wcth2.fra1.prod.ooni.io new file mode 100644 index 00000000..4f0e1f0e --- /dev/null +++ b/ansible/host_vars/wcth2.fra1.prod.ooni.io @@ -0,0 +1,3 @@ +ssl_domains: +- wcth2.fra1.prod.ooni.io +- wcth2.fra1.ooni.org From c868a4265f6c3e2d31de8f25b82d4d7490e056ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 20 Aug 2026 13:43:31 +0200 Subject: [PATCH 153/201] Build nginx config from ssl_domains --- ansible/deploy-wc-test-helper.yml | 2 ++ ansible/roles/oohelperd/templates/nginx-oohelperd.j2 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/deploy-wc-test-helper.yml b/ansible/deploy-wc-test-helper.yml index 9b65e354..0be5c91d 100644 --- a/ansible/deploy-wc-test-helper.yml +++ b/ansible/deploy-wc-test-helper.yml @@ -8,6 +8,8 @@ become: true vars: nginx_install_method: official_repo + ssl_domains: + - "{{ inventory_hostname }}" roles: - role: bootstrap - role: nginx diff --git a/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 index 0a97ae10..62803fc1 100644 --- a/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 +++ b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 @@ -28,7 +28,7 @@ server { # nginx >= 1.25.1, which is what nginx.org ships http2 on; - server_name {{ inventory_hostname }}; + server_name {{ ssl_domains | join (' ') }}; include /etc/nginx/ssl_modern.conf; ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; From 9b42b2910e8c1153b1a3f0afeeb4dd872bd37541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 20 Aug 2026 13:46:15 +0200 Subject: [PATCH 154/201] Fix ssl_domains host var precedence --- ansible/deploy-wc-test-helper.yml | 2 -- ansible/host_vars/wc.th.dev.ooni.io | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 ansible/host_vars/wc.th.dev.ooni.io diff --git a/ansible/deploy-wc-test-helper.yml b/ansible/deploy-wc-test-helper.yml index 0be5c91d..9b65e354 100644 --- a/ansible/deploy-wc-test-helper.yml +++ b/ansible/deploy-wc-test-helper.yml @@ -8,8 +8,6 @@ become: true vars: nginx_install_method: official_repo - ssl_domains: - - "{{ inventory_hostname }}" roles: - role: bootstrap - role: nginx diff --git a/ansible/host_vars/wc.th.dev.ooni.io b/ansible/host_vars/wc.th.dev.ooni.io new file mode 100644 index 00000000..971935a8 --- /dev/null +++ b/ansible/host_vars/wc.th.dev.ooni.io @@ -0,0 +1,2 @@ +ssl_domains: +- wc.th.dev.ooni.io From bc4453c1f29306deb5c32ae85829a9418668ad65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 20 Aug 2026 13:50:47 +0200 Subject: [PATCH 155/201] Write ssl_domains on the same line in dehydrated --- ansible/roles/dehydrated/templates/domains.txt.j2 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ansible/roles/dehydrated/templates/domains.txt.j2 b/ansible/roles/dehydrated/templates/domains.txt.j2 index 4b4a6c0e..b264f25d 100644 --- a/ansible/roles/dehydrated/templates/domains.txt.j2 +++ b/ansible/roles/dehydrated/templates/domains.txt.j2 @@ -1,3 +1 @@ -{% for d in ssl_domains %} -{{ d }} -{% endfor %} +{% ssl_domains | join(' ') %} From fbd7c8a7be3b01cd62987e1d142bd9ed74ccc07b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 20 Aug 2026 13:52:01 +0200 Subject: [PATCH 156/201] Revert change to domains.txt --- ansible/roles/dehydrated/templates/domains.txt.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/dehydrated/templates/domains.txt.j2 b/ansible/roles/dehydrated/templates/domains.txt.j2 index b264f25d..4cc1ec59 100644 --- a/ansible/roles/dehydrated/templates/domains.txt.j2 +++ b/ansible/roles/dehydrated/templates/domains.txt.j2 @@ -1 +1,3 @@ -{% ssl_domains | join(' ') %} +{% ssl_domains %} +{{ d }} +{% endfor %} From b9e9b76201b90bd04fe5d6e8e8c6db65d9444bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 20 Aug 2026 15:32:19 +0200 Subject: [PATCH 157/201] Fix server_name mappings and dehydrated setup for alternate hostnames --- ansible/host_vars/wcth0.fra1.prod.ooni.io | 4 ++-- ansible/host_vars/wcth1.fra1.prod.ooni.io | 4 ++-- ansible/host_vars/wcth2.fra1.prod.ooni.io | 4 ++-- ansible/roles/oohelperd/defaults/main.yml | 4 ++++ ansible/roles/oohelperd/templates/nginx-oohelperd.j2 | 4 ++-- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ansible/host_vars/wcth0.fra1.prod.ooni.io b/ansible/host_vars/wcth0.fra1.prod.ooni.io index 42dae12d..8300c119 100644 --- a/ansible/host_vars/wcth0.fra1.prod.ooni.io +++ b/ansible/host_vars/wcth0.fra1.prod.ooni.io @@ -1,3 +1,3 @@ ssl_domains: -- wcth0.fra1.prod.ooni.io -- wcth0.fra1.ooni.org +- "wcth0.fra1.prod.ooni.io wcth0.fra1.ooni.org" +nginx_server_name: "wcth0.fra1.prod.ooni.io wcth0.fra1.ooni.org" diff --git a/ansible/host_vars/wcth1.fra1.prod.ooni.io b/ansible/host_vars/wcth1.fra1.prod.ooni.io index ea1b7d98..78d0b0e6 100644 --- a/ansible/host_vars/wcth1.fra1.prod.ooni.io +++ b/ansible/host_vars/wcth1.fra1.prod.ooni.io @@ -1,3 +1,3 @@ ssl_domains: -- wcth1.fra1.prod.ooni.io -- wcth1.fra1.ooni.org +- "wcth1.fra1.prod.ooni.io wcth1.fra1.ooni.org" +nginx_server_name: "wcth1.fra1.prod.ooni.io wcth1.fra1.ooni.org" diff --git a/ansible/host_vars/wcth2.fra1.prod.ooni.io b/ansible/host_vars/wcth2.fra1.prod.ooni.io index 4f0e1f0e..2c248673 100644 --- a/ansible/host_vars/wcth2.fra1.prod.ooni.io +++ b/ansible/host_vars/wcth2.fra1.prod.ooni.io @@ -1,3 +1,3 @@ ssl_domains: -- wcth2.fra1.prod.ooni.io -- wcth2.fra1.ooni.org +- "wcth2.fra1.prod.ooni.io wcth2.fra1.ooni.org" +nginx_server_name: "wcth2.fra1.prod.ooni.io wcth2.fra1.ooni.org" diff --git a/ansible/roles/oohelperd/defaults/main.yml b/ansible/roles/oohelperd/defaults/main.yml index 1c4f7a86..e3696f33 100644 --- a/ansible/roles/oohelperd/defaults/main.yml +++ b/ansible/roles/oohelperd/defaults/main.yml @@ -8,3 +8,7 @@ oohelperd_checksum: "fff38b9366fecae400cb3f616ffa2a37dba211c33ee4e1c1b343bff1c38 # A test helper request measures the target before it can answer, so it needs # way more than the nginx default of 60s oohelperd_nginx_proxy_timeout: 900 +# It's important that when you generate certificate using dehydrated, the first +# domain in the list matches inventory_hostname so that the lookup for the +# certificate that was generated using dehydrated matches +nginx_server_name: "{{ inventory_hostname }}" diff --git a/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 index 62803fc1..fd6480bd 100644 --- a/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 +++ b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 @@ -9,7 +9,7 @@ proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M server { listen 80; listen [::]:80; - server_name {{ inventory_hostname }}; + server_name {{ nginx_server_name }}; # This server block is more specific than the catch-all one installed by # the dehydrated role, so it must serve the ACME challenge itself @@ -28,7 +28,7 @@ server { # nginx >= 1.25.1, which is what nginx.org ships http2 on; - server_name {{ ssl_domains | join (' ') }}; + server_name {{ nginx_server_name }}; include /etc/nginx/ssl_modern.conf; ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; From 6cadabd2cfee707fc5d12ab9b889bc1581da0566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 20 Aug 2026 15:35:13 +0200 Subject: [PATCH 158/201] Fix jinja template for loop --- ansible/roles/dehydrated/templates/domains.txt.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/dehydrated/templates/domains.txt.j2 b/ansible/roles/dehydrated/templates/domains.txt.j2 index 4cc1ec59..4b4a6c0e 100644 --- a/ansible/roles/dehydrated/templates/domains.txt.j2 +++ b/ansible/roles/dehydrated/templates/domains.txt.j2 @@ -1,3 +1,3 @@ -{% ssl_domains %} +{% for d in ssl_domains %} {{ d }} {% endfor %} From 25847f27ffac886984284f7377aebfe10f164551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 2 Sep 2026 15:55:31 +0200 Subject: [PATCH 159/201] Setup load balancer rules to map to the legacy ooniprobe based on header --- tf/environments/dev/main.tf | 53 ++++++++++++++++++- tf/modules/ooniapi_frontend/main.tf | 65 ++++++++++++++++++++++-- tf/modules/ooniapi_frontend/variables.tf | 4 ++ 3 files changed, 117 insertions(+), 5 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 95826431..b366b875 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -581,7 +581,7 @@ module "ooniapi_ooniprobe_deployer" { service_name = "ooniprobe" repo = "ooni/backend" - branch_name = "1237-check-in-to-th" + branch_name = "master" environment = local.environment trigger_path = "ooniapi/services/ooniprobe/**" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" @@ -651,6 +651,56 @@ module "ooniapi_ooniprobe" { ) } +# Legacy ooniprobe service, used to serve older probes. Identified by the +# X-Protocol-Version header that specifies the anonymous credentials protocol +# version. +module "ooniapi_ooniprobe_legacy" { + source = "../../modules/ooniapi_service" + + task_memory = 256 + + vpc_id = module.network.vpc_id + + service_name = "ooniprobe-legacy" + default_docker_image_url = "ooni/api-ooniprobe:20260824-f2dac67a" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_secrets = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.arn + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.arn + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn + } + + task_environment = { + FASTPATH_URL = "http://fastpath.${local.environment}.ooni.io:8472" + FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 3 # use a different one in prod + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + + ooniapi_service_security_groups = [ + # module.ooniapi_cluster.web_security_group_id + ] + + use_autoscaling = false + service_desired_count = 1 + + tags = merge( + local.tags, + { Name = "ooni-tier0-ooniprobe-legacy" } + ) +} + #### OONI Backend proxy service module "ooniapi_reverseproxy_deployer" { @@ -1301,6 +1351,7 @@ module "ooniapi_frontend" { ooniapi_oonirun_target_group_arn = module.ooniapi_oonirun.alb_target_group_id ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id + ooniapi_ooniprobe_legacy_target_group_arn = module.ooniapi_ooniprobe_legacy.alb_target_group_id ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id ooniapi_testlists_target_group_arn = module.ooniapi_testlists.alb_target_group_id diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index d6034f2e..2e1bee8e 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -377,22 +377,79 @@ resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_2" { } } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3" { +resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_legacy_version" { listener_arn = aws_alb_listener.ooniapi_listener_https.arn priority = 122 + action { + type = "forward" + target_group_arn = var.ooniapi_ooniprobe_legacy_target_group_arn + } + + condition { + path_pattern { + values = [ + "/api/v1/manifest*", + "/api/v1/submit_measurement*", + "/api/v1/sign_credential*" + ] + } + } + + condition { + http_header { + http_header_name = "X-Protocol-Version" + values = ["0.1.0"] + } + } +} + +resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_current_version" { + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 124 + action { type = "forward" target_group_arn = var.ooniapi_ooniprobe_target_group_arn } - # anonymous credentials condition { path_pattern { values = [ "/api/v1/manifest*", - "/api/v1/sign_credential*", - "/api/v1/submit_measurement*" + "/api/v1/submit_measurement*", + "/api/v1/sign_credential*" + ] + } + } + + # matches any value, but only when the header is present + condition { + http_header { + http_header_name = "X-Protocol-Version" + values = ["*"] + } + } +} + +resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_no_version" { + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 126 + + action { + type = "forward" + target_group_arn = var.ooniapi_ooniprobe_legacy_target_group_arn + } + + # No X-Protocol-Version header at all: rules 122/124 above already + # matched every request that does carry the header, so anything left + # here is header-less and should go to the legacy service. + condition { + path_pattern { + values = [ + "/api/v1/manifest*", + "/api/v1/submit_measurement*", + "/api/v1/sign_credential*" ] } } diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index d92c39e3..614056e6 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -28,6 +28,10 @@ variable "ooniapi_ooniprobe_target_group_arn" { description = "arn for the target group of the ooniprobe service" } +variable "ooniapi_ooniprobe_legacy_target_group_arn" { + description = "arn for the target group of the legacy ooniprobe service" +} + variable "ooniapi_oonifindings_target_group_arn" { description = "arn for the target group of the oonifindings service" } From 22b14b9cc2ef24475dfb47c285983b8ad53df47f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 2 Sep 2026 16:02:26 +0200 Subject: [PATCH 160/201] Add ooniprobe-legacy service to prod as well --- tf/environments/prod/main.tf | 51 ++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index b6980c4b..9cd5da00 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -966,6 +966,56 @@ module "ooniapi_ooniprobe" { ) } +# Legacy ooniprobe service, used to serve older probes. Identified by the +# X-Protocol-Version header that specifies the anonymous credentials protocol +# version. +module "ooniapi_ooniprobe_legacy" { + source = "../../modules/ooniapi_service" + + vpc_id = module.network.vpc_id + + service_name = "ooniprobe-legacy" + default_docker_image_url = "ooni/api-ooniprobe:20260824-f2dac67a" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + task_memory = 1024 + + task_secrets = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.arn + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn + } + + task_environment = { + # hardcoded IP for fastpath2.prod.prod.ooni.io + FASTPATH_URL = "http://10.0.0.32:8472" + FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 4 # be sure this is different from dev + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + use_autoscaling = false + service_desired_count = 1 + + tags = merge( + local.tags, + { Name = "ooni-tier0-ooniprobe-legacy" } + ) +} + ### Fastpath module "ooni_fastpath" { source = "../../modules/ooni_fastpath" @@ -1534,6 +1584,7 @@ module "ooniapi_frontend" { ooniapi_oonirun_target_group_arn = module.ooniapi_oonirun.alb_target_group_id ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id + ooniapi_ooniprobe_legacy_target_group_arn = module.ooniapi_ooniprobe_legacy.alb_target_group_id ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id ooniapi_testlists_target_group_arn = module.ooniapi_testlists.alb_target_group_id From d8d8d3c9e900d510ffe804c01f749fefd066bdb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 3 Sep 2026 11:12:02 +0200 Subject: [PATCH 161/201] Set first run to true for these services --- tf/environments/dev/main.tf | 3 +++ tf/environments/prod/main.tf | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index b366b875..3bf6abe1 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -657,6 +657,9 @@ module "ooniapi_ooniprobe" { module "ooniapi_ooniprobe_legacy" { source = "../../modules/ooniapi_service" + # First run should be set on first run to bootstrap the task definition + first_run = true + task_memory = 256 vpc_id = module.network.vpc_id diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 9cd5da00..ab6438a7 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -972,6 +972,9 @@ module "ooniapi_ooniprobe" { module "ooniapi_ooniprobe_legacy" { source = "../../modules/ooniapi_service" + # First run should be set on first run to bootstrap the task definition + first_run = true + vpc_id = module.network.vpc_id service_name = "ooniprobe-legacy" From bab971db74a67696d9b288aad97046d71666e545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 3 Sep 2026 11:55:10 +0200 Subject: [PATCH 162/201] Fix trying to create the service --- tf/environments/dev/main.tf | 2 +- tf/modules/ooniapi_service/main.tf | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 3bf6abe1..809632b5 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -658,7 +658,7 @@ module "ooniapi_ooniprobe_legacy" { source = "../../modules/ooniapi_service" # First run should be set on first run to bootstrap the task definition - first_run = true + # first_run = true task_memory = 256 diff --git a/tf/modules/ooniapi_service/main.tf b/tf/modules/ooniapi_service/main.tf index 84e9f38f..94bebfd8 100644 --- a/tf/modules/ooniapi_service/main.tf +++ b/tf/modules/ooniapi_service/main.tf @@ -126,6 +126,9 @@ resource "aws_ecs_service" "ooniapi_service" { lifecycle { create_before_destroy = true + # fixes an issue when AWS changes the capacity provider + # on its own, terraform has no record of that on its internal state + ignore_changes = [capacity_provider_strategy] } force_new_deployment = true From 8cd00e552f5be9edc6e7a300a39d502872b0199a Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 3 Sep 2026 15:45:04 +0530 Subject: [PATCH 163/201] feat: point fastpath dev builder to feat/wc-x-flags --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 95826431..8206c361 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -949,7 +949,7 @@ module "fastpath_builder" { service_name = "fastpath" repo = "ooni/backend" - branch_name = "master" + branch_name = "feat/wc-x-flags" environment = local.environment buildspec_path = "fastpath/buildspec.yml" trigger_path = "fastpath/**" From 731f80cbc8f47481f3dca2c30285864b8ce89169 Mon Sep 17 00:00:00 2001 From: decfox Date: Mon, 7 Sep 2026 12:24:45 +0530 Subject: [PATCH 164/201] feat: make fastpath image as env var for ansible role --- ansible/host_vars/fastpath.dev.ooni.io/vars.yml | 3 ++- ansible/roles/fastpath/defaults/main.yml | 3 +++ ansible/roles/fastpath/tasks/main.yml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml index e60bcb31..ed30f067 100644 --- a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml @@ -3,4 +3,5 @@ clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidev bucket_name: "ooni-data-eu-fra-test" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "3" -env: "dev" \ No newline at end of file +env: "dev" +fastpath_image: "ooni/fastpath:20260903-7c51eb5d" diff --git a/ansible/roles/fastpath/defaults/main.yml b/ansible/roles/fastpath/defaults/main.yml index 2a40e928..e9fd0dc0 100644 --- a/ansible/roles/fastpath/defaults/main.yml +++ b/ansible/roles/fastpath/defaults/main.yml @@ -4,5 +4,8 @@ tls_cert_dir: /var/lib/dehydrated/certs fastpath_user: fastpath fastpath_home: "/opt/{{ fastpath_user }}" +# Fastpath container image +fastpath_image: "ooni/fastpath:20260416-37f9e89c" + # Fastpath settings clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" diff --git a/ansible/roles/fastpath/tasks/main.yml b/ansible/roles/fastpath/tasks/main.yml index 948105bc..8b381b65 100644 --- a/ansible/roles/fastpath/tasks/main.yml +++ b/ansible/roles/fastpath/tasks/main.yml @@ -149,7 +149,7 @@ - name: Ensure fastpath is running community.docker.docker_container: name: fastpath - image: ooni/fastpath:20260416-37f9e89c + image: "{{ fastpath_image }}" state: started restart_policy: always user: "{{user_uid.stdout}}:{{user_gid.stdout}}" From c691fe629cde298887ecda1ae7df5a4ca0dbd3d2 Mon Sep 17 00:00:00 2001 From: decfox Date: Mon, 7 Sep 2026 17:53:52 +0530 Subject: [PATCH 165/201] chore: explicitly pass in fastpath image tags for prod env --- ansible/host_vars/fastpath.prod.ooni.io/vars.yml | 1 + ansible/host_vars/fastpath2.prod.ooni.io/vars.yml | 1 + ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml | 1 + ansible/roles/fastpath/defaults/main.yml | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml index f43f5acd..08b4f34a 100644 --- a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml @@ -4,3 +4,4 @@ bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "1" env: "prod" +fastpath_image: "ooni/fastpath:20260416-37f9e89c" diff --git a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml index 441cf286..b80d95bf 100644 --- a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml @@ -4,3 +4,4 @@ bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "4" env: "prod" +fastpath_image: "ooni/fastpath:20260416-37f9e89c" diff --git a/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml b/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml index 3918076b..b688ed41 100644 --- a/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml @@ -4,3 +4,4 @@ bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "5" env: "prod" +fastpath_image: "ooni/fastpath:20260416-37f9e89c" diff --git a/ansible/roles/fastpath/defaults/main.yml b/ansible/roles/fastpath/defaults/main.yml index e9fd0dc0..99e4bdbb 100644 --- a/ansible/roles/fastpath/defaults/main.yml +++ b/ansible/roles/fastpath/defaults/main.yml @@ -5,7 +5,7 @@ fastpath_user: fastpath fastpath_home: "/opt/{{ fastpath_user }}" # Fastpath container image -fastpath_image: "ooni/fastpath:20260416-37f9e89c" +fastpath_image: "ooni/fastpath:latest" # Fastpath settings clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" From 96ebe34e54fa6d4c86bf7c2f10deafd12ec55c5c Mon Sep 17 00:00:00 2001 From: decfox Date: Tue, 8 Sep 2026 16:43:25 +0530 Subject: [PATCH 166/201] chore: point fastpath prod deployments to latest docker image --- ansible/host_vars/fastpath.prod.ooni.io/vars.yml | 2 +- ansible/host_vars/fastpath2.prod.ooni.io/vars.yml | 2 +- ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml index 08b4f34a..b2847e9d 100644 --- a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml @@ -4,4 +4,4 @@ bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "1" env: "prod" -fastpath_image: "ooni/fastpath:20260416-37f9e89c" +fastpath_image: "ooni/fastpath:20260908-c4d5a24d" diff --git a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml index b80d95bf..1be99841 100644 --- a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml @@ -4,4 +4,4 @@ bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "4" env: "prod" -fastpath_image: "ooni/fastpath:20260416-37f9e89c" +fastpath_image: "ooni/fastpath:20260908-c4d5a24d" diff --git a/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml b/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml index b688ed41..430f3f49 100644 --- a/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml @@ -4,4 +4,4 @@ bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "5" env: "prod" -fastpath_image: "ooni/fastpath:20260416-37f9e89c" +fastpath_image: "ooni/fastpath:20260908-c4d5a24d" From ec62b76c79ef6bd8b4266e3ce8cd2813e2af2ae0 Mon Sep 17 00:00:00 2001 From: decfox Date: Mon, 21 Sep 2026 18:14:25 +0530 Subject: [PATCH 167/201] debug: point dev ooniprobe deployer to feat/experiment-versions --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 8206c361..0464f59e 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -581,7 +581,7 @@ module "ooniapi_ooniprobe_deployer" { service_name = "ooniprobe" repo = "ooni/backend" - branch_name = "1237-check-in-to-th" + branch_name = "feat/experiment-versions" environment = local.environment trigger_path = "ooniapi/services/ooniprobe/**" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" From 1eccb2bb914ca1c543affc47f46ae341ce9560ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 22 Sep 2026 11:47:57 +0200 Subject: [PATCH 168/201] Increase quota for default in notebook for in-host operations --- ansible/host_vars/notebook.ooni.org | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ansible/host_vars/notebook.ooni.org b/ansible/host_vars/notebook.ooni.org index 2b68cbfc..4ac6c934 100644 --- a/ansible/host_vars/notebook.ooni.org +++ b/ansible/host_vars/notebook.ooni.org @@ -134,6 +134,8 @@ admin_group_name: admin clickhouse_default_profiles: default: readonly: 2 + # 5 GB + max_memory_usage: 5005005000 write: readonly: 0 admin: @@ -154,7 +156,7 @@ clickhouse_default_users: networks: - "127.0.0.1" profile: default - quota: default + quota: notebook_default - user: name: write password_sha256_hex: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_notebook_write_password', profile='oonidevops_user_prod') | hash('sha256') }}" @@ -163,6 +165,17 @@ clickhouse_default_users: profile: write quota: default +clickhouse_custom_quotas_xml: + - quota: + name: notebook_default + duration: 3600 + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 + # No query execution time limit + # configure extra domains for dehydrated ssl_domains: - notebook1.htz-fns.prod.ooni.nu From d758d0573254e566ea4353a6d45985d17ee6c3cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 22 Sep 2026 12:26:15 +0200 Subject: [PATCH 169/201] Increase quota for the default user overall --- ansible/group_vars/clickhouse/vars.yml | 3 ++- ansible/host_vars/notebook.ooni.org | 15 +-------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 83d700b5..73275001 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -168,7 +168,8 @@ clickhouse_role_manage_settings_profiles: True clickhouse_default_profiles: default: readonly: 2 - max_memory_usage: 11001001000 + # 10 GB + max_memory_usage: 10010010000 use_uncompressed_cache: 0 load_balancing: random max_partitions_per_insert_block: 100 diff --git a/ansible/host_vars/notebook.ooni.org b/ansible/host_vars/notebook.ooni.org index 4ac6c934..2b68cbfc 100644 --- a/ansible/host_vars/notebook.ooni.org +++ b/ansible/host_vars/notebook.ooni.org @@ -134,8 +134,6 @@ admin_group_name: admin clickhouse_default_profiles: default: readonly: 2 - # 5 GB - max_memory_usage: 5005005000 write: readonly: 0 admin: @@ -156,7 +154,7 @@ clickhouse_default_users: networks: - "127.0.0.1" profile: default - quota: notebook_default + quota: default - user: name: write password_sha256_hex: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_notebook_write_password', profile='oonidevops_user_prod') | hash('sha256') }}" @@ -165,17 +163,6 @@ clickhouse_default_users: profile: write quota: default -clickhouse_custom_quotas_xml: - - quota: - name: notebook_default - duration: 3600 - queries: 0 - errors: 0 - result_rows: 0 - read_rows: 0 - execution_time: 0 - # No query execution time limit - # configure extra domains for dehydrated ssl_domains: - notebook1.htz-fns.prod.ooni.nu From ce277406f93457927837d25337bd107a3cbc51ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 23 Sep 2026 11:36:12 +0200 Subject: [PATCH 170/201] Add nginx rules for downloading the data dump --- ansible/roles/notebook/tasks/jupyterhub.yml | 14 ++++++++++++++ ansible/roles/notebook/templates/nginx-dump.j2 | 11 +++++++++++ .../roles/notebook/templates/nginx-jupyterhub.j2 | 2 ++ 3 files changed, 27 insertions(+) create mode 100644 ansible/roles/notebook/templates/nginx-dump.j2 diff --git a/ansible/roles/notebook/tasks/jupyterhub.yml b/ansible/roles/notebook/tasks/jupyterhub.yml index de4a271b..596b6c89 100644 --- a/ansible/roles/notebook/tasks/jupyterhub.yml +++ b/ansible/roles/notebook/tasks/jupyterhub.yml @@ -93,6 +93,20 @@ - jupyterhub - config +- name: Setup file dump nginx config + ansible.builtin.template: + src: nginx-dump.j2 + dest: /etc/nginx/notebook-dump.conf + owner: root + group: root + mode: "0644" + notify: + - Reload nginx + tags: + - oonidata + - jupyterhub + - config + - name: Setup oonidata nginx config ansible.builtin.template: src: nginx-jupyterhub.j2 diff --git a/ansible/roles/notebook/templates/nginx-dump.j2 b/ansible/roles/notebook/templates/nginx-dump.j2 new file mode 100644 index 00000000..838808e7 --- /dev/null +++ b/ansible/roles/notebook/templates/nginx-dump.j2 @@ -0,0 +1,11 @@ +# ansible-managed in ooni/devops.git +# Included from /etc/nginx/sites-enabled/01-jupyterhub + +# Static file downloads. Only exact file names are exposed, not the whole directory +location = /dump/intersection_domains_dump.csv.lz4 { + alias /mnt/data/fariba/intersection_domains_dump.csv.lz4; + + default_type application/octet-stream; + add_header Content-Disposition 'attachment'; + add_header Access-Control-Allow-Origin *; +} diff --git a/ansible/roles/notebook/templates/nginx-jupyterhub.j2 b/ansible/roles/notebook/templates/nginx-jupyterhub.j2 index 92eb9eba..c0fbe9e9 100644 --- a/ansible/roles/notebook/templates/nginx-jupyterhub.j2 +++ b/ansible/roles/notebook/templates/nginx-jupyterhub.j2 @@ -25,6 +25,8 @@ server { add_header Access-Control-Allow-Origin *; + include /etc/nginx/notebook-dump.conf; + ## JupyterHub configuration location / { proxy_pass http://127.0.0.1:8888; From 1b21bab7d2b466d760a93091a86b5abfb345df38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 23 Sep 2026 12:33:43 +0200 Subject: [PATCH 171/201] Point oonirun dev to master --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 0464f59e..f701cd91 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -965,7 +965,7 @@ module "ooniapi_oonirun_deployer" { service_name = "oonirun" repo = "ooni/backend" - branch_name = "oonirun-v2-1" + branch_name = "master" environment = local.environment buildspec_path = "ooniapi/services/oonirun/buildspec.yml" trigger_path = "ooniapi/services/oonirun/**" From f4f0ed2b447824181cf1f4176507325e03a626af Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 24 Sep 2026 09:12:49 +0200 Subject: [PATCH 172/201] Update fastpath_image to latest tag --- ansible/host_vars/fastpath.prod.ooni.io/vars.yml | 2 +- ansible/host_vars/fastpath2.prod.ooni.io/vars.yml | 2 +- ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml index b2847e9d..986a62f6 100644 --- a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml @@ -4,4 +4,4 @@ bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "1" env: "prod" -fastpath_image: "ooni/fastpath:20260908-c4d5a24d" +fastpath_image: "ooni/fastpath:20260921-ac774cb4" diff --git a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml index 1be99841..26123dcc 100644 --- a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml @@ -4,4 +4,4 @@ bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "4" env: "prod" -fastpath_image: "ooni/fastpath:20260908-c4d5a24d" +fastpath_image: "ooni/fastpath:20260921-ac774cb4" diff --git a/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml b/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml index 430f3f49..eef14c04 100644 --- a/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml @@ -4,4 +4,4 @@ bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "5" env: "prod" -fastpath_image: "ooni/fastpath:20260908-c4d5a24d" +fastpath_image: "ooni/fastpath:20260921-ac774cb4" From 425247dd52921940ae7bc02851245a85d6dcb5f5 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 24 Sep 2026 09:29:19 +0200 Subject: [PATCH 173/201] clean up old nginx apt sources use nginx_install_method: official repo --- ansible/deploy-fastpath.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ansible/deploy-fastpath.yml b/ansible/deploy-fastpath.yml index b9a8e502..5030fe1e 100644 --- a/ansible/deploy-fastpath.yml +++ b/ansible/deploy-fastpath.yml @@ -6,6 +6,27 @@ - fastpath2.prod.ooni.io - reuploaderfastpath.prod.ooni.io become: true + vars: + nginx_install_method: official_repo + pre_tasks: + - name: Find any nginx.org apt source files left over from a previous nginxinc.nginx install + ansible.builtin.find: + paths: /etc/apt/sources.list.d + patterns: "*.list" + contains: "nginx.org/packages" + register: _stale_nginx_apt_sources + tags: + - nginx + - packages + + - name: Remove stale nginx.org apt source files (avoids "Conflicting values set for option Signed-By") + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ _stale_nginx_apt_sources.files }}" + tags: + - nginx + - packages roles: - role: bootstrap - role: nginx From 8fd1280fd4f884519a0952c519a39cf877a296db Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 24 Sep 2026 09:37:15 +0200 Subject: [PATCH 174/201] fix repo path for ubuntu host compatibility --- ansible/roles/nginx/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/nginx/tasks/main.yml b/ansible/roles/nginx/tasks/main.yml index dbb2c3fc..bd70a7cd 100644 --- a/ansible/roles/nginx/tasks/main.yml +++ b/ansible/roles/nginx/tasks/main.yml @@ -71,7 +71,7 @@ - name: Add the nginx.org apt repository ansible.builtin.apt_repository: - repo: "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://nginx.org/packages/debian {{ ansible_distribution_release }} nginx" + repo: "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://nginx.org/packages/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} nginx" filename: nginx state: present From 43b216704c8fee065a02b4fe04c3834d227ef8c1 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 24 Sep 2026 10:45:53 +0200 Subject: [PATCH 175/201] match nginx.org/packages in sources.list.d files --- ansible/deploy-fastpath.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/deploy-fastpath.yml b/ansible/deploy-fastpath.yml index 5030fe1e..8795c8b7 100644 --- a/ansible/deploy-fastpath.yml +++ b/ansible/deploy-fastpath.yml @@ -13,7 +13,8 @@ ansible.builtin.find: paths: /etc/apt/sources.list.d patterns: "*.list" - contains: "nginx.org/packages" + contains: "nginx\\.org/packages" + read_whole_file: true register: _stale_nginx_apt_sources tags: - nginx From 0a5d9f68da0e30e322d1c7068c9a07e26718d9a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 28 Sep 2026 11:49:09 +0200 Subject: [PATCH 176/201] Remove intersection dump from notebook server --- ansible/roles/notebook/tasks/jupyterhub.yml | 14 -------------- ansible/roles/notebook/templates/nginx-dump.j2 | 11 ----------- .../roles/notebook/templates/nginx-jupyterhub.j2 | 2 -- 3 files changed, 27 deletions(-) delete mode 100644 ansible/roles/notebook/templates/nginx-dump.j2 diff --git a/ansible/roles/notebook/tasks/jupyterhub.yml b/ansible/roles/notebook/tasks/jupyterhub.yml index 596b6c89..de4a271b 100644 --- a/ansible/roles/notebook/tasks/jupyterhub.yml +++ b/ansible/roles/notebook/tasks/jupyterhub.yml @@ -93,20 +93,6 @@ - jupyterhub - config -- name: Setup file dump nginx config - ansible.builtin.template: - src: nginx-dump.j2 - dest: /etc/nginx/notebook-dump.conf - owner: root - group: root - mode: "0644" - notify: - - Reload nginx - tags: - - oonidata - - jupyterhub - - config - - name: Setup oonidata nginx config ansible.builtin.template: src: nginx-jupyterhub.j2 diff --git a/ansible/roles/notebook/templates/nginx-dump.j2 b/ansible/roles/notebook/templates/nginx-dump.j2 deleted file mode 100644 index 838808e7..00000000 --- a/ansible/roles/notebook/templates/nginx-dump.j2 +++ /dev/null @@ -1,11 +0,0 @@ -# ansible-managed in ooni/devops.git -# Included from /etc/nginx/sites-enabled/01-jupyterhub - -# Static file downloads. Only exact file names are exposed, not the whole directory -location = /dump/intersection_domains_dump.csv.lz4 { - alias /mnt/data/fariba/intersection_domains_dump.csv.lz4; - - default_type application/octet-stream; - add_header Content-Disposition 'attachment'; - add_header Access-Control-Allow-Origin *; -} diff --git a/ansible/roles/notebook/templates/nginx-jupyterhub.j2 b/ansible/roles/notebook/templates/nginx-jupyterhub.j2 index c0fbe9e9..92eb9eba 100644 --- a/ansible/roles/notebook/templates/nginx-jupyterhub.j2 +++ b/ansible/roles/notebook/templates/nginx-jupyterhub.j2 @@ -25,8 +25,6 @@ server { add_header Access-Control-Allow-Origin *; - include /etc/nginx/notebook-dump.conf; - ## JupyterHub configuration location / { proxy_pass http://127.0.0.1:8888; From 7dfda84bb3f2b48bcdc1eef47d2f2e7a82297658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 28 Sep 2026 12:36:08 +0200 Subject: [PATCH 177/201] Update docker tag in legacy --- tf/environments/dev/main.tf | 2 +- tf/environments/prod/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 99d15327..9e3fd3b1 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -665,7 +665,7 @@ module "ooniapi_ooniprobe_legacy" { vpc_id = module.network.vpc_id service_name = "ooniprobe-legacy" - default_docker_image_url = "ooni/api-ooniprobe:20260824-f2dac67a" + default_docker_image_url = "ooni/api-ooniprobe:20260921-da8b057f" stage = local.environment dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index ab6438a7..a4b4f5f6 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -978,7 +978,7 @@ module "ooniapi_ooniprobe_legacy" { vpc_id = module.network.vpc_id service_name = "ooniprobe-legacy" - default_docker_image_url = "ooni/api-ooniprobe:20260824-f2dac67a" + default_docker_image_url = "ooni/api-ooniprobe:20260921-da8b057f" stage = local.environment dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name From eeff8dd99f3b8cdff9579023425dfaea82ed3edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 28 Sep 2026 12:40:53 +0200 Subject: [PATCH 178/201] Already deployed; first_run no longer needed --- tf/environments/prod/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index a4b4f5f6..35f43a97 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -973,7 +973,7 @@ module "ooniapi_ooniprobe_legacy" { source = "../../modules/ooniapi_service" # First run should be set on first run to bootstrap the task definition - first_run = true + # first_run = true vpc_id = module.network.vpc_id From 4a7907c05b265334aa83792f40974d7dec393f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 29 Sep 2026 10:38:05 +0200 Subject: [PATCH 179/201] Add wcth to the metrics scraping --- ansible/roles/prometheus/templates/prometheus.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index eb854621..ba419185 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -87,6 +87,9 @@ scrape_configs: - http://0.do.th.prod.ooni.io:9001/metrics - http://1.do.th.prod.ooni.io:9001/metrics - http://2.do.th.prod.ooni.io:9001/metrics + - https://wcth0.fra1.prod.ooni.io/metrics + - https://wcth1.fra1.prod.ooni.io/metrics + - https://wcth2.fra1.prod.ooni.io/metrics relabel_configs: # set the scheme based on what has been parsed in the address - source_labels: [__address__] From 5da15b997e8d6d1729461b7973ceb4a7ad5f930d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 29 Sep 2026 10:40:47 +0200 Subject: [PATCH 180/201] Add dev test helper to monitoring --- ansible/roles/prometheus/templates/prometheus.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index ba419185..e67eea50 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -87,6 +87,7 @@ scrape_configs: - http://0.do.th.prod.ooni.io:9001/metrics - http://1.do.th.prod.ooni.io:9001/metrics - http://2.do.th.prod.ooni.io:9001/metrics + - https://wc.th.dev.ooni.io/metrics - https://wcth0.fra1.prod.ooni.io/metrics - https://wcth1.fra1.prod.ooni.io/metrics - https://wcth2.fra1.prod.ooni.io/metrics From 56e8345282bf5657d19d8ccd1346fdddfbdc18f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 29 Sep 2026 14:41:09 +0200 Subject: [PATCH 181/201] Fix node_exporter path --- ansible/roles/prometheus/templates/prometheus.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index e67eea50..1b16ad95 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -88,9 +88,9 @@ scrape_configs: - http://1.do.th.prod.ooni.io:9001/metrics - http://2.do.th.prod.ooni.io:9001/metrics - https://wc.th.dev.ooni.io/metrics - - https://wcth0.fra1.prod.ooni.io/metrics - - https://wcth1.fra1.prod.ooni.io/metrics - - https://wcth2.fra1.prod.ooni.io/metrics + - https://wcth0.fra1.prod.ooni.io/metrics/node_exporter + - https://wcth1.fra1.prod.ooni.io/metrics/node_exporter + - https://wcth2.fra1.prod.ooni.io/metrics/node_exporter relabel_configs: # set the scheme based on what has been parsed in the address - source_labels: [__address__] From a7f1d8be919d41e28bc85e4a3a9636976bc5d1c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 29 Sep 2026 14:50:39 +0200 Subject: [PATCH 182/201] Fix monitoring address --- ansible/roles/prometheus/templates/prometheus.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index 1b16ad95..7d126533 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -88,9 +88,9 @@ scrape_configs: - http://1.do.th.prod.ooni.io:9001/metrics - http://2.do.th.prod.ooni.io:9001/metrics - https://wc.th.dev.ooni.io/metrics - - https://wcth0.fra1.prod.ooni.io/metrics/node_exporter - - https://wcth1.fra1.prod.ooni.io/metrics/node_exporter - - https://wcth2.fra1.prod.ooni.io/metrics/node_exporter + - https://wcth0.fra1.prod.ooni.io:9100/metrics/node_exporter + - https://wcth1.fra1.prod.ooni.io:9100/metrics/node_exporter + - https://wcth2.fra1.prod.ooni.io:9100/metrics/node_exporter relabel_configs: # set the scheme based on what has been parsed in the address - source_labels: [__address__] From ee8d3ecabf45e8d01c16cc747841534d27d7526f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 29 Sep 2026 14:51:37 +0200 Subject: [PATCH 183/201] Fix url --- ansible/roles/prometheus/templates/prometheus.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index 7d126533..a5a6d735 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -88,9 +88,9 @@ scrape_configs: - http://1.do.th.prod.ooni.io:9001/metrics - http://2.do.th.prod.ooni.io:9001/metrics - https://wc.th.dev.ooni.io/metrics - - https://wcth0.fra1.prod.ooni.io:9100/metrics/node_exporter - - https://wcth1.fra1.prod.ooni.io:9100/metrics/node_exporter - - https://wcth2.fra1.prod.ooni.io:9100/metrics/node_exporter + - https://wcth0.fra1.prod.ooni.io:9001/metrics/node_exporter + - https://wcth1.fra1.prod.ooni.io:9001/metrics/node_exporter + - https://wcth2.fra1.prod.ooni.io:9001/metrics/node_exporter relabel_configs: # set the scheme based on what has been parsed in the address - source_labels: [__address__] From 848df1f93b7ed160579e7755cefda4c7e5df9f9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 29 Sep 2026 15:03:38 +0200 Subject: [PATCH 184/201] Fix oohelperd nginx config --- ansible/roles/oohelperd/templates/nginx-oohelperd.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 index fd6480bd..7da08869 100644 --- a/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 +++ b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 @@ -28,7 +28,7 @@ server { # nginx >= 1.25.1, which is what nginx.org ships http2 on; - server_name {{ nginx_server_name }}; + server_name {{ nginx_server_name }} *.th.ooni.org; include /etc/nginx/ssl_modern.conf; ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; From 9ccfa83a897fc288ed4d4c092fcf2648d3088b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 29 Sep 2026 17:13:27 +0200 Subject: [PATCH 185/201] Drop legacy droplet test helpers from digital ocean --- tf/environments/dev/main.tf | 16 -- tf/environments/prod/main.tf | 17 -- tf/modules/ooni_th_droplet/main.tf | 49 ----- tf/modules/ooni_th_droplet/outputs.tf | 10 - .../templates/cloud-init-docker.yml | 177 ------------------ .../ooni_th_droplet/templates/cloud-init.yml | 59 ------ tf/modules/ooni_th_droplet/variables.tf | 40 ---- 7 files changed, 368 deletions(-) delete mode 100644 tf/modules/ooni_th_droplet/main.tf delete mode 100644 tf/modules/ooni_th_droplet/outputs.tf delete mode 100644 tf/modules/ooni_th_droplet/templates/cloud-init-docker.yml delete mode 100644 tf/modules/ooni_th_droplet/templates/cloud-init.yml delete mode 100644 tf/modules/ooni_th_droplet/variables.tf diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 9e3fd3b1..7d06000b 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -393,22 +393,6 @@ moved { ### OONI Tier0 Backend Proxy -module "ooni_th_droplet" { - source = "../../modules/ooni_th_droplet" - - stage = local.environment - instance_location = "fra1" - instance_size = "s-1vcpu-1gb" - droplet_count = 1 - deployer_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] - metrics_password = data.aws_ssm_parameter.prometheus_metrics_password.arn - ssh_keys = [ - "3d:81:99:17:b5:d1:20:a5:fe:2b:14:96:67:93:d6:34", - "f6:4b:8b:e2:0e:d2:97:c5:45:5c:07:a6:fe:54:60:0e" - ] - dns_zone_ooni_io = local.dns_zone_ooni_io -} - resource "digitalocean_ssh_key" "oonidevops" { name = "oonidevops" public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 35f43a97..b8877903 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -425,23 +425,6 @@ moved { ### OONI Tier0 Backend Proxy -module "ooni_th_droplet" { - source = "../../modules/ooni_th_droplet" - - stage = local.environment - instance_location = "fra1" - instance_size = "s-1vcpu-1gb" - droplet_count = 3 - deployer_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] - metrics_password = data.aws_ssm_parameter.prometheus_metrics_password.arn - ssh_keys = [ - "3d:81:99:17:b5:d1:20:a5:fe:2b:14:96:67:93:d6:34", - "f6:4b:8b:e2:0e:d2:97:c5:45:5c:07:a6:fe:54:60:0e" - ] - - dns_zone_ooni_io = local.dns_zone_ooni_io -} - module "ooni_test_helpers_wc" { source = "../../modules/ooni_th_binary_droplet" diff --git a/tf/modules/ooni_th_droplet/main.tf b/tf/modules/ooni_th_droplet/main.tf deleted file mode 100644 index 74413a6f..00000000 --- a/tf/modules/ooni_th_droplet/main.tf +++ /dev/null @@ -1,49 +0,0 @@ -terraform { - required_providers { - digitalocean = { - source = "digitalocean/digitalocean" - version = "~> 2.0" - } - } -} - -data "cloudinit_config" "ooni_th_docker" { - gzip = false - base64_encode = false - - part { - filename = "init.cfg" - content_type = "text/cloud-config" - content = templatefile("${path.module}/templates/cloud-init-docker.yml", { - monitoring_ip = "5.9.112.244", - deployer_key = var.deployer_key, - metrics_password = var.metrics_password, - }) - } -} - -resource "digitalocean_droplet" "ooni_th_docker" { - image = "ubuntu-24-04-x64" - name = "${var.name}-docker-${var.stage}-${count.index}" - region = var.instance_location - size = var.instance_size - ipv6 = true - ssh_keys = var.ssh_keys - user_data = data.cloudinit_config.ooni_th_docker.rendered - count = var.droplet_count - - lifecycle { - create_before_destroy = true - ignore_changes = all - } -} -resource "aws_route53_record" "ooni_th" { - zone_id = var.dns_zone_ooni_io - name = "${each.key}.do.th.${var.stage}.ooni.io" - type = "A" - ttl = 60 - for_each = { - for d in digitalocean_droplet.ooni_th_docker : reverse(split("-", d.name))[0] => d.ipv4_address - } - records = [each.value] -} diff --git a/tf/modules/ooni_th_droplet/outputs.tf b/tf/modules/ooni_th_droplet/outputs.tf deleted file mode 100644 index fdcad4af..00000000 --- a/tf/modules/ooni_th_droplet/outputs.tf +++ /dev/null @@ -1,10 +0,0 @@ -output "droplet_ipv4_address" { - value = digitalocean_droplet.ooni_th_docker[*].ipv4_address -} - -output "droplet_addresses" { - # for why we use values, - # see: https://github.com/hashicorp/terraform/issues/23245#issuecomment-548391304 - # https://github.com/hashicorp/terraform/issues/22476 - value = values(aws_route53_record.ooni_th)[*].fqdn -} diff --git a/tf/modules/ooni_th_droplet/templates/cloud-init-docker.yml b/tf/modules/ooni_th_droplet/templates/cloud-init-docker.yml deleted file mode 100644 index 93135daf..00000000 --- a/tf/modules/ooni_th_droplet/templates/cloud-init-docker.yml +++ /dev/null @@ -1,177 +0,0 @@ -apt: - sources: - docker.list: - source: "deb [arch=amd64 signed-by=$KEY_FILE] https://download.docker.com/linux/ubuntu $RELEASE stable" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth - lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh - 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq - L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 - UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N - cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht - ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo - vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD - G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ - XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj - q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB - tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 - BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO - v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd - tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk - jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m - 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P - XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc - FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 - g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm - ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh - 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 - G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW - FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB - EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF - M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx - Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu - w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk - z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 - eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb - VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa - 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X - zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ - pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 - ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ - BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY - 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp - YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI - mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES - KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 - JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ - cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 - 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 - U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z - VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f - irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk - SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz - QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W - 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw - 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe - dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y - Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR - H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh - /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ - M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S - xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O - jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG - YT90qFF93M3v01BbxP+EIY2/9tiIPbrd - =0YYh - -----END PGP PUBLIC KEY BLOCK----- -package_update: true -packages: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin - - prometheus-node-exporter - - nginx - -users: - - name: deployer - ssh-authorized-keys: - - ${deployer_key} - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - groups: - - sudo - - docker - shell: /bin/bash - -write_files: - - path: /etc/ssh/sshd_config - content: | - PermitRootLogin no - PermitEmptyPasswords no - PasswordAuthentication no - KbdInteractiveAuthentication no - UsePAM yes - PubkeyAuthentication yes - X11Forwarding no - PrintMotd no - AcceptEnv LANG LC_* - AllowUsers deployer - - - path: /etc/docker/daemon.json - content: | - { - "ipv6": true, - "fixed-cidr-v6": "2001:db8:1::/64", - "log-driver": "json-file", - "log-opts": {"max-size": "100m", "max-file": "3"} - } - - - path: /etc/nginx/sites-available/default - content: | - proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M - max_size=5g inactive=24h use_temp_path=off; - - server { - listen 80; - server_name _; - gzip on; - resolver 127.0.0.1; - - # test helper application metrics - location /metrics { - allow ${monitoring_ip}; - deny all; - - proxy_pass http://127.0.0.1:8080; - } - - # local test helper - location / { - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - proxy_pass http://127.0.0.1:8080; - - proxy_cache thcache; - proxy_cache_min_uses 1; - proxy_cache_lock on; - proxy_cache_lock_timeout 30; - proxy_cache_lock_age 30; - proxy_cache_use_stale error timeout invalid_header updating; - # Cache POST without headers set by the test helper! - proxy_cache_methods POST; - proxy_cache_key "$request_uri|$request_body"; - proxy_cache_valid 200 10m; - proxy_cache_valid any 0; - add_header X-Cache-Status $upstream_cache_status; - } - } - - server { - listen 9001; - server_name localhost; - - allow ${monitoring_ip}; - deny all; - - # Metrics from node_exporter - location = /metrics { - proxy_pass http://127.0.0.1:9100; - } - } - -runcmd: - - sshd -t - - systemctl restart sshd - - systemctl restart docker - - ufw default deny incoming - - ufw default allow outgoing - - ufw allow 22/tcp - - ufw allow 80/tcp - - ufw allow 443/tcp - - ufw allow from ${monitoring_ip} proto tcp to any port 9001 - - ufw enable - - service nginx restart - - docker container rm -f oonith - - docker run -d -e PROMETHEUS_METRICS_PASSWORD='${metrics_password}' -p 8080:80 --restart unless-stopped --name oonith ooni/oonith-oohelperd:latest diff --git a/tf/modules/ooni_th_droplet/templates/cloud-init.yml b/tf/modules/ooni_th_droplet/templates/cloud-init.yml deleted file mode 100644 index 111502ef..00000000 --- a/tf/modules/ooni_th_droplet/templates/cloud-init.yml +++ /dev/null @@ -1,59 +0,0 @@ -apt: - sources: - ooni.list: - source: "deb [trusted=yes] https://ooni-internal-deb.s3.eu-central-1.amazonaws.com unstable main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mDMEYGISFRYJKwYBBAHaRw8BAQdA4VxoR0gSsH56BbVqYdK9HNQ0Dj2YFVbvKIIZ - JKlaW920Mk9PTkkgcGFja2FnZSBzaWduaW5nIDxjb250YWN0QG9wZW5vYnNlcnZh - dG9yeS5vcmc+iJYEExYIAD4WIQS1oI8BeW5/UhhhtEk3LR/ycfLdUAUCYGISFQIb - AwUJJZgGAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRA3LR/ycfLdUFk+AQCb - gsUQsAQGxUFvxk1XQ4RgEoh7wy2yTuK8ZCkSHJ0HWwD/f2OAjDigGq07uJPYw7Uo - Ih9+mJ/ubwiPMzUWF6RSdgu4OARgYhIVEgorBgEEAZdVAQUBAQdAx4p1KerwcIhX - HfM9LbN6Gi7z9j4/12JKYOvr0d0yC30DAQgHiH4EGBYIACYWIQS1oI8BeW5/Uhhh - tEk3LR/ycfLdUAUCYGISFQIbDAUJJZgGAAAKCRA3LR/ycfLdUL4cAQCs53fLphhy - 6JMwVhRs02LXi1lntUtw1c+EMn6t7XNM6gD+PXpbgSZwoV3ZViLqr58o9fZQtV3s - oN7jfdbznrWVigE= - =PtYb - -----END PGP PUBLIC KEY BLOCK----- -package_update: true -packages: - - oohelperd - - nginx - -write_files: - - path: /etc/nginx/sites-available/default - content: | - proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M - max_size=5g inactive=24h use_temp_path=off; - - server { - listen 80; - server_name _; - gzip on; - resolver 127.0.0.1; - # local test helper - location / { - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - proxy_pass http://127.0.0.1:8080; - - proxy_cache thcache; - proxy_cache_min_uses 1; - proxy_cache_lock on; - proxy_cache_lock_timeout 30; - proxy_cache_lock_age 30; - proxy_cache_use_stale error timeout invalid_header updating; - # Cache POST without headers set by the test helper! - proxy_cache_methods POST; - proxy_cache_key "$request_uri|$request_body"; - proxy_cache_valid 200 10m; - proxy_cache_valid any 0; - add_header X-Cache-Status $upstream_cache_status; - - } - } - -runcmd: - - service nginx restart diff --git a/tf/modules/ooni_th_droplet/variables.tf b/tf/modules/ooni_th_droplet/variables.tf deleted file mode 100644 index 522e3333..00000000 --- a/tf/modules/ooni_th_droplet/variables.tf +++ /dev/null @@ -1,40 +0,0 @@ -variable "stage" { - type = string -} - -variable "name" { - description = "Name of the droplets" - type = string - default = "ooni-wcth" -} - -variable "instance_location" { - type = string - default = "fra1" -} - -variable "instance_size" { - # s-2vcpu-4gb - type = string - default = "s-1vcpu-1gb" -} - -variable "droplet_count" { - default = 1 -} - -variable "ssh_keys" { - type = list(string) -} - -variable "deployer_key" { - type = string -} - -variable "metrics_password" { - type = string -} - -variable "dns_zone_ooni_io" { - type = string -} From addf016ca0b6df27bf00e729f0c01d1cd12a3c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 29 Sep 2026 17:14:14 +0200 Subject: [PATCH 186/201] Drop digital ocean test helpers from monitoring --- ansible/roles/prometheus/templates/prometheus.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index a5a6d735..243bc982 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -84,9 +84,6 @@ scrape_configs: - https://data2.htz-fsn.prod.ooni.nu/metrics/node_exporter - https://data3.htz-fsn.prod.ooni.nu/metrics/node_exporter - https://notebook1.htz-fsn.prod.ooni.nu/metrics/node_exporter - - http://0.do.th.prod.ooni.io:9001/metrics - - http://1.do.th.prod.ooni.io:9001/metrics - - http://2.do.th.prod.ooni.io:9001/metrics - https://wc.th.dev.ooni.io/metrics - https://wcth0.fra1.prod.ooni.io:9001/metrics/node_exporter - https://wcth1.fra1.prod.ooni.io:9001/metrics/node_exporter @@ -209,9 +206,6 @@ scrape_configs: password: '{{ prometheus_metrics_password }}' static_configs: - targets: - - 0.do.th.prod.ooni.io - - 1.do.th.prod.ooni.io - - 2.do.th.prod.ooni.io - wcth0.fra1.prod.ooni.io - wcth1.fra1.prod.ooni.io - wcth2.fra1.prod.ooni.io From 8d929ced7cbe9f603fdfc2da29da730cd64d457c Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 1 Oct 2026 01:10:53 +0200 Subject: [PATCH 187/201] add private api targets as blackbox job with interval these queries can be slow; poll every 5 minutes --- .../roles/prometheus/templates/prometheus.yml | 6 ++++ ansible/roles/prometheus/vars/main.yml | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index 243bc982..ccd5b006 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -22,6 +22,12 @@ scrape_configs: # Node exporter metrics {% for bbjob in blackbox_jobs %} - job_name: "{{ bbjob.name }}" +{% if bbjob.scrape_interval is defined %} + scrape_interval: {{ bbjob.scrape_interval }} +{% endif %} +{% if bbjob.scrape_timeout is defined %} + scrape_timeout: {{ bbjob.scrape_timeout }} +{% endif %} metrics_path: /probe params: module: diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index e9db75b1..bdb46315 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -109,6 +109,35 @@ blackbox_jobs: - "https://api.ooni.io/api/_/global_overview" - "https://api.ooni.org/api/_/global_overview" + - name: "ooni API private" + module: "https_2xx_json" + ignore_instance_down: true + scrape_interval: 5m + scrape_timeout: 30s + labels: + tier: "tier-1" + targets: + - "https://api.ooni.org/api/_/asn_by_month" + - "https://api.ooni.org/api/_/countries_by_month" + - "https://api.ooni.org/api/_/test_names" + - "https://api.ooni.org/api/_/countries" + - "https://api.ooni.org/api/_/test_coverage?probe_cc=IT" + - "https://api.ooni.org/api/_/website_networks?probe_cc=IT" + - "https://api.ooni.org/api/_/website_stats?probe_cc=IT&probe_asn=3269&input=https://www.google.com/" + - "https://api.ooni.org/api/_/website_urls?probe_cc=IT&probe_asn=AS3269" + - "https://api.ooni.org/api/_/vanilla_tor_stats?probe_cc=IT" + - "https://api.ooni.org/api/_/im_networks?probe_cc=IT" + - "https://api.ooni.org/api/_/im_stats?probe_cc=IT&probe_asn=AS3269&test_name=signal" + - "https://api.ooni.org/api/_/network_stats?probe_cc=IT" + - "https://api.ooni.org/api/_/country_overview?probe_cc=IT" + - "https://api.ooni.org/api/_/global_overview_by_month" + - "https://api.ooni.org/api/_/circumvention_stats_by_country" + - "https://api.ooni.org/api/_/circumvention_runtime_stats" + - "https://api.ooni.org/api/_/domain_metadata?domain=google.com" + - "https://api.ooni.org/api/_/asnmeta?asn=3269" + - "https://api.ooni.org/api/_/networks" + - "https://api.ooni.org/api/_/domains" + - name: "ooni API findings listing" module: "https_2xx_json" labels: From 55e622ea70bb8d8580e1769909bbf13ddec94797 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 1 Oct 2026 01:20:07 +0200 Subject: [PATCH 188/201] add api.oonio.io targets --- ansible/roles/prometheus/vars/main.yml | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index bdb46315..5cc81e78 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -138,6 +138,36 @@ blackbox_jobs: - "https://api.ooni.org/api/_/networks" - "https://api.ooni.org/api/_/domains" + # api.ooni.io goes to backend-fsn nginx, which caches some responses + - name: "ooni API private (legacy)" + module: "https_2xx_json" + ignore_instance_down: true + scrape_interval: 5m + scrape_timeout: 30s + labels: + tier: "tier-1" + targets: + - "https://api.ooni.io/api/_/asn_by_month" + - "https://api.ooni.io/api/_/countries_by_month" + - "https://api.ooni.io/api/_/test_names" + - "https://api.ooni.io/api/_/countries" + - "https://api.ooni.io/api/_/test_coverage?probe_cc=IT" + - "https://api.ooni.io/api/_/website_networks?probe_cc=IT" + - "https://api.ooni.io/api/_/website_stats?probe_cc=IT&probe_asn=3269&input=https://www.google.com/" + - "https://api.ooni.io/api/_/website_urls?probe_cc=IT&probe_asn=AS3269" + - "https://api.ooni.io/api/_/vanilla_tor_stats?probe_cc=IT" + - "https://api.ooni.io/api/_/im_networks?probe_cc=IT" + - "https://api.ooni.io/api/_/im_stats?probe_cc=IT&probe_asn=AS3269&test_name=signal" + - "https://api.ooni.io/api/_/network_stats?probe_cc=IT" + - "https://api.ooni.io/api/_/country_overview?probe_cc=IT" + - "https://api.ooni.io/api/_/global_overview_by_month" + - "https://api.ooni.io/api/_/circumvention_stats_by_country" + - "https://api.ooni.io/api/_/circumvention_runtime_stats" + - "https://api.ooni.io/api/_/domain_metadata?domain=google.com" + - "https://api.ooni.io/api/_/asnmeta?asn=3269" + - "https://api.ooni.io/api/_/networks" + - "https://api.ooni.io/api/_/domains" + - name: "ooni API findings listing" module: "https_2xx_json" labels: From 5392bc6cd5c893c787c59d488c88117c73f1b8da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 2 Oct 2026 11:34:16 +0200 Subject: [PATCH 189/201] Drop countly host from monitoring --- ansible/roles/prometheus/vars/main.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index 5cc81e78..1432d135 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -1,6 +1,5 @@ dom0_hosts: - ams-slack-1.ooni.org - - doams1-countly.ooni.nu blackbox_jobs: # TODO add these records to the ALB config @@ -248,11 +247,6 @@ blackbox_jobs: # end of API # - - name: "countly.ooni.io ping" - module: "http_2xx" - targets: - - "https://countly.ooni.io/o/ping" - - name: "slack inviter" module: "http_2xx" targets: From ea669d36d297281ceb1151d847a3c227d239d44a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 2 Oct 2026 13:06:50 +0200 Subject: [PATCH 190/201] testing pagination PR on dev --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 9e3fd3b1..1e648673 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1188,7 +1188,7 @@ module "ooniapi_oonimeasurements_deployer" { service_name = "oonimeasurements" repo = "ooni/backend" - branch_name = "master" + branch_name = "636-better-pagination" environment = local.environment trigger_path = "ooniapi/services/oonimeasurements/**" buildspec_path = "ooniapi/services/oonimeasurements/buildspec.yml" From 124a863134e128f28ac1922b784e5ef61ae4f926 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 00:59:35 +0000 Subject: [PATCH 191/201] Use per-service ClickHouse URLs in blue/green service secrets main replaced the shared clickhouse_write_url, clickhouse_readonly_url and clickhouse_readonly_test_url parameters with one per service (b7baff6). The blue/green service secrets are built from the same parameters as each service's ECS task_secrets, so they now point at the per-service ones again: ooniprobe, oonirun and oonimeasurements in dev, oonirun in prod. dev oonifindings no longer gets a CLICKHOUSE_URL, like its ECS task. Without this, terraform init fails on references to undeclared data sources after merging main. --- tf/environments/dev/main.tf | 7 +++---- tf/environments/prod/main.tf | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index c83c6ad1..8ded4312 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -284,20 +284,19 @@ locals { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.value ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value } oonirun = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.value } oonifindings = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value } ooniauth = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value @@ -311,7 +310,7 @@ locals { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_test_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.value ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value } } diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 39589634..e0c4d2e1 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -321,7 +321,7 @@ locals { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.value } oonifindings = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value From 1a7bdd6249785f1192ac9a21de05b0824eb154fe Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:01:55 +0000 Subject: [PATCH 192/201] deploy.py: log every command it runs Review: "We should add a log.info on this" (on run()). Every ssh, scp and aws command now goes through the logger with a timestamp, so the CodeBuild log shows what ran against which host and when, not just the start and end of each host's deploy. Secret values are only ever written to files, never put on a command line, so they don't reach the log. --- .../ooniapi_service_deployer/files/deploy.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 57cbf5c7..68ff53e3 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -18,6 +18,7 @@ # needed on top of docker compose. import json +import logging import os import subprocess import sys @@ -36,6 +37,8 @@ # host. Must match ooniapi_gateway_staging_dir in the Ansible role. STAGING_DIR = "/var/lib/ooniapi/deploy-staging" +log = logging.getLogger("deploy") + def require_env(name): value = os.environ.get(name) @@ -45,6 +48,9 @@ def require_env(name): def run(cmd, **kwargs): + # every remote and aws command goes through here; secret values are only + # ever written to files, never passed on a command line, so this is safe + log.info("running: %s", " ".join(cmd)) return subprocess.run(cmd, check=True, text=True, **kwargs) @@ -57,7 +63,9 @@ def ssh_output(user, host, remote_cmd): def ssh_succeeds(user, host, remote_cmd): - return subprocess.run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd]).returncode == 0 + cmd = ["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd] + log.info("running: %s", " ".join(cmd)) + return subprocess.run(cmd).returncode == 0 def scp(user, host, local_path, remote_path): @@ -93,14 +101,14 @@ def write_tmp(name, content, mode=0o644): def deploy_host(host, ctx): service = ctx["service"] user = ctx["user"] - print(f"=== {service}: deploying to {host} ===") + log.info(f"=== {service}: deploying to {host} ===") active_slot = ssh_output( user, host, f"cat /etc/ooniapi/{service}/active_slot 2>/dev/null || echo a" ).strip() or "a" target_slot = "b" if active_slot == "a" else "a" target_port = ctx["host_port_b"] if target_slot == "b" else ctx["host_port_a"] - print(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") + log.info(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") # compose file for the target slot compose_file = f"{service}-{target_slot}.yaml" @@ -145,15 +153,16 @@ def deploy_host(host, ctx): f" && sudo nginx -t && sudo systemctl reload nginx") ssh(user, host, f"echo {target_slot} | sudo tee /etc/ooniapi/{service}/active_slot > /dev/null") - print(f"=== {service} on {host}: now serving from slot {target_slot} ===") + log.info(f"=== {service} on {host}: now serving from slot {target_slot} ===") def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout) with open("imagedefinitions.json") as f: image_tag = json.load(f)[0]["imageUri"].rsplit(":", 1)[-1] service = require_env("SERVICE_NAME") - print(f"Deploying {service} image tag {image_tag}") + log.info(f"Deploying {service} image tag {image_tag}") with open("/tmp/deploy_key", "w") as f: f.write(secretsmanager_get(require_env("DEPLOY_SSH_KEY_SECRET_ARN"))) From ab096419aab1fc07bd32f0a23e30be26f70d429c Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:02:44 +0000 Subject: [PATCH 193/201] deploy.py: give a new slot as long as ECS gives a new task Review: "We should make sure 20s is enough". It wasn't comparable to what the same images get on ECS: the target groups (ooniapi_service) check /health every 30 s with a 5 s timeout and want 2 passes in a row, so a new task has about a minute to come up. The blue/green check gave 20 s, a single pass sufficed, and curl had no timeout of its own. Now each check has curl --max-time 5, the slot must pass twice in a row, and it has health_check_timeout seconds (default 120, per service) to do so. The time it took is logged, so the deadline can be tuned per service from real deploys; at 2 hosts per deploy it stays well within the deploy job's 20 minute build timeout. --- .../ooniapi_service_deployer/files/deploy.py | 26 +++++++++++++------ tf/modules/ooniapi_service_deployer/main.tf | 4 +++ .../ooniapi_service_deployer/variables.tf | 6 +++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 68ff53e3..426573fd 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -133,14 +133,23 @@ def deploy_host(host, ctx): ssh(user, host, f"sudo docker compose -f {compose_path} up -d --pull always --remove-orphans") - healthy = False - for _ in range(10): - if ssh_succeeds(user, host, f"curl -sf -o /dev/null http://127.0.0.1:{target_port}/health"): - healthy = True - break - time.sleep(2) - if not healthy: - sys.exit(f"{service} on {host}: slot {target_slot} failed health check, aborting deploy") + # like the ECS target groups' health check (ooniapi_service: timeout 5, + # healthy_threshold 2): each check may take up to 5 s, and the slot must + # pass twice in a row. ECS gives a new task about a minute to come up; + # the deadline defaults to twice that, and is logged so it can be tuned + started = time.monotonic() + passes = 0 + while passes < 2: + if ssh_succeeds(user, host, f"curl -sf --max-time 5 -o /dev/null http://127.0.0.1:{target_port}/health"): + passes += 1 + else: + passes = 0 + elapsed = time.monotonic() - started + if passes < 2 and elapsed > ctx["health_check_timeout"]: + sys.exit(f"{service} on {host}: slot {target_slot} not healthy after {elapsed:.0f}s, aborting deploy") + if passes < 2: + time.sleep(2) + log.info(f"{service} on {host}: slot {target_slot} healthy after {time.monotonic() - started:.0f}s") state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") upstream_file = f"{service}-upstream.conf" @@ -175,6 +184,7 @@ def main(): "image_tag": image_tag, "host_port_a": require_env("HOST_PORT_A"), "host_port_b": require_env("HOST_PORT_B"), + "health_check_timeout": int(os.environ.get("HEALTH_CHECK_TIMEOUT") or 120), "secrets": json.loads(secretsmanager_get(require_env("SERVICE_SECRETS_ARN"))), } diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index 46e90700..7a0ea9c3 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -347,6 +347,10 @@ resource "aws_codebuild_project" "deploy" { name = "DEPLOY_SSH_KEY_SECRET_ARN" value = var.deploy_ssh_key_secret_arn } + environment_variable { + name = "HEALTH_CHECK_TIMEOUT" + value = tostring(var.health_check_timeout) + } } logs_config { diff --git a/tf/modules/ooniapi_service_deployer/variables.tf b/tf/modules/ooniapi_service_deployer/variables.tf index b6056c78..9db59596 100644 --- a/tf/modules/ooniapi_service_deployer/variables.tf +++ b/tf/modules/ooniapi_service_deployer/variables.tf @@ -86,6 +86,12 @@ variable "host_port_b" { default = null } +variable "health_check_timeout" { + description = "Seconds a new blue/green slot has to pass its /health check twice in a row before the deploy is aborted. The ECS target groups give a new task about 60 s." + type = number + default = 120 +} + variable "container_port" { description = "Port the service listens on inside the container. Required when deploy_mode = \"blue_green\"." type = number From 0e205281421fa29851377c1429e168112b8371b7 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:03:42 +0000 Subject: [PATCH 194/201] deploy.py: restore the live upstream when nginx -t fails Review: "what happens if nginx -t fails". The new upstream conf was moved into conf.d before nginx -t, and left there when it failed. nginx kept serving its old config, but the next reload on that host (another service's deploy, a certificate renewal) would fail, and a restart would take every service on the host down. Now a failing nginx -t puts back the upstream of the slot that is still serving and checks it again, then aborts the deploy without reloading nginx or moving the active_slot marker. If nginx -t fails even then, the config is broken independently of this deploy, and the error says so. The restore uses the same mv the deploy user's sudoers already allows. --- .../ooniapi_service_deployer/files/deploy.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 426573fd..0d851ce0 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -151,15 +151,30 @@ def deploy_host(host, ctx): time.sleep(2) log.info(f"{service} on {host}: slot {target_slot} healthy after {time.monotonic() - started:.0f}s") - state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") upstream_file = f"{service}-upstream.conf" - upstream_content = s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") - upstream_content = upstream_content.replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) - upstream_local = write_tmp(upstream_file, upstream_content) - scp(user, host, upstream_local, f"{STAGING_DIR}/{upstream_file}") - ssh(user, host, - f"sudo mv {STAGING_DIR}/{upstream_file} /etc/nginx/conf.d/{upstream_file}" - f" && sudo nginx -t && sudo systemctl reload nginx") + upstream_template = s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") + + def install_upstream(live_slot): + state_a, state_b = ("", "down") if live_slot == "a" else ("down", "") + content = upstream_template.replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) + scp(user, host, write_tmp(upstream_file, content), f"{STAGING_DIR}/{upstream_file}") + ssh(user, host, f"sudo mv {STAGING_DIR}/{upstream_file} /etc/nginx/conf.d/{upstream_file}") + + install_upstream(target_slot) + if not ssh_succeeds(user, host, "sudo nginx -t"): + # nginx is still running its old config, but a config that fails + # nginx -t must not stay in conf.d: the next reload (another + # service's deploy, a certificate renewal) would fail, and a restart + # would take every service on this host down. Put back the upstream + # of the slot still serving, which nginx -t accepted before. + log.error(f"{service} on {host}: nginx -t failed with slot {target_slot}'s upstream, restoring slot {active_slot}'s") + install_upstream(active_slot) + if not ssh_succeeds(user, host, "sudo nginx -t"): + sys.exit(f"{service} on {host}: nginx -t still fails after restoring slot {active_slot}'s upstream:" + " the nginx config is broken independently of this deploy, fix it before reloading nginx") + sys.exit(f"{service} on {host}: nginx -t failed with slot {target_slot}'s upstream; restored slot" + f" {active_slot}'s, which is still serving. Aborting deploy") + ssh(user, host, "sudo systemctl reload nginx") ssh(user, host, f"echo {target_slot} | sudo tee /etc/ooniapi/{service}/active_slot > /dev/null") log.info(f"=== {service} on {host}: now serving from slot {target_slot} ===") From 608d3f0a9b1b379849cd959401b00be38a0a9125 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:04:58 +0000 Subject: [PATCH 195/201] Bound how long a slot taken out of rotation drains Review: "We should make sure there is some kind of timeout on connections while draining". After a flip, nginx workers from before the reload keep serving their in-flight requests to the old slot, with nothing bounding them but proxy_read_timeout (900 s), and long lived HTTP/2 connections can keep them longer. On ECS the ALB bounds the same thing with its deregistration delay, the AWS default of 300 s here. - ooniapi_gateway sets worker_shutdown_timeout to ooniapi_gateway_drain_timeout (300 s), through modules-enabled/, the only place included in nginx's main context. - The next deploy of a service replaces the slot that is draining, so deploy.py now waits until drain_timeout (300 s, per service) has passed since the previous flip, from the active_slot marker's mtime, before restarting it. A first deploy has no marker and does not wait. --- ansible/roles/ooniapi_gateway/defaults/main.yml | 7 +++++++ ansible/roles/ooniapi_gateway/tasks/main.yml | 14 ++++++++++++++ .../ooniapi_service_deployer/files/deploy.py | 14 ++++++++++++++ tf/modules/ooniapi_service_deployer/main.tf | 4 ++++ tf/modules/ooniapi_service_deployer/variables.tf | 6 ++++++ 5 files changed, 45 insertions(+) diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 0b28f33b..4f910dc7 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -66,3 +66,10 @@ ooniapi_gateway_deploy_user: deploy # moved/installed into place. Must match STAGING_DIR in # tf/modules/ooniapi_service_deployer/files/deploy.py. ooniapi_gateway_staging_dir: /var/lib/ooniapi/deploy-staging + +# How long nginx workers left over from a reload may keep serving in-flight +# requests (to the slot a deploy just took out of rotation) before they are +# closed. Matches the ALB's deregistration delay on ECS (AWS default, 300 s), +# and must match drain_timeout in ooniapi_service_deployer: a deploy waits +# this long after the previous flip before it restarts the drained slot. +ooniapi_gateway_drain_timeout: 300 diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 020790d3..2a9ac98a 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -203,6 +203,20 @@ mode: "0440" validate: "visudo -cf %s" +# worker_shutdown_timeout is only valid in nginx's main context, which +# conf.d can't reach; modules-enabled/*.conf is included there by both the +# nginx role's nginx.conf and Debian's stock one. +- name: Bound how long old nginx workers drain after a reload + ansible.builtin.copy: + dest: /etc/nginx/modules-enabled/90-ooniapi-gateway-drain.conf + content: | + # Managed by ansible - roles/ooniapi_gateway (ooniapi_gateway_drain_timeout) + worker_shutdown_timeout {{ ooniapi_gateway_drain_timeout }}s; + owner: root + group: root + mode: "0644" + notify: reload ooniapi gateway nginx + - name: Render the gateway vhost ansible.builtin.template: src: gateway.conf.j2 diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 0d851ce0..13892537 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -106,6 +106,19 @@ def deploy_host(host, ctx): active_slot = ssh_output( user, host, f"cat /etc/ooniapi/{service}/active_slot 2>/dev/null || echo a" ).strip() or "a" + + # the slot about to be replaced is the one the previous deploy took out + # of rotation: nginx workers from before that flip may still be serving + # requests to it, for up to drain_timeout (worker_shutdown_timeout in + # the ooniapi_gateway role). Don't restart it under them. + since_flip = int(ssh_output( + user, host, + f"echo $(( $(date +%s) - $(stat -c %Y /etc/ooniapi/{service}/active_slot 2>/dev/null || echo 0) ))", + ).strip()) + wait = ctx["drain_timeout"] - since_flip + if wait > 0: + log.info(f"{service} on {host}: previous deploy flipped {since_flip}s ago, waiting {wait}s for its old slot to drain") + time.sleep(wait) target_slot = "b" if active_slot == "a" else "a" target_port = ctx["host_port_b"] if target_slot == "b" else ctx["host_port_a"] log.info(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") @@ -200,6 +213,7 @@ def main(): "host_port_a": require_env("HOST_PORT_A"), "host_port_b": require_env("HOST_PORT_B"), "health_check_timeout": int(os.environ.get("HEALTH_CHECK_TIMEOUT") or 120), + "drain_timeout": int(os.environ.get("DRAIN_TIMEOUT") or 300), "secrets": json.loads(secretsmanager_get(require_env("SERVICE_SECRETS_ARN"))), } diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index 7a0ea9c3..5fbc02b3 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -351,6 +351,10 @@ resource "aws_codebuild_project" "deploy" { name = "HEALTH_CHECK_TIMEOUT" value = tostring(var.health_check_timeout) } + environment_variable { + name = "DRAIN_TIMEOUT" + value = tostring(var.drain_timeout) + } } logs_config { diff --git a/tf/modules/ooniapi_service_deployer/variables.tf b/tf/modules/ooniapi_service_deployer/variables.tf index 9db59596..5ce31a1e 100644 --- a/tf/modules/ooniapi_service_deployer/variables.tf +++ b/tf/modules/ooniapi_service_deployer/variables.tf @@ -92,6 +92,12 @@ variable "health_check_timeout" { default = 120 } +variable "drain_timeout" { + description = "Seconds requests may keep draining from a blue/green slot after it is taken out of rotation; must match ooniapi_gateway_drain_timeout on the hosts. A deploy waits this long after the previous one before restarting that slot. Mirrors the ALB deregistration delay (AWS default, 300 s)." + type = number + default = 300 +} + variable "container_port" { description = "Port the service listens on inside the container. Required when deploy_mode = \"blue_green\"." type = number From 16aa56bbbde02cbc96f8fd37110fb9a192195c40 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:18:27 +0000 Subject: [PATCH 196/201] Declare the ooniapi ALB routes in routes.yaml The routes of the API are going to be served by two frontends, the ALB for the services on ECS and the nginx gateway on the Hetzner hosts for those deployed blue/green, and their copies have already drifted: the gateway's hand written locations predate main's X-Protocol-Version rules. routes.yaml becomes the one place they are declared; this change has Terraform read it, the gateway follows. The 18 hand written listener rules become one aws_lb_listener_rule with for_each over routes.yaml, with moved blocks from their old addresses. Planned with fake ARNs (no state), the old and the new code produce the same 18 rules: same priorities, targets, conditions and tags. Applying should only move them in the state. The "hotfix" that made the oonimeasurements and testlists rules optional (count = arn != null ? 1 : 0) is removed rather than carried over. It had stopped working anyway: oonimeasurements_rule_3 lacked the count, so the frontend couldn't be planned without oonimeasurements. Both environments pass every target group, so those two inputs are now required and nullable = false. A missing service fails the plan with "Required variable not set" instead of silently losing its routes. The planned rules are unchanged. --- tf/modules/ooniapi_frontend/main.tf | 419 ++++++----------------- tf/modules/ooniapi_frontend/routes.yaml | 156 +++++++++ tf/modules/ooniapi_frontend/variables.tf | 4 +- 3 files changed, 254 insertions(+), 325 deletions(-) create mode 100644 tf/modules/ooniapi_frontend/routes.yaml diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 2e1bee8e..8b6d0650 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -261,378 +261,151 @@ resource "aws_alb_listener_rule" "ooniapi_th" { tags = var.tags } -resource "aws_lb_listener_rule" "ooniapi_ooniauth_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 108 +locals { + # routes.yaml is shared with the gateway on the Hetzner hosts + # (ansible/roles/ooniapi_gateway), so both serve the same routes + routes = yamldecode(file("${path.module}/routes.yaml")) - action { - type = "forward" - target_group_arn = var.ooniapi_ooniauth_target_group_arn + target_groups = { + ooniauth = var.ooniapi_ooniauth_target_group_arn + oonirun = var.ooniapi_oonirun_target_group_arn + ooniprobe = var.ooniapi_ooniprobe_target_group_arn + ooniprobe_legacy = var.ooniapi_ooniprobe_legacy_target_group_arn + oonifindings = var.ooniapi_oonifindings_target_group_arn + oonimeasurements = var.ooniapi_oonimeasurements_target_group_arn + testlists = var.ooniapi_testlists_target_group_arn } - condition { - path_pattern { - values = [ - "/api/v2/ooniauth/*", - "/api/v1/user_register", - "/api/v1/user_login", - "/api/v1/user_refresh_token", - "/api/_/account_metadata", - ] - } - } + listener_rules = { for r in local.routes : r.name => r } } -resource "aws_lb_listener_rule" "ooniapi_ooniauth_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 109 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniauth_target_group_arn - } - - condition { - host_header { - values = ["ooniauth.${local.direct_domain_suffix}"] - } - } -} +resource "aws_lb_listener_rule" "route" { + for_each = local.listener_rules -resource "aws_lb_listener_rule" "ooniapi_oonirun_rule" { listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 110 + priority = each.value.priority action { type = "forward" - target_group_arn = var.ooniapi_oonirun_target_group_arn + target_group_arn = local.target_groups[each.value.service] } - condition { - path_pattern { - values = ["/api/v2/oonirun/*"] + dynamic "condition" { + for_each = can(each.value.paths) ? [each.value.paths] : [] + content { + path_pattern { + values = condition.value + } } - } -} -resource "aws_lb_listener_rule" "ooniapi_oonirun_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 111 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonirun_target_group_arn - } - - condition { - host_header { - values = ["oonirun.${local.direct_domain_suffix}"] + dynamic "condition" { + for_each = try(each.value.direct_host, false) ? [each.value.service] : [] + content { + host_header { + values = ["${condition.value}.${local.direct_domain_suffix}"] + } } } -} - -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 120 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v2/ooniprobe/*", - "/api/v1/login", - "/api/v1/register", - "/api/v1/update/*", - "/api/v1/check-in*" - ] + dynamic "condition" { + for_each = can(each.value.http_header) ? [each.value.http_header] : [] + content { + http_header { + http_header_name = condition.value.name + values = condition.value.values + } } } } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_2" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 121 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/test-helpers*", - "/api/v1/test-list/urls", - "/report*", - "/api/_/show_countries_prioritization", - "/api/_/debug_prioritization" - ] - } - } +# the rules were one resource each before routes.yaml +moved { + from = aws_lb_listener_rule.ooniapi_ooniauth_rule + to = aws_lb_listener_rule.route["ooniauth_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_legacy_version" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 122 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_legacy_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/manifest*", - "/api/v1/submit_measurement*", - "/api/v1/sign_credential*" - ] - } - } - - condition { - http_header { - http_header_name = "X-Protocol-Version" - values = ["0.1.0"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniauth_rule_host + to = aws_lb_listener_rule.route["ooniauth_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_current_version" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 124 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/manifest*", - "/api/v1/submit_measurement*", - "/api/v1/sign_credential*" - ] - } - } - - # matches any value, but only when the header is present - condition { - http_header { - http_header_name = "X-Protocol-Version" - values = ["*"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonirun_rule + to = aws_lb_listener_rule.route["oonirun_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_no_version" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 126 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_legacy_target_group_arn - } - - # No X-Protocol-Version header at all: rules 122/124 above already - # matched every request that does carry the header, so anything left - # here is header-less and should go to the legacy service. - condition { - path_pattern { - values = [ - "/api/v1/manifest*", - "/api/v1/submit_measurement*", - "/api/v1/sign_credential*" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonirun_rule_host + to = aws_lb_listener_rule.route["oonirun_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_4" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 123 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/bouncer/net-tests*", - "/api/v1/geolookup*", - "/api/v1/collectors*", - "/api/v1/test-list/tor-targets", - "/api/v1/test-list/psiphon-config" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule + to = aws_lb_listener_rule.route["ooniprobe_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 125 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - - condition { - host_header { - values = ["ooniprobe.${local.direct_domain_suffix}"] - } - } - +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_2 + to = aws_lb_listener_rule.route["ooniprobe_rule_2"] } -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 130 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonifindings_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/incidents/*", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_legacy_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_legacy_version"] } -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 131 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonifindings_target_group_arn - } - condition { - host_header { - values = ["oonifindings.${local.direct_domain_suffix}"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_4 + to = aws_lb_listener_rule.route["ooniprobe_rule_4"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_host" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 139 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } - condition { - host_header { - values = ["oonimeasurements.${local.direct_domain_suffix}"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_current_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_current_version"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_1" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 140 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/measurements/*", - "/api/v1/raw_measurement", - "/api/v1/measurement_meta", - "/api/v1/measurements", - "/api/v1/torsf_stats" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_host + to = aws_lb_listener_rule.route["ooniprobe_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 142 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_no_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_no_version"] +} - condition { - path_pattern { - values = [ - "/api/v1/aggregation", - "/api/v1/aggregation/*", - "/api/v1/observations", - "/api/v1/analysis", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonifindings_rule + to = aws_lb_listener_rule.route["oonifindings_rule"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_3" { +moved { + from = aws_lb_listener_rule.ooniapi_oonifindings_rule_host + to = aws_lb_listener_rule.route["oonifindings_rule_host"] +} - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 143 +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_host[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_host"] +} - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_1[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_1"] +} - condition { - path_pattern { - values = [ - "/api/v1/detector/changepoints", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_2[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_2"] } -resource "aws_lb_listener_rule" "ooniapi_testlists_rule" { - # hotfix: to allow us to deploy the frontend without the testlists service - count = var.ooniapi_testlists_target_group_arn != null ? 1 : 0 - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 144 +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_3 + to = aws_lb_listener_rule.route["oonimeasurements_rule_3"] +} - action { - type = "forward" - target_group_arn = var.ooniapi_testlists_target_group_arn - } - condition { - path_pattern { - values = [ - "/api/_/url-submission/test-list/*", - "/api/_/url-priorities/list", - "/api/_/url-priorities/update", - "/api/v1/url-submission/submit", - "/api/v1/url-submission/update-url", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_testlists_rule[0] + to = aws_lb_listener_rule.route["testlists_rule"] } diff --git a/tf/modules/ooniapi_frontend/routes.yaml b/tf/modules/ooniapi_frontend/routes.yaml new file mode 100644 index 00000000..b32462df --- /dev/null +++ b/tf/modules/ooniapi_frontend/routes.yaml @@ -0,0 +1,156 @@ +# Routes of the OONI API, the one place they are declared. Read by: +# - Terraform, tf/modules/ooniapi_frontend: one ALB listener rule per entry +# (aws_lb_listener_rule.route[""]), for the services on ECS. +# Anything no route matches goes to reverseproxy. +# +# Each entry is one ALB rule, so the ALB's limits apply: at most 5 values +# over all its conditions. Fields: +# name resource key of the rule; keep stable, renaming one makes +# Terraform replace the rule +# priority ALB rule priority, lowest matches first +# service target service (ALB target group) +# paths ALB path patterns: exact, or ending in * for a prefix +# direct_host route . to the service +# http_header only match requests with this header and one of these values + +- name: ooniauth_rule + priority: 108 + service: ooniauth + paths: + - "/api/v2/ooniauth/*" + - "/api/v1/user_register" + - "/api/v1/user_login" + - "/api/v1/user_refresh_token" + - "/api/_/account_metadata" + +- name: ooniauth_rule_host + priority: 109 + service: ooniauth + direct_host: true + +- name: oonirun_rule + priority: 110 + service: oonirun + paths: + - "/api/v2/oonirun/*" + +- name: oonirun_rule_host + priority: 111 + service: oonirun + direct_host: true + +- name: ooniprobe_rule + priority: 120 + service: ooniprobe + paths: + - "/api/v2/ooniprobe/*" + - "/api/v1/login" + - "/api/v1/register" + - "/api/v1/update/*" + - "/api/v1/check-in*" + +- name: ooniprobe_rule_2 + priority: 121 + service: ooniprobe + paths: + - "/api/v1/test-helpers*" + - "/api/v1/test-list/urls" + - "/report*" + - "/api/_/show_countries_prioritization" + - "/api/_/debug_prioritization" + +- name: ooniprobe_rule_3_legacy_version + priority: 122 + service: ooniprobe_legacy + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + http_header: + name: X-Protocol-Version + values: ["0.1.0"] + +- name: ooniprobe_rule_4 + priority: 123 + service: ooniprobe + paths: + - "/bouncer/net-tests*" + - "/api/v1/geolookup*" + - "/api/v1/collectors*" + - "/api/v1/test-list/tor-targets" + - "/api/v1/test-list/psiphon-config" + +- name: ooniprobe_rule_3_current_version + priority: 124 + service: ooniprobe + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + http_header: + name: X-Protocol-Version + values: ["*"] + +- name: ooniprobe_rule_host + priority: 125 + service: ooniprobe + direct_host: true + +- name: ooniprobe_rule_3_no_version + priority: 126 + service: ooniprobe_legacy + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + +- name: oonifindings_rule + priority: 130 + service: oonifindings + paths: + - "/api/v1/incidents/*" + +- name: oonifindings_rule_host + priority: 131 + service: oonifindings + direct_host: true + +- name: oonimeasurements_rule_host + priority: 139 + service: oonimeasurements + direct_host: true + +- name: oonimeasurements_rule_1 + priority: 140 + service: oonimeasurements + paths: + - "/api/v1/measurements/*" + - "/api/v1/raw_measurement" + - "/api/v1/measurement_meta" + - "/api/v1/measurements" + - "/api/v1/torsf_stats" + +- name: oonimeasurements_rule_2 + priority: 142 + service: oonimeasurements + paths: + - "/api/v1/aggregation" + - "/api/v1/aggregation/*" + - "/api/v1/observations" + - "/api/v1/analysis" + +- name: oonimeasurements_rule_3 + priority: 143 + service: oonimeasurements + paths: + - "/api/v1/detector/changepoints" + +- name: testlists_rule + priority: 144 + service: testlists + paths: + - "/api/_/url-submission/test-list/*" + - "/api/_/url-priorities/list" + - "/api/_/url-priorities/update" + - "/api/v1/url-submission/submit" + - "/api/v1/url-submission/update-url" diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index 614056e6..47ff4f10 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -38,12 +38,12 @@ variable "ooniapi_oonifindings_target_group_arn" { variable "ooniapi_oonimeasurements_target_group_arn" { description = "arn for the target group of the oonimeasurements service" - default = null + nullable = false } variable "ooniapi_testlists_target_group_arn" { description = "arn for the target group of the testlists service" - default = null + nullable = false } variable "dns_zone_ooni_io" { From fbb91b852e0be63585cc370061eabceb7a3a0bd3 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:21:25 +0000 Subject: [PATCH 197/201] Generate the gateway's nginx locations from routes.yaml The gateway's locations were a hand written copy of the ALB rules, and had already fallen behind them (main's X-Protocol-Version rules). They are now generated from tf/modules/ooniapi_frontend/routes.yaml, which the ALB rules are generated from too, as are the per service direct-host vhosts. nginx can't follow ALB priorities: it picks an exact match, then the longest prefix. filter_plugins/ooniapi_routes.py turns each ALB pattern into an exact or ^~ prefix location and lists the routes nginx couldn't serve the same way (a wildcard other than a trailing *, the same path twice, a prefix of one service covering a path of another); the role fails on any. A plugin rather than inline Jinja, so the result doesn't depend on how a given ansible-core converts templated strings. Routes marked legacy (ooniprobe_legacy, being retired) are left to the ALB: the gateway doesn't run it, so those paths go to ooniprobe. Rendered with ansible-core 2.21 for all six services plus testlists proxied cross-cloud, the generated config has the same 46 locations, with the same match types and upstreams, as the hand written one. --- .../roles/ooniapi_gateway/defaults/main.yml | 4 + .../filter_plugins/ooniapi_routes.py | 63 +++++++++++++ ansible/roles/ooniapi_gateway/tasks/main.yml | 3 + .../roles/ooniapi_gateway/tasks/routes.yml | 14 +++ .../ooniapi_gateway/templates/gateway.conf.j2 | 89 ++++--------------- tf/modules/ooniapi_frontend/routes.yaml | 13 ++- 6 files changed, 112 insertions(+), 74 deletions(-) create mode 100644 ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py create mode 100644 ansible/roles/ooniapi_gateway/tasks/routes.yml diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 4f910dc7..4d5d50b8 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -57,6 +57,10 @@ ooniapi_gateway_testlists_upstream: "" # Docker network shared by every service's containers on this host. ooniapi_gateway_network_name: ooniapi +# Routes of the API, shared with the ALB (see the file's header); the +# gateway's locations are generated from them in tasks/main.yml +ooniapi_gateway_routes: "{{ lookup('ansible.builtin.file', role_path ~ '/../../../tf/modules/ooniapi_frontend/routes.yaml') | from_yaml }}" + # SSH user the deploy CodeBuild job connects as. Must match # `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer # Terraform module. diff --git a/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py new file mode 100644 index 00000000..61c0e0c1 --- /dev/null +++ b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py @@ -0,0 +1,63 @@ +# Filters turning tf/modules/ooniapi_frontend/routes.yaml (shared with the +# ALB) into the gateway's nginx locations; see tasks/main.yml. +# +# nginx picks a location by exact match, then longest prefix, where the ALB +# goes by rule priority. The two agree as long as every ALB pattern is exact +# or a prefix (a trailing *), and no prefix of one service covers a path of +# another; ooniapi_gateway_route_problems lists the routes where they would +# not. + + +def _locations(routes): + out = [] + for r in sorted(routes, key=lambda r: r["priority"]): + # the ALB sends these to ooniprobe_legacy, which the gateway doesn't + # run: they go to ooniprobe, whatever their X-Protocol-Version + if r.get("legacy"): + continue + for p in r.get("paths", []): + prefix = p.endswith("*") + out.append({ + "route": r["name"], + "service": r["service"], + "match": "^~" if prefix else "=", + "path": p[:-1] if prefix else p, + }) + return out + + +def ooniapi_gateway_locations(routes): + return _locations(routes) + + +def ooniapi_gateway_direct_hosts(routes): + return list(dict.fromkeys(r["service"] for r in routes if r.get("direct_host"))) + + +def ooniapi_gateway_route_problems(routes): + locations = _locations(routes) + problems = [] + seen = {} + for loc in locations: + if "*" in loc["path"] or "?" in loc["path"]: + problems.append(f"{loc['route']}: {loc['path']}: only exact paths and a trailing * can be served by nginx") + key = (loc["match"], loc["path"]) + if key in seen: + problems.append(f"{loc['route']}: {loc['path']} is also routed by {seen[key]}") + seen.setdefault(key, loc["route"]) + for a in locations: + if a["match"] != "^~": + continue + for b in locations: + if b["service"] != a["service"] and b["path"].startswith(a["path"]): + problems.append(f"{a['route']}: {a['path']}* covers {b['route']}'s {b['path']}, nginx would not follow ALB priorities") + return problems + + +class FilterModule: + def filters(self): + return { + "ooniapi_gateway_locations": ooniapi_gateway_locations, + "ooniapi_gateway_direct_hosts": ooniapi_gateway_direct_hosts, + "ooniapi_gateway_route_problems": ooniapi_gateway_route_problems, + } diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 2a9ac98a..7fc36e8b 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -217,6 +217,9 @@ mode: "0644" notify: reload ooniapi gateway nginx +- name: Derive and check the gateway's locations + ansible.builtin.import_tasks: routes.yml + - name: Render the gateway vhost ansible.builtin.template: src: gateway.conf.j2 diff --git a/ansible/roles/ooniapi_gateway/tasks/routes.yml b/ansible/roles/ooniapi_gateway/tasks/routes.yml new file mode 100644 index 00000000..a6478c7c --- /dev/null +++ b/ansible/roles/ooniapi_gateway/tasks/routes.yml @@ -0,0 +1,14 @@ +--- +# The gateway's locations are generated from routes.yaml, shared with the +# ALB, by filter_plugins/ooniapi_routes.py, which also checks nginx can +# serve them the way the ALB does. +- name: Derive the gateway's locations from routes.yaml + ansible.builtin.set_fact: + ooniapi_gateway_locations: "{{ ooniapi_gateway_routes | ooniapi_gateway_locations }}" + ooniapi_gateway_direct_hosts: "{{ ooniapi_gateway_routes | ooniapi_gateway_direct_hosts }}" + ooniapi_gateway_route_problems: "{{ ooniapi_gateway_routes | ooniapi_gateway_route_problems }}" + +- name: Check that nginx can serve routes.yaml like the ALB does + ansible.builtin.assert: + that: ooniapi_gateway_route_problems | length == 0 + fail_msg: "{{ ooniapi_gateway_route_problems }}" diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 index e87e46ef..b7a32a92 100644 --- a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -1,11 +1,12 @@ # Managed by ansible - roles/ooniapi_gateway/templates/gateway.conf.j2 # -# Single "gateway" vhost for the OONI tier0 API on this dedicated host, -# mirroring (1:1, by design) the AWS ALB listener rules in -# tf/modules/ooniapi_frontend/main.tf. Only rules for services actually -# running blue/green here ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }}) -# are handled; everything unmatched falls through to "reverseproxy", exactly -# like the ALB's default_action. +# Single "gateway" vhost for the OONI tier0 API on this dedicated host. Its +# locations are generated from tf/modules/ooniapi_frontend/routes.yaml, which +# the AWS ALB's listener rules are generated from too, so both route alike. +# Only routes of services running blue/green here +# ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }}) and of +# those proxied cross-cloud are handled; everything unmatched falls through +# to "reverseproxy", exactly like the ALB's default_action. # # Each service's own `-upstream.conf` (the `upstream { # ... }` block referenced below via proxy_pass) is dropped into this same @@ -63,70 +64,16 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 900; -{% if 'ooniauth' in service_names %} - ## --- ooniauth --- (ALB priority 108) - location ^~ /api/v2/ooniauth/ { proxy_pass http://ooniauth; } - location = /api/v1/user_register { proxy_pass http://ooniauth; } - location = /api/v1/user_login { proxy_pass http://ooniauth; } - location = /api/v1/user_refresh_token { proxy_pass http://ooniauth; } - location = /api/_/account_metadata { proxy_pass http://ooniauth; } -{% endif %} - -{% if 'oonirun' in service_names %} - ## --- oonirun --- (ALB priority 110) - location ^~ /api/v2/oonirun/ { proxy_pass http://oonirun; } -{% endif %} - -{% if 'ooniprobe' in service_names %} - ## --- ooniprobe --- (ALB priorities 120-123) - location ^~ /api/v2/ooniprobe/ { proxy_pass http://ooniprobe; } - location = /api/v1/login { proxy_pass http://ooniprobe; } - location = /api/v1/register { proxy_pass http://ooniprobe; } - location ^~ /api/v1/update/ { proxy_pass http://ooniprobe; } - location ^~ /api/v1/check-in { proxy_pass http://ooniprobe; } - location ^~ /api/v1/test-helpers { proxy_pass http://ooniprobe; } - location = /api/v1/test-list/urls { proxy_pass http://ooniprobe; } - location ^~ /report { proxy_pass http://ooniprobe; } - location = /api/_/show_countries_prioritization { proxy_pass http://ooniprobe; } - location = /api/_/debug_prioritization { proxy_pass http://ooniprobe; } - location ^~ /api/v1/manifest { proxy_pass http://ooniprobe; } - location ^~ /api/v1/sign_credential { proxy_pass http://ooniprobe; } - location ^~ /api/v1/submit_measurement { proxy_pass http://ooniprobe; } - location ^~ /bouncer/net-tests { proxy_pass http://ooniprobe; } - location ^~ /api/v1/geolookup { proxy_pass http://ooniprobe; } - location ^~ /api/v1/collectors { proxy_pass http://ooniprobe; } - location = /api/v1/test-list/tor-targets { proxy_pass http://ooniprobe; } - location = /api/v1/test-list/psiphon-config { proxy_pass http://ooniprobe; } -{% endif %} - -{% if 'oonifindings' in service_names %} - ## --- oonifindings --- (ALB priority 130) - location ^~ /api/v1/incidents/ { proxy_pass http://oonifindings; } -{% endif %} - -{% if 'oonimeasurements' in service_names %} - ## --- oonimeasurements --- (ALB priorities 140-143) - location ^~ /api/v1/measurements/ { proxy_pass http://oonimeasurements; } - location = /api/v1/raw_measurement { proxy_pass http://oonimeasurements; } - location = /api/v1/measurement_meta { proxy_pass http://oonimeasurements; } - location = /api/v1/measurements { proxy_pass http://oonimeasurements; } - location = /api/v1/torsf_stats { proxy_pass http://oonimeasurements; } - location = /api/v1/aggregation { proxy_pass http://oonimeasurements; } - location ^~ /api/v1/aggregation/ { proxy_pass http://oonimeasurements; } - location = /api/v1/observations { proxy_pass http://oonimeasurements; } - location = /api/v1/analysis { proxy_pass http://oonimeasurements; } - location = /api/v1/detector/changepoints { proxy_pass http://oonimeasurements; } -{% endif %} - -{% if ooniapi_gateway_testlists_upstream %} - ## --- testlists (ALB priority 144; not yet migrated off AWS, proxied - ## cross-cloud until it is) --- - location ^~ /api/_/url-submission/test-list/ { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/_/url-priorities/list { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/_/url-priorities/update { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/v1/url-submission/submit { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/v1/url-submission/update-url { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } +{% set remote_upstreams = {'testlists': 'https://' ~ ooniapi_gateway_testlists_upstream} if ooniapi_gateway_testlists_upstream else {} %} + ## --- generated from routes.yaml, see tasks/main.yml --- +{% for l in ooniapi_gateway_locations %} +{% if l.service in service_names %} + location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }}; } # {{ l.route }} +{% elif l.service in remote_upstreams %} + # {{ l.service }} is not yet migrated off AWS, proxied cross-cloud until it is + location {{ l.match }} {{ l.path }} { proxy_pass {{ remote_upstreams[l.service] }}; } # {{ l.route }} {% endif %} +{% endfor %} ## --- default: everything else goes to reverseproxy, same as the ALB's ## default_action. Covers legacy/unmigrated paths (oonith is NOT among @@ -137,8 +84,7 @@ server { } } -{% for service in service_names %} -{% if service != 'reverseproxy' %} +{% for service in ooniapi_gateway_direct_hosts if service in service_names %} server { listen 443 ssl http2; listen [::]:443 ssl http2; @@ -163,5 +109,4 @@ server { proxy_pass http://{{ service }}; } } -{% endif %} {% endfor %} diff --git a/tf/modules/ooniapi_frontend/routes.yaml b/tf/modules/ooniapi_frontend/routes.yaml index b32462df..3a0bb8fe 100644 --- a/tf/modules/ooniapi_frontend/routes.yaml +++ b/tf/modules/ooniapi_frontend/routes.yaml @@ -1,17 +1,24 @@ # Routes of the OONI API, the one place they are declared. Read by: # - Terraform, tf/modules/ooniapi_frontend: one ALB listener rule per entry # (aws_lb_listener_rule.route[""]), for the services on ECS. -# Anything no route matches goes to reverseproxy. +# - Ansible, roles/ooniapi_gateway: the nginx locations of the gateway on the +# Hetzner hosts, for the services deployed there blue/green. +# Anything no route matches goes to reverseproxy, on both. # # Each entry is one ALB rule, so the ALB's limits apply: at most 5 values # over all its conditions. Fields: # name resource key of the rule; keep stable, renaming one makes # Terraform replace the rule -# priority ALB rule priority, lowest matches first +# priority ALB rule priority, lowest matches first. The gateway can't +# order its locations by it, so a prefix of one service must +# not cover a path of another (checked by the gateway role) # service target service (ALB target group) # paths ALB path patterns: exact, or ending in * for a prefix # direct_host route . to the service # http_header only match requests with this header and one of these values +# (ALB only: they only choose between ooniprobe and its legacy) +# legacy ALB only: ooniprobe_legacy is being retired and does not run +# on the gateway, which sends these paths to ooniprobe - name: ooniauth_rule priority: 108 @@ -69,6 +76,7 @@ http_header: name: X-Protocol-Version values: ["0.1.0"] + legacy: true - name: ooniprobe_rule_4 priority: 123 @@ -103,6 +111,7 @@ - "/api/v1/manifest*" - "/api/v1/submit_measurement*" - "/api/v1/sign_credential*" + legacy: true - name: oonifindings_rule priority: 130 From e8a184ee2fa27f02d4a6e7e16cc5e1049844e5d2 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:23:28 +0000 Subject: [PATCH 198/201] ooniapi_gateway: use a syslog tag nginx accepts nginx only allows letters, digits and underscores in a syslog tag, so nginx -t rejected the gateway vhost ("syslog \"tag\" only allows alphanumeric characters and underscore", nginx 1.27) and it could never have been installed. The tag is now ooniapi_gateway. --- ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 index b7a32a92..76782f9e 100644 --- a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -32,8 +32,8 @@ server { listen [::]:443 ssl http2; server_name {{ ([ooniapi_gateway_primary_domain] + ooniapi_gateway_extra_server_names) | join(' ') }}; - access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; - error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + access_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info; client_max_body_size 200M; # for measurement POST, matches the legacy API vhost @@ -90,8 +90,8 @@ server { listen [::]:443 ssl http2; server_name {{ service }}.{{ ooniapi_gateway_direct_domain_suffix }}; - access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; - error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + access_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info; ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; From ac8df6d6a23b63045a170e9b9204b5a472bf81ff Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:23:39 +0000 Subject: [PATCH 199/201] ooniapi_gateway: cache responses of the read heavy routes Nothing caches oonimeasurements or oonirun today: on api.ooni.org every repeated request reaches uvicorn (only the reverseproxy's /api/_/ paths show x-cache-status: HIT), and for oonimeasurements that means ClickHouse, saturated on data2. The gateway now caches GET/HEAD responses of the routes routes.yaml marks cache (oonimeasurements, oonirun), as long as their Cache-Control allows, or 10 s without one, like the legacy ooni-api vhost's apicache. Requests with an Authorization header never use the cache (oonirun links can differ per user), and nginx doesn't store no-cache, private or max-age=0 responses, or any setting a cookie. Two zones, since nginx keeps an entry until it goes unused for inactive, expired or not: lists and aggregations valid for minutes would otherwise push out measurement bodies valid for a day. Sized per host as if it got all the traffic, from data2's query_log and responses of api.ooni.org: - long (raw_measurement, measurement_meta; valid 1 d): at most ~50K distinct bodies a day of ~76 KB (max seen ~160 KB), ~3.8 GB a day: max_size 8g, keys_zone 16m (~130K keys), inactive 1d. - short (everything else cached; valid 10 s to 1 h): ~7.5K requests an hour of up to ~60 KB, ~450 MB an hour if all were distinct: max_size 2g, keys_zone 8m, inactive 1h. The access log now records $upstream_cache_status, to tune these from real hit rates. Cache directives and X-Cache-Status are set per server, since an add_header in a location would drop the server's HSTS headers. Checked with nginx 1.27 and stub upstreams: lists, bodies (long zone) and header-less aggregations miss then hit; Authorization bypasses and isn't stored; no-cache isn't stored; ooniprobe and the default route aren't cached; every response keeps HSTS. --- .../roles/ooniapi_gateway/defaults/main.yml | 21 +++++++++++++++ .../filter_plugins/ooniapi_routes.py | 10 +++++++ ansible/roles/ooniapi_gateway/tasks/main.yml | 9 +++++++ .../ooniapi_gateway/templates/gateway.conf.j2 | 27 +++++++++++++++++-- tf/modules/ooniapi_frontend/routes.yaml | 11 ++++++++ 5 files changed, 76 insertions(+), 2 deletions(-) diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 4d5d50b8..6a78c903 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -61,6 +61,27 @@ ooniapi_gateway_network_name: ooniapi # gateway's locations are generated from them in tasks/main.yml ooniapi_gateway_routes: "{{ lookup('ansible.builtin.file', role_path ~ '/../../../tf/modules/ooniapi_frontend/routes.yaml') | from_yaml }}" +# Response caches for the routes marked cache/cache_long in routes.yaml. +# nginx keeps an entry until it goes unused for `inactive`, expired or not, +# so responses valid for minutes and measurements valid for a day get zones +# of their own, and the first can't push the second out. Sized per host, as +# if it got all the traffic (2026-10, data2 query_log and api.ooni.org): +# - long: measurement bodies (raw_measurement, measurement_meta), valid 1 d: +# at most ~50K distinct a day (28K measurement_meta + 25K raw lookups) of +# ~76 KB (max seen ~160 KB), ~3.8 GB a day; 2x that, ~130K keys. +# - short: lists, aggregations, oonirun (valid 10 s to 1 h): ~7.5K requests +# an hour of up to ~60 KB (aggregations 0.3-22 KB, lists 35-70 KB, +# observations ~135 KB), ~450 MB an hour if all were distinct; 4x that. +# 1 MB of keys_zone holds ~8K keys. nginx stores responses uncompressed. +ooniapi_gateway_cache_dir: /var/cache/nginx/ooniapi-gateway +ooniapi_gateway_cache_zones: + short: {keys_zone: 8m, max_size: 2g, inactive: 1h} + long: {keys_zone: 16m, max_size: 8g, inactive: 1d} +# for responses without Cache-Control, like /api/v1/aggregation's today; +# the same as the legacy ooni-api vhost's apicache +ooniapi_gateway_cache_default_valid: 10s +ooniapi_gateway_nginx_user: www-data + # SSH user the deploy CodeBuild job connects as. Must match # `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer # Terraform module. diff --git a/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py index 61c0e0c1..1a17b46d 100644 --- a/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py +++ b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py @@ -17,11 +17,17 @@ def _locations(routes): continue for p in r.get("paths", []): prefix = p.endswith("*") + cache = None + if p in r.get("cache_long", []): + cache = "long" + elif r.get("cache"): + cache = "short" out.append({ "route": r["name"], "service": r["service"], "match": "^~" if prefix else "=", "path": p[:-1] if prefix else p, + "cache": cache, }) return out @@ -38,6 +44,10 @@ def ooniapi_gateway_route_problems(routes): locations = _locations(routes) problems = [] seen = {} + for r in routes: + for p in r.get("cache_long", []): + if p not in r.get("paths", []): + problems.append(f"{r['name']}: cache_long {p} is not one of its paths") for loc in locations: if "*" in loc["path"] or "?" in loc["path"]: problems.append(f"{loc['route']}: {loc['path']}: only exact paths and a trailing * can be served by nginx") diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 7fc36e8b..5e812bc1 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -220,6 +220,15 @@ - name: Derive and check the gateway's locations ansible.builtin.import_tasks: routes.yml +- name: Ensure the gateway's cache directories exist + ansible.builtin.file: + path: "{{ ooniapi_gateway_cache_dir }}/{{ item }}" + state: directory + owner: "{{ ooniapi_gateway_nginx_user }}" + group: "{{ ooniapi_gateway_nginx_user }}" + mode: "0700" + loop: "{{ ooniapi_gateway_cache_zones.keys() | list }}" + - name: Render the gateway vhost ansible.builtin.template: src: gateway.conf.j2 diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 index 76782f9e..dca63a30 100644 --- a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -24,9 +24,15 @@ map $remote_addr $ooniapi_gateway_remote_addr_anon { default 0.0.0.0; } -log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon [$time_local] ' +log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon $upstream_cache_status [$time_local] ' '"$request" $status $body_bytes_sent rt:$request_time "$http_referer" "$http_user_agent"'; +# response caches, see ooniapi_gateway_cache_zones in defaults/main.yml +{% for name, zone in ooniapi_gateway_cache_zones.items() %} +proxy_cache_path {{ ooniapi_gateway_cache_dir }}/{{ name }} levels=1:2 keys_zone=ooniapi_{{ name }}:{{ zone.keys_zone }} + max_size={{ zone.max_size }} inactive={{ zone.inactive }} use_temp_path=off; +{% endfor %} + server { listen 443 ssl http2; listen [::]:443 ssl http2; @@ -64,11 +70,28 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 900; + # caching, for the locations of routes marked cache/cache_long in + # routes.yaml (proxy_cache is off elsewhere). Responses are cached as long + # as their Cache-Control allows (nginx skips no-cache, private, max-age=0 + # and those setting cookies), or ooniapi_gateway_cache_default_valid + # without one. Requests with credentials never use the cache. + proxy_cache_key "$scheme$host$request_uri"; + proxy_cache_methods GET HEAD; + proxy_cache_valid 200 301 302 {{ ooniapi_gateway_cache_default_valid }}; + proxy_cache_valid any 0; + proxy_cache_lock on; + proxy_cache_lock_timeout 30s; + proxy_cache_lock_age 30s; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_bypass $http_authorization; + proxy_no_cache $http_authorization; + add_header X-Cache-Status $upstream_cache_status always; + {% set remote_upstreams = {'testlists': 'https://' ~ ooniapi_gateway_testlists_upstream} if ooniapi_gateway_testlists_upstream else {} %} ## --- generated from routes.yaml, see tasks/main.yml --- {% for l in ooniapi_gateway_locations %} {% if l.service in service_names %} - location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }}; } # {{ l.route }} + location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }};{% if l.cache %} proxy_cache ooniapi_{{ l.cache }};{% endif %} } # {{ l.route }} {% elif l.service in remote_upstreams %} # {{ l.service }} is not yet migrated off AWS, proxied cross-cloud until it is location {{ l.match }} {{ l.path }} { proxy_pass {{ remote_upstreams[l.service] }}; } # {{ l.route }} diff --git a/tf/modules/ooniapi_frontend/routes.yaml b/tf/modules/ooniapi_frontend/routes.yaml index 3a0bb8fe..ea1e430c 100644 --- a/tf/modules/ooniapi_frontend/routes.yaml +++ b/tf/modules/ooniapi_frontend/routes.yaml @@ -19,6 +19,10 @@ # (ALB only: they only choose between ooniprobe and its legacy) # legacy ALB only: ooniprobe_legacy is being retired and does not run # on the gateway, which sends these paths to ooniprobe +# cache gateway only: cache GET/HEAD responses of these paths in the +# short lived zone (see roles/ooniapi_gateway defaults) +# cache_long gateway only: those of its paths to cache in the long lived zone +# instead, for responses of immutable measurements - name: ooniauth_rule priority: 108 @@ -40,6 +44,7 @@ service: oonirun paths: - "/api/v2/oonirun/*" + cache: true - name: oonirun_rule_host priority: 111 @@ -138,6 +143,10 @@ - "/api/v1/measurement_meta" - "/api/v1/measurements" - "/api/v1/torsf_stats" + cache: true + cache_long: + - "/api/v1/raw_measurement" + - "/api/v1/measurement_meta" - name: oonimeasurements_rule_2 priority: 142 @@ -147,12 +156,14 @@ - "/api/v1/aggregation/*" - "/api/v1/observations" - "/api/v1/analysis" + cache: true - name: oonimeasurements_rule_3 priority: 143 service: oonimeasurements paths: - "/api/v1/detector/changepoints" + cache: true - name: testlists_rule priority: 144 From 525c6575ce10451771ac252566d6fa9633099861 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:25:10 +0000 Subject: [PATCH 200/201] Invalidate a service's gateway cache when it is deployed A deploy can change what a service returns, but the gateway would keep serving responses cached from the old slot, for up to a day for measurement bodies. nginx without the commercial purge API can't drop entries, so each service's cache key now starts with a version: the image tag, which deploy.py writes into -upstream.conf as $ooniapi_cache_version_, the same file and the same reload that put the new slot in rotation. From then on the old entries are never looked up and age out of the zones; other services' caches are left alone. (The slot letter wouldn't do: slots alternate, so it would bring back responses from two deploys ago.) When nginx -t rejects the new upstream conf, deploy.py now puts the previous file back as it was, cache version included, rather than rendering it again. The image tag is checked to be a plain tag, since it ends up in an nginx string. The placeholder upstream conf the role seeds before a first deploy defines an empty version. Checked with nginx 1.27: after a deploy of oonimeasurements its cached list missed (new upstream response) then hit again, while oonirun's cached response kept hitting; redeploying the previous tag found that tag's entries again, as long as they were still valid. --- ansible/roles/ooniapi_gateway/tasks/main.yml | 3 ++ .../ooniapi_gateway/templates/gateway.conf.j2 | 8 +++-- .../ooniapi_service_deployer/files/deploy.py | 32 +++++++++++++------ .../templates/nginx_upstream.conf.tftpl | 8 +++++ 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 5e812bc1..8b1594c4 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -67,6 +67,9 @@ ansible.builtin.copy: dest: "/etc/nginx/conf.d/{{ item.item.name }}-upstream.conf" content: | + map "" $ooniapi_cache_version_{{ item.item.name }} { + default ""; + } upstream {{ item.item.name }} { server 127.0.0.1:{{ item.item.host_port_a }}; server 127.0.0.1:{{ item.item.host_port_b }} down; diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 index dca63a30..a6563a5b 100644 --- a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -74,8 +74,10 @@ server { # routes.yaml (proxy_cache is off elsewhere). Responses are cached as long # as their Cache-Control allows (nginx skips no-cache, private, max-age=0 # and those setting cookies), or ooniapi_gateway_cache_default_valid - # without one. Requests with credentials never use the cache. - proxy_cache_key "$scheme$host$request_uri"; + # without one. Requests with credentials never use the cache. Each cached + # location keys on its service's $ooniapi_cache_version_, the + # image tag its blue/green deploy writes to -upstream.conf, so a + # deploy invalidates that service's responses, and only those. proxy_cache_methods GET HEAD; proxy_cache_valid 200 301 302 {{ ooniapi_gateway_cache_default_valid }}; proxy_cache_valid any 0; @@ -91,7 +93,7 @@ server { ## --- generated from routes.yaml, see tasks/main.yml --- {% for l in ooniapi_gateway_locations %} {% if l.service in service_names %} - location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }};{% if l.cache %} proxy_cache ooniapi_{{ l.cache }};{% endif %} } # {{ l.route }} + location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }};{% if l.cache %} proxy_cache ooniapi_{{ l.cache }}; proxy_cache_key "$ooniapi_cache_version_{{ l.service }}$scheme$host$request_uri";{% endif %} } # {{ l.route }} {% elif l.service in remote_upstreams %} # {{ l.service }} is not yet migrated off AWS, proxied cross-cloud until it is location {{ l.match }} {{ l.path }} { proxy_pass {{ remote_upstreams[l.service] }}; } # {{ l.route }} diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 13892537..16eae8f1 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -20,6 +20,7 @@ import json import logging import os +import re import subprocess import sys import tempfile @@ -165,23 +166,31 @@ def deploy_host(host, ctx): log.info(f"{service} on {host}: slot {target_slot} healthy after {time.monotonic() - started:.0f}s") upstream_file = f"{service}-upstream.conf" - upstream_template = s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") - - def install_upstream(live_slot): - state_a, state_b = ("", "down") if live_slot == "a" else ("down", "") - content = upstream_template.replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) + upstream_path = f"/etc/nginx/conf.d/{upstream_file}" + state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") + # the image tag doubles as this service's cache version on the gateway + # (see nginx_upstream.conf.tftpl): new tag, new cache keys + upstream_content = ( + s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") + .replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) + .replace("__CACHE_VERSION__", ctx["image_tag"]) + ) + # kept as is, to put back if nginx rejects the new one + previous_content = ssh_output(user, host, f"cat {upstream_path}") + + def install_upstream(content): scp(user, host, write_tmp(upstream_file, content), f"{STAGING_DIR}/{upstream_file}") - ssh(user, host, f"sudo mv {STAGING_DIR}/{upstream_file} /etc/nginx/conf.d/{upstream_file}") + ssh(user, host, f"sudo mv {STAGING_DIR}/{upstream_file} {upstream_path}") - install_upstream(target_slot) + install_upstream(upstream_content) if not ssh_succeeds(user, host, "sudo nginx -t"): # nginx is still running its old config, but a config that fails # nginx -t must not stay in conf.d: the next reload (another # service's deploy, a certificate renewal) would fail, and a restart - # would take every service on this host down. Put back the upstream - # of the slot still serving, which nginx -t accepted before. + # would take every service on this host down. Put back the previous + # file, slot and cache version, which nginx -t accepted before. log.error(f"{service} on {host}: nginx -t failed with slot {target_slot}'s upstream, restoring slot {active_slot}'s") - install_upstream(active_slot) + install_upstream(previous_content) if not ssh_succeeds(user, host, "sudo nginx -t"): sys.exit(f"{service} on {host}: nginx -t still fails after restoring slot {active_slot}'s upstream:" " the nginx config is broken independently of this deploy, fix it before reloading nginx") @@ -197,6 +206,9 @@ def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout) with open("imagedefinitions.json") as f: image_tag = json.load(f)[0]["imageUri"].rsplit(":", 1)[-1] + # it ends up in an nginx string and in compose files + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", image_tag): + sys.exit(f"unexpected image tag: {image_tag!r}") service = require_env("SERVICE_NAME") log.info(f"Deploying {service} image tag {image_tag}") diff --git a/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl index d239d393..640eff7f 100644 --- a/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl +++ b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl @@ -1,3 +1,11 @@ +# Rendered by deploy.py on every deploy. __CACHE_VERSION__ becomes the +# deployed image tag: it is part of the gateway's cache key for this +# service, so the reload that puts the new slot in rotation also stops +# serving responses cached from the old one. +map "" $ooniapi_cache_version_${service_name} { + default "__CACHE_VERSION__"; +} + upstream ${service_name} { server 127.0.0.1:${host_port_a} __STATE_A__; server 127.0.0.1:${host_port_b} __STATE_B__; From d20cc6f884df102841e6df37dd7f6294934fd5d3 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 10:28:41 +0000 Subject: [PATCH 201/201] ooniapi_service_deployer: add deploy_mode "both" A service had to choose between ECS and the Hetzner hosts. Switching one to "blue_green" removes the ECS deploy action but leaves the ECS service running, so the ALB keeps sending api.ooni.org and .prod.ooni.io traffic to tasks that no longer get new versions, until DNS moves to the hosts. "both" deploys the same build artifact to ECS and then, in the same Deploy stage, runs the blue/green deploy to the hosts. The ECS deploy runs first, so the hosts never get a version ECS rejected. Either failing fails the pipeline. The migration can then keep the single DNS cutover of the original design while every service runs in "both": the hosts are proven before the move, and DNS can go back to the ALB, which still serves the current version, if they aren't. "ecs" and "blue_green" plan exactly as before. Planned with fake inputs, one service per mode, and the account id lookup replaced by a constant: the 6 and 13 resources are identical. A "both" service gets the 13 blue/green resources and a Deploy stage with ECS "Deploy" (run order 1) then CodeBuild "DeployBlueGreen" (run order 2). Both actions need their own name and namespace. The pipeline role already allows ecs:* and codebuild:StartBuild on any resource. The per-service comments in dev and prod now mention "both", and no longer say the blue/green deploy uses systemd (it uses Docker Compose since e11b071). --- tf/environments/dev/main.tf | 30 +++++++++------- tf/environments/prod/main.tf | 30 +++++++++------- tf/modules/ooniapi_service_deployer/main.tf | 29 ++++++++------- .../ooniapi_service_deployer/variables.tf | 36 ++++++++++--------- 4 files changed, 73 insertions(+), 52 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 8ded4312..f8a2fab6 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -653,8 +653,9 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniprobe" @@ -803,8 +804,9 @@ module "ooniapi_ooniprobe_legacy" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "reverseproxy" @@ -1122,8 +1124,9 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonirun" @@ -1183,8 +1186,9 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonifindings" @@ -1243,8 +1247,9 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniauth" @@ -1337,8 +1342,9 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonimeasurements" diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index e0c4d2e1..a1e6b79c 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -542,8 +542,9 @@ resource "aws_route53_record" "ooni_wc_th" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "reverseproxy" @@ -978,8 +979,9 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniprobe" @@ -1346,8 +1348,9 @@ resource "aws_iam_role_policy" "reuploader_role" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonirun" @@ -1408,8 +1411,9 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonifindings" @@ -1470,8 +1474,9 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniauth" @@ -1566,8 +1571,9 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonimeasurements" diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index 5fbc02b3..dc6f2165 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -5,6 +5,9 @@ data "aws_caller_identity" "current" {} locals { account_id = data.aws_caller_identity.current.account_id env_label = var.environment == "prod" ? "latest" : "dev" + + deploy_ecs = contains(["ecs", "both"], var.deploy_mode) + deploy_blue_green = contains(["blue_green", "both"], var.deploy_mode) } resource "aws_iam_policy" "codebuild" { @@ -161,10 +164,10 @@ resource "aws_codebuild_project" "ooniapi" { } } -## Docker Compose blue/green deploy (deploy_mode = "blue_green") +## Docker Compose blue/green deploy (deploy_mode = "blue_green" or "both") resource "aws_s3_object" "compose_file" { - for_each = var.deploy_mode == "blue_green" ? { a = var.host_port_a, b = var.host_port_b } : {} + for_each = local.deploy_blue_green ? { a = var.host_port_a, b = var.host_port_b } : {} bucket = var.deploy_bucket key = "${var.service_name}/${var.service_name}-${each.key}.yaml" @@ -182,7 +185,7 @@ resource "aws_s3_object" "compose_file" { } resource "aws_s3_object" "nginx_upstream" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 bucket = var.deploy_bucket key = "${var.service_name}/${var.service_name}-upstream.conf" @@ -196,7 +199,7 @@ resource "aws_s3_object" "nginx_upstream" { } resource "aws_s3_object" "deploy_script" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 bucket = var.deploy_bucket key = "${var.service_name}/deploy.py" @@ -206,7 +209,7 @@ resource "aws_s3_object" "deploy_script" { } resource "aws_iam_policy" "deploy" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 description = "Policy used in trust relationship with the blue/green deploy CodeBuild project" name = "codebuild-deploy-${var.service_name}-${var.aws_region}" @@ -260,7 +263,7 @@ resource "aws_iam_policy" "deploy" { } resource "aws_iam_role" "deploy" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 assume_role_policy = <