diff --git a/.github/workflows/check_ansible.yml b/.github/workflows/check_ansible.yml index 396a8afc..1c32eb4d 100644 --- a/.github/workflows/check_ansible.yml +++ b/.github/workflows/check_ansible.yml @@ -58,41 +58,41 @@ jobs: #- name: Setup tmate session # uses: mxschmitt/action-tmate@v3 - - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const commentTitle = "Ansible Run Output"; - const ansiblePlaybookOutput = `${{ steps.playbook.outputs.ansible_playbook}}`; - const parts = ansiblePlaybookOutput.split(/PLAY RECAP \*+/); - const ansiblePlaybookRecap = parts.length > 1 ? parts[1].trim() : ''; + #- uses: actions/github-script@v9 + # with: + # github-token: ${{ secrets.GITHUB_TOKEN }} + # script: | + # const commentTitle = "Ansible Run Output"; + # const ansiblePlaybookOutput = `${{ steps.playbook.outputs.ansible_playbook}}`; + # const parts = ansiblePlaybookOutput.split(/PLAY RECAP \*+/); + # const ansiblePlaybookRecap = parts.length > 1 ? parts[1].trim() : ''; - const commentBody = ` - #### Ansible Playbook Recap 🔍 + # const commentBody = ` + # #### Ansible Playbook Recap 🔍 - \`\`\`\n - ${ansiblePlaybookRecap} - \`\`\` + # \`\`\`\n + # ${ansiblePlaybookRecap} + # \`\`\` - #### Ansible playbook output 📖\`${{ steps.playbook.outcome }}\` + # #### Ansible playbook output 📖\`${{ steps.playbook.outcome }}\` -
Show Execution + #
Show Execution - \`\`\`\n - ${ansiblePlaybookOutput} - \`\`\` + # \`\`\`\n + # ${ansiblePlaybookOutput} + # \`\`\` -
+ #
- | | | - |-------------------|------------------------------------| - | Pusher | @${{ github.actor }} | - | Action | ${{ github.event_name }} | - | Working Directory | ${{ env.tf_actions_working_dir }} | - | Workflow | ${{ github.workflow }} | - | Last updated | ${(new Date()).toUTCString()} | - `; + # | | | + # |-------------------|------------------------------------| + # | Pusher | @${{ github.actor }} | + # | Action | ${{ github.event_name }} | + # | Working Directory | ${{ env.tf_actions_working_dir }} | + # | Workflow | ${{ github.workflow }} | + # | Last updated | ${(new Date()).toUTCString()} | + # `; - // Call the script to write the comment - const script = require('./scripts/ghactions/comment-on-pr.js'); - await script({github, context, core, commentTitle, commentBody}); + # // Call the script to write the comment + # const script = require('./scripts/ghactions/comment-on-pr.js'); + # await script({github, context, core, commentTitle, commentBody}); diff --git a/ansible/README.md b/ansible/README.md index 8507cd29..0f49c25e 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -191,6 +191,41 @@ When deploying files or updating files already existing on the hosts it can be u This helps track down how files on the host were modified and why. ::: +#### Tagging convention + +Tasks are tagged along two independent axes, so that `--tags`/`--skip-tags` can be used to run (or skip) a +meaningful slice of a playbook without guessing what else might be silently pulled in or left out: + +* **Component tag** — matches the role name (`nginx`, `dehydrated`, `nftables`, `prometheus_node_exporter`, + `docker`, `ooniapi_gateway`, ...). Selects everything belonging to that role. +* **Phase tag** — a small, fixed vocabulary describing *what kind* of operation the task performs, independent of + which role it lives in: + + | Tag | Meaning | + |--------------|----------------------------------------------------------------------------------------------------------------------| + | `packages` | apt/package installs — slow, safe to skip when nothing package-level changed | + | `config` | template/config file rendering — fast, safe to run often | + | `certs` | dehydrated cert issuance/renewal — rate-limited by Let's Encrypt, must be independently skippable (e.g. before DNS is cut over to a new host) | + | `network` | nftables rules and docker network setup | + | `service` | service enable/start/restart/reload operations | + | `monitoring` | prometheus/node_exporter wiring and health checks | + | `secrets` | sudoers rules and deploy-credential/password setup | + + Most tasks should carry exactly one component tag and one phase tag. + + Examples: + ``` + ./play -i inventory deploy-monitoring-proxy.yml -l monitoringproxy.prod.ooni.io --tags config + ./play -i inventory deploy-ooni-backend.yml -l backend-hel.ooni.org --skip-tags certs + ./play -i inventory deploy-clickhouse-proxy.yml -l clickhouseproxy.prod.ooni.io --tags monitoring + ``` + +:::note +This taxonomy is applied incrementally as roles are touched, not retrofitted across the whole codebase in one go +— currently `nginx`, `dehydrated`, and `prometheus_node_exporter` follow it. Don't assume every task elsewhere +already does. +::: + ### Platform specific known bugs On macOS you might run into this issue: https://github.com/ansible/ansible/issues/76322 diff --git a/ansible/deploy-airflow.yml b/ansible/deploy-airflow.yml index b070fce2..c6468a95 100644 --- a/ansible/deploy-airflow.yml +++ b/ansible/deploy-airflow.yml @@ -11,6 +11,9 @@ vars: airflow_public_fqdn: "airflow.prod.ooni.io" tls_cert_dir: /var/lib/dehydrated/certs + event_detector_panel_domain: detector-panel.prod.ooni.io ssl_domains: - "data1.htz-fsn.prod.ooni.nu" - "airflow.prod.ooni.io" + - "{{event_detector_panel_domain}}" + detector_panel_passwd: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/detector_panel_passwd', profile='oonidevops_user_prod') }}" diff --git a/ansible/deploy-clickhouse-proxy.yml b/ansible/deploy-clickhouse-proxy.yml index 69be5ebb..bf69e389 100644 --- a/ansible/deploy-clickhouse-proxy.yml +++ b/ansible/deploy-clickhouse-proxy.yml @@ -6,9 +6,10 @@ become: true roles: - role: bootstrap + - role: nginx - role: dehydrated - vars: - ssl_domains: + vars: + ssl_domains: - "{{ inventory_hostname }}" tls_cert_dir: /var/lib/dehydrated/certs - role: clickhouse_proxy diff --git a/ansible/deploy-clickhouse.yml b/ansible/deploy-clickhouse.yml index 50261e34..630d1736 100644 --- a/ansible/deploy-clickhouse.yml +++ b/ansible/deploy-clickhouse.yml @@ -1,13 +1,13 @@ --- - name: Deploy oonidata clickhouse hosts hosts: - - notebook1.htz-fsn.prod.ooni.nu + - notebook.ooni.org - data1.htz-fsn.prod.ooni.nu - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu become: true - tags: - - clickhouse roles: - - prometheus_node_exporter - - oonidata_clickhouse + - role: prometheus_node_exporter + tags: [node_exporter] + - role: oonidata_clickhouse + tags: [clickhouse] diff --git a/ansible/deploy-echo-test-helper.yml b/ansible/deploy-echo-test-helper.yml new file mode 100644 index 00000000..2e875c1e --- /dev/null +++ b/ansible/deploy-echo-test-helper.yml @@ -0,0 +1,23 @@ +--- +- name: Deploy test helpers + hosts: + - echo.th.dev.ooni.io + - echo.th.prod.ooni.io + become: true + roles: + - role: bootstrap + - role: prometheus_node_exporter + vars: + node_exporter_port: 9100 + node_exporter_host: "0.0.0.0" + prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:9100/metrics + use_https: false + http_port: 8080 # if we leave port 80, it's taken by nginx + - role: test_helpers + vars: + helper: ooechohelpd + # sha256 of https://github.com/ooni/ooniprobe-rs/releases/download/v0.1.6/ooechohelpd + test_helpers_checksum: 10511a8b918c2244eba9e17c32f3cec9efbc5ffa51d6ad7f59fe648cd0986615 + port: 80 diff --git a/ansible/deploy-fastpath.yml b/ansible/deploy-fastpath.yml index b9a8e502..8795c8b7 100644 --- a/ansible/deploy-fastpath.yml +++ b/ansible/deploy-fastpath.yml @@ -6,6 +6,28 @@ - fastpath2.prod.ooni.io - reuploaderfastpath.prod.ooni.io become: true + vars: + nginx_install_method: official_repo + pre_tasks: + - name: Find any nginx.org apt source files left over from a previous nginxinc.nginx install + ansible.builtin.find: + paths: /etc/apt/sources.list.d + patterns: "*.list" + contains: "nginx\\.org/packages" + read_whole_file: true + register: _stale_nginx_apt_sources + tags: + - nginx + - packages + + - name: Remove stale nginx.org apt source files (avoids "Conflicting values set for option Signed-By") + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ _stale_nginx_apt_sources.files }}" + tags: + - nginx + - packages roles: - role: bootstrap - role: nginx diff --git a/ansible/deploy-json-test-helper.yml b/ansible/deploy-json-test-helper.yml new file mode 100644 index 00000000..e591018d --- /dev/null +++ b/ansible/deploy-json-test-helper.yml @@ -0,0 +1,23 @@ +--- +- name: Deploy test helpers + hosts: + - json.th.dev.ooni.io + - json.th.prod.ooni.io + become: true + roles: + - role: bootstrap + - role: prometheus_node_exporter + vars: + node_exporter_port: 9100 + node_exporter_host: "0.0.0.0" + prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:9100/metrics + use_https: false + http_port: 8080 # if we leave port 80, it's taken by nginx + - role: test_helpers + vars: + helper: oojsonhelpd + # sha256 of https://github.com/ooni/ooniprobe-rs/releases/download/v0.1.6/oojsonhelpd + test_helpers_checksum: 3d977e1418ea398f520dff3de951e36010a294dc3d1b564c961542cc3729a224 + port: 80 diff --git a/ansible/deploy-monitoring-proxy.yml b/ansible/deploy-monitoring-proxy.yml index edeb3bf2..945021f9 100644 --- a/ansible/deploy-monitoring-proxy.yml +++ b/ansible/deploy-monitoring-proxy.yml @@ -6,13 +6,13 @@ become: true roles: - role: bootstrap + - role: nginx + tags: nginx - role: dehydrated vars: ssl_domains: - "{{ inventory_hostname }}" tls_cert_dir: /var/lib/dehydrated/certs - - role: nginx - tags: nginx - role: monitoring_proxy vars: monitoring_proxy_public_fqdn: "{{ inventory_hostname }}" diff --git a/ansible/deploy-notebook.yml b/ansible/deploy-notebook.yml index a9a528ee..5f4306ec 100644 --- a/ansible/deploy-notebook.yml +++ b/ansible/deploy-notebook.yml @@ -1,6 +1,6 @@ --- - name: Deploy notebook host - hosts: notebook1.htz-fsn.prod.ooni.nu + hosts: notebook.ooni.org become: true tags: - notebook @@ -8,6 +8,7 @@ notebook_domain: "notebook.ooni.org" ssl_domains: - "{{ inventory_hostname }}" - - "notebook.ooni.org" roles: + - nginx + - dehydrated - notebook diff --git a/ansible/deploy-ooni-backend.yml b/ansible/deploy-ooni-backend.yml index 7e056905..3306f780 100644 --- a/ansible/deploy-ooni-backend.yml +++ b/ansible/deploy-ooni-backend.yml @@ -8,6 +8,7 @@ admin_group_name: adm - role: base-backend - role: nftables + - role: nginx - role: dehydrated tags: dehydrated expand: yes diff --git a/ansible/deploy-wc-test-helper.yml b/ansible/deploy-wc-test-helper.yml new file mode 100644 index 00000000..9b65e354 --- /dev/null +++ b/ansible/deploy-wc-test-helper.yml @@ -0,0 +1,19 @@ +--- +- name: Deploy the Web Connectivity test helper + hosts: + - wc.th.dev.ooni.io + - wcth0.fra1.prod.ooni.io + - wcth1.fra1.prod.ooni.io + - wcth2.fra1.prod.ooni.io + become: true + vars: + nginx_install_method: official_repo + roles: + - role: bootstrap + - role: nginx + - role: dehydrated + - role: prometheus_node_exporter + vars: + use_https: true + https_port: 9001 + - role: oohelperd diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 7de27f93..73275001 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -5,7 +5,7 @@ nftables_clickhouse_allow: ip: 23.88.74.249 - fqdn: data3.htz-fsn.prod.ooni.nu ip: 168.119.7.188 - - fqdn: notebook1.htz-fsn.prod.ooni.nu + - fqdn: notebook.ooni.org ip: 138.201.19.39 - fqdn: backend-hel.ooni.org ip: 65.108.192.151 @@ -21,6 +21,8 @@ nftables_clickhouse_allow: ip: "{{ lookup('dig', 'fastpath.prod.ooni.io/A') }}" - fqdn: fastpath2.prod.ooni.io ip: "{{ lookup('dig', 'fastpath2.prod.ooni.io/A') }}" + - fqdn: reuploaderfastpath.prod.ooni.io + ip: "{{ lookup('dig', 'reuploaderfastpath.prod.ooni.io/A') }}" nftables_zookeeper_allow: - fqdn: data1.htz-fsn.prod.ooni.nu @@ -29,7 +31,7 @@ nftables_zookeeper_allow: ip: 23.88.74.249 - fqdn: data3.htz-fsn.prod.ooni.nu ip: 168.119.7.188 - - fqdn: notebook1.htz-fsn.prod.ooni.nu + - fqdn: notebook.ooni.org ip: 138.201.19.39 clickhouse_version: 24.8.6.70 @@ -94,11 +96,11 @@ clickhouse_keeper: hostname: clickhouse1.prod.ooni.io port: 9234 - #- keeper_server: - # server: data2.htz-fsn.prod.ooni.nu - # id: 2 - # hostname: clickhouse2.prod.ooni.io - # port: 9234 + - keeper_server: + server: data2.htz-fsn.prod.ooni.nu + id: 2 + hostname: clickhouse2.prod.ooni.io + port: 9234 - keeper_server: server: data3.htz-fsn.prod.ooni.nu @@ -106,24 +108,15 @@ clickhouse_keeper: hostname: clickhouse3.prod.ooni.io port: 9234 - - keeper_server: - server: notebook1.htz-fsn.prod.ooni.nu - id: 4 - hostname: notebook1.htz-fsn.prod.ooni.nu - port: 9234 - clickhouse_zookeeper: - node: host: clickhouse1.prod.ooni.io port: 9181 -# - node: -# host: clickhouse2.prod.ooni.io -# port: 9181 - node: - host: clickhouse3.prod.ooni.io + host: clickhouse2.prod.ooni.io port: 9181 - node: - host: notebook1.htz-fsn.prod.ooni.nu + host: clickhouse3.prod.ooni.io port: 9181 clickhouse_remote_servers: @@ -171,10 +164,12 @@ clickhouse_distributed_ddl: cleanup_delay_period: 60 max_tasks_in_queue: 1000 +clickhouse_role_manage_settings_profiles: True clickhouse_default_profiles: default: readonly: 2 - max_memory_usage: 11001001000 + # 10 GB + max_memory_usage: 10010010000 use_uncompressed_cache: 0 load_balancing: random max_partitions_per_insert_block: 100 @@ -232,48 +227,168 @@ clickhouse_default_users: profile: write quota: default -clickhouse_role_manage_users: true +clickhouse_role_manage_users: True clickhouse_custom_users: + - user: + name: fastpath + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: "fastpath" + databases: [ooni, oonitest] + - user: name: oonimeasurements - password_type: sha256_password - password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') }}" + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') | hash('sha256') }}" networks: - "IP '0.0.0.0/0'" settings: - # 500 MB - - "max_memory_usage = 501001000" + # 1 GB + - "max_memory_usage = 1001001000" # 60 seconds - "max_execution_time = 30" + # 500 GB + - "max_bytes_to_read = 501001001000" + # 5 B + - "max_rows_to_read = 5001001000" + # 5s + - "timeout_before_checking_execution_speed = 5" + # 50k + - "max_result_rows = 51000" profile: - readonly - quota: "oonimeasurements" + quota: oonimeasurements + databases: [ooni, oonitest] + + - user: + name: ooniprobe + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_ooniprobe_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: ooniprobe + databases: [ooni, oonitest] + + - user: + name: oonirun + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonirun_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: oonirun + databases: [ooni, oonitest] + + - user: + name: oonitestlists + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: oonitestlists + databases: [ooni, oonitest] + + - user: + name: notebook + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_notebook_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: notebook databases: [ooni] -# TODO: this quota was created by hand since it wasn't working in the idealista playbook -clickhouse_role_manage_quotas: false +clickhouse_role_manage_quotas: True clickhouse_custom_quotas: - # quota over a 10 minute window - quota: name: oonimeasurements - settings: - - "INTERVAL 10 minute MAX queries = 12000, MAX errors = 1000, MAX execution_time = 1000" - to: - - oonimeasurements + duration: 600 + queries: 12000 + errors: 1000 + result_rows: 0 + read_rows: 0 + execution_time: 1000 + + # no limits set + - quota: + name: ooniprobe + duration: 0 + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 -clickhouse_role_manage_grants: true -clickhouse_role_manage_roles: true + # no limits set + - quota: + name: oonirun + duration: 0 + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 + + - quota: + name: oonitestlists + duration: 600 + queries: 12000 + errors: 1000 + result_rows: 0 + read_rows: 0 + execution_time: 1000 + + # no limits set + - quota: + name: notebook + duration: 0 + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 + +clickhouse_role_manage_grants: True clickhouse_custom_grants: - on: - databases: [ooni] + databases: [ooni, oonitest] tables: ["*"] privileges: [SELECT] - to: [oonimeasurements] + to: [ooniprobe, oonimeasurements, oonirun, fastpath, notebook] + +- on: + databases: [ooni, oonitest] + tables: [url_priorities] + privileges: [INSERT] + to: [oonitestlists] + +- on: + databases: [ooni, oonitest] + tables: [faulty_measurements] + privileges: [INSERT] + to: [ooniprobe] + +- on: + databases: [ooni, oonitest] + tables: [fastpath, obs_web, obs_openvpn, jsonl, new_jsonl] + privileges: [INSERT] + to: [fastpath] clickhouse_custom_grant_roles: - roles: [oonimeasurements] to: [oonimeasurements] +clickhouse_role_manage_roles: True clickhouse_custom_roles: - role: name: oonimeasurements diff --git a/ansible/group_vars/prod/vars.yml b/ansible/group_vars/prod/vars.yml index afec7630..79566b91 100644 --- a/ansible/group_vars/prod/vars.yml +++ b/ansible/group_vars/prod/vars.yml @@ -1,7 +1,3 @@ prometheus_metrics_password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}" -tailscale_authkey: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/tailscale_authkey_devops', profile='oonidevops_user_prod') }}" -tailscale_tags: - - "devops-prod" -tailscale_oauth_ephemeral: false admin_usernames: [ art, mehul ] non_admin_usernames: [ ] diff --git a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml index fbd3d273..ed30f067 100644 --- a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml @@ -1,6 +1,7 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_dev') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" bucket_name: "ooni-data-eu-fra-test" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "3" -env: "dev" \ No newline at end of file +env: "dev" +fastpath_image: "ooni/fastpath:20260903-7c51eb5d" diff --git a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml index b9692f7a..986a62f6 100644 --- a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml @@ -1,6 +1,7 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@data1.htz-fsn.prod.ooni.nu/ooni" +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "1" env: "prod" +fastpath_image: "ooni/fastpath:20260921-ac774cb4" diff --git a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml index 17a5af5a..26123dcc 100644 --- a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml @@ -1,6 +1,7 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@data1.htz-fsn.prod.ooni.nu/ooni" +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "4" env: "prod" +fastpath_image: "ooni/fastpath:20260921-ac774cb4" diff --git a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu b/ansible/host_vars/notebook.ooni.org similarity index 99% rename from ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu rename to ansible/host_vars/notebook.ooni.org index 1090f8cf..2b68cbfc 100644 --- a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu +++ b/ansible/host_vars/notebook.ooni.org @@ -145,6 +145,7 @@ clickhouse_listen_hosts: # this only applies to SQL managed users clickhouse_role_manage_users: false +clickhouse_role_manage_grants: false # these are handled via config files clickhouse_default_users: - user: diff --git a/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml b/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml index f1507236..eef14c04 100644 --- a/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/reuploaderfastpath.prod.ooni.io/vars.yml @@ -1,6 +1,7 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@data1.htz-fsn.prod.ooni.nu/ooni" +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "5" env: "prod" +fastpath_image: "ooni/fastpath:20260921-ac774cb4" diff --git a/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml b/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml index 7ab99273..b6497100 100644 --- a/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml +++ b/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml @@ -1,3 +1,4 @@ jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_dev') }}" github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_dev') }}" log_level: "debug" +clickhouse_url: "clickhouse://oonitestlists:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/ooni" diff --git a/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml b/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml index 59e4bb4c..fb061292 100644 --- a/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml @@ -1,4 +1,4 @@ jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_prod') }}" github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_prod') }}" log_level: "info" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" +clickhouse_url: "clickhouse://oonitestlists:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" diff --git a/ansible/host_vars/wc.th.dev.ooni.io b/ansible/host_vars/wc.th.dev.ooni.io new file mode 100644 index 00000000..971935a8 --- /dev/null +++ b/ansible/host_vars/wc.th.dev.ooni.io @@ -0,0 +1,2 @@ +ssl_domains: +- wc.th.dev.ooni.io diff --git a/ansible/host_vars/wcth0.fra1.prod.ooni.io b/ansible/host_vars/wcth0.fra1.prod.ooni.io new file mode 100644 index 00000000..8300c119 --- /dev/null +++ b/ansible/host_vars/wcth0.fra1.prod.ooni.io @@ -0,0 +1,3 @@ +ssl_domains: +- "wcth0.fra1.prod.ooni.io wcth0.fra1.ooni.org" +nginx_server_name: "wcth0.fra1.prod.ooni.io wcth0.fra1.ooni.org" diff --git a/ansible/host_vars/wcth1.fra1.prod.ooni.io b/ansible/host_vars/wcth1.fra1.prod.ooni.io new file mode 100644 index 00000000..78d0b0e6 --- /dev/null +++ b/ansible/host_vars/wcth1.fra1.prod.ooni.io @@ -0,0 +1,3 @@ +ssl_domains: +- "wcth1.fra1.prod.ooni.io wcth1.fra1.ooni.org" +nginx_server_name: "wcth1.fra1.prod.ooni.io wcth1.fra1.ooni.org" diff --git a/ansible/host_vars/wcth2.fra1.prod.ooni.io b/ansible/host_vars/wcth2.fra1.prod.ooni.io new file mode 100644 index 00000000..2c248673 --- /dev/null +++ b/ansible/host_vars/wcth2.fra1.prod.ooni.io @@ -0,0 +1,3 @@ +ssl_domains: +- "wcth2.fra1.prod.ooni.io wcth2.fra1.ooni.org" +nginx_server_name: "wcth2.fra1.prod.ooni.io wcth2.fra1.ooni.org" diff --git a/ansible/inventory b/ansible/inventory index 9a878bf1..754130cd 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -5,7 +5,7 @@ ghs_ams ## Role tags [clickhouse] -notebook1.htz-fsn.prod.ooni.nu +notebook.ooni.org data1.htz-fsn.prod.ooni.nu data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu @@ -17,7 +17,7 @@ data1.htz-fsn.prod.ooni.nu [htz_fsn] monitoring.ooni.org -notebook1.htz-fsn.prod.ooni.nu +notebook.ooni.org data1.htz-fsn.prod.ooni.nu data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu @@ -35,6 +35,12 @@ ams-ps.ooni.nu #mia-echoth.ooni.nu #mia-httpth.ooni.nu +[do_fra] +wc.th.dev.ooni.io +wcth0.fra1.prod.ooni.io +wcth1.fra1.prod.ooni.io +wcth2.fra1.prod.ooni.io + [aws-proxy] clickhouseproxy.dev.ooni.io clickhouseproxy.prod.ooni.io @@ -48,6 +54,10 @@ openvpn2.htz-fsn.prod.ooni.nu [aws-backend] fastpath.dev.ooni.io fastpath.prod.ooni.io +json.th.dev.ooni.io +echo.th.dev.ooni.io +json.th.prod.ooni.io +echo.th.prod.ooni.io fastpath2.prod.ooni.io reuploaderfastpath.prod.ooni.io anonc.dev.ooni.io diff --git a/ansible/requirements/ansible-galaxy.yml b/ansible/requirements/ansible-galaxy.yml index 997d7a83..5172cbec 100644 --- a/ansible/requirements/ansible-galaxy.yml +++ b/ansible/requirements/ansible-galaxy.yml @@ -1,12 +1,8 @@ roles: - - src: willshersystems.sshd - version: v0.25.0 - src: nginxinc.nginx - version: 0.24.3 + version: 0.26.0 - src: geerlingguy.certbot version: 5.2.0 - - src: artis3n.tailscale - version: v4.5.0 - src: https://github.com/idealista/clickhouse_role scm: git version: 3.5.1 @@ -18,4 +14,4 @@ roles: version: 7.4.7 collections: - name: community.docker # manages containers - version: 4.6.1 \ No newline at end of file + version: 4.6.1 diff --git a/ansible/roles/bootstrap/handlers/main.yml b/ansible/roles/bootstrap/handlers/main.yml index a9c712a4..2bdb126d 100644 --- a/ansible/roles/bootstrap/handlers/main.yml +++ b/ansible/roles/bootstrap/handlers/main.yml @@ -9,7 +9,7 @@ state: restarted - name: Test systemd-resolved - ansible.builtin.shell: resolvectl query go.dnscheck.tools --cache=no + ansible.builtin.shell: resolvectl query dnscheck.tools --cache=no - name: Restart systemd-journald ansible.builtin.systemd_service: diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index 2b65c611..5b5da161 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -26,6 +26,7 @@ - git - htop - iotop + - locales - lsof - lvm2 - man-db @@ -44,6 +45,24 @@ update_cache: yes install_recommends: no +- name: Generate en_US.UTF-8 locale + community.general.locale_gen: + name: en_US.UTF-8 + state: present + +- name: Set system locale + ansible.builtin.lineinfile: + path: /etc/default/locale + create: yes + regexp: "^{{ item.key }}=" + line: "{{ item.key }}={{ item.value }}" + owner: root + group: root + mode: "0644" + loop: + - { key: "LANG", value: "en_US.UTF-8" } + - { key: "LC_ALL", value: "en_US.UTF-8" } + - name: Set timezone community.general.timezone: name: Etc/UTC diff --git a/ansible/roles/dehydrated/handlers/main.yml b/ansible/roles/dehydrated/handlers/main.yml index a539854a..71d72a81 100644 --- a/ansible/roles/dehydrated/handlers/main.yml +++ b/ansible/roles/dehydrated/handlers/main.yml @@ -3,6 +3,11 @@ name: nginx state: reloaded +- name: restart nginx + service: + name: nginx + state: restarted + - name: reload nftables service: name: nftables @@ -13,10 +18,3 @@ name: dehydrated state: restarted enabled: yes - -- name: reload nginx and restart dehydrated - service: - name: nginx - state: reloaded - notify: - restart dehydrated \ No newline at end of file diff --git a/ansible/roles/dehydrated/meta/main.yml b/ansible/roles/dehydrated/meta/main.yml deleted file mode 100644 index 0e72e865..00000000 --- a/ansible/roles/dehydrated/meta/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -dependencies: - - nginx -... - diff --git a/ansible/roles/dehydrated/tasks/main.yml b/ansible/roles/dehydrated/tasks/main.yml index ecf09a5d..56af5a5d 100644 --- a/ansible/roles/dehydrated/tasks/main.yml +++ b/ansible/roles/dehydrated/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Installs packages - tags: dehydrated + tags: [dehydrated, packages] apt: install_recommends: false cache_valid_time: 86400 @@ -23,7 +23,7 @@ # HOOK="/etc/dehydrated/hook.sh" - name: Add ACME dedicated sites-enabled file - tags: dehydrated + tags: [dehydrated, certs] ansible.builtin.template: src: templates/letsencrypt-http # the server block matches all SSL FQDNs and must be @@ -33,7 +33,7 @@ owner: root - name: Add canary file to ensure /.well-known/acme-challenge is reachable by let's encrypt - tags: dehydrated + tags: [dehydrated, certs] ansible.builtin.copy: content: | Generated by ansible using ansible/roles/dehydrated/tasks/main.yml. @@ -44,23 +44,23 @@ owner: root - name: reload nftables service - tags: dehydrated + tags: [dehydrated, network] shell: systemctl reload nftables.service - name: Configure domains {{ ssl_domains }} # https://github.com/dehydrated-io/dehydrated/blob/master/docs/domains_txt.md - tags: dehydrated + tags: [dehydrated, certs] template: src: templates/domains.txt.j2 dest: /etc/dehydrated/domains.txt - name: Register account if needed - tags: dehydrated + tags: [dehydrated, certs] ansible.builtin.shell: cmd: "test -d /var/lib/dehydrated/accounts || dehydrated --register --accept-terms" - name: Install dehydrated.service - tags: dehydrated + tags: [dehydrated, certs] template: src: templates/dehydrated.service dest: /etc/systemd/system/dehydrated.service @@ -68,7 +68,7 @@ owner: root - name: Install dehydrated.timer - tags: dehydrated + tags: [dehydrated, certs] template: src: templates/dehydrated.timer dest: /etc/systemd/system/dehydrated.timer @@ -76,20 +76,27 @@ owner: root - name: Ensure timer runs - tags: dehydrated + tags: [dehydrated, certs] systemd: name: dehydrated.timer state: started enabled: yes notify: - # creates: - # /var/lib/dehydrated/certs//chain.pem cert.pem privkey.pem fullchain.pem + - restart nginx - # Note that we need to restart dehydrated ensuring that nginx reloads before dehydrated restarts. - # When we first run dehydrated with the tasks above it creates an nginx rule that is required - # to pass the ACME challenge. - # If nginx doesn't picks this rule before dehydrated runs again, the ACME challenge will fail - # crashing the playbook - # - # See: https://github.com/ooni/devops/pull/235#discussion_r2053664605 - - reload nginx and restart dehydrated \ No newline at end of file +- name: Flush all handlers + tags: [dehydrated, certs] + meta: flush_handlers + +- name: Ensure nginx is running + tags: [dehydrated, certs] + ansible.builtin.systemd_service: + state: started + name: nginx + +- name: Force the restarting of the dehydrated service + tags: [dehydrated, certs] + ansible.builtin.systemd_service: + state: restarted + daemon_reload: true + name: dehydrated diff --git a/ansible/roles/dehydrated/templates/domains.txt.j2 b/ansible/roles/dehydrated/templates/domains.txt.j2 index 5850d203..4b4a6c0e 100644 --- a/ansible/roles/dehydrated/templates/domains.txt.j2 +++ b/ansible/roles/dehydrated/templates/domains.txt.j2 @@ -1 +1,3 @@ -{% for d in ssl_domains %}{{ d }} {% endfor %} +{% for d in ssl_domains %} +{{ d }} +{% endfor %} diff --git a/ansible/roles/fastpath/defaults/main.yml b/ansible/roles/fastpath/defaults/main.yml index 1a15aea5..99e4bdbb 100644 --- a/ansible/roles/fastpath/defaults/main.yml +++ b/ansible/roles/fastpath/defaults/main.yml @@ -4,5 +4,8 @@ tls_cert_dir: /var/lib/dehydrated/certs fastpath_user: fastpath fastpath_home: "/opt/{{ fastpath_user }}" +# Fastpath container image +fastpath_image: "ooni/fastpath:latest" + # Fastpath settings -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" \ No newline at end of file +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" diff --git a/ansible/roles/fastpath/tasks/main.yml b/ansible/roles/fastpath/tasks/main.yml index 948105bc..8b381b65 100644 --- a/ansible/roles/fastpath/tasks/main.yml +++ b/ansible/roles/fastpath/tasks/main.yml @@ -149,7 +149,7 @@ - name: Ensure fastpath is running community.docker.docker_container: name: fastpath - image: ooni/fastpath:20260416-37f9e89c + image: "{{ fastpath_image }}" state: started restart_policy: always user: "{{user_uid.stdout}}:{{user_gid.stdout}}" diff --git a/ansible/roles/nginx/defaults/main.yml b/ansible/roles/nginx/defaults/main.yml index 4c0ac11a..41a890b4 100644 --- a/ansible/roles/nginx/defaults/main.yml +++ b/ansible/roles/nginx/defaults/main.yml @@ -1 +1,10 @@ nginx_user: nginx + +# "nginxinc_role" (default, unchanged): install via the nginxinc.nginx galaxy +# role, as today. +# "official_repo": install directly from nginx.org's own apt repo, following +# https://nginx.org/en/linux_packages.html#Debian, with a pinned signing-key +# fingerprint check. Opt in per-playbook; the nginxinc_role default is left +# alone for existing callers since switching apt repos changes what nginx +# package/version they get on their next run. +nginx_install_method: nginxinc_role diff --git a/ansible/roles/nginx/files/ssl_modern.conf b/ansible/roles/nginx/files/ssl_modern.conf index 9ad7c11d..538d3097 100644 --- a/ansible/roles/nginx/files/ssl_modern.conf +++ b/ansible/roles/nginx/files/ssl_modern.conf @@ -1,4 +1,4 @@ # Oldest compatible clients: Firefox 27, Chrome 30, IE 11 on Windows 7, Edge, Opera 17, Safari 9, Android 5.0, and Java 8 -ssl_protocols TLSv1.2; +ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; # NB: technically, it does not require ssl_dhparam as it has no DHE, only ECDHE. diff --git a/ansible/roles/nginx/tasks/main.yml b/ansible/roles/nginx/tasks/main.yml index 9af2a9b4..bd70a7cd 100644 --- a/ansible/roles/nginx/tasks/main.yml +++ b/ansible/roles/nginx/tasks/main.yml @@ -12,10 +12,84 @@ tags: - nginx - nftables + - network -- name: install nginx +- name: install nginx via the nginxinc.nginx galaxy role include_role: name: nginxinc.nginx + tags: + - nginx + - packages + when: nginx_install_method == 'nginxinc_role' + +# Follows https://nginx.org/en/linux_packages.html#Debian directly instead +# of a third-party role. +- name: Install nginx from nginx.org's official apt repository + when: nginx_install_method == 'official_repo' + tags: + - nginx + - packages + block: + - name: Install nginx's prerequisites + ansible.builtin.apt: + name: + - curl + - gnupg2 + - ca-certificates + state: present + update_cache: true + + - name: Download nginx's official signing key + ansible.builtin.get_url: + url: https://nginx.org/keys/nginx_signing.key + dest: /tmp/nginx_signing.key + mode: "0644" + + - name: Dearmor nginx's signing key into the apt keyring + ansible.builtin.command: gpg --dearmor --yes --output /usr/share/keyrings/nginx-archive-keyring.gpg /tmp/nginx_signing.key + args: + creates: /usr/share/keyrings/nginx-archive-keyring.gpg + + - name: Read the fingerprint of the downloaded key + ansible.builtin.command: gpg --with-colons --show-keys /usr/share/keyrings/nginx-archive-keyring.gpg + register: nginx_key_check + changed_when: false + + # Pin against nginx.org's gpg key fingerprint: + # 573B FD6B 3D8F BC64 1079 A6AB ABF5 BD82 7BD9 BF62, documented at + # (https://nginx.org/en/linux_packages.html) + - name: Verify the downloaded key's fingerprint matches nginx.org's published one + ansible.builtin.assert: + that: + - nginx_key_check.stdout is + search('^fpr:::::::::573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62:$', multiline=true) + fail_msg: >- + nginx-archive-keyring.gpg's fingerprint does not match the + expected 573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62 -- refusing to + trust it. + success_msg: "nginx-archive-keyring.gpg's fingerprint matches the expected value." + + - name: Add the nginx.org apt repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://nginx.org/packages/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} nginx" + filename: nginx + state: present + + - name: Pin nginx.org packages above Debian's own nginx package + ansible.builtin.copy: + dest: /etc/apt/preferences.d/99nginx + content: | + Package: * + Pin: origin nginx.org + Pin: release o=nginx + Pin-Priority: 900 + mode: "0644" + + - name: Install nginx + ansible.builtin.apt: + name: nginx + state: present + update_cache: true # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4 # Guide https://wiki.mozilla.org/Security/Server_Side_TLS#Pre-defined_DHE_groups @@ -28,6 +102,7 @@ - ssl_modern.conf tags: - nginx + - config - name: remove `default` vhost file: path={{item}} state=absent @@ -38,6 +113,7 @@ - /etc/nginx/sites-enabled/default tags: - nginx + - config - name: Create nginx sites directory ansible.builtin.file: @@ -45,9 +121,11 @@ state: directory tags: - nginx + - config - name: set nginx.conf template: src=nginx.conf dest=/etc/nginx/nginx.conf mode=0444 notify: reload nginx tags: - nginx + - config diff --git a/ansible/roles/notebook/tasks/jupyterhub.yml b/ansible/roles/notebook/tasks/jupyterhub.yml index ea9e0e04..de4a271b 100644 --- a/ansible/roles/notebook/tasks/jupyterhub.yml +++ b/ansible/roles/notebook/tasks/jupyterhub.yml @@ -106,14 +106,3 @@ - jupyterhub - config -- ansible.builtin.include_role: - name: nginx - tags: - - oonidata - - nginx - -- ansible.builtin.include_role: - name: dehydrated - tags: - - oonidata - - dehydrated diff --git a/ansible/roles/notebook/templates/nginx-jupyterhub.j2 b/ansible/roles/notebook/templates/nginx-jupyterhub.j2 index 79c954a4..92eb9eba 100644 --- a/ansible/roles/notebook/templates/nginx-jupyterhub.j2 +++ b/ansible/roles/notebook/templates/nginx-jupyterhub.j2 @@ -5,6 +5,7 @@ map $http_upgrade $connection_upgrade { '' close; } +# Comment this block when running dehydrated for the first time server { listen 80; server_name {{ notebook_domain }}; diff --git a/ansible/roles/oohelperd/defaults/main.yml b/ansible/roles/oohelperd/defaults/main.yml new file mode 100644 index 00000000..e3696f33 --- /dev/null +++ b/ansible/roles/oohelperd/defaults/main.yml @@ -0,0 +1,14 @@ +# The Web Connectivity test helper (oohelperd) is built and released as part of +# https://github.com/ooni/probe-cli +oohelperd_version: v3.30.0 +oohelperd_base_url: "https://github.com/ooni/probe-cli/releases/download/{{ oohelperd_version }}" +# sha256 of {{ oohelperd_base_url }}/oohelperd-linux-amd64 +oohelperd_checksum: "fff38b9366fecae400cb3f616ffa2a37dba211c33ee4e1c1b343bff1c384d48e" + +# A test helper request measures the target before it can answer, so it needs +# way more than the nginx default of 60s +oohelperd_nginx_proxy_timeout: 900 +# It's important that when you generate certificate using dehydrated, the first +# domain in the list matches inventory_hostname so that the lookup for the +# certificate that was generated using dehydrated matches +nginx_server_name: "{{ inventory_hostname }}" diff --git a/ansible/roles/oohelperd/handlers/main.yml b/ansible/roles/oohelperd/handlers/main.yml new file mode 100644 index 00000000..71ffc659 --- /dev/null +++ b/ansible/roles/oohelperd/handlers/main.yml @@ -0,0 +1,4 @@ +- name: reload nginx + service: + name: nginx + state: reloaded diff --git a/ansible/roles/oohelperd/tasks/main.yml b/ansible/roles/oohelperd/tasks/main.yml new file mode 100644 index 00000000..c20d5e63 --- /dev/null +++ b/ansible/roles/oohelperd/tasks/main.yml @@ -0,0 +1,93 @@ +--- +- name: Flush all handlers now + ansible.builtin.meta: flush_handlers + +- name: Create the oohelperd user + tags: [oohelperd] + ansible.builtin.user: + name: oohelperd + shell: /usr/sbin/nologin + create_home: no + system: yes + +- name: Download the oohelperd binary + tags: [oohelperd] + ansible.builtin.get_url: + url: "{{ oohelperd_base_url }}/oohelperd-linux-amd64" + dest: /usr/local/bin/oohelperd + checksum: "sha256:{{ oohelperd_checksum }}" + mode: "0755" + owner: root + register: oohelperd_binary + +- name: Create the ooni configuration directory + tags: [oohelperd, config] + ansible.builtin.file: + path: /etc/ooni + state: directory + owner: root + mode: "0755" + +# Read by systemd as root, hence not readable by the oohelperd user itself +- name: Write the oohelperd environment file + tags: [oohelperd, secrets] + no_log: true + ansible.builtin.template: + src: oohelperd.env + dest: /etc/ooni/oohelperd.env + owner: root + group: root + mode: "0600" + register: oohelperd_env + +- name: Create the oohelperd.service file + tags: [oohelperd] + ansible.builtin.template: + src: oohelperd.service + dest: /etc/systemd/system/oohelperd.service + mode: "0644" + owner: root + register: oohelperd_unit + +- name: reload systemd + tags: [oohelperd] + ansible.builtin.systemd_service: + daemon_reload: yes + when: oohelperd_unit.changed + +- name: Start oohelperd + tags: [oohelperd] + ansible.builtin.systemd_service: + name: oohelperd.service + state: started + enabled: yes + +- name: Restart oohelperd if its binary, unit file or environment changed + tags: [oohelperd] + ansible.builtin.systemd_service: + name: oohelperd.service + state: restarted + when: oohelperd_binary.changed or oohelperd_unit.changed or oohelperd_env.changed + +# Expose the test helper to the probes +- name: Add the oohelperd nginx vhost + tags: [oohelperd, nginx, config] + ansible.builtin.template: + src: nginx-oohelperd.j2 + dest: /etc/nginx/sites-enabled/02-oohelperd + mode: "0644" + owner: root + notify: + - reload nginx + +- name: Verify oohelperd is answering requests + tags: [oohelperd] + # GET / is oohelperd's health check, it answers "Hello OONItarian!" + ansible.builtin.uri: + url: http://127.0.0.1:8080/ + return_content: true + register: oohelperd_check + when: not ansible_check_mode + until: "'OONItarian' in oohelperd_check.content" + retries: 5 + delay: 2 diff --git a/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 new file mode 100644 index 00000000..7da08869 --- /dev/null +++ b/ansible/roles/oohelperd/templates/nginx-oohelperd.j2 @@ -0,0 +1,73 @@ +# Generated by ansible +# roles/oohelperd/templates/nginx-oohelperd.j2 + +# The probes send the same measurement request to the test helper over and +# over, so responses are cached, keyed by the request body since they are POSTs +proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M + max_size=5g inactive=24h use_temp_path=off; + +server { + listen 80; + listen [::]:80; + server_name {{ nginx_server_name }}; + + # This server block is more specific than the catch-all one installed by + # the dehydrated role, so it must serve the ACME challenge itself + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + # nginx >= 1.25.1, which is what nginx.org ships + http2 on; + + server_name {{ nginx_server_name }} *.th.ooni.org; + include /etc/nginx/ssl_modern.conf; + + ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; + + # Test helper application metrics. oohelperd protects them with basic auth + # using the password in /etc/ooni/oohelperd.env (user "prom"). Kept out of + # the cache so that prometheus always scrapes fresh values. + location = /metrics { + proxy_pass http://127.0.0.1:8080; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Local test helper + location / { + proxy_pass http://127.0.0.1:8080; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout {{ oohelperd_nginx_proxy_timeout }}; + + proxy_cache thcache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + # Cache POST without headers set by the test helper! + proxy_cache_methods POST; + proxy_cache_key "$request_uri|$request_body"; + proxy_cache_valid 200 10m; + proxy_cache_valid any 0; + add_header X-Cache-Status $upstream_cache_status; + } +} diff --git a/ansible/roles/oohelperd/templates/oohelperd.env b/ansible/roles/oohelperd/templates/oohelperd.env new file mode 100644 index 00000000..80167997 --- /dev/null +++ b/ansible/roles/oohelperd/templates/oohelperd.env @@ -0,0 +1,5 @@ +# Generated by ansible +# roles/oohelperd/templates/oohelperd.env + +# Password for the "prom" user on oohelperd's own /metrics endpoint +PROMETHEUS_METRICS_PASSWORD={{ prometheus_metrics_password }} diff --git a/ansible/roles/oohelperd/templates/oohelperd.service b/ansible/roles/oohelperd/templates/oohelperd.service new file mode 100644 index 00000000..b09d034d --- /dev/null +++ b/ansible/roles/oohelperd/templates/oohelperd.service @@ -0,0 +1,30 @@ +# Generated by ansible +# roles/oohelperd/templates/oohelperd.service + +[Unit] +Description=OONI Web Connectivity test helper +Documentation=https://github.com/ooni/probe-cli +After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +Type=simple +# nginx terminates TLS in front of it, so it only listens on loopback +ExecStart=/usr/local/bin/oohelperd -api-endpoint 127.0.0.1:8080 -pprof-endpoint 127.0.0.1:6061 + +# Holds PROMETHEUS_METRICS_PASSWORD, used to protect /metrics +EnvironmentFile=/etc/ooni/oohelperd.env +Restart=on-failure +RestartSec=5 +User=oohelperd +Group=oohelperd +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +PrivateTmp=yes +# Every measurement opens several connections towards the target +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 0b28f33b..6a78c903 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -57,6 +57,31 @@ ooniapi_gateway_testlists_upstream: "" # Docker network shared by every service's containers on this host. ooniapi_gateway_network_name: ooniapi +# Routes of the API, shared with the ALB (see the file's header); the +# gateway's locations are generated from them in tasks/main.yml +ooniapi_gateway_routes: "{{ lookup('ansible.builtin.file', role_path ~ '/../../../tf/modules/ooniapi_frontend/routes.yaml') | from_yaml }}" + +# Response caches for the routes marked cache/cache_long in routes.yaml. +# nginx keeps an entry until it goes unused for `inactive`, expired or not, +# so responses valid for minutes and measurements valid for a day get zones +# of their own, and the first can't push the second out. Sized per host, as +# if it got all the traffic (2026-10, data2 query_log and api.ooni.org): +# - long: measurement bodies (raw_measurement, measurement_meta), valid 1 d: +# at most ~50K distinct a day (28K measurement_meta + 25K raw lookups) of +# ~76 KB (max seen ~160 KB), ~3.8 GB a day; 2x that, ~130K keys. +# - short: lists, aggregations, oonirun (valid 10 s to 1 h): ~7.5K requests +# an hour of up to ~60 KB (aggregations 0.3-22 KB, lists 35-70 KB, +# observations ~135 KB), ~450 MB an hour if all were distinct; 4x that. +# 1 MB of keys_zone holds ~8K keys. nginx stores responses uncompressed. +ooniapi_gateway_cache_dir: /var/cache/nginx/ooniapi-gateway +ooniapi_gateway_cache_zones: + short: {keys_zone: 8m, max_size: 2g, inactive: 1h} + long: {keys_zone: 16m, max_size: 8g, inactive: 1d} +# for responses without Cache-Control, like /api/v1/aggregation's today; +# the same as the legacy ooni-api vhost's apicache +ooniapi_gateway_cache_default_valid: 10s +ooniapi_gateway_nginx_user: www-data + # SSH user the deploy CodeBuild job connects as. Must match # `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer # Terraform module. @@ -66,3 +91,10 @@ ooniapi_gateway_deploy_user: deploy # moved/installed into place. Must match STAGING_DIR in # tf/modules/ooniapi_service_deployer/files/deploy.py. ooniapi_gateway_staging_dir: /var/lib/ooniapi/deploy-staging + +# How long nginx workers left over from a reload may keep serving in-flight +# requests (to the slot a deploy just took out of rotation) before they are +# closed. Matches the ALB's deregistration delay on ECS (AWS default, 300 s), +# and must match drain_timeout in ooniapi_service_deployer: a deploy waits +# this long after the previous flip before it restarts the drained slot. +ooniapi_gateway_drain_timeout: 300 diff --git a/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py new file mode 100644 index 00000000..1a17b46d --- /dev/null +++ b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py @@ -0,0 +1,73 @@ +# Filters turning tf/modules/ooniapi_frontend/routes.yaml (shared with the +# ALB) into the gateway's nginx locations; see tasks/main.yml. +# +# nginx picks a location by exact match, then longest prefix, where the ALB +# goes by rule priority. The two agree as long as every ALB pattern is exact +# or a prefix (a trailing *), and no prefix of one service covers a path of +# another; ooniapi_gateway_route_problems lists the routes where they would +# not. + + +def _locations(routes): + out = [] + for r in sorted(routes, key=lambda r: r["priority"]): + # the ALB sends these to ooniprobe_legacy, which the gateway doesn't + # run: they go to ooniprobe, whatever their X-Protocol-Version + if r.get("legacy"): + continue + for p in r.get("paths", []): + prefix = p.endswith("*") + cache = None + if p in r.get("cache_long", []): + cache = "long" + elif r.get("cache"): + cache = "short" + out.append({ + "route": r["name"], + "service": r["service"], + "match": "^~" if prefix else "=", + "path": p[:-1] if prefix else p, + "cache": cache, + }) + return out + + +def ooniapi_gateway_locations(routes): + return _locations(routes) + + +def ooniapi_gateway_direct_hosts(routes): + return list(dict.fromkeys(r["service"] for r in routes if r.get("direct_host"))) + + +def ooniapi_gateway_route_problems(routes): + locations = _locations(routes) + problems = [] + seen = {} + for r in routes: + for p in r.get("cache_long", []): + if p not in r.get("paths", []): + problems.append(f"{r['name']}: cache_long {p} is not one of its paths") + for loc in locations: + if "*" in loc["path"] or "?" in loc["path"]: + problems.append(f"{loc['route']}: {loc['path']}: only exact paths and a trailing * can be served by nginx") + key = (loc["match"], loc["path"]) + if key in seen: + problems.append(f"{loc['route']}: {loc['path']} is also routed by {seen[key]}") + seen.setdefault(key, loc["route"]) + for a in locations: + if a["match"] != "^~": + continue + for b in locations: + if b["service"] != a["service"] and b["path"].startswith(a["path"]): + problems.append(f"{a['route']}: {a['path']}* covers {b['route']}'s {b['path']}, nginx would not follow ALB priorities") + return problems + + +class FilterModule: + def filters(self): + return { + "ooniapi_gateway_locations": ooniapi_gateway_locations, + "ooniapi_gateway_direct_hosts": ooniapi_gateway_direct_hosts, + "ooniapi_gateway_route_problems": ooniapi_gateway_route_problems, + } diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 020790d3..8b1594c4 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -67,6 +67,9 @@ ansible.builtin.copy: dest: "/etc/nginx/conf.d/{{ item.item.name }}-upstream.conf" content: | + map "" $ooniapi_cache_version_{{ item.item.name }} { + default ""; + } upstream {{ item.item.name }} { server 127.0.0.1:{{ item.item.host_port_a }}; server 127.0.0.1:{{ item.item.host_port_b }} down; @@ -203,6 +206,32 @@ mode: "0440" validate: "visudo -cf %s" +# worker_shutdown_timeout is only valid in nginx's main context, which +# conf.d can't reach; modules-enabled/*.conf is included there by both the +# nginx role's nginx.conf and Debian's stock one. +- name: Bound how long old nginx workers drain after a reload + ansible.builtin.copy: + dest: /etc/nginx/modules-enabled/90-ooniapi-gateway-drain.conf + content: | + # Managed by ansible - roles/ooniapi_gateway (ooniapi_gateway_drain_timeout) + worker_shutdown_timeout {{ ooniapi_gateway_drain_timeout }}s; + owner: root + group: root + mode: "0644" + notify: reload ooniapi gateway nginx + +- name: Derive and check the gateway's locations + ansible.builtin.import_tasks: routes.yml + +- name: Ensure the gateway's cache directories exist + ansible.builtin.file: + path: "{{ ooniapi_gateway_cache_dir }}/{{ item }}" + state: directory + owner: "{{ ooniapi_gateway_nginx_user }}" + group: "{{ ooniapi_gateway_nginx_user }}" + mode: "0700" + loop: "{{ ooniapi_gateway_cache_zones.keys() | list }}" + - name: Render the gateway vhost ansible.builtin.template: src: gateway.conf.j2 diff --git a/ansible/roles/ooniapi_gateway/tasks/routes.yml b/ansible/roles/ooniapi_gateway/tasks/routes.yml new file mode 100644 index 00000000..a6478c7c --- /dev/null +++ b/ansible/roles/ooniapi_gateway/tasks/routes.yml @@ -0,0 +1,14 @@ +--- +# The gateway's locations are generated from routes.yaml, shared with the +# ALB, by filter_plugins/ooniapi_routes.py, which also checks nginx can +# serve them the way the ALB does. +- name: Derive the gateway's locations from routes.yaml + ansible.builtin.set_fact: + ooniapi_gateway_locations: "{{ ooniapi_gateway_routes | ooniapi_gateway_locations }}" + ooniapi_gateway_direct_hosts: "{{ ooniapi_gateway_routes | ooniapi_gateway_direct_hosts }}" + ooniapi_gateway_route_problems: "{{ ooniapi_gateway_routes | ooniapi_gateway_route_problems }}" + +- name: Check that nginx can serve routes.yaml like the ALB does + ansible.builtin.assert: + that: ooniapi_gateway_route_problems | length == 0 + fail_msg: "{{ ooniapi_gateway_route_problems }}" diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 index e87e46ef..a6563a5b 100644 --- a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -1,11 +1,12 @@ # Managed by ansible - roles/ooniapi_gateway/templates/gateway.conf.j2 # -# Single "gateway" vhost for the OONI tier0 API on this dedicated host, -# mirroring (1:1, by design) the AWS ALB listener rules in -# tf/modules/ooniapi_frontend/main.tf. Only rules for services actually -# running blue/green here ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }}) -# are handled; everything unmatched falls through to "reverseproxy", exactly -# like the ALB's default_action. +# Single "gateway" vhost for the OONI tier0 API on this dedicated host. Its +# locations are generated from tf/modules/ooniapi_frontend/routes.yaml, which +# the AWS ALB's listener rules are generated from too, so both route alike. +# Only routes of services running blue/green here +# ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }}) and of +# those proxied cross-cloud are handled; everything unmatched falls through +# to "reverseproxy", exactly like the ALB's default_action. # # Each service's own `-upstream.conf` (the `upstream { # ... }` block referenced below via proxy_pass) is dropped into this same @@ -23,16 +24,22 @@ map $remote_addr $ooniapi_gateway_remote_addr_anon { default 0.0.0.0; } -log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon [$time_local] ' +log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon $upstream_cache_status [$time_local] ' '"$request" $status $body_bytes_sent rt:$request_time "$http_referer" "$http_user_agent"'; +# response caches, see ooniapi_gateway_cache_zones in defaults/main.yml +{% for name, zone in ooniapi_gateway_cache_zones.items() %} +proxy_cache_path {{ ooniapi_gateway_cache_dir }}/{{ name }} levels=1:2 keys_zone=ooniapi_{{ name }}:{{ zone.keys_zone }} + max_size={{ zone.max_size }} inactive={{ zone.inactive }} use_temp_path=off; +{% endfor %} + server { listen 443 ssl http2; listen [::]:443 ssl http2; server_name {{ ([ooniapi_gateway_primary_domain] + ooniapi_gateway_extra_server_names) | join(' ') }}; - access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; - error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + access_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info; client_max_body_size 200M; # for measurement POST, matches the legacy API vhost @@ -63,70 +70,35 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 900; -{% if 'ooniauth' in service_names %} - ## --- ooniauth --- (ALB priority 108) - location ^~ /api/v2/ooniauth/ { proxy_pass http://ooniauth; } - location = /api/v1/user_register { proxy_pass http://ooniauth; } - location = /api/v1/user_login { proxy_pass http://ooniauth; } - location = /api/v1/user_refresh_token { proxy_pass http://ooniauth; } - location = /api/_/account_metadata { proxy_pass http://ooniauth; } -{% endif %} - -{% if 'oonirun' in service_names %} - ## --- oonirun --- (ALB priority 110) - location ^~ /api/v2/oonirun/ { proxy_pass http://oonirun; } -{% endif %} - -{% if 'ooniprobe' in service_names %} - ## --- ooniprobe --- (ALB priorities 120-123) - location ^~ /api/v2/ooniprobe/ { proxy_pass http://ooniprobe; } - location = /api/v1/login { proxy_pass http://ooniprobe; } - location = /api/v1/register { proxy_pass http://ooniprobe; } - location ^~ /api/v1/update/ { proxy_pass http://ooniprobe; } - location ^~ /api/v1/check-in { proxy_pass http://ooniprobe; } - location ^~ /api/v1/test-helpers { proxy_pass http://ooniprobe; } - location = /api/v1/test-list/urls { proxy_pass http://ooniprobe; } - location ^~ /report { proxy_pass http://ooniprobe; } - location = /api/_/show_countries_prioritization { proxy_pass http://ooniprobe; } - location = /api/_/debug_prioritization { proxy_pass http://ooniprobe; } - location ^~ /api/v1/manifest { proxy_pass http://ooniprobe; } - location ^~ /api/v1/sign_credential { proxy_pass http://ooniprobe; } - location ^~ /api/v1/submit_measurement { proxy_pass http://ooniprobe; } - location ^~ /bouncer/net-tests { proxy_pass http://ooniprobe; } - location ^~ /api/v1/geolookup { proxy_pass http://ooniprobe; } - location ^~ /api/v1/collectors { proxy_pass http://ooniprobe; } - location = /api/v1/test-list/tor-targets { proxy_pass http://ooniprobe; } - location = /api/v1/test-list/psiphon-config { proxy_pass http://ooniprobe; } -{% endif %} - -{% if 'oonifindings' in service_names %} - ## --- oonifindings --- (ALB priority 130) - location ^~ /api/v1/incidents/ { proxy_pass http://oonifindings; } -{% endif %} - -{% if 'oonimeasurements' in service_names %} - ## --- oonimeasurements --- (ALB priorities 140-143) - location ^~ /api/v1/measurements/ { proxy_pass http://oonimeasurements; } - location = /api/v1/raw_measurement { proxy_pass http://oonimeasurements; } - location = /api/v1/measurement_meta { proxy_pass http://oonimeasurements; } - location = /api/v1/measurements { proxy_pass http://oonimeasurements; } - location = /api/v1/torsf_stats { proxy_pass http://oonimeasurements; } - location = /api/v1/aggregation { proxy_pass http://oonimeasurements; } - location ^~ /api/v1/aggregation/ { proxy_pass http://oonimeasurements; } - location = /api/v1/observations { proxy_pass http://oonimeasurements; } - location = /api/v1/analysis { proxy_pass http://oonimeasurements; } - location = /api/v1/detector/changepoints { proxy_pass http://oonimeasurements; } -{% endif %} - -{% if ooniapi_gateway_testlists_upstream %} - ## --- testlists (ALB priority 144; not yet migrated off AWS, proxied - ## cross-cloud until it is) --- - location ^~ /api/_/url-submission/test-list/ { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/_/url-priorities/list { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/_/url-priorities/update { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/v1/url-submission/submit { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/v1/url-submission/update-url { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + # caching, for the locations of routes marked cache/cache_long in + # routes.yaml (proxy_cache is off elsewhere). Responses are cached as long + # as their Cache-Control allows (nginx skips no-cache, private, max-age=0 + # and those setting cookies), or ooniapi_gateway_cache_default_valid + # without one. Requests with credentials never use the cache. Each cached + # location keys on its service's $ooniapi_cache_version_, the + # image tag its blue/green deploy writes to -upstream.conf, so a + # deploy invalidates that service's responses, and only those. + proxy_cache_methods GET HEAD; + proxy_cache_valid 200 301 302 {{ ooniapi_gateway_cache_default_valid }}; + proxy_cache_valid any 0; + proxy_cache_lock on; + proxy_cache_lock_timeout 30s; + proxy_cache_lock_age 30s; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_bypass $http_authorization; + proxy_no_cache $http_authorization; + add_header X-Cache-Status $upstream_cache_status always; + +{% set remote_upstreams = {'testlists': 'https://' ~ ooniapi_gateway_testlists_upstream} if ooniapi_gateway_testlists_upstream else {} %} + ## --- generated from routes.yaml, see tasks/main.yml --- +{% for l in ooniapi_gateway_locations %} +{% if l.service in service_names %} + location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }};{% if l.cache %} proxy_cache ooniapi_{{ l.cache }}; proxy_cache_key "$ooniapi_cache_version_{{ l.service }}$scheme$host$request_uri";{% endif %} } # {{ l.route }} +{% elif l.service in remote_upstreams %} + # {{ l.service }} is not yet migrated off AWS, proxied cross-cloud until it is + location {{ l.match }} {{ l.path }} { proxy_pass {{ remote_upstreams[l.service] }}; } # {{ l.route }} {% endif %} +{% endfor %} ## --- default: everything else goes to reverseproxy, same as the ALB's ## default_action. Covers legacy/unmigrated paths (oonith is NOT among @@ -137,15 +109,14 @@ server { } } -{% for service in service_names %} -{% if service != 'reverseproxy' %} +{% for service in ooniapi_gateway_direct_hosts if service in service_names %} server { listen 443 ssl http2; listen [::]:443 ssl http2; server_name {{ service }}.{{ ooniapi_gateway_direct_domain_suffix }}; - access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; - error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + access_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info; ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; @@ -163,5 +134,4 @@ server { proxy_pass http://{{ service }}; } } -{% endif %} {% endfor %} diff --git a/ansible/roles/oonidata_airflow/handlers/main.yml b/ansible/roles/oonidata_airflow/handlers/main.yml index f6dda47d..e7434941 100644 --- a/ansible/roles/oonidata_airflow/handlers/main.yml +++ b/ansible/roles/oonidata_airflow/handlers/main.yml @@ -2,3 +2,13 @@ ansible.builtin.systemd_service: name: nginx state: reloaded + +- name: reload systemd + ansible.builtin.systemd: + daemon_reload: true + +- name: restart detector-panel + ansible.builtin.systemd: + name: detector-panel.service + state: restarted + enabled: true diff --git a/ansible/roles/oonidata_airflow/tasks/main.yml b/ansible/roles/oonidata_airflow/tasks/main.yml index a466a7c3..09d8eeed 100644 --- a/ansible/roles/oonidata_airflow/tasks/main.yml +++ b/ansible/roles/oonidata_airflow/tasks/main.yml @@ -46,7 +46,7 @@ - name: "Install pipeline with pip" ansible.builtin.shell: - cmd: "{{ miniconda_install_dir }}/bin/pip install --upgrade /opt/airflow/oonidata/oonipipeline/" + cmd: "{{ miniconda_install_dir }}/bin/pip install --upgrade '/opt/airflow/oonidata/oonipipeline/[analysis]'" chdir: "{{ miniconda_install_dir }}" become_user: miniconda tags: @@ -97,3 +97,29 @@ tags: - oonidata - config + +# Event detector panel +- name: Deploy detector-panel systemd unit + ansible.builtin.template: + src: templates/detector-panel.service + dest: /etc/systemd/system/detector-panel.service + owner: root + mode: '0644' + notify: + - reload systemd + - restart detector-panel + tags: + - detector_panel + +- name: Setup event detector password file and ensure permissions are set + community.general.htpasswd: + path: "/etc/ooni/detector_panel_passwd" + name: admin + password: "{{ detector_panel_passwd }}" + owner: root + group: "nginx" + mode: 0640 + notify: + - reload nginx + tags: + - detector_panel diff --git a/ansible/roles/oonidata_airflow/templates/detector-panel.service b/ansible/roles/oonidata_airflow/templates/detector-panel.service new file mode 100644 index 00000000..18480675 --- /dev/null +++ b/ansible/roles/oonidata_airflow/templates/detector-panel.service @@ -0,0 +1,14 @@ +[Unit] +Description=OONI Pipeline Events Panel +After=network.target + +[Service] +Type=simple +ExecStart=/opt/miniconda/bin/python -m oonipipeline.main events-panel --port 8501 +Restart=on-failure +RestartSec=5 +User=oonipipeline +WorkingDirectory=/opt/miniconda + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 index 14b8dae6..6d201f36 100644 --- a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 +++ b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 @@ -50,3 +50,30 @@ server { proxy_buffering off; } } + +server { + listen 443 ssl http2; + include /etc/nginx/ssl_intermediate.conf; + ssl_certificate /var/lib/dehydrated/certs/{{event_detector_panel_domain}}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{event_detector_panel_domain}}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{event_detector_panel_domain}}/chain.pem; + server_name {{ event_detector_panel_domain }}; + access_log /var/log/nginx/{{ event_detector_panel_domain }}.access.log; + error_log /var/log/nginx/{{ event_detector_panel_domain }}.log warn; + + location / { + auth_basic "Administrator's Area"; + auth_basic_user_file /etc/ooni/detector_panel_passwd; + proxy_pass http://127.0.0.1:8501; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + client_max_body_size 100M; + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header X-Scheme $scheme; + proxy_buffering off; + } +} diff --git a/ansible/roles/prometheus/files/alert_ooni.yml b/ansible/roles/prometheus/files/alert_ooni.yml index e43397ed..a84b2732 100644 --- a/ansible/roles/prometheus/files/alert_ooni.yml +++ b/ansible/roles/prometheus/files/alert_ooni.yml @@ -27,4 +27,10 @@ groups: summary: 'Mirror {{ $labels.instance }} lags by {{ $value | humanizeDuration }}' description: '{{ if eq $labels.instance "ooni.torproject.org:443" }}Run `make update-site`.{{ end }}' + - alert: TestHelperDown + expr: up{domain!=""} != 1 + for: 5m + annotations: + summary: 'test helper down: {{ $labels.instance }} ({{ $labels.domain }})' + ... diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index ed675d0d..ccd5b006 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -22,6 +22,12 @@ scrape_configs: # Node exporter metrics {% for bbjob in blackbox_jobs %} - job_name: "{{ bbjob.name }}" +{% if bbjob.scrape_interval is defined %} + scrape_interval: {{ bbjob.scrape_interval }} +{% endif %} +{% if bbjob.scrape_timeout is defined %} + scrape_timeout: {{ bbjob.scrape_timeout }} +{% endif %} metrics_path: /probe params: module: @@ -84,9 +90,10 @@ scrape_configs: - https://data2.htz-fsn.prod.ooni.nu/metrics/node_exporter - https://data3.htz-fsn.prod.ooni.nu/metrics/node_exporter - https://notebook1.htz-fsn.prod.ooni.nu/metrics/node_exporter - - http://0.do.th.prod.ooni.io:9001/metrics - - http://1.do.th.prod.ooni.io:9001/metrics - - http://2.do.th.prod.ooni.io:9001/metrics + - https://wc.th.dev.ooni.io/metrics + - https://wcth0.fra1.prod.ooni.io:9001/metrics/node_exporter + - https://wcth1.fra1.prod.ooni.io:9001/metrics/node_exporter + - https://wcth2.fra1.prod.ooni.io:9001/metrics/node_exporter relabel_configs: # set the scheme based on what has been parsed in the address - source_labels: [__address__] @@ -205,9 +212,9 @@ scrape_configs: password: '{{ prometheus_metrics_password }}' static_configs: - targets: - - 0.do.th.prod.ooni.io - - 1.do.th.prod.ooni.io - - 2.do.th.prod.ooni.io + - wcth0.fra1.prod.ooni.io + - wcth1.fra1.prod.ooni.io + - wcth2.fra1.prod.ooni.io - job_name: 'ooni-web' scrape_interval: 5m scheme: https diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index 1da7f053..1432d135 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -1,8 +1,5 @@ dom0_hosts: - ams-slack-1.ooni.org - - doams1-countly.ooni.nu - - mia-echoth.ooni.nu - - mia-httpth.ooni.nu blackbox_jobs: # TODO add these records to the ALB config @@ -25,6 +22,9 @@ blackbox_jobs: - "https://4.th.ooni.org/" - "https://5.th.ooni.org/" - "https://6.th.ooni.org/" + - "https://wcth0.fra1.prod.ooni.io/" + - "https://wcth1.fra1.prod.ooni.io/" + - "https://wcth2.fra1.prod.ooni.io/" - "https://d33d1gs9kpq1c5.cloudfront.net/status" - name: "ooni collector" @@ -48,22 +48,27 @@ blackbox_jobs: # cloudfront - "https://d3kr4emv7f56qa.cloudfront.net/bouncer/net-tests" - # IP addresses are used for test-helpers in monitoring configuration for some - # historical reason hopefully remembered by @hellais. + - name: "ooni explorer homepage" + module: "http_2xx" + targets: + - "https://explorer.ooni.org/" + + # We use IP monitoring since this is what is served to users through check-in. - name: "ooni tcp echo" module: "ooni_tcp_echo_ok" + ignore_instance_down: true # covered by the more specific TestHelperDown alert + labels: + domain: "echo.th.prod.ooni.io" targets: - - "{{ lookup('dig', 'c.echo.th.ooni.io/A') }}:80" + - "{{ lookup('dig', 'echo.th.prod.ooni.io/A') }}:80" - name: "ooni http return json headers" module: "ooni_http_return_json_headers_ok" + ignore_instance_down: true # covered by the more specific TestHelperDown alert + labels: + domain: "json.th.prod.ooni.io" targets: - - "http://{{ lookup('dig', 'a.http.th.ooni.io/A') }}:80" - - - name: "ooni explorer homepage" - module: "http_2xx" - targets: - - "https://explorer.ooni.org/" + - "http://{{ lookup('dig', 'json.th.prod.ooni.io/A') }}:80" # API # @@ -103,6 +108,65 @@ blackbox_jobs: - "https://api.ooni.io/api/_/global_overview" - "https://api.ooni.org/api/_/global_overview" + - name: "ooni API private" + module: "https_2xx_json" + ignore_instance_down: true + scrape_interval: 5m + scrape_timeout: 30s + labels: + tier: "tier-1" + targets: + - "https://api.ooni.org/api/_/asn_by_month" + - "https://api.ooni.org/api/_/countries_by_month" + - "https://api.ooni.org/api/_/test_names" + - "https://api.ooni.org/api/_/countries" + - "https://api.ooni.org/api/_/test_coverage?probe_cc=IT" + - "https://api.ooni.org/api/_/website_networks?probe_cc=IT" + - "https://api.ooni.org/api/_/website_stats?probe_cc=IT&probe_asn=3269&input=https://www.google.com/" + - "https://api.ooni.org/api/_/website_urls?probe_cc=IT&probe_asn=AS3269" + - "https://api.ooni.org/api/_/vanilla_tor_stats?probe_cc=IT" + - "https://api.ooni.org/api/_/im_networks?probe_cc=IT" + - "https://api.ooni.org/api/_/im_stats?probe_cc=IT&probe_asn=AS3269&test_name=signal" + - "https://api.ooni.org/api/_/network_stats?probe_cc=IT" + - "https://api.ooni.org/api/_/country_overview?probe_cc=IT" + - "https://api.ooni.org/api/_/global_overview_by_month" + - "https://api.ooni.org/api/_/circumvention_stats_by_country" + - "https://api.ooni.org/api/_/circumvention_runtime_stats" + - "https://api.ooni.org/api/_/domain_metadata?domain=google.com" + - "https://api.ooni.org/api/_/asnmeta?asn=3269" + - "https://api.ooni.org/api/_/networks" + - "https://api.ooni.org/api/_/domains" + + # api.ooni.io goes to backend-fsn nginx, which caches some responses + - name: "ooni API private (legacy)" + module: "https_2xx_json" + ignore_instance_down: true + scrape_interval: 5m + scrape_timeout: 30s + labels: + tier: "tier-1" + targets: + - "https://api.ooni.io/api/_/asn_by_month" + - "https://api.ooni.io/api/_/countries_by_month" + - "https://api.ooni.io/api/_/test_names" + - "https://api.ooni.io/api/_/countries" + - "https://api.ooni.io/api/_/test_coverage?probe_cc=IT" + - "https://api.ooni.io/api/_/website_networks?probe_cc=IT" + - "https://api.ooni.io/api/_/website_stats?probe_cc=IT&probe_asn=3269&input=https://www.google.com/" + - "https://api.ooni.io/api/_/website_urls?probe_cc=IT&probe_asn=AS3269" + - "https://api.ooni.io/api/_/vanilla_tor_stats?probe_cc=IT" + - "https://api.ooni.io/api/_/im_networks?probe_cc=IT" + - "https://api.ooni.io/api/_/im_stats?probe_cc=IT&probe_asn=AS3269&test_name=signal" + - "https://api.ooni.io/api/_/network_stats?probe_cc=IT" + - "https://api.ooni.io/api/_/country_overview?probe_cc=IT" + - "https://api.ooni.io/api/_/global_overview_by_month" + - "https://api.ooni.io/api/_/circumvention_stats_by_country" + - "https://api.ooni.io/api/_/circumvention_runtime_stats" + - "https://api.ooni.io/api/_/domain_metadata?domain=google.com" + - "https://api.ooni.io/api/_/asnmeta?asn=3269" + - "https://api.ooni.io/api/_/networks" + - "https://api.ooni.io/api/_/domains" + - name: "ooni API findings listing" module: "https_2xx_json" labels: @@ -183,11 +247,6 @@ blackbox_jobs: # end of API # - - name: "countly.ooni.io ping" - module: "http_2xx" - targets: - - "https://countly.ooni.io/o/ping" - - name: "slack inviter" module: "http_2xx" targets: diff --git a/ansible/roles/prometheus_node_exporter/defaults/main.yml b/ansible/roles/prometheus_node_exporter/defaults/main.yml index 9066e94c..0cdc0011 100644 --- a/ansible/roles/prometheus_node_exporter/defaults/main.yml +++ b/ansible/roles/prometheus_node_exporter/defaults/main.yml @@ -18,3 +18,7 @@ node_exporter_options: '' node_exporter_state: started node_exporter_enabled: true node_exporter_restart: on-failure +http_port: 9001 +https_port: 443 +use_https: false +monitoring_server_ip: "{{ lookup('dig', 'monitoring.ooni.org') }}" diff --git a/ansible/roles/prometheus_node_exporter/handlers/main.yml b/ansible/roles/prometheus_node_exporter/handlers/main.yml index 4ec66003..092c7285 100644 --- a/ansible/roles/prometheus_node_exporter/handlers/main.yml +++ b/ansible/roles/prometheus_node_exporter/handlers/main.yml @@ -18,3 +18,9 @@ service: name: node_exporter state: restarted + +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded diff --git a/ansible/roles/prometheus_node_exporter/tasks/install.yml b/ansible/roles/prometheus_node_exporter/tasks/install.yml index 47ed566e..1ff74425 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/install.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/install.yml @@ -4,6 +4,9 @@ failed_when: false changed_when: false register: node_exporter_version_check + tags: + - prometheus_node_exporter + - packages - name: Download and unarchive node_exporter into temporary location. unarchive: @@ -15,6 +18,9 @@ node_exporter_version_check.stdout is not defined or node_exporter_version not in node_exporter_version_check.stdout register: node_exporter_download_check + tags: + - prometheus_node_exporter + - packages - name: Move node_exporter binary into place. copy: @@ -26,11 +32,17 @@ when: > node_exporter_download_check is changed or node_exporter_version_check.stdout | length == 0 + tags: + - prometheus_node_exporter + - packages - name: Create node_exporter group. group: name: "{{ prometheus_group }}" state: present + tags: + - prometheus_node_exporter + - config - name: Create node_exporter user. user: @@ -38,6 +50,9 @@ shell: /sbin/nologin group: "{{ prometheus_group }}" state: present + tags: + - prometheus_node_exporter + - config - name: Add a user to a password file and ensure permissions are set community.general.htpasswd: @@ -48,9 +63,10 @@ group: "{{ 'nginx' if use_nginx else prometheus_group }}" mode: 0640 tags: + - prometheus_node_exporter - monitoring - node_exporter - - config + - secrets - name: Copy the node_exporter systemd unit file. template: @@ -58,18 +74,27 @@ dest: /etc/systemd/system/node_exporter.service mode: 0644 register: node_exporter_service + tags: + - prometheus_node_exporter + - config - name: Reload systemd daemon if unit file is changed. systemd: daemon_reload: true notify: restart node_exporter when: node_exporter_service is changed + tags: + - prometheus_node_exporter + - service - name: Ensure node_exporter is running and enabled at boot. service: name: node_exporter state: "{{ node_exporter_state }}" enabled: "{{ node_exporter_enabled }}" + tags: + - prometheus_node_exporter + - service - name: Verify node_exporter is responding to requests. uri: @@ -77,3 +102,6 @@ return_content: true register: metrics_output failed_when: "'Metrics' not in metrics_output.content" + tags: + - prometheus_node_exporter + - monitoring diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index 21726b00..35e32438 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -1,26 +1,10 @@ -- ansible.builtin.include_role: - name: nginx - tags: - - nginx - - node_exporter - when: use_nginx - -- ansible.builtin.include_role: - name: dehydrated - tags: - - oonidata - - dehydrated - vars: - ssl_domains: - - "{{ inventory_hostname }}" - when: use_https - - name: create ooni configuration directory ansible.builtin.file: path: "/etc/ooni/" state: directory owner: root tags: + - prometheus_node_exporter - monitoring - node_exporter - config @@ -33,9 +17,36 @@ notify: - Restart nginx tags: + - prometheus_node_exporter - monitoring - node_exporter - config when: use_nginx +- name: Allow traffic on the node exporter http_port from the monitoring host only + become: true + tags: [prometheus_node_exporter, monitoring, node_exporter, nftables] + ansible.builtin.blockinfile: + path: "/etc/ooni/nftables/tcp/{{ http_port }}.nft" + create: yes + mode: "0644" + block: | + add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport {{ http_port }} counter accept comment "node exporter metrics" + when: use_nginx and not use_https + notify: + - reload nftables + +- name: Allow traffic on the node exporter https_port from the monitoring host only + become: true + tags: [prometheus_node_exporter, monitoring, node_exporter, nftables] + ansible.builtin.blockinfile: + path: "/etc/ooni/nftables/tcp/{{ http_port }}.nft" + create: yes + mode: "0644" + block: | + add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport {{ https_port }} counter accept comment "node exporter metrics" + when: use_nginx and use_https + notify: + - reload nftables + - include_tasks: install.yml diff --git a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 index 6e7de3d5..5af3c130 100644 --- a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 +++ b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 @@ -2,7 +2,7 @@ server { {% if use_https %} - listen 443 ssl http2; + listen {{ https_port }} ssl http2; server_name {{ inventory_hostname }}; include /etc/nginx/ssl_intermediate.conf; @@ -11,7 +11,7 @@ server { ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; {% else %} - listen 80; + listen {{ http_port }}; server_name {{ inventory_hostname }}; {% endif %} @@ -30,4 +30,4 @@ server { } {% endfor %} -} \ No newline at end of file +} diff --git a/ansible/roles/ssh_users/tasks/main.yml b/ansible/roles/ssh_users/tasks/main.yml index 6ab0e805..77330df0 100644 --- a/ansible/roles/ssh_users/tasks/main.yml +++ b/ansible/roles/ssh_users/tasks/main.yml @@ -52,16 +52,17 @@ mode: 0400 with_items: "{{ admin_usernames | union(non_admin_usernames) }}" -- name: configure sshd +- name: configure sshd_config tags: ssh_users - include_role: - name: willshersystems.sshd - vars: - sshd_skip_defaults: false - sshd: - AllowUsers: "{{ admin_usernames | union(non_admin_usernames) | sort | join(' ') }}" + template: + src: sshd_config + dest: /etc/ssh/sshd_config + owner: root + group: root + mode: 0440 + validate: 'sshd -t -f %s' -- name: Enesure sudoers dir exists +- name: Ensure sudoers dir exists tags: ssh_users ansible.builtin.file: path: /etc/sudoers.d @@ -85,8 +86,8 @@ path: /etc/sudoers.d/adm state: absent -- name: reload sshd +- name: reload ssh tags: ssh_users ansible.builtin.systemd_service: - name: sshd + name: ssh state: reloaded diff --git a/ansible/roles/ssh_users/templates/allow_users b/ansible/roles/ssh_users/templates/allow_users new file mode 100644 index 00000000..e69de29b diff --git a/ansible/roles/ssh_users/templates/sshd_config b/ansible/roles/ssh_users/templates/sshd_config new file mode 100644 index 00000000..04a795a9 --- /dev/null +++ b/ansible/roles/ssh_users/templates/sshd_config @@ -0,0 +1,13 @@ +Include /etc/ssh/sshd_config.d/*.conf +AllowUsers {{ admin_usernames | union(non_admin_usernames) | sort | join(' ') }} +PasswordAuthentication No +KbdInteractiveAuthentication no +UsePAM yes +X11Forwarding yes +PrintMotd no +AcceptEnv LANG LC_* +Subsystem sftp /usr/lib/openssh/sftp-server +Ciphers chacha20-poly1305@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr +# mlkem768x25519-sha256 is excluded, but we should enable it once we have all hosts on trixie +KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256 +MACs hmac-sha2-512,hmac-sha2-256 diff --git a/ansible/roles/tailnet/tasks/main.yml b/ansible/roles/tailnet/tasks/main.yml deleted file mode 100644 index 86bc4b3d..00000000 --- a/ansible/roles/tailnet/tasks/main.yml +++ /dev/null @@ -1,4 +0,0 @@ -- ansible.builtin.include_role: - name: artis3n.tailscale - tags: - - tailnet diff --git a/ansible/roles/test_helpers/defaults/main.yml b/ansible/roles/test_helpers/defaults/main.yml new file mode 100644 index 00000000..d3d76986 --- /dev/null +++ b/ansible/roles/test_helpers/defaults/main.yml @@ -0,0 +1,5 @@ +test_helpers_version: v0.1.6 +test_helpers_base_url: "https://github.com/ooni/ooniprobe-rs/releases/download/{{ test_helpers_version }}" + +# monitoring server IP +monitoring_server_ip: "{{ lookup('dig', 'monitoring.ooni.org') }}" diff --git a/ansible/roles/test_helpers/handlers/main.yml b/ansible/roles/test_helpers/handlers/main.yml new file mode 100644 index 00000000..1d820eff --- /dev/null +++ b/ansible/roles/test_helpers/handlers/main.yml @@ -0,0 +1,10 @@ +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded + +- name: reload nginx + service: + name: nginx + state: reloaded diff --git a/ansible/roles/test_helpers/tasks/main.yml b/ansible/roles/test_helpers/tasks/main.yml new file mode 100644 index 00000000..02b57055 --- /dev/null +++ b/ansible/roles/test_helpers/tasks/main.yml @@ -0,0 +1,76 @@ +--- + +# For prometheus scrape requests. +- name: Allow traffic on port 9100 from the monitoring host only + become: true + tags: prometheus-proxy + blockinfile: + path: /etc/ooni/nftables/tcp/9100.nft + create: yes + block: | + add rule inet filter input ip saddr {{ monitoring_server_ip }} tcp dport 9100 counter accept comment "node exporter" + notify: + - reload nftables + +- name: Allow traffic on the {{ helper }} test helper port + become: true + tags: test-helpers + blockinfile: + path: "/etc/ooni/nftables/tcp/{{ port }}.nft" + create: yes + block: | + add rule inet filter input tcp dport {{ port }} counter accept comment "{{ helper }} test helper" + notify: + - reload nftables + +# Create test helpers user +- name: Create the testhelpers user + ansible.builtin.user: + name: "testhelpers" + shell: "/bin/bash" + create_home: no + system: yes + become: yes + +# Install test helpers +- name: Download the {{ helper }} test helper binary + ansible.builtin.get_url: + url: "{{ test_helpers_base_url }}/{{ helper }}" + dest: "/usr/local/bin/{{ helper }}" + checksum: "sha256:{{ test_helpers_checksum }}" + mode: '0755' + force: yes + become: yes + register: helper_binary + +# Create systemd units + +- name: Create .service file + tags: test-helpers + ansible.builtin.template: + src: templates/{{helper}}.service + dest: /etc/systemd/system/{{helper}}.service + mode: '0755' + owner: root + register: helper_unit + +- name: reload systemd + tags: test-helpers + ansible.builtin.systemd_service: + daemon_reload: yes + when: helper_unit.changed + +- name: Start helper + tags: test-helpers + systemd: + name: "{{helper}}.service" + state: started + enabled: yes + +- name: Restart helper if its binary or unit file changed + tags: test-helpers + ansible.builtin.systemd_service: + name: "{{ helper }}.service" + state: restarted + become: yes + when: helper_binary.changed or helper_unit.changed diff --git a/ansible/roles/test_helpers/templates/ooechohelpd.service b/ansible/roles/test_helpers/templates/ooechohelpd.service new file mode 100644 index 00000000..2ba60f1c --- /dev/null +++ b/ansible/roles/test_helpers/templates/ooechohelpd.service @@ -0,0 +1,22 @@ +[Unit] +Description=Test helper that will start an echo session on request +After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +Type=simple +ExecStart=/usr/local/bin/ooechohelpd --port {{port}} +Restart=on-failure +RestartSec=5 +User=testhelpers +Group=testhelpers +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +PrivateTmp=yes +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/test_helpers/templates/oojsonhelpd.service b/ansible/roles/test_helpers/templates/oojsonhelpd.service new file mode 100644 index 00000000..de3c4367 --- /dev/null +++ b/ansible/roles/test_helpers/templates/oojsonhelpd.service @@ -0,0 +1,22 @@ +[Unit] +Description=Test helper that will respond with a json showing the headers it received +After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +Type=simple +ExecStart=/usr/local/bin/oojsonhelpd --port {{port}} +Restart=on-failure +RestartSec=5 +User=testhelpers +Group=testhelpers +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +PrivateTmp=yes +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/test_helpers/vars/main.yml b/ansible/roles/test_helpers/vars/main.yml new file mode 100644 index 00000000..3d224a6d --- /dev/null +++ b/ansible/roles/test_helpers/vars/main.yml @@ -0,0 +1,4 @@ + +# choices: oojsonhelpd, ooechohelpd +helper: "oojsonhelpd" +port: "80" diff --git a/docs/Runbooks.md b/docs/Runbooks.md index 766c53e0..c1b14ffa 100644 --- a/docs/Runbooks.md +++ b/docs/Runbooks.md @@ -1191,12 +1191,12 @@ In order to add new users to the jupyterlab notebook server hosted `notebook.ooni.org` the steps are: 1. Open a PR adding an entry to this list: - https://github.com/ooni/devops/blob/main/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu (we need ssh key, email and username) + https://github.com/ooni/devops/blob/main/ansible/host_vars/notebook.ooni.org (we need ssh key, email and username) 2. An OONI core team member will run: ``` -./play deploy-bootstrap.yml -l notebook1.htz-fsn.prod.ooni.nu --diff -i inventory +./play deploy-bootstrap.yml -l notebook.ooni.org --diff -i inventory ``` -3. An OONI core team member logs into `notebook1.htz-fsn.prod.ooni.nu` and runs: +3. An OONI core team member logs into `notebook.ooni.org` and runs: ``` sudo passwd NEWUSERNAME ``` @@ -1243,11 +1243,11 @@ The notebooks server uses the system’s actual user accounts to authenticate ac ### Procedure 0. Ask the person requesting a new account for the following details: **name**, **username** (used to log in to the server), and **SSH key**. -1. Create a new user entry in the [notebook server’s host vars](https://github.com/ooni/devops/blob/main/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu). Remember to also add the username to the `non_admin_usernames` variable +1. Create a new user entry in the [notebook server’s host vars](https://github.com/ooni/devops/blob/main/ansible/host_vars/notebook.ooni.org). Remember to also add the username to the `non_admin_usernames` variable 2. In the `ansible` directory, run: ``` - ./play -i inventory deploy-bootstrap.yml -l notebook1.htz-fsn.prod.ooni.nu --diff + ./play -i inventory deploy-bootstrap.yml -l notebook.ooni.org --diff ``` 3. Log in to `notebook.ooni.org` via SSH. 4. Set the default password for the new user using: diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 31805934..f8a2fab6 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -237,16 +237,20 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } -data "aws_ssm_parameter" "clickhouse_readonly_url" { - name = "/oonidevops/secrets/clickhouse_readonly_url" +data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" { + name = "/oonidevops/secrets/clickhouse_oonimeasurements_url" } -data "aws_ssm_parameter" "clickhouse_readonly_test_url" { - name = "/oonidevops/secrets/clickhouse_readonly_test_url" +data "aws_ssm_parameter" "clickhouse_oonimeasurements_test_url" { + name = "/oonidevops/secrets/clickhouse_oonimeasurements_test_url" } -data "aws_ssm_parameter" "clickhouse_write_url" { - name = "/oonidevops/secrets/clickhouse_write_url" +data "aws_ssm_parameter" "clickhouse_ooniprobe_url" { + name = "/oonidevops/secrets/clickhouse_ooniprobe_url" +} + +data "aws_ssm_parameter" "clickhouse_oonirun_url" { + name = "/oonidevops/secrets/clickhouse_oonirun_url" } data "aws_ssm_parameter" "account_id_hashing_key" { @@ -280,20 +284,19 @@ locals { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.value ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value } oonirun = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.value } oonifindings = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value } ooniauth = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value @@ -307,7 +310,7 @@ locals { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_test_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.value ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value } } @@ -480,19 +483,21 @@ moved { ### OONI Tier0 Backend Proxy -module "ooni_th_droplet" { - source = "../../modules/ooni_th_droplet" - - stage = local.environment - instance_location = "fra1" - instance_size = "s-1vcpu-1gb" - droplet_count = 1 - deployer_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] - metrics_password = data.aws_ssm_parameter.prometheus_metrics_password.arn - ssh_keys = [ - "3d:81:99:17:b5:d1:20:a5:fe:2b:14:96:67:93:d6:34", - "f6:4b:8b:e2:0e:d2:97:c5:45:5c:07:a6:fe:54:60:0e" - ] +resource "digitalocean_ssh_key" "oonidevops" { + name = "oonidevops" + public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] +} + + +module "ooni_test_helpers_wc" { + source = "../../modules/ooni_th_binary_droplet" + + stage = local.environment + name = "ooniwcth" + hostname = "wc.th" + + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] + dns_zone_ooni_io = local.dns_zone_ooni_io } @@ -648,13 +653,14 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniprobe" repo = "ooni/backend" - branch_name = "master" + branch_name = "feat/experiment-versions" environment = local.environment trigger_path = "ooniapi/services/ooniprobe/**" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" @@ -703,7 +709,7 @@ module "ooniapi_ooniprobe" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.arn ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn } @@ -740,13 +746,67 @@ module "ooniapi_ooniprobe" { ) } +# Legacy ooniprobe service, used to serve older probes. Identified by the +# X-Protocol-Version header that specifies the anonymous credentials protocol +# version. +module "ooniapi_ooniprobe_legacy" { + source = "../../modules/ooniapi_service" + + # First run should be set on first run to bootstrap the task definition + # first_run = true + + task_memory = 256 + + vpc_id = module.network.vpc_id + + service_name = "ooniprobe-legacy" + default_docker_image_url = "ooni/api-ooniprobe:20260921-da8b057f" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_secrets = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.arn + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.arn + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn + } + + task_environment = { + FASTPATH_URL = "http://fastpath.${local.environment}.ooni.io:8472" + FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 3 # use a different one in prod + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + + ooniapi_service_security_groups = [ + # module.ooniapi_cluster.web_security_group_id + ] + + use_autoscaling = false + service_desired_count = 1 + + tags = merge( + local.tags, + { Name = "ooni-tier0-ooniprobe-legacy" } + ) +} + #### OONI Backend proxy service module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "reverseproxy" @@ -1050,7 +1110,7 @@ module "fastpath_builder" { service_name = "fastpath" repo = "ooni/backend" - branch_name = "fix-fastpath" + branch_name = "feat/wc-x-flags" environment = local.environment buildspec_path = "fastpath/buildspec.yml" trigger_path = "fastpath/**" @@ -1064,13 +1124,14 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonirun" repo = "ooni/backend" - branch_name = "oonirun-v2-1" + branch_name = "master" environment = local.environment buildspec_path = "ooniapi/services/oonirun/buildspec.yml" trigger_path = "ooniapi/services/oonirun/**" @@ -1106,7 +1167,7 @@ module "ooniapi_oonirun" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.arn } ooniapi_service_security_groups = [ @@ -1125,8 +1186,9 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonifindings" @@ -1167,7 +1229,6 @@ module "ooniapi_oonifindings" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn } ooniapi_service_security_groups = [ @@ -1186,8 +1247,9 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniauth" @@ -1280,13 +1342,14 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonimeasurements" repo = "ooni/backend" - branch_name = "master" + branch_name = "636-better-pagination" environment = local.environment trigger_path = "ooniapi/services/oonimeasurements/**" buildspec_path = "ooniapi/services/oonimeasurements/buildspec.yml" @@ -1332,7 +1395,7 @@ module "ooniapi_oonimeasurements" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_test_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.arn ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.arn } @@ -1446,7 +1509,7 @@ module "testlists_builder" { service_name = "testlists" repo = "ooni/backend" - branch_name = "master" + branch_name = "fix_1238_testlists_worktree" environment = local.environment buildspec_path = "ooniapi/services/testlists/buildspec.yml" trigger_path = "ooniapi/services/testlists/**" @@ -1467,6 +1530,7 @@ module "ooniapi_frontend" { ooniapi_oonirun_target_group_arn = module.ooniapi_oonirun.alb_target_group_id ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id + ooniapi_ooniprobe_legacy_target_group_arn = module.ooniapi_ooniprobe_legacy.alb_target_group_id ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id ooniapi_testlists_target_group_arn = module.ooniapi_testlists.alb_target_group_id @@ -1575,77 +1639,6 @@ module "ooni_monitoring" { tags = local.tags } -### Anonymous credentials testing instance -module "ooni_anonc" { - source = "../../modules/ec2" - - stage = local.environment - - vpc_id = module.network.vpc_id - subnet_id = module.network.vpc_subnet_public[0].id - private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block - dns_zone_ooni_io = local.dns_zone_ooni_io - - key_name = module.adm_iam_roles.oonidevops_key_name - instance_type = "t3a.small" - - name = "anonc" - ingress_rules = [{ - from_port = 22, - to_port = 22, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 80, # for dehydrated challenge - to_port = 80, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 443, # for the POC hosting - to_port = 443, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 9100, # for node exporter metrics - to_port = 9100, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"], - }] - - egress_rules = [{ - from_port = 0, - to_port = 0, - protocol = "-1", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 0, - to_port = 0, - protocol = "-1", - ipv6_cidr_blocks = ["::/0"], - }] - - sg_prefix = "oonianonc" - tg_prefix = "anon" - - disk_size = 20 - - tags = merge( - local.tags, - { Name = "ooni-tier0-anonc" } - ) -} - -resource "aws_route53_record" "anonc_alias" { - zone_id = local.dns_zone_ooni_io - name = "anonc.${local.environment}.ooni.io" - type = "CNAME" - ttl = 300 - - records = [ - module.ooni_anonc.aws_instance_public_dns - ] -} - # Jump host for accessing postgres module "ooni_jumphost" { source = "../../modules/ec2" diff --git a/tf/environments/prod/dns_records.tf b/tf/environments/prod/dns_records.tf index 1bbc99a5..061686de 100644 --- a/tf/environments/prod/dns_records.tf +++ b/tf/environments/prod/dns_records.tf @@ -38,14 +38,6 @@ resource "aws_route53_record" "deb-ooni-org-_CNAME_" { zone_id = local.dns_root_zone_ooni_org } -resource "aws_route53_record" "deb-ci-ooni-org-_A_" { - name = "deb-ci.ooni.org" - records = ["188.166.93.143"] - ttl = "1799" - type = "A" - zone_id = local.dns_root_zone_ooni_org -} - resource "aws_route53_record" "docs-ooni-org-_CNAME_" { name = "docs.ooni.org" records = ["cname.vercel-dns.com"] diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index f3543a25..a1e6b79c 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -239,16 +239,24 @@ data "aws_ssm_parameter" "oonipg_url" { name = "/oonidevops/secrets/ooni-tier0-postgres/postgresql_write_url" } -data "aws_ssm_parameter" "clickhouse_readonly_url" { - name = "/oonidevops/secrets/clickhouse_readonly_url" +data "aws_ssm_parameter" "clickhouse_write_url" { + name = "/oonidevops/secrets/clickhouse_write_url" } data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" { name = "/oonidevops/secrets/clickhouse_oonimeasurements_url" } -data "aws_ssm_parameter" "clickhouse_write_url" { - name = "/oonidevops/secrets/clickhouse_write_url" +data "aws_ssm_parameter" "clickhouse_oonimeasurements_test_url" { + name = "/oonidevops/secrets/clickhouse_oonimeasurements_test_url" +} + +data "aws_ssm_parameter" "clickhouse_ooniprobe_url" { + name = "/oonidevops/secrets/clickhouse_ooniprobe_url" +} + +data "aws_ssm_parameter" "clickhouse_oonirun_url" { + name = "/oonidevops/secrets/clickhouse_oonirun_url" } data "aws_ssm_parameter" "account_id_hashing_key" { @@ -313,7 +321,7 @@ locals { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.value } oonifindings = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value @@ -383,6 +391,10 @@ resource "random_id" "artifact_id" { byte_length = 4 } +data "aws_s3_bucket" "ooniprobe_failed_reports_2026_04_10" { + bucket = "ooniprobe-failed-reports-eu-central-1-1d24426a" +} + resource "aws_s3_bucket" "ooniprobe_failed_reports" { bucket = "ooniprobe-failed-reports-${var.aws_region}-${random_id.artifact_id.hex}" } @@ -503,28 +515,36 @@ moved { ### OONI Tier0 Backend Proxy -module "ooni_th_droplet" { - source = "../../modules/ooni_th_droplet" +module "ooni_test_helpers_wc" { + source = "../../modules/ooni_th_binary_droplet" stage = local.environment instance_location = "fra1" - instance_size = "s-1vcpu-1gb" - droplet_count = 3 - deployer_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] - metrics_password = data.aws_ssm_parameter.prometheus_metrics_password.arn - ssh_keys = [ - "3d:81:99:17:b5:d1:20:a5:fe:2b:14:96:67:93:d6:34", - "f6:4b:8b:e2:0e:d2:97:c5:45:5c:07:a6:fe:54:60:0e" - ] + name = "ooniwcth${count.index}-fra1" + hostname = "wcth${count.index}.fra1" + + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] dns_zone_ooni_io = local.dns_zone_ooni_io + count = 3 +} + +resource "aws_route53_record" "ooni_wc_th" { + zone_id = local.dns_root_zone_ooni_org + name = "wcth${count.index}.fra1.ooni.org" + type = "A" + ttl = 60 + + count = 3 + records = [module.ooni_test_helpers_wc[count.index].droplet_ipv4_address] } module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "reverseproxy" @@ -959,8 +979,9 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniprobe" @@ -1037,7 +1058,7 @@ module "ooniapi_ooniprobe" { ] use_autoscaling = false - service_desired_count = 2 + service_desired_count = 4 # max_desired_count = 8 # autoscale_policies = [ # { @@ -1053,6 +1074,59 @@ module "ooniapi_ooniprobe" { ) } +# Legacy ooniprobe service, used to serve older probes. Identified by the +# X-Protocol-Version header that specifies the anonymous credentials protocol +# version. +module "ooniapi_ooniprobe_legacy" { + source = "../../modules/ooniapi_service" + + # First run should be set on first run to bootstrap the task definition + # first_run = true + + vpc_id = module.network.vpc_id + + service_name = "ooniprobe-legacy" + default_docker_image_url = "ooni/api-ooniprobe:20260921-da8b057f" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + task_memory = 1024 + + task_secrets = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.arn + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn + } + + task_environment = { + # hardcoded IP for fastpath2.prod.prod.ooni.io + FASTPATH_URL = "http://10.0.0.32:8472" + FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 4 # be sure this is different from dev + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + use_autoscaling = false + service_desired_count = 1 + + tags = merge( + local.tags, + { Name = "ooni-tier0-ooniprobe-legacy" } + ) +} + ### Fastpath module "ooni_fastpath" { source = "../../modules/ooni_fastpath" @@ -1145,13 +1219,138 @@ module "fastpath_builder" { } +#### Test Helpers Machines +# + +# Registers the same oonidevops keypair used for the EC2 instances (see +# module.adm_iam_roles) as a DigitalOcean account key, so it can be installed +# on droplets via their ssh_keys argument too. +resource "digitalocean_ssh_key" "oonidevops" { + name = "oonidevops" + public_key = jsondecode(data.aws_secretsmanager_secret_version.deploy_key.secret_string)["public_key"] +} + +module "ooni_test_helpers_json" { + source = "../../modules/ooni_th_binary_droplet" + + stage = local.environment + name = "oonijsonth" + hostname = "json.th" + + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] + + dns_zone_ooni_io = local.dns_zone_ooni_io +} + +# Echo test helper requires a dedicated machine bc it's a tcp server, +# not an HTTP server. It's impossible to reroute using nginx +module "ooni_test_helpers_echo" { + source = "../../modules/ooni_th_binary_droplet" + + stage = local.environment + name = "ooniechoth" + hostname = "echo.th" + + ssh_keys = [digitalocean_ssh_key.oonidevops.fingerprint] + + dns_zone_ooni_io = local.dns_zone_ooni_io +} + + +module "reuploader_builder" { + source = "../../modules/ooni_docker_build" + trigger_tag = "" + + service_name = "reuploader" + repo = "ooni/backend" + branch_name = "master" + environment = local.environment + buildspec_path = "reuploader/buildspec.yml" + trigger_path = "reuploader/**" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket +} + +module "reuploader" { + source = "../../modules/scheduled_service" + + task_memory = 256 + + vpc_id = module.network.vpc_id + + first_run = true + service_name = "reuploader" + default_docker_image_url = "ooni/reuploader:20260617-8b35a38f" + schedule_expression = "cron(30 0 * * ? 2000-2199)" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + scheduled_task_cluster = module.ooniapi_cluster.cluster_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_environment = { + AWS_REGION = var.aws_region + BATCH_SIZE = 50000 + S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket + FASTPATH_API = "http://${module.ooni_reuploader_fastpath.aws_instance_private_ip}:8472" + LOG_LEVEL = "DEBUG" + } + + task_secrets = { + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + tags = merge( + local.tags, + { Name = "ooni-tier0-reuploader" } + ) +} + +# For reuploader accessing the failed reports s3 bucket +resource "aws_iam_role_policy" "reuploader_role" { + name = "${local.name}-task-role" + role = module.reuploader.task_role_name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "" + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" + }, + { + Sid = "" + Effect = "Allow" + Action = ["s3:ListBucket"] + Resource = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn + }, + { + Sid = "" + Effect = "Allow" + Action = ["s3:DeleteObject"] + Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" + } + ] + }) +} + + + + #### OONI Run service module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonirun" @@ -1194,7 +1393,7 @@ module "ooniapi_oonirun" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.arn } ooniapi_service_security_groups = [ @@ -1212,8 +1411,9 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonifindings" @@ -1274,8 +1474,9 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniauth" @@ -1370,8 +1571,9 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonimeasurements" @@ -1561,6 +1763,7 @@ module "ooniapi_frontend" { ooniapi_oonirun_target_group_arn = module.ooniapi_oonirun.alb_target_group_id ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id + ooniapi_ooniprobe_legacy_target_group_arn = module.ooniapi_ooniprobe_legacy.alb_target_group_id ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id ooniapi_testlists_target_group_arn = module.ooniapi_testlists.alb_target_group_id @@ -1842,3 +2045,15 @@ resource "aws_route53_record" "jumphost_alias" { module.ooni_jumphost.aws_instance_public_dns ] } + +resource "aws_route53_record" "detector_panel_alias" { + zone_id = local.dns_zone_ooni_io + name = "detector-panel.${local.environment}.ooni.io" + type = "A" + ttl = 300 + + records = [ + # Airflow host + "142.132.254.225" + ] +} diff --git a/tf/modules/adm_iam_roles/main.tf b/tf/modules/adm_iam_roles/main.tf index aa5c525f..10fb15f9 100644 --- a/tf/modules/adm_iam_roles/main.tf +++ b/tf/modules/adm_iam_roles/main.tf @@ -53,7 +53,8 @@ resource "aws_iam_policy" "oonidevops" { "secretsmanager:*", "cloudhsm:*", "athena:*", - "glue:*" + "glue:*", + "events:*" ], "Resource": "*" } diff --git a/tf/modules/ooni_th_binary_droplet/main.tf b/tf/modules/ooni_th_binary_droplet/main.tf new file mode 100644 index 00000000..f0d3c1f6 --- /dev/null +++ b/tf/modules/ooni_th_binary_droplet/main.tf @@ -0,0 +1,31 @@ +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + } +} + +resource "digitalocean_droplet" "ooni_th" { + image = "debian-13-x64" + name = "${var.name}-${var.stage}" + region = var.instance_location + size = var.instance_size + ipv6 = true + ssh_keys = var.ssh_keys + + lifecycle { + create_before_destroy = true + ignore_changes = all + } +} + +resource "aws_route53_record" "ooni_th" { + zone_id = var.dns_zone_ooni_io + name = "${var.hostname}.${var.stage}.ooni.io" + type = "A" + ttl = 60 + + records = [digitalocean_droplet.ooni_th.ipv4_address] +} diff --git a/tf/modules/ooni_th_binary_droplet/outputs.tf b/tf/modules/ooni_th_binary_droplet/outputs.tf new file mode 100644 index 00000000..e091c016 --- /dev/null +++ b/tf/modules/ooni_th_binary_droplet/outputs.tf @@ -0,0 +1,7 @@ +output "droplet_ipv4_address" { + value = digitalocean_droplet.ooni_th.ipv4_address +} + +output "fqdn" { + value = aws_route53_record.ooni_th.fqdn +} diff --git a/tf/modules/ooni_th_binary_droplet/variables.tf b/tf/modules/ooni_th_binary_droplet/variables.tf new file mode 100644 index 00000000..b257b344 --- /dev/null +++ b/tf/modules/ooni_th_binary_droplet/variables.tf @@ -0,0 +1,33 @@ +variable "stage" { + type = string +} + +variable "name" { + description = "Name of the droplet (will be suffixed with -)" + type = string +} + +variable "hostname" { + description = "DNS label for the helper, e.g. \"json.th\" or \"echo.th\". The record is created as ..ooni.io" + type = string +} + +variable "instance_location" { + type = string + default = "fra1" +} + +variable "instance_size" { + type = string + default = "s-1vcpu-1gb" +} + +variable "ssh_keys" { + description = "Fingerprints of the DigitalOcean account SSH keys to grant root access to" + type = list(string) +} + +variable "dns_zone_ooni_io" { + description = "id of the DNS zone for ooni_io" + type = string +} diff --git a/tf/modules/ooni_th_droplet/main.tf b/tf/modules/ooni_th_droplet/main.tf deleted file mode 100644 index 74413a6f..00000000 --- a/tf/modules/ooni_th_droplet/main.tf +++ /dev/null @@ -1,49 +0,0 @@ -terraform { - required_providers { - digitalocean = { - source = "digitalocean/digitalocean" - version = "~> 2.0" - } - } -} - -data "cloudinit_config" "ooni_th_docker" { - gzip = false - base64_encode = false - - part { - filename = "init.cfg" - content_type = "text/cloud-config" - content = templatefile("${path.module}/templates/cloud-init-docker.yml", { - monitoring_ip = "5.9.112.244", - deployer_key = var.deployer_key, - metrics_password = var.metrics_password, - }) - } -} - -resource "digitalocean_droplet" "ooni_th_docker" { - image = "ubuntu-24-04-x64" - name = "${var.name}-docker-${var.stage}-${count.index}" - region = var.instance_location - size = var.instance_size - ipv6 = true - ssh_keys = var.ssh_keys - user_data = data.cloudinit_config.ooni_th_docker.rendered - count = var.droplet_count - - lifecycle { - create_before_destroy = true - ignore_changes = all - } -} -resource "aws_route53_record" "ooni_th" { - zone_id = var.dns_zone_ooni_io - name = "${each.key}.do.th.${var.stage}.ooni.io" - type = "A" - ttl = 60 - for_each = { - for d in digitalocean_droplet.ooni_th_docker : reverse(split("-", d.name))[0] => d.ipv4_address - } - records = [each.value] -} diff --git a/tf/modules/ooni_th_droplet/outputs.tf b/tf/modules/ooni_th_droplet/outputs.tf deleted file mode 100644 index fdcad4af..00000000 --- a/tf/modules/ooni_th_droplet/outputs.tf +++ /dev/null @@ -1,10 +0,0 @@ -output "droplet_ipv4_address" { - value = digitalocean_droplet.ooni_th_docker[*].ipv4_address -} - -output "droplet_addresses" { - # for why we use values, - # see: https://github.com/hashicorp/terraform/issues/23245#issuecomment-548391304 - # https://github.com/hashicorp/terraform/issues/22476 - value = values(aws_route53_record.ooni_th)[*].fqdn -} diff --git a/tf/modules/ooni_th_droplet/templates/cloud-init-docker.yml b/tf/modules/ooni_th_droplet/templates/cloud-init-docker.yml deleted file mode 100644 index 93135daf..00000000 --- a/tf/modules/ooni_th_droplet/templates/cloud-init-docker.yml +++ /dev/null @@ -1,177 +0,0 @@ -apt: - sources: - docker.list: - source: "deb [arch=amd64 signed-by=$KEY_FILE] https://download.docker.com/linux/ubuntu $RELEASE stable" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth - lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh - 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq - L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 - UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N - cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht - ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo - vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD - G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ - XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj - q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB - tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 - BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO - v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd - tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk - jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m - 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P - XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc - FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 - g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm - ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh - 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 - G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW - FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB - EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF - M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx - Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu - w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk - z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 - eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb - VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa - 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X - zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ - pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 - ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ - BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY - 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp - YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI - mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES - KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 - JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ - cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 - 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 - U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z - VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f - irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk - SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz - QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W - 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw - 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe - dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y - Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR - H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh - /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ - M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S - xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O - jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG - YT90qFF93M3v01BbxP+EIY2/9tiIPbrd - =0YYh - -----END PGP PUBLIC KEY BLOCK----- -package_update: true -packages: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin - - prometheus-node-exporter - - nginx - -users: - - name: deployer - ssh-authorized-keys: - - ${deployer_key} - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - groups: - - sudo - - docker - shell: /bin/bash - -write_files: - - path: /etc/ssh/sshd_config - content: | - PermitRootLogin no - PermitEmptyPasswords no - PasswordAuthentication no - KbdInteractiveAuthentication no - UsePAM yes - PubkeyAuthentication yes - X11Forwarding no - PrintMotd no - AcceptEnv LANG LC_* - AllowUsers deployer - - - path: /etc/docker/daemon.json - content: | - { - "ipv6": true, - "fixed-cidr-v6": "2001:db8:1::/64", - "log-driver": "json-file", - "log-opts": {"max-size": "100m", "max-file": "3"} - } - - - path: /etc/nginx/sites-available/default - content: | - proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M - max_size=5g inactive=24h use_temp_path=off; - - server { - listen 80; - server_name _; - gzip on; - resolver 127.0.0.1; - - # test helper application metrics - location /metrics { - allow ${monitoring_ip}; - deny all; - - proxy_pass http://127.0.0.1:8080; - } - - # local test helper - location / { - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - proxy_pass http://127.0.0.1:8080; - - proxy_cache thcache; - proxy_cache_min_uses 1; - proxy_cache_lock on; - proxy_cache_lock_timeout 30; - proxy_cache_lock_age 30; - proxy_cache_use_stale error timeout invalid_header updating; - # Cache POST without headers set by the test helper! - proxy_cache_methods POST; - proxy_cache_key "$request_uri|$request_body"; - proxy_cache_valid 200 10m; - proxy_cache_valid any 0; - add_header X-Cache-Status $upstream_cache_status; - } - } - - server { - listen 9001; - server_name localhost; - - allow ${monitoring_ip}; - deny all; - - # Metrics from node_exporter - location = /metrics { - proxy_pass http://127.0.0.1:9100; - } - } - -runcmd: - - sshd -t - - systemctl restart sshd - - systemctl restart docker - - ufw default deny incoming - - ufw default allow outgoing - - ufw allow 22/tcp - - ufw allow 80/tcp - - ufw allow 443/tcp - - ufw allow from ${monitoring_ip} proto tcp to any port 9001 - - ufw enable - - service nginx restart - - docker container rm -f oonith - - docker run -d -e PROMETHEUS_METRICS_PASSWORD='${metrics_password}' -p 8080:80 --restart unless-stopped --name oonith ooni/oonith-oohelperd:latest diff --git a/tf/modules/ooni_th_droplet/templates/cloud-init.yml b/tf/modules/ooni_th_droplet/templates/cloud-init.yml deleted file mode 100644 index 111502ef..00000000 --- a/tf/modules/ooni_th_droplet/templates/cloud-init.yml +++ /dev/null @@ -1,59 +0,0 @@ -apt: - sources: - ooni.list: - source: "deb [trusted=yes] https://ooni-internal-deb.s3.eu-central-1.amazonaws.com unstable main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mDMEYGISFRYJKwYBBAHaRw8BAQdA4VxoR0gSsH56BbVqYdK9HNQ0Dj2YFVbvKIIZ - JKlaW920Mk9PTkkgcGFja2FnZSBzaWduaW5nIDxjb250YWN0QG9wZW5vYnNlcnZh - dG9yeS5vcmc+iJYEExYIAD4WIQS1oI8BeW5/UhhhtEk3LR/ycfLdUAUCYGISFQIb - AwUJJZgGAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRA3LR/ycfLdUFk+AQCb - gsUQsAQGxUFvxk1XQ4RgEoh7wy2yTuK8ZCkSHJ0HWwD/f2OAjDigGq07uJPYw7Uo - Ih9+mJ/ubwiPMzUWF6RSdgu4OARgYhIVEgorBgEEAZdVAQUBAQdAx4p1KerwcIhX - HfM9LbN6Gi7z9j4/12JKYOvr0d0yC30DAQgHiH4EGBYIACYWIQS1oI8BeW5/Uhhh - tEk3LR/ycfLdUAUCYGISFQIbDAUJJZgGAAAKCRA3LR/ycfLdUL4cAQCs53fLphhy - 6JMwVhRs02LXi1lntUtw1c+EMn6t7XNM6gD+PXpbgSZwoV3ZViLqr58o9fZQtV3s - oN7jfdbznrWVigE= - =PtYb - -----END PGP PUBLIC KEY BLOCK----- -package_update: true -packages: - - oohelperd - - nginx - -write_files: - - path: /etc/nginx/sites-available/default - content: | - proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M - max_size=5g inactive=24h use_temp_path=off; - - server { - listen 80; - server_name _; - gzip on; - resolver 127.0.0.1; - # local test helper - location / { - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - proxy_pass http://127.0.0.1:8080; - - proxy_cache thcache; - proxy_cache_min_uses 1; - proxy_cache_lock on; - proxy_cache_lock_timeout 30; - proxy_cache_lock_age 30; - proxy_cache_use_stale error timeout invalid_header updating; - # Cache POST without headers set by the test helper! - proxy_cache_methods POST; - proxy_cache_key "$request_uri|$request_body"; - proxy_cache_valid 200 10m; - proxy_cache_valid any 0; - add_header X-Cache-Status $upstream_cache_status; - - } - } - -runcmd: - - service nginx restart diff --git a/tf/modules/ooni_th_droplet/variables.tf b/tf/modules/ooni_th_droplet/variables.tf deleted file mode 100644 index 522e3333..00000000 --- a/tf/modules/ooni_th_droplet/variables.tf +++ /dev/null @@ -1,40 +0,0 @@ -variable "stage" { - type = string -} - -variable "name" { - description = "Name of the droplets" - type = string - default = "ooni-wcth" -} - -variable "instance_location" { - type = string - default = "fra1" -} - -variable "instance_size" { - # s-2vcpu-4gb - type = string - default = "s-1vcpu-1gb" -} - -variable "droplet_count" { - default = 1 -} - -variable "ssh_keys" { - type = list(string) -} - -variable "deployer_key" { - type = string -} - -variable "metrics_password" { - type = string -} - -variable "dns_zone_ooni_io" { - type = string -} diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index d6034f2e..8b6d0650 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -261,321 +261,151 @@ resource "aws_alb_listener_rule" "ooniapi_th" { tags = var.tags } -resource "aws_lb_listener_rule" "ooniapi_ooniauth_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 108 +locals { + # routes.yaml is shared with the gateway on the Hetzner hosts + # (ansible/roles/ooniapi_gateway), so both serve the same routes + routes = yamldecode(file("${path.module}/routes.yaml")) - action { - type = "forward" - target_group_arn = var.ooniapi_ooniauth_target_group_arn + target_groups = { + ooniauth = var.ooniapi_ooniauth_target_group_arn + oonirun = var.ooniapi_oonirun_target_group_arn + ooniprobe = var.ooniapi_ooniprobe_target_group_arn + ooniprobe_legacy = var.ooniapi_ooniprobe_legacy_target_group_arn + oonifindings = var.ooniapi_oonifindings_target_group_arn + oonimeasurements = var.ooniapi_oonimeasurements_target_group_arn + testlists = var.ooniapi_testlists_target_group_arn } - condition { - path_pattern { - values = [ - "/api/v2/ooniauth/*", - "/api/v1/user_register", - "/api/v1/user_login", - "/api/v1/user_refresh_token", - "/api/_/account_metadata", - ] - } - } + listener_rules = { for r in local.routes : r.name => r } } -resource "aws_lb_listener_rule" "ooniapi_ooniauth_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 109 +resource "aws_lb_listener_rule" "route" { + for_each = local.listener_rules - action { - type = "forward" - target_group_arn = var.ooniapi_ooniauth_target_group_arn - } - - condition { - host_header { - values = ["ooniauth.${local.direct_domain_suffix}"] - } - } -} - -resource "aws_lb_listener_rule" "ooniapi_oonirun_rule" { listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 110 + priority = each.value.priority action { type = "forward" - target_group_arn = var.ooniapi_oonirun_target_group_arn + target_group_arn = local.target_groups[each.value.service] } - condition { - path_pattern { - values = ["/api/v2/oonirun/*"] + dynamic "condition" { + for_each = can(each.value.paths) ? [each.value.paths] : [] + content { + path_pattern { + values = condition.value + } } - - } -} - -resource "aws_lb_listener_rule" "ooniapi_oonirun_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 111 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonirun_target_group_arn } - condition { - host_header { - values = ["oonirun.${local.direct_domain_suffix}"] + dynamic "condition" { + for_each = try(each.value.direct_host, false) ? [each.value.service] : [] + content { + host_header { + values = ["${condition.value}.${local.direct_domain_suffix}"] + } } } -} - -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 120 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v2/ooniprobe/*", - "/api/v1/login", - "/api/v1/register", - "/api/v1/update/*", - "/api/v1/check-in*" - ] + dynamic "condition" { + for_each = can(each.value.http_header) ? [each.value.http_header] : [] + content { + http_header { + http_header_name = condition.value.name + values = condition.value.values + } } } } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_2" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 121 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/test-helpers*", - "/api/v1/test-list/urls", - "/report*", - "/api/_/show_countries_prioritization", - "/api/_/debug_prioritization" - ] - } - } +# the rules were one resource each before routes.yaml +moved { + from = aws_lb_listener_rule.ooniapi_ooniauth_rule + to = aws_lb_listener_rule.route["ooniauth_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 122 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - # anonymous credentials - condition { - path_pattern { - values = [ - "/api/v1/manifest*", - "/api/v1/sign_credential*", - "/api/v1/submit_measurement*" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniauth_rule_host + to = aws_lb_listener_rule.route["ooniauth_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_4" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 123 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/bouncer/net-tests*", - "/api/v1/geolookup*", - "/api/v1/collectors*", - "/api/v1/test-list/tor-targets", - "/api/v1/test-list/psiphon-config" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonirun_rule + to = aws_lb_listener_rule.route["oonirun_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 125 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - - condition { - host_header { - values = ["ooniprobe.${local.direct_domain_suffix}"] - } - } - +moved { + from = aws_lb_listener_rule.ooniapi_oonirun_rule_host + to = aws_lb_listener_rule.route["oonirun_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 130 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonifindings_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/incidents/*", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule + to = aws_lb_listener_rule.route["ooniprobe_rule"] } -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 131 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonifindings_target_group_arn - } - condition { - host_header { - values = ["oonifindings.${local.direct_domain_suffix}"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_2 + to = aws_lb_listener_rule.route["ooniprobe_rule_2"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_host" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 139 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } - condition { - host_header { - values = ["oonimeasurements.${local.direct_domain_suffix}"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_legacy_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_legacy_version"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_1" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 140 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/measurements/*", - "/api/v1/raw_measurement", - "/api/v1/measurement_meta", - "/api/v1/measurements", - "/api/v1/torsf_stats" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_4 + to = aws_lb_listener_rule.route["ooniprobe_rule_4"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_current_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_current_version"] +} - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 142 +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_host + to = aws_lb_listener_rule.route["ooniprobe_rule_host"] +} - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_no_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_no_version"] +} - condition { - path_pattern { - values = [ - "/api/v1/aggregation", - "/api/v1/aggregation/*", - "/api/v1/observations", - "/api/v1/analysis", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonifindings_rule + to = aws_lb_listener_rule.route["oonifindings_rule"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_3" { +moved { + from = aws_lb_listener_rule.ooniapi_oonifindings_rule_host + to = aws_lb_listener_rule.route["oonifindings_rule_host"] +} - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 143 +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_host[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_host"] +} - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_1[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_1"] +} - condition { - path_pattern { - values = [ - "/api/v1/detector/changepoints", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_2[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_2"] } -resource "aws_lb_listener_rule" "ooniapi_testlists_rule" { - # hotfix: to allow us to deploy the frontend without the testlists service - count = var.ooniapi_testlists_target_group_arn != null ? 1 : 0 - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 144 +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_3 + to = aws_lb_listener_rule.route["oonimeasurements_rule_3"] +} - action { - type = "forward" - target_group_arn = var.ooniapi_testlists_target_group_arn - } - condition { - path_pattern { - values = [ - "/api/_/url-submission/test-list/*", - "/api/_/url-priorities/list", - "/api/_/url-priorities/update", - "/api/v1/url-submission/submit", - "/api/v1/url-submission/update-url", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_testlists_rule[0] + to = aws_lb_listener_rule.route["testlists_rule"] } diff --git a/tf/modules/ooniapi_frontend/routes.yaml b/tf/modules/ooniapi_frontend/routes.yaml new file mode 100644 index 00000000..ea1e430c --- /dev/null +++ b/tf/modules/ooniapi_frontend/routes.yaml @@ -0,0 +1,176 @@ +# Routes of the OONI API, the one place they are declared. Read by: +# - Terraform, tf/modules/ooniapi_frontend: one ALB listener rule per entry +# (aws_lb_listener_rule.route[""]), for the services on ECS. +# - Ansible, roles/ooniapi_gateway: the nginx locations of the gateway on the +# Hetzner hosts, for the services deployed there blue/green. +# Anything no route matches goes to reverseproxy, on both. +# +# Each entry is one ALB rule, so the ALB's limits apply: at most 5 values +# over all its conditions. Fields: +# name resource key of the rule; keep stable, renaming one makes +# Terraform replace the rule +# priority ALB rule priority, lowest matches first. The gateway can't +# order its locations by it, so a prefix of one service must +# not cover a path of another (checked by the gateway role) +# service target service (ALB target group) +# paths ALB path patterns: exact, or ending in * for a prefix +# direct_host route . to the service +# http_header only match requests with this header and one of these values +# (ALB only: they only choose between ooniprobe and its legacy) +# legacy ALB only: ooniprobe_legacy is being retired and does not run +# on the gateway, which sends these paths to ooniprobe +# cache gateway only: cache GET/HEAD responses of these paths in the +# short lived zone (see roles/ooniapi_gateway defaults) +# cache_long gateway only: those of its paths to cache in the long lived zone +# instead, for responses of immutable measurements + +- name: ooniauth_rule + priority: 108 + service: ooniauth + paths: + - "/api/v2/ooniauth/*" + - "/api/v1/user_register" + - "/api/v1/user_login" + - "/api/v1/user_refresh_token" + - "/api/_/account_metadata" + +- name: ooniauth_rule_host + priority: 109 + service: ooniauth + direct_host: true + +- name: oonirun_rule + priority: 110 + service: oonirun + paths: + - "/api/v2/oonirun/*" + cache: true + +- name: oonirun_rule_host + priority: 111 + service: oonirun + direct_host: true + +- name: ooniprobe_rule + priority: 120 + service: ooniprobe + paths: + - "/api/v2/ooniprobe/*" + - "/api/v1/login" + - "/api/v1/register" + - "/api/v1/update/*" + - "/api/v1/check-in*" + +- name: ooniprobe_rule_2 + priority: 121 + service: ooniprobe + paths: + - "/api/v1/test-helpers*" + - "/api/v1/test-list/urls" + - "/report*" + - "/api/_/show_countries_prioritization" + - "/api/_/debug_prioritization" + +- name: ooniprobe_rule_3_legacy_version + priority: 122 + service: ooniprobe_legacy + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + http_header: + name: X-Protocol-Version + values: ["0.1.0"] + legacy: true + +- name: ooniprobe_rule_4 + priority: 123 + service: ooniprobe + paths: + - "/bouncer/net-tests*" + - "/api/v1/geolookup*" + - "/api/v1/collectors*" + - "/api/v1/test-list/tor-targets" + - "/api/v1/test-list/psiphon-config" + +- name: ooniprobe_rule_3_current_version + priority: 124 + service: ooniprobe + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + http_header: + name: X-Protocol-Version + values: ["*"] + +- name: ooniprobe_rule_host + priority: 125 + service: ooniprobe + direct_host: true + +- name: ooniprobe_rule_3_no_version + priority: 126 + service: ooniprobe_legacy + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + legacy: true + +- name: oonifindings_rule + priority: 130 + service: oonifindings + paths: + - "/api/v1/incidents/*" + +- name: oonifindings_rule_host + priority: 131 + service: oonifindings + direct_host: true + +- name: oonimeasurements_rule_host + priority: 139 + service: oonimeasurements + direct_host: true + +- name: oonimeasurements_rule_1 + priority: 140 + service: oonimeasurements + paths: + - "/api/v1/measurements/*" + - "/api/v1/raw_measurement" + - "/api/v1/measurement_meta" + - "/api/v1/measurements" + - "/api/v1/torsf_stats" + cache: true + cache_long: + - "/api/v1/raw_measurement" + - "/api/v1/measurement_meta" + +- name: oonimeasurements_rule_2 + priority: 142 + service: oonimeasurements + paths: + - "/api/v1/aggregation" + - "/api/v1/aggregation/*" + - "/api/v1/observations" + - "/api/v1/analysis" + cache: true + +- name: oonimeasurements_rule_3 + priority: 143 + service: oonimeasurements + paths: + - "/api/v1/detector/changepoints" + cache: true + +- name: testlists_rule + priority: 144 + service: testlists + paths: + - "/api/_/url-submission/test-list/*" + - "/api/_/url-priorities/list" + - "/api/_/url-priorities/update" + - "/api/v1/url-submission/submit" + - "/api/v1/url-submission/update-url" diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index d92c39e3..47ff4f10 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -28,18 +28,22 @@ variable "ooniapi_ooniprobe_target_group_arn" { description = "arn for the target group of the ooniprobe service" } +variable "ooniapi_ooniprobe_legacy_target_group_arn" { + description = "arn for the target group of the legacy ooniprobe service" +} + variable "ooniapi_oonifindings_target_group_arn" { description = "arn for the target group of the oonifindings service" } variable "ooniapi_oonimeasurements_target_group_arn" { description = "arn for the target group of the oonimeasurements service" - default = null + nullable = false } variable "ooniapi_testlists_target_group_arn" { description = "arn for the target group of the testlists service" - default = null + nullable = false } variable "dns_zone_ooni_io" { diff --git a/tf/modules/ooniapi_service/main.tf b/tf/modules/ooniapi_service/main.tf index 84e9f38f..94bebfd8 100644 --- a/tf/modules/ooniapi_service/main.tf +++ b/tf/modules/ooniapi_service/main.tf @@ -126,6 +126,9 @@ resource "aws_ecs_service" "ooniapi_service" { lifecycle { create_before_destroy = true + # fixes an issue when AWS changes the capacity provider + # on its own, terraform has no record of that on its internal state + ignore_changes = [capacity_provider_strategy] } force_new_deployment = true diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 57cbf5c7..16eae8f1 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -18,7 +18,9 @@ # needed on top of docker compose. import json +import logging import os +import re import subprocess import sys import tempfile @@ -36,6 +38,8 @@ # host. Must match ooniapi_gateway_staging_dir in the Ansible role. STAGING_DIR = "/var/lib/ooniapi/deploy-staging" +log = logging.getLogger("deploy") + def require_env(name): value = os.environ.get(name) @@ -45,6 +49,9 @@ def require_env(name): def run(cmd, **kwargs): + # every remote and aws command goes through here; secret values are only + # ever written to files, never passed on a command line, so this is safe + log.info("running: %s", " ".join(cmd)) return subprocess.run(cmd, check=True, text=True, **kwargs) @@ -57,7 +64,9 @@ def ssh_output(user, host, remote_cmd): def ssh_succeeds(user, host, remote_cmd): - return subprocess.run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd]).returncode == 0 + cmd = ["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd] + log.info("running: %s", " ".join(cmd)) + return subprocess.run(cmd).returncode == 0 def scp(user, host, local_path, remote_path): @@ -93,14 +102,27 @@ def write_tmp(name, content, mode=0o644): def deploy_host(host, ctx): service = ctx["service"] user = ctx["user"] - print(f"=== {service}: deploying to {host} ===") + log.info(f"=== {service}: deploying to {host} ===") active_slot = ssh_output( user, host, f"cat /etc/ooniapi/{service}/active_slot 2>/dev/null || echo a" ).strip() or "a" + + # the slot about to be replaced is the one the previous deploy took out + # of rotation: nginx workers from before that flip may still be serving + # requests to it, for up to drain_timeout (worker_shutdown_timeout in + # the ooniapi_gateway role). Don't restart it under them. + since_flip = int(ssh_output( + user, host, + f"echo $(( $(date +%s) - $(stat -c %Y /etc/ooniapi/{service}/active_slot 2>/dev/null || echo 0) ))", + ).strip()) + wait = ctx["drain_timeout"] - since_flip + if wait > 0: + log.info(f"{service} on {host}: previous deploy flipped {since_flip}s ago, waiting {wait}s for its old slot to drain") + time.sleep(wait) target_slot = "b" if active_slot == "a" else "a" target_port = ctx["host_port_b"] if target_slot == "b" else ctx["host_port_a"] - print(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") + log.info(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") # compose file for the target slot compose_file = f"{service}-{target_slot}.yaml" @@ -125,35 +147,71 @@ def deploy_host(host, ctx): ssh(user, host, f"sudo docker compose -f {compose_path} up -d --pull always --remove-orphans") - healthy = False - for _ in range(10): - if ssh_succeeds(user, host, f"curl -sf -o /dev/null http://127.0.0.1:{target_port}/health"): - healthy = True - break - time.sleep(2) - if not healthy: - sys.exit(f"{service} on {host}: slot {target_slot} failed health check, aborting deploy") + # like the ECS target groups' health check (ooniapi_service: timeout 5, + # healthy_threshold 2): each check may take up to 5 s, and the slot must + # pass twice in a row. ECS gives a new task about a minute to come up; + # the deadline defaults to twice that, and is logged so it can be tuned + started = time.monotonic() + passes = 0 + while passes < 2: + if ssh_succeeds(user, host, f"curl -sf --max-time 5 -o /dev/null http://127.0.0.1:{target_port}/health"): + passes += 1 + else: + passes = 0 + elapsed = time.monotonic() - started + if passes < 2 and elapsed > ctx["health_check_timeout"]: + sys.exit(f"{service} on {host}: slot {target_slot} not healthy after {elapsed:.0f}s, aborting deploy") + if passes < 2: + time.sleep(2) + log.info(f"{service} on {host}: slot {target_slot} healthy after {time.monotonic() - started:.0f}s") - state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") upstream_file = f"{service}-upstream.conf" - upstream_content = s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") - upstream_content = upstream_content.replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) - upstream_local = write_tmp(upstream_file, upstream_content) - scp(user, host, upstream_local, f"{STAGING_DIR}/{upstream_file}") - ssh(user, host, - f"sudo mv {STAGING_DIR}/{upstream_file} /etc/nginx/conf.d/{upstream_file}" - f" && sudo nginx -t && sudo systemctl reload nginx") + upstream_path = f"/etc/nginx/conf.d/{upstream_file}" + state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") + # the image tag doubles as this service's cache version on the gateway + # (see nginx_upstream.conf.tftpl): new tag, new cache keys + upstream_content = ( + s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") + .replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) + .replace("__CACHE_VERSION__", ctx["image_tag"]) + ) + # kept as is, to put back if nginx rejects the new one + previous_content = ssh_output(user, host, f"cat {upstream_path}") + + def install_upstream(content): + scp(user, host, write_tmp(upstream_file, content), f"{STAGING_DIR}/{upstream_file}") + ssh(user, host, f"sudo mv {STAGING_DIR}/{upstream_file} {upstream_path}") + + install_upstream(upstream_content) + if not ssh_succeeds(user, host, "sudo nginx -t"): + # nginx is still running its old config, but a config that fails + # nginx -t must not stay in conf.d: the next reload (another + # service's deploy, a certificate renewal) would fail, and a restart + # would take every service on this host down. Put back the previous + # file, slot and cache version, which nginx -t accepted before. + log.error(f"{service} on {host}: nginx -t failed with slot {target_slot}'s upstream, restoring slot {active_slot}'s") + install_upstream(previous_content) + if not ssh_succeeds(user, host, "sudo nginx -t"): + sys.exit(f"{service} on {host}: nginx -t still fails after restoring slot {active_slot}'s upstream:" + " the nginx config is broken independently of this deploy, fix it before reloading nginx") + sys.exit(f"{service} on {host}: nginx -t failed with slot {target_slot}'s upstream; restored slot" + f" {active_slot}'s, which is still serving. Aborting deploy") + ssh(user, host, "sudo systemctl reload nginx") ssh(user, host, f"echo {target_slot} | sudo tee /etc/ooniapi/{service}/active_slot > /dev/null") - print(f"=== {service} on {host}: now serving from slot {target_slot} ===") + log.info(f"=== {service} on {host}: now serving from slot {target_slot} ===") def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout) with open("imagedefinitions.json") as f: image_tag = json.load(f)[0]["imageUri"].rsplit(":", 1)[-1] + # it ends up in an nginx string and in compose files + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", image_tag): + sys.exit(f"unexpected image tag: {image_tag!r}") service = require_env("SERVICE_NAME") - print(f"Deploying {service} image tag {image_tag}") + log.info(f"Deploying {service} image tag {image_tag}") with open("/tmp/deploy_key", "w") as f: f.write(secretsmanager_get(require_env("DEPLOY_SSH_KEY_SECRET_ARN"))) @@ -166,6 +224,8 @@ def main(): "image_tag": image_tag, "host_port_a": require_env("HOST_PORT_A"), "host_port_b": require_env("HOST_PORT_B"), + "health_check_timeout": int(os.environ.get("HEALTH_CHECK_TIMEOUT") or 120), + "drain_timeout": int(os.environ.get("DRAIN_TIMEOUT") or 300), "secrets": json.loads(secretsmanager_get(require_env("SERVICE_SECRETS_ARN"))), } diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index 46e90700..dc6f2165 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -5,6 +5,9 @@ data "aws_caller_identity" "current" {} locals { account_id = data.aws_caller_identity.current.account_id env_label = var.environment == "prod" ? "latest" : "dev" + + deploy_ecs = contains(["ecs", "both"], var.deploy_mode) + deploy_blue_green = contains(["blue_green", "both"], var.deploy_mode) } resource "aws_iam_policy" "codebuild" { @@ -161,10 +164,10 @@ resource "aws_codebuild_project" "ooniapi" { } } -## Docker Compose blue/green deploy (deploy_mode = "blue_green") +## Docker Compose blue/green deploy (deploy_mode = "blue_green" or "both") resource "aws_s3_object" "compose_file" { - for_each = var.deploy_mode == "blue_green" ? { a = var.host_port_a, b = var.host_port_b } : {} + for_each = local.deploy_blue_green ? { a = var.host_port_a, b = var.host_port_b } : {} bucket = var.deploy_bucket key = "${var.service_name}/${var.service_name}-${each.key}.yaml" @@ -182,7 +185,7 @@ resource "aws_s3_object" "compose_file" { } resource "aws_s3_object" "nginx_upstream" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 bucket = var.deploy_bucket key = "${var.service_name}/${var.service_name}-upstream.conf" @@ -196,7 +199,7 @@ resource "aws_s3_object" "nginx_upstream" { } resource "aws_s3_object" "deploy_script" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 bucket = var.deploy_bucket key = "${var.service_name}/deploy.py" @@ -206,7 +209,7 @@ resource "aws_s3_object" "deploy_script" { } resource "aws_iam_policy" "deploy" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 description = "Policy used in trust relationship with the blue/green deploy CodeBuild project" name = "codebuild-deploy-${var.service_name}-${var.aws_region}" @@ -260,7 +263,7 @@ resource "aws_iam_policy" "deploy" { } resource "aws_iam_role" "deploy" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 assume_role_policy = <