Blue green podman: add "both" mode, add nginx cache, improve logging, shared route generation

.github/workflows/check_ansible.yml

@@ -58,41 +58,41 @@ jobs:
       #- name: Setup tmate session
       #  uses: mxschmitt/action-tmate@v3
 
-      - uses: actions/github-script@v6
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const commentTitle = "Ansible Run Output";
-            const ansiblePlaybookOutput = `${{ steps.playbook.outputs.ansible_playbook}}`;
-            const parts = ansiblePlaybookOutput.split(/PLAY RECAP \*+/);
-            const ansiblePlaybookRecap = parts.length > 1 ? parts[1].trim() : '';
+      #- uses: actions/github-script@v9
+      #  with:
+      #    github-token: ${{ secrets.GITHUB_TOKEN }}
+      #    script: |
+      #      const commentTitle = "Ansible Run Output";
+      #      const ansiblePlaybookOutput = `${{ steps.playbook.outputs.ansible_playbook}}`;
+      #      const parts = ansiblePlaybookOutput.split(/PLAY RECAP \*+/);
+      #      const ansiblePlaybookRecap = parts.length > 1 ? parts[1].trim() : '';
 
-            const commentBody = `
-            #### Ansible Playbook Recap 🔍
+      #      const commentBody = `
+      #      #### Ansible Playbook Recap 🔍
 
-            \`\`\`\n
-            ${ansiblePlaybookRecap}
-            \`\`\`
+      #      \`\`\`\n
+      #      ${ansiblePlaybookRecap}
+      #      \`\`\`
 
-            #### Ansible playbook output 📖\`${{ steps.playbook.outcome }}\`
+      #      #### Ansible playbook output 📖\`${{ steps.playbook.outcome }}\`
 
-            <details><summary>Show Execution</summary>
+      #      <details><summary>Show Execution</summary>
 
-            \`\`\`\n
-            ${ansiblePlaybookOutput}
-            \`\`\`
+      #      \`\`\`\n
+      #      ${ansiblePlaybookOutput}
+      #      \`\`\`
 
-            </details>
+      #      </details>
 
-            |                   |                                    |
-            |-------------------|------------------------------------|
-            | Pusher            | @${{ github.actor }}               |
-            | Action            |  ${{ github.event_name }}          |
-            | Working Directory |  ${{ env.tf_actions_working_dir }} |
-            | Workflow          |  ${{ github.workflow }}            |
-            | Last updated      |  ${(new Date()).toUTCString()}     |
-            `;
+      #      |                   |                                    |
+      #      |-------------------|------------------------------------|
+      #      | Pusher            | @${{ github.actor }}               |
+      #      | Action            |  ${{ github.event_name }}          |
+      #      | Working Directory |  ${{ env.tf_actions_working_dir }} |
+      #      | Workflow          |  ${{ github.workflow }}            |
+      #      | Last updated      |  ${(new Date()).toUTCString()}     |
+      #      `;
 
-            // Call the script to write the comment
-            const script = require('./scripts/ghactions/comment-on-pr.js');
-            await script({github, context, core, commentTitle, commentBody});
+      #      // Call the script to write the comment
+      #      const script = require('./scripts/ghactions/comment-on-pr.js');
+      #      await script({github, context, core, commentTitle, commentBody});

ansible/README.md

@@ -191,6 +191,41 @@ When deploying files or updating files already existing on the hosts it can be u
 This helps track down how files on the host were modified and why.
 :::
 
+#### Tagging convention
+
+Tasks are tagged along two independent axes, so that `--tags`/`--skip-tags` can be used to run (or skip) a
+meaningful slice of a playbook without guessing what else might be silently pulled in or left out:
+
+* **Component tag** — matches the role name (`nginx`, `dehydrated`, `nftables`, `prometheus_node_exporter`,
+  `docker`, `ooniapi_gateway`, ...). Selects everything belonging to that role.
+* **Phase tag** — a small, fixed vocabulary describing *what kind* of operation the task performs, independent of
+  which role it lives in:
+
+  | Tag          | Meaning                                                                                                             |
+  |--------------|----------------------------------------------------------------------------------------------------------------------|
+  | `packages`   | apt/package installs — slow, safe to skip when nothing package-level changed                                      |
+  | `config`     | template/config file rendering — fast, safe to run often                                                           |
+  | `certs`      | dehydrated cert issuance/renewal — rate-limited by Let's Encrypt, must be independently skippable (e.g. before DNS is cut over to a new host) |
+  | `network`    | nftables rules and docker network setup                                                                            |
+  | `service`    | service enable/start/restart/reload operations                                                                     |
+  | `monitoring` | prometheus/node_exporter wiring and health checks                                                                  |
+  | `secrets`    | sudoers rules and deploy-credential/password setup                                                                 |
+
+  Most tasks should carry exactly one component tag and one phase tag.
+
+  Examples:
+  ```
+  ./play -i inventory deploy-monitoring-proxy.yml -l monitoringproxy.prod.ooni.io --tags config
+  ./play -i inventory deploy-ooni-backend.yml -l backend-hel.ooni.org --skip-tags certs
+  ./play -i inventory deploy-clickhouse-proxy.yml -l clickhouseproxy.prod.ooni.io --tags monitoring
+  ```
+
+:::note
+This taxonomy is applied incrementally as roles are touched, not retrofitted across the whole codebase in one go
+— currently `nginx`, `dehydrated`, and `prometheus_node_exporter` follow it. Don't assume every task elsewhere
+already does.
+:::
+
 ### Platform specific known bugs
 
 On macOS you might run into this issue: https://github.com/ansible/ansible/issues/76322

ansible/deploy-airflow.yml

@@ -11,6 +11,9 @@
   vars:
     airflow_public_fqdn: "airflow.prod.ooni.io"
     tls_cert_dir: /var/lib/dehydrated/certs
+    event_detector_panel_domain: detector-panel.prod.ooni.io
     ssl_domains:
       - "data1.htz-fsn.prod.ooni.nu"
       - "airflow.prod.ooni.io"
+      - "{{event_detector_panel_domain}}"
+    detector_panel_passwd: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/detector_panel_passwd', profile='oonidevops_user_prod') }}"

ansible/deploy-clickhouse-proxy.yml

@@ -6,9 +6,10 @@
   become: true
   roles:
     - role: bootstrap
+    - role: nginx
     - role: dehydrated
-      vars: 
-        ssl_domains: 
+      vars:
+        ssl_domains:
           - "{{ inventory_hostname }}"
         tls_cert_dir: /var/lib/dehydrated/certs
     - role: clickhouse_proxy

ansible/deploy-clickhouse.yml

@@ -1,13 +1,13 @@
 ---
 - name: Deploy oonidata clickhouse hosts
   hosts:
-    - notebook1.htz-fsn.prod.ooni.nu
+    - notebook.ooni.org
     - data1.htz-fsn.prod.ooni.nu
     - data2.htz-fsn.prod.ooni.nu
     - data3.htz-fsn.prod.ooni.nu
   become: true
-  tags:
-    - clickhouse
   roles:
-    - prometheus_node_exporter
-    - oonidata_clickhouse
+    - role: prometheus_node_exporter
+      tags: [node_exporter]
+    - role: oonidata_clickhouse
+      tags: [clickhouse]

ansible/deploy-echo-test-helper.yml

@@ -0,0 +1,23 @@
+---
+- name: Deploy test helpers
+  hosts:
+    - echo.th.dev.ooni.io
+    - echo.th.prod.ooni.io
+  become: true
+  roles:
+    - role: bootstrap
+    - role: prometheus_node_exporter
+      vars:
+        node_exporter_port: 9100
+        node_exporter_host: "0.0.0.0"
+        prometheus_nginx_proxy_config:
+          - location: /metrics/node_exporter
+            proxy_pass: http://127.0.0.1:9100/metrics
+        use_https: false
+        http_port: 8080 # if we leave port 80, it's taken by nginx
+    - role: test_helpers
+      vars:
+        helper: ooechohelpd
+        # sha256 of https://github.com/ooni/ooniprobe-rs/releases/download/v0.1.6/ooechohelpd
+        test_helpers_checksum: 10511a8b918c2244eba9e17c32f3cec9efbc5ffa51d6ad7f59fe648cd0986615
+        port: 80

ansible/deploy-fastpath.yml

@@ -6,6 +6,28 @@
     - fastpath2.prod.ooni.io
     - reuploaderfastpath.prod.ooni.io
   become: true
+  vars:
+    nginx_install_method: official_repo
+  pre_tasks:
+    - name: Find any nginx.org apt source files left over from a previous nginxinc.nginx install
+      ansible.builtin.find:
+        paths: /etc/apt/sources.list.d
+        patterns: "*.list"
+        contains: "nginx\\.org/packages"
+        read_whole_file: true
+      register: _stale_nginx_apt_sources
+      tags:
+        - nginx
+        - packages
+
+    - name: Remove stale nginx.org apt source files (avoids "Conflicting values set for option Signed-By")
+      ansible.builtin.file:
+        path: "{{ item.path }}"
+        state: absent
+      loop: "{{ _stale_nginx_apt_sources.files }}"
+      tags:
+        - nginx
+        - packages
   roles:
     - role: bootstrap
     - role: nginx

ansible/deploy-json-test-helper.yml

@@ -0,0 +1,23 @@
+---
+- name: Deploy test helpers
+  hosts:
+    - json.th.dev.ooni.io
+    - json.th.prod.ooni.io
+  become: true
+  roles:
+    - role: bootstrap
+    - role: prometheus_node_exporter
+      vars:
+        node_exporter_port: 9100
+        node_exporter_host: "0.0.0.0"
+        prometheus_nginx_proxy_config:
+          - location: /metrics/node_exporter
+            proxy_pass: http://127.0.0.1:9100/metrics
+        use_https: false
+        http_port: 8080 # if we leave port 80, it's taken by nginx
+    - role: test_helpers
+      vars:
+        helper: oojsonhelpd
+        # sha256 of https://github.com/ooni/ooniprobe-rs/releases/download/v0.1.6/oojsonhelpd
+        test_helpers_checksum: 3d977e1418ea398f520dff3de951e36010a294dc3d1b564c961542cc3729a224
+        port: 80

ansible/deploy-monitoring-proxy.yml

@@ -6,13 +6,13 @@
   become: true
   roles:
     - role: bootstrap
+    - role: nginx
+      tags: nginx
     - role: dehydrated
       vars:
         ssl_domains:
           - "{{ inventory_hostname }}"
         tls_cert_dir: /var/lib/dehydrated/certs
-    - role: nginx
-      tags: nginx
     - role: monitoring_proxy
       vars:
         monitoring_proxy_public_fqdn: "{{ inventory_hostname }}"

ansible/deploy-notebook.yml

@@ -1,13 +1,14 @@
 ---
 - name: Deploy notebook host
-  hosts: notebook1.htz-fsn.prod.ooni.nu
+  hosts: notebook.ooni.org
   become: true
   tags:
     - notebook
   vars:
     notebook_domain: "notebook.ooni.org"
     ssl_domains:
       - "{{ inventory_hostname }}"
-      - "notebook.ooni.org"
   roles:
+    - nginx
+    - dehydrated
     - notebook

ansible/deploy-ooni-backend.yml

@@ -8,6 +8,7 @@
         admin_group_name: adm
     - role: base-backend
     - role: nftables
+    - role: nginx
     - role: dehydrated
       tags: dehydrated
       expand: yes

ansible/deploy-wc-test-helper.yml

@@ -0,0 +1,19 @@
+---
+- name: Deploy the Web Connectivity test helper
+  hosts:
+    - wc.th.dev.ooni.io
+    - wcth0.fra1.prod.ooni.io
+    - wcth1.fra1.prod.ooni.io
+    - wcth2.fra1.prod.ooni.io
+  become: true
+  vars:
+    nginx_install_method: official_repo
+  roles:
+    - role: bootstrap
+    - role: nginx
+    - role: dehydrated
+    - role: prometheus_node_exporter
+      vars:
+        use_https: true
+        https_port: 9001
+    - role: oohelperd