diff --git a/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_5m_default.yaml b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_5m_default.yaml new file mode 100644 index 00000000..bd721fc5 --- /dev/null +++ b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_5m_default.yaml @@ -0,0 +1,50 @@ +# @package _global_ +# Generated by recommended_sketch_configs.py; do not edit. +experiment_params: + experiment: + - mode: sketchdb + server: sketchdb + query_prometheus_too: true + monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: + - name: prometheus + url: http://localhost:9090 + - name: sketchdb + url: http://localhost:8088 + exporters: + only_start_if_queries_exist: true + exporter_list: + cluster_data_exporter: + provider: alibaba + port: 40000 + data_type: msresource + data_year: 2022 + parts_mode: part-index + part_index: 0 + scrape_timeout: 10s + query_groups: + - id: 1 + queries: + - sum by (ms_name) (sum_over_time(alibaba_microservice_cpu_usage[5m])) + repetition_delay_ms: 10000 + client_options: + repetitions: 20 + query_time_offset: 10 + starting_delay: 390 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + metrics: + - metric: alibaba_microservice_cpu_usage + labels: + - instance + - job + - ms_name + - ms_instance_id + - node_id + exporter: cluster_data_exporter +cluster_data_directory: /data/cluster_traces/alibaba_msmetrics +prometheus: + scrape_interval: 10s diff --git a/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_5m_recommended.yaml b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_5m_recommended.yaml new file mode 100644 index 00000000..593b834c --- /dev/null +++ b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_5m_recommended.yaml @@ -0,0 +1,54 @@ +# @package _global_ +# Generated by recommended_sketch_configs.py; do not edit. +experiment_params: + experiment: + - mode: sketchdb + server: sketchdb + query_prometheus_too: true + monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: + - name: prometheus + url: http://localhost:9090 + - name: sketchdb + url: http://localhost:8088 + exporters: + only_start_if_queries_exist: true + exporter_list: + cluster_data_exporter: + provider: alibaba + port: 40000 + data_type: msresource + data_year: 2022 + parts_mode: part-index + part_index: 0 + scrape_timeout: 10s + query_groups: + - id: 1 + queries: + - sum by (ms_name) (sum_over_time(alibaba_microservice_cpu_usage[5m])) + repetition_delay_ms: 10000 + client_options: + repetitions: 20 + query_time_offset: 10 + starting_delay: 390 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + metrics: + - metric: alibaba_microservice_cpu_usage + labels: + - instance + - job + - ms_name + - ms_instance_id + - node_id + exporter: cluster_data_exporter +cluster_data_directory: /data/cluster_traces/alibaba_msmetrics +prometheus: + scrape_interval: 10s +sketch_parameters: + CountMinSketch: + depth: 3 + width: 16384 diff --git a/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_instant_default.yaml b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_instant_default.yaml new file mode 100644 index 00000000..6f54463e --- /dev/null +++ b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_instant_default.yaml @@ -0,0 +1,50 @@ +# @package _global_ +# Generated by recommended_sketch_configs.py; do not edit. +experiment_params: + experiment: + - mode: sketchdb + server: sketchdb + query_prometheus_too: true + monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: + - name: prometheus + url: http://localhost:9090 + - name: sketchdb + url: http://localhost:8088 + exporters: + only_start_if_queries_exist: true + exporter_list: + cluster_data_exporter: + provider: alibaba + port: 40000 + data_type: msresource + data_year: 2022 + parts_mode: part-index + part_index: 0 + scrape_timeout: 10s + query_groups: + - id: 1 + queries: + - sum by (ms_name) (alibaba_microservice_cpu_usage) + repetition_delay_ms: 10000 + client_options: + repetitions: 20 + query_time_offset: 10 + starting_delay: 90 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + metrics: + - metric: alibaba_microservice_cpu_usage + labels: + - instance + - job + - ms_name + - ms_instance_id + - node_id + exporter: cluster_data_exporter +cluster_data_directory: /data/cluster_traces/alibaba_msmetrics +prometheus: + scrape_interval: 10s diff --git a/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_instant_recommended.yaml b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_instant_recommended.yaml new file mode 100644 index 00000000..dac39e20 --- /dev/null +++ b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/alibaba_v2022_ms_cpu_by_msname_instant_recommended.yaml @@ -0,0 +1,54 @@ +# @package _global_ +# Generated by recommended_sketch_configs.py; do not edit. +experiment_params: + experiment: + - mode: sketchdb + server: sketchdb + query_prometheus_too: true + monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: + - name: prometheus + url: http://localhost:9090 + - name: sketchdb + url: http://localhost:8088 + exporters: + only_start_if_queries_exist: true + exporter_list: + cluster_data_exporter: + provider: alibaba + port: 40000 + data_type: msresource + data_year: 2022 + parts_mode: part-index + part_index: 0 + scrape_timeout: 10s + query_groups: + - id: 1 + queries: + - sum by (ms_name) (alibaba_microservice_cpu_usage) + repetition_delay_ms: 10000 + client_options: + repetitions: 20 + query_time_offset: 10 + starting_delay: 90 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + metrics: + - metric: alibaba_microservice_cpu_usage + labels: + - instance + - job + - ms_name + - ms_instance_id + - node_id + exporter: cluster_data_exporter +cluster_data_directory: /data/cluster_traces/alibaba_msmetrics +prometheus: + scrape_interval: 10s +sketch_parameters: + CountMinSketch: + depth: 3 + width: 16384 diff --git a/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_5m_default.yaml b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_5m_default.yaml new file mode 100644 index 00000000..9a163036 --- /dev/null +++ b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_5m_default.yaml @@ -0,0 +1,47 @@ +# @package _global_ +# Generated by recommended_sketch_configs.py; do not edit. +experiment_params: + experiment: + - mode: sketchdb + server: sketchdb + query_prometheus_too: true + monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: + - name: prometheus + url: http://localhost:9090 + - name: sketchdb + url: http://localhost:8088 + exporters: + only_start_if_queries_exist: true + exporter_list: + cluster_data_exporter: + provider: google + port: 40000 + metrics: mean-cpu-usage-rate + parts_mode: part-index + part_index: 0 + scrape_timeout: 1s + query_groups: + - id: 1 + queries: + - sum by (job_id) (sum_over_time(google_mean_cpu_usage_rate_0[5m])) + repetition_delay_ms: 5000 + client_options: + repetitions: 20 + query_time_offset: 10 + starting_delay: 390 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + metrics: + - metric: google_mean_cpu_usage_rate_0 + labels: + - instance + - job + - job_id + - task_index + - machine_id + exporter: cluster_data_exporter +cluster_data_directory: /data/cluster_traces/google diff --git a/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_5m_recommended.yaml b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_5m_recommended.yaml new file mode 100644 index 00000000..0f815477 --- /dev/null +++ b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_5m_recommended.yaml @@ -0,0 +1,51 @@ +# @package _global_ +# Generated by recommended_sketch_configs.py; do not edit. +experiment_params: + experiment: + - mode: sketchdb + server: sketchdb + query_prometheus_too: true + monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: + - name: prometheus + url: http://localhost:9090 + - name: sketchdb + url: http://localhost:8088 + exporters: + only_start_if_queries_exist: true + exporter_list: + cluster_data_exporter: + provider: google + port: 40000 + metrics: mean-cpu-usage-rate + parts_mode: part-index + part_index: 0 + scrape_timeout: 1s + query_groups: + - id: 1 + queries: + - sum by (job_id) (sum_over_time(google_mean_cpu_usage_rate_0[5m])) + repetition_delay_ms: 5000 + client_options: + repetitions: 20 + query_time_offset: 10 + starting_delay: 390 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + metrics: + - metric: google_mean_cpu_usage_rate_0 + labels: + - instance + - job + - job_id + - task_index + - machine_id + exporter: cluster_data_exporter +cluster_data_directory: /data/cluster_traces/google +sketch_parameters: + CountMinSketch: + depth: 3 + width: 4096 diff --git a/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_instant_default.yaml b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_instant_default.yaml new file mode 100644 index 00000000..d68ab47c --- /dev/null +++ b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_instant_default.yaml @@ -0,0 +1,47 @@ +# @package _global_ +# Generated by recommended_sketch_configs.py; do not edit. +experiment_params: + experiment: + - mode: sketchdb + server: sketchdb + query_prometheus_too: true + monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: + - name: prometheus + url: http://localhost:9090 + - name: sketchdb + url: http://localhost:8088 + exporters: + only_start_if_queries_exist: true + exporter_list: + cluster_data_exporter: + provider: google + port: 40000 + metrics: mean-cpu-usage-rate + parts_mode: part-index + part_index: 0 + scrape_timeout: 1s + query_groups: + - id: 1 + queries: + - sum by (job_id) (google_mean_cpu_usage_rate_0) + repetition_delay_ms: 5000 + client_options: + repetitions: 20 + query_time_offset: 10 + starting_delay: 90 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + metrics: + - metric: google_mean_cpu_usage_rate_0 + labels: + - instance + - job + - job_id + - task_index + - machine_id + exporter: cluster_data_exporter +cluster_data_directory: /data/cluster_traces/google diff --git a/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_instant_recommended.yaml b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_instant_recommended.yaml new file mode 100644 index 00000000..c6fd091c --- /dev/null +++ b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_by_job_id_instant_recommended.yaml @@ -0,0 +1,51 @@ +# @package _global_ +# Generated by recommended_sketch_configs.py; do not edit. +experiment_params: + experiment: + - mode: sketchdb + server: sketchdb + query_prometheus_too: true + monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: + - name: prometheus + url: http://localhost:9090 + - name: sketchdb + url: http://localhost:8088 + exporters: + only_start_if_queries_exist: true + exporter_list: + cluster_data_exporter: + provider: google + port: 40000 + metrics: mean-cpu-usage-rate + parts_mode: part-index + part_index: 0 + scrape_timeout: 1s + query_groups: + - id: 1 + queries: + - sum by (job_id) (google_mean_cpu_usage_rate_0) + repetition_delay_ms: 5000 + client_options: + repetitions: 20 + query_time_offset: 10 + starting_delay: 90 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + metrics: + - metric: google_mean_cpu_usage_rate_0 + labels: + - instance + - job + - job_id + - task_index + - machine_id + exporter: cluster_data_exporter +cluster_data_directory: /data/cluster_traces/google +sketch_parameters: + CountMinSketch: + depth: 3 + width: 4096 diff --git a/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_p99_instant_default.yaml b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_p99_instant_default.yaml new file mode 100644 index 00000000..3be876db --- /dev/null +++ b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_p99_instant_default.yaml @@ -0,0 +1,47 @@ +# @package _global_ +# Generated by recommended_sketch_configs.py; do not edit. +experiment_params: + experiment: + - mode: sketchdb + server: sketchdb + query_prometheus_too: true + monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: + - name: prometheus + url: http://localhost:9090 + - name: sketchdb + url: http://localhost:8088 + exporters: + only_start_if_queries_exist: true + exporter_list: + cluster_data_exporter: + provider: google + port: 40000 + metrics: mean-cpu-usage-rate + parts_mode: part-index + part_index: 0 + scrape_timeout: 1s + query_groups: + - id: 1 + queries: + - quantile(0.99, google_mean_cpu_usage_rate_0) + repetition_delay_ms: 5000 + client_options: + repetitions: 20 + query_time_offset: 10 + starting_delay: 90 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + metrics: + - metric: google_mean_cpu_usage_rate_0 + labels: + - instance + - job + - job_id + - task_index + - machine_id + exporter: cluster_data_exporter +cluster_data_directory: /data/cluster_traces/google diff --git a/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_p99_instant_recommended.yaml b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_p99_instant_recommended.yaml new file mode 100644 index 00000000..e26d8cd7 --- /dev/null +++ b/asap-tools/experiments/config/experiment_type/recommended_sketch_configs/google_2011_cpu_p99_instant_recommended.yaml @@ -0,0 +1,50 @@ +# @package _global_ +# Generated by recommended_sketch_configs.py; do not edit. +experiment_params: + experiment: + - mode: sketchdb + server: sketchdb + query_prometheus_too: true + monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: + - name: prometheus + url: http://localhost:9090 + - name: sketchdb + url: http://localhost:8088 + exporters: + only_start_if_queries_exist: true + exporter_list: + cluster_data_exporter: + provider: google + port: 40000 + metrics: mean-cpu-usage-rate + parts_mode: part-index + part_index: 0 + scrape_timeout: 1s + query_groups: + - id: 1 + queries: + - quantile(0.99, google_mean_cpu_usage_rate_0) + repetition_delay_ms: 5000 + client_options: + repetitions: 20 + query_time_offset: 10 + starting_delay: 90 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + metrics: + - metric: google_mean_cpu_usage_rate_0 + labels: + - instance + - job + - job_id + - task_index + - machine_id + exporter: cluster_data_exporter +cluster_data_directory: /data/cluster_traces/google +sketch_parameters: + DatasketchesKLL: + K: 200 diff --git a/asap-tools/experiments/recommended_sketch_configs/README.md b/asap-tools/experiments/recommended_sketch_configs/README.md new file mode 100644 index 00000000..70e97871 --- /dev/null +++ b/asap-tools/experiments/recommended_sketch_configs/README.md @@ -0,0 +1,128 @@ +# Recommended sketch configs, end to end + +Runs the sketch configs that sketch-bench recommends for the dataset-analysis +queries (`asap-tools/dataset-analysis`) through ASAPQuery on replayed cluster +traces, and compares the measured error with sketch-bench's prediction and +with ASAPQuery's default sketch parameters. The planner is unchanged: each +config is passed as the global `sketch_parameters` override. + +## Generate the configs + +```bash +cd asap-tools/experiments +python recommended_sketch_configs/recommended_sketch_configs.py generate \ + --recommendations /docs/figures/saturation/grid_cost/recommendations.csv +``` + +This writes two configs per query under +`config/experiment_type/recommended_sketch_configs/`: `_recommended` +(the recommended config) and `_default` (config.yaml's +`sketch_parameters`: CMS 3x1024, KLL K=20). Families with no planner sketch +(CountSketch, DDSketch, CMS-heap top-k for non-topk queries) and tables with +no exporter (Alibaba MCRRTUpdate, CallGraph) are skipped and listed. + +| dataset-analysis query | PromQL run | planner sketch | +|---|---|---| +| google_2011 `cpu_by_job_id` instant | `sum by (job_id) (google_mean_cpu_usage_rate_0)` | CMS | +| google_2011 `cpu_by_job_id` 5m | `sum by (job_id) (sum_over_time(google_mean_cpu_usage_rate_0[5m]))` | CMS | +| google_2011 `cpu_p99` instant | `quantile(0.99, google_mean_cpu_usage_rate_0)` | KLL | +| alibaba_v2022 `ms_cpu_by_msname` instant | `sum by (ms_name) (alibaba_microservice_cpu_usage)` | CMS | +| alibaba_v2022 `ms_cpu_by_msname` 5m | `sum by (ms_name) (sum_over_time(alibaba_microservice_cpu_usage[5m]))` | CMS | + +`cpu_p99` has no range run: its range form `quantile_over_time` is a +per-series quantile in PromQL, not the single value stream sketch-bench +measured. + +## Run + +Each config runs one `sketchdb` mode with `query_prometheus_too`, so every +ASAP answer has a Prometheus answer for the same timestamp. On a single +machine, with the local provider: + +```bash +python experiment_run_e2e.py \ + experiment_type=recommended_sketch_configs/ experiment.name= \ + '~providers.cloudlab' '+providers.local.home_dir=' controller.punting=false +``` + +Trace data goes under `/data/cluster_traces/{google,alibaba_msmetrics}`: +Google `part-00000-of-00500.csv.gz` (task_usage) and Alibaba +`MSMetrics_0.csv.gz` (`MSMetricsUpdate_0.tar.gz` extracted and sorted by +timestamp, as `cluster_data_exporter/bin/alibaba/sort_and_format.sh` does). +The Alibaba configs scrape every 10 s: one scrape holds about 470k series +(117 MB) and takes about 5 s. + +Then: + +```bash +python recommended_sketch_configs/recommended_sketch_configs.py summarize \ + --recommendations --experiments-dir /experiment_outputs +``` + +## Results + +One run per config on an idle 56-core node (CloudLab clnode109), 20 query +repetitions each, medians over repetitions. Measured error is the ARE over +the 100 largest keys (sketch-bench's CMS metric) for key queries and the rank +error for the p99 query (from the replayed values, see below). Targets are +dataset-analysis's: ARE <= 0.05, rank error <= 0.01. + +| query | config | predicted error | measured error | target met | all-keys ARE | QE peak RSS (MB) | ASAP / Prometheus p50 latency (ms) | +|---|---|---|---|---|---|---|---| +| google cpu_by_job_id instant | rec. CMS 3x4096 | 0.042 | 0.00005 | yes | 0.68 | 99 | 124 / 1639 | +| | default CMS 3x1024 | | 0.0031 | yes | 57 | 79 | 116 / 1847 | +| google cpu_by_job_id 5m | rec. CMS 3x4096 | 0.043 | 0.00005 | yes | 0.70 | 90 | 100 / 3804 | +| | default CMS 3x1024 | | 0.0031 | yes | 58 | 74 | 92 / 4262 | +| google cpu_p99 instant | rec. KLL k=200 | 0.0020 | 0.0019 (value err. 10.6%) | yes | | 44 | 3.5 / 1860 | +| | default KLL K=20 | | 0.0153 (value err. 46%) | no | | 43 | 3.7 / 1857 | +| | planner default K=500 | | 0.0004 (value err. 2.1%) | yes | | 46 | 3.3 / 2070 | +| alibaba ms_cpu_by_msname instant | rec. CMS 3x16384 | 0.021 | 0.0013 | yes | 2.2 | 317 | 629 / 3235 | +| | default CMS 3x1024 | | 0.20 | no | 258 | 275 | 620 / 3130 | +| alibaba ms_cpu_by_msname 5m | rec. CMS 3x16384 | 0.021 | 0.0013 | yes | 2.2 | 282 | 678 / 4350 | +| | default CMS 3x1024 | | 0.20 | no | 251 | 258 | 640 / 4494 | + +- Every recommended config met its target, and the predictions were upper + bounds: CMS 16x to 900x below the estimate (the estimate is the worst case + over the whole trace, rounded toward the harder side; a run replays only + the first window of one file), KLL k=200 within 5% of it (0.0019 vs + 0.0020). +- The defaults miss the target where the recommendation is larger than them: + Alibaba `sum by (ms_name)` (28k keys) at 3x1024 has 20% error on its + largest keys, and KLL K=20 (config.yaml's e2e default) has 1.5% rank error + for p99. On Google `sum by (job_id)` (3k keys) 3x1024 already meets the + target, so the recommended 3x4096 is larger than this replay needed. +- The sketch size does not change ASAP query latency here; the larger + CMS configs add 16 to 42 MB of query-engine RSS across all retained windows. +- The all-keys ARE shows CMS overestimating small keys: it stays far above + the target for every config, which is why the target is on the 100 largest + keys. + +The p99 rank error is computed offline: the exporter had exported the +93,719 part-0 rows starting by 615 s (aggregation_type 0, the 5-minute window +starting at 600 s), and Prometheus's p99 matched their exact p99 (0.1194); +rank error = |F(estimate) - 0.99| over those values. + +`post_experiment/single_experiment/calculate_fidelity.py` was also run on +every experiment. For the key queries its per-key time-series metrics are +NaN or inf: some keys sum to 0 (division by zero) and gauges replayed within +one trace window are constant over the run (correlation undefined). For p99 +its MAPE is 13.6% (k=200), 43% (K=20) and 3.7% (K=500). + +## Gaps + +- One run per config, one trace file per dataset, and only its first + minutes (Google replays at 1/10 speed, so a run sees the first 5-minute + window of trace time; Alibaba sees the first 5 to 10 minutes). +- Google metrics are the exporter's `aggregation_type` 0 series, which hold + most rows; dataset-analysis's `cpu_rate` includes both types. +- CountSketch, DDSketch and the top-k family have no planner sketch, so their + recommendations are not run. No query in the sets uses HLL. +- Memory is the query engine's whole RSS, not per-sketch bytes; sketch-bench + reports 49 KB (3x4096) and 197 KB (3x16384) per sketch. + +## Reproducing + +- Running these configs end to end with the local provider needs the runner fixes in ProjectASAP/ASAPQuery#751. +- `results_summary.csv` is the summary table of the runs above. +- p99 rank error is computed offline against the replayed trace values: + `python kll_rank_error.py --trace --outputs ` diff --git a/asap-tools/experiments/recommended_sketch_configs/kll_rank_error.py b/asap-tools/experiments/recommended_sketch_configs/kll_rank_error.py new file mode 100644 index 00000000..ba0d1dc0 --- /dev/null +++ b/asap-tools/experiments/recommended_sketch_configs/kll_rank_error.py @@ -0,0 +1,68 @@ +"""Rank error of ASAP's p99 estimates against the replayed Google CPU values. + +During a run the exporter has exported the part-0 task_usage rows with +start_time <= --cutoff-us (default: the 5-minute window starting at 600 s) +and aggregation_type 0; F is their empirical CDF and the rank error of an +estimate is |F(estimate) - 0.99|. +""" + +import argparse +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..")) +sys.path.insert( + 0, os.path.join(HERE, "../../../asap-common/dependencies/py/promql_utilities") +) +from post_experiment.lib.results_loader import load_results # noqa: E402 +from recommended_sketch_configs import ( # noqa: E402 + P99, + P99_REPLAY_CUTOFF_US, + rank_errors, + replayed_google_cpu_values, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--trace", required=True, help="task_usage part-00000-of-00500.csv.gz" + ) + parser.add_argument( + "--outputs", required=True, help="experiment_outputs directory of the runs" + ) + parser.add_argument("--cutoff-us", type=int, default=P99_REPLAY_CUTOFF_US) + parser.add_argument( + "--runs", + default="recommended,default,k500", + help="suffixes of google_2011_cpu_p99_instant_", + ) + args = parser.parse_args() + + v = replayed_google_cpu_values(args.trace, args.cutoff_us) + print(f"{len(v)} values, exact p99 {np.quantile(v, P99):.6f}") + for name in args.runs.split(","): + out = os.path.join( + args.outputs, + f"google_2011_cpu_p99_instant_{name}", + "sketchdb", + "prometheus_client_output", + ) + results = load_results(out) + est = [ + list(q.result.values())[0] + for q in results["sketchdb"][0].query_results + if q.result + ] + err = rank_errors(v, est, P99) + print( + f"{name}: n={len(est)} rank_err median={np.median(err):.4f} " + f"mean={np.mean(err):.4f} max={np.max(err):.4f}" + ) + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/recommended_sketch_configs/recommended_sketch_configs.py b/asap-tools/experiments/recommended_sketch_configs/recommended_sketch_configs.py new file mode 100644 index 00000000..d0ec7a87 --- /dev/null +++ b/asap-tools/experiments/recommended_sketch_configs/recommended_sketch_configs.py @@ -0,0 +1,503 @@ +"""Run sketch-bench's recommended sketch configs end to end in ASAPQuery. + +`generate` reads sketch-bench's recommendations.csv (smallest config per +dataset, query, range and sketch family that meets the query's accuracy +target, computed from the dataset-analysis skew summary) and writes two +experiment_type configs per query: one with the recommended config as the +planner's global `sketch_parameters` override and a `default` twin that keeps +config.yaml's sketch parameters. + +`summarize` reads finished experiments and prints, per query and config, the +measured error against Prometheus (ARE over the 100 largest keys for key +queries, the metric sketch-bench's CMS estimate uses, with ARE over all keys as +a second column; rank error against the replayed trace values for the p99 +query) next to the predicted error, plus query latencies. + +Usage (from asap-tools/experiments): + python recommended_sketch_configs/recommended_sketch_configs.py generate \ + --recommendations .../recommendations.csv + python recommended_sketch_configs/recommended_sketch_configs.py summarize \ + --recommendations .../recommendations.csv --experiments-dir +""" + +import argparse +import csv +import gzip +import json +import os +import re +import sys +from typing import Dict, List, Optional, Tuple + +import numpy as np +import yaml + +EXPERIMENTS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DATASET_ANALYSIS_DIR = os.path.join( + os.path.dirname(EXPERIMENTS_DIR), "dataset-analysis" +) +DEFAULT_OUTPUT_DIR = os.path.join( + EXPERIMENTS_DIR, "config", "experiment_type", "recommended_sketch_configs" +) + +# Queries run by default: (dataset, query_id, range). +DEFAULT_QUERIES = [ + ("google_2011", "cpu_by_job_id", "instant"), + ("google_2011", "cpu_by_job_id", "5m"), + ("google_2011", "cpu_p99", "instant"), + ("alibaba_v2022", "ms_cpu_by_msname", "instant"), + ("alibaba_v2022", "ms_cpu_by_msname", "5m"), +] + +# dataset-analysis (dataset, table) -> cluster_data_exporter config, metric +# names per value column and label names per label column. Tables without an +# exporter (Alibaba MSRTMCR and CallGraph) are absent. +EXPORTERS: Dict[Tuple[str, str], dict] = { + ("google_2011", "task_usage"): { + "exporter": { + "provider": "google", + "port": 40000, + "metrics": "mean-cpu-usage-rate", + "parts_mode": "part-index", + "part_index": 0, + "scrape_timeout": "1s", + }, + "data_subdir": "google", + # The exporter splits each column by the trace's aggregation_type; 0 + # holds almost every row. + "metrics": { + "cpu_rate": "google_mean_cpu_usage_rate_0", + }, + "labels": { + "job_id": "job_id", + "task_index": "task_index", + "machine_id": "machine_id", + }, + }, + ("alibaba_v2022", "MSMetrics"): { + "exporter": { + "provider": "alibaba", + "port": 40000, + "data_type": "msresource", + "data_year": 2022, + "parts_mode": "part-index", + "part_index": 0, + "scrape_timeout": "10s", + }, + # One scrape holds about 470k series (117 MB) and takes about 5 s. + "scrape_interval": "10s", + "data_subdir": "alibaba_msmetrics", + "metrics": { + "cpu_utilization": "alibaba_microservice_cpu_usage", + "memory_utilization": "alibaba_microservice_memory_usage", + }, + "labels": { + "msname": "ms_name", + "msinstanceid": "ms_instance_id", + "nodeid": "node_id", + }, + }, + ("alibaba_v2022", "NodeMetrics"): { + "exporter": { + "provider": "alibaba", + "port": 40000, + "data_type": "node", + "data_year": 2022, + "parts_mode": "part-index", + "part_index": 0, + }, + "data_subdir": "alibaba_nodemetrics", + "metrics": { + "cpu_utilization": "alibaba_node_cpu_usage", + "memory_utilization": "alibaba_node_memory_usage", + }, + "labels": {"nodeid": "node_id"}, + }, +} + +# Planner sketch_parameters key per sketch-bench family. CountSketch and +# DDSketch have no planner equivalent; the CMS-heap top-k family only applies +# to topk queries, which the dataset-analysis query sets do not contain. +PLANNER_FAMILIES = {"cms": "CountMinSketch", "kll": "DatasketchesKLL"} + +# Query client timing. A range query needs one full range of data before its +# first answer, so its starting delay grows with the range. +REPETITIONS = 20 +REPETITION_DELAY_MS = 5000 +MIN_STARTING_DELAY_S = 90 + +# remote_monitor.py keyword of the containerized query engine. +QUERY_ENGINE_MONITOR_KEYWORD = "sketchdb-queryengine-rust" + +# The only quantile query is cpu_p99. A run replays Google at 1/10 speed, so by +# its queries the exporter has exported the rows starting by 615 s (the +# 5-minute window starting at 600 s). +P99 = 0.99 +P99_REPLAY_CUTOFF_US = 615_000_000 + + +def parse_duration_s(text: str) -> int: + """'5m' -> 300.""" + match = re.fullmatch(r"(\d+)([smh])", text) + if match is None: + raise ValueError(f"Unsupported duration: {text}") + return int(match.group(1)) * {"s": 1, "m": 60, "h": 3600}[match.group(2)] + + +def planner_sketch_parameters(family: str, config: str) -> dict: + """recommendations.csv family and config -> planner sketch_parameters. + + 'rows=3 cols=4096' -> {'CountMinSketch': {'depth': 3, 'width': 4096}}; + 'k=200' -> {'DatasketchesKLL': {'K': 200}}. + """ + fields = dict(item.split("=") for item in config.split()) + if family == "cms": + return { + "CountMinSketch": { + "depth": int(fields["rows"]), + "width": int(fields["cols"]), + } + } + if family == "kll": + return {"DatasketchesKLL": {"K": int(fields["k"])}} + raise ValueError(f"No planner sketch for family {family}") + + +def translate_promql(promql: str, exporter: dict) -> str: + """Rename dataset-analysis metric and label names to the exporter's.""" + names = {**exporter["metrics"], **exporter["labels"]} + pattern = r"\b(" + "|".join(re.escape(name) for name in names) + r")\b" + return re.sub(pattern, lambda m: names[m.group(1)], promql) + + +def load_queries(dataset: str) -> Dict[str, dict]: + path = os.path.join(DATASET_ANALYSIS_DIR, "queries", f"{dataset}.yaml") + with open(path) as f: + spec = yaml.safe_load(f) + # Some query ids appear twice (key and value forms); keep the first. + queries: Dict[str, dict] = {} + for query in spec["queries"]: + queries.setdefault(query["id"], query) + return queries + + +def load_recommendations(path: str) -> List[dict]: + with open(path) as f: + return list(csv.DictReader(f)) + + +def experiment_name(dataset: str, query_id: str, range_: str, variant: str) -> str: + return f"{dataset}_{query_id}_{range_}_{variant}" + + +def build_experiment_config( + query_spec: dict, + range_: str, + exporter: dict, + cluster_data_root: str, + sketch_parameters: Optional[dict], +) -> dict: + """One experiment_type config (package _global_) for one query.""" + if range_ == "instant": + promql = query_spec["promql"] + starting_delay = MIN_STARTING_DELAY_S + else: + promql = query_spec["promql_range"].format(range=range_) + starting_delay = MIN_STARTING_DELAY_S + parse_duration_s(range_) + # The planner needs queries no more often than the scrape interval. + repetition_delay_ms = REPETITION_DELAY_MS + if "scrape_interval" in exporter: + scrape_interval_ms = 1000 * parse_duration_s(exporter["scrape_interval"]) + repetition_delay_ms = max(repetition_delay_ms, scrape_interval_ms) + query = translate_promql(promql, exporter) + labels = ["instance", "job"] + list(exporter["labels"].values()) + metric = exporter["metrics"][query_spec["value"]] + + experiment_params = { + # One mode that queries both servers, so each ASAP answer has a + # Prometheus answer for the same timestamp. + "experiment": [ + {"mode": "sketchdb", "server": "sketchdb", "query_prometheus_too": True} + ], + "monitoring": {"tool": "prometheus", "deployment_mode": "bare_metal"}, + "servers": [ + {"name": "prometheus", "url": "http://localhost:9090"}, + {"name": "sketchdb", "url": "http://localhost:8088"}, + ], + "exporters": { + "only_start_if_queries_exist": True, + "exporter_list": {"cluster_data_exporter": dict(exporter["exporter"])}, + }, + "query_groups": [ + { + "id": 1, + "queries": [query], + "repetition_delay_ms": repetition_delay_ms, + "client_options": { + "repetitions": REPETITIONS, + "query_time_offset": 10, + "starting_delay": starting_delay, + }, + "controller_options": {"accuracy_sla": 0.99, "latency_sla": 1}, + } + ], + "metrics": [ + { + "metric": metric, + "labels": labels, + "exporter": "cluster_data_exporter", + } + ], + } + config: dict = { + "experiment_params": experiment_params, + "cluster_data_directory": os.path.join( + cluster_data_root, exporter["data_subdir"] + ), + } + if "scrape_interval" in exporter: + config["prometheus"] = {"scrape_interval": exporter["scrape_interval"]} + if sketch_parameters is not None: + config["sketch_parameters"] = sketch_parameters + return config + + +def generate( + recommendations: List[dict], + queries: List[Tuple[str, str, str]], + cluster_data_root: str, +) -> Tuple[Dict[str, dict], List[str]]: + """Return (experiment name -> config, skipped notes).""" + configs: Dict[str, dict] = {} + skipped: List[str] = [] + for dataset, query_id, range_ in queries: + query_spec = load_queries(dataset)[query_id] + exporter = EXPORTERS.get((dataset, query_spec["table"])) + if exporter is None: + skipped.append( + f"{dataset}/{query_id}: no exporter for table {query_spec['table']}" + ) + continue + rows = [ + r + for r in recommendations + if (r["dataset"], r["query_id"], r["range"]) == (dataset, query_id, range_) + ] + if not rows: + skipped.append(f"{dataset}/{query_id}/{range_}: no recommendation") + continue + for row in rows: + if row["family"] not in PLANNER_FAMILIES: + skipped.append( + f"{dataset}/{query_id}/{range_}: {row['family']} " + f"{row['config']} (no planner sketch for this family)" + ) + continue + if row["meets_target"] != "True": + skipped.append( + f"{dataset}/{query_id}/{range_}: {row['family']} " + f"{row['config']} does not meet the target" + ) + continue + name = experiment_name(dataset, query_id, range_, "recommended") + configs[name] = build_experiment_config( + query_spec, + range_, + exporter, + cluster_data_root, + planner_sketch_parameters(row["family"], row["config"]), + ) + name = experiment_name(dataset, query_id, range_, "default") + configs[name] = build_experiment_config( + query_spec, range_, exporter, cluster_data_root, None + ) + return configs, skipped + + +def write_configs(configs: Dict[str, dict], output_dir: str) -> None: + os.makedirs(output_dir, exist_ok=True) + for name, config in configs.items(): + with open(os.path.join(output_dir, f"{name}.yaml"), "w") as f: + f.write("# @package _global_\n") + f.write("# Generated by recommended_sketch_configs.py; do not edit.\n") + yaml.safe_dump(config, f, sort_keys=False) + + +def are_top_keys(exact: Dict, estimate: Dict, num_keys: int = 100) -> float: + """Mean |estimate - exact| / exact over the num_keys largest exact keys. + + A key missing from the estimate counts as estimate 0. + """ + keys = sorted(exact, key=lambda k: exact[k], reverse=True)[:num_keys] + keys = [k for k in keys if exact[k] != 0] + errors = [abs(estimate.get(k, 0.0) - exact[k]) / abs(exact[k]) for k in keys] + return float(np.mean(errors)) if errors else float("nan") + + +def replayed_google_cpu_values(trace: str, cutoff_us: int) -> np.ndarray: + """Sorted mean CPU usage of the task_usage rows the exporter replayed. + + Rows with start_time <= cutoff_us and aggregation_type 0 (the series the + p99 query reads). + """ + values = [] + with gzip.open(trace, "rt") as f: + for line in f: + c = line.rstrip("\n").split(",") + if int(c[0]) <= cutoff_us and c[18] in ("", "0") and c[5] != "": + values.append(float(c[5])) + return np.sort(values) + + +def rank_errors(sorted_values: np.ndarray, estimates: List[float], q: float): + """|F(estimate) - q| per estimate, F the empirical CDF of sorted_values.""" + ranks = np.searchsorted(sorted_values, estimates, side="right") + return np.abs(ranks / len(sorted_values) - q) + + +def monitor_output_path(experiment_dir: str) -> str: + """Written by remote_monitor.py when the run finishes.""" + return os.path.join( + experiment_dir, "sketchdb", "remote_monitor_output", "monitor_output.json" + ) + + +def summarize_experiment( + experiment_dir: str, quantile_values: Optional[np.ndarray] = None +) -> dict: + """Measured error and latency of one finished experiment. + + With quantile_values (sorted replayed values), the measured error is the + p99 rank error instead of the ARE, and there is no all-keys column. + """ + sys.path.insert(0, EXPERIMENTS_DIR) + from post_experiment.lib.results_loader import load_results + + results = load_results( + os.path.join(experiment_dir, "sketchdb", "prometheus_client_output") + ) + exact = results["prometheus"][0].query_results + estimate = results["sketchdb"][0].query_results + errors = [] + errors_all_keys = [] + for exact_rep, estimate_rep in zip(exact, estimate): + if not (exact_rep.result and estimate_rep.result): + continue + if quantile_values is not None: + (value,) = estimate_rep.result.values() + errors.append(float(rank_errors(quantile_values, [value], P99)[0])) + continue + errors.append(are_top_keys(exact_rep.result, estimate_rep.result)) + errors_all_keys.append( + are_top_keys(exact_rep.result, estimate_rep.result, len(exact_rep.result)) + ) + latencies = { + server: [r.latency for r in results[server][0].query_results if r.latency] + for server in ("prometheus", "sketchdb") + } + with open(monitor_output_path(experiment_dir)) as f: + monitor = json.load(f) + peak_rss_mb = { + process["keyword"]: max(process["memory_info"]) / 1e6 + for process in monitor.values() + } + return { + "measured_error": float(np.nanmedian(errors)) if errors else float("nan"), + "measured_error_all_keys": ( + float(np.nanmedian(errors_all_keys)) if errors_all_keys else "" + ), + "answered": f"{len(errors)}/{len(exact)}", + "asap_latency_ms": 1000 * float(np.median(latencies["sketchdb"])), + "prom_latency_ms": 1000 * float(np.median(latencies["prometheus"])), + "asap_peak_rss_mb": peak_rss_mb[QUERY_ENGINE_MONITOR_KEYWORD], + } + + +def summarize( + recommendations: List[dict], experiments_dir: str, google_trace: str +) -> List[dict]: + """One row per finished recommended or default experiment.""" + rows = [] + quantile_values = None + for row in recommendations: + if row["family"] not in PLANNER_FAMILIES: + continue + for variant in ("recommended", "default"): + name = experiment_name( + row["dataset"], row["query_id"], row["range"], variant + ) + experiment_dir = os.path.join(experiments_dir, name) + if not os.path.isdir(experiment_dir): + continue + if not os.path.exists(monitor_output_path(experiment_dir)): + print(f"skipped unfinished {name}", file=sys.stderr) + continue + if row["family"] == "kll" and quantile_values is None: + quantile_values = replayed_google_cpu_values( + google_trace, P99_REPLAY_CUTOFF_US + ) + summary = summarize_experiment( + experiment_dir, quantile_values if row["family"] == "kll" else None + ) + recommended = variant == "recommended" + rows.append( + { + "experiment": name, + "family": row["family"], + "config": row["config"] if recommended else "default", + # Only the recommended config has a prediction. + "predicted_error": float(row["est_error"]) if recommended else "", + "target": float(row["target"]), + **summary, + "meets_target": summary["measured_error"] <= float(row["target"]), + } + ) + return rows + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + gen = sub.add_parser("generate") + gen.add_argument("--recommendations", required=True) + gen.add_argument("--output-dir", default=DEFAULT_OUTPUT_DIR) + gen.add_argument("--cluster-data-root", default="/data/cluster_traces") + summ = sub.add_parser("summarize") + summ.add_argument("--recommendations", required=True) + summ.add_argument("--experiments-dir", required=True) + summ.add_argument("--output-csv") + summ.add_argument( + "--google-trace", + default="/data/cluster_traces/google/part-00000-of-00500.csv.gz", + help="task_usage part the p99 runs replayed, for their rank error", + ) + args = parser.parse_args() + + recommendations = load_recommendations(args.recommendations) + if args.command == "generate": + configs, skipped = generate( + recommendations, DEFAULT_QUERIES, args.cluster_data_root + ) + write_configs(configs, args.output_dir) + for name in configs: + print(f"wrote {name}") + for note in skipped: + print(f"skipped {note}") + return + + rows = summarize(recommendations, args.experiments_dir, args.google_trace) + if not rows: + print("No finished experiments found") + return + writer = csv.DictWriter(sys.stdout, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + if args.output_csv: + with open(args.output_csv, "w") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/recommended_sketch_configs/results_summary.csv b/asap-tools/experiments/recommended_sketch_configs/results_summary.csv new file mode 100644 index 00000000..0ec1e81f --- /dev/null +++ b/asap-tools/experiments/recommended_sketch_configs/results_summary.csv @@ -0,0 +1,11 @@ +experiment,family,config,predicted_error,target,measured_error,measured_error_all_keys,answered,asap_latency_ms,prom_latency_ms,asap_peak_rss_mb,meets_target +alibaba_v2022_ms_cpu_by_msname_instant_recommended,cms,rows=3 cols=16384,0.020573849606818375,0.05,0.0012830390988093434,2.2210426168040818,20/20,628.7422180175781,3235.399007797241,316.6208,True +alibaba_v2022_ms_cpu_by_msname_instant_default,cms,default,0.020573849606818375,0.05,0.1995992177052795,257.7338964738049,20/20,619.9700832366943,3130.3480863571167,274.51392,False +alibaba_v2022_ms_cpu_by_msname_5m_recommended,cms,rows=3 cols=16384,0.021473512975033845,0.05,0.001300943674761616,2.1992003698696116,20/20,677.9897212982178,4349.95698928833,282.226688,True +alibaba_v2022_ms_cpu_by_msname_5m_default,cms,default,0.021473512975033845,0.05,0.19694224821114614,250.54328076568058,20/20,640.0543451309204,4493.685245513916,257.98656,False +google_2011_cpu_by_job_id_instant_recommended,cms,rows=3 cols=4096,0.04232304463607425,0.05,4.5252910014443425e-05,0.6798157955982075,20/20,124.03130531311035,1638.5921239852905,98.885632,True +google_2011_cpu_by_job_id_instant_default,cms,default,0.04232304463607425,0.05,0.003137059413584803,57.16644243494396,20/20,115.76724052429199,1847.3784923553467,78.856192,True +google_2011_cpu_by_job_id_5m_recommended,cms,rows=3 cols=4096,0.04301674918691996,0.05,4.902173506506417e-05,0.7003128433987799,20/20,99.68626499176025,3803.6561012268066,90.025984,True +google_2011_cpu_by_job_id_5m_default,cms,default,0.04301674918691996,0.05,0.0031331704745136144,57.60383734423756,20/20,91.85421466827393,4261.834502220154,73.785344,True +google_2011_cpu_p99_instant_recommended,kll,k=200,0.002039960011051183,0.01,0.10553291066016723,0.10553291066016723,20/20,3.455996513366699,1860.2386713027954,44.056576,False +google_2011_cpu_p99_instant_default,kll,default,0.002039960011051183,0.01,0.458229353342154,0.458229353342154,20/20,3.654956817626953,1857.015609741211,43.003904,False diff --git a/asap-tools/experiments/tests/test_recommended_sketch_configs.py b/asap-tools/experiments/tests/test_recommended_sketch_configs.py new file mode 100644 index 00000000..6b2b45ab --- /dev/null +++ b/asap-tools/experiments/tests/test_recommended_sketch_configs.py @@ -0,0 +1,255 @@ +"""Tests for recommended_sketch_configs.py (config generation and error metric).""" + +import os +import tempfile +import unittest +from unittest import mock + +import numpy as np +from hydra import compose, initialize_config_dir + +from recommended_sketch_configs import recommended_sketch_configs as rsc + +CONFIG_DIR = os.path.join(rsc.EXPERIMENTS_DIR, "config") + + +def recommendation(dataset, query_id, range_, family, config, meets_target="True"): + return { + "dataset": dataset, + "query_id": query_id, + "range": range_, + "family": family, + "config": config, + "est_error": "0.04", + "target": "0.05", + "meets_target": meets_target, + } + + +class PlannerSketchParametersTest(unittest.TestCase): + def test_cms_rows_cols_become_depth_width(self): + self.assertEqual( + rsc.planner_sketch_parameters("cms", "rows=3 cols=4096"), + {"CountMinSketch": {"depth": 3, "width": 4096}}, + ) + + def test_kll_k(self): + self.assertEqual( + rsc.planner_sketch_parameters("kll", "k=200"), + {"DatasketchesKLL": {"K": 200}}, + ) + + def test_family_without_planner_sketch_raises(self): + with self.assertRaises(ValueError): + rsc.planner_sketch_parameters("countsketch", "rows=3 cols=4096") + + +class TranslatePromqlTest(unittest.TestCase): + def test_alibaba_metric_and_label_names(self): + exporter = rsc.EXPORTERS[("alibaba_v2022", "MSMetrics")] + self.assertEqual( + rsc.translate_promql( + "sum by (msname) (sum_over_time(cpu_utilization[5m]))", exporter + ), + "sum by (ms_name) (sum_over_time(alibaba_microservice_cpu_usage[5m]))", + ) + + def test_only_whole_names_are_renamed(self): + # A label that merely starts with a dataset name is not renamed. + exporter = rsc.EXPORTERS[("alibaba_v2022", "MSMetrics")] + self.assertEqual( + rsc.translate_promql("sum by (msname_x) (m)", exporter), + "sum by (msname_x) (m)", + ) + + +class GenerateTest(unittest.TestCase): + def setUp(self): + self.recommendations = [ + recommendation( + "google_2011", "cpu_by_job_id", "5m", "cms", "rows=3 cols=4096" + ), + recommendation( + "google_2011", "cpu_by_job_id", "5m", "countsketch", "rows=3 cols=4096" + ), + recommendation("google_2011", "cpu_p99", "instant", "kll", "k=200"), + recommendation("google_2011", "cpu_p99", "instant", "dd", "alpha=0.01"), + ] + + def test_recommended_and_default_twin(self): + configs, _ = rsc.generate( + self.recommendations, + [("google_2011", "cpu_by_job_id", "5m")], + "/traces", + ) + self.assertEqual( + sorted(configs), + [ + "google_2011_cpu_by_job_id_5m_default", + "google_2011_cpu_by_job_id_5m_recommended", + ], + ) + recommended = configs["google_2011_cpu_by_job_id_5m_recommended"] + default = configs["google_2011_cpu_by_job_id_5m_default"] + self.assertEqual( + recommended["sketch_parameters"], + {"CountMinSketch": {"depth": 3, "width": 4096}}, + ) + self.assertNotIn("sketch_parameters", default) + group = recommended["experiment_params"]["query_groups"][0] + self.assertEqual( + group["queries"], + ["sum by (job_id) (sum_over_time(google_mean_cpu_usage_rate_0[5m]))"], + ) + # A range query waits one full range before its first answer. + self.assertEqual( + group["client_options"]["starting_delay"], + rsc.MIN_STARTING_DELAY_S + 300, + ) + self.assertEqual(recommended["cluster_data_directory"], "/traces/google") + + def test_alibaba_scrape_interval_bounds_query_rate(self): + # One Alibaba MSMetrics scrape takes about 5 s, and the planner rejects + # queries repeated faster than the scrape interval. + configs, _ = rsc.generate( + [ + recommendation( + "alibaba_v2022", + "ms_cpu_by_msname", + "instant", + "cms", + "rows=3 cols=16384", + ) + ], + [("alibaba_v2022", "ms_cpu_by_msname", "instant")], + "/traces", + ) + config = configs["alibaba_v2022_ms_cpu_by_msname_instant_recommended"] + self.assertEqual(config["prometheus"], {"scrape_interval": "10s"}) + group = config["experiment_params"]["query_groups"][0] + self.assertEqual(group["repetition_delay_ms"], 10000) + self.assertEqual( + group["queries"], ["sum by (ms_name) (alibaba_microservice_cpu_usage)"] + ) + + def test_families_without_planner_sketch_are_skipped(self): + configs, skipped = rsc.generate( + self.recommendations, + [ + ("google_2011", "cpu_by_job_id", "5m"), + ("google_2011", "cpu_p99", "instant"), + ], + "/traces", + ) + self.assertEqual(len(configs), 4) + self.assertEqual(len(skipped), 2) + self.assertIn("countsketch", skipped[0]) + self.assertIn("dd", skipped[1]) + + def test_query_without_exporter_is_skipped(self): + # MSRTMCR (call-rate) data has no cluster_data_exporter. + configs, skipped = rsc.generate( + [ + recommendation( + "alibaba_v2022", + "mcr_by_msname", + "instant", + "cms", + "rows=3 cols=16384", + ) + ], + [("alibaba_v2022", "mcr_by_msname", "instant")], + "/traces", + ) + self.assertEqual(configs, {}) + self.assertIn("no exporter", skipped[0]) + + def test_config_that_misses_target_is_skipped(self): + configs, skipped = rsc.generate( + [ + recommendation( + "google_2011", "cpu_p99", "instant", "kll", "k=200", "False" + ) + ], + [("google_2011", "cpu_p99", "instant")], + "/traces", + ) + self.assertEqual(configs, {}) + self.assertIn("does not meet the target", skipped[0]) + + +class ComposeGeneratedConfigTest(unittest.TestCase): + def test_recommended_config_overrides_config_yaml_sketch_parameters(self): + # The generated files are package _global_ so sketch_parameters lands at + # the top level, where the runner reads it, not under experiment_params. + name = "recommended_sketch_configs/google_2011_cpu_p99_instant_recommended" + with initialize_config_dir(version_base=None, config_dir=CONFIG_DIR): + cfg = compose( + "config", + overrides=[f"experiment_type={name}"], + ) + self.assertEqual(cfg.sketch_parameters.DatasketchesKLL.K, 200) + self.assertEqual( + list(cfg.experiment_params.query_groups[0].queries), + ["quantile(0.99, google_mean_cpu_usage_rate_0)"], + ) + + +class AreTopKeysTest(unittest.TestCase): + def test_largest_keys_only(self): + exact = {"a": 100.0, "b": 10.0, "c": 1.0} + estimate = {"a": 110.0, "b": 10.0, "c": 5.0} + self.assertAlmostEqual(rsc.are_top_keys(exact, estimate, num_keys=2), 0.05) + + def test_missing_estimate_counts_as_zero(self): + self.assertEqual(rsc.are_top_keys({"a": 4.0}, {}), 1.0) + + +class RankErrorsTest(unittest.TestCase): + def test_distance_of_estimate_rank_from_quantile(self): + values = np.arange(1.0, 101.0) + np.testing.assert_allclose( + rsc.rank_errors(values, [99.0, 95.0, 1000.0], 0.99), [0.0, 0.04, 0.01] + ) + + +class SummarizeTest(unittest.TestCase): + # The p99 runs were once scored by relative value error against the + # rank-error target, and default rows showed the recommended prediction. + def summarize(self, family): + rec = recommendation("google_2011", "cpu_p99", "instant", family, "k=200") + with tempfile.TemporaryDirectory() as tmp: + for variant in ("recommended", "default"): + name = rsc.experiment_name("google_2011", "cpu_p99", "instant", variant) + os.makedirs(os.path.dirname(rsc.monitor_output_path(f"{tmp}/{name}"))) + open(rsc.monitor_output_path(f"{tmp}/{name}"), "w").close() + with mock.patch.object( + rsc, "summarize_experiment", return_value={"measured_error": 0.002} + ) as summarize_experiment, mock.patch.object( + rsc, "replayed_google_cpu_values", return_value="values" + ) as replayed: + rows = rsc.summarize([rec], tmp, "trace.csv.gz") + return rows, summarize_experiment, replayed + + def test_kll_is_scored_on_replayed_values(self): + rows, summarize_experiment, replayed = self.summarize("kll") + replayed.assert_called_once_with("trace.csv.gz", rsc.P99_REPLAY_CUTOFF_US) + for call in summarize_experiment.call_args_list: + self.assertEqual(call.args[1], "values") + + def test_cms_does_not_read_the_trace(self): + _, summarize_experiment, replayed = self.summarize("cms") + replayed.assert_not_called() + for call in summarize_experiment.call_args_list: + self.assertIsNone(call.args[1]) + + def test_only_recommended_row_has_a_prediction(self): + rows, _, _ = self.summarize("kll") + self.assertEqual( + [(r["config"], r["predicted_error"]) for r in rows], + [("k=200", 0.04), ("default", "")], + ) + + +if __name__ == "__main__": + unittest.main()