diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md index 3b606a4f5..ee2118d2e 100644 --- a/packages/gooddata-eval/AGENTS.md +++ b/packages/gooddata-eval/AGENTS.md @@ -156,6 +156,7 @@ Run this before guessing a `--model` string. | `GD_EVAL_AGENT_ID` | which agent to drive, same as `--agent-id` | | `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` | `--langfuse`, `--langfuse-dataset` | | `GOODDATA_EVAL_CHAT_*` | SSE retry, backoff and timeout knobs | +| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary runs are not labelled | | `GD_EVAL_TIMERS` | same as `--timers` | A gitignored `.env` at the repo root is the normal place for these; load it with diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index d8381c907..0a3edb07c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -13,12 +13,15 @@ protocol, not on this class. """ +import functools import json import logging import os +import re import time from dataclasses import dataclass, field from typing import Any, Callable, Iterable, TypeVar +from urllib.parse import quote import httpx @@ -135,6 +138,45 @@ def _float_env(name: str, default: float) -> float: # 4-turn conversation run to 4x the budget, which is not what a user would sit through. _ITEM_TIMEOUT_S = _float_env("GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S", 0.0) +# Comma-separated `key=value` labels, e.g. `model_version=gpt-5.2,github_run_id=42`, sent as +# W3C baggage on every chat request. gen-ai's Langfuse span processor copies `langfuse_*` +# baggage onto each span it starts, so every observation of the conversation -- the cost-bearing +# generations, retries and the title trace included -- carries them from creation. Langfuse v4 +# cannot add them to an observation afterwards. A value cannot contain a comma, and a key must be +# a plain token (letters, digits, `_`, `-`); any other key is skipped with a warning, because httpx +# refuses a non-ASCII header and gen-ai's OTel propagator drops or rewrites the rest. +_TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS" +_TRACE_LABEL_KEY = re.compile(r"[A-Za-z0-9_-]+") + + +@functools.cache +def _warn_skipped_trace_label(key: str) -> None: + """Warn once per key per process: every ChatClient re-reads the labels.""" + _log.warning("%s: skipping label with key %r, keys must match [A-Za-z0-9_-]+", _TRACE_LABELS_ENV, key) + + +def _trace_baggage() -> dict[str, str]: + """The `baggage` header for the labels in ``GOODDATA_EVAL_TRACE_LABELS``, or none. + + Each label becomes trace metadata, and ``model_version`` -- the combo the report groups + on -- also becomes the observation ``version``, the dimension Langfuse groups cost by. + Values are percent-encoded: a space or comma would otherwise end the baggage entry. + """ + labels: dict[str, str] = {} + for pair in os.getenv(_TRACE_LABELS_ENV, "").split(","): + key, sep, value = pair.partition("=") + key = key.strip() + if not sep or not key: + continue + if _TRACE_LABEL_KEY.fullmatch(key): + labels[key] = value.strip() + else: + _warn_skipped_trace_label(key) + entries = [f"langfuse_metadata_{k}={quote(v, safe='')}" for k, v in labels.items()] + if labels.get("model_version"): + entries.insert(0, f"langfuse_version={quote(labels['model_version'], safe='')}") + return {"baggage": ",".join(entries)} if entries else {} + def set_default_turn_timeout(seconds: float | None) -> None: """Set the per-turn budget every ChatClient built afterwards inherits. @@ -466,7 +508,7 @@ def __init__( the ``enableAiContextSetup`` feature flag is on for the organization. """ self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations" - self._auth = {"Authorization": f"Bearer {token}"} + self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()} # 0/None disables the cap. Also lowered onto the read timeout: the wall-clock check # fires between events, so a turn that goes silent needs the transport to give up too. budget = _TURN_TIMEOUT_S if turn_timeout_s is None else turn_timeout_s diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index e40dae503..cf007c60b 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -1,5 +1,8 @@ # (C) 2026 GoodData Corporation import json +import logging +from collections.abc import Callable +from urllib.parse import unquote import httpx import pytest @@ -1053,3 +1056,77 @@ def handler(request): ) client.ask(item) assert captured["body"]["userContext"] == _ATTACHMENT + + +def _baggage_of(requests: list[httpx.Request]) -> list[dict[str, str] | None]: + """Each request's `baggage` header as {key: decoded value}, or None when absent.""" + out: list[dict[str, str] | None] = [] + for r in requests: + raw = r.headers.get("baggage") + out.append(None if raw is None else {k: unquote(v) for k, v in (e.split("=", 1) for e in raw.split(","))}) + return out + + +def _record_requests(requests: list[httpx.Request]) -> Callable[[httpx.Request], httpx.Response]: + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.url.path.endswith("/conversations"): + return httpx.Response(200, json={"conversationId": "c1"}) + return httpx.Response(200, content=_OK_SSE) + + return handler + + +def test_trace_labels_ride_every_request_as_langfuse_baggage(monkeypatch: pytest.MonkeyPatch) -> None: + """gen-ai's Langfuse span processor copies `langfuse_*` baggage onto every span it starts, so + the labels reach the root generation and its cost-bearing children at creation.""" + monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "model_version=gpt-5.2 openai,github_run_id=42") + requests: list[httpx.Request] = [] + client = _client_with_handler(_record_requests(requests)) + + client.send_message(client.create_conversation(), "q") + + expected = { + "langfuse_version": "gpt-5.2 openai", + "langfuse_metadata_model_version": "gpt-5.2 openai", + "langfuse_metadata_github_run_id": "42", + } + assert _baggage_of(requests) == [expected, expected] + # Encoded, not raw: a space or comma in a value would otherwise split or end the entry. + assert " " not in requests[0].headers["baggage"] + + +def test_trace_labels_without_a_model_version_set_no_langfuse_version(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "github_run_id=42") + requests: list[httpx.Request] = [] + _client_with_handler(_record_requests(requests)).create_conversation() + + assert _baggage_of(requests) == [{"langfuse_metadata_github_run_id": "42"}] + + +@pytest.mark.parametrize("key", ["modèle", "run id", "c++"]) +def test_trace_label_with_a_non_token_key_is_skipped_with_one_warning( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, key: str +) -> None: + """httpx refuses a non-ASCII header, gen-ai's OTel propagator drops a key with a space, and + it reads `+` as a space; each would otherwise fail the run or mislabel the trace.""" + monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", f"github_run_id=42,{key}=1") + requests: list[httpx.Request] = [] + + for _ in range(2): + _client_with_handler(_record_requests(requests)).create_conversation() + + assert _baggage_of(requests) == [{"langfuse_metadata_github_run_id": "42"}] * 2 + assert [r.levelno for r in caplog.records if repr(key) in r.getMessage()] == [logging.WARNING] + + +@pytest.mark.parametrize("raw", [None, "", "no-equals-sign", " , "]) +def test_no_trace_labels_send_no_baggage(monkeypatch: pytest.MonkeyPatch, raw: str | None) -> None: + if raw is None: + monkeypatch.delenv("GOODDATA_EVAL_TRACE_LABELS", raising=False) + else: + monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", raw) + requests: list[httpx.Request] = [] + _client_with_handler(_record_requests(requests)).create_conversation() + + assert _baggage_of(requests) == [None]