Add trace components

Cargo.toml

@@ -2,9 +2,23 @@
 resolver = "2"
 members = [
     "components/client",
+    "components/trace",
+    "components/trace-client",
+    "components/trace-handler",
+    "components/trace-componentized-client",
+    "components/trace-types",
+    "crates/test-harness",
 ]
 
 [workspace.dependencies]
+bytes = "1"
+http = "1"
+http-body-util = "0.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+test-harness = { path = "./crates/test-harness" }
+tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync"] }
+wasmtime = { version = "49", features = ["component-model-async"] }
+wasmtime-wasi = { version = "49", features = ["p3"] }
+wasmtime-wasi-http = { version = "49", default-features = false, features = ["p3", "component-model-async"] }
 wit-bindgen = "0.62.0"

Makefile

@@ -1,13 +1,16 @@
 SHELL := /bin/bash
 
-WKG_CONFIG := $(CURDIR)/.config/wasm-pkg/config.toml
+# prefer paths are relative to the root of the repository, where make runs, to keep the logs readable
+
+WKG_CONFIG := .config/wasm-pkg/config.toml
 
 export RUST_BACKTRACE ?= 1
 export WASMTIME_BACKTRACE_DETAILS ?= 1
 
-COMPONENTS_DIR := $(abspath target/components)
-TOOLS_DIR := $(abspath target/tools)
-export PATH := $(TOOLS_DIR)/bin:$(PATH)
+COMPONENTS_DIR := target/components
+TOOLS_DIR := target/tools
+# absolute, tools also run from other directories, e.g. `cd components && wkg fetch`
+export PATH := $(abspath $(TOOLS_DIR))/bin:$(PATH)
 
 # cargo binstall downloads prebuilt binaries, without it the tools are built with cargo install
 CARGO_INSTALL := $(if $(shell command -v cargo-binstall 2> /dev/null),cargo binstall --no-confirm --disable-telemetry,cargo install)
@@ -125,7 +128,7 @@ wit/deps: wkg.toml $(shell find wit -type f -name "*.wit" -not -path "deps") | $
 	wkg fetch --config $(WKG_CONFIG)
 
 components/wit/deps: wit/deps components/wkg.toml $(shell find components/wit -type f -name "*.wit" -not -path "deps") | $(call tool,wkg)
-	( cd components && wkg fetch --config $(WKG_CONFIG) )
+	( cd components && wkg fetch --config ../$(WKG_CONFIG) )
 
 # sign published components with cosign, `SIGN=false` to push without signing, e.g. to a local registry
 SIGN ?= true

README.md

@@ -3,6 +3,7 @@
 A collection of utility components that remix wasi:http types and interfaces.
 
 - [Components](#components)
+  - [Choosing a component variant](#choosing-a-component-variant)
 - [Build](#build)
 - [Community](#community)
   - [Code of Conduct](#code-of-conduct)
@@ -15,31 +16,43 @@ A collection of utility components that remix wasi:http types and interfaces.
 ## Components
 
 - [`client`](./components/client/)
+- [`trace`](./components/trace/)
+- [`trace-client`](./components/trace-client/)
+- [`trace-handler`](./components/trace-handler/)
+- [`trace-componentized-client`](./components/trace-componentized-client/)
+- [`trace-types`](./components/trace-types/)
 
-## Build
+### Choosing a component variant
+
+Due to resource types being unique to the instance that defines them in the Component Model, fine grain composition of the `wasi:http` interfaces can be persnickety. Pick the most specific component that covers the interfaces the target component imports. While the base component is more universal, a larger surface area asks the host for capabilities the target component doesn't use.
+
+For example, with the `trace-*` components:
+
+| The target component imports                                  | Use                                                                      |
+| ------------------------------------------------------------- | ------------------------------------------------------------------------ |
+| `wasi:http/types`                                             | [`trace-types`](./components/trace-types/)                               |
+| `wasi:http/types` and `wasi:http/client`                      | [`trace-client`](./components/trace-client/)                             |
+| `wasi:http/types` and `wasi:http/handler`                     | [`trace-handler`](./components/trace-handler/)                           |
+| `wasi:http/types`, `wasi:http/client` and `wasi:http/handler` | [`trace`](./components/trace/)                                           |
+| `componentized:http/client`                                   | [`trace-componentized-client`](./components/trace-componentized-client/) |
 
-A [dev container](https://containers.dev) is available that contains the necessary tools and configuration out of the box.
+`trace-client`, `trace-handler` and `trace` also trace `wasi:http/types`, don't combine them with `trace-types`.
+
+The `wasi:http/handler` exported by `trace-handler` and `trace` takes requests created with their exported `wasi:http/types`. Use them in front of a component that forwards requests with `wasi:http/handler`, a host serving incoming requests can't call them directly.
+
+## Build
 
 Prereqs:
 - a rust toolchain
-- [`wasm-tools`](https://github.com/bytecodealliance/wasm-tools)
-- [`wkg`](https://github.com/bytecodealliance/wasm-pkg-tools)
+- [`cargo-binstall`](https://github.com/cargo-bins/cargo-binstall), optional, to download prebuilt tools instead of building them
 
 ```sh
 make components
 ```
 
 The build creates each component in [`components`](./components) into `target/components`, e.g. the client at `target/components/client/client.wasm`, along with `target/components/interface.wasm`, the `componentized:http` WIT package. Each component is also built with debug info, e.g. `target/components/client/client.debug.wasm`.
 
-```sh
-make test
-```
-
-The WIT dependencies in each `wit/deps` directory are fetched rather than committed, pinned by the `wkg.lock` files. The make targets fetch them as needed. To fetch or update them directly, e.g. before building the Rust components with `cargo`, whose bindings are generated from the WIT:
-
-```sh
-make wit
-```
+The cli tools the build uses, [`static-config`](https://github.com/componentized/static-config), [`wasm-tools`](https://github.com/bytecodealliance/wasm-tools), [`wac`](https://github.com/bytecodealliance/wac) and [`wkg`](https://github.com/bytecodealliance/wasm-pkg-tools), are pinned in [`tools/Cargo.toml`](./tools/Cargo.toml) and installed into `target/tools` as needed, or ahead of time with `make tools`. Dependabot bumps the pinned versions.
 
 ## Community
 

components/trace-client/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "trace-client"
+version = "0.1.0"
+edition = "2024"
+license = "Apache-2.0"
+
+[lib]
+crate-type = ["cdylib"]
+# shared with the other trace components
+path = "../../crates/trace/src/lib.rs"
+
+[features]
+default = ["client"]
+client = []
+handler = []
+
+[dependencies]
+wit-bindgen = { workspace = true, features = ["async-spawn"] }
+
+[dev-dependencies]
+test-harness = { workspace = true }
+tokio = { workspace = true }
+wasmtime = { workspace = true }

components/trace-client/README.md

@@ -0,0 +1,13 @@
+# `trace-client`
+
+Virtualizes the wasi:http/types and wasi:http/client interfaces logging all method calls at the TRACE level and 'componentized-trace' component.
+
+Built from the sources shared with the other trace components in [`crates/trace`](../../crates/trace/src/lib.rs).
+
+## The `trace-client` World
+
+- exports `wasi:http/types`
+- exports `wasi:http/client`
+- imports `wasi:http/types`
+- imports `wasi:http/client`
+- imports `wasi:logging/logging`

components/trace-client/tests/client.rs

@@ -0,0 +1,116 @@
+//! Tests for the `wasi:http/client` operations traced by `trace-client`.
+
+use test_harness::{
+    ErrorCode, Harness, LogEntry, Method, ResourceAny, Scheme, SentRequest, UPSTREAM_HEADER,
+    UPSTREAM_STATUS,
+};
+
+fn traced(message: &str) -> LogEntry {
+    LogEntry::trace("componentized-trace", message)
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn exports_types_and_client() -> wasmtime::Result<()> {
+    let trace = Harness::new("trace-client").build().await?;
+    assert!(trace.exports().exports_types());
+    assert!(trace.exports().exports_client());
+    assert!(!trace.exports().exports_handler());
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn send_traced() -> wasmtime::Result<()> {
+    let mut trace = Harness::new("trace-client").build().await?;
+    let (status, header) = trace
+        .run(async |accessor, trace| {
+            let request = trace
+                .new_request(
+                    accessor,
+                    Method::Get,
+                    Scheme::Https,
+                    "example.com",
+                    "/hello",
+                )
+                .await?;
+            let response: ResourceAny = trace
+                .wasi_http_client()
+                .call_send(accessor, request)
+                .await?
+                .expect("send");
+            // the response is a traced resource
+            let types = trace.wasi_http_types();
+            let status = types
+                .response()
+                .call_get_status_code(accessor, response)
+                .await?;
+            let headers = types
+                .response()
+                .call_get_headers(accessor, response)
+                .await?;
+            let header = types
+                .fields()
+                .call_get(accessor, headers, UPSTREAM_HEADER.0.to_string())
+                .await?;
+            Ok((status, header))
+        })
+        .await?;
+    // the upstream response is passed through
+    assert_eq!(status, UPSTREAM_STATUS);
+    assert_eq!(header, vec![UPSTREAM_HEADER.1.as_bytes().to_vec()]);
+    assert_eq!(
+        trace.recorder().requests(),
+        vec![SentRequest {
+            method: "GET".to_string(),
+            uri: "https://example.com/hello".to_string(),
+        }]
+    );
+    assert_eq!(
+        trace.recorder().masked_logs(),
+        vec![
+            traced("OPERATION=wasi:http/types#fields.new"),
+            traced("OPERATION=wasi:http/types#request.new"),
+            traced("OPERATION=wasi:http/types#request.set-method SELF=&---- METHOD=get"),
+            traced("OPERATION=wasi:http/types#request.set-scheme SELF=&---- SCHEME=some<https>"),
+            traced(
+                "OPERATION=wasi:http/types#request.set-authority SELF=&---- AUTHORITY=some<example.com>"
+            ),
+            traced(
+                "OPERATION=wasi:http/types#request.set-path-with-query SELF=&---- PATH-WITH-QUERY=some</hello>"
+            ),
+            traced("OPERATION=wasi:http/client#send REQUEST=&----"),
+            traced("OPERATION=wasi:http/types#response.get-status-code SELF=&----"),
+            traced("OPERATION=wasi:http/types#response.get-headers SELF=&----"),
+            traced("OPERATION=wasi:http/types#fields.get SELF=&---- NAME=x-upstream"),
+        ]
+    );
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn send_error_passed_through() -> wasmtime::Result<()> {
+    let mut trace = Harness::new("trace-client").build().await?;
+    let result = trace
+        .run(async |accessor, trace| {
+            let scheme = Scheme::Other("gopher".to_string());
+            let request = trace
+                .new_request(accessor, Method::Get, scheme, "example.com", "/")
+                .await?;
+            let result = trace
+                .wasi_http_client()
+                .call_send(accessor, request)
+                .await?;
+            Ok(result.map(|_| ()))
+        })
+        .await?;
+    // the upstream error is passed through, the request never left
+    assert!(
+        matches!(result, Err(ErrorCode::HttpProtocolError)),
+        "unsupported scheme: {result:?}"
+    );
+    assert_eq!(trace.recorder().requests(), vec![]);
+    assert_eq!(
+        trace.recorder().masked_logs().last(),
+        Some(&traced("OPERATION=wasi:http/client#send REQUEST=&----"))
+    );
+    Ok(())
+}

components/trace-client/tests/handles.rs

@@ -0,0 +1,55 @@
+//! The handle logged for a resource must be the same in every message about it, including when
+//! it is passed to `wasi:http/client#send`.
+
+use test_harness::{ErrorCode, Harness, Method, Scheme, ready};
+
+fn same_handle(values: Vec<String>, resource: &str) -> String {
+    let first = values
+        .first()
+        .unwrap_or_else(|| panic!("no handle logged for the {resource}"))
+        .clone();
+    assert!(
+        values.iter().all(|value| *value == first),
+        "the {resource} was logged with different handles: {values:?}"
+    );
+    first
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn send_logs_the_request_handle() -> wasmtime::Result<()> {
+    let mut trace = Harness::new("trace-client").build().await?;
+    trace
+        .run(async |accessor, trace| {
+            let request = trace
+                .new_request(accessor, Method::Get, Scheme::Https, "example.com", "/")
+                .await?;
+            let response = trace
+                .wasi_http_client()
+                .call_send(accessor, request)
+                .await?
+                .expect("send");
+            let responses = trace.wasi_http_types().response();
+            responses.call_get_status_code(accessor, response).await?;
+            responses.call_get_headers(accessor, response).await?;
+            let res = ready(accessor, Ok::<(), ErrorCode>(()))?;
+            responses.call_consume_body(accessor, response, res).await?;
+            Ok(())
+        })
+        .await?;
+    let recorder = trace.recorder();
+    let request = same_handle(
+        recorder.handle_values("wasi:http/types#request."),
+        "request",
+    );
+    assert_eq!(
+        recorder.handle_values("wasi:http/client#send"),
+        vec![request],
+        "send logs the handle of the request it was passed"
+    );
+    // the response from upstream is a new resource, its handle is stable too
+    same_handle(
+        recorder.handle_values("wasi:http/types#response."),
+        "response",
+    );
+    Ok(())
+}

components/trace-componentized-client/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "trace-componentized-client"
+version = "0.1.0"
+edition = "2024"
+license = "Apache-2.0"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+wit-bindgen = { workspace = true }
+
+[dev-dependencies]
+test-harness = { workspace = true }
+tokio = { workspace = true }
+wasmtime = { workspace = true }

components/trace-componentized-client/README.md

@@ -0,0 +1,9 @@
+# `trace-componentized-client`
+
+Virtualizes the componentized:http/client interface logging all method calls at the TRACE level and 'componentized-trace' component.
+
+## The `trace-componentized-client` World
+
+- exports `componentized:http/client`
+- imports `componentized:http/client`
+- imports `wasi:logging/logging`