perf: faster browser tasks: settling, hedged and warmed Jev calls, quorum decisions, early pages, fewer rescues (openhuman#7000)

.env.example

@@ -47,6 +47,12 @@ TINYCOMPUTER_LAB_SELF_EMAIL=
 # How the live browser examples read a page: by sight (the default), or
 # `tree` for the accessibility tree alone (docs/technical/specs/browser-sight.md).
 # TINYCOMPUTER_BROWSER_PERCEPTION=tree
+# How a page settles after an action: prompt (default) or steady.
+# TINYCOMPUTER_BROWSER_SETTLE=steady
+# task_live: plan inside StartTask as OpenHuman does; the browser opens
+# while it plans, at the one address the task names, unless PRELAUNCH is 0.
+# TASK_PLAN=in-task
+# TINYCOMPUTER_BROWSER_PRELAUNCH=0
 
 # The on-screen cursor's overlay helper, when it is not beside the module.
 # TINYCOMPUTER_CURSOR_OVERLAY=/path/to/tinycomputer-cursor-overlay
@@ -79,6 +85,9 @@ TINYCOMPUTER_LAB_SELF_EMAIL=
 # default 5).
 # TINYCOMPUTER_RESCUE_MODEL=
 # TASK_RESCUES=5
+# task_live: a file of elements earlier runs learned; read before the task,
+# saved after it.
+# TASK_MEMORY=target/task-live/memory/amazon.json
 # The reasoning model task_live shapes a finished task's answer with
 # (default openai/gpt-6-luna on OpenRouter), and a JSON TaskOutput file
 # asking for it.

MODULE.md

@@ -61,8 +61,12 @@ is optional:
   base URL; and `rescue_route` (`api_key`, `provider`, `endpoint_url`,
   `sdk_name`) gives the rescuer a route and key of its own;
 - `browser`: how every browser launches — `executable` (the Chrome or
-  Chromium binary), `user_agent`, `args` (an array of launch arguments), and
-  `perception` (`sight`, the default, or `tree`: how a task reads a page).
+  Chromium binary), `user_agent`, `args` (an array of launch arguments),
+  `perception` (`sight`, the default, or `tree`: how a task reads a page),
+  `settle` (`prompt`, the default, or `steady`: how long a page is let
+  settle after an action), and `prelaunch` (a boolean, `true` by default:
+  open a browser-only task's browser while it is planned, at the one web
+  address the task's text names, if it names one).
 
 Configuration is delivered as sensitive host-control traffic and is never
 shown to monitors.

crates/tinycomputer-browser/src/fake/mod.rs

@@ -10,13 +10,15 @@ use serde_json::{Value, json};
 use crate::engine::{Engine, Launcher, Reply};
 
 type Script = dyn Fn(&Value) -> Option<Value> + Send + Sync;
+type Stall = dyn Fn(&Value) -> bool + Send + Sync;
 
 /// Records every command; answers from an optional override, else like the
-/// engine would.
+/// engine would, and never answers a command it is told to stall on.
 #[derive(Clone)]
 pub(crate) struct Fake {
     sent: Arc<Mutex<Vec<Value>>>,
     script: Arc<Script>,
+    stall: Arc<Stall>,
 }
 
 impl std::fmt::Debug for Fake {
@@ -36,9 +38,20 @@ impl Fake {
         Self {
             sent: Arc::new(Mutex::new(Vec::new())),
             script: Arc::new(script),
+            stall: Arc::new(|_| false),
         }
     }
 
+    /// The same fake, never answering a command `stall` picks, as a page
+    /// being replaced may not.
+    pub(crate) fn stalling(
+        mut self,
+        stall: impl Fn(&Value) -> bool + Send + Sync + 'static,
+    ) -> Self {
+        self.stall = Arc::new(stall);
+        self
+    }
+
     pub(crate) fn actions(&self) -> Vec<String> {
         self.sent
             .lock()
@@ -131,6 +144,9 @@ pub(crate) fn default_reply(command: &Value) -> Value {
 impl Engine for Fake {
     fn execute(&mut self, command: Value) -> Reply<'_> {
         self.sent.lock().unwrap().push(command.clone());
+        if (self.stall)(&command) {
+            return Box::pin(std::future::pending());
+        }
         let reply = (self.script)(&command).unwrap_or_else(|| default_reply(&command));
         Box::pin(async move { reply })
     }

crates/tinycomputer-browser/src/lib.rs

@@ -47,6 +47,6 @@ pub use error::{Error, Result};
 pub use linked::AgentBrowser;
 pub use outputs::SWEEP_INTERVAL;
 pub use sessions::{Browser, MAX_SESSIONS};
-pub use surface::{BrowserSurface, Denoised, Perception};
+pub use surface::{BrowserSurface, Denoised, Perception, Settle};
 pub use tinycomputer_bus::browser::*;
 pub use tinycomputer_cursor::{CursorPace, ProcessOverlay, ScreenCursor};

crates/tinycomputer-browser/src/surface/mod.rs

@@ -22,14 +22,18 @@ mod sight;
 mod tabs;
 mod tree;
 mod uncover;
+mod watch;
 
 pub use sight::Denoised;
 
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 
 use serde_json::json;
 use tinycomputer_bus::DesktopResponse;
-use tinycomputer_bus::browser::{Action, SessionId, SessionOptions};
+use tinycomputer_bus::browser::{
+    Action, NavigateRequest, SessionId, SessionOptions, SnapshotRequest,
+};
 use tinycomputer_core::Platform;
 use tinycomputer_core::surface::Screen;
 use tinycomputer_cursor::ScreenCursor;
@@ -51,6 +55,13 @@ const SETTLE_MS: u64 = 400;
 /// that polls forever is never idle, so this is a cap, not an expectation.
 const NETWORK_IDLE_MS: u64 = 2_000;
 
+/// The longest [`Settle::Prompt`] waits for the requests that change the
+/// page to end. Live on Amazon, each action that opened a page sent 100+
+/// requests for over 2 s, while what the task needed (the results, a
+/// product's title and Add to Cart, the cart's subtotal) showed after
+/// 0.7–1.2 s.
+const QUIET_MS: u64 = 1_000;
+
 /// The longest one reading of the page may take, by sight or as a tree. A
 /// reading sent while a page was being replaced waited out the browser's
 /// own deadline live, 30 s for sight and again for the tree, so one look
@@ -62,16 +73,49 @@ const READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
 pub enum Perception {
     /// As a person looks at it: what is drawn and on top, the words on and
     /// beside each control, and which boxes take text, read from the
-    /// rendered page. Falls back to the accessibility tree when it cannot
-    /// reach what it sees (a shadow root, a frame in front) or the reading
-    /// fails.
+    /// rendered page. One shadow root's controls are read from the
+    /// accessibility tree beside it. Falls back to the tree alone when it
+    /// cannot reach what it sees (two shadow roots showing controls, a frame
+    /// in front, a shadow root the tree cannot read) or the reading fails.
     #[default]
     Sight,
     /// Through the accessibility tree alone: roles and names as the page's
     /// markup declares them.
     Tree,
 }
 
+/// How a [`BrowserSurface`] lets the page settle after an action, before
+/// the page is read again.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub enum Settle {
+    /// Wait for the network to go idle — 500 ms with nothing in flight,
+    /// counted only after a first quiet receive window, so at least about
+    /// 1.1 s, and at most 2 s (`NETWORK_IDLE_MS`) — then pause 400 ms
+    /// (`SETTLE_MS`) more.
+    Steady,
+    /// Wait for the network to go quiet, counting the 500 ms from the start
+    /// and only the requests that can change the page (its document,
+    /// scripts, stylesheets, fetched data), those the action sent included,
+    /// at most 1 s (`QUIET_MS`); then only until the page stops changing:
+    /// no DOM change for 120 ms (`STILL_MS`) and no finite CSS animation
+    /// running, over at least two drawn frames, at most 400 ms (`SETTLE_MS`).
+    /// An idle page is read again after about 0.6 s instead of 1.6 s; a busy
+    /// one still waits for its requests. The default: over 44 live runs it
+    /// cost no run its outcome.
+    #[default]
+    Prompt,
+}
+
+/// How long the page must go without a DOM change, under [`Settle::Prompt`],
+/// to count as still.
+const STILL_MS: u64 = 120;
+
+/// The longest an early load ([`BrowserSurface::open_at`]) waits for its
+/// page's `load` event: beyond nine in ten live first pages (6.4 s). A plan
+/// drafted sooner waits for it no longer, and a page not drawn by then is
+/// loaded again by the step that browses there.
+const EARLY_LOAD_MS: u64 = 10_000;
+
 /// One browser session, lazily opened, as a [`Surface`].
 #[derive(Clone)]
 pub struct BrowserSurface {
@@ -82,7 +126,15 @@ pub struct BrowserSurface {
     platform: Platform,
     cursor: Arc<ScreenCursor>,
     perception: Perception,
+    settle: Settle,
     denoised: Arc<Mutex<Denoised>>,
+    /// Set once the surface is let go ([`BrowserSurface::close`]): it is
+    /// then not opened early again.
+    closed: Arc<AtomicBool>,
+    /// The session opened early ([`BrowserSurface::open_at`]) and the
+    /// address loaded in it, until the page is first read or another address
+    /// is loaded: a navigation there, in that session, finds it loaded.
+    opened_at: Arc<Mutex<Option<(SessionId, String)>>>,
 }
 
 impl std::fmt::Debug for BrowserSurface {
@@ -93,6 +145,8 @@ impl std::fmt::Debug for BrowserSurface {
             .field("platform", &self.platform)
             .field("cursor", &self.cursor)
             .field("perception", &self.perception)
+            .field("settle", &self.settle)
+            .field("opened_at", &self.opened_at)
             .finish_non_exhaustive()
     }
 }
@@ -115,10 +169,21 @@ impl BrowserSurface {
             platform: Platform::current(),
             cursor: Arc::new(ScreenCursor::off()),
             perception: Perception::default(),
+            settle: Settle::default(),
             denoised: Arc::new(Mutex::new(Denoised::default())),
+            closed: Arc::new(AtomicBool::new(false)),
+            opened_at: Arc::default(),
         }
     }
 
+    /// The same surface, settling after an action with `settle`
+    /// ([`Settle::Prompt`] unless told otherwise).
+    #[must_use]
+    pub fn with_settle(mut self, settle: Settle) -> Self {
+        self.settle = settle;
+        self
+    }
+
     /// The same surface, reading pages with `perception`
     /// ([`Perception::Sight`] unless told otherwise).
     #[must_use]
@@ -137,6 +202,44 @@ impl BrowserSurface {
         self
     }
 
+    /// Opens the session now, if it is not open yet, rather than at the first
+    /// call that needs it: a task that will run on the browser can open it
+    /// while its plan is drafted. Whether the session is open. A surface
+    /// already let go ([`BrowserSurface::close`]) is not opened: a task
+    /// cancelled while its browser was being opened early is left holding
+    /// none.
+    #[must_use]
+    pub fn open(&self) -> bool {
+        self.session_slot(true).is_ok()
+    }
+
+    /// Opens the session early, as [`BrowserSurface::open`] does, and loads
+    /// `url` in it: the page a task names, loaded while its plan is drafted,
+    /// waiting for its `load` at most `EARLY_LOAD_MS`. Until the page is
+    /// first read, a navigation to the same place (one page's two
+    /// addresses: `https://` or not, `www.` or not, a trailing slash or not)
+    /// in the same session finds it loaded and loads nothing. Whether the
+    /// page loaded; a surface already let go opens nothing, and one let go
+    /// meanwhile keeps no page.
+    #[must_use]
+    pub fn open_at(&self, url: &str) -> bool {
+        let Ok(id) = self.session_slot(true) else {
+            return false;
+        };
+        let request = NavigateRequest {
+            timeout_ms: Some(EARLY_LOAD_MS),
+            ..NavigateRequest::new(url)
+        };
+        let loaded = self.navigate_in(&id, request).ok;
+        if loaded
+            && !self.closed.load(Ordering::Acquire)
+            && let Ok(mut opened) = self.opened_at.lock()
+        {
+            *opened = Some((id, url.to_owned()));
+        }
+        loaded
+    }
+
     /// The session this surface drives, once one is open.
     #[must_use]
     pub fn session(&self) -> Option<SessionId> {
@@ -157,6 +260,10 @@ impl BrowserSurface {
     /// Closes the session, if one is open, without waiting for it: safe to
     /// call from async code, where a blocking surface call is not.
     pub fn close(&self) {
+        // Before the slot is taken: an early open that has not begun yet
+        // finds it set, and one under way finishes first and is closed.
+        self.closed.store(true, Ordering::Release);
+        let _opened = self.early_page();
         let Some(id) = self
             .session
             .lock()
@@ -176,13 +283,23 @@ impl BrowserSurface {
     }
 
     fn ensure_session(&self) -> Result<SessionId> {
+        self.session_slot(false)
+    }
+
+    /// The open session, opening one if there is none; when `early`, not on
+    /// a surface already let go. `closed` is read under the slot's lock, so
+    /// an early open and a close cannot both miss each other.
+    fn session_slot(&self, early: bool) -> Result<SessionId> {
         let mut slot = self
             .session
             .lock()
             .map_err(|_| Error::failed("the browser surface was poisoned by a panic"))?;
         if let Some(id) = slot.as_ref() {
             return Ok(id.clone());
         }
+        if early && self.closed.load(Ordering::Acquire) {
+            return Err(Error::failed("the browser surface was let go"));
+        }
         let info = self.block(self.browser.open_session(self.options.clone()))?;
         *slot = Some(info.id.clone());
         Ok(info.id)
@@ -204,11 +321,67 @@ impl BrowserSurface {
             .ok()?
             .ok()?;
         let result = reply.get("result")?;
-        let screen = sight::screen(result)?;
+        let mut screen = sight::screen(result)?;
+        match sight::shadows(result).as_slice() {
+            [] => {}
+            [shadow] => self.read_shadow(&id, shadow, &mut screen)?,
+            // A tree snapshot's refs last until the next one: one host's
+            // subtree can be read beside sight, not two.
+            _ => return None,
+        }
         self.keep_denoised(sight::denoised(result));
         Some(screen)
     }
 
+    /// Adds to `screen` what the tree reads under `shadow`'s host: the
+    /// controls a selector cannot reach, and the host and what the page puts
+    /// in its slots, which sight leaves to the tree, under the label of the
+    /// layer they draw, after everything sight read. `None` when the subtree
+    /// cannot be read, so the tree reads the whole page instead.
+    fn read_shadow(
+        &self,
+        id: &SessionId,
+        shadow: &sight::Shadow,
+        screen: &mut Screen,
+    ) -> Option<()> {
+        let request = SnapshotRequest {
+            selector: Some(sight::selector(&shadow.host)),
+            ..SnapshotRequest::default()
+        };
+        let reading = self.browser.snapshot(id, request);
+        let snapshot = self
+            .block(async { tokio::time::timeout(READ_TIMEOUT, reading).await })
+            .ok()?
+            .ok()?;
+        let part = tree::screen(&snapshot.tree, &snapshot.title);
+        let after = screen
+            .candidates
+            .iter()
+            .chain(&screen.text_nodes)
+            .map(|node| node.order + 1)
+            .max()
+            .unwrap_or_default();
+        let placed = |mut node: tinycomputer_core::surface::Candidate| {
+            if let Some(label) = &shadow.label {
+                node.path.insert(0, label.clone());
+            }
+            node.order += after;
+            node
+        };
+        screen
+            .candidates
+            .extend(part.candidates.into_iter().map(placed));
+        screen
+            .text_nodes
+            .extend(part.text_nodes.into_iter().map(placed));
+        for line in part.context {
+            if !screen.context.contains(&line) {
+                screen.context.push(line);
+            }
+        }
+        Some(())
+    }
+
     fn keep_denoised(&self, denoised: Denoised) {
         if let Ok(mut kept) = self.denoised.lock() {
             *kept = denoised;