diff --git a/crates/tinymemory-integrations/src/cortex/README.md b/crates/tinymemory-integrations/src/cortex/README.md index d1a3e4fa..82ee4dbc 100644 --- a/crates/tinymemory-integrations/src/cortex/README.md +++ b/crates/tinymemory-integrations/src/cortex/README.md @@ -178,7 +178,9 @@ as prefixes, so they cannot be labelled and are filtered only client-side. scopes, or, for an unscoped read, every kind scope the engine holds. With nothing to read the answer is empty and nothing is sent. The answer comes from the pack holding the most admitted events. The answer route is - called **once** with `use_pack_id`. Hosted omits a null + called **once** with `use_pack_id` (again, after every pack is built + anew, when the packs were dropped in between, up to three rounds: + CortexDB drops every pack on any forget). Hosted omits a null `answer_instructions`, because its schema is strict; Direct sends `null`. Citations come from the packs' decoded events, filtered (reach included), merged rank by rank (the most specific node's first), one per item, capped diff --git a/crates/tinymemory-integrations/src/cortex/engine/beliefs.rs b/crates/tinymemory-integrations/src/cortex/engine/beliefs.rs index e837ce41..39b3410f 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/beliefs.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/beliefs.rs @@ -34,7 +34,7 @@ use tinymemory_api::{ }; use super::CortexEngine; -use super::fetch::recall_body; +use super::fetch::{recall_body, whole_items_budget}; use super::items::hit; use crate::cortex::descriptor::{CortexWire, Route}; use crate::cortex::envelope::parse_scope; @@ -174,6 +174,7 @@ impl CortexEngine { body["budgets"]["per_layer_limits"] = json!({ "events": 0, "facts": 0, "episodes": 0, "understanding": 0, "beliefs": limit, }); + body["budgets"]["max_tokens"] = json!(whole_items_budget(limit)); let pack = self.log.recall(&body).await?; Ok(beliefs_in(&pack, "/layers/beliefs")) } diff --git a/crates/tinymemory-integrations/src/cortex/engine/beliefs_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/beliefs_tests.rs index 352d8307..bd303a88 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/beliefs_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/beliefs_tests.rs @@ -98,6 +98,14 @@ async fn a_built_scope_s_beliefs_are_read_with_and_without_a_query() { let belief_pack = state.seen.lock().unwrap().recalls.last().cloned().unwrap(); assert_eq!(belief_pack["include"], json!(["beliefs"])); assert_eq!(belief_pack["view"], "granular"); + let asked = belief_pack["budgets"]["per_layer_limits"]["beliefs"] + .as_u64() + .unwrap(); + assert_eq!( + belief_pack["budgets"]["max_tokens"].as_u64(), + Some(asked * crate::cortex::envelope::chunks::MAX_EVENT_TEXT_BYTES as u64), + "room for each belief asked for" + ); assert_eq!(ranked[0].meta.namespace, node); let listed = engine .beliefs(BeliefsRequest::new(Reach::exact(node), 5)) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_tests.rs index ca0374d0..eae5723d 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_tests.rs @@ -320,3 +320,46 @@ async fn a_store_succeeds_when_ranked_recall_is_down() { assert_eq!(listed.items.len(), 1, "the settle probe is best-effort"); } } + +#[tokio::test] +async fn an_answer_whose_pack_expired_recalls_that_scope_again() { + for (engine, state) in both().await { + for item in items() { + engine.store(item).await.unwrap(); + } + let mut req = RecallRequest::new("which editor helix", 2); + req.filter = MetaFilter::kinds([ItemKind::Learning]); + state.seen.lock().unwrap().recalls.clear(); + state.expire_packs.store(1, Ordering::SeqCst); + let answer = engine.recall(req.clone()).await.unwrap(); + assert_eq!(answer.answer, "grounded answer for which editor helix"); + { + let seen = state.seen.lock().unwrap(); + assert_eq!(seen.recalls.len(), 2, "the one scope, packed again"); + assert_eq!(seen.recalls[0], seen.recalls[1], "the same pack request"); + assert_eq!(seen.answers.len(), 2, "answered from the fresh pack"); + } + + // With several scopes every pack is read again, so the citations + // come from packs read after whatever dropped them, not before. + let mut wide = RecallRequest::new("which editor helix", 2); + wide.filter = MetaFilter::kinds([ItemKind::Learning, ItemKind::Conversation]); + state.seen.lock().unwrap().recalls.clear(); + state.expire_packs.store(1, Ordering::SeqCst); + engine.recall(wide).await.unwrap(); + assert_eq!( + state.seen.lock().unwrap().recalls.len(), + 4, + "two scopes, each packed in both rounds" + ); + + state.seen.lock().unwrap().answers.clear(); + state.expire_packs.store(3, Ordering::SeqCst); + assert!( + matches!(engine.recall(req).await, Err(Error::NotFound(_))), + "a bounded retry, not forever" + ); + assert_eq!(state.seen.lock().unwrap().answers.len(), 3); + state.expire_packs.store(0, Ordering::SeqCst); + } +} diff --git a/crates/tinymemory-integrations/src/cortex/engine/recall.rs b/crates/tinymemory-integrations/src/cortex/engine/recall.rs index 3adcee31..bc0abfbb 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/recall.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/recall.rs @@ -9,7 +9,9 @@ //! order, not by relevance, so it would answer from an arbitrary sample. //! With no scope to read, the answer is empty and nothing is sent. //! -//! The answer route is asked once with `use_pack_id`, so it answers from +//! The answer route is asked once with `use_pack_id` (again, after every +//! pack is built anew, when CortexDB dropped the packs in between, up to +//! three rounds: it drops every pack on any forget), so it answers from //! exactly the evidence that pack holds: with several packs, the one holding //! the most admitted events, the most specific node on a tie. **The answer //! text is grounded on that one pack, while the citations come from every @@ -43,6 +45,10 @@ use crate::cortex::error::{Error, Result}; /// Recall packs built at once when a reach spans several scopes. const PACKS_AT_ONCE: usize = 4; +/// Rounds (packs, then one answer) at most per recall: the first, and one +/// after each time CortexDB dropped the packs before the answer used one. +const ANSWER_ATTEMPTS: usize = 3; + /// The derived layers a pack also draws on, besides events. const DERIVED_LAYERS: [&str; 4] = ["facts", "beliefs", "episodes", "understanding"]; @@ -71,6 +77,13 @@ fn pack_budgets(limit: usize) -> Value { Value::Object(layers) } +/// The id a recall pack names. +fn pack_id_of(pack: &Value) -> Result<&str> { + pack.get("pack_id") + .and_then(Value::as_str) + .ok_or_else(|| Error::Engine("CortexDB recall omitted pack_id".to_string())) +} + /// The answer request body. /// /// The hosted route's schema is strict (an unknown key, or a `null` @@ -120,39 +133,20 @@ impl CortexEngine { let mut ordered: Vec<&KindScope> = scopes.iter().collect(); ordered.sort_by_key(|scope| std::cmp::Reverse(scope.namespace.depth())); let paths: Vec = ordered.into_iter().map(|s| s.path.clone()).collect(); - let req = &req; - let packs: Vec<(String, Value)> = stream::iter(paths) - .map(|path| async move { - let pack = self.pack(req, &path).await?; - Ok::<_, Error>((path, pack)) - }) - .buffered(PACKS_AT_ONCE) - .try_collect() - .await?; - let per_pack: Vec> = packs - .iter() - .map(|(_, pack)| ranked(pack, None, &req.filter)) - .collect(); - let chosen = per_pack - .iter() - .enumerate() - .max_by_key(|(index, events)| (events.len(), std::cmp::Reverse(*index))) - .map_or(0, |(index, _)| index); - let (scope, pack) = &packs[chosen]; - let pack_id = pack - .get("pack_id") - .and_then(Value::as_str) - .ok_or_else(|| Error::Engine("CortexDB recall omitted pack_id".to_string()))?; - let answered = self - .log - .answer(&answer_body( - self.wire(), - scope, - &req.question, - pack_id, - req.instructions.as_deref(), - )) - .await?; + // A pack lives 60 s, and CortexDB drops every pack it holds once + // anything is forgotten (measured on 0.10.4: a forget in another + // scope turns the next `use_pack_id` into a 404). On a 404 every + // pack is built again, so the answer and the citations both come + // from packs read after that forget, up to [`ANSWER_ATTEMPTS`] + // rounds in all. + let mut round = 1; + let (per_pack, answered) = loop { + let (per_pack, answered) = self.answer_round(&req, &paths).await?; + match answered { + Err(Error::NotFound(_)) if round < ANSWER_ATTEMPTS => round += 1, + answered => break (per_pack, answered?), + } + }; let answer = answered .get("answer") .and_then(Value::as_str) @@ -183,6 +177,43 @@ impl CortexEngine { } impl CortexEngine { + /// One round of recall: a pack per path (four at a time), decoded and + /// filtered, and the answer route asked once with the pack holding the + /// most admitted events, the most specific node on a tie. The answer's + /// own error is handed back, so the caller can tell a dropped pack. + async fn answer_round( + &self, + req: &RecallRequest, + paths: &[String], + ) -> Result<(Vec>, Result)> { + let packs: Vec<(String, Value)> = stream::iter(paths.to_vec()) + .map(|path| async move { + let pack = self.pack(req, &path).await?; + Ok::<_, Error>((path, pack)) + }) + .buffered(PACKS_AT_ONCE) + .try_collect() + .await?; + let per_pack: Vec> = packs + .iter() + .map(|(_, pack)| ranked(pack, None, &req.filter)) + .collect(); + let chosen = per_pack + .iter() + .enumerate() + .max_by_key(|(index, events)| (events.len(), std::cmp::Reverse(*index))) + .map_or(0, |(index, _)| index); + let (scope, pack) = &packs[chosen]; + let body = answer_body( + self.wire(), + scope, + &req.question, + pack_id_of(pack)?, + req.instructions.as_deref(), + ); + Ok((per_pack, self.log.answer(&body).await)) + } + /// A recall pack for `req` over exactly `scope`, sized for `req.limit` /// citations. It includes the derived layers the answer route reads, /// after the events the citations come from. diff --git a/crates/tinymemory-integrations/src/cortex/envelope/mod_tests.rs b/crates/tinymemory-integrations/src/cortex/envelope/mod_tests.rs index f5665fc5..5837e491 100644 --- a/crates/tinymemory-integrations/src/cortex/envelope/mod_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/envelope/mod_tests.rs @@ -287,3 +287,51 @@ fn a_whitespace_only_document_keeps_its_text() { assert_eq!(rebuild(&envelopes).unwrap(), item, "{body:?}"); } } + +#[test] +fn a_whole_read_needs_every_piece_of_one_agreed_layout() { + let item = long_document(); + let pieces = Envelope::for_item(&item, "id").unwrap(); + assert!(pieces.len() > 1); + assert_eq!(rebuild_whole(&pieces), Some(item.clone())); + assert_eq!(rebuild_whole(&pieces[1..]), None, "a piece missing"); + + let relaid = |edit: &dyn Fn(&mut ChunkInfo)| { + let mut changed = pieces.clone(); + edit(changed[0].chunk.as_mut().unwrap()); + rebuild_whole(&changed) + }; + assert_eq!(relaid(&|chunk| chunk.count += 1), None, "counts disagree"); + assert_eq!( + relaid(&|chunk| chunk.index = 99), + None, + "an index past the count" + ); + let mut zero = pieces[..1].to_vec(); + zero[0].chunk = Some(ChunkInfo { + index: 0, + count: 0, + pages: None, + section: None, + }); + assert_eq!(rebuild_whole(&zero), None, "a zero count"); + + let mut whole = Envelope::for_item(&StoreItem::document("x", meta()), "id") + .unwrap() + .remove(0); + whole.text = match &item { + StoreItem::Document { + body: DocumentBody::Text(text), + .. + } => text.clone(), + _ => unreachable!("a document"), + }; + whole.title.clone_from(&pieces[0].title); + whole.meta = pieces[0].meta.clone(); + let mixed = vec![pieces[1].clone(), whole]; + assert_eq!( + rebuild_whole(&mixed), + Some(item), + "the same item written whole before chunking reads as that body" + ); +} diff --git a/crates/tinymemory-integrations/src/cortex/envelope/rebuild.rs b/crates/tinymemory-integrations/src/cortex/envelope/rebuild.rs index 90a2d7f0..7ae2e849 100644 --- a/crates/tinymemory-integrations/src/cortex/envelope/rebuild.rs +++ b/crates/tinymemory-integrations/src/cortex/envelope/rebuild.rs @@ -83,35 +83,52 @@ pub(crate) fn rebuild(envelopes: &[Envelope]) -> Option { /// does not list yet. A `fetch` hit or a `recall` citation is one piece and /// uses [`rebuild`]. pub(crate) fn rebuild_whole(envelopes: &[Envelope]) -> Option { - if let Some(count) = envelopes.iter().find_map(|e| Some(e.chunk.as_ref()?.count)) { - let held: HashSet = envelopes - .iter() - .filter_map(|envelope| Some(envelope.chunk.as_ref()?.index)) - .collect(); - if (0..count).any(|index| !held.contains(&index)) { - log::debug!( - "[cortex] chunked document {:?} is missing pieces; not returned whole", - envelopes[0].id - ); - return None; - } + if !pieces_complete(envelopes) { + log::debug!( + "[cortex] chunked document {:?} is missing pieces or disagrees on its layout; \ + not returned whole", + envelopes.first().map(|envelope| &envelope.id) + ); + return None; } rebuild(envelopes) } -/// A document's text from its envelopes: the first one's, or the pieces of -/// a chunked document in index order, each once. +/// Whether `envelopes` hold a whole item: one written as a single event +/// (a document's unchunked envelope is its whole body), or every piece of a +/// chunked document, all agreeing on one positive count with every index +/// below it and each index present. +fn pieces_complete(envelopes: &[Envelope]) -> bool { + if envelopes.iter().any(|envelope| envelope.chunk.is_none()) { + return true; + } + let Some(count) = envelopes.iter().find_map(|e| Some(e.chunk.as_ref()?.count)) else { + return true; + }; + let mut held = HashSet::new(); + for chunk in envelopes + .iter() + .filter_map(|envelope| envelope.chunk.as_ref()) + { + if chunk.count != count || chunk.index >= count { + return false; + } + held.insert(chunk.index); + } + count > 0 && held.len() == count as usize +} + +/// A document's text from its envelopes: an unchunked envelope's (the whole +/// body: the same item written before chunking, since an id is a content +/// digest), else the pieces of a chunked document in index order, each once. fn document_text(envelopes: &[Envelope]) -> String { + if let Some(whole) = envelopes.iter().find(|envelope| envelope.chunk.is_none()) { + return whole.text.clone(); + } let mut pieces: Vec<(u32, &str)> = envelopes .iter() .filter_map(|envelope| Some((envelope.chunk.as_ref()?.index, envelope.text.as_str()))) .collect(); - if pieces.is_empty() { - return envelopes - .first() - .map(|envelope| envelope.text.clone()) - .unwrap_or_default(); - } pieces.sort_by_key(|(index, _)| *index); pieces.dedup_by_key(|(index, _)| *index); pieces.into_iter().map(|(_, text)| text).collect() diff --git a/crates/tinymemory-integrations/src/cortex/testing/mod.rs b/crates/tinymemory-integrations/src/cortex/testing/mod.rs index d80a39be..a714c4b7 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/mod.rs @@ -77,6 +77,9 @@ pub(crate) struct Double { pub(crate) rate_limit_forget: AtomicUsize, /// Recall answers 500. pub(crate) recall_down: AtomicBool, + /// Answers refuse their `use_pack_id` as expired (404) this many times, + /// as CortexDB does once anything is forgotten after the pack was built. + pub(crate) expire_packs: AtomicUsize, /// Once the next write is applied, rate limit this many listings and /// hide the listing this many more times: (429s, hidden). Lets a test /// aim at the reads a write makes after it is sent, not the replay diff --git a/crates/tinymemory-integrations/src/cortex/testing/routes.rs b/crates/tinymemory-integrations/src/cortex/testing/routes.rs index 74d0675f..3451f082 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/routes.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/routes.rs @@ -297,6 +297,9 @@ async fn answer( if body["use_pack_id"].as_str() != Some("pack_test") { return fail(&state, 400, "MISSING_PACK"); } + if take_one(&state.expire_packs) { + return fail(&state, 404, "NOT_FOUND"); + } ok( &state, 200, diff --git a/crates/tinymemory-integrations/tests/live_cortexdb.rs b/crates/tinymemory-integrations/tests/live_cortexdb.rs index dfd6990e..e3fe9f20 100644 --- a/crates/tinymemory-integrations/tests/live_cortexdb.rs +++ b/crates/tinymemory-integrations/tests/live_cortexdb.rs @@ -102,8 +102,16 @@ async fn list_until(engine: &CortexEngine, filter: &MetaFilter, want: usize) -> } } +/// Held by each live test for its whole run, so they reach the server one +/// at a time. Run together, one test's forgets drop the packs another is +/// about to answer from, and load the server enough that forgetting a long +/// document (seconds per ~240 KiB event on 0.10.4) outlasts the request +/// timeout. +static ONE_AT_A_TIME: futures::lock::Mutex<()> = futures::lock::Mutex::new(()); + #[tokio::test] async fn the_live_server_upholds_the_contract() { + let _alone = ONE_AT_A_TIME.lock().await; for (wire, engine) in live_engines() { eprintln!("conformance on {wire}"); tinymemory_api::conformance::run(&engine) @@ -114,6 +122,7 @@ async fn the_live_server_upholds_the_contract() { #[tokio::test] async fn documents_conversations_and_learnings_round_trip_into_context() { + let _alone = ONE_AT_A_TIME.lock().await; for (wire, engine) in live_engines() { eprintln!("round trip on {wire}"); round_trip(&engine).await; @@ -264,6 +273,7 @@ async fn round_trip(engine: &CortexEngine) { /// section, and `forget` removes every piece. #[tokio::test] async fn a_long_document_round_trips_in_pieces() { + let _alone = ONE_AT_A_TIME.lock().await; for (wire, engine) in live_engines() { eprintln!("long document on {wire}"); let workspace = format!("ws-long-{}", run_id()); @@ -276,7 +286,11 @@ async fn a_long_document_round_trips_in_pieces() { .collect(); let body = pages.join("\u{c}"); // A document splits once its envelope passes the 256 KiB chunk - // target (not the 1 MiB event limit): this one is several pieces. + // target (not the 1 MiB event limit): this one is three pieces, so + // two boundaries are crossed. It stays under 1 MiB because CortexDB + // 0.10.4 takes seconds to forget a ~240 KiB event (8 to 13 s + // measured), and a longer document's forget can outlast the 60 s + // request timeout. assert!( body.len() > 2 * 256 * 1024, "over twice the chunk target: {} bytes", @@ -351,7 +365,7 @@ async fn a_long_document_round_trips_in_pieces() { ); let report = engine - .forget(ForgetTarget::Ids(vec![receipt.id])) + .forget(ForgetTarget::Ids(vec![receipt.id.clone()])) .await .expect("forget"); assert_eq!(report.forgotten, 1); @@ -362,7 +376,18 @@ async fn a_long_document_round_trips_in_pieces() { .expect("list after forget") .items .is_empty(), - "no piece is left behind" + "the document is gone" ); + // `list` hides a document missing a piece, so ask ranked recall, + // which hits single pieces: none may be left behind. + let deadline = Instant::now() + VISIBILITY; + loop { + let page = engine.fetch(fetch.clone()).await.expect("fetch"); + if !page.hits.iter().any(|hit| hit.id == receipt.id) { + break; + } + assert!(Instant::now() < deadline, "a piece outlived forget"); + tokio::time::sleep(Duration::from_millis(500)).await; + } } } diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 411c6dc3..32c30114 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -14,7 +14,7 @@ rustdoc next to the code. | [operations.md](operations.md) | Step-by-step semantics of store, store_many, fetch, recall, list, forget, explore and get | | [namespaces.md](namespaces.md) | The memory tree: `Namespace`, `Segment`, `Reach`, and what each operation does with them | | [cortex.md](cortex.md) | The CortexDB engine: wires, scopes, envelopes, recall | -| [cortex-wire.md](cortex-wire.md), [cortex-flows.md](cortex-flows.md) | The CortexDB wire formats and the step-by-step request flows | +| [cortex-wire.md](cortex-wire.md), [cortex-flows.md](cortex-flows.md), [cortex-chunks.md](cortex-chunks.md) | The CortexDB wire formats, the step-by-step request flows, and chunked documents | | [tools.md](tools.md) | `tinymemory-tools`: the seven agent tools, host-fixed scoping, `context.md` | | [lifecycle.md](lifecycle.md) | The agent memory lifecycle: the standard layout (brain, conversations, learnings), holistic recall, pre- and post-turn, compaction, background belief builds | | [integrations.md](integrations.md) | `tinymemory-integrations`: registry and config, documents, sources, safety, legacy import | diff --git a/docs/architecture/cortex-chunks.md b/docs/architecture/cortex-chunks.md new file mode 100644 index 00000000..b7de6bb3 --- /dev/null +++ b/docs/architecture/cortex-chunks.md @@ -0,0 +1,51 @@ +# CortexDB: chunked documents + +How the CortexDB engine writes a document too long for one event, and how it +reads one back. The event layout and labels are in +[cortex-wire.md](cortex-wire.md); the request flows in +[cortex-flows.md](cortex-flows.md). + +CortexDB refuses an experience whose flattened text is over 1 MiB +(`422 INVALID_ENVELOPE`, 0.10.4 API §6.10), and a document's event text is +its whole envelope. So (`envelope/chunks.rs`): + +- **When.** Measured as a piece would be written (the envelope with its + `chunk` field): a document whose encoded envelope fits in + `DOCUMENT_CHUNK_TARGET_BYTES` (256 KiB) is one event, byte-identical to an + unchunked one (no `chunk` field). A longer one is split. The target is the + only granularity knob: at `0` every page and every section becomes its own + event, except that a page or section too big for one event is still cut + into several (see Where). +- **Where.** First at page breaks (the form feed the PDF converter puts + between pages), then before markdown heading lines; a stretch of only + whitespace (or page breaks) never becomes a piece of its own but joins the + unit next to it, except in a text that is nothing else, which is then one + piece, so every byte is kept. These units are packed greedily, in order, + up to the target. A unit over the target is cut at blank lines, then line + ends, then characters. Sizes are JSON-escaped bytes plus the envelope + around the piece (its metadata and the `chunk` field at full width, with + a page range reserved only when the document marks pages). +- **Limit.** No event over `MAX_EVENT_TEXT_BYTES` (768 KiB of encoded + envelope, a quarter under the server's limit) is ever sent: every event of + a batch is encoded and checked before the first write, and an item over it + (a learning or a conversation turn that long) is `Error::InvalidRequest`. +- **Identity and replay.** Every piece carries the item's id and label, so + replay detection, `forget` by id or filter, and `get` see all of them; a + store that failed part way writes only the missing pieces. Until then + `get` and `list` do not return the document (never a truncated body); + `fetch` and `recall` still hit the pieces that are there. +- **Reads.** `get` and `list` give the whole document (pieces in index + order). `fetch` and `recall` give one hit or citation per document, as for + every item: its best-ranked piece, with the + item's id and its metadata plus, when known, a `page:` (or + `page:-`) tag and a `section:` tag: a document without + page breaks gets no page tag, a piece before the first heading no section + tag. These tags are read-side metadata, not part of the item's identity. + `pages` is an inclusive range `[first, last]`, counted from 1, so a piece + on one page has `first == last`. Readable CortexDB labels for page and + section are not written yet. +- **Why one piece per target rather than per page.** CortexDB 0.10.4 + already fragments every event over about 500 bytes for retrieval + (`matched_fragments`) and serves an over-budget event as an excerpt, and a + hosted write is billed per event, so splitting is used only to stay under + the limit, along the document's structure. diff --git a/docs/architecture/cortex-flows.md b/docs/architecture/cortex-flows.md index eef2b0d5..f7713e21 100644 --- a/docs/architecture/cortex-flows.md +++ b/docs/architecture/cortex-flows.md @@ -152,8 +152,8 @@ Only `Hybrid`. Other modes fail `Error::Unsupported` before any request. ## Recall -Recall builds a pack, asks the answer route **once** with `use_pack_id`, and -cites from the pack. +Recall builds a pack, asks the answer route **once** with `use_pack_id` +(again, from every pack built anew, when the packs were dropped in between, step 5), and cites from the packs. 1. Resolve the scopes: a reach's kind scopes (its node and, when it inherits, every ancestor; below it too for a subtree reach), or, with **no reach** (an @@ -178,7 +178,13 @@ cites from the pack. by default. 5. Ask the answer route with that pack's scope and `use_pack_id`. A response without `answer` text is `Error::Engine`. `model` is - `diagnostics.answer_model`. + `diagnostics.answer_model`. A pack lives 60 s, and CortexDB drops every + pack it holds once anything is forgotten (measured on 0.10.4: a forget in + an unrelated scope turns the next `use_pack_id` into a 404). On that 404 + the round is repeated from step 1's packs: every scope's pack is built + again and steps 3 to 5 run on the new ones, so the answer and the + citations both come from packs read after the drop. At most three rounds; + a third 404 is returned as `Error::NotFound`. 6. **Citations** come from the packs' decoded events, merged rank by rank (each pack's best first), one per item, the most specific node's first, capped at `limit`, with `score: None` and the diff --git a/docs/architecture/cortex-wire.md b/docs/architecture/cortex-wire.md index cc3f216c..2fde8b86 100644 --- a/docs/architecture/cortex-wire.md +++ b/docs/architecture/cortex-wire.md @@ -199,6 +199,12 @@ ones). Response fields read: `answer` (required, string) and `diagnostics.answer_model` (optional, becomes `RecallAnswer.model`). +A 404 means the pack is gone: packs live 60 s, and CortexDB drops every +pack it holds once anything is forgotten. Every scope's pack is then built +again and the answer asked from the new chosen pack, up to three rounds in +all (see +[recall](cortex-flows.md#recall), step 5). + `answer_instructions` is the request's instructions when set. When unset, Direct sends `null` and TinyHumans **omits the key**: its answer schema is strict (an unknown key, or a `null` instructions, is a 400). @@ -266,7 +272,7 @@ The beliefs land in a derived layer, read two ways: ```json { "scope": "…", "query": "…", - "budgets": { "max_tokens": 786432, + "budgets": { "max_tokens": 6291456, "per_layer_limits": { "events": 0, "facts": 0, "episodes": 0, "understanding": 0, "beliefs": 8 } } } ``` @@ -359,58 +365,11 @@ double enforces). parent-scope sample. - A filter whose `kinds` admits nothing reads no scopes. -## Chunked documents - -CortexDB refuses an experience whose flattened text is over 1 MiB -(`422 INVALID_ENVELOPE`, 0.10.4 API §6.10), and a document's event text is -its whole envelope. So (`envelope/chunks.rs`): - -- **When.** Measured as a piece would be written (the envelope with its - `chunk` field): a document whose encoded envelope fits in - `DOCUMENT_CHUNK_TARGET_BYTES` (256 KiB) is one event, byte-identical to an - unchunked one (no `chunk` field). A longer one is split. The target is the - only granularity knob: at `0` every page and every section becomes its own - event, except that a page or section too big for one event is still cut - into several (see Where). -- **Where.** First at page breaks (the form feed the PDF converter puts - between pages), then before markdown heading lines; a stretch of only - whitespace (or page breaks) never becomes a piece of its own but joins the - unit next to it, except in a text that is nothing else, which is then one - piece, so every byte is kept. These units are packed greedily, in order, - up to the target. A unit over the target is cut at blank lines, then line - ends, then characters. Sizes are JSON-escaped bytes plus the envelope - around the piece (its metadata and the `chunk` field at full width, with - a page range reserved only when the document marks pages). -- **Limit.** No event over `MAX_EVENT_TEXT_BYTES` (768 KiB of encoded - envelope, a quarter under the server's limit) is ever sent: every event of - a batch is encoded and checked before the first write, and an item over it - (a learning or a conversation turn that long) is `Error::InvalidRequest`. -- **Identity and replay.** Every piece carries the item's id and label, so - replay detection, `forget` by id or filter, and `get` see all of them; a - store that failed part way writes only the missing pieces. Until then - `get` and `list` do not return the document (never a truncated body); - `fetch` and `recall` still hit the pieces that are there. -- **Reads.** `get` and `list` give the whole document (pieces in index - order). `fetch` and `recall` give one hit or citation per document, as for - every item: its best-ranked piece, with the - item's id and its metadata plus, when known, a `page:<n>` (or - `page:<first>-<last>`) tag and a `section:<title>` tag: a document without - page breaks gets no page tag, a piece before the first heading no section - tag. These tags are read-side metadata, not part of the item's identity. - `pages` is an inclusive range `[first, last]`, counted from 1, so a piece - on one page has `first == last`. Readable CortexDB labels for page and - section are not written yet. -- **Why one piece per target rather than per page.** CortexDB 0.10.4 - already fragments every event over about 500 bytes for retrieval - (`matched_fragments`) and serves an over-budget event as an excerpt, and a - hosted write is billed per event, so splitting is used only to stay under - the limit, along the document's structure. - ## The v2 envelope A learning is one event; a conversation is one event per turn, appended in order. A document is one event unless its envelope would pass 256 KiB; then it -is one event per piece (see [Chunked documents](#chunked-documents)). +is one event per piece (see [cortex-chunks.md](cortex-chunks.md)). CortexDB's experience schema is closed (an unknown field is a 422), so the structured data rides in the one free-form field: the event's `content.text` is a JSON **envelope**: diff --git a/docs/specs/memory-v2.md b/docs/specs/memory-v2.md index 87ead2e7..4659d95f 100644 --- a/docs/specs/memory-v2.md +++ b/docs/specs/memory-v2.md @@ -237,7 +237,7 @@ credentialed cleartext non-loopback endpoint, all as `Error::Config`. - **Wires.** `Direct` (`v1/experience`, `v1/events`, `v1/recall`, `v1/forget`, `v1/answer`) and `TinyHumans` (`memory/*` with `{success,data}` envelopes), as in the v1 adapter. - **Store.** - - Each item becomes one experience: a conversation becomes a bulk append of its turns, and a document whose envelope would pass 256 KiB becomes an ordered append of its pieces, cut at page breaks and headings, and a page or section still over the target cut again at blank lines, then line ends, then characters (`chunk {index, count, pages?, section?}` in the envelope; `pages` an inclusive `[first, last]` range counted from 1, only when the text marks pages; `section` only under a heading). No event over 768 KiB of encoded envelope is sent (CortexDB refuses one over 1 MiB): an item that cannot fit even split (a learning or a turn that long, or a document whose title and metadata leave no room for a piece and which does not fit whole) is `InvalidRequest`, and nothing of its batch is sent. `get`/`list` reassemble a chunked document, returning it only when every piece is present, and a ranked read gives one hit per document, its best-ranked piece, with `page:`/`section:` tags. + - Each item becomes one experience: a conversation becomes a bulk append of its turns, and a document whose encoded envelope as a chunked piece would pass 256 KiB becomes an ordered append of its pieces, cut at page breaks and headings, and a page or section still over the target cut again at blank lines, then line ends, then characters (`chunk {index, count, pages?, section?}` in the envelope; `pages` an inclusive `[first, last]` range counted from 1, only when the text marks pages; `section` only under a heading). No event over 768 KiB of encoded envelope is sent (CortexDB refuses one over 1 MiB): an item that cannot fit even split (a learning or a turn that long, or a document whose title and metadata leave no room for a piece and which does not fit whole) is `InvalidRequest`, and nothing of its batch is sent. `get`/`list` reassemble a chunked document, returning it only when every piece is present, and a ranked read gives one hit per document, its best-ranked piece, with `page:`/`section:` tags. - The envelope carries `{v:2, kind, meta, title?, learning_kind?, confidence?}`, and `meta` maps to scope labels where CortexDB can filter. - Writes wait for the indexed barrier, keeping the v1 `await_readable` behaviour. - **Scope.** One scope per item kind *per namespace node*, under the TinyMemory root `app:tinymemory` (which the hosted backend further roots under the tenant): the root node keeps `app:tinymemory/app:{documents,conversations,learnings}`, and a node adds its segments in between, e.g. `app:tinymemory/team:acme/agent:writer/app:learnings`. Namespace segments map to CortexDB's built-in `agent`, `team`, `user`, `ws` and `project` types and the kind leaf uses `app`, because CortexDB v0.10+ refuses scope types outside the deployment's `allowed_scope_types` (`422 UNREGISTERED_SCOPE_TYPE`); every shipped preset allows all of them. A `MetaFilter`'s `kinds` and `reach` pick the scopes read, each admitted kind at each node: an ordinary reach (`Reach::of`, `inherit: true`) reads its node **and every ancestor**, an exact reach its node alone, and a subtree reach its node and every node below; those nodes are known, and only a subtree reach or an unscoped read discovers nodes, from the registered scopes (`v1/scopes/list` / `memory/scopes`). Reads are always exact (`view: "granular"`, sent explicitly because public recall defaults to `holistic`), never server-side traversal.