Expose ID constants and improve generated stdlib API docs

.github/workflows/docgen.yml

@@ -36,6 +36,10 @@ jobs:
         with:
           deno-version: v2.x
 
+      - name: Test doc generator
+        working-directory: tools/docgen
+        run: deno task test
+
       - name: Generate reference
         working-directory: tools/docgen
         run: deno task gen --src "$GITHUB_WORKSPACE/_stdlib/wurst" --out "$GITHUB_WORKSPACE"

tools/docgen/README.md

@@ -26,6 +26,8 @@ deno task gen
 deno task gen --src /path/to/WurstStdlib2/wurst --out ./_scratch
 # parse-only coverage report, writes nothing:
 deno task check
+# regression tests for constant values, constructor docs, and rawcode links:
+deno task test
 ```
 
 ## How parsing works
@@ -35,6 +37,12 @@ declaration (`function` / `class` / `interface` / `enum` / `tuple` / `module` /
 Wurst enforces indented blocks, so there is no nesting to worry about. A line-based pass is
 enough. See `parser.ts` for the details and `types.ts` for the data model.
 
+Both tab and four-space indentation are supported. Public class constants retain their
+initializer values and documentation in the member list, alongside constructors and methods.
+Class constants have stable anchors such as `#abilityids-blizzard`. Compact generated rawcode
+comments (`'AHbz' / AbilityIds.blizzard`) link to those anchors when the referenced constant is
+part of the generated API. Code examples and unknown references are left unchanged.
+
 A package's summary is, in order of preference: a doc block directly above the `package` line;
 the doc of the entity whose name matches the package (e.g. `HashMap`); or the first multi-line
 narrative block in the file.

tools/docgen/api_test.ts

@@ -0,0 +1,89 @@
+import { parseFile } from "./parser.ts";
+import { renderPackagePage } from "./emit.ts";
+import { hotdocToMarkdown } from "./hotdoc.ts";
+
+function expect(value: boolean, message: string) {
+  if (!value) throw new Error(message);
+}
+
+Deno.test("indented hotdoc examples preserve relative code indentation", () => {
+  const result = hotdocToMarkdown(
+    "    > @compiletime function createMyUnit()\n" +
+      "    >     new UnitDefinition(UNIT_ID_GEN.next(), UnitIds.footman)\n" +
+      '    >         ..setName("My Footman")',
+  );
+  expect(
+    result === "```wurst\n@compiletime function createMyUnit()\n" +
+        "    new UnitDefinition(UNIT_ID_GEN.next(), UnitIds.footman)\n" +
+        '        ..setName("My Footman")\n```',
+    "Example contains comment indentation",
+  );
+});
+
+Deno.test("class constants retain values and docs while constructors retain their own docs", () => {
+  const pkg = parseFile({
+    category: "_wurst/assets",
+    sourcePath: "wurst/_wurst/assets/AbilityIds.wurst",
+    text: `package AbilityIds
+public class AbilityIds
+    /** Blizzard's base rawcode. */
+    static constant blizzard = 'AHbz'
+    private static constant hidden = 'XXXX'
+    /** Choose a base ID. */
+    construct(int baseId)
+    function setCooldown(real value)
+        constant localConstant = 1
+`,
+  });
+  const members = pkg.entities[0].members;
+  const constant = members.find((m) => m.name === "blizzard");
+  expect(constant?.signature === "static constant blizzard = 'AHbz'", "Missing constant value");
+  expect(constant?.doc === "Blizzard's base rawcode.", "Constant doc is detached");
+  expect(!members.some((m) => m.name === "hidden"), "Private constant leaked");
+  expect(!members.some((m) => m.name === "localConstant"), "Function local leaked into class API");
+  expect(
+    members.find((m) => m.name === "construct")?.doc === "Choose a base ID.",
+    "Constructor doc is detached",
+  );
+  expect(
+    members.find((m) => m.name === "setCooldown")?.doc === "",
+    "Constructor doc leaked onto method",
+  );
+  const page = renderPackagePage(pkg, { outDir: "", curated: new Map(), includes: new Set() });
+  expect(page.includes("static constant blizzard = 'AHbz'"), "Rendered docs omit rawcode");
+  expect(page.includes("Choose a base ID."), "Rendered docs omit constructor documentation");
+  expect(page.includes('id="abilityids-blizzard"'), "Constant has no link target");
+});
+
+Deno.test("compact rawcode docs link to known constants without changing code examples", () => {
+  const pkg = parseFile({
+    category: "objediting",
+    sourcePath: "wurst/objediting/AbilityObjEditing.wurst",
+    text: `package AbilityObjEditing
+/** 'AHbz' / AbilityIds.blizzard */
+public class AbilityDefinitionArchMageBlizzard
+    /** > new AbilityDefinitionArchMageBlizzard(AbilityIds.blizzard) */
+    construct(int newId)
+`,
+  });
+  const href = "/stdlib/ref/_wurst/assets/AbilityIds.html#abilityids-blizzard";
+  const page = renderPackagePage(pkg, {
+    outDir: "",
+    curated: new Map(),
+    includes: new Set(),
+    referenceLinks: new Map([["AbilityIds.blizzard", href]]),
+  });
+  expect(
+    page.includes(`'AHbz' / [AbilityIds.blizzard](${href})`),
+    "Rawcode reference is not linked",
+  );
+  expect(
+    page.includes("new AbilityDefinitionArchMageBlizzard(AbilityIds.blizzard)"),
+    "Code example changed",
+  );
+  const unlinked = renderPackagePage(pkg, { outDir: "", curated: new Map(), includes: new Set() });
+  expect(
+    unlinked.includes("'AHbz' / AbilityIds.blizzard"),
+    "Unknown references must remain readable",
+  );
+});

tools/docgen/deno.json

@@ -1,5 +1,6 @@
 {
   "tasks": {
+    "test": "deno test api_test.ts",
     "gen": "deno run --allow-read --allow-write main.ts",
     "check": "deno run --allow-read main.ts --check"
   },

tools/docgen/emit.ts

@@ -11,12 +11,28 @@ export interface EmitContext {
   curated: Map<string, string>;
   /** Package names that have a _includes/stdlib_curated/<Package>.md transclude. */
   includes: Set<string>;
+  /** Qualified class constant names -> API reference anchors. */
+  referenceLinks?: Map<string, string>;
 }
 
 const REF_DIR = ["_doc", "stdlib", "ref"];
 
 export async function emitAll(packages: PackageDoc[], ctx: EmitContext): Promise<string[]> {
   const written: string[] = [];
+  const referenceLinks = new Map<string, string>();
+  for (const pkg of packages) {
+    for (const entity of pkg.entities) {
+      for (const member of entity.members.filter((m) => m.kind === "constant")) {
+        referenceLinks.set(
+          `${entity.name}.${member.name}`,
+          `/stdlib/ref/${pkg.category.replace(/^\./, "root")}/${pkg.package}.html#${
+            constantAnchor(entity.name, member.name)
+          }`,
+        );
+      }
+    }
+  }
+  ctx = { ...ctx, referenceLinks };
 
   // 1. JSON index.
   const jsonPath = join(ctx.outDir, "_data", "stdlib_index.json");
@@ -62,7 +78,10 @@ export function renderPackagePage(pkg: PackageDoc, ctx: EmitContext): string {
   if (pkg.summary) body.push(hotdocToMarkdown(pkg.summary), "");
   body.push(`**[Source on GitHub](${pkg.githubUrl})**`, "");
   if (curatedPath) {
-    body.push(`> 📖 Read the **[detailed guide](${curatedPath})** for hand-written examples and background.`, "");
+    body.push(
+      `> 📖 Read the **[detailed guide](${curatedPath})** for hand-written examples and background.`,
+      "",
+    );
   }
   if (ctx.includes.has(pkg.package)) {
     body.push(`{% include stdlib_curated/${pkg.package}.md %}`, "");
@@ -73,7 +92,7 @@ export function renderPackagePage(pkg: PackageDoc, ctx: EmitContext): string {
     body.push(`**Re-exports:** ${links}`, "");
   }
 
-  body.push(...renderEntities(pkg.entities));
+  body.push(...renderEntities(pkg.entities, ctx));
 
   return frontmatter(fm) + body.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
 }
@@ -94,18 +113,18 @@ const GROUPS: KindGroup[] = [
   { heading: "Constants", kinds: ["constant"] },
 ];
 
-function renderEntities(entities: Entity[]): string[] {
+function renderEntities(entities: Entity[], ctx: EmitContext): string[] {
   const out: string[] = [];
   for (const group of GROUPS) {
     const items = entities.filter((e) => group.kinds.includes(e.kind));
     if (items.length === 0) continue;
     out.push(`## ${group.heading}`, "");
-    for (const e of items) out.push(...renderEntity(e));
+    for (const e of items) out.push(...renderEntity(e, ctx));
   }
   return out;
 }
 
-function renderEntity(e: Entity): string[] {
+function renderEntity(e: Entity, ctx: EmitContext): string[] {
   const out: string[] = [];
   const title = e.receiver ? `${e.receiver}.${e.name}` : e.name;
   out.push(`### ${title}`, "");
@@ -116,21 +135,34 @@ function renderEntity(e: Entity): string[] {
   if (e.configurable) {
     out.push(`> 🔧 **Configurable.** Override it in your map's config package.`, "");
   }
-  if (e.doc) out.push(hotdocToMarkdown(e.doc), "");
+  if (e.doc) {
+    // The generated rawcode comment is a prose reference, not an executable code example.
+    const reference = e.doc.trim().match(/^'([^']{4})' \/ (\w+\.\w+)$/);
+    const href = reference ? ctx.referenceLinks?.get(reference[2]) : undefined;
+    out.push(
+      href ? `'${reference![1]}' / [${reference![2]}](${href})` : hotdocToMarkdown(e.doc),
+      "",
+    );
+  }
   if (e.enumMembers.length > 0) {
     out.push("**Values:** " + e.enumMembers.map((m) => `\`${m}\``).join(", "), "");
   }
   if (e.members.length > 0) {
     out.push("**Members:**", "");
-    for (const m of e.members) out.push(...renderMember(m));
+    for (const m of e.members) out.push(...renderMember(m, e.name));
     out.push("");
   }
   return out;
 }
 
-function renderMember(m: Entity): string[] {
+function constantAnchor(className: string, memberName: string): string {
+  return `${className.toLowerCase()}-${memberName}`;
+}
+
+function renderMember(m: Entity, className: string): string[] {
   const sig = m.signature.replace(/^function\s+/, "");
-  const head = `- \`${sig}\``;
+  const anchor = m.kind === "constant" ? `<a id="${constantAnchor(className, m.name)}"></a> ` : "";
+  const head = `- ${anchor}\`${sig}\``;
   if (!m.doc && !m.deprecated.flag) return [head];
   const lines: string[] = [head];
   if (m.deprecated.flag) {

tools/docgen/hotdoc.ts

@@ -22,7 +22,13 @@ export function hotdocToMarkdown(doc: string): string {
       }
       while (block.length && block[block.length - 1] === "") block.pop();
       out.push("```wurst");
-      out.push(...block);
+      // Comment indentation is not part of the code example; retain only relative indentation.
+      const nonBlank = block.filter((l) => l.trim() !== "");
+      let prefix = nonBlank[0]?.match(/^\s*/)?.[0] ?? "";
+      for (const line of nonBlank) {
+        while (!line.startsWith(prefix)) prefix = prefix.slice(0, -1);
+      }
+      out.push(...block.map((l) => l.slice(prefix.length)));
       out.push("```");
       continue;
     }

tools/docgen/parser.ts

@@ -169,7 +169,8 @@ export function parseFile(input: ParseInput): PackageDoc {
     };
 
     const isMember = inContainer !== null &&
-      (decl.kind === "function" || decl.kind === "extension-function");
+      indent === inContainer.indent + 1 &&
+      (decl.kind === "function" || decl.kind === "extension-function" || decl.kind === "constant");
     const skipDecl = annos.skip;
 
     // Reset per-declaration state up front; container/enum reads advance `i` themselves.
@@ -271,7 +272,13 @@ function matchDeclaration(rest: string): DeclMatch | null {
     return { kind: "tuple", name: m[1], typeParams: "", receiver: null, hasParens: true };
   }
   if (s.startsWith("constant ")) {
-    return { kind: "constant", name: constantName(s), typeParams: "", receiver: null, hasParens: false };
+    return {
+      kind: "constant",
+      name: constantName(s),
+      typeParams: "",
+      receiver: null,
+      hasParens: false,
+    };
   }
   // Extension function: function <Recv>.<name>(...)
   if ((m = s.match(/^function\s+([A-Za-z_]\w*(?:<[^>]*>)?)\.([A-Za-z_]\w*)(<[^>]*>)?\s*\(/))) {
@@ -285,7 +292,13 @@ function matchDeclaration(rest: string): DeclMatch | null {
   }
   // Normal function.
   if ((m = s.match(/^function\s+([A-Za-z_]\w*)(<[^>]*>)?\s*\(/))) {
-    return { kind: "function", name: m[1], typeParams: m[2] ?? "", receiver: null, hasParens: true };
+    return {
+      kind: "function",
+      name: m[1],
+      typeParams: m[2] ?? "",
+      receiver: null,
+      hasParens: true,
+    };
   }
   // Constructor (only meaningful inside a container; caller decides).
   if (/^construct\s*\(/.test(s)) {
@@ -458,9 +471,13 @@ function resolveSummary(prePackage: string | null, entities: Entity[], pkgName:
 // --- small helpers -----------------------------------------------------------
 
 function leadingTabs(s: string): number {
-  let n = 0;
-  while (n < s.length && s[n] === "\t") n++;
-  return n;
+  let columns = 0;
+  for (const char of s) {
+    if (char === "\t") columns += 4;
+    else if (char === " ") columns++;
+    else break;
+  }
+  return Math.floor(columns / 4);
 }
 
 function parensBalanced(s: string): boolean {

tools/docgen/types.ts

@@ -29,7 +29,7 @@ export interface Entity {
   doc: string;
   deprecated: Deprecation;
   configurable: boolean;
-  /** Public methods/constructors of a class/interface/module; empty otherwise. */
+  /** Public methods, constructors, and constants of a class/interface/module; empty otherwise. */
   members: Entity[];
   /** Enum case names (inline comments stripped); empty for non-enums. */
   enumMembers: string[];