diff --git a/WASM-PROPOSAL.md b/WASM-PROPOSAL.md index 08d67ef..0d9dc92 100644 --- a/WASM-PROPOSAL.md +++ b/WASM-PROPOSAL.md @@ -607,13 +607,19 @@ call cost. Exit: numbers published in this document, all tests and reviews pass. > **Status: HARNESS BASELINE PASS (2026-06-12).** `spikes/wasm-phase5/` > now automates the Phase 5 parity/performance gate shape before the production > wasm worker exists. On k-uce it ran the full native network suite (`83/83`), -> the starter-focused parity subset (`14/14`, non-vacuous), a heuristic `site/` +> the starter-focused parity subset (`14/14`), a heuristic code-focused `site/` > cross-request/static-state audit, and a warmed native benchmark baseline for > the template-heavy doc page, sqlite page, and component-heavy starter page. -> Reports were written under `/tmp/uce/wasm-phase5/`, and the harness ended with -> `PHASE5 HARNESS: PASS`. True Phase 5 completion still requires passing the -> same harness against a real wasm worker URL and adding worker-internal probes -> for workspace birth and component-call overhead budgets. +> The harness now gates case counts (`network >= 80`, `starter >= 10`) so broken +> filters cannot pass vacuously, and it runs a throwaway warmup suite before the +> measured gate to absorb cold unit-cache timeouts. Informational native medians +> from the 2026-06-12 baseline are: template-heavy doc `313.8 ms`, sqlite page +> `3.4 ms`, starter dashboard `41.1 ms`. A durable snapshot lives in +> `spikes/wasm-phase5/reports/native-baseline-2026-06-12.md`; paired wasm/native +> gate runs still recompute the native medians for the actual ≤2× comparison. +> True Phase 5 completion still requires passing the same harness against a real +> wasm worker URL and adding worker-internal probes for workspace birth and +> component-call overhead budgets. **Phase 6 — second plane (deferred until wanted).** Cross-instance call mechanism (props-in/output-out, UCEB1), first Plane B diff --git a/spikes/wasm-phase5/README.md b/spikes/wasm-phase5/README.md index 212cd0b..45bb459 100644 --- a/spikes/wasm-phase5/README.md +++ b/spikes/wasm-phase5/README.md @@ -17,15 +17,27 @@ Expected final line: PHASE5 HARNESS: PASS ``` +The harness first runs a throwaway warmup pass to populate the unit cache after +binary rebuilds. The warmup excludes the stateful `site tests tasks` case so it +cannot perturb the measured task-lifecycle check. The measured full-suite and +starter-subset passes then become the gate. + Artifacts are written under `/tmp/uce/wasm-phase5/`: -- `native-network.json` — full native network suite result. -- `native-starter.json` — starter-focused parity subset; this protects against - a vacuous `--match starter` exit gate. +- `native-network-warmup.json` — throwaway warmup result; allowed to fail on cold + compile timeouts. +- `native-network.json` — measured full native network suite result. The harness + fails if fewer than 80 cases run, so an empty or broken suite cannot pass. +- `native-starter.json` — measured starter-focused parity subset. The harness + fails if fewer than 10 cases run, which protects against a vacuous + `--match starter` exit gate. - `site-static-audit.{json,md}` — candidate cross-request/static-state risks in - `site/` for the §3.2 semantic change audit. + `site/` for the §3.2 semantic change audit. By default it scans code-like + `.uce` and `.h` files only; pass `--include-doc-text` to include `.txt` docs + as `documentation` severity findings. - `benchmark.{json,md}` — warmed native baseline for the three Phase 5 budget - pages: + pages. The default is 20 samples per target; use a higher `--samples` value for + noisy shared-host gate runs if needed: - `template-heavy-doc`: `/doc/singlepage.uce` - `sqlite-page`: `/demo/sqlite.uce` - `component-heavy-starter`: `/examples/uce-starter/?dashboard` @@ -36,6 +48,9 @@ page latency. Workspace birth and internal component call overhead budgets still need worker-internal probes; this harness documents the gap rather than faking those numbers. +A durable informational baseline snapshot is kept in `reports/`. Paired +native/WASM runs still recompute native medians for the actual budget decision. + Current scope: - Native parity and baseline collection are automated. diff --git a/spikes/wasm-phase5/audit_site_statics.py b/spikes/wasm-phase5/audit_site_statics.py index 069ccd4..e903501 100755 --- a/spikes/wasm-phase5/audit_site_statics.py +++ b/spikes/wasm-phase5/audit_site_statics.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 -"""Phase 5 audit for cross-request/static state risks in site files.""" +"""Phase 5 audit for cross-request/static state risks in site code files.""" from __future__ import annotations +import argparse import json from dataclasses import asdict, dataclass from pathlib import Path @@ -17,6 +18,8 @@ PATTERNS = [ ] SKIP_PARTS = {".git", "tmp", "work", "bin", "pkg", "__pycache__"} +CODE_SUFFIXES = {".uce", ".h"} +DOC_SUFFIXES = {".txt"} @dataclass @@ -24,24 +27,28 @@ class Finding: path: str line: int kind: str + severity: str text: str note: str -def iter_files() -> list[Path]: +def iter_files(include_doc_text: bool) -> list[Path]: + suffixes = CODE_SUFFIXES | (DOC_SUFFIXES if include_doc_text else set()) result: list[Path] = [] for path in SITE.rglob("*"): if not path.is_file(): continue if any(part in SKIP_PARTS for part in path.parts): continue - if path.suffix not in {".uce", ".h", ".txt"}: + if path.suffix not in suffixes: continue result.append(path) return sorted(result) -def note_for(kind: str, text: str) -> str: +def note_for(kind: str, severity: str) -> str: + if severity == "documentation": + return "Documentation prose mention; useful for terminology review, not a direct static-state migration finding." if kind in {"once hook", "init hook"}: return "Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state." if kind == "background task": @@ -49,16 +56,21 @@ def note_for(kind: str, text: str) -> str: return "Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace." -def scan() -> list[Finding]: +def severity_for(path: Path) -> str: + return "documentation" if path.suffix in DOC_SUFFIXES else "code" + + +def scan(include_doc_text: bool) -> list[Finding]: findings: list[Finding] = [] - for path in iter_files(): + for path in iter_files(include_doc_text): try: lines = path.read_text(encoding="utf-8").splitlines() except UnicodeDecodeError: continue + severity = severity_for(path) for lineno, line in enumerate(lines, 1): stripped = line.strip() - if stripped.startswith("//") or stripped.startswith("# "): + if severity == "code" and (stripped.startswith("//") or stripped.startswith("# ")): continue for kind, needle in PATTERNS: if needle in line: @@ -66,8 +78,9 @@ def scan() -> list[Finding]: path=str(path.relative_to(ROOT)), line=lineno, kind=kind, + severity=severity, text=stripped[:180], - note=note_for(kind, stripped), + note=note_for(kind, severity), )) return findings @@ -79,17 +92,25 @@ def write_reports(findings: list[Finding], out_dir: Path) -> None: if not findings: lines.append("No candidate cross-request/static-state patterns found.") else: - lines.extend(["| file | line | kind | code | note |", "|---|---:|---|---|---|"]) + code_count = sum(1 for f in findings if f.severity == "code") + doc_count = sum(1 for f in findings if f.severity == "documentation") + lines.append(f"Findings: {code_count} code, {doc_count} documentation prose.") + lines.append("") + lines.extend(["| file | line | severity | kind | code | note |", "|---|---:|---|---|---|---|"]) for f in findings: code = f.text.replace("|", "\\|") note = f.note.replace("|", "\\|") - lines.append(f"| {f.path} | {f.line} | {f.kind} | `{code}` | {note} |") + lines.append(f"| {f.path} | {f.line} | {f.severity} | {f.kind} | `{code}` | {note} |") (out_dir / "site-static-audit.md").write_text("\n".join(lines) + "\n") def main() -> int: - out_dir = Path("/tmp/uce/wasm-phase5") - findings = scan() + parser = argparse.ArgumentParser(description="Phase 5 site static/cross-request-state audit") + parser.add_argument("--include-doc-text", action="store_true", help="include site/doc .txt prose as documentation-severity findings") + parser.add_argument("--out-dir", default="/tmp/uce/wasm-phase5") + args = parser.parse_args() + out_dir = Path(args.out_dir) + findings = scan(args.include_doc_text) write_reports(findings, out_dir) print(f"Found {len(findings)} candidate static/cross-request patterns") print(f"wrote {out_dir / 'site-static-audit.json'} and {out_dir / 'site-static-audit.md'}") diff --git a/spikes/wasm-phase5/benchmark.py b/spikes/wasm-phase5/benchmark.py index 379a05a..c7750a7 100755 --- a/spikes/wasm-phase5/benchmark.py +++ b/spikes/wasm-phase5/benchmark.py @@ -63,9 +63,6 @@ def request_once(base_url: str, host_header: str, path: str, timeout: float) -> def measure_backend(backend: str, base_url: str, host_header: str, targets: list[Target], warmups: int, samples: int, timeout: float) -> list[Measurement]: results: list[Measurement] = [] for target in targets: - path = urlparse(target.url).path - if "?" in target.url: - path = target.url for _ in range(warmups): request_once(base_url, host_header, target.url, timeout) durations: list[float] = [] @@ -127,7 +124,7 @@ def main() -> int: parser.add_argument("--wasm-base-url", default="", help="optional wasm worker URL; omitted until worker exists") parser.add_argument("--host-header", default="uce.openfu.com") parser.add_argument("--warmups", type=int, default=2) - parser.add_argument("--samples", type=int, default=5) + parser.add_argument("--samples", type=int, default=20) parser.add_argument("--timeout", type=float, default=10.0) parser.add_argument("--out-dir", default="/tmp/uce/wasm-phase5") args = parser.parse_args() diff --git a/spikes/wasm-phase5/reports/native-baseline-2026-06-12.json b/spikes/wasm-phase5/reports/native-baseline-2026-06-12.json new file mode 100644 index 0000000..28ad94c --- /dev/null +++ b/spikes/wasm-phase5/reports/native-baseline-2026-06-12.json @@ -0,0 +1,104 @@ +[ + { + "backend": "native", + "target": "template-heavy-doc", + "url": "/doc/singlepage.uce", + "ok": true, + "status": 200, + "samples_ms": [ + 318.72209906578064, + 340.9293442964554, + 319.705568253994, + 324.0259513258934, + 350.93583166599274, + 313.20811808109283, + 317.0944079756737, + 310.312956571579, + 307.89367109537125, + 308.1864267587662, + 311.681292951107, + 313.39504569768906, + 314.3857270479202, + 312.5988021492958, + 313.25943768024445, + 312.9996135830879, + 334.4864323735237, + 314.2518773674965, + 309.7490146756172, + 317.4726217985153 + ], + "median_ms": 313.8234615325928, + "mean_ms": 318.2647120207548, + "min_ms": 307.89367109537125, + "max_ms": 350.93583166599274, + "note": "" + }, + { + "backend": "native", + "target": "sqlite-page", + "url": "/demo/sqlite.uce", + "ok": true, + "status": 200, + "samples_ms": [ + 3.4144148230552673, + 6.088584661483765, + 3.463640809059143, + 3.498159348964691, + 3.5023540258407593, + 6.214611232280731, + 3.563329577445984, + 3.3720433712005615, + 3.330707550048828, + 3.3808723092079163, + 3.322914242744446, + 3.3655911684036255, + 3.374151885509491, + 3.358304500579834, + 3.3307820558547974, + 3.268897533416748, + 3.360658884048462, + 3.3061057329177856, + 6.233863532543182, + 3.432638943195343 + ], + "median_ms": 3.3775120973587036, + "mean_ms": 3.809131309390068, + "min_ms": 3.268897533416748, + "max_ms": 6.233863532543182, + "note": "" + }, + { + "backend": "native", + "target": "component-heavy-starter", + "url": "/examples/uce-starter/?dashboard", + "ok": true, + "status": 200, + "samples_ms": [ + 74.01428371667862, + 40.9795418381691, + 40.995217859745026, + 41.04568809270859, + 40.724173188209534, + 41.10313206911087, + 40.59913754463196, + 41.28593951463699, + 41.10439121723175, + 40.649913251399994, + 42.06441342830658, + 42.074643075466156, + 43.475523591041565, + 41.8255478143692, + 41.068583726882935, + 41.116394102573395, + 41.09777510166168, + 42.58237034082413, + 41.312023997306824, + 70.98117470741272 + ], + "median_ms": 41.11039265990257, + "mean_ms": 44.50499340891838, + "min_ms": 40.59913754463196, + "max_ms": 74.01428371667862, + "note": "" + } +] diff --git a/spikes/wasm-phase5/reports/native-baseline-2026-06-12.md b/spikes/wasm-phase5/reports/native-baseline-2026-06-12.md new file mode 100644 index 0000000..413facc --- /dev/null +++ b/spikes/wasm-phase5/reports/native-baseline-2026-06-12.md @@ -0,0 +1,19 @@ +# Phase 5 native baseline — 2026-06-12 + +Host: `k-uce` / `uce.openfu.com` via localhost with Host header. + +This is a durable informational snapshot. The Phase 5 gate recomputes native medians during paired native/WASM runs; these numbers are not hard-coded budgets. + +- Warmup suite: 82/82 passed; excludes `site tests tasks` to avoid perturbing task lifecycle state. +- Measured full network suite: 83/83 passed. +- Measured starter subset: 14/14 passed. +- Static audit default scan: 50 code findings, documentation prose excluded by default. + +| target | median ms | mean ms | samples | +|---|---:|---:|---:| +| template-heavy-doc | 313.8 | 318.3 | 20 | +| sqlite-page | 3.4 | 3.8 | 20 | +| component-heavy-starter | 41.1 | 44.5 | 20 | + +Raw benchmark JSON: `native-baseline-2026-06-12.json`. +Static audit snapshot: `site-static-audit-2026-06-12.md`. diff --git a/spikes/wasm-phase5/reports/site-static-audit-2026-06-12.md b/spikes/wasm-phase5/reports/site-static-audit-2026-06-12.md new file mode 100644 index 0000000..a199d6f --- /dev/null +++ b/spikes/wasm-phase5/reports/site-static-audit-2026-06-12.md @@ -0,0 +1,56 @@ +# Phase 5 site static-state audit + +Findings: 50 code, 0 documentation prose. + +| file | line | severity | kind | code | note | +|---|---:|---|---|---|---| +| site/demo/index.uce | 76 | code | background task | `` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/once-init.uce | 1 | code | static local/global | `static s64 demo_worker_init_count = 0;` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. | +| site/demo/once-init.uce | 2 | code | static local/global | `static s64 demo_component_hits = 0;` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. | +| site/demo/once-init.uce | 4 | code | init hook | `INIT(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. | +| site/demo/once-init.uce | 10 | code | once hook | `ONCE(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. | +| site/demo/once-init.uce | 44 | code | once hook | `ONCE() and INIT()` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. | +| site/demo/once-init.uce | 44 | code | init hook | `ONCE() and INIT()` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. | +| site/demo/once-init.uce | 47 | code | once hook | `This page calls the same named component twice. `ONCE()` should only run once for the request, while `INIT()` should stay stable for the currently loaded worker copy.` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. | +| site/demo/once-init.uce | 47 | code | init hook | `This page calls the same named component twice. `ONCE()` should only run once for the request, while `INIT()` should stay stable for the currently loaded worker copy.` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. | +| site/demo/task-status.uce | 11 | code | background task | `String task_name = first(context.get["task-name"], "example-task");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task-status.uce | 13 | code | background task | `print("Task Name: ", task_name, "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task-status.uce | 14 | code | background task | `print("Task ID: ", task_pid(task_name), "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task-status.uce | 15 | code | background task | `print("Task Running: ", task_pid(task_name) == 0 ? "no" : "yes", "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task.uce | 13 | code | background task | `String task_name = first(context.get["task-name"], "example-task");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task.uce | 37 | code | background task | `` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task.uce | 46 | code | background task | `load(document.getElementById('task-status'), 'task-status.uce?task-name=');` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task.uce | 56 | code | background task | `print("Task Name: ", task_name, "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task.uce | 57 | code | background task | `print("Task ID: ", task_pid(task_name), "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task.uce | 58 | code | background task | `print("Task Running: ", task_pid(task_name) == 0 ? "no" : "yes", "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task.uce | 71 | code | background task | `print("New Task ID: ", task(task_name, []() {` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task_repeat.uce | 13 | code | background task | `String task_name = first(context.get["task-name"], "example-task");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task_repeat.uce | 37 | code | background task | `` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task_repeat.uce | 46 | code | background task | `load(document.getElementById('task-status'), 'task-status.uce?task-name=');` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task_repeat.uce | 56 | code | background task | `print("Task Name: ", task_name, "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task_repeat.uce | 57 | code | background task | `print("Task ID: ", task_pid(task_name), "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task_repeat.uce | 58 | code | background task | `print("Task Running: ", task_pid(task_name) == 0 ? "no" : "yes", "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/demo/task_repeat.uce | 71 | code | background task | `print("New Task ID: ", task_repeat(task_name, 5, []() {` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/examples/uce-starter/components/data/widgets.uce | 3 | code | once hook | `ONCE(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. | +| site/examples/uce-starter/components/workspace/primitives.uce | 3 | code | once hook | `ONCE(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. | +| site/examples/uce-starter/lib/user.class.h | 13 | code | static local/global | `static String session_key()` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. | +| site/examples/uce-starter/lib/user.class.h | 18 | code | static local/global | `static String normalize_email(String email)` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. | +| site/examples/uce-starter/lib/user.class.h | 23 | code | static local/global | `static String hash_id(String raw)` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. | +| site/examples/uce-starter/lib/user.class.h | 45 | code | static local/global | `static String password_hash(String password, String salt)` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. | +| site/examples/uce-starter/lib/user.class.h | 53 | code | static local/global | `static DValue read_json_file(String file_name)` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. | +| site/examples/uce-starter/lib/user.class.h | 64 | code | static local/global | `static bool write_json_file(String file_name, DValue data)` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. | +| site/examples/uce-starter/views/dashboard.uce | 3 | code | once hook | `ONCE(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. | +| site/info/index.uce | 269 | code | static local/global | `
  • nginx serves static files directly from `site/`
  • ` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. | +| site/tests/preprocessor.uce | 3 | code | once hook | `ONCE(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. | +| site/tests/tasks.uce | 25 | code | background task | `pid_t repeat_existing = task_pid("site-tests-repeat");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 27 | code | background task | `task_kill(repeat_existing, 15);` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 37 | code | background task | `repeat_pid = task_repeat("site-tests-repeat", 1.0, []() {` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 55 | code | background task | `pid_t seen_short_pid = task_pid("site-tests-short");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 56 | code | background task | `pid_t seen_repeat_pid = task_pid("site-tests-repeat");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 57 | code | background task | `pid_t seen_timeout_pid = task_pid("site-tests-timeout");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 58 | code | background task | `pid_t seen_unsafe_key_pid = task_pid("site-tests/../unsafe key");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 59 | code | background task | `int short_alive = seen_short_pid == 0 ? -1 : task_kill(seen_short_pid, 0);` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 60 | code | background task | `int repeat_alive = seen_repeat_pid == 0 ? -1 : task_kill(seen_repeat_pid, 0);` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 72 | code | background task | `check("task_pid() + task_kill(pid, 0)", short_alive == 0, "kill(0) result=" + std::to_string(short_alive));` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 73 | code | background task | `check("task_repeat()", repeat_pid != 0 && seen_repeat_pid != 0, "started=" + std::to_string(repeat_pid) + " seen=" + std::to_string(seen_repeat_pid));` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | +| site/tests/tasks.uce | 77 | code | background task | `check("task_kill() rejects negative pid", task_kill(-1, 0) == -1, "kill(-1, 0) rejected");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. | diff --git a/spikes/wasm-phase5/run_phase5.sh b/spikes/wasm-phase5/run_phase5.sh index 2bcf391..0b8432a 100755 --- a/spikes/wasm-phase5/run_phase5.sh +++ b/spikes/wasm-phase5/run_phase5.sh @@ -7,21 +7,52 @@ cd "$(dirname "$0")/../.." OUT=/tmp/uce/wasm-phase5 mkdir -p "$OUT" -python3 tests/run_network_tests.py --include-internal --json-report "$OUT/native-network.json" -python3 tests/run_network_tests.py --include-internal --match starter --json-report "$OUT/native-starter.json" -python3 spikes/wasm-phase5/audit_site_statics.py -python3 spikes/wasm-phase5/benchmark.py --out-dir "$OUT" +# The network suite can show transient cold-compile timeouts immediately after a +# binary rebuild. Run a throwaway pass first so the measured gate is a warmed +# parity signal rather than a compiler-cache signal. The warmup is allowed to +# fail; the measured pass below is not. +python3 tests/run_network_tests.py --include-internal --exclude 'site tests tasks' --json-report "$OUT/native-network-warmup.json" || true -python3 - <<'PY' +network_status=0 +starter_status=0 +benchmark_status=0 +python3 tests/run_network_tests.py --include-internal --json-report "$OUT/native-network.json" || network_status=$? +python3 tests/run_network_tests.py --include-internal --match starter --json-report "$OUT/native-starter.json" || starter_status=$? +python3 spikes/wasm-phase5/audit_site_statics.py --out-dir "$OUT" +python3 spikes/wasm-phase5/benchmark.py --out-dir "$OUT" || benchmark_status=$? + +NETWORK_STATUS=$network_status STARTER_STATUS=$starter_status BENCHMARK_STATUS=$benchmark_status python3 - <<'PY' import json +import os from pathlib import Path + +MIN_NETWORK_CASES = 80 +MIN_STARTER_CASES = 10 +MIN_BENCHMARKS = 3 out = Path('/tmp/uce/wasm-phase5') network = json.loads((out / 'native-network.json').read_text()) starter = json.loads((out / 'native-starter.json').read_text()) bench = json.loads((out / 'benchmark.json').read_text()) failures = [r for r in network if not r['ok']] + [r for r in starter if not r['ok']] + [r for r in bench if not r['ok']] -if failures: +structural_failures = [] +if len(network) < MIN_NETWORK_CASES: + structural_failures.append(f"network suite ran {len(network)} cases, expected at least {MIN_NETWORK_CASES}") +if len(starter) < MIN_STARTER_CASES: + structural_failures.append(f"starter subset ran {len(starter)} cases, expected at least {MIN_STARTER_CASES}") +if len(bench) < MIN_BENCHMARKS: + structural_failures.append(f"benchmark ran {len(bench)} rows, expected at least {MIN_BENCHMARKS}") +nonzero = { + 'network_status': int(os.environ['NETWORK_STATUS']), + 'starter_status': int(os.environ['STARTER_STATUS']), + 'benchmark_status': int(os.environ['BENCHMARK_STATUS']), +} +for name, status in nonzero.items(): + if status != 0: + structural_failures.append(f"{name} exited {status}") +if failures or structural_failures: print('PHASE5 HARNESS: FAIL') + for failure in structural_failures: + print(failure) for failure in failures: print(failure) raise SystemExit(1) diff --git a/tests/run_network_tests.py b/tests/run_network_tests.py index 9a19037..a35d0ee 100755 --- a/tests/run_network_tests.py +++ b/tests/run_network_tests.py @@ -222,6 +222,7 @@ def filter_cases( name_pattern: Optional[str], tags: List[str], include_internal: bool, + excluded_names: List[str], ) -> List[NetworkTestCase]: filtered = list(cases) if plugins: @@ -233,6 +234,9 @@ def filter_cases( filtered = [case for case in filtered if matcher.search(case.name)] if tags: filtered = [case for case in filtered if any(tag in case.tags for tag in tags)] + if excluded_names: + excluded = set(excluded_names) + filtered = [case for case in filtered if case.name not in excluded] return filtered @@ -273,6 +277,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--plugin-dir", default=str(base_dir / "plugins"), help="directory containing plugin files") parser.add_argument("--plugin", action="append", default=[], help="limit to one or more plugin basenames") parser.add_argument("--match", help="regex filter for case names") + parser.add_argument("--exclude", action="append", default=[], help="exclude an exact test case name after other filters") parser.add_argument("--tag", action="append", default=[], help="only run cases with at least one matching tag") parser.add_argument("--include-internal", action="store_true", help="include tests tagged internal in the run") parser.add_argument("--list", action="store_true", help="list discovered tests without running them") @@ -316,7 +321,7 @@ def main(argv: Optional[List[str]] = None) -> int: for plugin_path in plugin_files: register_plugin_cases(registry, plugin_path) - cases = filter_cases(registry.cases, args.plugin, args.match, args.tag, args.include_internal) + cases = filter_cases(registry.cases, args.plugin, args.match, args.tag, args.include_internal, args.exclude) if not cases: print("No tests matched the selected filters", file=sys.stderr) return 2