fix: harden WASM phase 5 harness

This commit is contained in:
udo 2026-06-12 20:13:07 +00:00
parent 961df2d542
commit 89b5499c8f
9 changed files with 287 additions and 33 deletions

View File

@ -607,13 +607,19 @@ call cost. Exit: numbers published in this document, all tests and reviews pass.
> **Status: HARNESS BASELINE PASS (2026-06-12).** `spikes/wasm-phase5/`
> now automates the Phase 5 parity/performance gate shape before the production
> wasm worker exists. On k-uce it ran the full native network suite (`83/83`),
> the starter-focused parity subset (`14/14`, non-vacuous), a heuristic `site/`
> the starter-focused parity subset (`14/14`), a heuristic code-focused `site/`
> cross-request/static-state audit, and a warmed native benchmark baseline for
> the template-heavy doc page, sqlite page, and component-heavy starter page.
> Reports were written under `/tmp/uce/wasm-phase5/`, and the harness ended with
> `PHASE5 HARNESS: PASS`. True Phase 5 completion still requires passing the
> same harness against a real wasm worker URL and adding worker-internal probes
> for workspace birth and component-call overhead budgets.
> The harness now gates case counts (`network >= 80`, `starter >= 10`) so broken
> filters cannot pass vacuously, and it runs a throwaway warmup suite before the
> measured gate to absorb cold unit-cache timeouts. Informational native medians
> from the 2026-06-12 baseline are: template-heavy doc `313.8 ms`, sqlite page
> `3.4 ms`, starter dashboard `41.1 ms`. A durable snapshot lives in
> `spikes/wasm-phase5/reports/native-baseline-2026-06-12.md`; paired wasm/native
> gate runs still recompute the native medians for the actual ≤2× comparison.
> True Phase 5 completion still requires passing the same harness against a real
> wasm worker URL and adding worker-internal probes for workspace birth and
> component-call overhead budgets.
**Phase 6 — second plane (deferred until wanted).**
Cross-instance call mechanism (props-in/output-out, UCEB1), first Plane B

View File

@ -17,15 +17,27 @@ Expected final line:
PHASE5 HARNESS: PASS
```
The harness first runs a throwaway warmup pass to populate the unit cache after
binary rebuilds. The warmup excludes the stateful `site tests tasks` case so it
cannot perturb the measured task-lifecycle check. The measured full-suite and
starter-subset passes then become the gate.
Artifacts are written under `/tmp/uce/wasm-phase5/`:
- `native-network.json` — full native network suite result.
- `native-starter.json` — starter-focused parity subset; this protects against
a vacuous `--match starter` exit gate.
- `native-network-warmup.json` — throwaway warmup result; allowed to fail on cold
compile timeouts.
- `native-network.json` — measured full native network suite result. The harness
fails if fewer than 80 cases run, so an empty or broken suite cannot pass.
- `native-starter.json` — measured starter-focused parity subset. The harness
fails if fewer than 10 cases run, which protects against a vacuous
`--match starter` exit gate.
- `site-static-audit.{json,md}` — candidate cross-request/static-state risks in
`site/` for the §3.2 semantic change audit.
`site/` for the §3.2 semantic change audit. By default it scans code-like
`.uce` and `.h` files only; pass `--include-doc-text` to include `.txt` docs
as `documentation` severity findings.
- `benchmark.{json,md}` — warmed native baseline for the three Phase 5 budget
pages:
pages. The default is 20 samples per target; use a higher `--samples` value for
noisy shared-host gate runs if needed:
- `template-heavy-doc`: `/doc/singlepage.uce`
- `sqlite-page`: `/demo/sqlite.uce`
- `component-heavy-starter`: `/examples/uce-starter/?dashboard`
@ -36,6 +48,9 @@ page latency. Workspace birth and internal component call overhead budgets still
need worker-internal probes; this harness documents the gap rather than faking
those numbers.
A durable informational baseline snapshot is kept in `reports/`. Paired
native/WASM runs still recompute native medians for the actual budget decision.
Current scope:
- Native parity and baseline collection are automated.

View File

@ -1,7 +1,8 @@
#!/usr/bin/env python3
"""Phase 5 audit for cross-request/static state risks in site files."""
"""Phase 5 audit for cross-request/static state risks in site code files."""
from __future__ import annotations
import argparse
import json
from dataclasses import asdict, dataclass
from pathlib import Path
@ -17,6 +18,8 @@ PATTERNS = [
]
SKIP_PARTS = {".git", "tmp", "work", "bin", "pkg", "__pycache__"}
CODE_SUFFIXES = {".uce", ".h"}
DOC_SUFFIXES = {".txt"}
@dataclass
@ -24,24 +27,28 @@ class Finding:
path: str
line: int
kind: str
severity: str
text: str
note: str
def iter_files() -> list[Path]:
def iter_files(include_doc_text: bool) -> list[Path]:
suffixes = CODE_SUFFIXES | (DOC_SUFFIXES if include_doc_text else set())
result: list[Path] = []
for path in SITE.rglob("*"):
if not path.is_file():
continue
if any(part in SKIP_PARTS for part in path.parts):
continue
if path.suffix not in {".uce", ".h", ".txt"}:
if path.suffix not in suffixes:
continue
result.append(path)
return sorted(result)
def note_for(kind: str, text: str) -> str:
def note_for(kind: str, severity: str) -> str:
if severity == "documentation":
return "Documentation prose mention; useful for terminology review, not a direct static-state migration finding."
if kind in {"once hook", "init hook"}:
return "Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state."
if kind == "background task":
@ -49,16 +56,21 @@ def note_for(kind: str, text: str) -> str:
return "Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace."
def scan() -> list[Finding]:
def severity_for(path: Path) -> str:
return "documentation" if path.suffix in DOC_SUFFIXES else "code"
def scan(include_doc_text: bool) -> list[Finding]:
findings: list[Finding] = []
for path in iter_files():
for path in iter_files(include_doc_text):
try:
lines = path.read_text(encoding="utf-8").splitlines()
except UnicodeDecodeError:
continue
severity = severity_for(path)
for lineno, line in enumerate(lines, 1):
stripped = line.strip()
if stripped.startswith("//") or stripped.startswith("# "):
if severity == "code" and (stripped.startswith("//") or stripped.startswith("# ")):
continue
for kind, needle in PATTERNS:
if needle in line:
@ -66,8 +78,9 @@ def scan() -> list[Finding]:
path=str(path.relative_to(ROOT)),
line=lineno,
kind=kind,
severity=severity,
text=stripped[:180],
note=note_for(kind, stripped),
note=note_for(kind, severity),
))
return findings
@ -79,17 +92,25 @@ def write_reports(findings: list[Finding], out_dir: Path) -> None:
if not findings:
lines.append("No candidate cross-request/static-state patterns found.")
else:
lines.extend(["| file | line | kind | code | note |", "|---|---:|---|---|---|"])
code_count = sum(1 for f in findings if f.severity == "code")
doc_count = sum(1 for f in findings if f.severity == "documentation")
lines.append(f"Findings: {code_count} code, {doc_count} documentation prose.")
lines.append("")
lines.extend(["| file | line | severity | kind | code | note |", "|---|---:|---|---|---|---|"])
for f in findings:
code = f.text.replace("|", "\\|")
note = f.note.replace("|", "\\|")
lines.append(f"| {f.path} | {f.line} | {f.kind} | `{code}` | {note} |")
lines.append(f"| {f.path} | {f.line} | {f.severity} | {f.kind} | `{code}` | {note} |")
(out_dir / "site-static-audit.md").write_text("\n".join(lines) + "\n")
def main() -> int:
out_dir = Path("/tmp/uce/wasm-phase5")
findings = scan()
parser = argparse.ArgumentParser(description="Phase 5 site static/cross-request-state audit")
parser.add_argument("--include-doc-text", action="store_true", help="include site/doc .txt prose as documentation-severity findings")
parser.add_argument("--out-dir", default="/tmp/uce/wasm-phase5")
args = parser.parse_args()
out_dir = Path(args.out_dir)
findings = scan(args.include_doc_text)
write_reports(findings, out_dir)
print(f"Found {len(findings)} candidate static/cross-request patterns")
print(f"wrote {out_dir / 'site-static-audit.json'} and {out_dir / 'site-static-audit.md'}")

View File

@ -63,9 +63,6 @@ def request_once(base_url: str, host_header: str, path: str, timeout: float) ->
def measure_backend(backend: str, base_url: str, host_header: str, targets: list[Target], warmups: int, samples: int, timeout: float) -> list[Measurement]:
results: list[Measurement] = []
for target in targets:
path = urlparse(target.url).path
if "?" in target.url:
path = target.url
for _ in range(warmups):
request_once(base_url, host_header, target.url, timeout)
durations: list[float] = []
@ -127,7 +124,7 @@ def main() -> int:
parser.add_argument("--wasm-base-url", default="", help="optional wasm worker URL; omitted until worker exists")
parser.add_argument("--host-header", default="uce.openfu.com")
parser.add_argument("--warmups", type=int, default=2)
parser.add_argument("--samples", type=int, default=5)
parser.add_argument("--samples", type=int, default=20)
parser.add_argument("--timeout", type=float, default=10.0)
parser.add_argument("--out-dir", default="/tmp/uce/wasm-phase5")
args = parser.parse_args()

View File

@ -0,0 +1,104 @@
[
{
"backend": "native",
"target": "template-heavy-doc",
"url": "/doc/singlepage.uce",
"ok": true,
"status": 200,
"samples_ms": [
318.72209906578064,
340.9293442964554,
319.705568253994,
324.0259513258934,
350.93583166599274,
313.20811808109283,
317.0944079756737,
310.312956571579,
307.89367109537125,
308.1864267587662,
311.681292951107,
313.39504569768906,
314.3857270479202,
312.5988021492958,
313.25943768024445,
312.9996135830879,
334.4864323735237,
314.2518773674965,
309.7490146756172,
317.4726217985153
],
"median_ms": 313.8234615325928,
"mean_ms": 318.2647120207548,
"min_ms": 307.89367109537125,
"max_ms": 350.93583166599274,
"note": ""
},
{
"backend": "native",
"target": "sqlite-page",
"url": "/demo/sqlite.uce",
"ok": true,
"status": 200,
"samples_ms": [
3.4144148230552673,
6.088584661483765,
3.463640809059143,
3.498159348964691,
3.5023540258407593,
6.214611232280731,
3.563329577445984,
3.3720433712005615,
3.330707550048828,
3.3808723092079163,
3.322914242744446,
3.3655911684036255,
3.374151885509491,
3.358304500579834,
3.3307820558547974,
3.268897533416748,
3.360658884048462,
3.3061057329177856,
6.233863532543182,
3.432638943195343
],
"median_ms": 3.3775120973587036,
"mean_ms": 3.809131309390068,
"min_ms": 3.268897533416748,
"max_ms": 6.233863532543182,
"note": ""
},
{
"backend": "native",
"target": "component-heavy-starter",
"url": "/examples/uce-starter/?dashboard",
"ok": true,
"status": 200,
"samples_ms": [
74.01428371667862,
40.9795418381691,
40.995217859745026,
41.04568809270859,
40.724173188209534,
41.10313206911087,
40.59913754463196,
41.28593951463699,
41.10439121723175,
40.649913251399994,
42.06441342830658,
42.074643075466156,
43.475523591041565,
41.8255478143692,
41.068583726882935,
41.116394102573395,
41.09777510166168,
42.58237034082413,
41.312023997306824,
70.98117470741272
],
"median_ms": 41.11039265990257,
"mean_ms": 44.50499340891838,
"min_ms": 40.59913754463196,
"max_ms": 74.01428371667862,
"note": ""
}
]

View File

@ -0,0 +1,19 @@
# Phase 5 native baseline — 2026-06-12
Host: `k-uce` / `uce.openfu.com` via localhost with Host header.
This is a durable informational snapshot. The Phase 5 gate recomputes native medians during paired native/WASM runs; these numbers are not hard-coded budgets.
- Warmup suite: 82/82 passed; excludes `site tests tasks` to avoid perturbing task lifecycle state.
- Measured full network suite: 83/83 passed.
- Measured starter subset: 14/14 passed.
- Static audit default scan: 50 code findings, documentation prose excluded by default.
| target | median ms | mean ms | samples |
|---|---:|---:|---:|
| template-heavy-doc | 313.8 | 318.3 | 20 |
| sqlite-page | 3.4 | 3.8 | 20 |
| component-heavy-starter | 41.1 | 44.5 | 20 |
Raw benchmark JSON: `native-baseline-2026-06-12.json`.
Static audit snapshot: `site-static-audit-2026-06-12.md`.

View File

@ -0,0 +1,56 @@
# Phase 5 site static-state audit
Findings: 50 code, 0 documentation prose.
| file | line | severity | kind | code | note |
|---|---:|---|---|---|---|
| site/demo/index.uce | 76 | code | background task | `<? if(allow_server_demos) { render_card("task_repeat.uce", "Task Repeat", "Recurring task scheduling"); } ?>` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/once-init.uce | 1 | code | static local/global | `static s64 demo_worker_init_count = 0;` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. |
| site/demo/once-init.uce | 2 | code | static local/global | `static s64 demo_component_hits = 0;` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. |
| site/demo/once-init.uce | 4 | code | init hook | `INIT(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. |
| site/demo/once-init.uce | 10 | code | once hook | `ONCE(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. |
| site/demo/once-init.uce | 44 | code | once hook | `ONCE() and INIT()` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. |
| site/demo/once-init.uce | 44 | code | init hook | `ONCE() and INIT()` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. |
| site/demo/once-init.uce | 47 | code | once hook | `This page calls the same named component twice. `ONCE()` should only run once for the request, while `INIT()` should stay stable for the currently loaded worker copy.` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. |
| site/demo/once-init.uce | 47 | code | init hook | `This page calls the same named component twice. `ONCE()` should only run once for the request, while `INIT()` should stay stable for the currently loaded worker copy.` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. |
| site/demo/task-status.uce | 11 | code | background task | `String task_name = first(context.get["task-name"], "example-task");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task-status.uce | 13 | code | background task | `print("Task Name: ", task_name, "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task-status.uce | 14 | code | background task | `print("Task ID: ", task_pid(task_name), "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task-status.uce | 15 | code | background task | `print("Task Running: ", task_pid(task_name) == 0 ? "no" : "yes", "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task.uce | 13 | code | background task | `String task_name = first(context.get["task-name"], "example-task");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task.uce | 37 | code | background task | `<input type="text" name="task-name" value="<?= task_name ?>"/>` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task.uce | 46 | code | background task | `load(document.getElementById('task-status'), 'task-status.uce?task-name=<?= uri_encode(task_name) ?>');` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task.uce | 56 | code | background task | `print("Task Name: ", task_name, "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task.uce | 57 | code | background task | `print("Task ID: ", task_pid(task_name), "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task.uce | 58 | code | background task | `print("Task Running: ", task_pid(task_name) == 0 ? "no" : "yes", "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task.uce | 71 | code | background task | `print("New Task ID: ", task(task_name, []() {` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task_repeat.uce | 13 | code | background task | `String task_name = first(context.get["task-name"], "example-task");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task_repeat.uce | 37 | code | background task | `<input type="text" name="task-name" value="<?= task_name ?>"/>` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task_repeat.uce | 46 | code | background task | `load(document.getElementById('task-status'), 'task-status.uce?task-name=<?= uri_encode(task_name) ?>');` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task_repeat.uce | 56 | code | background task | `print("Task Name: ", task_name, "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task_repeat.uce | 57 | code | background task | `print("Task ID: ", task_pid(task_name), "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task_repeat.uce | 58 | code | background task | `print("Task Running: ", task_pid(task_name) == 0 ? "no" : "yes", "\n");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/demo/task_repeat.uce | 71 | code | background task | `print("New Task ID: ", task_repeat(task_name, 5, []() {` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/examples/uce-starter/components/data/widgets.uce | 3 | code | once hook | `ONCE(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. |
| site/examples/uce-starter/components/workspace/primitives.uce | 3 | code | once hook | `ONCE(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. |
| site/examples/uce-starter/lib/user.class.h | 13 | code | static local/global | `static String session_key()` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. |
| site/examples/uce-starter/lib/user.class.h | 18 | code | static local/global | `static String normalize_email(String email)` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. |
| site/examples/uce-starter/lib/user.class.h | 23 | code | static local/global | `static String hash_id(String raw)` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. |
| site/examples/uce-starter/lib/user.class.h | 45 | code | static local/global | `static String password_hash(String password, String salt)` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. |
| site/examples/uce-starter/lib/user.class.h | 53 | code | static local/global | `static DValue read_json_file(String file_name)` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. |
| site/examples/uce-starter/lib/user.class.h | 64 | code | static local/global | `static bool write_json_file(String file_name, DValue data)` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. |
| site/examples/uce-starter/views/dashboard.uce | 3 | code | once hook | `ONCE(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. |
| site/info/index.uce | 269 | code | static local/global | `<li>nginx serves static files directly from `site/`</li>` | Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace. |
| site/tests/preprocessor.uce | 3 | code | once hook | `ONCE(Request& context)` | Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state. |
| site/tests/tasks.uce | 25 | code | background task | `pid_t repeat_existing = task_pid("site-tests-repeat");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 27 | code | background task | `task_kill(repeat_existing, 15);` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 37 | code | background task | `repeat_pid = task_repeat("site-tests-repeat", 1.0, []() {` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 55 | code | background task | `pid_t seen_short_pid = task_pid("site-tests-short");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 56 | code | background task | `pid_t seen_repeat_pid = task_pid("site-tests-repeat");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 57 | code | background task | `pid_t seen_timeout_pid = task_pid("site-tests-timeout");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 58 | code | background task | `pid_t seen_unsafe_key_pid = task_pid("site-tests/../unsafe key");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 59 | code | background task | `int short_alive = seen_short_pid == 0 ? -1 : task_kill(seen_short_pid, 0);` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 60 | code | background task | `int repeat_alive = seen_repeat_pid == 0 ? -1 : task_kill(seen_repeat_pid, 0);` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 72 | code | background task | `check("task_pid() + task_kill(pid, 0)", short_alive == 0, "kill(0) result=" + std::to_string(short_alive));` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 73 | code | background task | `check("task_repeat()", repeat_pid != 0 && seen_repeat_pid != 0, "started=" + std::to_string(repeat_pid) + " seen=" + std::to_string(seen_repeat_pid));` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |
| site/tests/tasks.uce | 77 | code | background task | `check("task_kill() rejects negative pid", task_kill(-1, 0) == -1, "kill(-1, 0) rejected");` | Task APIs cross request lifetimes; verify they are host handles, not guest statics. |

View File

@ -7,21 +7,52 @@ cd "$(dirname "$0")/../.."
OUT=/tmp/uce/wasm-phase5
mkdir -p "$OUT"
python3 tests/run_network_tests.py --include-internal --json-report "$OUT/native-network.json"
python3 tests/run_network_tests.py --include-internal --match starter --json-report "$OUT/native-starter.json"
python3 spikes/wasm-phase5/audit_site_statics.py
python3 spikes/wasm-phase5/benchmark.py --out-dir "$OUT"
# The network suite can show transient cold-compile timeouts immediately after a
# binary rebuild. Run a throwaway pass first so the measured gate is a warmed
# parity signal rather than a compiler-cache signal. The warmup is allowed to
# fail; the measured pass below is not.
python3 tests/run_network_tests.py --include-internal --exclude 'site tests tasks' --json-report "$OUT/native-network-warmup.json" || true
python3 - <<'PY'
network_status=0
starter_status=0
benchmark_status=0
python3 tests/run_network_tests.py --include-internal --json-report "$OUT/native-network.json" || network_status=$?
python3 tests/run_network_tests.py --include-internal --match starter --json-report "$OUT/native-starter.json" || starter_status=$?
python3 spikes/wasm-phase5/audit_site_statics.py --out-dir "$OUT"
python3 spikes/wasm-phase5/benchmark.py --out-dir "$OUT" || benchmark_status=$?
NETWORK_STATUS=$network_status STARTER_STATUS=$starter_status BENCHMARK_STATUS=$benchmark_status python3 - <<'PY'
import json
import os
from pathlib import Path
MIN_NETWORK_CASES = 80
MIN_STARTER_CASES = 10
MIN_BENCHMARKS = 3
out = Path('/tmp/uce/wasm-phase5')
network = json.loads((out / 'native-network.json').read_text())
starter = json.loads((out / 'native-starter.json').read_text())
bench = json.loads((out / 'benchmark.json').read_text())
failures = [r for r in network if not r['ok']] + [r for r in starter if not r['ok']] + [r for r in bench if not r['ok']]
if failures:
structural_failures = []
if len(network) < MIN_NETWORK_CASES:
structural_failures.append(f"network suite ran {len(network)} cases, expected at least {MIN_NETWORK_CASES}")
if len(starter) < MIN_STARTER_CASES:
structural_failures.append(f"starter subset ran {len(starter)} cases, expected at least {MIN_STARTER_CASES}")
if len(bench) < MIN_BENCHMARKS:
structural_failures.append(f"benchmark ran {len(bench)} rows, expected at least {MIN_BENCHMARKS}")
nonzero = {
'network_status': int(os.environ['NETWORK_STATUS']),
'starter_status': int(os.environ['STARTER_STATUS']),
'benchmark_status': int(os.environ['BENCHMARK_STATUS']),
}
for name, status in nonzero.items():
if status != 0:
structural_failures.append(f"{name} exited {status}")
if failures or structural_failures:
print('PHASE5 HARNESS: FAIL')
for failure in structural_failures:
print(failure)
for failure in failures:
print(failure)
raise SystemExit(1)

View File

@ -222,6 +222,7 @@ def filter_cases(
name_pattern: Optional[str],
tags: List[str],
include_internal: bool,
excluded_names: List[str],
) -> List[NetworkTestCase]:
filtered = list(cases)
if plugins:
@ -233,6 +234,9 @@ def filter_cases(
filtered = [case for case in filtered if matcher.search(case.name)]
if tags:
filtered = [case for case in filtered if any(tag in case.tags for tag in tags)]
if excluded_names:
excluded = set(excluded_names)
filtered = [case for case in filtered if case.name not in excluded]
return filtered
@ -273,6 +277,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--plugin-dir", default=str(base_dir / "plugins"), help="directory containing plugin files")
parser.add_argument("--plugin", action="append", default=[], help="limit to one or more plugin basenames")
parser.add_argument("--match", help="regex filter for case names")
parser.add_argument("--exclude", action="append", default=[], help="exclude an exact test case name after other filters")
parser.add_argument("--tag", action="append", default=[], help="only run cases with at least one matching tag")
parser.add_argument("--include-internal", action="store_true", help="include tests tagged internal in the run")
parser.add_argument("--list", action="store_true", help="list discovered tests without running them")
@ -316,7 +321,7 @@ def main(argv: Optional[List[str]] = None) -> int:
for plugin_path in plugin_files:
register_plugin_cases(registry, plugin_path)
cases = filter_cases(registry.cases, args.plugin, args.match, args.tag, args.include_internal)
cases = filter_cases(registry.cases, args.plugin, args.match, args.tag, args.include_internal, args.exclude)
if not cases:
print("No tests matched the selected filters", file=sys.stderr)
return 2