From 961df2d542ff6554e32da9ee28e73f5fc59b5975 Mon Sep 17 00:00:00 2001 From: udo Date: Fri, 12 Jun 2026 19:55:35 +0000 Subject: [PATCH] phase 5 --- .gitignore | 12 +- WASM-PROPOSAL.md | 125 ++++- site/tests/core.uce | 16 + spikes/wasm-phase0/FINDINGS.md | 9 + spikes/wasm-phase0/loader.cpp | 6 +- spikes/wasm-phase2/README.md | 51 ++ spikes/wasm-phase2/build_loader.sh | 16 + spikes/wasm-phase2/build_modules.sh | 21 + spikes/wasm-phase2/core.cpp | 142 ++++++ spikes/wasm-phase2/hostcalls.syms | 2 + spikes/wasm-phase2/loader.cpp | 280 +++++++++++ spikes/wasm-phase2/page.uce | 12 + spikes/wasm-phase3/README.md | 63 +++ spikes/wasm-phase3/build_loader.sh | 16 + spikes/wasm-phase3/build_modules.sh | 70 +++ spikes/wasm-phase3/core.cpp | 188 ++++++++ spikes/wasm-phase3/generated/helper.cpp | 6 + spikes/wasm-phase3/generated/page.uce.cpp | 72 +++ spikes/wasm-phase3/hostcalls.syms | 2 + spikes/wasm-phase3/loader.cpp | 554 ++++++++++++++++++++++ spikes/wasm-phase3/page.uce | 12 + spikes/wasm-phase4/README.md | 77 +++ spikes/wasm-phase4/build_runner.sh | 16 + spikes/wasm-phase4/runner.cpp | 298 ++++++++++++ spikes/wasm-phase5/README.md | 45 ++ spikes/wasm-phase5/audit_site_statics.py | 100 ++++ spikes/wasm-phase5/benchmark.py | 153 ++++++ spikes/wasm-phase5/run_phase5.sh | 31 ++ src/lib/uce_lib.h | 1 + src/lib/wasm_trace.h | 226 +++++++++ tests/plugins/uce_starter_parity.py | 66 +++ 31 files changed, 2681 insertions(+), 7 deletions(-) create mode 100644 spikes/wasm-phase2/README.md create mode 100755 spikes/wasm-phase2/build_loader.sh create mode 100755 spikes/wasm-phase2/build_modules.sh create mode 100644 spikes/wasm-phase2/core.cpp create mode 100644 spikes/wasm-phase2/hostcalls.syms create mode 100644 spikes/wasm-phase2/loader.cpp create mode 100644 spikes/wasm-phase2/page.uce create mode 100644 spikes/wasm-phase3/README.md create mode 100755 spikes/wasm-phase3/build_loader.sh create mode 100755 spikes/wasm-phase3/build_modules.sh create mode 100644 spikes/wasm-phase3/core.cpp create mode 100644 spikes/wasm-phase3/generated/helper.cpp create mode 100644 spikes/wasm-phase3/generated/page.uce.cpp create mode 100644 spikes/wasm-phase3/hostcalls.syms create mode 100644 spikes/wasm-phase3/loader.cpp create mode 100644 spikes/wasm-phase3/page.uce create mode 100644 spikes/wasm-phase4/README.md create mode 100755 spikes/wasm-phase4/build_runner.sh create mode 100644 spikes/wasm-phase4/runner.cpp create mode 100644 spikes/wasm-phase5/README.md create mode 100755 spikes/wasm-phase5/audit_site_statics.py create mode 100755 spikes/wasm-phase5/benchmark.py create mode 100755 spikes/wasm-phase5/run_phase5.sh create mode 100644 src/lib/wasm_trace.h create mode 100644 tests/plugins/uce_starter_parity.py diff --git a/.gitignore b/.gitignore index a219215..5fe7c3f 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,14 @@ pkg/* # Python cache artifacts __pycache__/ -*.pyc +*.py[cod] +*$py.class +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Editor swap artifacts +*.swp +*.swo diff --git a/WASM-PROPOSAL.md b/WASM-PROPOSAL.md index 44498c3..08d67ef 100644 --- a/WASM-PROPOSAL.md +++ b/WASM-PROPOSAL.md @@ -1,6 +1,10 @@ # WASM-PROPOSAL: WebAssembly Unit Runtime for UCE -- **Status:** updated design guide (2026-06-12) +- **Status:** design guide; Phase 0–4 spikes complete and gated (2026-06-12). + Next: production implementation, starting with the Phase 3 work plan + (item 1, real `uce_lib` core compile); the starter parity bar + (`tests/run_network_tests.py --match starter`, 14 cases) is already green + against the native backend. - **Scope:** replace the native unit pipeline (generated C++ → clang → `.so` → `dlopen`) with per-unit WebAssembly modules executed in a per-request, runtime-linked workspace, exposing the same API surface to page code, @@ -346,7 +350,10 @@ Per `load(unit)` into a workspace: `__table_base` (append to shared table). 6. Instantiate with bases; resolve `GOT.*` imports against the workspace symbol registry (core symbols + previously loaded units); register the - unit's exports. + unit's exports. NB: data exports of PIC modules are `__memory_base`-relative + offsets — add the owning unit's base when registering/resolving (core + exports are absolute; the core is non-PIC). See the Phase 0 FINDINGS + erratum; the Phase 3 spike's `self-got` marker exists to catch this. 7. Register entry points in the path → table-index dispatch map. `uce_host_component_resolve(path)` consults the dispatch map and calls @@ -476,6 +483,16 @@ statically-linked unit + core to validate codegen and membrane without the loader. Exit: one real `.uce` page (e.g. `site/tests/core.uce`) renders correctly through the membrane. Scaffolding is marked throwaway. +> **Status: MEMBRANE SCAFFOLD DONE (2026-06-12).** Temporary scaffold checked +> in under `spikes/wasm-phase2/`: a WASM reactor core subset owns memory, +> `Request`, `DValue`, UCEB1, and output buffering; the Wasmtime host implements +> the initial `uce_host_ctx_read`/`uce_host_log` membrane and passes a UCEB1 +> request context into the guest; a statically linked `.uce` render entry runs +> through that membrane. Validation passed on k-uce with `PHASE2 EXIT CRITERION: +> PASS`. The scaffold does not yet exercise UCE preprocessor-emitted page C++; +> generator-emitted units through this membrane are explicitly deferred into the +> Phase 3 loader path. Full dynamic unit loading also remains Phase 3 as planned. + **Phase 3 — the loader + workspace.** Implement §6 in full: dylink parsing, base allocation, GOT resolution, ABI/import verification, lazy mid-request loading, path dispatch. Per-request @@ -483,13 +500,101 @@ workspace birth/drop (plain memcpy birth is fine here; CoW is Phase 4). Exit: the uce-starter renders end-to-end with components loading lazily; `tests/run_network_tests.py --match starter` passes against the wasm worker. +> **Status: SPIKE PASS (2026-06-12).** `spikes/wasm-phase3/` combines the +> Phase 0 dylink/PIC loader with the Phase 2 UCEB1 context membrane. The spike +> builds a core workspace reactor and a separate generated-shape `.uce.cpp` PIC +> side module, parses `dylink.0`, allocates memory/table bases, resolves +> `env.*`/`GOT.*` imports, runs relocations/constructors, calls +> `__uce_set_current_request` and `__uce_render`, and reads output from +> core-owned memory. The fixture now exercises nonzero table placement, +> `GOT.func`, deferred self-resolved `GOT.mem`, and generated-style +> `html_escape(...)` expression output. It passed on k-uce with `PHASE3 EXIT +> CRITERION: PASS`. +> +> The self-GOT fixture caught a real loader bug (2026-06-12, fixed): deferred +> `GOT.mem` entries were patched with the unit's exported symbol values +> verbatim, but a PIC module's data exports are offsets relative to its +> `__memory_base` — the loader must add the base. The bug rendered silently +> wrong values and wrote into core memory at low addresses; both spike loaders +> are fixed and the Phase 3 exit gate now asserts every GOT-derived output +> marker (`self-got`/`callback`/`each`/`map`) so a regression cannot pass. +> Details in the `spikes/wasm-phase0/FINDINGS.md` erratum. +> +> **Production Phase 3 work plan** (ordered by dependency/risk; spike-proven +> mechanics not repeated here): +> +> 1. **Compile the real `uce_lib` as `core.wasm`** — the last big unknown; all +> spikes used a hand-stubbed `Request`. Forces: `types.h` allocator gate as +> a real `#ifdef` (replacing the spike's copied-header text patch), `sys.h` +> signal/fork/socket carve-outs, the libc++ closure strategy +> (`--whole-archive` vs keep-list), and splitting connectors out of the core +> (the MySQL client library cannot compile to wasm). Gates items 2–6. +> 2. **WASI decision (record in §11 when made)** — spikes stub all WASI imports +> with traps; real pages call `time()` (7× in uce-starter), which wasi-libc +> routes to `clock_time_get`. Recommended: zero-WASI core — route +> time/random/env through the §5.1 hostcalls so there is exactly one +> membrane to audit. +> 3. **Generator changes** (small, parallelizable): emit a logical include +> instead of the absolute `uce_lib.h` path; add the PIC side-module build to +> the compile-on-miss artifact cache. +> 4. **Productionize the loader into `src/`** (§6): `uce.abi` stamping, +> import-discipline verification, a name→funcptr registry in the core +> replacing the spike's per-symbol `core_table_index_of_*` helpers, an +> explicit export name-collision policy (the spike silently prefers core +> exports over unit definitions, e.g. `context`), multi-unit bump placement, +> retained unaligned allocation pointers for unload, hardened/fuzzed binary +> parsing. Plain memcpy workspace birth; CoW stays Phase 4. +> 5. **Starter-scoped hostcalls (~12, not the full ≤60)**: `respond` / +> `stream_write`, `time`/`random`/`env`, `session_get`/`set`, +> `http_request` (OAuth callback), `component_resolve`. The starter uses no +> sqlite/mysql/file APIs — connectors can wait for Phase 5 parity. +> 6. **`component_resolve` + path dispatch + lazy loading** — the only §6 step +> with zero spike coverage (all spikes load one unit eagerly). The starter's +> 71 `component()` calls across 47 files are the stress test; worth a +> focused spike before worker integration. +> 7. **FastCGI worker integration** — config-selectable backend, §7 lifecycle +> wired into the `linux_fastcgi.cpp` flow. +> 8. **Starter parity tests — DONE (2026-06-12).** +> `tests/plugins/uce_starter_parity.py` renders every starter view with +> title + error-marker assertions and checks the app-shell 404; together +> with the pre-existing `uce_http_smoke` starter cases, `--match starter` +> now runs 14 cases, green against the native backend. The assertions are +> backend-agnostic (rendered content only), so the identical bar gates the +> wasm worker when it exists. + **Phase 4 — production mechanics.** Core snapshot + CoW birth; bump-allocator flag; epoch/memory limits; trap → error-page path with guest stack traces (this supersedes the signal/longjmp machinery and closes RECOMMENDATIONS.md 1.5 structurally); handle-table cleanup (closes 1.7/5.1); artifact/ABI versioning end-to-end. -Exit: kill-tests (deliberate null-deref page, infinite-loop page, OOM page) -produce clean error pages and an unharmed worker. +Exit: kill-tests (deliberate out-of-bounds page, stack-exhaustion page, +infinite-loop page, OOM page) produce clean error pages and an unharmed +worker. (A literal null-deref page is not in the list: address 0 is valid +wasm linear memory, so it does not trap — see §10.) + +> **Status: KILL-TEST SPIKE PASS (2026-06-12).** `spikes/wasm-phase4/` +> validates the production mechanics that can be proven before the real wasm +> worker exists: reusable compiled artifacts as a core-snapshot proxy, fresh +> per-request stores as workspace birth/drop, **both** CPU-limit mechanisms +> (fuel and epoch interruption — production default is epoch per the Phase 0 +> findings; Wasmtime reports epoch traps as `interrupt`), a store memory +> limiter proven load-bearing (the OOM fixture grows within its own declared +> max so only the limiter can deny it), trap capture with the exit gate +> asserting a wasm backtrace and the expected cause per kill, handle closers +> checked to run exactly once and while the store is alive, and — after all +> six kills — a healthy request served by the same engine (the "unharmed +> worker" half of the exit criterion). Kill fixtures are genuinely trapping +> faults: unreachable, OOB access, stack exhaustion, fuel/epoch-limited +> infinite loops, limiter-denied growth. A literal null deref is deliberately +> absent (does not trap in wasm; risk recorded in §10). Passed on k-uce with +> `PHASE4 EXIT CRITERION: PASS`. The trap-trace summarizer now exists as +> `src/lib/wasm_trace.h` (collapses repeated frames, demangles symbols, +> splits cause/detail) and is double-gated: live traps in the spike runner, +> canned-message checks in `site/tests/core.uce` via the native suite. Not +> yet production: OS-level CoW core snapshots, wiring trace summaries into +> the error-page UI, the unit name-section policy for readable frames, +> `linux_fastcgi.cpp` wiring, real connector handle cleanup, and artifact/ABI +> versioning. **Phase 5 — parity & performance.** Full network suite green on the wasm worker; differential native-vs-wasm runs @@ -499,6 +604,17 @@ component-heavy starter page) with budgets: ≤2× native page latency, workspace birth ≤100µs, internal component call overhead within 10× native call cost. Exit: numbers published in this document, all tests and reviews pass. +> **Status: HARNESS BASELINE PASS (2026-06-12).** `spikes/wasm-phase5/` +> now automates the Phase 5 parity/performance gate shape before the production +> wasm worker exists. On k-uce it ran the full native network suite (`83/83`), +> the starter-focused parity subset (`14/14`, non-vacuous), a heuristic `site/` +> cross-request/static-state audit, and a warmed native benchmark baseline for +> the template-heavy doc page, sqlite page, and component-heavy starter page. +> Reports were written under `/tmp/uce/wasm-phase5/`, and the harness ended with +> `PHASE5 HARNESS: PASS`. True Phase 5 completion still requires passing the +> same harness against a real wasm worker URL and adding worker-internal probes +> for workspace birth and component-call overhead budgets. + **Phase 6 — second plane (deferred until wanted).** Cross-instance call mechanism (props-in/output-out, UCEB1), first Plane B language binding, loader enforcement of the cross-plane context rule (§4). @@ -520,6 +636,7 @@ only on that; both backends share the Phase 1 C ABI. | Performance regression beyond budget | Phase 5 gates; bump allocator and placement memoization in reserve; native backend retained | | Multi-module DWARF / debugging story | Trap stack traces cover the production case (better than today); accept weaker interactive debugging; keep native backend for local deep-debugging | | Unit-statics semantic change breaks existing pages | Phase 5 audit of `site/`; documented migration note; host-side cache facility if a real need surfaces | +| Null-pointer dereferences do not trap (wasm address 0 is valid linear memory) — a buggy page writes low workspace memory and renders silently wrong output instead of erroring | Accepted: damage is confined to the request's workspace and dropped at request end (strictly better than native SIGSEGV). Kill-tests use genuinely trapping faults (OOB, stack exhaustion, `__builtin_trap`). Optional later: null-check instrumentation at a measured perf cost | ## 11. Decisions diff --git a/site/tests/core.uce b/site/tests/core.uce index 3984cff..845e1bf 100644 --- a/site/tests/core.uce +++ b/site/tests/core.uce @@ -236,6 +236,22 @@ RENDER(Request& context) tree["nested"]["ok"].set_bool(true); check("DValue map access", tree["suite"].to_string() == "core" && tree["nested"]["api"].to_string() == "dvalue", json_encode(tree)); + // wasm_trace.h (WASM-PROPOSAL Phase 4): canned Wasmtime-format messages — + // the live trap path is gated by the phase-4 spike runner + String trace_msg = "error while executing at wasm backtrace:\n 0: 0x9f - units!_Z6renderR7Request\n"; + for(int trace_i = 1; trace_i <= 40; trace_i++) + trace_msg += " " + std::to_string(trace_i) + ": 0x20 - units!_Z9recursionv\n"; + trace_msg += "\nCaused by:\n wasm trap: call stack exhausted\n"; + WasmTraceSummary trace_summary = wasm_trace_summarize(trace_msg); + String trace_text = wasm_trace_format(trace_summary); + check("wasm_trace_summarize() collapse + demangle", trace_summary.parsed && trace_summary.total_frames == 41 && trace_summary.cause == "wasm trap: call stack exhausted" && trace_summary.frames.size() == 2 && contains(trace_text, "×40") && contains(trace_text, "recursion()"), trace_text); + + String fault_msg = "error while executing at wasm backtrace:\n 0: 0x2a - !\n\nCaused by:\n 0: memory fault at wasm address 0x20000 in linear memory of size 0x10000\n 1: wasm trap: out of bounds memory access\n"; + WasmTraceSummary fault_summary = wasm_trace_summarize(fault_msg); + check("wasm_trace_summarize() cause/detail split", fault_summary.cause == "wasm trap: out of bounds memory access" && contains(fault_summary.detail, "memory fault at wasm address 0x20000") && fault_summary.total_frames == 1, wasm_trace_format(fault_summary)); + + check("wasm_trace_collapse() raw passthrough", wasm_trace_collapse("plain host error") == "plain host error", wasm_trace_collapse("plain host error")); + site_tests_summary(passed, failed, skipped, "These assertions intentionally stay pure and side-effect free so they remain safe on the public site."); site_tests_page_end(); } diff --git a/spikes/wasm-phase0/FINDINGS.md b/spikes/wasm-phase0/FINDINGS.md index 7805086..6d2eb94 100644 --- a/spikes/wasm-phase0/FINDINGS.md +++ b/spikes/wasm-phase0/FINDINGS.md @@ -125,6 +125,15 @@ addresses read from core's exported data-symbol globals) → instantiate → patch deferred GOT entries → `__wasm_apply_data_relocs` → `__wasm_call_ctors` → call the entry export. +- **Erratum (found in Phase 3): self-resolved `GOT.mem` values must add + `__memory_base`.** A PIC module's exported data symbols are i32 globals + holding offsets *relative to its `__memory_base`*, not absolute addresses — + the linker adds the base when patching deferred GOT entries (there is no + `__wasm_apply_global_relocs` export to do it). Copying the export verbatim + reads/writes core memory at low addresses and renders silently wrong values; + the Phase 3 fixture's `self-got`/`callback` markers exist to catch exactly + this. `GOT.mem` entries resolved from the *core's* exports are absolute + already (the core is non-PIC) and need no adjustment. - **GOT.func is resolved guest-side**: the core exports a helper returning `(intptr_t)&func` — taking the address forces a link-time elem entry and a wasm function pointer *is* its table index. No host-side funcref injection diff --git a/spikes/wasm-phase0/loader.cpp b/spikes/wasm-phase0/loader.cpp index 902216f..f5e358c 100644 --- a/spikes/wasm-phase0/loader.cpp +++ b/spikes/wasm-phase0/loader.cpp @@ -394,9 +394,11 @@ int main(int argc, char** argv) CHECK(own, "GOT.mem.%s defined neither by core nor by unit", nm.c_str()); wasm_val_t v; wasm_global_get(own, &v); - wasm_val_t nv = WASM_I32_VAL(v.of.i32); + // dylink ABI: a PIC module's exported data symbols are offsets relative + // to its __memory_base; the linker must add the base when resolving + wasm_val_t nv = WASM_I32_VAL(memory_base + v.of.i32); wasm_global_set(got, &nv); - printf("self-resolved GOT.mem.%s = %d\n", nm.c_str(), v.of.i32); + printf("self-resolved GOT.mem.%s = %d (offset %d)\n", nm.c_str(), memory_base + v.of.i32, v.of.i32); } if(unit.func("__wasm_apply_data_relocs")) diff --git a/spikes/wasm-phase2/README.md b/spikes/wasm-phase2/README.md new file mode 100644 index 0000000..387ed03 --- /dev/null +++ b/spikes/wasm-phase2/README.md @@ -0,0 +1,51 @@ +# spikes/wasm-phase2 — core module + membrane scaffold + +Phase 2 validates the next WASM step without the Phase 3 dynamic loader. The +scaffold compiles a native UCE core subset (`Request`, `DValue`, UCEB1, print +buffering) plus one statically linked real `.uce` page into a WASM reactor. +A small Wasmtime C-API host provides the first membrane hostcalls, sends a UCEB1 +request context into the guest, invokes render, and reads the response from +linear memory. `uce_host_ctx_read(ptr, cap)` follows a length-query contract: +`ptr == 0` or `cap == 0` returns the required length; a short non-zero buffer +also returns the required length without a partial copy; an adequately sized +buffer receives the full context and returns the copied length. + +This is intentionally not the final worker. It proves the Phase 2 membrane path: +core-owned DValue/UCEB1 code runs in WASM and a `.uce` render entry sees a +host-provided request context through the membrane. The checked-in page uses the +same `RENDER(Request&)` entry shape as generated units, but it is included +directly by the scaffold rather than emitted by the UCE preprocessor. + +Run on `k-uce`: + +```bash +bash spikes/wasm-phase2/build_modules.sh +bash spikes/wasm-phase2/build_loader.sh +/tmp/uce/wasm-phase2/loader +``` + +Expected final line: + +```text +PHASE2 EXIT CRITERION: PASS +``` + +Files: + +- `core.cpp` — WASM reactor core subset and membrane decode/render entry. +- `page.uce` — real UCE page statically linked for Phase 2 only. +- `loader.cpp` — host runner with `uce_host_ctx_read` and `uce_host_log`. + Its small host-side UCEB1 encoder is spike-only; the production UCE server + host should use the native `ucb_encode()` implementation to avoid format drift. +- `build_modules.sh` — wasi-sdk build for `/tmp/uce/wasm-phase2/core.wasm`. +- `build_loader.sh` — Wasmtime C-API build for `/tmp/uce/wasm-phase2/loader`. + +Deferred to Phase 3: + +- PIC side modules and dylink loader integration. +- Lazy unit/component loading. +- Full `uce_lib` hostcall surface beyond the minimal context/log membrane. +- Generator-emitted units (`.uce` preprocessor output with literal markup, + `__uce_set_current_request`, and generated includes) rendering through this + membrane; Phase 0 only proved those generated units can compile as side + modules. diff --git a/spikes/wasm-phase2/build_loader.sh b/spikes/wasm-phase2/build_loader.sh new file mode 100755 index 0000000..269363b --- /dev/null +++ b/spikes/wasm-phase2/build_loader.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Phase 2: build the minimal host/membrane runner against Wasmtime's wasm-c-api. +set -e +cd "$(dirname "$0")" + +OUT=/tmp/uce/wasm-phase2 +mkdir -p "$OUT" + +c++ -std=c++17 -O2 loader.cpp \ + -I/opt/wasmtime/include \ + -L/opt/wasmtime/lib \ + -Wl,-rpath,/opt/wasmtime/lib \ + -lwasmtime \ + -o "$OUT/loader" + +echo "built: $OUT/loader" diff --git a/spikes/wasm-phase2/build_modules.sh b/spikes/wasm-phase2/build_modules.sh new file mode 100755 index 0000000..f201939 --- /dev/null +++ b/spikes/wasm-phase2/build_modules.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Phase 2: build the native DValue/UCEB1 core subset plus one statically +# linked real .uce page as a WASM reactor. Runs on k-uce; artifacts go under +# /tmp/uce/wasm-phase2. +set -e +cd "$(dirname "$0")" + +SDK=/opt/wasi-sdk +OUT=/tmp/uce/wasm-phase2 +mkdir -p "$OUT" + +"$SDK/bin/clang++" --target=wasm32-wasip1 -mexec-model=reactor \ + -O1 -fno-exceptions \ + -I../.. -I../../src/lib \ + core.cpp -o "$OUT/core.wasm" \ + -Wl,--export-all \ + -Wl,--export=__heap_base \ + -Wl,--allow-undefined-file=hostcalls.syms + +echo "--- phase2 core.wasm ---" +ls -la "$OUT/core.wasm" diff --git a/spikes/wasm-phase2/core.cpp b/spikes/wasm-phase2/core.cpp new file mode 100644 index 0000000..ada5f58 --- /dev/null +++ b/spikes/wasm-phase2/core.cpp @@ -0,0 +1,142 @@ +// WASM-PROPOSAL Phase 2 — native UCE core subset compiled to WASM. +// +// This scaffold validates the Phase 2 membrane without the Phase 3 dynamic +// loader: the core owns memory/libc++/DValue/UCEB1, imports a tiny hostcall +// surface, decodes a host-provided UCEB1 request context, and invokes one +// statically linked real .uce page. + +#include +#include + +#include "../../src/lib/types.h" +#include "../../src/lib/dvalue.cpp" + +extern "C" { +size_t uce_host_ctx_read(char* buf, size_t cap); +void uce_host_log(int level, const char* buf, size_t len); +} + +#define RENDER(X) extern "C" void __uce_render(X) +#include "page.uce" + +static Request g_request; +static ByteStream g_ob; +static String g_output; + +SharedUnit::~SharedUnit() {} + +String nibble(String div, String& haystack) +{ + auto pos = haystack.find(div); + if(pos == String::npos) + { + auto result = haystack; + haystack.clear(); + return(result); + } + auto result = haystack.substr(0, pos); + haystack.erase(0, pos + div.length()); + return(result); +} + +void Request::ob_start() +{ + ob_stack.push_back(new ByteStream()); + ob = ob_stack.back(); +} + +void Request::set_status(s32 code, String reason) +{ + if(reason == "") + reason = code == 200 ? "OK" : "Status"; + response_code = "HTTP/1.1 " + std::to_string(code) + " " + reason; +} + +Request::~Request() +{ + for(auto* stream : ob_stack) + delete stream; + ob_stack.clear(); +} + +static void phase2_clear_ob_stack() +{ + for(auto* stream : g_request.ob_stack) + delete stream; + g_request.ob_stack.clear(); +} + +static void phase2_apply_context(DValue& root) +{ + g_request.call = root; + g_request.params.clear(); + DValue* params = root.key("params"); + if(params) + { + params->each([&](const DValue& item, String key) { + g_request.params[key] = item.to_string(); + }); + } +} + +extern "C" { + +void* uce_alloc(size_t len) +{ + return(malloc(len)); +} + +void uce_free(void* ptr) +{ + free(ptr); +} + +int uce_phase2_render() +{ + context = &g_request; + phase2_clear_ob_stack(); + g_ob.str(""); + g_ob.clear(); + g_request.ob = &g_ob; + g_request.out = ""; + g_output = ""; + + size_t ctx_required = uce_host_ctx_read(0, 0); + if(ctx_required == 0) + return(10); + char* ctx_buf = (char*)malloc(ctx_required); + if(ctx_buf == 0) + return(11); + size_t ctx_len = uce_host_ctx_read(ctx_buf, ctx_required); + if(ctx_len != ctx_required) + { + free(ctx_buf); + return(12); + } + DValue decoded; + String error; + bool ok = ucb_decode(String(ctx_buf, ctx_len), decoded, &error); + free(ctx_buf); + if(!ok) + { + uce_host_log(3, error.data(), error.size()); + return(20); + } + + phase2_apply_context(decoded); + __uce_render(g_request); + g_output = g_ob.str(); + return(0); +} + +const char* uce_phase2_output_data() +{ + return(g_output.data()); +} + +size_t uce_phase2_output_size() +{ + return(g_output.size()); +} + +} diff --git a/spikes/wasm-phase2/hostcalls.syms b/spikes/wasm-phase2/hostcalls.syms new file mode 100644 index 0000000..6a18b04 --- /dev/null +++ b/spikes/wasm-phase2/hostcalls.syms @@ -0,0 +1,2 @@ +uce_host_ctx_read +uce_host_log diff --git a/spikes/wasm-phase2/loader.cpp b/spikes/wasm-phase2/loader.cpp new file mode 100644 index 0000000..b58e540 --- /dev/null +++ b/spikes/wasm-phase2/loader.cpp @@ -0,0 +1,280 @@ +// WASM-PROPOSAL Phase 2 — minimal membrane host. +// Instantiates the statically linked core/page module, serves a UCEB1 request +// context via hostcall, invokes render, and reads the response from guest +// memory. Dynamic linking is intentionally Phase 3 work. + +#include + +#include +#include +#include +#include +#include +#include + +#define FAIL(...) do { fprintf(stderr, "FAIL: " __VA_ARGS__); fprintf(stderr, "\n"); exit(1); } while(0) +#define CHECK(cond, ...) do { if(!(cond)) FAIL(__VA_ARGS__); } while(0) + +static wasm_store_t* g_store = nullptr; +static wasm_memory_t* g_memory = nullptr; +static std::string g_context; + +static std::vector read_file(const char* fn) +{ + FILE* f = fopen(fn, "rb"); + CHECK(f, "cannot open %s", fn); + fseek(f, 0, SEEK_END); + long n = ftell(f); + fseek(f, 0, SEEK_SET); + std::vector buf(n); + CHECK(fread(buf.data(), 1, n, f) == (size_t)n, "short read on %s", fn); + fclose(f); + return buf; +} + +static void append_varuint(std::string& out, uint64_t value) +{ + while(value >= 0x80) + { + out.push_back((char)((value & 0x7f) | 0x80)); + value >>= 7; + } + out.push_back((char)value); +} + +struct Node +{ + std::string scalar; + bool is_list = false; + std::vector> children; +}; + +static Node scalar(const char* value) +{ + Node n; + n.scalar = value; + return n; +} + +static void encode_node(std::string& out, const Node& node) +{ + out.push_back(node.is_list ? 1 : 0); + append_varuint(out, node.scalar.size()); + out.append(node.scalar); + append_varuint(out, node.children.size()); + for(const auto& child : node.children) + { + append_varuint(out, child.first.size()); + out.append(child.first); + encode_node(out, child.second); + } +} + +static std::string make_context() +{ + Node params; + params.children.push_back({"HTTP_HOST", scalar("phase2.example.test")}); + params.children.push_back({"SCRIPT_URL", scalar("/spikes/wasm-phase2/page.uce")}); + + Node nested; + nested.children.push_back({"answer", scalar("42")}); + + Node root; + root.children.push_back({"params", params}); + root.children.push_back({"route", scalar("/spikes/wasm-phase2/page.uce")}); + root.children.push_back({"nested", nested}); + + std::string out = "UCEB"; + out.push_back((char)1); + encode_node(out, root); + return out; +} + +static wasm_trap_t* host_ctx_read(void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results) +{ + (void)env; + uint32_t ptr = args->data[0].of.i32; + uint32_t cap = args->data[1].of.i32; + if(ptr == 0 || cap == 0 || cap < g_context.size()) + { + results->data[0] = WASM_I32_VAL((int32_t)g_context.size()); + return nullptr; + } + CHECK(g_memory, "host ctx_read called before memory export was captured"); + uint8_t* mem = (uint8_t*)wasm_memory_data(g_memory); + size_t mem_size = wasm_memory_data_size(g_memory); + CHECK((size_t)ptr <= mem_size, "ctx_read pointer outside memory"); + CHECK((size_t)ptr + g_context.size() <= mem_size, "ctx_read buffer outside memory"); + memcpy(mem + ptr, g_context.data(), g_context.size()); + results->data[0] = WASM_I32_VAL((int32_t)g_context.size()); + return nullptr; +} + +static wasm_trap_t* host_log(void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results) +{ + (void)env; (void)results; + int level = args->data[0].of.i32; + uint32_t ptr = args->data[1].of.i32; + uint32_t len = args->data[2].of.i32; + uint8_t* mem = (uint8_t*)wasm_memory_data(g_memory); + size_t mem_size = wasm_memory_data_size(g_memory); + if((size_t)ptr + len <= mem_size) + fprintf(stderr, "[guest log %d] %.*s\n", level, (int)len, (const char*)mem + ptr); + return nullptr; +} + +static wasm_trap_t* stub_callback(void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results) +{ + (void)args; (void)results; + char msg[256]; + snprintf(msg, sizeof(msg), "unimplemented import called: %s", (const char*)env); + wasm_message_t message; + wasm_byte_vec_new(&message, strlen(msg) + 1, msg); + wasm_trap_t* trap = wasm_trap_new(g_store, &message); + wasm_byte_vec_delete(&message); + return trap; +} + +static wasm_func_t* make_func(wasm_store_t* store, std::vector params, std::vector results, wasm_func_callback_with_env_t cb, void* env = nullptr) +{ + wasm_valtype_vec_t ps; + wasm_valtype_vec_new_uninitialized(&ps, params.size()); + for(size_t i = 0; i < params.size(); i++) + ps.data[i] = wasm_valtype_new(params[i]); + wasm_valtype_vec_t rs; + wasm_valtype_vec_new_uninitialized(&rs, results.size()); + for(size_t i = 0; i < results.size(); i++) + rs.data[i] = wasm_valtype_new(results[i]); + wasm_functype_t* ft = wasm_functype_new(&ps, &rs); + wasm_func_t* fn = wasm_func_new_with_env(store, ft, cb, env, nullptr); + wasm_functype_delete(ft); + return fn; +} + +struct Instance +{ + wasm_module_t* module = nullptr; + wasm_instance_t* instance = nullptr; + wasm_extern_vec_t exports = WASM_EMPTY_VEC; + std::map by_name; + + void index_exports() + { + wasm_exporttype_vec_t types = WASM_EMPTY_VEC; + wasm_module_exports(module, &types); + wasm_instance_exports(instance, &exports); + for(size_t i = 0; i < types.size; i++) + { + const wasm_name_t* nm = wasm_exporttype_name(types.data[i]); + std::string key(nm->data, nm->size); + while(!key.empty() && key.back() == '\0') key.pop_back(); + by_name[key] = exports.data[i]; + } + wasm_exporttype_vec_delete(&types); + } + + wasm_func_t* func(const char* name) + { + auto it = by_name.find(name); + return it == by_name.end() ? nullptr : wasm_extern_as_func(it->second); + } +}; + +static int32_t call_i32(Instance& inst, const char* name) +{ + wasm_func_t* f = inst.func(name); + CHECK(f, "missing export func %s", name); + wasm_val_t result_buf[1] = { WASM_INIT_VAL }; + wasm_val_vec_t args = WASM_EMPTY_VEC; + wasm_val_vec_t results = { wasm_func_result_arity(f), result_buf }; + wasm_val_vec_t no_results = WASM_EMPTY_VEC; + wasm_trap_t* trap = wasm_func_call(f, &args, results.size ? &results : &no_results); + if(trap) + { + wasm_message_t msg; + wasm_trap_message(trap, &msg); + FAIL("trap during %s: %.*s", name, (int)msg.size, msg.data); + } + return results.size ? result_buf[0].of.i32 : 0; +} + +int main(int argc, char** argv) +{ + const char* core_path = argc > 1 ? argv[1] : "/tmp/uce/wasm-phase2/core.wasm"; + g_context = make_context(); + + wasm_engine_t* engine = wasm_engine_new(); + CHECK(engine, "engine"); + g_store = wasm_store_new(engine); + CHECK(g_store, "store"); + + std::vector bytes = read_file(core_path); + wasm_byte_vec_t bv; + wasm_byte_vec_new(&bv, bytes.size(), (const char*)bytes.data()); + Instance core; + core.module = wasm_module_new(g_store, &bv); + wasm_byte_vec_delete(&bv); + CHECK(core.module, "core module load failed"); + + wasm_importtype_vec_t imports = WASM_EMPTY_VEC; + wasm_module_imports(core.module, &imports); + std::vector externs(imports.size); + std::vector owned_funcs; + for(size_t i = 0; i < imports.size; i++) + { + const wasm_name_t* mod_n = wasm_importtype_module(imports.data[i]); + const wasm_name_t* name_n = wasm_importtype_name(imports.data[i]); + std::string mod(mod_n->data, mod_n->size); + std::string name(name_n->data, name_n->size); + while(!mod.empty() && mod.back() == '\0') mod.pop_back(); + while(!name.empty() && name.back() == '\0') name.pop_back(); + const wasm_externtype_t* et = wasm_importtype_type(imports.data[i]); + CHECK(wasm_externtype_kind(et) == WASM_EXTERN_FUNC, "unexpected non-func import %s.%s", mod.c_str(), name.c_str()); + + wasm_func_t* fn = nullptr; + if(mod == "env" && name == "uce_host_ctx_read") + fn = make_func(g_store, {WASM_I32, WASM_I32}, {WASM_I32}, host_ctx_read); + else if(mod == "env" && name == "uce_host_log") + fn = make_func(g_store, {WASM_I32, WASM_I32, WASM_I32}, {}, host_log); + else + { + char* label = strdup((mod + "." + name).c_str()); + const wasm_functype_t* ft = wasm_externtype_as_functype_const(et); + fn = wasm_func_new_with_env(g_store, ft, stub_callback, label, nullptr); + } + CHECK(fn, "failed to create import %s.%s", mod.c_str(), name.c_str()); + owned_funcs.push_back(fn); + externs[i] = wasm_func_as_extern(fn); + } + + wasm_extern_vec_t iv = { externs.size(), externs.data() }; + wasm_trap_t* trap = nullptr; + core.instance = wasm_instance_new(g_store, core.module, &iv, &trap); + if(trap) + { + wasm_message_t msg; + wasm_trap_message(trap, &msg); + FAIL("trap during instantiation: %.*s", (int)msg.size, msg.data); + } + CHECK(core.instance, "instantiation failed"); + core.index_exports(); + g_memory = wasm_extern_as_memory(core.by_name.count("memory") ? core.by_name["memory"] : nullptr); + CHECK(g_memory, "core does not export memory"); + if(core.func("_initialize")) + call_i32(core, "_initialize"); + + int rc = call_i32(core, "uce_phase2_render"); + CHECK(rc == 0, "render returned %d", rc); + int32_t data = call_i32(core, "uce_phase2_output_data"); + int32_t size = call_i32(core, "uce_phase2_output_size"); + uint8_t* mem = (uint8_t*)wasm_memory_data(g_memory); + CHECK((size_t)data + (size_t)size <= wasm_memory_data_size(g_memory), "output outside memory"); + std::string output((const char*)mem + data, size); + printf("---- phase2 output (%d bytes) ----\n%s", size, output.c_str()); + printf("----------------------------------\n"); + CHECK(output.find("PHASE2 PAGE OK") != std::string::npos, "missing page marker"); + CHECK(output.find("host=phase2.example.test") != std::string::npos, "missing context param"); + CHECK(output.find("answer=42") != std::string::npos, "missing nested context value"); + printf("PHASE2 EXIT CRITERION: PASS\n"); + return 0; +} diff --git a/spikes/wasm-phase2/page.uce b/spikes/wasm-phase2/page.uce new file mode 100644 index 0000000..cf3e5f5 --- /dev/null +++ b/spikes/wasm-phase2/page.uce @@ -0,0 +1,12 @@ +// WASM-PROPOSAL Phase 2 scaffold page. +// This is a real .uce source file with a normal RENDER entry point. The +// Phase 2 core statically links it to validate the host-context membrane +// before the dynamic loader is introduced in Phase 3. + +RENDER(Request& context) +{ + print("PHASE2 PAGE OK\n"); + print("host=", context.params["HTTP_HOST"], "\n"); + print("route=", context.call["route"].to_string(), "\n"); + print("answer=", context.call["nested"]["answer"].to_string(), "\n"); +} diff --git a/spikes/wasm-phase3/README.md b/spikes/wasm-phase3/README.md new file mode 100644 index 0000000..cec7324 --- /dev/null +++ b/spikes/wasm-phase3/README.md @@ -0,0 +1,63 @@ +# spikes/wasm-phase3 — dynamic loader + workspace scaffold + +Phase 3 combines the Phase 0 dynamic PIC-module loader with the Phase 2 UCEB1 +request-context membrane. It is still a spike, not the production wasm worker, +but it validates the load-bearing pieces together: + +1. build a core WASM reactor that owns memory, allocator, `Request`, `DValue`, + UCEB1, and output buffering; +2. build a separate PIC side module with `dylink.0` from a generated-shape UCE + page C++ file; +3. parse `dylink.0`, allocate `__memory_base`/`__table_base`, resolve + `env.*`, `GOT.mem.*`, and `GOT.func.*` imports against the core/workspace; +4. instantiate the unit, run relocations/constructors, set its Request pointer, + call `__uce_render`, and read rendered output from core-owned memory. + +The fixture intentionally exercises the loader branches that are easiest to +accidentally leave dark: non-zero `table_size` / `__table_base`, `GOT.func` +resolution (`phase3_core_print`), and deferred self-resolution of unit-owned +`GOT.mem` entries. The loader output prints those resolutions before the final +pass marker. Its generated-shape expressions call `html_escape(...)` so the +side-module import surface includes the first ordinary UCE helper dependency +rather than only `types.h`/`DValue`. + +Run on `k-uce`: + +```bash +bash spikes/wasm-phase3/build_modules.sh +bash spikes/wasm-phase3/build_loader.sh +/tmp/uce/wasm-phase3/loader +``` + +Expected final line: + +```text +PHASE3 EXIT CRITERION: PASS +``` + +Files: + +- `core.cpp` — core/workspace scaffold and UCEB1 membrane decode. +- `page.uce` — source page for the generated-shape fixture. +- `generated/page.uce.cpp` — checked-in UCE-preprocessor-shaped side-module + fixture: `uce_lib.h` include, `__uce_set_current_request`, `__uce_render`, + literal markup lowered to `print(R"(...)")`, expression prints wrapped in + `html_escape(...)`, callback/lambda/table exercises, and self-GOT data. +- `loader.cpp` — dynamic linker/runner adapted from Phase 0. +- `hostcalls.syms` — explicit core hostcall allowlist. + +Still deferred to production Phase 3/4 work: + +- running the actual UCE preprocessor inside this build script rather than using + a checked-in generated-shape fixture; +- lazy component/path dispatch for a full site tree; +- uce-starter end-to-end under a wasm FastCGI worker; +- productionizing the side-module allocator gate as a real `#ifdef` in + `types.h` instead of this spike's copied-header text patch; +- parser hardening beyond the spike's basic magic/version/bounds/alignment + checks, including fuzzing malformed wasm/dylink sections before accepting + cache or third-party units; +- import-policy hardening beyond this spike's fail-fast resolver; +- workspace birth/drop integration with the real server lifecycle, including + retaining unaligned allocation pointers needed to unload units or drop a + workspace cleanly. diff --git a/spikes/wasm-phase3/build_loader.sh b/spikes/wasm-phase3/build_loader.sh new file mode 100755 index 0000000..8f698af --- /dev/null +++ b/spikes/wasm-phase3/build_loader.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Phase 3: build the dynamic-loader/membrane runner against Wasmtime's wasm-c-api. +set -e +cd "$(dirname "$0")" + +OUT=/tmp/uce/wasm-phase3 +mkdir -p "$OUT" + +c++ -std=c++17 -O2 loader.cpp \ + -I/opt/wasmtime/include \ + -L/opt/wasmtime/lib \ + -Wl,-rpath,/opt/wasmtime/lib \ + -lwasmtime \ + -o "$OUT/loader" + +echo "built: $OUT/loader" diff --git a/spikes/wasm-phase3/build_modules.sh b/spikes/wasm-phase3/build_modules.sh new file mode 100755 index 0000000..8b7fc20 --- /dev/null +++ b/spikes/wasm-phase3/build_modules.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Phase 3: build a core reactor plus one generated-shape PIC side module. +# Runs on k-uce; artifacts go under /tmp/uce/wasm-phase3. +set -e +cd "$(dirname "$0")" + +SDK=/opt/wasi-sdk +ROOT=$(cd ../.. && pwd) +OUT=/tmp/uce/wasm-phase3 +SHIM="$OUT/shim" +mkdir -p "$OUT" "$SHIM" + +cat > "$SHIM/uce_lib.h" <<'EOF' +#pragma once +#include "types.h" +String html_escape(String s); +EOF +cp "$ROOT/src/lib/types.h" "$SHIM/types.h" +python3 - <<'PY' +from pathlib import Path +p = Path('/tmp/uce/wasm-phase3/shim/types.h') +s = p.read_text() +start = s.index('void * operator new(decltype(sizeof(0)) n) noexcept(false)') +end = s.index('void operator delete(void * p) throw()') +end = s.index('\n}', end) + 3 +replacement = '''// WASM side-module shim: allocator is owned by the core module.\nvoid * operator new(decltype(sizeof(0)) n) noexcept(false);\nvoid operator delete(void * p) throw();\nvoid operator delete(void * p, decltype(sizeof(0)) n) noexcept;\n''' +p.write_text(s[:start] + replacement + s[end:]) +PY +cat > "$SHIM/signal.h" <<'EOF' +#pragma once +typedef int sig_atomic_t; +typedef void (*sighandler_t)(int); +#define SIGTERM 15 +int raise(int); +sighandler_t signal(int, sighandler_t); +EOF + +"$SDK/bin/clang++" --target=wasm32-wasip1 -mexec-model=reactor \ + -O1 -fno-exceptions \ + -I../.. -I../../src/lib \ + core.cpp -o "$OUT/core.wasm" \ + -Wl,--export-all \ + -Wl,--import-table \ + -Wl,--export=__stack_pointer \ + -Wl,--export=__heap_base \ + -Wl,--allow-undefined-file=hostcalls.syms \ + -Wl,--undefined=_ZTVN10__cxxabiv117__class_type_infoE + +"$SDK/bin/clang++" --target=wasm32-wasip1 -fPIC -fvisibility=default \ + -fvisibility-inlines-hidden \ + -O1 -fno-exceptions \ + -I"$SHIM" -I"$ROOT/src/lib" \ + -c generated/page.uce.cpp -o "$OUT/page.o" + +"$SDK/bin/clang++" --target=wasm32-wasip1 -fPIC -fvisibility=default \ + -fvisibility-inlines-hidden \ + -O1 -fno-exceptions \ + -I"$SHIM" -I"$ROOT/src/lib" \ + -c generated/helper.cpp -o "$OUT/helper.o" + +"$SDK/bin/wasm-ld" -shared --experimental-pic \ + --unresolved-symbols=import-dynamic \ + "$OUT/page.o" "$OUT/helper.o" -o "$OUT/page.wasm" \ + --export=__uce_set_current_request \ + --export=__uce_render + +echo "--- phase3 core.wasm ---" +ls -la "$OUT/core.wasm" +echo "--- phase3 page.wasm ---" +ls -la "$OUT/page.wasm" diff --git a/spikes/wasm-phase3/core.cpp b/spikes/wasm-phase3/core.cpp new file mode 100644 index 0000000..69685cf --- /dev/null +++ b/spikes/wasm-phase3/core.cpp @@ -0,0 +1,188 @@ +// WASM-PROPOSAL Phase 3 — dynamic-loader core scaffold. +// +// This core owns the UCE heap, Request, DValue/UCEB1, and output buffer. The +// Phase 3 loader dynamically links a PIC side module generated in the same +// shape as UCE preprocessor output, sets its Request* context, and invokes its +// __uce_render entry. + +#include +#include +#include + +#include "../../src/lib/types.h" +#include "../../src/lib/dvalue.cpp" + +extern "C" { +size_t uce_host_ctx_read(char* buf, size_t cap); +void uce_host_log(int level, const char* buf, size_t len); +} + +extern "C" void phase3_core_print(const char* data, size_t len) +{ + if(context && context->ob) + context->ob->write(data, len); +} + +extern "C" void phase3_core_invoke_callback(void (*cb)(int), int value) +{ + cb(value); +} + +extern "C" void phase3_core_invoke_function(std::function* f, int value) +{ + String out = "

fn=" + std::to_string((*f)(value)) + "

\n"; + phase3_core_print(out.data(), out.size()); +} + +extern "C" intptr_t core_table_index_of_phase3_core_print() +{ + return(reinterpret_cast(phase3_core_print)); +} + +static Request g_request; +static ByteStream g_ob; +static String g_output; + +SharedUnit::~SharedUnit() {} + +String html_escape(String s) +{ + String result; + for(char c : s) + { + switch(c) + { + case '&': result += "&"; break; + case '<': result += "<"; break; + case '>': result += ">"; break; + case '"': result += """; break; + case '\'': result += "'"; break; + default: result.push_back(c); break; + } + } + return(result); +} + +String nibble(String div, String& haystack) +{ + auto pos = haystack.find(div); + if(pos == String::npos) + { + auto result = haystack; + haystack.clear(); + return(result); + } + auto result = haystack.substr(0, pos); + haystack.erase(0, pos + div.length()); + return(result); +} + +void Request::ob_start() +{ + ob_stack.push_back(new ByteStream()); + ob = ob_stack.back(); +} + +void Request::set_status(s32 code, String reason) +{ + if(reason == "") + reason = code == 200 ? "OK" : "Status"; + response_code = "HTTP/1.1 " + std::to_string(code) + " " + reason; +} + +Request::~Request() +{ + for(auto* stream : ob_stack) + delete stream; + ob_stack.clear(); +} + +static void phase3_clear_ob_stack() +{ + for(auto* stream : g_request.ob_stack) + delete stream; + g_request.ob_stack.clear(); +} + +static void phase3_apply_context(DValue& root) +{ + g_request.call = root; + g_request.params.clear(); + DValue* params = root.key("params"); + if(params) + { + params->each([&](const DValue& item, String key) { + g_request.params[key] = item.to_string(); + }); + } +} + +extern "C" { + +void* uce_alloc(size_t len) +{ + return(malloc(len)); +} + +void uce_free(void* ptr) +{ + free(ptr); +} + +int uce_phase3_prepare() +{ + context = &g_request; + phase3_clear_ob_stack(); + g_ob.str(""); + g_ob.clear(); + g_request.ob = &g_ob; + g_request.out = ""; + g_output = ""; + + size_t ctx_required = uce_host_ctx_read(0, 0); + if(ctx_required == 0) + return(10); + char* ctx_buf = (char*)malloc(ctx_required); + if(ctx_buf == 0) + return(11); + size_t ctx_len = uce_host_ctx_read(ctx_buf, ctx_required); + if(ctx_len != ctx_required) + { + free(ctx_buf); + return(12); + } + + DValue decoded; + String error; + bool ok = ucb_decode(String(ctx_buf, ctx_len), decoded, &error); + free(ctx_buf); + if(!ok) + { + uce_host_log(3, error.data(), error.size()); + return(20); + } + phase3_apply_context(decoded); + return(0); +} + +Request* uce_phase3_request() +{ + return(&g_request); +} + +void uce_phase3_finish() +{ + g_output = g_ob.str(); +} + +const char* uce_phase3_output_data() +{ + return(g_output.data()); +} + +size_t uce_phase3_output_size() +{ + return(g_output.size()); +} + +} diff --git a/spikes/wasm-phase3/generated/helper.cpp b/spikes/wasm-phase3/generated/helper.cpp new file mode 100644 index 0000000..fba26fe --- /dev/null +++ b/spikes/wasm-phase3/generated/helper.cpp @@ -0,0 +1,6 @@ +extern int phase3_unit_state; + +extern "C" int phase3_helper_state_plus(int value) +{ + return(phase3_unit_state + value); +} diff --git a/spikes/wasm-phase3/generated/page.uce.cpp b/spikes/wasm-phase3/generated/page.uce.cpp new file mode 100644 index 0000000..3084365 --- /dev/null +++ b/spikes/wasm-phase3/generated/page.uce.cpp @@ -0,0 +1,72 @@ +#include "uce_lib.h" +#include +#include +#include + +extern "C" void phase3_core_print(const char* data, size_t len); +extern "C" void phase3_core_invoke_callback(void (*cb)(int), int value); +extern "C" void phase3_core_invoke_function(std::function* f, int value); +extern "C" int phase3_helper_state_plus(int value); + +int phase3_unit_state = 40; +int phase3_weak_data __attribute__((weak)) = 0; + +static void phase3_unit_callback(int value) +{ + String out = "

callback=" + std::to_string(value + phase3_unit_state) + "

\n"; + print(out); +} + +#ifndef UCE_SET_CURRENT_REQUEST_DEFINED +#define UCE_SET_CURRENT_REQUEST_DEFINED + +/*load_declarations*/ + +extern "C" void __uce_set_current_request(Request* _request) +{ + context = _request; + /*load_units*/ +} + +#endif +#line 4 "spikes/wasm-phase3/page.uce" +extern "C" void __uce_render(Request& context) +{ + print(R"(
+

PHASE3 PAGE OK

+

host=)"); + print(html_escape(context.params["HTTP_HOST"])); + print(R"(

+

route=)"); + print(html_escape(context.call["route"].to_string())); + print(R"(

+

answer=)"); + print(html_escape(context.call["nested"]["answer"].to_string())); + print(R"(

+)"); + context.call["nested"].each([&](const DValue& value, String key) { + print(R"(

each=)"); + print(html_escape(key + ":" + value.to_string())); + print(R"(

+)"); + }); + print(R"(

self-got=)"); + phase3_weak_data = 7; + print(std::to_string(phase3_helper_state_plus(2) + phase3_weak_data - 7)); + print(R"(

+)"); + std::map phase3_map; + phase3_map.emplace(std::piecewise_construct, std::forward_as_tuple("piece"), std::forward_as_tuple("wise")); + print(R"(

map=)"); + print(html_escape(phase3_map["piece"])); + print(R"(

+)"); + phase3_core_invoke_callback(phase3_unit_callback, 2); + void (*volatile print_ptr)(const char*, size_t) = phase3_core_print; + print_ptr("

got-func-ok

\n", 19); + auto* fn = new std::function([](int value) { return(value * 3); }); + phase3_core_invoke_function(fn, 14); + delete fn; + print(R"(
+)"); +} diff --git a/spikes/wasm-phase3/hostcalls.syms b/spikes/wasm-phase3/hostcalls.syms new file mode 100644 index 0000000..6a18b04 --- /dev/null +++ b/spikes/wasm-phase3/hostcalls.syms @@ -0,0 +1,2 @@ +uce_host_ctx_read +uce_host_log diff --git a/spikes/wasm-phase3/loader.cpp b/spikes/wasm-phase3/loader.cpp new file mode 100644 index 0000000..7e4fd2a --- /dev/null +++ b/spikes/wasm-phase3/loader.cpp @@ -0,0 +1,554 @@ +// WASM-PROPOSAL Phase 3 — dynamic loader + membrane scaffold. +// +// Embeds a wasm runtime through the standard wasm-c-api and links a core +// workspace module with a generated-shape PIC UCE side module: +// +// 1. instantiate core.wasm (owns memory/table/allocator/Request/DValue) +// 2. parse page.wasm's dylink.0 → data size/align, table slots needed +// 3. allocate __memory_base by calling core malloc, and __table_base by +// appending to the shared funcref table +// 4. build the unit's import vector: env.memory / env.__indirect_function_table / +// env.__stack_pointer from core; env.* functions from core exports; +// GOT.mem.* and GOT.func.* through the same rules as Phase 0 +// 5. instantiate unit, patch deferred GOT entries, run relocations/ctors +// 6. prepare a UCEB1 request context in core, set the unit's Request*, call +// __uce_render, then read output back out of core-owned linear memory +// +// Everything here is deliberately validation-grade: fail loudly, never guess. + +#include + +#include +#include +#include +#include +#include +#include + +#define FAIL(...) do { fprintf(stderr, "FAIL: " __VA_ARGS__); fprintf(stderr, "\n"); exit(1); } while(0) +#define CHECK(cond, ...) do { if(!(cond)) FAIL(__VA_ARGS__); } while(0) + +static std::vector read_file(const char* fn) +{ + FILE* f = fopen(fn, "rb"); + CHECK(f, "cannot open %s", fn); + fseek(f, 0, SEEK_END); + long n = ftell(f); + fseek(f, 0, SEEK_SET); + std::vector buf(n); + CHECK(fread(buf.data(), 1, n, f) == (size_t)n, "short read on %s", fn); + fclose(f); + return buf; +} + +// ---- minimal dylink.0 parser ------------------------------------------- + +struct DylinkInfo +{ + uint32_t mem_size = 0; + uint32_t mem_align = 0; // power of 2 + uint32_t table_size = 0; + uint32_t table_align = 0; + bool found = false; +}; + +static uint64_t read_uleb(const uint8_t* buf, size_t& pos, size_t end) +{ + uint64_t result = 0; + int shift = 0; + while(true) + { + CHECK(pos < end, "truncated uleb in wasm custom section"); + CHECK(shift < 64, "oversized uleb in wasm custom section"); + uint8_t b = buf[pos++]; + result |= (uint64_t)(b & 0x7f) << shift; + if(!(b & 0x80)) + return result; + shift += 7; + } +} + +static DylinkInfo parse_dylink(const std::vector& wasm) +{ + DylinkInfo info; + CHECK(wasm.size() >= 8 && !memcmp(wasm.data(), "\0asm", 4), "not a wasm module"); + CHECK(wasm[4] == 1 && wasm[5] == 0 && wasm[6] == 0 && wasm[7] == 0, "unsupported wasm binary version"); + size_t pos = 8; + while(pos < wasm.size()) + { + uint8_t sec_id = wasm[pos++]; + uint64_t size = read_uleb(wasm.data(), pos, wasm.size()); + CHECK(size <= wasm.size() - pos, "wasm section exceeds file size"); + size_t end = pos + size; + if(sec_id == 0) + { + uint64_t name_len = read_uleb(wasm.data(), pos, end); + CHECK(name_len <= end - pos, "custom section name exceeds section size"); + std::string name((const char*)wasm.data() + pos, name_len); + pos += name_len; + if(name == "dylink.0") + { + while(pos < end) + { + uint8_t sub = wasm[pos++]; + uint64_t sub_len = read_uleb(wasm.data(), pos, end); + CHECK(sub_len <= end - pos, "dylink subsection exceeds section size"); + size_t sub_end = pos + sub_len; + if(sub == 1) // WASM_DYLINK_MEM_INFO + { + info.mem_size = read_uleb(wasm.data(), pos, sub_end); + info.mem_align = read_uleb(wasm.data(), pos, sub_end); + info.table_size = read_uleb(wasm.data(), pos, sub_end); + info.table_align = read_uleb(wasm.data(), pos, sub_end); + CHECK(!info.found, "duplicate dylink.0 mem_info subsection"); + CHECK(info.mem_align < 31 && info.table_align < 31, "unsupported dylink alignment"); + info.found = true; + } + pos = sub_end; + } + } + } + pos = end; + } + return info; +} + +// ---- tiny UCEB1 request-context encoder --------------------------------- + +static void append_varuint(std::string& out, uint64_t value) +{ + while(value >= 0x80) + { + out.push_back((char)((value & 0x7f) | 0x80)); + value >>= 7; + } + out.push_back((char)value); +} + +struct Node +{ + std::string scalar; + std::vector> children; +}; + +static Node scalar(const char* value) +{ + Node n; + n.scalar = value; + return n; +} + +static void encode_node(std::string& out, const Node& node) +{ + out.push_back(0); + append_varuint(out, node.scalar.size()); + out.append(node.scalar); + append_varuint(out, node.children.size()); + for(const auto& child : node.children) + { + append_varuint(out, child.first.size()); + out.append(child.first); + encode_node(out, child.second); + } +} + +static std::string make_context() +{ + Node params; + params.children.push_back({"HTTP_HOST", scalar("phase3.example.test")}); + params.children.push_back({"SCRIPT_URL", scalar("/spikes/wasm-phase3/page.uce")}); + Node nested; + nested.children.push_back({"answer", scalar("42")}); + Node root; + root.children.push_back({"params", params}); + root.children.push_back({"route", scalar("/spikes/wasm-phase3/page.uce")}); + root.children.push_back({"nested", nested}); + std::string out = "UCEB"; + out.push_back((char)1); + encode_node(out, root); + return out; +} + +// ---- host imports -------------------------------------------------------- + +static wasm_memory_t* g_memory = nullptr; +static std::string g_context; + +static void set_i32_result(wasm_val_t* result, int32_t value) +{ + result->kind = WASM_I32; + result->of.i32 = value; +} + +static wasm_trap_t* host_ctx_read(void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results) +{ + (void)env; + uint32_t ptr = args->data[0].of.i32; + uint32_t cap = args->data[1].of.i32; + if(ptr == 0 || cap == 0 || cap < g_context.size()) + { + set_i32_result(&results->data[0], (int32_t)g_context.size()); + return nullptr; + } + CHECK(g_memory, "host ctx_read called before memory export was captured"); + uint8_t* mem = (uint8_t*)wasm_memory_data(g_memory); + size_t mem_size = wasm_memory_data_size(g_memory); + CHECK((size_t)ptr + g_context.size() <= mem_size, "ctx_read buffer outside memory"); + memcpy(mem + ptr, g_context.data(), g_context.size()); + set_i32_result(&results->data[0], (int32_t)g_context.size()); + return nullptr; +} + +static wasm_trap_t* host_log(void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results) +{ + (void)env; (void)results; + int level = args->data[0].of.i32; + uint32_t ptr = args->data[1].of.i32; + uint32_t len = args->data[2].of.i32; + uint8_t* mem = (uint8_t*)wasm_memory_data(g_memory); + size_t mem_size = wasm_memory_data_size(g_memory); + if((size_t)ptr + len <= mem_size) + fprintf(stderr, "[guest log %d] %.*s\n", level, (int)len, (const char*)mem + ptr); + return nullptr; +} + +// ---- named trap stubs for unsatisfied (WASI) imports -------------------- + +static wasm_store_t* g_store_for_stubs = nullptr; + +static wasm_trap_t* stub_callback(void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results) +{ + (void)args; (void)results; + fprintf(stderr, "[stub called: %s]\n", (const char*)env); + char msg[256]; + snprintf(msg, sizeof(msg), "unimplemented host import called: %s", (const char*)env); + wasm_message_t message; + wasm_byte_vec_new(&message, strlen(msg) + 1, msg); + wasm_trap_t* trap = wasm_trap_new(g_store_for_stubs, &message); + wasm_byte_vec_delete(&message); + return trap; +} + +// ---- instance wrapper: name → extern map -------------------------------- + +struct Instance +{ + wasm_module_t* module = nullptr; + wasm_instance_t* instance = nullptr; + wasm_extern_vec_t exports = WASM_EMPTY_VEC; + std::map by_name; + + void index_exports() + { + wasm_exporttype_vec_t types = WASM_EMPTY_VEC; + wasm_module_exports(module, &types); + wasm_instance_exports(instance, &exports); + CHECK(types.size == exports.size, "export type/extern count mismatch"); + for(size_t i = 0; i < types.size; i++) + { + const wasm_name_t* nm = wasm_exporttype_name(types.data[i]); + std::string key(nm->data, nm->size); + // some wasm-c-api impls include the trailing NUL in name size + while(!key.empty() && key.back() == '\0') + key.pop_back(); + by_name[key] = exports.data[i]; + } + wasm_exporttype_vec_delete(&types); + } + + wasm_func_t* func(const char* name) + { + auto it = by_name.find(name); + return it == by_name.end() ? nullptr : wasm_extern_as_func(it->second); + } + wasm_global_t* global(const char* name) + { + auto it = by_name.find(name); + return it == by_name.end() ? nullptr : wasm_extern_as_global(it->second); + } +}; + +static wasm_store_t* g_store = nullptr; + +static void report_trap(wasm_trap_t* trap, const char* what) +{ + if(!trap) + return; + wasm_message_t msg; + wasm_trap_message(trap, &msg); + FAIL("trap during %s: %.*s", what, (int)msg.size, msg.data); +} + +static int32_t call_i32(Instance& inst, const char* name, std::vector argv = {}) +{ + wasm_func_t* f = inst.func(name); + CHECK(f, "missing export func %s", name); + wasm_val_t args_buf[4]; + CHECK(argv.size() <= 4, "call_i32 argv overflow for %s", name); + for(size_t i = 0; i < argv.size(); i++) + args_buf[i] = WASM_I32_VAL(argv[i]); + wasm_val_t results_buf[1] = { WASM_INIT_VAL }; + wasm_val_vec_t args = { argv.size(), args_buf }; + wasm_val_vec_t results = { 1, results_buf }; + size_t result_arity = wasm_func_result_arity(f); + wasm_val_vec_t no_results = WASM_EMPTY_VEC; + wasm_trap_t* trap = wasm_func_call(f, &args, result_arity ? &results : &no_results); + report_trap(trap, name); + return result_arity ? results_buf[0].of.i32 : 0; +} + +static wasm_global_t* make_i32_global(int32_t value, wasm_mutability_t mut) +{ + wasm_globaltype_t* gt = wasm_globaltype_new(wasm_valtype_new(WASM_I32), mut); + wasm_val_t val = WASM_I32_VAL(value); + wasm_global_t* g = wasm_global_new(g_store, gt, &val); + CHECK(g, "wasm_global_new failed (host-created globals unsupported?)"); + wasm_globaltype_delete(gt); + return g; +} + +int main(int argc, char** argv) +{ + const char* core_path = argc > 1 ? argv[1] : "/tmp/uce/wasm-phase3/core.wasm"; + const char* unit_path = argc > 2 ? argv[2] : "/tmp/uce/wasm-phase3/page.wasm"; + g_context = make_context(); + + wasm_engine_t* engine = wasm_engine_new(); + CHECK(engine, "engine"); + g_store = wasm_store_new(engine); + CHECK(g_store, "store"); + g_store_for_stubs = g_store; + + // ---- 1. core module --------------------------------------------------- + std::vector core_bytes = read_file(core_path); + wasm_byte_vec_t core_bv; + wasm_byte_vec_new(&core_bv, core_bytes.size(), (const char*)core_bytes.data()); + Instance core; + core.module = wasm_module_new(g_store, &core_bv); + wasm_byte_vec_delete(&core_bv); + CHECK(core.module, "core module load failed"); + + // the shared funcref table is host-created (core links with --import-table) + // because WAMR cannot grow a table from the host: size it up front as + // core's declared minimum (= its own elem needs) plus headroom for unit + // module table regions. __table_base allocation bumps from the minimum. + wasm_table_t* table = nullptr; + uint32_t table_next_free = 0; + const uint32_t TABLE_HEADROOM = 2048; + + wasm_importtype_vec_t core_imports = WASM_EMPTY_VEC; + wasm_module_imports(core.module, &core_imports); + std::vector core_import_externs(core_imports.size); + for(size_t i = 0; i < core_imports.size; i++) + { + const wasm_name_t* mod = wasm_importtype_module(core_imports.data[i]); + const wasm_name_t* nm = wasm_importtype_name(core_imports.data[i]); + const wasm_externtype_t* et = wasm_importtype_type(core_imports.data[i]); + std::string name(nm->data, nm->size); + if(wasm_externtype_kind(et) == WASM_EXTERN_TABLE) + { + CHECK(name.rfind("__indirect_function_table", 0) == 0, + "unexpected core table import %s", name.c_str()); + const wasm_tabletype_t* tt = wasm_externtype_as_tabletype_const(et); + uint32_t core_min = wasm_tabletype_limits(tt)->min; + wasm_limits_t limits = { core_min + TABLE_HEADROOM, core_min + TABLE_HEADROOM }; + wasm_tabletype_t* host_tt = wasm_tabletype_new(wasm_valtype_new(WASM_FUNCREF), &limits); + table = wasm_table_new(g_store, host_tt, nullptr); + CHECK(table, "wasm_table_new failed (host-created tables unsupported?)"); + wasm_tabletype_delete(host_tt); + table_next_free = core_min; + printf("host table created: size=%u, core region=[0,%u)\n", core_min + TABLE_HEADROOM, core_min); + core_import_externs[i] = wasm_table_as_extern(table); + continue; + } + CHECK(wasm_externtype_kind(et) == WASM_EXTERN_FUNC, + "core has unexpected non-func import %.*s.%s", + (int)mod->size, mod->data, name.c_str()); + const wasm_functype_t* ft = wasm_externtype_as_functype_const(et); + wasm_func_t* import_func = nullptr; + std::string mod_name(mod->data, mod->size); + while(!mod_name.empty() && mod_name.back() == '\0') mod_name.pop_back(); + while(!name.empty() && name.back() == '\0') name.pop_back(); + if(mod_name == "env" && name == "uce_host_ctx_read") + import_func = wasm_func_new_with_env(g_store, ft, host_ctx_read, nullptr, nullptr); + else if(mod_name == "env" && name == "uce_host_log") + import_func = wasm_func_new_with_env(g_store, ft, host_log, nullptr, nullptr); + else + { + // named trap stub; leaks the name string, fine for a spike + char* label = strdup((mod_name + "." + name).c_str()); + import_func = wasm_func_new_with_env(g_store, ft, stub_callback, label, nullptr); + } + CHECK(import_func, "stub func creation failed for %s", name.c_str()); + core_import_externs[i] = wasm_func_as_extern(import_func); + } + CHECK(table, "core does not import __indirect_function_table — rebuild with --import-table"); + wasm_extern_vec_t core_iv = { core_import_externs.size(), core_import_externs.data() }; + wasm_trap_t* trap = nullptr; + core.instance = wasm_instance_new(g_store, core.module, &core_iv, &trap); + report_trap(trap, "core instantiation"); + CHECK(core.instance, "core instantiation failed"); + core.index_exports(); + printf("core instantiated: %zu exports\n", core.by_name.size()); + + // reactor init (runs ctors). Wasmtime does not auto-run _initialize; + // WAMR does — and calling it twice trips wasi-libc's double-init guard + // (__builtin_trap → "unreachable"), which cost us a debugging round. + if(core.func("_initialize")) + call_i32(core, "_initialize"); + + wasm_memory_t* memory = wasm_extern_as_memory(core.by_name.count("memory") ? core.by_name["memory"] : nullptr); + CHECK(memory, "core does not export memory"); + g_memory = memory; + + // ---- 2./3. dylink + base allocation ------------------------------------ + std::vector unit_bytes = read_file(unit_path); + DylinkInfo dl = parse_dylink(unit_bytes); + CHECK(dl.found, "unit has no dylink.0 mem_info"); + printf("dylink.0: memsize=%u memalign=2^%u tablesize=%u\n", dl.mem_size, dl.mem_align, dl.table_size); + + uint32_t align = 1u << dl.mem_align; + uint32_t raw = (uint32_t)call_i32(core, "malloc", { (int32_t)(dl.mem_size + align) }); + CHECK(raw, "core malloc returned 0"); + uint32_t memory_base = (raw + (align - 1)) & ~(align - 1); + + uint32_t table_base = table_next_free; + table_next_free += dl.table_size; + CHECK(table_next_free <= wasm_table_size(table), "table headroom exhausted"); + printf("bases: __memory_base=%u __table_base=%u (table size %u)\n", + memory_base, table_base, wasm_table_size(table)); + + // ---- 4. unit import resolution ----------------------------------------- + wasm_byte_vec_t unit_bv; + wasm_byte_vec_new(&unit_bv, unit_bytes.size(), (const char*)unit_bytes.data()); + Instance unit; + unit.module = wasm_module_new(g_store, &unit_bv); + wasm_byte_vec_delete(&unit_bv); + CHECK(unit.module, "unit module load failed"); + + wasm_importtype_vec_t unit_imports = WASM_EMPTY_VEC; + wasm_module_imports(unit.module, &unit_imports); + std::vector unit_import_externs(unit_imports.size); + // GOT.mem entries that must be self-resolved from the unit's own exports + // after instantiation (weak data the core does not define) + std::vector> deferred_got; + + for(size_t i = 0; i < unit_imports.size; i++) + { + const wasm_name_t* mod_n = wasm_importtype_module(unit_imports.data[i]); + const wasm_name_t* nm_n = wasm_importtype_name(unit_imports.data[i]); + std::string mod(mod_n->data, mod_n->size); + std::string nm(nm_n->data, nm_n->size); + const wasm_externtype_t* et = wasm_importtype_type(unit_imports.data[i]); + wasm_externkind_t kind = wasm_externtype_kind(et); + wasm_extern_t* resolved = nullptr; + + if(mod == "env" && nm == "memory") + resolved = wasm_memory_as_extern(memory); + else if(mod == "env" && nm == "__indirect_function_table") + resolved = wasm_table_as_extern(table); + else if(mod == "env" && nm == "__stack_pointer") + { + CHECK(core.global("__stack_pointer"), "core does not export __stack_pointer"); + resolved = core.by_name["__stack_pointer"]; + } + else if(mod == "env" && nm == "__memory_base") + resolved = wasm_global_as_extern(make_i32_global((int32_t)memory_base, WASM_CONST)); + else if(mod == "env" && nm == "__table_base") + resolved = wasm_global_as_extern(make_i32_global((int32_t)table_base, WASM_CONST)); + else if(mod == "env" && kind == WASM_EXTERN_FUNC) + { + auto it = core.by_name.find(nm); + CHECK(it != core.by_name.end(), "unresolved unit func import env.%s", nm.c_str()); + resolved = it->second; + } + else if(mod == "GOT.mem") + { + wasm_global_t* src = core.global(nm.c_str()); + if(src) + { + wasm_val_t v; + wasm_global_get(src, &v); + resolved = wasm_global_as_extern(make_i32_global(v.of.i32, WASM_VAR)); + } + else + { + // provisional 0; patched from the unit's own export post-instantiation + wasm_global_t* g = make_i32_global(0, WASM_VAR); + deferred_got.push_back({ nm, g }); + resolved = wasm_global_as_extern(g); + } + } + else if(mod == "GOT.func") + { + // resolved guest-side: the core exports a helper returning + // (intptr_t)&func, which on wasm is the function's table index — + // no host-side funcref injection needed (WAMR forbids it anyway) + std::string helper = "core_table_index_of_" + nm; + CHECK(core.func(helper.c_str()), "no GOT.func resolver %s in core", helper.c_str()); + int32_t slot = call_i32(core, helper.c_str()); + CHECK(slot > 0, "GOT.func resolver %s returned %d", helper.c_str(), slot); + printf("resolved GOT.func.%s = table[%d]\n", nm.c_str(), slot); + resolved = wasm_global_as_extern(make_i32_global(slot, WASM_VAR)); + } + CHECK(resolved, "unhandled unit import %s.%s (kind %d)", mod.c_str(), nm.c_str(), (int)kind); + unit_import_externs[i] = resolved; + } + + // ---- 5. instantiate unit, patch GOT, run init --------------------------- + wasm_extern_vec_t unit_iv = { unit_import_externs.size(), unit_import_externs.data() }; + trap = nullptr; + unit.instance = wasm_instance_new(g_store, unit.module, &unit_iv, &trap); + report_trap(trap, "unit instantiation"); + CHECK(unit.instance, "unit instantiation failed"); + unit.index_exports(); + printf("unit instantiated: %zu exports\n", unit.by_name.size()); + + for(auto& [nm, got] : deferred_got) + { + wasm_global_t* own = unit.global(nm.c_str()); + CHECK(own, "GOT.mem.%s defined neither by core nor by unit", nm.c_str()); + wasm_val_t v; + wasm_global_get(own, &v); + // dylink ABI: a PIC module's exported data symbols are offsets relative + // to its __memory_base; the linker must add the base when resolving + wasm_val_t nv = WASM_I32_VAL((int32_t)(memory_base + (uint32_t)v.of.i32)); + wasm_global_set(got, &nv); + printf("self-resolved GOT.mem.%s = %u (offset %d)\n", nm.c_str(), memory_base + (uint32_t)v.of.i32, v.of.i32); + } + + if(unit.func("__wasm_apply_data_relocs")) + call_i32(unit, "__wasm_apply_data_relocs"); + if(unit.func("__wasm_call_ctors")) + call_i32(unit, "__wasm_call_ctors"); + + // ---- 6. membrane prepare, render and read back --------------------------- + int32_t rc = call_i32(core, "uce_phase3_prepare"); + CHECK(rc == 0, "uce_phase3_prepare returned %d", rc); + int32_t request_ptr = call_i32(core, "uce_phase3_request"); + CHECK(request_ptr != 0, "core returned null Request*"); + call_i32(unit, "__uce_set_current_request", { request_ptr }); + call_i32(unit, "__uce_render", { request_ptr }); + call_i32(core, "uce_phase3_finish"); + + int32_t out_ptr = call_i32(core, "uce_phase3_output_data"); + int32_t out_len = call_i32(core, "uce_phase3_output_size"); + byte_t* mem = wasm_memory_data(memory); // may have moved if memory grew + + printf("---- phase3 unit output (%d bytes) ----\n%.*s\n--------------------------------------\n", + out_len, out_len, mem + out_ptr); + + bool ok = out_len > 0 && + memmem(mem + out_ptr, out_len, "PHASE3 PAGE OK", 14) && + memmem(mem + out_ptr, out_len, "host=phase3.example.test", 24) && + memmem(mem + out_ptr, out_len, "answer=42", 9) && + memmem(mem + out_ptr, out_len, "each=answer:42", 14) && + memmem(mem + out_ptr, out_len, "self-got=42", 11) && + memmem(mem + out_ptr, out_len, "map=wise", 8) && + memmem(mem + out_ptr, out_len, "callback=42", 11) && + memmem(mem + out_ptr, out_len, "got-func-ok", 11) && + memmem(mem + out_ptr, out_len, "fn=42", 5); + + printf(ok ? "PHASE3 EXIT CRITERION: PASS\n" : "PHASE3 EXIT CRITERION: FAIL (see output above)\n"); + return ok ? 0 : 1; +} diff --git a/spikes/wasm-phase3/page.uce b/spikes/wasm-phase3/page.uce new file mode 100644 index 0000000..457da9e --- /dev/null +++ b/spikes/wasm-phase3/page.uce @@ -0,0 +1,12 @@ +// Phase 3 generated-unit fixture source. The checked-in generated/page.uce.cpp +// is the UCE-preprocessor-shaped output used by the wasm side-module build. + +RENDER(Request& context) +{ + <>
+

PHASE3 PAGE OK

+

host=

+

route=

+

answer=

+
+} diff --git a/spikes/wasm-phase4/README.md b/spikes/wasm-phase4/README.md new file mode 100644 index 0000000..1ce9511 --- /dev/null +++ b/spikes/wasm-phase4/README.md @@ -0,0 +1,77 @@ +# spikes/wasm-phase4 — production mechanics kill-test spike + +Phase 4 validates the operational mechanics needed before a WASM worker can be +trusted in production. This spike is intentionally smaller than the future UCE +worker integration: it uses tiny WAT modules instead of generated UCE units, but +it exercises the Wasmtime controls and workspace cleanup paths the worker will +use. It is also deliberately the first spike on the Wasmtime-specific C++ API +(`wasmtime.hh`) — fuel, epochs, store limiters, and snapshots do not exist in +the portable `wasm.h` surface, so runtime-agnosticism ends here by design. + +Run on `k-uce`: + +```bash +bash spikes/wasm-phase4/build_runner.sh +/tmp/uce/wasm-phase4/runner +``` + +Expected final line: + +```text +PHASE4 EXIT CRITERION: PASS +``` + +What this proves: + +- **Reusable compiled artifact / snapshot proxy:** modules are compiled once and + then instantiated in fresh stores. Not OS CoW yet, but it validates the shape + of "shared core artifact + per-request workspace birth". +- **Unharmed worker:** after all six kill cases, the same engine serves a + healthy request that completes with the expected value. This — not merely + "didn't crash" — is the worker half of the Phase 4 exit criterion. +- **CPU limits, both mechanisms:** fuel (deterministic, per-instruction cost) + and epoch interruption (near-zero overhead, ticker thread, traps as + `interrupt`). Production default is **epoch** per the Phase 0 findings; fuel + is validated as the deterministic fallback. Note: the engine epoch only + advances, so every store must set its own deadline. +- **Memory limit, actually load-bearing:** the OOM fixture grows by 10 pages — + within its declared max (100) — so only the store limiter (2 pages) can deny + the growth. The denial is observed by the guest (`memory.grow` → -1) and + converted to a trap. +- **Trap-to-error-page data path:** every kill case's gate asserts the captured + message contains a `wasm backtrace` and the expected cause. Fixture frames + show `` — readable production traces require units to keep their + name section (or a symbolication side-file in the artifact cache). +- **Trace summarizer (`src/lib/wasm_trace.h`):** trap messages are rendered + through the production collapse facility (repeated frames → `×N` lines, + mangled symbols demangled, cause/detail split out). The stack-exhaustion + case gates it on live trap output here; `site/tests/core.uce` gates the + parsing on canned messages in the native suite. +- **Handle cleanup with falsifiable checks:** closers must run exactly once per + handle and *while the store is still alive* (production closers may flush + guest-resident state); destructor-only cleanup fails the ordering check. + +Kill fixtures (all genuinely trap): + +- `unreachable`: `__builtin_trap` analog. +- `oob-access`: wild pointer; load at 128 KiB from a 64 KiB memory. +- `stack-exhaustion`: runaway recursion → `call stack exhausted`. +- `infinite-loop-fuel` / `infinite-loop-epoch`: same loop, both CPU limits. +- `oom-limiter`: limiter-denied `memory.grow` → guest converts -1 to a trap. + +**A literal C++ null-pointer dereference is deliberately absent:** address 0 is +valid wasm linear memory, so `*(int*)nullptr` does not trap — it silently +writes inside the workspace, which is then dropped at request end (still +strictly better than a native SIGSEGV). See WASM-PROPOSAL §10 for the recorded +risk/policy. + +Still deferred to production Phase 4: + +- real core snapshot + OS-level CoW birth; +- wiring `wasm_trace.h` summaries into the UCE error-page UI (the summarizer + itself is done and gated); name-section policy for unit artifacts; +- wiring these traps into `linux_fastcgi.cpp` / the WASM worker backend; +- real runtime handle-table cleanup for sqlite/mysql/sockets/tasks; +- artifact ABI versioning end-to-end; +- memory-limit policy for guest allocators that return failure instead of + trapping (bump allocator vs dlmalloc behavior under the limiter). diff --git a/spikes/wasm-phase4/build_runner.sh b/spikes/wasm-phase4/build_runner.sh new file mode 100755 index 0000000..5f41037 --- /dev/null +++ b/spikes/wasm-phase4/build_runner.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Phase 4: build the Wasmtime kill-test runner. +set -e +cd "$(dirname "$0")" + +OUT=/tmp/uce/wasm-phase4 +mkdir -p "$OUT" + +c++ -std=c++17 -O2 -pthread runner.cpp \ + -I/opt/wasmtime/include \ + -L/opt/wasmtime/lib \ + -Wl,-rpath,/opt/wasmtime/lib \ + -lwasmtime \ + -o "$OUT/runner" + +echo "built: $OUT/runner" diff --git a/spikes/wasm-phase4/runner.cpp b/spikes/wasm-phase4/runner.cpp new file mode 100644 index 0000000..5ef0ca0 --- /dev/null +++ b/spikes/wasm-phase4/runner.cpp @@ -0,0 +1,298 @@ +// WASM-PROPOSAL Phase 4 — production-mechanics kill-test spike. +// +// Uses Wasmtime's C++ API directly (this spike is deliberately where the +// portable wasm.h surface ends: fuel, epochs, store limiters, and snapshots +// are Wasmtime-specific by nature; the runtime was selected in Phase 0). +// +// Validates, per kill case, that: the guest trap is captured with a wasm +// backtrace, the workspace handle closers run exactly once and while the +// store is still alive, and — after all kills — the same engine still serves +// a healthy request (the actual meaning of "unharmed worker"). +// +// Note: a literal C++ null-pointer dereference does NOT trap in wasm — +// address 0 is valid linear memory; the damage stays inside the dropped +// workspace (see WASM-PROPOSAL §10). The kill fixtures below therefore use +// faults that genuinely trap: unreachable (__builtin_trap analog), an +// out-of-bounds access (wild pointer), stack exhaustion (runaway recursion), +// fuel/epoch exhaustion (infinite loop), and a limiter-denied memory.grow. + +#include + +#include +#include +#include +#include +#include + +#include "../../src/lib/wasm_trace.h" + +using namespace wasmtime; + +// Handle-table proxy. The checks here are deliberately falsifiable: closers +// must run exactly once per handle, and they must run while the store (and +// thus guest memory) is still alive — production closers may need to flush +// guest-resident state. A destructor-only cleanup fails the ordering check. +struct Workspace +{ + int handles_open = 0; + int handles_closed = 0; + bool cleanup_ran = false; + bool closed_while_store_alive = false; + + void open_handles(int count) + { + handles_open = count; + } + + void cleanup(bool store_alive) + { + if(cleanup_ran) + return; + cleanup_ran = true; + closed_while_store_alive = store_alive; + for(int i = 0; i < handles_open; i++) + handles_closed++; + } + + ~Workspace() + { + cleanup(false); + } + + bool ok() const + { + return(cleanup_ran && handles_closed == handles_open && closed_while_store_alive); + } +}; + +static Module compile_wat(Engine& engine, const std::string& wat) +{ + auto module = Module::compile(engine, wat); + if(!module) + { + std::cerr << "compile failed: " << module.err_ref().message() << "\n"; + exit(1); + } + return module.ok(); +} + +struct RunResult +{ + bool trapped = false; + std::string message; + bool cleanup_ok = false; + bool has_value = false; + int32_t value = 0; +}; + +static RunResult run_case(Engine& engine, const Module& module, uint64_t fuel, int64_t memory_limit, bool epoch_ticker = false) +{ + RunResult result; + Workspace workspace; + workspace.open_handles(2); + { + Store store(engine); + store.limiter(memory_limit, -1, -1, -1, -1); + Store::Context cx(store); + // epoch_interruption is enabled engine-wide and the engine epoch only + // advances; every store needs a deadline beyond the current epoch or + // it traps immediately. The epoch kill case gets a deadline of 1 tick + // and a ticker thread; everything else gets effectively-unbounded. + cx.set_epoch_deadline(epoch_ticker ? 1 : 1'000'000'000); + auto fuel_result = cx.set_fuel(fuel); + if(!fuel_result) + { + result.trapped = true; + result.message = fuel_result.err_ref().message(); + workspace.cleanup(true); + result.cleanup_ok = workspace.ok(); + return result; + } + + auto instance = Instance::create(cx, module, {}); + if(!instance) + { + result.trapped = true; + result.message = instance.err_ref().message(); + workspace.cleanup(true); + result.cleanup_ok = workspace.ok(); + return result; + } + + auto run_export = instance.ok_ref().get(cx, "run"); + if(!run_export || !std::get_if(&*run_export)) + { + result.trapped = true; + result.message = "missing run export"; + workspace.cleanup(true); + result.cleanup_ok = workspace.ok(); + return result; + } + + std::thread ticker; + if(epoch_ticker) + ticker = std::thread([&engine] { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + engine.increment_epoch(); + }); + + Func run = *std::get_if(&*run_export); + auto call = run.call(cx, std::vector{}); + if(ticker.joinable()) + ticker.join(); + if(!call) + { + result.trapped = true; + result.message = call.err_ref().message(); + } + else + { + result.message = "completed without trap"; + auto values = call.ok(); + if(!values.empty()) + { + result.has_value = true; + result.value = values[0].i32(); + } + } + workspace.cleanup(true); + result.cleanup_ok = workspace.ok(); + } + return result; +} + +static void print_case(const char* label, const RunResult& result) +{ + std::cout << "--- " << label << " ---\n"; + std::cout << "trapped=" << (result.trapped ? "yes" : "no") << "\n"; + if(result.trapped) + std::cout << "summary:\n" << wasm_trace_collapse(result.message) << "\n"; + else + std::cout << "message=" << result.message << "\n"; + std::cout << "cleanup=" << (result.cleanup_ok ? "ok" : "failed") << "\n"; +} + +static void check_kill(const char* label, const RunResult& result, const char* expect_in_message) +{ + print_case(label, result); + bool has_trace = result.message.find("wasm backtrace") != std::string::npos; + bool has_cause = result.message.find(expect_in_message) != std::string::npos; + if(!result.trapped || !result.cleanup_ok || !has_trace || !has_cause) + { + std::cerr << "PHASE4 EXIT CRITERION: FAIL at " << label + << (result.trapped ? "" : " (no trap)") + << (result.cleanup_ok ? "" : " (cleanup)") + << (has_trace ? "" : " (no backtrace)") + << (has_cause ? "" : " (wrong cause)") << "\n"; + exit(1); + } +} + +static void check_healthy(const char* label, const RunResult& result) +{ + print_case(label, result); + if(result.trapped || !result.cleanup_ok || !result.has_value || result.value != 42) + { + std::cerr << "PHASE4 EXIT CRITERION: FAIL at " << label << " (worker harmed)\n"; + exit(1); + } +} + +int main() +{ + Config config; + config.consume_fuel(true); + config.epoch_interruption(true); + Engine engine(std::move(config)); + + // Compiled modules are reused across stores: a deliberately small proxy + // for the future core snapshot — per-request workspaces (stores) are + // born fresh while compilation work is shared. + + // __builtin_trap analog; a real C++ *nullptr does not trap (see header) + Module unreachable_module = compile_wat(engine, R"wat( + (module + (func (export "run") + unreachable)) + )wat"); + + // wild/OOB pointer: load at 128 KiB from a 64 KiB memory + Module oob_module = compile_wat(engine, R"wat( + (module + (memory 1 1) + (func (export "run") + (drop (i32.load (i32.const 131072))))) + )wat"); + + // runaway recursion → call stack exhausted + Module stack_module = compile_wat(engine, R"wat( + (module + (func $f (export "run") + (call $f))) + )wat"); + + Module loop_module = compile_wat(engine, R"wat( + (module + (func (export "run") + (loop br 0))) + )wat"); + + // grows by 10 pages: within the module's own declared max (100), so only + // the store limiter (2 pages) can deny it — this proves the limiter is + // load-bearing, not the declared max + Module oom_module = compile_wat(engine, R"wat( + (module + (memory 1 100) + (func (export "run") + i32.const 10 + memory.grow + i32.const -1 + i32.eq + (if (then unreachable)))) + )wat"); + + Module healthy_module = compile_wat(engine, R"wat( + (module + (func (export "run") (result i32) + i32.const 42)) + )wat"); + + const int64_t MEM_LIMIT = 2 * 65536; + const uint64_t FUEL = 10'000; + const uint64_t FUEL_PLENTY = 1'000'000'000'000; + + check_kill("unreachable", run_case(engine, unreachable_module, FUEL, MEM_LIMIT), "unreachable"); + check_kill("oob-access", run_case(engine, oob_module, FUEL, MEM_LIMIT), "out of bounds"); + + RunResult stack_result = run_case(engine, stack_module, FUEL_PLENTY, MEM_LIMIT); + check_kill("stack-exhaustion", stack_result, "call stack exhausted"); + + // the trace formatter (src/lib/wasm_trace.h) is part of the gate: the + // recursion frames (Wasmtime caps the displayed backtrace at 20) must + // collapse to a bounded summary + WasmTraceSummary stack_summary = wasm_trace_summarize(stack_result.message); + if(!stack_summary.parsed || stack_summary.total_frames < 10 + || stack_summary.frames.size() > 12 + || stack_summary.cause.find("call stack exhausted") == std::string::npos + || wasm_trace_format(stack_summary).find("×") == std::string::npos) + { + std::cerr << "PHASE4 EXIT CRITERION: FAIL at trace-collapse (parsed=" + << stack_summary.parsed << " total=" << stack_summary.total_frames + << " lines=" << stack_summary.frames.size() + << " cause=" << stack_summary.cause << ")\n"; + exit(1); + } + std::cout << "--- trace-collapse ---\n" << stack_summary.total_frames + << " raw frames -> " << stack_summary.frames.size() << " summary lines\n"; + check_kill("infinite-loop-fuel", run_case(engine, loop_module, FUEL, MEM_LIMIT), "fuel"); + // Wasmtime reports epoch-deadline traps as "interrupt" + check_kill("infinite-loop-epoch", run_case(engine, loop_module, FUEL_PLENTY, MEM_LIMIT, true), "interrupt"); + check_kill("oom-limiter", run_case(engine, oom_module, FUEL, MEM_LIMIT), "unreachable"); + + // after every kill above, the same engine must still serve a request — + // this is the "unharmed worker" half of the Phase 4 exit criterion + check_healthy("healthy-after-kills", run_case(engine, healthy_module, FUEL, MEM_LIMIT)); + + std::cout << "PHASE4 EXIT CRITERION: PASS\n"; + return 0; +} diff --git a/spikes/wasm-phase5/README.md b/spikes/wasm-phase5/README.md new file mode 100644 index 0000000..212cd0b --- /dev/null +++ b/spikes/wasm-phase5/README.md @@ -0,0 +1,45 @@ +# spikes/wasm-phase5 — parity, audit, and performance harness + +Phase 5's production exit requires the full network suite to pass on the WASM +worker and performance numbers to be published. The production worker is not in +this branch yet, so this spike builds the Phase 5 harness and records the native +baseline that the WASM worker must match. + +Run on `k-uce`: + +```bash +bash spikes/wasm-phase5/run_phase5.sh +``` + +Expected final line: + +```text +PHASE5 HARNESS: PASS +``` + +Artifacts are written under `/tmp/uce/wasm-phase5/`: + +- `native-network.json` — full native network suite result. +- `native-starter.json` — starter-focused parity subset; this protects against + a vacuous `--match starter` exit gate. +- `site-static-audit.{json,md}` — candidate cross-request/static-state risks in + `site/` for the §3.2 semantic change audit. +- `benchmark.{json,md}` — warmed native baseline for the three Phase 5 budget + pages: + - `template-heavy-doc`: `/doc/singlepage.uce` + - `sqlite-page`: `/demo/sqlite.uce` + - `component-heavy-starter`: `/examples/uce-starter/?dashboard` + +`benchmark.py` also accepts `--wasm-base-url` once a WASM worker endpoint exists. +When provided, it compares WASM medians against the Phase 5 budget of ≤2× native +page latency. Workspace birth and internal component call overhead budgets still +need worker-internal probes; this harness documents the gap rather than faking +those numbers. + +Current scope: + +- Native parity and baseline collection are automated. +- WASM parity/performance comparison is ready but blocked on the production WASM + worker endpoint. +- The static-state audit is heuristic and intentionally conservative; findings + must be reviewed by a human before migration work is scheduled. diff --git a/spikes/wasm-phase5/audit_site_statics.py b/spikes/wasm-phase5/audit_site_statics.py new file mode 100755 index 0000000..069ccd4 --- /dev/null +++ b/spikes/wasm-phase5/audit_site_statics.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Phase 5 audit for cross-request/static state risks in site files.""" +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SITE = ROOT / "site" + +PATTERNS = [ + ("static local/global", "static "), + ("once hook", "ONCE("), + ("init hook", "INIT("), + ("background task", "task_"), +] + +SKIP_PARTS = {".git", "tmp", "work", "bin", "pkg", "__pycache__"} + + +@dataclass +class Finding: + path: str + line: int + kind: str + text: str + note: str + + +def iter_files() -> list[Path]: + result: list[Path] = [] + for path in SITE.rglob("*"): + if not path.is_file(): + continue + if any(part in SKIP_PARTS for part in path.parts): + continue + if path.suffix not in {".uce", ".h", ".txt"}: + continue + result.append(path) + return sorted(result) + + +def note_for(kind: str, text: str) -> str: + if kind in {"once hook", "init hook"}: + return "Audit behavior under per-request wasm workspaces; ONCE/INIT may need host-side cache semantics if used for cross-request state." + if kind == "background task": + return "Task APIs cross request lifetimes; verify they are host handles, not guest statics." + return "Check whether state is request-local, immutable, or intentionally persistent; unit statics reset per wasm workspace." + + +def scan() -> list[Finding]: + findings: list[Finding] = [] + for path in iter_files(): + try: + lines = path.read_text(encoding="utf-8").splitlines() + except UnicodeDecodeError: + continue + for lineno, line in enumerate(lines, 1): + stripped = line.strip() + if stripped.startswith("//") or stripped.startswith("# "): + continue + for kind, needle in PATTERNS: + if needle in line: + findings.append(Finding( + path=str(path.relative_to(ROOT)), + line=lineno, + kind=kind, + text=stripped[:180], + note=note_for(kind, stripped), + )) + return findings + + +def write_reports(findings: list[Finding], out_dir: Path) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "site-static-audit.json").write_text(json.dumps([asdict(f) for f in findings], indent=2) + "\n") + lines = ["# Phase 5 site static-state audit", ""] + if not findings: + lines.append("No candidate cross-request/static-state patterns found.") + else: + lines.extend(["| file | line | kind | code | note |", "|---|---:|---|---|---|"]) + for f in findings: + code = f.text.replace("|", "\\|") + note = f.note.replace("|", "\\|") + lines.append(f"| {f.path} | {f.line} | {f.kind} | `{code}` | {note} |") + (out_dir / "site-static-audit.md").write_text("\n".join(lines) + "\n") + + +def main() -> int: + out_dir = Path("/tmp/uce/wasm-phase5") + findings = scan() + write_reports(findings, out_dir) + print(f"Found {len(findings)} candidate static/cross-request patterns") + print(f"wrote {out_dir / 'site-static-audit.json'} and {out_dir / 'site-static-audit.md'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/spikes/wasm-phase5/benchmark.py b/spikes/wasm-phase5/benchmark.py new file mode 100755 index 0000000..379a05a --- /dev/null +++ b/spikes/wasm-phase5/benchmark.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Phase 5 native/WASM benchmark harness. + +This runner intentionally has no third-party dependencies. It records a warmed +native HTTP baseline now and can compare against a future wasm worker by passing +--wasm-base-url. Results are written as JSON plus a small Markdown table. +""" +from __future__ import annotations + +import argparse +import http.client +import json +import statistics +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from urllib.parse import urlparse + + +@dataclass +class Target: + name: str + url: str + expected: str + + +@dataclass +class Measurement: + backend: str + target: str + url: str + ok: bool + status: int + samples_ms: list[float] + median_ms: float + mean_ms: float + min_ms: float + max_ms: float + note: str = "" + + +def request_once(base_url: str, host_header: str, path: str, timeout: float) -> tuple[int, bytes, float]: + parsed = urlparse(base_url) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + raise SystemExit(f"invalid base url: {base_url}") + port = parsed.port or (443 if parsed.scheme == "https" else 80) + conn_cls = http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection + headers = {} + if host_header: + headers["Host"] = host_header + started = time.perf_counter() + conn = conn_cls(parsed.hostname, port, timeout=timeout) + try: + conn.request("GET", path, headers=headers) + response = conn.getresponse() + body = response.read() + elapsed_ms = (time.perf_counter() - started) * 1000.0 + return response.status, body, elapsed_ms + finally: + conn.close() + + +def measure_backend(backend: str, base_url: str, host_header: str, targets: list[Target], warmups: int, samples: int, timeout: float) -> list[Measurement]: + results: list[Measurement] = [] + for target in targets: + path = urlparse(target.url).path + if "?" in target.url: + path = target.url + for _ in range(warmups): + request_once(base_url, host_header, target.url, timeout) + durations: list[float] = [] + status = 0 + ok = True + note = "" + for _ in range(samples): + status, body, elapsed_ms = request_once(base_url, host_header, target.url, timeout) + durations.append(elapsed_ms) + text = body.decode("utf-8", errors="replace") + if status != 200: + ok = False + note = f"HTTP {status}" + elif target.expected and target.expected not in text: + ok = False + note = f"missing marker {target.expected!r}" + results.append(Measurement( + backend=backend, + target=target.name, + url=target.url, + ok=ok, + status=status, + samples_ms=durations, + median_ms=statistics.median(durations), + mean_ms=statistics.fmean(durations), + min_ms=min(durations), + max_ms=max(durations), + note=note, + )) + return results + + +def compare(results: list[Measurement]) -> list[str]: + native = {r.target: r for r in results if r.backend == "native"} + lines = ["| target | backend | median ms | mean ms | budget | status |", "|---|---:|---:|---:|---|---|"] + for result in results: + budget = "baseline" + status = "PASS" if result.ok else "FAIL" + if result.backend != "native" and result.target in native: + limit = native[result.target].median_ms * 2.0 + budget = f"≤ {limit:.1f} ms" + if result.median_ms > limit: + status = "FAIL" + lines.append(f"| {result.target} | {result.backend} | {result.median_ms:.1f} | {result.mean_ms:.1f} | {budget} | {status} {result.note} |") + return lines + + +def build_targets() -> list[Target]: + return [ + Target("template-heavy-doc", "/doc/singlepage.uce", "UCE API"), + Target("sqlite-page", "/demo/sqlite.uce", "SQLite"), + Target("component-heavy-starter", "/examples/uce-starter/?dashboard", "Dashboard"), + ] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Phase 5 native/wasm benchmark harness") + parser.add_argument("--native-base-url", default="http://localhost:80") + parser.add_argument("--wasm-base-url", default="", help="optional wasm worker URL; omitted until worker exists") + parser.add_argument("--host-header", default="uce.openfu.com") + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--samples", type=int, default=5) + parser.add_argument("--timeout", type=float, default=10.0) + parser.add_argument("--out-dir", default="/tmp/uce/wasm-phase5") + args = parser.parse_args() + + targets = build_targets() + results = measure_backend("native", args.native_base_url, args.host_header, targets, args.warmups, args.samples, args.timeout) + if args.wasm_base_url: + results.extend(measure_backend("wasm", args.wasm_base_url, args.host_header, targets, args.warmups, args.samples, args.timeout)) + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "benchmark.json").write_text(json.dumps([asdict(r) for r in results], indent=2) + "\n") + md_lines = ["# Phase 5 benchmark report", "", *compare(results), ""] + if not args.wasm_base_url: + md_lines.append("WASM worker URL was not provided; this report is the native baseline that future wasm runs compare against.") + (out_dir / "benchmark.md").write_text("\n".join(md_lines) + "\n") + print("\n".join(md_lines)) + print(f"wrote {out_dir / 'benchmark.json'} and {out_dir / 'benchmark.md'}") + return 0 if all(r.ok for r in results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/spikes/wasm-phase5/run_phase5.sh b/spikes/wasm-phase5/run_phase5.sh new file mode 100755 index 0000000..2bcf391 --- /dev/null +++ b/spikes/wasm-phase5/run_phase5.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Phase 5 harness: native parity, site static audit, and native perf baseline. +# Run on k-uce from repo root. +set -euo pipefail +cd "$(dirname "$0")/../.." + +OUT=/tmp/uce/wasm-phase5 +mkdir -p "$OUT" + +python3 tests/run_network_tests.py --include-internal --json-report "$OUT/native-network.json" +python3 tests/run_network_tests.py --include-internal --match starter --json-report "$OUT/native-starter.json" +python3 spikes/wasm-phase5/audit_site_statics.py +python3 spikes/wasm-phase5/benchmark.py --out-dir "$OUT" + +python3 - <<'PY' +import json +from pathlib import Path +out = Path('/tmp/uce/wasm-phase5') +network = json.loads((out / 'native-network.json').read_text()) +starter = json.loads((out / 'native-starter.json').read_text()) +bench = json.loads((out / 'benchmark.json').read_text()) +failures = [r for r in network if not r['ok']] + [r for r in starter if not r['ok']] + [r for r in bench if not r['ok']] +if failures: + print('PHASE5 HARNESS: FAIL') + for failure in failures: + print(failure) + raise SystemExit(1) +print('PHASE5 HARNESS: PASS') +print(f"network_cases={len(network)} starter_cases={len(starter)} benchmarks={len(bench)}") +print(f"reports={out}") +PY diff --git a/src/lib/uce_lib.h b/src/lib/uce_lib.h index 7130f2e..7be85d1 100644 --- a/src/lib/uce_lib.h +++ b/src/lib/uce_lib.h @@ -4,6 +4,7 @@ #include "types.h" #include "hash.h" #include "functionlib.h" +#include "wasm_trace.h" #include "sys.h" #include "uri.h" #include "cli.h" diff --git a/src/lib/wasm_trace.h b/src/lib/wasm_trace.h new file mode 100644 index 0000000..f102b71 --- /dev/null +++ b/src/lib/wasm_trace.h @@ -0,0 +1,226 @@ +#pragma once + +// Host-side summarizer for Wasmtime trap/error messages (WASM-PROPOSAL +// Phase 4: trap → error-page path). Turns a raw runtime message — which for +// stack exhaustion contains hundreds of identical frames — into a compact, +// error-page-ready summary: headline cause, optional detail (e.g. faulting +// address), and a backtrace with consecutive repeated frames collapsed. +// +// Pure string processing with no wasm-runtime dependency, so it is testable +// from the native suite and reusable by both the phase-4 spike runner and +// the future wasm worker. Tolerant of unrecognized input: an unparsed +// message is passed through as the cause, never dropped. + +#include +#include +#include + +#if defined(__GNUG__) +#include +#endif + +struct WasmTraceSummary +{ + std::string cause; // "wasm trap: ..." headline, or the raw message if unparsed + std::string detail; // secondary cause lines, e.g. "memory fault at wasm address ..." + std::vector frames; // collapsed display lines + size_t total_frames = 0; + bool parsed = false; // a wasm backtrace section was recognized +}; + +inline std::string wasm_trace_demangle(const std::string& name) +{ +#if defined(__GNUG__) + // frame names look like "module!symbol"; demangle the symbol part + auto bang = name.find('!'); + std::string prefix = bang == std::string::npos ? "" : name.substr(0, bang + 1); + std::string symbol = bang == std::string::npos ? name : name.substr(bang + 1); + if(symbol.rfind("_Z", 0) == 0) + { + int status = 0; + char* demangled = abi::__cxa_demangle(symbol.c_str(), 0, 0, &status); + if(status == 0 && demangled) + { + std::string result = prefix + demangled; + free(demangled); + return(result); + } + if(demangled) + free(demangled); + } +#endif + return(name); +} + +inline WasmTraceSummary wasm_trace_summarize(const std::string& message, size_t max_frame_lines = 12) +{ + WasmTraceSummary summary; + + struct Frame + { + std::string name; + std::string address; + }; + std::vector raw_frames; + std::vector cause_lines; + + auto trim = [](const std::string& s) { + size_t a = s.find_first_not_of(" \t\r"); + if(a == std::string::npos) + return(std::string()); + size_t b = s.find_last_not_of(" \t\r"); + return(s.substr(a, b - a + 1)); + }; + + // a frame or numbered-cause line starts with "N:"; strip that prefix + auto strip_index = [&](const std::string& line, bool& had_index) { + had_index = false; + size_t i = 0; + while(i < line.size() && line[i] >= '0' && line[i] <= '9') + i++; + if(i == 0 || i >= line.size() || line[i] != ':') + return(line); + had_index = true; + return(trim(line.substr(i + 1))); + }; + + bool in_backtrace = false; + bool in_caused_by = false; + size_t pos = 0; + while(pos <= message.size()) + { + size_t eol = message.find('\n', pos); + std::string line = trim(message.substr(pos, (eol == std::string::npos ? message.size() : eol) - pos)); + pos = eol == std::string::npos ? message.size() + 1 : eol + 1; + if(line.empty()) + continue; + + if(line.find("wasm backtrace") != std::string::npos) + { + summary.parsed = true; + in_backtrace = true; + in_caused_by = false; + continue; + } + if(line.rfind("Caused by", 0) == 0) + { + in_backtrace = false; + in_caused_by = true; + continue; + } + + bool had_index = false; + std::string body = strip_index(line, had_index); + if(in_backtrace && had_index) + { + Frame frame; + // optional "0xADDR - " prefix before the frame name + if(body.rfind("0x", 0) == 0) + { + size_t dash = body.find(" - "); + if(dash != std::string::npos) + { + frame.address = body.substr(0, dash); + body = trim(body.substr(dash + 3)); + } + } + frame.name = wasm_trace_demangle(body); + raw_frames.push_back(frame); + continue; + } + if(in_caused_by || line.rfind("wasm trap:", 0) == 0) + { + cause_lines.push_back(body); + continue; + } + } + + summary.total_frames = raw_frames.size(); + + // headline = the "wasm trap:" line; everything else in Caused by → detail + for(auto& line : cause_lines) + { + if(line.rfind("wasm trap:", 0) == 0) + summary.cause = line; + else + summary.detail += (summary.detail.empty() ? "" : "; ") + line; + } + if(summary.cause.empty()) + summary.cause = summary.parsed && !cause_lines.empty() ? cause_lines.front() : trim(message); + + // collapse consecutive identical frame names + struct Group + { + std::string name; + std::string address; + size_t first = 0; + size_t count = 0; + }; + std::vector groups; + for(size_t i = 0; i < raw_frames.size(); i++) + { + if(!groups.empty() && groups.back().name == raw_frames[i].name) + { + groups.back().count++; + continue; + } + Group group; + group.name = raw_frames[i].name; + group.address = raw_frames[i].address; + group.first = i; + group.count = 1; + groups.push_back(group); + } + + auto group_line = [](const Group& g) { + std::string line; + if(g.count == 1) + { + line = "#" + std::to_string(g.first); + if(!g.address.empty()) + line += " " + g.address; + line += " " + g.name; + } + else + { + line = "#" + std::to_string(g.first) + "..#" + std::to_string(g.first + g.count - 1) + + " " + g.name + " ×" + std::to_string(g.count); + } + return(line); + }; + + if(groups.size() <= max_frame_lines) + { + for(auto& g : groups) + summary.frames.push_back(group_line(g)); + } + else + { + // keep the top of the stack and the outermost group + for(size_t i = 0; i + 2 < max_frame_lines && i < groups.size(); i++) + summary.frames.push_back(group_line(groups[i])); + summary.frames.push_back("... " + std::to_string(groups.size() - (max_frame_lines - 1)) + " more frame groups ..."); + summary.frames.push_back(group_line(groups.back())); + } + + return(summary); +} + +inline std::string wasm_trace_format(const WasmTraceSummary& summary) +{ + std::string out = summary.cause; + if(!summary.detail.empty()) + out += "\n " + summary.detail; + if(!summary.frames.empty()) + { + out += "\nbacktrace (" + std::to_string(summary.total_frames) + " frame" + (summary.total_frames == 1 ? "" : "s") + "):"; + for(auto& line : summary.frames) + out += "\n " + line; + } + return(out); +} + +inline std::string wasm_trace_collapse(const std::string& message, size_t max_frame_lines = 12) +{ + return(wasm_trace_format(wasm_trace_summarize(message, max_frame_lines))); +} diff --git a/tests/plugins/uce_starter_parity.py b/tests/plugins/uce_starter_parity.py new file mode 100644 index 0000000..8cf08b6 --- /dev/null +++ b/tests/plugins/uce_starter_parity.py @@ -0,0 +1,66 @@ +# Parity harness for the uce-starter example app. +# +# WASM-PROPOSAL Phase 3/5 exit criterion: `run_network_tests.py --match starter` +# must pass against the wasm worker backend. These cases are written against +# the native backend first so the bar exists — and is green — before the wasm +# worker does. Keep assertions backend-agnostic (rendered content only), so +# the same cases gate both backends unchanged. + +ERROR_MARKERS = [ + "uce compile error", + "fatal signal during request", + "uncaught exception during request", +] + +BASE = "/examples/uce-starter/index.uce" + +# (case name suffix, query-string route, expected ) +VIEWS = [ + ("landing", "", "Home | UCE Starter"), + ("dashboard", "?dashboard", "Dashboard | UCE Starter"), + ("gauges", "?gauges", "Gauges | UCE Starter"), + ("features", "?features", "Features | UCE Starter"), + ("components", "?page1", "Components | UCE Starter"), + ("workspace", "?workspace", "Workspace | UCE Starter"), +] + + +def _check_body(body, title): + body_lower = body.lower() + for marker in ERROR_MARKERS: + if marker in body_lower: + raise Exception("response body contained error marker %r" % marker) + needle = "<title>%s" % title + if needle not in body: + raise Exception("expected %r in response body" % needle) + + +def _make_view_case(name, query, title): + def run(context): + response = context.expect_status(BASE + query, 200) + _check_body(response.text, title) + return "rendered %s with expected title and no error markers" % name + + return run + + +def _unknown_route_case(context): + # the starter app must render its own 404 view through the app shell, + # not fall through to a bare server error + response = context.expect_status(BASE + "?does-not-exist", 404) + _check_body(response.text, "404 Not Found | UCE Starter") + return "starter app rendered its own 404 page" + + +def register(registry): + for name, query, title in VIEWS: + registry.case( + "starter view " + name, + _make_view_case(name, query, title), + tags=["http", "uce", "starter", "parity"], + ) + registry.case( + "starter unknown route 404", + _unknown_route_case, + tags=["http", "uce", "starter", "parity"], + )