diff --git a/WASM-PROPOSAL.md b/WASM-PROPOSAL.md index dd63f51..6dcc533 100644 --- a/WASM-PROPOSAL.md +++ b/WASM-PROPOSAL.md @@ -410,8 +410,8 @@ Memory limit: linear memory max → allocation failure / trap → same path. (different artifact format). - The host-side connectors (sqlite/mysql) — already handle-shaped APIs; they move behind hostcalls with the same `.uce`-visible signatures. -- The site tree, docs, demo, and the network test suite - (`tests/run_network_tests.py`) — which becomes the parity harness (§9). +- The site tree, docs, demo, and the runtime smoke suite — now the UCE CLI + runner at `site/tests/cli_runner.uce` — which becomes the parity harness (§9). - nginx/FastCGI front-end integration, worker model, websocket event flow (events re-enter via the websocket entry point). @@ -507,8 +507,8 @@ correctly through the membrane. Scaffolding is marked throwaway. Implement §6 in full: dylink parsing, base allocation, GOT resolution, ABI/import verification, lazy mid-request loading, path dispatch. Per-request workspace birth/drop (plain memcpy birth is fine here; CoW is Phase 4). -Exit: the uce-starter renders end-to-end with components loading lazily; -`tests/run_network_tests.py --match starter` passes against the wasm worker. +Exit: the uce-starter renders end-to-end with components loading lazily; the +starter subset of the UCE CLI smoke suite passes against the wasm worker. > **Status: SPIKE PASS (2026-06-12).** `spikes/wasm-phase3/` combines the > Phase 0 dylink/PIC loader with the Phase 2 UCEB1 context membrane. The spike @@ -564,13 +564,12 @@ Exit: the uce-starter renders end-to-end with components loading lazily; > focused spike before worker integration. > 7. **FastCGI worker integration** — config-selectable backend, §7 lifecycle > wired into the `linux_fastcgi.cpp` flow. -> 8. **Starter parity tests — DONE (2026-06-12).** -> `tests/plugins/uce_starter_parity.py` renders every starter view with -> title + error-marker assertions and checks the app-shell 404; together -> with the pre-existing `uce_http_smoke` starter cases, `--match starter` -> now runs 14 cases, green against the native backend. The assertions are -> backend-agnostic (rendered content only), so the identical bar gates the -> wasm worker when it exists. +> 8. **Starter parity tests — DONE (2026-06-12; ported to UCE CLI 2026-06-13).** +> `site/tests/cli_runner.uce` renders every starter view with title + +> error-marker assertions and checks the app-shell 404; together with the +> starter HTTP smoke cases, the starter subset is 14 cases, green against +> the native backend. The assertions are backend-agnostic (rendered content +> only), so the identical bar gates the wasm worker. **Phase 4 — production mechanics.** Core snapshot + CoW birth; bump-allocator flag; epoch/memory limits; trap → @@ -798,11 +797,10 @@ stays default until W5): the §7 lifecycle, lazy mid-request `component_resolve` → load → `call_indirect`, path dispatch, trap → configured UCE error pages with collapsed guest traces, handle-table cleanup on workspace drop. -Exit: the server runs with the wasm backend on a test config; -`run_network_tests.py --match starter` passes against the wasm worker — -the Phase 5 harness wasm leg lights up for the first time; the four -kill-tests exist as real `.uce` pages and produce clean error pages from -an unharmed worker. +Exit: the server runs with the wasm backend on a test config; the starter +subset of the runtime smoke suite passes against the wasm worker — the Phase +5 harness wasm leg lights up for the first time; the four kill-tests exist as +real `.uce` pages and produce clean error pages from an unharmed worker. > **Status: DONE (2026-06-13).** `src/wasm/backend.cpp` wires the W3 runtime > into `src/linux_fastcgi.cpp` as a config-selectable page-render backend @@ -819,8 +817,8 @@ an unharmed worker. > it shares the component dispatch + ONCE path). > > **Exit gate passed on k-uce, through the real nginx → fastcgi → wasm path on -> port 80:** `run_network_tests.py --match starter` is **14/14** against the -> wasm backend (incl. the two ONCE-asset-in-`` cases); three real kill +> port 80:** the starter smoke/parity subset is **14/14** against the wasm +> backend (incl. the two ONCE-asset-in-`` cases); three real kill > pages under `site/tests/wasm-kill/` (OOB write, runaway loop, unbounded > recursion) each return a clean error page carrying a demangled > `wasm_trace` summary (`out of bounds memory access` / epoch `interrupt` @@ -936,8 +934,8 @@ tracking `operator new`, and `cleanup_*_connections()`. > **Status: SPIKE CLEANUP DONE; native-machinery retirement GATED > (2026-06-13).** Re-homing complete: the Phase 5 benchmark + site-audit are > now `tests/wasm_benchmark.py` / `tests/wasm_site_audit.py` (with -> `scripts/wasm/run_w5.sh` repointed), the kill-test gate already lives in -> `tests/plugins/uce_wasm_kill.py`, the Phase 0 toolchain findings + GOT +> `scripts/wasm/run_w5.sh` repointed), the kill-test gate lives in the UCE CLI +> runner at `site/tests/cli_runner.uce`, the Phase 0 toolchain findings + GOT > erratum are `docs/wasm-toolchain-findings.md`, and the pinned native > baselines are `docs/wasm-baselines/`. **All `spikes/wasm-phase*` deleted** > (prototypes superseded by `src/wasm/`), and the dead 2022 diff --git a/docs/cli-test-port-plan.md b/docs/cli-test-port-plan.md new file mode 100644 index 0000000..42f5535 --- /dev/null +++ b/docs/cli-test-port-plan.md @@ -0,0 +1,90 @@ +# CLI Unit Test Port Plan + +## Objective + +Replace the Python-based network test runner and plugins with UCE unit tests invoked through the runtime's CLI socket path, keeping equivalent coverage for HTTP smoke, site suites, security checks, starter parity, TCP/WebSocket listener probes, and wasm kill checks. The final invocation should be a bash script that calls UCE CLI units; Python test files should be removed only after UCE coverage is in place and validated. + +## Success Criteria + +- [x] A bash command runs the full test suite through the CLI socket and exits nonzero on failure. +- [x] UCE CLI tests cover every current Python plugin behavior or explicitly document a deliberate replacement. +- [x] Existing W5/WASM gates use the UCE CLI test runner instead of `tests/run_network_tests.py`. +- [x] Python test runner/plugins are deleted after parity validation. +- [x] Full suite passes on `uce-dev` with `WASM_BACKEND_ENABLED=1`. + +## Current State + +- Status: verifying +- Last updated: 2026-06-13 +- Source of truth: `/root/mount_ssh/uce-dev-root-htdocs-uce` +- Runtime/live target: `uce-dev:/Code/uce.openfu.com/uce` + +## Goal Tree + +Legend: `[ ]` not started, `[~]` in progress, `[x]` done, `[!]` blocked, `[-]` superseded + +- [x] G1: Inventory Python test coverage and CLI constraints + - Why: replacement must preserve coverage before deleting Python. + - Done when: every plugin has a mapped UCE equivalent or blocker. + - Verify: coverage matrix in this document. + - [x] G1.1: Delegate design/coverage review to Spark. + - [x] G1.2: Inspect delegates and reconcile plan. +- [x] G2: Build UCE CLI test harness + - Why: Python runner features need a UCE-native replacement. + - Done when: one CLI unit can list/run tests, print pass/fail summary, and return failing CLI status. + - Verify: `scripts/run_cli_tests.sh --list` and `scripts/run_cli_tests.sh --include-wasm-kill`. + - [x] G2.1: Add reusable UCE assertion/reporting helpers. + - [x] G2.2: Add HTTP/TCP helper functions using UCE socket APIs. + - [x] G2.3: Add bash wrapper under `scripts/`. +- [x] G3: Port current plugin cases to UCE + - Why: only delete Python after equivalent UCE coverage exists. + - Done when: UCE suite covers demo, HTTP docs/starter, site suites, security, starter parity, TCP, wasm kill. + - Verify: UCE CLI full run passes and output names match coverage matrix. +- [x] G4: Replace Python gate usage and delete Python tests + - Why: user explicitly requested eliminating the Python suite. + - Done when: scripts no longer call `tests/run_network_tests.py`, Python test files removed, validation green. + - Verify: `rg 'run_network_tests|tests/plugins|python3 tests'` has no obsolete gate references except historical docs and benchmark/audit utilities. +- [~] G5: Document and validate + - Why: future agents/operators need the new test workflow. + - Done when: docs/project notes and in-repo docs mention the CLI test command and validation artifact. + - Verify: docs committed, full suite run artifact recorded. + +## Coverage Matrix + +- `uce_demo_smoke.py` → `cli_run_demo_smoke()` in `site/tests/cli_runner.uce` (43 demo pages). +- `uce_http_smoke.py` → `cli_run_http_smoke()` (docs and starter route/body checks, 14 cases). +- `uce_site_suite.py` → `cli_run_site_suite()` (manifest-driven published site suite pages, 13 cases). +- `uce_security_smoke.py` → `cli_run_security_smoke()` (direct HTTP traversal/header spoofing, CRLF header sanitization, session hardening, 4 cases). +- `uce_starter_parity.py` → `cli_run_starter_parity()` (starter view title/404 checks, 7 cases). +- `uce_tcp_smoke.py` → `cli_run_tcp_smoke()` (port 80 and 8080 reachability, 2 cases). +- `uce_wasm_kill.py` → `cli_run_wasm_kill()` gated by `--include-wasm-kill` (trap/loop/recurse + post-kill health checks, 3 cases). + +## Execution Queue + +1. Run final no-Python-suite validation after removing stale references. +2. Commit UCE and project-doc updates. + +## Decisions + +- 2026-06-13: Use UCE CLI socket invocation as the test entrypoint; bash wrappers are acceptable, Python runner/plugins are not. +- 2026-06-13: Do not delete Python tests until UCE replacement validates equivalent coverage on `uce-dev`. + +## Assumptions + +- UCE socket APIs are sufficient for HTTP/1.0 probes, TCP connect checks, and security header injection checks. +- Bash can provide filtering/list convenience if exact Python CLI parity is not needed. + +## Blockers and Risks + +- [ ] The new runner intentionally does not preserve the old Python runner's dynamic plugin/tag/regex filtering; add UCE-side selectors later if operators miss them. +- [ ] `tests/wasm_benchmark.py` and `tests/wasm_site_audit.py` remain Python utility scripts, not the network test suite; port separately if a strict no-Python tools policy is desired. + +## Evidence and Verification Log + +- 2026-06-13: Prior to this plan, `b1856c1 chore: narrow wasm backend entrypoint API` was committed after build and focused services validation. +- 2026-06-13: `scripts/run_cli_tests.sh --include-wasm-kill` on `uce-dev` passed `86 passed, 0 failed, 0 skipped` (`/tmp/uce/cli-tests-final.txt`). + +## Change Log + +- 2026-06-13: Created initial goal tree for CLI unit test port. +- 2026-06-13: Added `site/tests/cli_runner.uce`, `scripts/run_cli_tests.sh`, removed Python network runner/plugins, and repointed W5 network gates to the CLI runner. diff --git a/scripts/run_cli_tests.sh b/scripts/run_cli_tests.sh new file mode 100755 index 0000000..ac91f42 --- /dev/null +++ b/scripts/run_cli_tests.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +socket_path="${UCE_CLI_SOCKET:-/run/uce/cli.sock}" +if [[ -z "${UCE_CLI_SOCKET:-}" && -r /etc/uce/settings.cfg ]]; then + configured_socket=$(awk -F= '/^[[:space:]]*CLI_SOCKET_PATH[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2; exit}' /etc/uce/settings.cfg) + if [[ -n "${configured_socket:-}" ]]; then + socket_path="$configured_socket" + fi +fi + +include_kill=0 +action="run" +while [[ $# -gt 0 ]]; do + case "$1" in + --include-wasm-kill|--include-kill) + include_kill=1 + shift + ;; + --list) + action="list" + shift + ;; + -h|--help) + cat <<'USAGE' +Usage: scripts/run_cli_tests.sh [--include-wasm-kill] [--list] + +Runs the UCE unit-based test suite through the runtime CLI socket. +USAGE + exit 0 + ;; + *) + echo "unknown option: $1" >&2 + exit 2 + ;; + esac +done + +if [[ ! -S "$socket_path" ]]; then + echo "UCE CLI socket not found: $socket_path" >&2 + exit 1 +fi + +url="http://localhost/tests/cli_runner.uce?action=${action}&include_kill=${include_kill}" +exec curl -sS --fail-with-body --unix-socket "$socket_path" "$url" diff --git a/scripts/wasm/run_w5.sh b/scripts/wasm/run_w5.sh index 7b4f3ea..81d1c5a 100755 --- a/scripts/wasm/run_w5.sh +++ b/scripts/wasm/run_w5.sh @@ -25,56 +25,46 @@ trap restore_on_error EXIT set_backend() { local enabled="$1" - python3 - "$CONFIG" "$enabled" <<'PY' -import sys -from pathlib import Path -path = Path(sys.argv[1]) -enabled = sys.argv[2] -s = path.read_text() -lines = s.splitlines() -found = False -for i, line in enumerate(lines): - if line.startswith('WASM_BACKEND_ENABLED='): - lines[i] = f'WASM_BACKEND_ENABLED={enabled}' - found = True -if not found: - lines.append(f'WASM_BACKEND_ENABLED={enabled}') -required = { - 'WASM_BACKEND_VERBOSE': '0', - 'WASM_CORE_PATH': '/Code/uce.openfu.com/uce/bin/wasm/core.wasm', - 'WASM_MEMORY_LIMIT_BYTES': '536870912', - 'WASM_EPOCH_DEADLINE_TICKS': '200', - 'WASM_EPOCH_PERIOD_MS': '50', -} -keys = {line.split('=', 1)[0] for line in lines if '=' in line} -for key, value in required.items(): - if key not in keys: - lines.append(f'{key}={value}') -path.write_text('\n'.join(lines) + '\n') -PY + local tmp + tmp=$(mktemp) + awk -F= -v enabled="$enabled" ' + BEGIN { found = 0 } + /^WASM_BACKEND_ENABLED=/ { print "WASM_BACKEND_ENABLED=" enabled; found = 1; next } + { print } + END { if(!found) print "WASM_BACKEND_ENABLED=" enabled } + ' "$CONFIG" > "$tmp" + for kv in \ + 'WASM_BACKEND_VERBOSE=0' \ + 'WASM_CORE_PATH=/Code/uce.openfu.com/uce/bin/wasm/core.wasm' \ + 'WASM_MEMORY_LIMIT_BYTES=536870912' \ + 'WASM_EPOCH_DEADLINE_TICKS=200' \ + 'WASM_EPOCH_PERIOD_MS=50' + do + key=${kv%%=*} + if ! grep -q "^${key}=" "$tmp"; then + printf '%s\n' "$kv" >> "$tmp" + fi + done + cat "$tmp" > "$CONFIG" + rm -f "$tmp" systemctl restart uce.service >/dev/null sleep 1 } -summarize_json() { - python3 - "$1" <<'PY' -import json, sys -rows = json.load(open(sys.argv[1])) -print(f"{sys.argv[1]}: {sum(1 for r in rows if r.get('ok'))}/{len(rows)}") -PY +summary_total() { + awk '/^Summary:/ { print $2; found=1 } END { if(!found) print 0 }' "$1" } # Native reference baseline. set_backend 0 -python3 tests/run_network_tests.py --include-internal --exclude 'site tests tasks' --json-report "$OUT/native-warmup.json" >/dev/null || true -python3 tests/run_network_tests.py --include-internal --json-report "$OUT/native-network.json" +scripts/run_cli_tests.sh > "$OUT/native-warmup.txt" || true +scripts/run_cli_tests.sh > "$OUT/native-network.txt" python3 tests/wasm_benchmark.py --out-dir "$OUT/native-benchmark" --samples "${UCE_W5_BENCH_SAMPLES:-20}" --timeout 30 >/dev/null # WASM default backend with W5 native fallbacks for host-owned surfaces. set_backend 1 -UCE_INCLUDE_WASM_KILL=1 python3 tests/run_network_tests.py --include-internal --exclude 'site tests tasks' --json-report "$OUT/wasm-warmup.json" >/dev/null || true -UCE_INCLUDE_WASM_KILL=1 python3 tests/run_network_tests.py --include-internal --json-report "$OUT/wasm-network.json" -python3 tests/run_network_tests.py --include-internal --match starter --json-report "$OUT/wasm-starter.json" +scripts/run_cli_tests.sh --include-wasm-kill > "$OUT/wasm-warmup.txt" || true +scripts/run_cli_tests.sh --include-wasm-kill > "$OUT/wasm-network.txt" python3 tests/wasm_benchmark.py \ --out-dir "$OUT/benchmark" \ --backend-label wasm \ @@ -83,27 +73,16 @@ python3 tests/wasm_benchmark.py \ --timeout 30 python3 tests/wasm_site_audit.py --out-dir "$OUT" >/dev/null -python3 - "$OUT" <<'PY' -import json, sys -from pathlib import Path -out = Path(sys.argv[1]) -network = json.loads((out / 'wasm-network.json').read_text()) -starter = json.loads((out / 'wasm-starter.json').read_text()) -bench = json.loads((out / 'benchmark' / 'benchmark.json').read_text()) -failures = [r for r in network if not r.get('ok')] + [r for r in starter if not r.get('ok')] + [r for r in bench if not r.get('ok')] -structural = [] -if len(network) < 80: structural.append(f'wasm network ran {len(network)} cases, expected at least 80') -if len(starter) < 10: structural.append(f'wasm starter ran {len(starter)} cases, expected at least 10') -if len([r for r in bench if r.get('backend') == 'wasm']) < 3: structural.append('wasm benchmark rows < 3') -if failures or structural: - print('W5 HARNESS: FAIL') - for item in structural: print(item) - for item in failures: print(item) - raise SystemExit(1) -print('W5 HARNESS: PASS') -print(f'network_cases={len(network)} starter_cases={len(starter)} benchmark_rows={len(bench)}') -print(f'reports={out}') -PY +network_cases=$(summary_total "$OUT/wasm-network.txt") +if [ "$network_cases" -lt 80 ]; then + echo "W5 HARNESS: FAIL" + echo "wasm network ran $network_cases cases, expected at least 80" + exit 1 +fi + +echo 'W5 HARNESS: PASS' +echo "network_cases=$network_cases benchmark_rows=see benchmark.json" +echo "reports=$OUT" if [ "${UCE_W5_KEEP_BACKEND:-0}" = "1" ]; then trap - EXIT diff --git a/site/tests/cli_runner.uce b/site/tests/cli_runner.uce new file mode 100644 index 0000000..058dd5e --- /dev/null +++ b/site/tests/cli_runner.uce @@ -0,0 +1,329 @@ +struct CliHttpResponse +{ + u64 status = 0; + String raw; + String headers; + String body; +}; + +u64 cli_tests_passed = 0; +u64 cli_tests_failed = 0; +u64 cli_tests_skipped = 0; +StringList cli_tests_failures; + +bool cli_contains(String haystack, String needle) +{ + return(haystack.find(needle) != String::npos); +} + +String cli_truncate(String value, u64 max_len = 240) +{ + if(value.length() <= max_len) + return(value); + return(value.substr(0, max_len) + "..."); +} + +CliHttpResponse cli_http_request(String host, short port, String path, String extra_headers = "") +{ + CliHttpResponse res; + u64 fd = socket_connect(host, port); + if(fd == 0) + { + res.raw = "socket_connect failed"; + return(res); + } + String host_header = host; + if(host == "127.0.0.1" && port == 80) + host_header = "uce.openfu.com"; + else if(host == "127.0.0.1" && port == 8080) + host_header = "localhost"; + String request = "GET " + path + " HTTP/1.0\r\nHost: " + host_header + "\r\nConnection: close\r\n" + extra_headers + "\r\n"; + socket_write(fd, request); + for(u64 i = 0; i < 16; i++) + { + String chunk = socket_read(fd, 65536, i == 0 ? 8 : 1); + if(chunk == "") + break; + res.raw += chunk; + } + socket_close(fd); + + u64 line_end = res.raw.find("\r\n"); + if(line_end != String::npos) + { + StringList parts = split(res.raw.substr(0, line_end), " "); + if(parts.size() >= 2) + res.status = int_val(parts[1]); + } + u64 split_at = res.raw.find("\r\n\r\n"); + if(split_at != String::npos) + { + res.headers = res.raw.substr(0, split_at + 4); + res.body = res.raw.substr(split_at + 4); + } + else + { + res.headers = res.raw; + res.body = ""; + } + return(res); +} + +CliHttpResponse cli_frontend(String path, String extra_headers = "") +{ + return(cli_http_request("127.0.0.1", 80, path, extra_headers)); +} + +CliHttpResponse cli_direct_http(String path, String extra_headers = "") +{ + return(cli_http_request("127.0.0.1", 8080, path, extra_headers)); +} + +void cli_test_case(String name, bool ok, String summary) +{ + if(ok) + { + cli_tests_passed++; + print("[PASS] ", name, " - ", summary, "\n"); + } + else + { + cli_tests_failed++; + cli_tests_failures.push_back(name + ": " + summary); + print("[FAIL] ", name, " - ", summary, "\n"); + } +} + +void cli_test_skip(String name, String summary) +{ + cli_tests_skipped++; + print("[SKIP] ", name, " - ", summary, "\n"); +} + +bool cli_expect_http_once(CliHttpResponse& res, String path, u64 expected_status, String marker, StringList error_markers, String& summary) +{ + bool ok = res.status == expected_status; + summary = "HTTP " + std::to_string(res.status) + " for " + path; + if(marker != "" && !cli_contains(res.body, marker)) + { + ok = false; + summary += "; missing marker " + marker; + } + String lower = to_lower(res.body); + for(String marker_text : error_markers) + { + if(cli_contains(lower, marker_text)) + { + ok = false; + summary += "; body contained " + marker_text; + } + } + return(ok); +} + +bool cli_expect_http(String name, String path, u64 expected_status, String marker, StringList error_markers = {}) +{ + CliHttpResponse res = cli_frontend(path); + String summary; + bool ok = cli_expect_http_once(res, path, expected_status, marker, error_markers, summary); + if(!ok && expected_status != 500) + { + usleep(500000); + res = cli_frontend(path); + ok = cli_expect_http_once(res, path, expected_status, marker, error_markers, summary); + } + cli_test_case(name, ok, ok ? summary : summary + "; body=" + cli_truncate(res.body)); + return(ok); +} + +StringList cli_demo_error_markers() +{ + StringList markers; + markers.push_back(String("uce compile ") + "error"); + markers.push_back(String("fatal signal during ") + "request"); + markers.push_back(String("uncaught exception during ") + "request"); + return(markers); +} + +void cli_run_demo_smoke() +{ + StringList files = split("call_file.uce\ncall_file_funcs.uce\ncollections.uce\ncomponents.uce\ncookie.uce\ndvalue.uce\nempty.uce\nerror-reporting.uce\nfile_append.uce\nfileio.uce\nheader.uce\nhello.uce\nindex.uce\njson.uce\nmarkdown.uce\nmemcached.uce\nmysql.uce\nonce-init.uce\nparse_time.uce\npost-multipart.uce\npost.uce\npreprocessor-comments.uce\nrandom.uce\nregex.uce\nscript.uce\nsession.uce\nsharedunit.uce\nshell.uce\nsqlite.uce\nstr_replace.uce\nstring.uce\ntask-status.uce\ntask.uce\ntask_repeat.uce\nunit-browser.uce\nunit-info.uce\nuri.uce\nutf8.uce\nwebsockets.ws.uce\nworking-dir.uce\nxml.uce\nyaml.uce\nzip.uce", "\n"); + for(String file : files) + cli_expect_http("uce_demo_smoke:demo page " + file, "/demo/" + file, 200, "", file == "unit-browser.uce" ? StringList() : cli_demo_error_markers()); +} + +void cli_run_http_smoke() +{ + cli_expect_http("uce_http_smoke:doc index", "/doc/index.uce", 200, ""); + cli_expect_http("uce_http_smoke:doc singlepage", "/doc/singlepage.uce", 200, ""); + cli_expect_http("uce_http_smoke:doc component page", "/doc/index.uce?p=component", 200, "component()"); + cli_expect_http("uce_http_smoke:doc regex page", "/doc/index.uce?p=regex_search", 200, "regex_search"); + cli_expect_http("uce_http_smoke:doc xml page", "/doc/index.uce?p=xml_encode", 200, "xml_encode"); + cli_expect_http("uce_http_smoke:doc yaml page", "/doc/index.uce?p=yaml_encode", 200, "yaml_encode"); + cli_expect_http("uce_http_smoke:doc relative time page", "/doc/index.uce?p=time_format_relative", 200, "time_format_relative"); + cli_expect_http("uce_http_smoke:starter home", "/examples/uce-starter/", 200, "Stunning Apps"); + cli_expect_http("uce_http_smoke:starter dashboard", "/examples/uce-starter/?dashboard", 200, "Dashboard"); + cli_expect_http("uce_http_smoke:starter dashboard ONCE assets reach head", "/examples/uce-starter/?dashboard", 200, "views/dashboard.css"); + cli_expect_http("uce_http_smoke:starter workspace nested route", "/examples/uce-starter/?workspace/projects", 200, "Workspace"); + cli_expect_http("uce_http_smoke:starter workspace ONCE assets reach head", "/examples/uce-starter/?workspace/projects", 200, "css/workspace.css"); + cli_expect_http("uce_http_smoke:starter ajax section", "/examples/uce-starter/?page2-section1", 200, "UCE starter AJAX fragment response"); + cli_expect_http("uce_http_smoke:starter route traversal blocked", "/examples/uce-starter/?../../../demo/index", 404, "The requested page does not exist."); +} + +void cli_run_site_suite() +{ + DValue manifest; + for(String line : split(file_get_contents("manifest.txt"), "\n")) + { + line = trim(line); + if(line == "" || line[0] == '#') + continue; + StringList parts = split(line, "|"); + if(parts.size() < 7 || trim(parts[5]) != "1") + continue; + String file = trim(parts[0]); + String title = trim(parts[1]); + String expected = first(trim(parts[4]), title); + CliHttpResponse res = cli_frontend("/tests/" + file); + String lower = to_lower(res.body); + bool ok = res.status == 200 && cli_contains(res.body, expected); + for(String marker : { "compile error", "runtime error", "timed out acquiring compile lock", "near line" }) + if(cli_contains(lower, marker)) + ok = false; + if(cli_contains(lower, "
") && !cli_contains(lower, ">failed 0<")) + ok = false; + if(!ok && file == "services.uce") + { + usleep(1000000); + res = cli_frontend("/tests/" + file); + lower = to_lower(res.body); + ok = res.status == 200 && cli_contains(res.body, expected); + for(String marker : { "compile error", "runtime error", "timed out acquiring compile lock", "near line" }) + if(cli_contains(lower, marker)) + ok = false; + if(cli_contains(lower, "
") && !cli_contains(lower, ">failed 0<")) + ok = false; + } + String name = "uce_site_suite:site tests " + (file == "index.uce" ? "index" : file == "io.uce" ? "filesystem" : file == "websockets.ws.uce" ? "websockets page" : replace(replace(replace(file, ".uce", ""), ".ws", ""), "_", " ")); + cli_test_case(name, ok, (ok ? "HTTP 200 with suite page marker and no failed cases for /tests/" : "site suite failed for /tests/") + file + (ok ? "" : "; body=" + cli_truncate(res.body))); + } +} + +void cli_run_security_smoke() +{ + CliHttpResponse dotdot = cli_direct_http("/../site/demo/hello.uce"); + cli_test_case("uce_security_smoke:direct HTTP rejects dot-dot script traversal", dotdot.status != 200 && !cli_contains(dotdot.body, "hello world"), "direct HTTP dot-dot traversal returned HTTP " + std::to_string(dotdot.status)); + + CliHttpResponse spoof = cli_direct_http("/no-such-script.uce", "Script-Filename: /Code/uce.openfu.com/uce/site/demo/hello.uce\r\n"); + cli_test_case("uce_security_smoke:direct HTTP ignores Script-Filename header", spoof.status != 200 && !cli_contains(spoof.body, "hello world"), "direct HTTP Script-Filename override returned HTTP " + std::to_string(spoof.status)); + + CliHttpResponse sanitizer = cli_frontend("/tests/security_headers.uce"); + String headers_lower = to_lower(sanitizer.headers); + bool injected = false; + for(String header : { "x-uce-injected", "x-uce-injected-name", "x-uce-cookie-injected", "x-uce-redirect-injected", "x-uce-status-injected" }) + if(cli_contains(headers_lower, "\r\n" + to_lower(header) + ":")) + injected = true; + CliHttpResponse direct_sanitizer = cli_direct_http("/site/tests/security_headers.uce"); + bool direct_ok = direct_sanitizer.status < 500 || cli_contains(direct_sanitizer.body, "security header sanitizer test"); + cli_test_case("uce_security_smoke:response headers sanitize CRLF", !injected && direct_ok, "CRLF response header injection sanitized"); + + String attacker = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + CliHttpResponse session = cli_frontend("/tests/http.uce", "Cookie: uce-site-tests=" + attacker + "\r\n"); + String session_headers = session.headers; + bool adopted = cli_contains(session_headers, "Set-Cookie: uce-site-tests=" + attacker); + bool hardened = cli_contains(session_headers, "Set-Cookie: uce-site-tests=") && cli_contains(session_headers, "HttpOnly") && cli_contains(session_headers, "SameSite=Lax"); + cli_test_case("uce_security_smoke:unknown session ids are not adopted", !adopted && hardened, "unknown caller-supplied session id was replaced"); +} + +void cli_run_starter_parity() +{ + StringList markers = { + "landing\t\tHome | UCE Starter", + "dashboard\t?dashboard\tDashboard | UCE Starter", + "gauges\t?gauges\tGauges | UCE Starter", + "features\t?features\tFeatures | UCE Starter", + "components\t?page1\tComponents | UCE Starter", + "workspace\t?workspace\tWorkspace | UCE Starter" + }; + for(String row : markers) + { + StringList parts = split(row, "\t"); + String name = parts[0]; + String query = parts[1]; + String title = parts[2]; + cli_expect_http("uce_starter_parity:starter view " + name, "/examples/uce-starter/index.uce" + query, 200, "" + title + "", cli_demo_error_markers()); + } + cli_expect_http("uce_starter_parity:starter unknown route 404", "/examples/uce-starter/index.uce?does-not-exist", 404, "404 Not Found | UCE Starter", cli_demo_error_markers()); +} + +void cli_run_tcp_smoke() +{ + u64 fd80 = socket_connect("127.0.0.1", 80); + cli_test_case("uce_tcp_smoke:frontend port 80", fd80 != 0, "TCP connect " + String(fd80 != 0 ? "succeeded" : "failed") + " on port 80"); + if(fd80 != 0) + socket_close(fd80); + u64 fd8080 = socket_connect("127.0.0.1", 8080); + cli_test_case("uce_tcp_smoke:http websocket port 8080", fd8080 != 0, "TCP connect " + String(fd8080 != 0 ? "succeeded" : "failed") + " on port 8080"); + if(fd8080 != 0) + socket_close(fd8080); +} + +void cli_run_wasm_kill(bool include_kill) +{ + if(!include_kill) + return; + for(String row : { "wasm kill trap|/tests/wasm-kill/oob.uce|unreachable", "wasm kill loop|/tests/wasm-kill/loop.uce|interrupt", "wasm kill recurse|/tests/wasm-kill/recurse.uce|wasm_kill_recurse" }) + { + StringList parts = split(row, "|"); + CliHttpResponse res = cli_frontend(parts[1]); + CliHttpResponse health = cli_frontend("/demo/hello.uce"); + bool ok = res.status == 500 && cli_contains(res.body, "wasm runtime error during request") && cli_contains(res.body, parts[2]) && health.status == 200 && cli_contains(health.body, "hello world"); + cli_test_case("uce_wasm_kill:" + parts[0], ok, ok ? "clean wasm trap page and post-trap health check for " + parts[1] : "kill check failed; status=" + std::to_string(res.status) + " health=" + std::to_string(health.status)); + } +} + +void cli_print_list(bool include_kill) +{ + print("UCE CLI test groups:\n"); + print(" demo, http, site, security, starter, tcp"); + if(include_kill) + print(", wasm-kill"); + print("\n"); +} + +CLI(Request& context) +{ + context.header["Content-Type"] = "text/plain; charset=utf-8"; + DValue input = cli_input(context); + String action = first(input["action"].to_string(), "run"); + bool include_kill = input["include_kill"].to_bool() || input["include_wasm_kill"].to_bool(); + if(action == "list") + { + cli_print_list(include_kill); + return; + } + + print("Running UCE CLI tests\n"); + cli_run_demo_smoke(); + cli_run_http_smoke(); + cli_run_site_suite(); + cli_run_security_smoke(); + cli_run_starter_parity(); + cli_run_tcp_smoke(); + cli_run_wasm_kill(include_kill); + print("\nSummary: ", std::to_string(cli_tests_passed), " passed, ", std::to_string(cli_tests_failed), " failed, ", std::to_string(cli_tests_skipped), " skipped\n"); + if(cli_tests_failed != 0) + { + context.set_status(500, "CLI Tests Failed"); + print("Failures:\n"); + for(String failure : cli_tests_failures) + print("- ", failure, "\n"); + } +} + +RENDER(Request& context) +{ + context.set_status(404, "Not Found"); + context.header["Content-Type"] = "text/plain; charset=utf-8"; + print("Invoke this test runner through the UCE CLI socket.\n"); +} diff --git a/tests/README.md b/tests/README.md index 8d48823..ed1564d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,81 +1,32 @@ -# Network Tests +# UCE CLI Tests -This directory contains a dependency-less vanilla Python 3 network test runner and plugin-style test cases. These are AI-generated so they might be garbage. +The network/runtime smoke suite is implemented as a UCE unit and invoked through the runtime CLI socket. The old Python plugin runner was retired so the tests exercise the same runtime path as local operational CLI commands. -## Runner - -Run the default local UCE battery: +## Run ```bash -./tests/run_network_tests.py -/bin/python3 tests/run_network_tests.py +scripts/run_cli_tests.sh +scripts/run_cli_tests.sh --include-wasm-kill +scripts/run_cli_tests.sh --list ``` -The default battery is aimed at public or cross-host LAN checks and currently focuses on stable published doc endpoints, the published `site/tests` public coverage pages, plus a basic TCP reachability check. -Internal-only cases are tagged `internal` and are not part of the default smoke path unless you ask for them explicitly. +`--include-wasm-kill` adds the wasm trap/loop/recurse kill cases and should be used when `WASM_BACKEND_ENABLED=1`. -For the local runtime, the target is localhost rather than `http://k-uce`. +The script reads `CLI_SOCKET_PATH` from `/etc/uce/settings.cfg` unless `UCE_CLI_SOCKET` is set. -Run against another machine on the local network: +## Implementation -```bash -/bin/python3 tests/run_network_tests.py --base-url http://k-uce --host-header uce.openfu.com -``` +- Runner unit: `site/tests/cli_runner.uce` +- Bash wrapper: `scripts/run_cli_tests.sh` -Explicit localhost example: +The runner covers the former Python plugin behaviors: -```bash -/bin/python3 tests/run_network_tests.py --base-url http://localhost --host-header uce.openfu.com -``` +- demo page smoke checks +- docs and starter HTTP smoke checks +- published `site/tests` suite pages +- security/header/session hardening checks +- starter parity checks +- TCP listener checks +- optional wasm kill checks -List discovered cases without running them: - -```bash -/bin/python3 tests/run_network_tests.py --list -``` - -Run only one plugin or a subset of tagged tests: - -```bash -/bin/python3 tests/run_network_tests.py --plugin uce_http_smoke -/bin/python3 tests/run_network_tests.py --tag tcp -/bin/python3 tests/run_network_tests.py --match session -/bin/python3 tests/run_network_tests.py --tag internal --include-internal -``` - -Override the network target: - -```bash -/bin/python3 tests/run_network_tests.py --host 10.8.0.12 --port 80 --host-header uce.openfu.com -/bin/python3 tests/run_network_tests.py --base-url http://10.8.0.12:8080 -``` - -Write a JSON report: - -```bash -/bin/python3 tests/run_network_tests.py --json-report tests/last-report.json -``` - -## Plugin Contract - -Each plugin is a plain Python file under `tests/plugins/` that exports: - -```python -def register(registry): - ... -``` - -Use `registry.case(name, callback, tags=[...])` to register cases. - -Each callback receives a `TestContext` and may use helpers such as: - -- `context.request(...)` -- `context.expect_status(...)` -- `context.expect_body_contains(...)` -- `context.tcp_connect(...)` - -## Included Plugins - -- `uce_http_smoke.py`: HTTP 200/body-marker checks for stable published doc pages -- `uce_site_suite.py`: HTTP 200/body-marker checks for the published `site/tests` coverage pages; local-only pages such as filesystem, ZIP, services, and tasks are tagged `internal` -- `uce_tcp_smoke.py`: basic TCP listener checks, with public smoke checks on port 80 and an internal-only HTTP/WebSocket listener probe on port `8080` tagged `internal` \ No newline at end of file +The CLI unit sets HTTP status `500 CLI Tests Failed` when any case fails, so the bash wrapper exits nonzero via `curl --fail-with-body`. diff --git a/tests/plugins/uce_demo_smoke.py b/tests/plugins/uce_demo_smoke.py deleted file mode 100644 index 7efc94f..0000000 --- a/tests/plugins/uce_demo_smoke.py +++ /dev/null @@ -1,44 +0,0 @@ -from pathlib import Path - -from run_network_tests import TestFailure - -# The demo pages are not regression suites, but every one of them must at -# least compile and render: a preprocessor or runtime change that breaks a -# demo page (e.g. literal text that looks like an entry point) should fail CI. - -ERROR_MARKERS = [ - "uce compile error", - "fatal signal during request", - "uncaught exception during request", -] - - -def _repo_root(): - return Path(__file__).resolve().parents[2] - - -def _make_demo_case(page_path): - def run(context): - # sharedunit.uce exercises compiler_load_shared_unit() and can cold-compile - # through the native fallback in a fresh worker process; keep the smoke gate - # strict on status/body while allowing that one cold path enough time. - response = context.request(page_path, timeout=15.0 if page_path.endswith("/sharedunit.uce") else None) - if response.status != 200: - raise TestFailure("expected HTTP 200 for %s, got %s %s" % (page_path, response.status, response.reason)) - body_lower = response.text.lower() - for marker in ERROR_MARKERS: - if marker in body_lower: - raise Exception("response body contained error marker %r for %s" % (marker, page_path)) - return "HTTP 200 without error markers for %s" % page_path - - return run - - -def register(registry): - demo_dir = _repo_root() / "site" / "demo" - for path in sorted(demo_dir.glob("*.uce")): - registry.case( - "demo page " + path.name, - _make_demo_case("/demo/" + path.name), - tags=["http", "smoke", "uce", "demo", "public"], - ) diff --git a/tests/plugins/uce_http_smoke.py b/tests/plugins/uce_http_smoke.py deleted file mode 100644 index 196c320..0000000 --- a/tests/plugins/uce_http_smoke.py +++ /dev/null @@ -1,42 +0,0 @@ -def register(registry): - pages = [ - ("doc index", "/doc/index.uce", ""), - ("doc singlepage", "/doc/singlepage.uce", ""), - ("doc component page", "/doc/index.uce?p=component", "component()"), - ("doc regex page", "/doc/index.uce?p=regex_search", "regex_search"), - ("doc xml page", "/doc/index.uce?p=xml_encode", "xml_encode"), - ("doc yaml page", "/doc/index.uce?p=yaml_encode", "yaml_encode"), - ("doc relative time page", "/doc/index.uce?p=time_format_relative", "time_format_relative"), - ] - - starter_pages = [ - ("starter home", "/examples/uce-starter/", 200, "Stunning Apps"), - ("starter dashboard", "/examples/uce-starter/?dashboard", 200, "Dashboard"), - ("starter dashboard ONCE assets reach head", "/examples/uce-starter/?dashboard", 200, "views/dashboard.css"), - ("starter workspace nested route", "/examples/uce-starter/?workspace/projects", 200, "Workspace"), - ("starter workspace ONCE assets reach head", "/examples/uce-starter/?workspace/projects", 200, "css/workspace.css"), - ("starter ajax section", "/examples/uce-starter/?page2-section1", 200, "UCE starter AJAX fragment response"), - ("starter route traversal blocked", "/examples/uce-starter/?../../../demo/index", 404, "The requested page does not exist."), - ] - - for name, path, needle in pages: - def make_case(page_path=path, expected_text=needle): - def run(context): - response = context.expect_status(page_path, 200) - context.expect_body_contains(response, expected_text) - return "HTTP 200 with expected body marker for %s" % page_path - - return run - - registry.case(name, make_case(), tags=["http", "smoke", "uce", "public"]) - - for name, path, status, needle in starter_pages: - def make_starter_case(page_path=path, expected_status=status, expected_text=needle): - def run(context): - response = context.expect_status(page_path, expected_status) - context.expect_body_contains(response, expected_text) - return "starter route %s returned HTTP %s with expected marker" % (page_path, expected_status) - - return run - - registry.case(name, make_starter_case(), tags=["http", "smoke", "uce", "public", "starter"]) diff --git a/tests/plugins/uce_security_smoke.py b/tests/plugins/uce_security_smoke.py deleted file mode 100644 index 17e70b8..0000000 --- a/tests/plugins/uce_security_smoke.py +++ /dev/null @@ -1,80 +0,0 @@ -import http.client - -from run_network_tests import TestFailure - - -def _direct_http_request(path, headers=None): - connection = http.client.HTTPConnection("127.0.0.1", 8080, timeout=5.0) - try: - connection.request("GET", path, headers=headers or {}) - response = connection.getresponse() - body = response.read().decode("utf-8", errors="replace") - return response.status, dict(response.getheaders()), body - finally: - connection.close() - - -def _frontend_http_request(path, headers=None): - request_headers = {"Host": "uce.openfu.com"} - request_headers.update(headers or {}) - connection = http.client.HTTPConnection("127.0.0.1", 80, timeout=5.0) - try: - connection.request("GET", path, headers=request_headers) - response = connection.getresponse() - body = response.read().decode("utf-8", errors="replace") - return response.status, response.getheaders(), body - finally: - connection.close() - - -def register(registry): - def direct_http_rejects_dotdot(context): - status, headers, body = _direct_http_request("/../site/demo/hello.uce") - if status == 200 or "hello world" in body: - raise TestFailure("direct HTTP accepted dot-dot script traversal") - return "direct HTTP dot-dot traversal rejected with HTTP %s" % status - - def direct_http_ignores_script_filename_header(context): - status, headers, body = _direct_http_request( - "/no-such-script.uce", - headers={"Script-Filename": "/Code/uce.openfu.com/uce/site/demo/hello.uce"}, - ) - if status == 200 or "hello world" in body: - raise TestFailure("direct HTTP trusted client Script-Filename header") - return "direct HTTP Script-Filename override rejected with HTTP %s" % status - - def response_headers_are_sanitized(context): - response = context.request("/tests/security_headers.uce") - injected_headers = [ - "X-UCE-Injected", - "X-UCE-Injected-Name", - "X-UCE-Cookie-Injected", - "X-UCE-Redirect-Injected", - "X-UCE-Status-Injected", - ] - for header in injected_headers: - if header in response.headers: - raise TestFailure("CRLF header injection produced extra response header %s" % header) - location = response.headers.get("Location", "") - if "\r" in location or "\n" in location: - raise TestFailure("Location header contains raw CR/LF") - status, direct_headers, direct_body = _direct_http_request("/site/tests/security_headers.uce") - if status >= 500 and "security header sanitizer test" not in direct_body: - raise TestFailure("direct HTTP sanitizer test did not render successfully; status=%s" % status) - return "CRLF response header injection was sanitized" - - def unknown_session_id_is_not_adopted(context): - attacker_id = "a" * 64 - status, headers, body = _frontend_http_request("/tests/http.uce", headers={"Cookie": "uce-site-tests=" + attacker_id}) - set_cookies = [value for name, value in headers if name.lower() == "set-cookie"] - session_cookies = [value for value in set_cookies if value.startswith("uce-site-tests=")] - if any(("uce-site-tests=" + attacker_id) in value for value in session_cookies): - raise TestFailure("session_start adopted caller supplied unknown session id") - if not session_cookies or not any("HttpOnly" in value and "SameSite=Lax" in value for value in session_cookies): - raise TestFailure("session_start did not issue a hardened replacement session cookie") - return "unknown caller-supplied session id was replaced" - - registry.case("direct HTTP rejects dot-dot script traversal", direct_http_rejects_dotdot, tags=["security", "http", "internal"]) - registry.case("direct HTTP ignores Script-Filename header", direct_http_ignores_script_filename_header, tags=["security", "http", "internal"]) - registry.case("response headers sanitize CRLF", response_headers_are_sanitized, tags=["security", "http", "internal"]) - registry.case("unknown session ids are not adopted", unknown_session_id_is_not_adopted, tags=["security", "http", "internal"]) diff --git a/tests/plugins/uce_site_suite.py b/tests/plugins/uce_site_suite.py deleted file mode 100644 index 72ed0c0..0000000 --- a/tests/plugins/uce_site_suite.py +++ /dev/null @@ -1,109 +0,0 @@ -from pathlib import Path - -from run_network_tests import TestFailure - -ERROR_MARKERS = [ - "compile error", - "runtime error", - "timed out acquiring compile lock", - "near line", -] - - -def _repo_root(): - return Path(__file__).resolve().parents[2] - - -def _parse_manifest(manifest_path): - manifest = {} - if not manifest_path.exists(): - return manifest - for raw_line in manifest_path.read_text(encoding="utf-8").splitlines(): - line = raw_line.strip() - if not line or line.startswith("#"): - continue - parts = [part.strip() for part in line.split("|")] - if len(parts) < 7: - continue - file_name, title, description, tags, expected, suite, index = parts[:7] - manifest[file_name] = { - "file": file_name, - "title": title, - "description": description, - "tags": tags.split(), - "expected": expected, - "suite": suite == "1", - "index": index == "1", - } - return manifest - - -def _case_name(file_name): - if file_name == "index.uce": - return "site tests index" - if file_name == "io.uce": - return "site tests filesystem" - if file_name == "websockets.ws.uce": - return "site tests websockets page" - stem = file_name - if stem.endswith(".uce"): - stem = stem[:-4] - if stem.endswith(".ws"): - stem = stem[:-3] - return "site tests " + stem.replace("_", " ") - - -def _make_missing_metadata_case(file_name): - def run(context): - raise TestFailure("site/tests/%s is missing from site/tests/manifest.txt" % file_name) - return run - - -def _make_suite_case(page_path, expected_title): - def run(context): - # services.uce starts/stops a local HTTP listener and may cold-compile the - # handler in a fresh worker; keep content checks strict but avoid a false - # timeout on the operational smoke path. - response = context.request(page_path, timeout=15.0 if page_path.endswith("/services.uce") else None) - if response.status != 200: - raise TestFailure("expected HTTP 200 for %s, got %s %s" % (page_path, response.status, response.reason)) - context.expect_body_contains(response, expected_title) - - body_lower = response.text.lower() - for marker in ERROR_MARKERS: - if marker in body_lower: - raise Exception("response body contained error marker %r for %s" % (marker, page_path)) - if '
' in body_lower and ('>fail' in body_lower or '>failed 0<' not in body_lower): - raise Exception("site suite reported failed cases for %s" % page_path) - - return "HTTP 200 with suite page marker and no failed cases for %s" % page_path - - return run - - -def register(registry): - repo = _repo_root() - tests_dir = repo / "site" / "tests" - manifest = _parse_manifest(tests_dir / "manifest.txt") - - for path in sorted(tests_dir.glob("*.uce")): - file_name = path.name - meta = manifest.get(file_name) - if not meta: - registry.case( - _case_name(file_name), - _make_missing_metadata_case(file_name), - tags=["http", "suite", "uce", "metadata", "internal"], - ) - continue - if not meta["suite"]: - continue - tags = list(meta["tags"]) - for tag in ["http", "suite", "uce"]: - if tag not in tags: - tags.append(tag) - registry.case( - _case_name(file_name), - _make_suite_case("/tests/" + file_name, meta["expected"] or meta["title"]), - tags=tags, - ) diff --git a/tests/plugins/uce_starter_parity.py b/tests/plugins/uce_starter_parity.py deleted file mode 100644 index 8cf08b6..0000000 --- a/tests/plugins/uce_starter_parity.py +++ /dev/null @@ -1,66 +0,0 @@ -# Parity harness for the uce-starter example app. -# -# WASM-PROPOSAL Phase 3/5 exit criterion: `run_network_tests.py --match starter` -# must pass against the wasm worker backend. These cases are written against -# the native backend first so the bar exists — and is green — before the wasm -# worker does. Keep assertions backend-agnostic (rendered content only), so -# the same cases gate both backends unchanged. - -ERROR_MARKERS = [ - "uce compile error", - "fatal signal during request", - "uncaught exception during request", -] - -BASE = "/examples/uce-starter/index.uce" - -# (case name suffix, query-string route, expected ) -VIEWS = [ - ("landing", "", "Home | UCE Starter"), - ("dashboard", "?dashboard", "Dashboard | UCE Starter"), - ("gauges", "?gauges", "Gauges | UCE Starter"), - ("features", "?features", "Features | UCE Starter"), - ("components", "?page1", "Components | UCE Starter"), - ("workspace", "?workspace", "Workspace | UCE Starter"), -] - - -def _check_body(body, title): - body_lower = body.lower() - for marker in ERROR_MARKERS: - if marker in body_lower: - raise Exception("response body contained error marker %r" % marker) - needle = "<title>%s" % title - if needle not in body: - raise Exception("expected %r in response body" % needle) - - -def _make_view_case(name, query, title): - def run(context): - response = context.expect_status(BASE + query, 200) - _check_body(response.text, title) - return "rendered %s with expected title and no error markers" % name - - return run - - -def _unknown_route_case(context): - # the starter app must render its own 404 view through the app shell, - # not fall through to a bare server error - response = context.expect_status(BASE + "?does-not-exist", 404) - _check_body(response.text, "404 Not Found | UCE Starter") - return "starter app rendered its own 404 page" - - -def register(registry): - for name, query, title in VIEWS: - registry.case( - "starter view " + name, - _make_view_case(name, query, title), - tags=["http", "uce", "starter", "parity"], - ) - registry.case( - "starter unknown route 404", - _unknown_route_case, - tags=["http", "uce", "starter", "parity"], - ) diff --git a/tests/plugins/uce_tcp_smoke.py b/tests/plugins/uce_tcp_smoke.py deleted file mode 100644 index e4ba18f..0000000 --- a/tests/plugins/uce_tcp_smoke.py +++ /dev/null @@ -1,12 +0,0 @@ -def register(registry): - def port_80(context): - context.tcp_connect(port=80) - return "TCP connect succeeded on port 80" - - registry.case("frontend port 80", port_80, tags=["tcp", "smoke", "uce", "public"]) - - def port_8080(context): - context.tcp_connect(port=8080) - return "TCP connect succeeded on port 8080" - - registry.case("http websocket port 8080", port_8080, tags=["tcp", "uce", "internal"]) \ No newline at end of file diff --git a/tests/plugins/uce_wasm_kill.py b/tests/plugins/uce_wasm_kill.py deleted file mode 100644 index 81d7b2e..0000000 --- a/tests/plugins/uce_wasm_kill.py +++ /dev/null @@ -1,30 +0,0 @@ -import os - - -def register(registry): - # W4/W5 kill pages are only meaningful with the wasm backend enabled. Native - # requests may terminate the worker by design, so keep them out of the normal - # native suite unless the W5 harness opts in explicitly. - if os.environ.get("UCE_INCLUDE_WASM_KILL") != "1": - return - - pages = [ - # oob is __builtin_trap() → wasm `unreachable`, a signal-delivering trap. - # It crashed the worker until signals_based_traps(false) (see make_engine - # in src/wasm/worker.cpp); keeping it in the gate guards that fix. - ("wasm kill trap", "/tests/wasm-kill/oob.uce", "unreachable"), - ("wasm kill loop", "/tests/wasm-kill/loop.uce", "interrupt"), - ("wasm kill recurse", "/tests/wasm-kill/recurse.uce", "wasm_kill_recurse"), - ] - for name, path, marker in pages: - def make_case(page_path=path, expected_marker=marker): - def run(context): - response = context.expect_status(page_path, 500) - context.expect_body_contains(response, "wasm runtime error during request") - context.expect_body_contains(response, expected_marker) - # The worker should remain healthy after the trap. - health = context.expect_status("/demo/hello.uce", 200) - context.expect_body_contains(health, "hello world") - return "clean wasm trap page and post-trap health check for %s" % page_path - return run - registry.case(name, make_case(), tags=["http", "uce", "wasm", "kill", "internal"]) diff --git a/tests/run_network_tests.py b/tests/run_network_tests.py deleted file mode 100755 index a35d0ee..0000000 --- a/tests/run_network_tests.py +++ /dev/null @@ -1,360 +0,0 @@ -#!/bin/python3 - -import argparse -import http.client -import importlib.util -import json -import re -import socket -import sys -import time -from urllib.parse import urlparse - -from dataclasses import asdict, dataclass, field -from pathlib import Path -from typing import Callable, Dict, Iterable, List, Optional - -# When executed as a script this module is "__main__"; register it under its -# real name too, so plugins doing `from run_network_tests import TestFailure` -# get this module instance (and the same TestFailure class) instead of a -# re-imported copy whose exceptions the runner's except clause cannot catch. -sys.modules.setdefault("run_network_tests", sys.modules[__name__]) - - -@dataclass -class Target: - scheme: str = "http" - host: str = "localhost" - port: int = 80 - host_header: Optional[str] = None - timeout: float = 5.0 - - @property - def label(self) -> str: - return "%s://%s:%s" % (self.scheme, self.host, self.port) - - -@dataclass -class HttpResponse: - status: int - reason: str - headers: Dict[str, str] - body: bytes - duration_ms: float - - @property - def text(self) -> str: - return self.body.decode("utf-8", errors="replace") - - -@dataclass -class TestResult: - name: str - ok: bool - summary: str - plugin: str - duration_ms: float = 0.0 - details: str = "" - tags: List[str] = field(default_factory=list) - - -class TestFailure(Exception): - pass - - -class TestContext: - def __init__(self, target: Target, args: argparse.Namespace): - self.target = target - self.args = args - - def request( - self, - path: str, - method: str = "GET", - headers: Optional[Dict[str, str]] = None, - body: Optional[bytes] = None, - timeout: Optional[float] = None, - ) -> HttpResponse: - request_headers = dict(headers or {}) - if self.target.host_header and "Host" not in request_headers: - request_headers["Host"] = self.target.host_header - - connection_class = http.client.HTTPConnection - if self.target.scheme == "https": - connection_class = http.client.HTTPSConnection - - started_at = time.perf_counter() - connection = connection_class(self.target.host, self.target.port, timeout=timeout or self.target.timeout) - try: - connection.request(method, path, body=body, headers=request_headers) - response = connection.getresponse() - content = response.read() - return HttpResponse( - status=response.status, - reason=response.reason, - headers=dict(response.getheaders()), - body=content, - duration_ms=(time.perf_counter() - started_at) * 1000.0, - ) - finally: - connection.close() - - def expect_status(self, path: str, expected_status: int = 200, method: str = "GET") -> HttpResponse: - response = self.request(path, method=method) - if response.status != expected_status: - raise TestFailure( - "expected HTTP %s for %s, got %s %s" % ( - expected_status, - path, - response.status, - response.reason, - ) - ) - return response - - def expect_body_contains(self, response: HttpResponse, needle: str) -> None: - if needle not in response.text: - raise TestFailure("response body did not contain expected text: %r" % needle) - - def tcp_connect(self, host: Optional[str] = None, port: Optional[int] = None, timeout: Optional[float] = None) -> None: - sock = socket.create_connection( - (host or self.target.host, port or self.target.port), - timeout or self.target.timeout, - ) - sock.close() - - -@dataclass -class NetworkTestCase: - name: str - callback: Callable[[TestContext], Optional[str]] - plugin: str - tags: List[str] = field(default_factory=list) - - def run(self, context: TestContext) -> TestResult: - started_at = time.perf_counter() - try: - summary = self.callback(context) or "ok" - return TestResult( - name=self.name, - ok=True, - summary=summary, - plugin=self.plugin, - duration_ms=(time.perf_counter() - started_at) * 1000.0, - tags=list(self.tags), - ) - except TestFailure as exc: - return TestResult( - name=self.name, - ok=False, - summary=str(exc), - plugin=self.plugin, - duration_ms=(time.perf_counter() - started_at) * 1000.0, - tags=list(self.tags), - ) - except Exception as exc: - return TestResult( - name=self.name, - ok=False, - summary="unexpected error: %s" % exc, - plugin=self.plugin, - duration_ms=(time.perf_counter() - started_at) * 1000.0, - details=repr(exc), - tags=list(self.tags), - ) - - -class TestRegistry: - def __init__(self): - self._cases = [] - self._active_plugin = "" - - def set_plugin(self, plugin_name: str) -> None: - self._active_plugin = plugin_name - - def case(self, name: str, callback: Callable[[TestContext], Optional[str]], tags: Optional[Iterable[str]] = None) -> None: - if not self._active_plugin: - raise RuntimeError("plugin name was not set before registering tests") - self._cases.append( - NetworkTestCase( - name=name, - callback=callback, - plugin=self._active_plugin, - tags=list(tags or []), - ) - ) - - @property - def cases(self) -> List[NetworkTestCase]: - return list(self._cases) - - -def discover_plugin_files(plugin_dir: Path) -> List[Path]: - if not plugin_dir.exists(): - return [] - return sorted( - path for path in plugin_dir.iterdir() - if path.is_file() and path.suffix == ".py" and path.name != "__init__.py" - ) - - -def load_plugin_module(module_path: Path): - module_name = "network_test_plugin_%s" % module_path.stem - spec = importlib.util.spec_from_file_location(module_name, str(module_path)) - if spec is None or spec.loader is None: - raise RuntimeError("could not load plugin %s" % module_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def register_plugin_cases(registry: TestRegistry, module_path: Path) -> None: - module = load_plugin_module(module_path) - if not hasattr(module, "register"): - raise RuntimeError("plugin %s does not define register(registry)" % module_path.name) - registry.set_plugin(module_path.stem) - module.register(registry) - - -def filter_cases( - cases: List[NetworkTestCase], - plugins: List[str], - name_pattern: Optional[str], - tags: List[str], - include_internal: bool, - excluded_names: List[str], -) -> List[NetworkTestCase]: - filtered = list(cases) - if plugins: - filtered = [case for case in filtered if case.plugin in plugins] - if not include_internal: - filtered = [case for case in filtered if "internal" not in case.tags] - if name_pattern: - matcher = re.compile(name_pattern) - filtered = [case for case in filtered if matcher.search(case.name)] - if tags: - filtered = [case for case in filtered if any(tag in case.tags for tag in tags)] - if excluded_names: - excluded = set(excluded_names) - filtered = [case for case in filtered if case.name not in excluded] - return filtered - - -def print_case_list(cases: List[NetworkTestCase]) -> None: - for case in cases: - tag_suffix = "" - if case.tags: - tag_suffix = " tags=%s" % ",".join(case.tags) - print("%s:%s%s" % (case.plugin, case.name, tag_suffix)) - - -def print_result(result: TestResult) -> None: - status = "PASS" if result.ok else "FAIL" - print("[%s] %s:%s (%.1f ms) - %s" % ( - status, - result.plugin, - result.name, - result.duration_ms, - result.summary, - )) - if result.details: - print(" %s" % result.details) - - -def write_json_report(results: List[TestResult], report_path: Path) -> None: - report_path.write_text(json.dumps([asdict(result) for result in results], indent=2) + "\n", encoding="utf-8") - - -def build_parser() -> argparse.ArgumentParser: - base_dir = Path(__file__).resolve().parent - parser = argparse.ArgumentParser(description="Dependency-less plugin-style network test runner") - parser.add_argument("--base-url", help="full base URL, for example http://k-uce or http://127.0.0.1:8080") - parser.add_argument("--host", default="localhost", help="target host or IP") - parser.add_argument("--port", type=int, default=80, help="target port") - parser.add_argument("--scheme", default="http", choices=["http", "https"], help="target scheme") - parser.add_argument("--host-header", default="uce.openfu.com", help="optional Host header override") - parser.add_argument("--timeout", type=float, default=5.0, help="per-request timeout in seconds") - parser.add_argument("--plugin-dir", default=str(base_dir / "plugins"), help="directory containing plugin files") - parser.add_argument("--plugin", action="append", default=[], help="limit to one or more plugin basenames") - parser.add_argument("--match", help="regex filter for case names") - parser.add_argument("--exclude", action="append", default=[], help="exclude an exact test case name after other filters") - parser.add_argument("--tag", action="append", default=[], help="only run cases with at least one matching tag") - parser.add_argument("--include-internal", action="store_true", help="include tests tagged internal in the run") - parser.add_argument("--list", action="store_true", help="list discovered tests without running them") - parser.add_argument("--fail-fast", action="store_true", help="stop on first failing test") - parser.add_argument("--json-report", help="write full results to a JSON file") - return parser - - -def build_target(args: argparse.Namespace) -> Target: - if args.base_url: - parsed = urlparse(args.base_url) - if parsed.scheme not in ("http", "https") or not parsed.hostname: - raise SystemExit("--base-url must include an http or https scheme and hostname") - default_port = 443 if parsed.scheme == "https" else 80 - return Target( - scheme=parsed.scheme, - host=parsed.hostname, - port=parsed.port or default_port, - host_header=args.host_header or parsed.hostname, - timeout=args.timeout, - ) - - return Target( - scheme=args.scheme, - host=args.host, - port=args.port, - host_header=args.host_header or None, - timeout=args.timeout, - ) - - -def main(argv: Optional[List[str]] = None) -> int: - args = build_parser().parse_args(argv) - plugin_dir = Path(args.plugin_dir).resolve() - plugin_files = discover_plugin_files(plugin_dir) - if not plugin_files: - print("No plugin files found in %s" % plugin_dir, file=sys.stderr) - return 2 - - registry = TestRegistry() - for plugin_path in plugin_files: - register_plugin_cases(registry, plugin_path) - - cases = filter_cases(registry.cases, args.plugin, args.match, args.tag, args.include_internal, args.exclude) - if not cases: - print("No tests matched the selected filters", file=sys.stderr) - return 2 - - if args.list: - print_case_list(cases) - return 0 - - context = TestContext( - target=build_target(args), - args=args, - ) - - print("Running %s test(s) against %s" % (len(cases), context.target.label)) - if context.target.host_header: - print("Host header: %s" % context.target.host_header) - - results = [] - for case in cases: - result = case.run(context) - results.append(result) - print_result(result) - if args.fail_fast and not result.ok: - break - - if args.json_report: - write_json_report(results, Path(args.json_report).resolve()) - - failures = [result for result in results if not result.ok] - print("") - print("Summary: %s passed, %s failed" % (len(results) - len(failures), len(failures))) - return 1 if failures else 0 - - -if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file