fix: harden WASM phase 5 harness
This commit is contained in:
@@ -7,21 +7,52 @@ cd "$(dirname "$0")/../.."
|
||||
OUT=/tmp/uce/wasm-phase5
|
||||
mkdir -p "$OUT"
|
||||
|
||||
python3 tests/run_network_tests.py --include-internal --json-report "$OUT/native-network.json"
|
||||
python3 tests/run_network_tests.py --include-internal --match starter --json-report "$OUT/native-starter.json"
|
||||
python3 spikes/wasm-phase5/audit_site_statics.py
|
||||
python3 spikes/wasm-phase5/benchmark.py --out-dir "$OUT"
|
||||
# The network suite can show transient cold-compile timeouts immediately after a
|
||||
# binary rebuild. Run a throwaway pass first so the measured gate is a warmed
|
||||
# parity signal rather than a compiler-cache signal. The warmup is allowed to
|
||||
# fail; the measured pass below is not.
|
||||
python3 tests/run_network_tests.py --include-internal --exclude 'site tests tasks' --json-report "$OUT/native-network-warmup.json" || true
|
||||
|
||||
python3 - <<'PY'
|
||||
network_status=0
|
||||
starter_status=0
|
||||
benchmark_status=0
|
||||
python3 tests/run_network_tests.py --include-internal --json-report "$OUT/native-network.json" || network_status=$?
|
||||
python3 tests/run_network_tests.py --include-internal --match starter --json-report "$OUT/native-starter.json" || starter_status=$?
|
||||
python3 spikes/wasm-phase5/audit_site_statics.py --out-dir "$OUT"
|
||||
python3 spikes/wasm-phase5/benchmark.py --out-dir "$OUT" || benchmark_status=$?
|
||||
|
||||
NETWORK_STATUS=$network_status STARTER_STATUS=$starter_status BENCHMARK_STATUS=$benchmark_status python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
MIN_NETWORK_CASES = 80
|
||||
MIN_STARTER_CASES = 10
|
||||
MIN_BENCHMARKS = 3
|
||||
out = Path('/tmp/uce/wasm-phase5')
|
||||
network = json.loads((out / 'native-network.json').read_text())
|
||||
starter = json.loads((out / 'native-starter.json').read_text())
|
||||
bench = json.loads((out / 'benchmark.json').read_text())
|
||||
failures = [r for r in network if not r['ok']] + [r for r in starter if not r['ok']] + [r for r in bench if not r['ok']]
|
||||
if failures:
|
||||
structural_failures = []
|
||||
if len(network) < MIN_NETWORK_CASES:
|
||||
structural_failures.append(f"network suite ran {len(network)} cases, expected at least {MIN_NETWORK_CASES}")
|
||||
if len(starter) < MIN_STARTER_CASES:
|
||||
structural_failures.append(f"starter subset ran {len(starter)} cases, expected at least {MIN_STARTER_CASES}")
|
||||
if len(bench) < MIN_BENCHMARKS:
|
||||
structural_failures.append(f"benchmark ran {len(bench)} rows, expected at least {MIN_BENCHMARKS}")
|
||||
nonzero = {
|
||||
'network_status': int(os.environ['NETWORK_STATUS']),
|
||||
'starter_status': int(os.environ['STARTER_STATUS']),
|
||||
'benchmark_status': int(os.environ['BENCHMARK_STATUS']),
|
||||
}
|
||||
for name, status in nonzero.items():
|
||||
if status != 0:
|
||||
structural_failures.append(f"{name} exited {status}")
|
||||
if failures or structural_failures:
|
||||
print('PHASE5 HARNESS: FAIL')
|
||||
for failure in structural_failures:
|
||||
print(failure)
|
||||
for failure in failures:
|
||||
print(failure)
|
||||
raise SystemExit(1)
|
||||
|
||||
Reference in New Issue
Block a user