From d95fb381838a6394532c335fedf9ddd08145288c Mon Sep 17 00:00:00 2001 From: udo Date: Sun, 19 Jul 2026 03:02:15 +0000 Subject: [PATCH] Bound request-time Wasm compilation --- docs/setup.md | 11 +- docs/wasm-runtime-architecture.md | 20 +- scripts/run_cli_tests.sh | 1 + scripts/test_dependency_invalidation.sh | 11 +- scripts/test_wasm_compile_timeout.sh | 206 +++++++++++ scripts/test_wasm_invocation_timeout.sh | 2 +- src/lib/compiler-parser.cpp | 2 +- src/lib/compiler.cpp | 461 ++++++++++++++++++++++-- src/lib/compiler.h | 5 + src/lib/sys.cpp | 179 +++++++++ src/lib/sys.h | 3 + src/linux_fastcgi.cpp | 85 ++++- src/wasm/backend.cpp | 14 +- src/wasm/backend.h | 3 +- src/wasm/worker.cpp | 146 +++----- 15 files changed, 1002 insertions(+), 147 deletions(-) create mode 100755 scripts/test_wasm_compile_timeout.sh diff --git a/docs/setup.md b/docs/setup.md index c0320ba..f769d3a 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -237,7 +237,16 @@ Important settings: the Wasm backend from starting. The invocation timeout defaults to `30000` and is enforced to the epoch ticker's period resolution. Blocking host helpers retain their own shorter limits and are capped to the remaining invocation - budget where the underlying operation is cancellable. + budget where the underlying operation is cancellable. Cold entry compilation, + explicit `unit_compile()`, dynamic component JIT, transitive `#load` work, + compiler and registry lock waits, and the configured runtime-error page all + consume the same request budget. A timed-out compiler process group is killed + without replacing the prior artifact set or persisting a compile-failure + backoff. Request compiles require a zero child exit status and complete staged + output; generated C++, exports, source map, Wasm, metadata, cached-module + invalidation, and diagnostics are published or rolled back as one guarded + generation. Proactive and offline precompile work remain independent of + request invocation deadlines. After editing settings, restart UCE: diff --git a/docs/wasm-runtime-architecture.md b/docs/wasm-runtime-architecture.md index 5425711..a468a51 100644 --- a/docs/wasm-runtime-architecture.md +++ b/docs/wasm-runtime-architecture.md @@ -202,14 +202,23 @@ The graceful signal handler belongs to the parent and render workers. Generic Epoch interruption measures uninterrupted guest CPU segments. A separate absolute workspace invocation deadline starts before app-owned entry-unit -loading and initialization and remains unchanged through the selected handler, -ONCE, and every nested component/unit call. The common hostcall membrane checks +readiness and cold compilation, loading, and initialization. It remains +unchanged through the selected handler, ONCE, every nested component/unit call, +and configured runtime-error rendering. The common hostcall membrane checks that deadline before and after every native call and re-arms the store with the smaller of the remaining absolute budget and the CPU-segment budget. A cheap hostcall loop therefore cannot renew an invocation indefinitely. Blocking host helpers retain operation-specific limits and cap them to the remaining invocation budget where the underlying operation is cancellable. Forked task callbacks receive a fresh invocation deadline capped by the task lifetime. +Synchronous compiler locks, transitive `#load` compilation, and compiler child +processes consume that same deadline. Compiler children run in a dedicated +process group; timeout kills the group, retains the previous generation, and +never records a caller-budget timeout as a persisted source failure. Bounded +compiles require a valid zero child exit, cap captured output, and stage +generated C++, exports, source maps, Wasm, and metadata. Publication removes +the prior serialized module and diagnostics inside the same rollback-protected +generation while repeatedly checking the live request deadline. `request_perf()` reports worker module-cache hits and misses and divides a miss into artifact lookup, wasm read, custom-section parse, serialized-module @@ -556,6 +565,13 @@ header free-functions are `inline`. The wasm backend exposes only declarations function table, enabling `InstancePre`. A separately built legacy `--import-table` core must still pass the demo and 64-request pool-isolation groups before the compatibility fallback changes. + `scripts/test_wasm_compile_timeout.sh` runs an isolated one-worker runtime + with a two-second request budget. It covers cold entry JIT, guest + `unit_compile()`, dynamic component and transitive `#load` compilation, + held unit and registry locks, silent nonzero and missing-output compiler + results, compiler descendants, staged-output timeout, prior-generation + hashes, same-worker recovery, configured-error-page boundedness, residue, + and deadline-independent offline precompile. `scripts/test_cold_component_deadline.sh` separately compiles a deliberately cold component that exceeds the development epoch window and proves the parent request still renders it. The focused shell gates create temporary diff --git a/scripts/run_cli_tests.sh b/scripts/run_cli_tests.sh index 2308649..c78ebb7 100755 --- a/scripts/run_cli_tests.sh +++ b/scripts/run_cli_tests.sh @@ -72,6 +72,7 @@ if [[ "$action" == "run" ]]; then timeout --signal=TERM --kill-after=5s 175s scripts/test_parallel_proactive_compile.sh scripts/test_cold_component_deadline.sh scripts/test_wasm_invocation_timeout.sh + timeout --signal=TERM --kill-after=5s 130s scripts/test_wasm_compile_timeout.sh scripts/test_nested_component_props.sh scripts/test_component_once_prefetch.sh scripts/test_relative_component_cache.sh diff --git a/scripts/test_dependency_invalidation.sh b/scripts/test_dependency_invalidation.sh index 1bcf2e5..b6b838f 100755 --- a/scripts/test_dependency_invalidation.sh +++ b/scripts/test_dependency_invalidation.sh @@ -398,7 +398,7 @@ rm -f "$cache_dir/parent.uce.cwasm" for _ in {1..16}; do assert_marker parent dependency-marker-c; done race_unit="$source_dir/source-race.uce" -race_cpp="$cache_dir/source-race.uce.cpp" +race_cpp="" { printf '%s\n' 'String source_race_marker() { return("source-race-marker-a"); }' for i in $(seq 1 3500); do @@ -414,16 +414,17 @@ rm -f "$cache_dir/source-race.uce.cpp" "$cache_dir/source-race.uce.wasm" "$cache ) & race_pid=$! deadline=$((SECONDS + 10)) -while [[ ! -e "$race_cpp" && $SECONDS -lt $deadline ]]; do +while [[ -z "$race_cpp" && $SECONDS -lt $deadline ]]; do + race_cpp=$(find "$cache_dir" -maxdepth 1 -name 'source-race.uce.invocation-*.cpp' -print -quit 2>/dev/null || true) if ! kill -0 "$race_pid" 2>/dev/null; then - echo "source-race compile finished before publishing generated C++" >&2 + echo "source-race compile finished before staging generated C++" >&2 cat "$race_output" "$race_error" >&2 || true exit 1 fi sleep 0.02 done -if [[ ! -e "$race_cpp" ]]; then - echo "source-race compile did not publish generated C++ before deadline" >&2 +if [[ -z "$race_cpp" || ! -e "$race_cpp" ]]; then + echo "source-race compile did not stage generated C++ before deadline" >&2 exit 1 fi if ! grep -q 'source-race-marker-a' "$race_cpp"; then diff --git a/scripts/test_wasm_compile_timeout.sh b/scripts/test_wasm_compile_timeout.sh new file mode 100755 index 0000000..8ffca21 --- /dev/null +++ b/scripts/test_wasm_compile_timeout.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +error_page_path=$(sed -n '/^bool render_wasm_error_page(/,/^}/p' src/linux_fastcgi.cpp) +[[ "$error_page_path" == *'get_shared_unit_bounded('* && "$error_page_path" == *'wasm_invocation_remaining_ms(invocation_deadline)'* ]] +[[ "$error_page_path" != *'get_shared_unit(&request'* ]] + +if [[ "${1:-}" != --inside ]]; then + exec timeout --signal=TERM --kill-after=5s 120s unshare --mount --fork --kill-child=TERM "$0" --inside +fi + +root="/tmp/uce-compile-timeout-$$" +site="$root/site" +work="$root/work" +settings="$root/settings.cfg" +log="$root/service.log" +server_pid="" + +cleanup() { + status=$? + if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then + kill -TERM "$server_pid" 2>/dev/null || true + for _ in $(seq 1 100); do kill -0 "$server_pid" 2>/dev/null || break; sleep 0.02; done + kill -KILL "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + fi + if (( status != 0 )) && [[ -r "$log" ]]; then cat "$log" >&2; fi + rm -rf "$root" + return "$status" +} +trap cleanup EXIT +mkdir -p "$site/components" "$work" "$root/run" "$root/session" "$root/upload" +sed -E '/^[[:space:]]*(BIN_DIRECTORY|PRECOMPILE_FILES_IN|SITE_DIRECTORY|FCGI_SOCKET_PATH|FCGI_PORT|CLI_SOCKET_PATH|WS_BROKER_SOCKET_PATH|HTTP_PORT|HTTP_DOCUMENT_ROOT|SESSION_PATH|TMP_UPLOAD_PATH|WASM_CORE_PATH|WASM_COMPILE_SCRIPT|WASM_INVOCATION_TIMEOUT_MS|WASM_EPOCH_PERIOD_MS|PROACTIVE_COMPILE_ENABLED|WORKER_COUNT|COMPILE_FAILURE_RETRY_SECONDS)[[:space:]]*=/d' \ + /etc/uce/settings.cfg >"$settings" +cat >>"$settings" <"$root/compile" <'$root/compiler-pid' + (sleep 3; touch '$root/late-descendant') & + cp "\$2/\$(basename "\$source_file").wasm" "\$2/\$5" + cp "\$2/\$(basename "\$source_file").wasm.source-map" "\$2/\$5.source-map" + sleep 60 +fi +if [[ -r '$root/hold' ]] && grep -Fxq "\$source_file" '$root/hold'; then + printf '%s\n' "\$\$" >'$root/compiler-pid' + (sleep 3; touch '$root/late-descendant') & + sleep 60 +fi +if [[ -r '$root/silent-fail' ]] && grep -Fxq "\$source_file" '$root/silent-fail'; then exit 23; fi +if [[ -r '$root/empty-success' ]] && grep -Fxq "\$source_file" '$root/empty-success'; then exit 0; fi +if [[ -r '$root/delay' ]] && grep -Fxq "\$source_file" '$root/delay'; then sleep 1; fi +exec '$(pwd)/scripts/compile_wasm_unit' "\$@" +SHIM +chmod +x "$root/compile" + +cat >"$site/driver.uce" <<'UCE' +CLI(Request& context) { + if(context.get["health"] == "1") { print(request_perf()["worker_pid"].to_u64(), "|health"); return; } + if(context.get["dynamic"] == "1") { print(component("slow-component.uce")); return; } + String target = first(context.get["target"], "slow.uce"); + print(unit_compile(target) ? "compiled" : "failed"); +} +UCE +printf '%s\n' 'CLI(Request& context) { print("entry"); }' >"$site/cold-entry.uce" +printf '%s\n' 'CLI(Request& context) { print("slow"); }' >"$site/slow.uce" +printf '%s\n' 'CLI(Request& context) { print("silent"); }' >"$site/silent.uce" +printf '%s\n' 'CLI(Request& context) { print("empty"); }' >"$site/empty.uce" +printf '%s\n' '#load "transitive-child.uce"' 'CLI(Request& context) { print("parent"); }' >"$site/transitive.uce" +printf '%s\n' 'String transitive_value() { return("a"); }' >"$site/transitive-child.uce" +printf '%s\n' 'COMPONENT(Request& context) { print("component"); }' >"$site/components/slow-component.uce" + +timeout 40s env UCE_PRECOMPILE_FILES_IN="$site" UCE_PRECOMPILE_BIN_DIRECTORY="$work" UCE_PRECOMPILE_JOBS=1 bin/uce_fastcgi.linux.bin --precompile >"$root/precompile.log" 2>&1 || { cat "$root/precompile.log" >&2; exit 1; } +cache="$(scripts/unit_cache_directory "$work")$(realpath "$site")" +generation=$(scripts/unit_cache_directory "$work") +rm -f "$cache/cold-entry.uce."* "$cache/components/slow-component.uce."* + +timeout --signal=TERM --kill-after=5s 90s bin/uce_fastcgi.linux.bin >"$log" 2>&1 & +server_pid=$! +for _ in $(seq 1 400); do [[ -S "$root/run/cli.sock" ]] && break; sleep 0.02; done +[[ -S "$root/run/cli.sock" ]] || { cat "$log" >&2; exit 1; } + +request() { + local spec="$1" path="${1%%\?*}" + if [[ "$spec" == *\?* ]]; then + local query="${spec#*\?}" + IFS='&' read -r -a params <<<"$query" + timeout 5s scripts/uce-cli --get "$path" "${params[@]}" + else + timeout 5s scripts/uce-cli "$path" + fi +} +driver="" +for _ in $(seq 1 100); do + set +e + driver=$(request '/driver.uce?health=1' 2>/dev/null) + rc=$? + set -e + (( rc == 0 )) && [[ "$driver" == *'|health' ]] && break + sleep 0.05 +done +[[ "$driver" == *'|health' ]] || { echo "driver did not become ready: $driver" >&2; exit 1; } +worker_pid=${driver%%|*} +[[ "$driver" == "$worker_pid|health" ]] +[[ "$(request /driver.uce)" == compiled ]] +before=$(sha256sum "$cache/slow.uce.wasm" "$cache/slow.uce.wasm.source-map" "$cache/slow.uce.cpp" "$cache/slow.uce.exports.txt" "$cache/slow.uce.meta.txt") + +assert_timeout() { + local url="$1" source="$2" started elapsed output + printf '%s\n' "$source" >"$root/hold" + rm -f "$root/compiler-pid" "$root/late-descendant" + started=$(date +%s%N) + set +e + output=$(request "$url" 2>&1) + set -e + elapsed=$(( ($(date +%s%N) - started) / 1000000 )) + [[ "$output" == *UCE_INVOCATION_TIMEOUT:* ]] || { echo "$url lacked timeout: $output" >&2; exit 1; } + (( elapsed >= 1700 && elapsed <= 3500 )) || { echo "$url took ${elapsed}ms" >&2; exit 1; } + [[ "$(request '/driver.uce?health=1')" == "$worker_pid|health" ]] + [[ -s "$root/compiler-pid" ]] + compiler_pid=$(<"$root/compiler-pid") + ! kill -0 "$compiler_pid" 2>/dev/null || { echo "compiler $compiler_pid survived" >&2; exit 1; } + sleep 0.1 + [[ ! -e "$root/late-descendant" ]] + rm -f "$root/hold" +} + +assert_timeout /driver.uce "$site/slow.uce" +[[ "$before" == "$(sha256sum "$cache/slow.uce.wasm" "$cache/slow.uce.wasm.source-map" "$cache/slow.uce.cpp" "$cache/slow.uce.exports.txt" "$cache/slow.uce.meta.txt")" ]] +[[ ! -e "$cache/slow.uce.compile.txt" && ! -e "$cache/slow.uce.wasm-check.txt" ]] +! find "$cache" -name '*.invocation-*' -print -quit | grep -q . +[[ "$(request /driver.uce)" == compiled ]] +before=$(sha256sum "$cache/slow.uce.wasm" "$cache/slow.uce.wasm.source-map" "$cache/slow.uce.cpp" "$cache/slow.uce.exports.txt" "$cache/slow.uce.meta.txt") + +printf '%s\n' "$site/silent.uce" >"$root/silent-fail" +[[ "$(request '/driver.uce?target=silent.uce')" == failed ]] +grep -q 'status 23' "$cache/silent.uce.compile.txt" +rm -f "$root/silent-fail" +printf '%s\n' "$site/empty.uce" >"$root/empty-success" +[[ "$(request '/driver.uce?target=empty.uce')" == failed ]] +grep -q 'without creating' "$cache/empty.uce.compile.txt" +rm -f "$root/empty-success" +! find "$cache" -name '*.invocation-*' -print -quit | grep -q . + +printf '%s\n' "$site/slow.uce" >"$root/stage-timeout" +assert_timeout /driver.uce "$site/slow.uce" +rm -f "$root/stage-timeout" +[[ "$before" == "$(sha256sum "$cache/slow.uce.wasm" "$cache/slow.uce.wasm.source-map" "$cache/slow.uce.cpp" "$cache/slow.uce.exports.txt" "$cache/slow.uce.meta.txt")" ]] + +transitive_before=$(sha256sum "$cache/transitive.uce.wasm" "$cache/transitive.uce.wasm.source-map" "$cache/transitive.uce.cpp" "$cache/transitive.uce.exports.txt" "$cache/transitive.uce.meta.txt") +printf '%s\n' 'String transitive_value() { return("b"); }' >"$site/transitive-child.uce" +assert_timeout '/driver.uce?target=transitive.uce' "$site/transitive-child.uce" +[[ "$transitive_before" == "$(sha256sum "$cache/transitive.uce.wasm" "$cache/transitive.uce.wasm.source-map" "$cache/transitive.uce.cpp" "$cache/transitive.uce.exports.txt" "$cache/transitive.uce.meta.txt")" ]] + +assert_timeout '/driver.uce?dynamic=1' "$site/components/slow-component.uce" +assert_timeout /cold-entry.uce "$site/cold-entry.uce" + +printf '%s\n' "$site/slow.uce" >"$root/delay" +timeout 25s env UCE_PRECOMPILE_FILES_IN="$site" UCE_PRECOMPILE_BIN_DIRECTORY="$work" UCE_PRECOMPILE_JOBS=1 bin/uce_fastcgi.linux.bin --precompile >/dev/null +rm -f "$root/delay" + +exec 8>"$generation/known-uce-files.txt.lock" +flock 8 +started=$(date +%s%N) +set +e +registry_locked=$(request /driver.uce 2>&1) +set -e +elapsed=$(( ($(date +%s%N) - started) / 1000000 )) +flock -u 8 +[[ "$registry_locked" == *UCE_INVOCATION_TIMEOUT:* && "$elapsed" -le 3500 ]] + +exec 9>"$cache/slow.uce.wasm.lock" +flock 9 +started=$(date +%s%N) +set +e +locked=$(request /driver.uce 2>&1) +set -e +elapsed=$(( ($(date +%s%N) - started) / 1000000 )) +flock -u 9 +[[ "$locked" == *UCE_INVOCATION_TIMEOUT:* && "$elapsed" -le 3500 ]] + +echo "wasm compile timeout passed (entry, explicit, dynamic, unit/registry locks, error page, process group, staging, native precompile)" diff --git a/scripts/test_wasm_invocation_timeout.sh b/scripts/test_wasm_invocation_timeout.sh index 06c91dd..346eb56 100755 --- a/scripts/test_wasm_invocation_timeout.sh +++ b/scripts/test_wasm_invocation_timeout.sh @@ -113,7 +113,7 @@ for unit in hostcall-loop.uce legacy-shell.uce sleep.uce; do expected_ms=5000 fi else - grep -q 'UCE_INVOCATION_TIMEOUT:' "$body_file" || { echo "$unit lacked invocation-timeout classification" >&2; exit 1; } + grep -q 'UCE_INVOCATION_TIMEOUT:' "$body_file" || { echo "$unit lacked invocation-timeout classification: $(cat "$body_file")" >&2; exit 1; } expected_ms=$invocation_ms fi (( elapsed_ms >= expected_ms - 1000 && elapsed_ms <= expected_ms + 3000 )) || { echo "$unit completed in ${elapsed_ms}ms; expected about ${expected_ms}ms" >&2; exit 1; } diff --git a/src/lib/compiler-parser.cpp b/src/lib/compiler-parser.cpp index 6293020..32fb484 100644 --- a/src/lib/compiler-parser.cpp +++ b/src/lib/compiler-parser.cpp @@ -567,7 +567,7 @@ String compiler_preprocess_shared_unit_char_wise(Request* context, SharedUnit* s String resolved_unit = unit_name; if(resolved_unit != "" && resolved_unit[0] != '/') resolved_unit = expand_path(resolved_unit, su->src_path); - SharedUnit* sub_su = (resolved_unit == "" ? 0 : get_shared_unit(context, resolved_unit)); + SharedUnit* sub_su = (resolved_unit == "" ? 0 : get_shared_unit_for_preprocess(context, resolved_unit)); if(sub_su) parsed_content.append("#include \"" + sub_su->bin_path + "/" + sub_su->pre_file_name + "\"\n"); } diff --git a/src/lib/compiler.cpp b/src/lib/compiler.cpp index 44c2158..e5be256 100644 --- a/src/lib/compiler.cpp +++ b/src/lib/compiler.cpp @@ -3,6 +3,7 @@ #include "hash.h" #include "../wasm/abi.h" #include +#include #include #include #include @@ -52,6 +53,49 @@ struct SharedUnitCompileCheck bool needs_compile = false; }; +struct CompilerDeadline +{ + using Clock = std::chrono::steady_clock; + Clock::time_point expires_at; + bool timed_out = false; + bool operational_failure = false; + String operational_error; + bool dependency_failure = false; + String dependency_error; + + explicit CompilerDeadline(u64 timeout_ms) : expires_at(Clock::now() + std::chrono::milliseconds(timeout_ms)) {} + + u64 remaining_ms() const + { + auto now = Clock::now(); + if(now >= expires_at) + return(0); + return((u64)std::chrono::duration_cast(expires_at - now).count()); + } + + bool expire_if_needed() + { + if(remaining_ms() > 0) + return(false); + timed_out = true; + return(true); + } +}; + +static std::atomic compiler_invocation_stage_counter(0); +static thread_local CompilerDeadline* compiler_active_deadline = 0; + +class CompilerDeadlineScope +{ + CompilerDeadline* previous; +public: + explicit CompilerDeadlineScope(CompilerDeadline* deadline) : previous(compiler_active_deadline) + { + compiler_active_deadline = deadline; + } + ~CompilerDeadlineScope() { compiler_active_deadline = previous; } +}; + struct UnitSourceSignatureEntry { std::chrono::steady_clock::time_point checked_at; @@ -304,6 +348,92 @@ String compiler_source_map_path(String wasm_path) return(wasm_path + ".source-map"); } +static bool compiler_publish_staged_artifacts(SharedUnit* su, String staged_pre_name, String staged_api_name, + String staged_wasm_name, String staged_map_name, String staged_meta_name, CompilerDeadline* deadline, String& error) +{ + struct Artifact + { + String staged; + String canonical; + String previous; + bool remove_only = false; + bool existed = false; + bool published = false; + }; + std::vector artifacts = { + { staged_pre_name, compiler_generated_cpp_path(su), staged_pre_name + ".previous" }, + { staged_api_name, su->api_file_name, staged_api_name + ".previous" }, + { staged_map_name, compiler_source_map_path(su->wasm_name), staged_map_name + ".previous" }, + { "", compiler_cached_wasm_path(su->wasm_name), staged_wasm_name + ".cached.previous", true }, + { staged_wasm_name, su->wasm_name, staged_wasm_name + ".previous" }, + { "", su->compile_output_file_name, staged_wasm_name + ".compile.previous", true }, + { "", su->wasm_check_file_name, staged_wasm_name + ".check.previous", true }, + { staged_meta_name, su->meta_file_name, staged_meta_name + ".previous" } + }; + auto rollback = [&]() { + for(auto it = artifacts.rbegin(); it != artifacts.rend(); ++it) + { + if(it->published) + { + if(it->existed) + { + if(rename(it->previous.c_str(), it->canonical.c_str()) != 0) + error += "\ncould not restore previous bounded compile artifact " + it->canonical + ": " + String(std::strerror(errno)); + } + else + file_unlink(it->canonical); + } + file_unlink(it->previous); + } + }; + for(auto& artifact : artifacts) + { + if(deadline && deadline->expire_if_needed()) + { + error = "bounded compile deadline expired while preserving prior artifacts"; + for(auto& cleanup : artifacts) + file_unlink(cleanup.previous); + return(false); + } + file_unlink(artifact.previous); + artifact.existed = file_exists(artifact.canonical); + if(artifact.existed && link(artifact.canonical.c_str(), artifact.previous.c_str()) != 0) + { + error = "could not preserve previous bounded compile artifacts: " + String(std::strerror(errno)); + for(auto& cleanup : artifacts) + file_unlink(cleanup.previous); + return(false); + } + } + for(auto& artifact : artifacts) + { + if(deadline && deadline->expire_if_needed()) + { + error = "bounded compile deadline expired during artifact publication"; + rollback(); + return(false); + } + if(artifact.remove_only) + file_unlink(artifact.canonical); + else if(rename(artifact.staged.c_str(), artifact.canonical.c_str()) != 0) + { + error = "could not publish bounded compile artifacts: " + String(std::strerror(errno)); + rollback(); + return(false); + } + artifact.published = true; + } + if(deadline && deadline->expire_if_needed()) + { + error = "bounded compile deadline expired during artifact publication"; + rollback(); + return(false); + } + for(auto& artifact : artifacts) + file_unlink(artifact.previous); + return(true); +} + void compiler_unlink_unit_wasm_artifacts(SharedUnit* su) { file_unlink(su->wasm_name); @@ -437,6 +567,35 @@ int compiler_open_lock_file(String file_name, String purpose, bool nonblocking = return(fdlock); } +int compiler_open_lock_file_bounded(String file_name, String purpose, CompilerDeadline* deadline) +{ + if(!deadline) + return(compiler_open_lock_file(file_name, purpose)); + auto lock_dir = dirname(file_name); + if(lock_dir != "") + mkdir(lock_dir); + int fdlock = open(file_name.c_str(), O_RDWR | O_CREAT, 0666); + if(fdlock == -1) + return(-1); + fcntl(fdlock, F_SETFD, FD_CLOEXEC); + while(true) + { + if(flock(fdlock, LOCK_EX | LOCK_NB) == 0) + return(fdlock); + if(errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) + { + close(fdlock); + return(-1); + } + if(deadline->expire_if_needed()) + { + close(fdlock); + return(-2); + } + usleep(1000); + } +} + void compiler_close_lock_file(int fdlock) { if(fdlock == -1) @@ -445,6 +604,18 @@ void compiler_close_lock_file(int fdlock) close(fdlock); } +static void compiler_mark_source_generation_nonblocking(Request* context) +{ + if(!context || !context->server) + return; + String file_name = compiler_source_generation_file_name(context); + int fdlock = compiler_open_lock_file(file_name + ".lock", "source-generation", true); + if(fdlock < 0) + return; + file_put_contents(file_name, std::to_string(getpid()) + ":" + std::to_string((u64)(time_precise() * 1000000.0)) + "\n"); + compiler_close_lock_file(fdlock); +} + String compiler_normalize_unit_path(Request* context, String file_name) { file_name = trim(file_name); @@ -947,7 +1118,7 @@ String compiler_format_source_read_failure(Request* context, SharedUnit* su, Str return(result); } -void compile_shared_unit(Request* context, SharedUnit* su) +void compile_shared_unit_bounded(Request* context, SharedUnit* su, CompilerDeadline* deadline) { f64 comp_start = time_precise(); @@ -955,9 +1126,13 @@ void compile_shared_unit(Request* context, SharedUnit* su) { su->compiler_messages = "source file not found (" + su->file_name + ")"; file_put_contents(su->compile_output_file_name, su->compiler_messages + "\n"); - compiler_untrack_known_unit(context, su->file_name); + if(!deadline) + compiler_untrack_known_unit(context, su->file_name); compiler_record_compile_result(su, time_precise() - comp_start, false, "missing_source", su->compiler_messages); - compiler_mark_source_generation(context); + if(deadline) + compiler_mark_source_generation_nonblocking(context); + else + compiler_mark_source_generation(context); return; } struct stat source_info; @@ -974,39 +1149,180 @@ void compile_shared_unit(Request* context, SharedUnit* su) compiler_unlink_unit_wasm_artifacts(su); compiler_record_compile_result(su, time_precise() - comp_start, false, "compile_error", su->compiler_messages); printf("%s \n", compiler_format_source_read_failure(context, su, su->compiler_messages).c_str()); - compiler_mark_source_generation(context); + if(deadline) + compiler_mark_source_generation_nonblocking(context); + else + compiler_mark_source_generation(context); return; } - shell_exec("mkdir -p " + shell_escape(su->pre_path)); + mkdir(su->pre_path); String compiled_input_signature; + String staged_wasm_file_name; + String staged_wasm_name; + String staged_map_name; + String staged_pre_file_name; + String staged_pre_name; + String staged_api_name; + String staged_meta_name; + bool publication_failed = false; + if(deadline) + { + u64 stage_id = compiler_invocation_stage_counter.fetch_add(1, std::memory_order_relaxed) + 1; + String stage_suffix = ".invocation-" + std::to_string((u64)getpid()) + "-" + std::to_string(stage_id); + staged_wasm_file_name = su->src_file_name + stage_suffix + ".wasm"; + staged_wasm_name = su->bin_path + "/" + staged_wasm_file_name; + staged_map_name = compiler_source_map_path(staged_wasm_name); + staged_pre_file_name = su->src_file_name + stage_suffix + ".cpp"; + staged_pre_name = su->pre_path + "/" + staged_pre_file_name; + staged_api_name = su->bin_path + "/" + su->src_file_name + stage_suffix + ".exports.txt"; + staged_meta_name = su->bin_path + "/" + su->src_file_name + stage_suffix + ".meta.txt"; + file_unlink(staged_wasm_name); + file_unlink(staged_map_name); + file_unlink(staged_pre_name); + file_unlink(staged_api_name); + file_unlink(staged_meta_name); + } for(u64 attempt = 0; attempt < 2; attempt++) { + if(deadline && deadline->expire_if_needed()) + break; su->api_declarations.clear(); compiled_input_signature = compiler_unit_input_signature(context, su); - file_put_contents(su->pre_path + "/" + su->pre_file_name, preprocess_shared_unit(context, su)); - file_put_contents(su->api_file_name, join(su->api_declarations, "\n")); + String generated_source; + { + CompilerDeadlineScope deadline_scope(deadline); + generated_source = preprocess_shared_unit(context, su); + } + if(deadline && deadline->dependency_failure) + { + su->compiler_messages = first(deadline->dependency_error, "transitive dependency compilation failed"); + break; + } + if(deadline && (deadline->timed_out || deadline->operational_failure)) + break; + if(!file_put_contents(deadline ? staged_pre_name : su->pre_path + "/" + su->pre_file_name, generated_source) || + !file_put_contents(deadline ? staged_api_name : su->api_file_name, join(su->api_declarations, "\n"))) + { + su->compiler_messages = "could not write generated bounded compile inputs"; + break; + } - su->compiler_messages = trim(shell_exec(shell_escape(compiler_wasm_compile_script(context))+" "+ + String compile_command = shell_escape(compiler_wasm_compile_script(context))+" "+ shell_escape(su->src_path)+" "+ shell_escape(su->bin_path)+" "+ shell_escape(su->file_name)+" "+ - shell_escape(su->pre_file_name)+" "+ - shell_escape(su->wasm_file_name)+" "+ - shell_escape(compiler_unit_bin_directory(context)) - )); + shell_escape(deadline ? staged_pre_file_name : su->pre_file_name)+" "+ + shell_escape(deadline ? staged_wasm_file_name : su->wasm_file_name)+" "+ + shell_escape(compiler_unit_bin_directory(context)); + if(deadline) + { + u64 remaining_ms = deadline->remaining_ms(); + if(remaining_ms == 0) + { + deadline->timed_out = true; + break; + } + DValue execution = process_exec(compile_command + " 2>&1", "", StringMap(), remaining_ms, 1024 * 1024); + if(execution["timed_out"].to_bool()) + { + deadline->timed_out = true; + break; + } + su->compiler_messages = trim(execution["stdout"].to_string() + execution["stderr"].to_string()); + if(execution["output_truncated"].to_bool()) + su->compiler_messages += (su->compiler_messages == "" ? "" : "\n") + String("compiler output truncated at 1048576 bytes"); + if(execution["exit_code"].to_s64(-1) != 0 && su->compiler_messages == "") + su->compiler_messages = "wasm compile script exited with status " + std::to_string(execution["exit_code"].to_s64(-1)); + } + else + su->compiler_messages = trim(shell_exec(compile_command)); - if(su->compiler_messages.length() == 0 && !file_exists(su->wasm_name)) - su->compiler_messages = "wasm compile script completed without creating " + su->wasm_name; + String compiled_wasm_name = deadline ? staged_wasm_name : su->wasm_name; + if(su->compiler_messages.length() == 0 && !file_exists(compiled_wasm_name)) + su->compiler_messages = "wasm compile script completed without creating " + compiled_wasm_name; if(su->compiler_messages.length() > 0) break; String current_input_signature = compiler_unit_input_signature(context, su); if(current_input_signature == compiled_input_signature) + { + if(deadline) + { + if(deadline->expire_if_needed()) + break; + String source_map = file_get_contents(staged_map_name); + if(source_map == "") + su->compiler_messages = "bounded wasm compile did not create a source map"; + else + { + if(!file_put_contents(staged_map_name, replace(source_map, staged_pre_name, compiler_generated_cpp_path(su))) || + !file_put_contents(staged_meta_name, compiler_unit_metadata_text(context, su, compiled_input_signature))) + { + su->compiler_messages = "could not stage bounded compile metadata"; + publication_failed = true; + } + else + publication_failed = !compiler_publish_staged_artifacts(su, staged_pre_name, staged_api_name, + staged_wasm_name, staged_map_name, staged_meta_name, deadline, su->compiler_messages); + } + } break; - compiler_unlink_unit_wasm_artifacts(su); + } + if(deadline) + { + file_unlink(staged_wasm_name); + file_unlink(staged_map_name); + file_unlink(staged_pre_name); + file_unlink(staged_api_name); + file_unlink(staged_meta_name); + } + else + compiler_unlink_unit_wasm_artifacts(su); if(attempt == 1) su->compiler_messages = "source changed during wasm compile; retry required"; } + if(deadline && deadline->timed_out) + { + file_unlink(staged_wasm_name); + file_unlink(staged_map_name); + file_unlink(staged_pre_name); + file_unlink(staged_api_name); + file_unlink(staged_meta_name); + su->compiler_messages = "UCE_INVOCATION_TIMEOUT: unit compilation exceeded the invocation deadline"; + compiler_record_compile_result(su, time_precise() - comp_start, false, "compile_timeout", su->compiler_messages); + return; + } + if(deadline && deadline->operational_failure) + { + file_unlink(staged_wasm_name); + file_unlink(staged_map_name); + file_unlink(staged_pre_name); + file_unlink(staged_api_name); + file_unlink(staged_meta_name); + su->compiler_messages = first(deadline->operational_error, "transitive dependency compilation failed"); + compiler_record_compile_result(su, time_precise() - comp_start, false, "dependency_error", su->compiler_messages); + return; + } + if(publication_failed) + { + file_unlink(staged_wasm_name); + file_unlink(staged_map_name); + file_unlink(staged_pre_name); + file_unlink(staged_api_name); + file_unlink(staged_meta_name); + compiler_record_compile_result(su, time_precise() - comp_start, false, "publish_error", su->compiler_messages); + deadline->operational_failure = true; + deadline->operational_error = su->compiler_messages; + return; + } + if(deadline) + { + file_unlink(staged_wasm_name); + file_unlink(staged_map_name); + file_unlink(staged_pre_name); + file_unlink(staged_api_name); + file_unlink(staged_meta_name); + } if(su->compiler_messages.length() > 0) { @@ -1014,15 +1330,19 @@ void compile_shared_unit(Request* context, SharedUnit* su) file_put_contents(su->compile_output_file_name, raw_messages + "\n"); file_put_contents(su->wasm_check_file_name, raw_messages + "\n"); file_put_contents(su->meta_file_name, compiler_unit_metadata_text(context, su, compiled_input_signature)); - compiler_unlink_unit_wasm_artifacts(su); + if(!publication_failed) + compiler_unlink_unit_wasm_artifacts(su); compiler_record_compile_result(su, time_precise() - comp_start, false, "compile_error", raw_messages); printf("%s \n", compiler_format_compile_failure(context, su, raw_messages).c_str()); } else { su->last_compiled = file_mtime(su->wasm_name); - file_unlink(compiler_cached_wasm_path(su->wasm_name)); - file_put_contents(su->meta_file_name, compiler_unit_metadata_text(context, su, compiled_input_signature)); + if(!deadline) + { + file_unlink(compiler_cached_wasm_path(su->wasm_name)); + file_put_contents(su->meta_file_name, compiler_unit_metadata_text(context, su, compiled_input_signature)); + } file_unlink(su->compile_output_file_name); file_unlink(su->wasm_check_file_name); compiler_record_compile_result( @@ -1036,10 +1356,18 @@ void compile_shared_unit(Request* context, SharedUnit* su) (su->pre_path + "/" + su->pre_file_name).c_str(), time_precise() - comp_start); } - compiler_mark_source_generation(context); + if(deadline) + compiler_mark_source_generation_nonblocking(context); + else + compiler_mark_source_generation(context); } -SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name, bool force_recompile, bool retry_current_failure = false) +void compile_shared_unit(Request* context, SharedUnit* su) +{ + compile_shared_unit_bounded(context, su, 0); +} + +SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name, bool force_recompile, bool retry_current_failure = false, CompilerDeadline* deadline = 0) { file_name = compiler_normalize_unit_path(context, file_name); bool bypass_cached_result = force_recompile || retry_current_failure; @@ -1054,7 +1382,13 @@ SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name setup_unit_paths(context, su, file_name); bool can_serve_stale = !bypass_cached_result && compiler_unit_can_serve_stale_artifact(context, file_name); - int fdlock = compiler_open_lock_file(su->wasm_name + ".lock", "shared-unit:" + file_name, can_serve_stale); + int fdlock = deadline ? compiler_open_lock_file_bounded(su->wasm_name + ".lock", "shared-unit:" + file_name, deadline) : + compiler_open_lock_file(su->wasm_name + ".lock", "shared-unit:" + file_name, can_serve_stale); + if(deadline && fdlock == -2) + { + delete su; + return(0); + } if(fdlock == -2 && file_exists(su->wasm_name)) { auto state = inspect_shared_unit_filesystem(context, su); @@ -1106,7 +1440,7 @@ SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name } } else - compile_shared_unit(context, su); + compile_shared_unit_bounded(context, su, deadline); } else { @@ -1116,6 +1450,13 @@ SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name su->compile_error_status = ""; } + if(deadline && (deadline->timed_out || deadline->operational_failure)) + { + compiler_close_lock_file(fdlock); + delete su; + return(0); + } + auto observed_state = inspect_shared_unit_filesystem(context, su); compiler_record_observed_filesystem_state(su, observed_state); @@ -1130,6 +1471,37 @@ SharedUnit* get_shared_unit(Request* context, String file_name) return(compiler_get_shared_unit_internal(context, file_name, false)); } +SharedUnit* get_shared_unit_for_preprocess(Request* context, String file_name) +{ + if(!compiler_active_deadline) + return(get_shared_unit(context, file_name)); + auto result = compiler_get_shared_unit_internal(context, file_name, false, false, compiler_active_deadline); + if(result && trim(result->compiler_messages) != "") + { + compiler_active_deadline->dependency_failure = true; + compiler_active_deadline->dependency_error = "transitive dependency compilation failed: " + file_name + "\n" + trim(result->compiler_messages); + return(0); + } + if(!result) + return(0); + if(!file_exists(compiler_generated_cpp_path(result))) + { + compiler_active_deadline->dependency_failure = true; + compiler_active_deadline->dependency_error = "transitive dependency generated source unavailable: " + file_name; + return(0); + } + return(result); +} + +SharedUnit* get_shared_unit_bounded(Request* context, String file_name, u64 timeout_ms, bool* timed_out) +{ + CompilerDeadline deadline(timeout_ms); + auto result = compiler_get_shared_unit_internal(context, file_name, false, false, &deadline); + if(timed_out) + *timed_out = deadline.timed_out; + return(result); +} + String compiler_source_generation(Request* context) { if(!context || !context->server) @@ -1435,6 +1807,25 @@ void compiler_track_known_unit(Request* context, String file_name) }); } +static bool compiler_track_known_unit_bounded(Request* context, String file_name, CompilerDeadline* deadline) +{ + file_name = compiler_normalize_unit_path(context, file_name); + if(file_name == "" || !compiler_is_known_unit_file(file_name)) + return(true); + String lock_file_name = compiler_registry_lock_file_name(context); + int fdlock = compiler_open_lock_file_bounded(lock_file_name, "compiler-registry", deadline); + if(fdlock < 0) + return(false); + auto files = compiler_read_known_units_unlocked(context); + if(std::find(files.begin(), files.end(), file_name) == files.end()) + { + files.push_back(file_name); + compiler_write_known_units_unlocked(context, files); + } + compiler_close_lock_file(fdlock); + return(true); +} + void compiler_untrack_known_unit(Request* context, String file_name) { file_name = compiler_normalize_unit_path(context, file_name); @@ -1591,3 +1982,29 @@ bool unit_compile(String path) auto su = compiler_get_shared_unit_internal(context, resolved_path, true); return(su && trim(su->compiler_messages) == "" && file_exists(su->wasm_name)); } + +bool unit_compile_bounded(Request* request, String path, u64 timeout_ms, bool* timed_out) +{ + if(timed_out) + *timed_out = false; + if(!request || timeout_ms == 0) + { + if(timed_out) + *timed_out = timeout_ms == 0; + return(false); + } + CompilerDeadline deadline(timeout_ms); + String resolved_path = compiler_resolve_unit_path(request, path); + if(resolved_path == "") + return(false); + if(!compiler_track_known_unit_bounded(request, resolved_path, &deadline)) + { + if(timed_out) + *timed_out = deadline.timed_out; + return(false); + } + auto su = compiler_get_shared_unit_internal(request, resolved_path, true, false, &deadline); + if(timed_out) + *timed_out = deadline.timed_out; + return(su && trim(su->compiler_messages) == "" && file_exists(su->wasm_name)); +} diff --git a/src/lib/compiler.h b/src/lib/compiler.h index 1473a9d..053d17f 100644 --- a/src/lib/compiler.h +++ b/src/lib/compiler.h @@ -25,6 +25,11 @@ String compiler_unit_wasm_path(Request* context, String source_file); void setup_unit_paths(Request* context, SharedUnit* su, String file_name); void compile_shared_unit(Request* context, SharedUnit* su); SharedUnit* get_shared_unit(Request* context, String file_name); +#ifndef __UCE_WASM_UNIT__ +SharedUnit* get_shared_unit_for_preprocess(Request* context, String file_name); +SharedUnit* get_shared_unit_bounded(Request* context, String file_name, u64 timeout_ms, bool* timed_out); +bool unit_compile_bounded(Request* context, String path, u64 timeout_ms, bool* timed_out); +#endif String compiler_error_page_unit(Request* context, String config_key); bool compiler_unit_compile_pending(Request* context, String file_name); bool compiler_unit_compile_in_progress(Request* context, String file_name); diff --git a/src/lib/sys.cpp b/src/lib/sys.cpp index 456484d..7f1bf4b 100644 --- a/src/lib/sys.cpp +++ b/src/lib/sys.cpp @@ -1438,6 +1438,185 @@ bool child_exit_status_take(pid_t pid, int& status, unsigned int since) return(false); } +namespace { + +class ProcessSigchldBlock +{ + sigset_t previous; + bool blocked = false; +public: + ProcessSigchldBlock() + { + sigset_t mask; + sigemptyset(&mask); + sigaddset(&mask, SIGCHLD); + blocked = pthread_sigmask(SIG_BLOCK, &mask, &previous) == 0; + } + void restore() + { + if(blocked) + { + pthread_sigmask(SIG_SETMASK, &previous, 0); + blocked = false; + } + } + ~ProcessSigchldBlock() { restore(); } +}; + +} + +DValue process_exec(String cmd, String input, StringMap env, u64 timeout_ms, u64 output_limit) +{ + DValue result; + result["exit_code"] = (f64)-1; + result["stdout"] = ""; + result["stderr"] = ""; + result["timed_out"].set_bool(false); + result["output_truncated"].set_bool(false); + if(timeout_ms == 0) + timeout_ms = 5000; + int inpipe[2] = {-1, -1}, outpipe[2] = {-1, -1}, errpipe[2] = {-1, -1}; + if(pipe(inpipe) != 0 || pipe(outpipe) != 0 || pipe(errpipe) != 0) + { + for(int fd : {inpipe[0], inpipe[1], outpipe[0], outpipe[1], errpipe[0], errpipe[1]}) + if(fd >= 0) + close(fd); + result["stderr"] = "pipe failed"; + return(result); + } + ProcessSigchldBlock sigchld; + unsigned int child_status_snapshot_value = child_exit_status_snapshot(); + pid_t pid = fork(); + if(pid < 0) + { + close(inpipe[0]); close(inpipe[1]); close(outpipe[0]); close(outpipe[1]); close(errpipe[0]); close(errpipe[1]); + result["stderr"] = "fork failed"; + return(result); + } + if(pid == 0) + { + sigchld.restore(); + setpgid(0, 0); + pid_t expected_parent = getppid(); + prctl(PR_SET_PDEATHSIG, SIGKILL); + if(getppid() != expected_parent) + _exit(127); + dup2(inpipe[0], 0); dup2(outpipe[1], 1); dup2(errpipe[1], 2); + close(inpipe[0]); close(inpipe[1]); close(outpipe[0]); close(outpipe[1]); close(errpipe[0]); close(errpipe[1]); + for(auto& value : env) + setenv(value.first.c_str(), value.second.c_str(), 1); + execl("/bin/sh", "sh", "-c", cmd.c_str(), (char*)0); + _exit(127); + } + setpgid(pid, pid); + close(inpipe[0]); close(outpipe[1]); close(errpipe[1]); + fcntl(inpipe[1], F_SETFL, fcntl(inpipe[1], F_GETFL, 0) | O_NONBLOCK); + fcntl(outpipe[0], F_SETFL, fcntl(outpipe[0], F_GETFL, 0) | O_NONBLOCK); + fcntl(errpipe[0], F_SETFL, fcntl(errpipe[0], F_GETFL, 0) | O_NONBLOCK); + size_t input_offset = 0; + bool input_open = true, output_open = true, error_open = true, exited = false, status_valid = false; + int status = 0; + u64 now_ms = monotonic_ms(); + u64 deadline = timeout_ms > UINT64_MAX - now_ms ? UINT64_MAX : now_ms + timeout_ms; + auto append_output = [&](String key, const char* data, size_t length) { + String current = result[key].to_string(); + u64 captured = result["stdout"].to_string().size() + result["stderr"].to_string().size(); + size_t accepted = length; + if(output_limit > 0) + { + u64 available = captured < output_limit ? output_limit - captured : 0; + accepted = std::min(accepted, (size_t)available); + if(accepted < length) + result["output_truncated"].set_bool(true); + } + if(accepted > 0) + result[key] = current + String(data, accepted); + }; + while(output_open || error_open || !exited) + { + if(!exited) + { + pid_t waited = waitpid(pid, &status, WNOHANG); + if(waited == pid) + { + exited = true; + status_valid = true; + } + else if(waited < 0 && errno == ECHILD) + { + u64 transfer_deadline = monotonic_ms() + 50; + do + { + status_valid = child_exit_status_take(pid, status, child_status_snapshot_value); + if(!status_valid) + sched_yield(); + } while(!status_valid && monotonic_ms() < transfer_deadline); + exited = true; + if(!status_valid) + result["stderr"] = result["stderr"].to_string() + "lost child exit status"; + } + } + if(input_open) + { + if(input_offset < input.size()) + { + ssize_t written = write(inpipe[1], input.data() + input_offset, input.size() - input_offset); + if(written > 0) + input_offset += (size_t)written; + else if(written < 0 && errno != EINTR && errno != EAGAIN && errno != EWOULDBLOCK) + { + close(inpipe[1]); + input_open = false; + } + } + else + { + close(inpipe[1]); + input_open = false; + } + } + char buffer[4096]; + ssize_t length; + while((length = read(outpipe[0], buffer, sizeof(buffer))) > 0) + append_output("stdout", buffer, (size_t)length); + if(length == 0 && output_open) + { + close(outpipe[0]); + output_open = false; + } + while((length = read(errpipe[0], buffer, sizeof(buffer))) > 0) + append_output("stderr", buffer, (size_t)length); + if(length == 0 && error_open) + { + close(errpipe[0]); + error_open = false; + } + if((output_open || error_open || !exited) && monotonic_ms() >= deadline) + { + result["timed_out"].set_bool(true); + kill(-pid, SIGKILL); + kill(pid, SIGKILL); + if(!exited) + status_valid = waitpid(pid, &status, 0) == pid; + exited = true; + if(input_open) { close(inpipe[1]); input_open = false; } + if(output_open) { close(outpipe[0]); output_open = false; } + if(error_open) { close(errpipe[0]); error_open = false; } + } + if(output_open || error_open || !exited) + { + u64 remaining_ms = deadline > monotonic_ms() ? deadline - monotonic_ms() : 0; + if(remaining_ms > 0) + usleep((useconds_t)std::min(1000, remaining_ms * 1000)); + } + } + if(status_valid && WIFEXITED(status)) + result["exit_code"] = (f64)WEXITSTATUS(status); + else if(status_valid && WIFSIGNALED(status)) + result["exit_code"] = (f64)(128 + WTERMSIG(status)); + return(result); +} + pid_t spawn_subprocess(std::function exec_after_spawn) { parent_pid = getpid(); diff --git a/src/lib/sys.h b/src/lib/sys.h index 090bcb5..8659897 100644 --- a/src/lib/sys.h +++ b/src/lib/sys.h @@ -30,6 +30,9 @@ bool child_exit_status_take(pid_t pid, int& status, unsigned int since); struct DValue; String shell_exec(String cmd); +#if !defined(__UCE_WASM_CORE__) && !defined(__UCE_WASM_UNIT__) +DValue process_exec(String cmd, String input, StringMap env, u64 timeout_ms, u64 output_limit = 0); +#endif DValue http_request(DValue req); u64 http_request_async(DValue req); DValue shell_exec(DValue spec); diff --git a/src/linux_fastcgi.cpp b/src/linux_fastcgi.cpp index 0957ae4..6f50ccd 100644 --- a/src/linux_fastcgi.cpp +++ b/src/linux_fastcgi.cpp @@ -6,6 +6,7 @@ // Minimal FastCGI client: connection brokers forward to a clean-engine worker. #include "lib/fcgi_forward.h" #include +#include #include #include #include @@ -43,6 +44,16 @@ static volatile sig_atomic_t request_fault_frame_count = 0; void close_inherited_server_sockets(); u64 request_seed_from_time(f64 time_value); + +using WasmInvocationClock = std::chrono::steady_clock; +u64 wasm_invocation_remaining_ms(WasmInvocationClock::time_point deadline) +{ + auto now = WasmInvocationClock::now(); + if(now >= deadline) + return(0); + return((u64)std::chrono::duration_cast(deadline - now).count()); +} + void prepare_request_body_maps(Request& request); Request* set_active_request(Request& request) @@ -73,7 +84,8 @@ void clear_request_output(Request& request) request.ob_start(); } -bool render_wasm_error_page(Request& request, String config_key, s32 status_code, String status_reason, DValue error_info) +bool render_wasm_error_page(Request& request, String config_key, s32 status_code, String status_reason, DValue error_info, + WasmInvocationClock::time_point invocation_deadline, u64 invocation_budget_ms) { if(!(request.params["REQUEST_METHOD"] != "" && request.ob && request.ob_stack.size() > 0)) return(false); @@ -93,13 +105,22 @@ bool render_wasm_error_page(Request& request, String config_key, s32 status_code request.header["Content-Type"] = first(request.server->config["CONTENT_TYPE"], "text/html; charset=utf-8"); String unit = compiler_normalize_unit_path(&request, unit_file); + bool compile_timed_out = false; if(!wasm_backend_should_handle(request, unit)) - get_shared_unit(&request, unit); + { + u64 remaining_ms = wasm_invocation_remaining_ms(invocation_deadline); + if(remaining_ms == 0) + compile_timed_out = true; + else + get_shared_unit_bounded(&request, unit, remaining_ms, &compile_timed_out); + } ob_start(); String wasm_error = ""; - if(wasm_backend_should_handle(request, unit)) - wasm_error = wasm_backend_serve(request, unit, "render"); + if(compile_timed_out) + wasm_error = "UCE_INVOCATION_TIMEOUT: wasm invocation exceeded " + std::to_string(invocation_budget_ms) + " ms"; + else if(wasm_backend_should_handle(request, unit)) + wasm_error = wasm_backend_serve(request, unit, "render", wasm_invocation_remaining_ms(invocation_deadline)); else wasm_error = "error page wasm unit unavailable after compile: " + unit_file; String html = ob_get_close(); @@ -122,7 +143,8 @@ bool render_wasm_error_page(Request& request, String config_key, s32 status_code return(true); } -void render_request_failure(Request& request, String title, String details, String trace, int status_code = 500) +void render_request_failure(Request& request, String title, String details, String trace, + WasmInvocationClock::time_point invocation_deadline, u64 invocation_budget_ms, int status_code = 500) { request.response_code = request_status_line(request, status_code, "Internal Server Error"); request.header.clear(); @@ -149,7 +171,8 @@ void render_request_failure(Request& request, String title, String details, Stri error_info["signal_name"] = signal_name((int)request_fault_signal); } error_info["trace"] = trace; - if(render_wasm_error_page(request, "page_runtime_error", status_code, "Internal Server Error", error_info)) + if(render_wasm_error_page(request, "page_runtime_error", status_code, "Internal Server Error", error_info, + invocation_deadline, invocation_budget_ms)) { request.err = "UCE runtime error: " + title + (details != "" ? " (" + details + ")" : ""); restore_active_request(previous_context); @@ -388,6 +411,8 @@ int handle_cli_complete(FastCGIRequest& request) request.random_index = 0; request.random_seed = request_seed_from_time(request.stats.time_start); request.ob_start(); + u64 invocation_budget_ms = wasm_backend_invocation_timeout_ms(request); + auto invocation_deadline = WasmInvocationClock::now() + std::chrono::milliseconds(invocation_budget_ms); String method = trim(request.params["REQUEST_METHOD"]); String command = trim(first(request.params["DOCUMENT_URI"], request.params["REQUEST_URI"])); @@ -441,15 +466,26 @@ int handle_cli_complete(FastCGIRequest& request) // been removed, so a unit that still cannot be served by wasm is a // request failure instead of a fallback path. SharedUnit* cli_compile_state = 0; + bool cli_compile_timed_out = false; bool cli_wasm_ready = wasm_backend_should_handle(request, cli_unit); if(!cli_wasm_ready) { - cli_compile_state = get_shared_unit(&request, cli_unit); - cli_wasm_ready = wasm_backend_should_handle(request, cli_unit); + u64 remaining_ms = wasm_invocation_remaining_ms(invocation_deadline); + if(remaining_ms == 0) + cli_compile_timed_out = true; + else + cli_compile_state = get_shared_unit_bounded(&request, cli_unit, remaining_ms, &cli_compile_timed_out); + if(!cli_compile_timed_out) + cli_wasm_ready = wasm_backend_should_handle(request, cli_unit); } - if(cli_wasm_ready) + if(cli_compile_timed_out) { - String wasm_error = wasm_backend_serve(request, cli_unit, "cli"); + request.set_status(500, "Internal Server Error"); + print("UCE_INVOCATION_TIMEOUT: wasm invocation exceeded ", invocation_budget_ms, " ms\n"); + } + else if(cli_wasm_ready) + { + String wasm_error = wasm_backend_serve(request, cli_unit, "cli", wasm_invocation_remaining_ms(invocation_deadline)); if(wasm_error != "") { request.set_status(500, "Internal Server Error"); @@ -476,11 +512,13 @@ int handle_cli_complete(FastCGIRequest& request) } catch(const std::exception& e) { - render_request_failure(request, "uncaught exception during CLI request", e.what(), backtrace_capture(32, 1), 500); + render_request_failure(request, "uncaught exception during CLI request", e.what(), backtrace_capture(32, 1), + invocation_deadline, invocation_budget_ms, 500); } catch(...) { - render_request_failure(request, "unknown uncaught exception during CLI request", "", backtrace_capture(32, 1), 500); + render_request_failure(request, "unknown uncaught exception during CLI request", "", backtrace_capture(32, 1), + invocation_deadline, invocation_budget_ms, 500); } for(auto &f : request.uploaded_files) @@ -522,6 +560,8 @@ int handle_complete(FastCGIRequest& request) { request.random_index = 0; request.random_seed = request_seed_from_time(request.stats.time_start); request.ob_start(); + u64 invocation_budget_ms = wasm_backend_invocation_timeout_ms(request); + auto invocation_deadline = WasmInvocationClock::now() + std::chrono::milliseconds(invocation_budget_ms); request_fault_request = &request; request_fault_active = 1; request_fault_signal = 0; @@ -557,7 +597,7 @@ int handle_complete(FastCGIRequest& request) { request_fault_active = 0; restore_request_fault_handlers(); request.stats.wasm_backend_started = time_precise(); - String wasm_error = wasm_backend_serve(request, entry_unit, handler); + String wasm_error = wasm_backend_serve(request, entry_unit, handler, wasm_invocation_remaining_ms(invocation_deadline)); request.stats.wasm_backend_finished = time_precise(); install_request_fault_handlers(); request_fault_active = 1; @@ -605,13 +645,25 @@ int handle_complete(FastCGIRequest& request) { // recheck. Native execution has been removed, so a unit that still cannot // be served by wasm becomes a clean 500 request failure. SharedUnit* entry_compile_state = 0; + bool entry_compile_timed_out = false; auto wasm_ready = [&](const String& unit) -> bool { if(wasm_backend_should_handle(request, unit)) return(true); - entry_compile_state = get_shared_unit(&request, unit); - return(wasm_backend_should_handle(request, unit)); + u64 remaining_ms = wasm_invocation_remaining_ms(invocation_deadline); + if(remaining_ms == 0) + entry_compile_timed_out = true; + else + entry_compile_state = get_shared_unit_bounded(&request, unit, remaining_ms, &entry_compile_timed_out); + return(!entry_compile_timed_out && wasm_backend_should_handle(request, unit)); }; auto fail_wasm_unavailable = [&](const String& handler) { + if(entry_compile_timed_out) + { + failure_title = "wasm invocation timeout"; + failure_details = ""; + failure_trace = "UCE_INVOCATION_TIMEOUT: wasm invocation exceeded " + std::to_string(invocation_budget_ms) + " ms"; + return; + } failure_title = "wasm unit unavailable after compile"; String compile_error = entry_compile_state ? first(entry_compile_state->compiler_messages, entry_compile_state->compile_error_status) : String(""); failure_details = compile_error != "" ? compile_error : handler + " handler could not be served by wasm"; @@ -691,7 +743,8 @@ int handle_complete(FastCGIRequest& request) { restore_request_fault_handlers(); if(failure_title != "") - render_request_failure(request, failure_title, failure_details, failure_trace, 500); + render_request_failure(request, failure_title, failure_details, failure_trace, + invocation_deadline, invocation_budget_ms, 500); for( auto &f : request.uploaded_files) { diff --git a/src/wasm/backend.cpp b/src/wasm/backend.cpp index eff0287..8000efd 100644 --- a/src/wasm/backend.cpp +++ b/src/wasm/backend.cpp @@ -178,15 +178,25 @@ bool wasm_backend_should_handle(Request& request, const String& entry_unit) return(true); } +u64 wasm_backend_invocation_timeout_ms(Request& request) +{ + if(!request.server) + return(0); + u64 timeout_ms = to_u64(first(request.server->config["WASM_INVOCATION_TIMEOUT_MS"], "30000"), 0); + return(timeout_ms > 0 && timeout_ms <= 86400000 ? timeout_ms : 0); +} + // Serve a request through a wasm workspace using the unit handler selected by // `kind` (page render / cli / serve_http, with an optional named serve_http // handler). Populates the native Request (status/headers/cookies/session/body) // so the existing transport writes the response unchanged. Returns "" on // success, or a collapsed error/trace string for the caller to route into the // configured error page. -String wasm_backend_serve(Request& request, const String& entry_unit, const String& handler = "render") +String wasm_backend_serve(Request& request, const String& entry_unit, const String& handler, u64 timeout_cap_ms) { - WasmResponse response = wasm_worker_serve(*g_wasm_worker, request, entry_unit, handler); + if(timeout_cap_ms == 0) + return("UCE_INVOCATION_TIMEOUT: wasm invocation exceeded " + std::to_string(wasm_backend_invocation_timeout_ms(request)) + " ms"); + WasmResponse response = wasm_worker_serve(*g_wasm_worker, request, entry_unit, handler, timeout_cap_ms); request.stats.wasm_dispatch_us = response.dispatch_us; request.stats.wasm_workspace_complete_us = response.workspace_complete_us; request.stats.wasm_entry_invoke_us = response.entry_invoke_us; diff --git a/src/wasm/backend.h b/src/wasm/backend.h index de20767..e03d4f9 100644 --- a/src/wasm/backend.h +++ b/src/wasm/backend.h @@ -9,6 +9,7 @@ // True if this request can be served by the wasm backend for the named unit // artifact; handler-agnostic, the caller names the handler. bool wasm_backend_should_handle(Request& request, const String& entry_unit); +u64 wasm_backend_invocation_timeout_ms(Request& request); // Initialize the process-local engine after fork and before the worker accepts // requests. Returns "" on success or the retained backend initialization error. @@ -23,7 +24,7 @@ String wasm_serialize_module_artifact(const String& wasm_path); // "render", "cli", "websocket", "serve_http", "serve_http:named" — and populate // the native Request. Returns "" on success or a collapsed error. The handler is // just an export name; there is no per-mode machinery. -String wasm_backend_serve(Request& request, const String& entry_unit, const String& handler = "render"); +String wasm_backend_serve(Request& request, const String& entry_unit, const String& handler = "render", u64 timeout_cap_ms = UINT64_MAX); // Join the per-process epoch ticker before the worker process exits. void wasm_backend_shutdown(); diff --git a/src/wasm/worker.cpp b/src/wasm/worker.cpp index a6e0f58..2ea30e8 100644 --- a/src/wasm/worker.cpp +++ b/src/wasm/worker.cpp @@ -542,90 +542,9 @@ static void uce_job_reap() } } -static DValue uce_process_exec(String cmd, String input, StringMap env, u64 timeout_ms) -{ - DValue r; - r["exit_code"] = (f64)-1; - r["stdout"] = ""; - r["stderr"] = ""; - r["timed_out"].set_bool(false); - if(timeout_ms == 0) timeout_ms = 5000; - int inpipe[2], outpipe[2], errpipe[2]; - if(pipe(inpipe) || pipe(outpipe) || pipe(errpipe)) { r["stderr"]="pipe failed"; return(r); } - // The process-wide handler reaps background children; this caller owns this child's status. - WasmSigchldBlock sigchld; - unsigned int child_status_snapshot = child_exit_status_snapshot(); - pid_t pid = fork(); - if(pid < 0) - { - close(inpipe[0]); close(inpipe[1]); close(outpipe[0]); close(outpipe[1]); close(errpipe[0]); close(errpipe[1]); - r["stderr"] = "fork failed"; - return(r); - } - if(pid == 0) - { - sigchld.restore(); - setpgid(0, 0); - dup2(inpipe[0], 0); dup2(outpipe[1], 1); dup2(errpipe[1], 2); - close(inpipe[0]); close(inpipe[1]); close(outpipe[0]); close(outpipe[1]); close(errpipe[0]); close(errpipe[1]); - for(auto& kv : env) setenv(kv.first.c_str(), kv.second.c_str(), 1); - execl("/bin/sh", "sh", "-c", cmd.c_str(), (char*)0); - _exit(127); - } - setpgid(pid, pid); - close(inpipe[0]); close(outpipe[1]); close(errpipe[1]); - fcntl(inpipe[1], F_SETFL, fcntl(inpipe[1], F_GETFL, 0) | O_NONBLOCK); - fcntl(outpipe[0], F_SETFL, fcntl(outpipe[0], F_GETFL, 0) | O_NONBLOCK); - fcntl(errpipe[0], F_SETFL, fcntl(errpipe[0], F_GETFL, 0) | O_NONBLOCK); - size_t input_off = 0; bool in_open = true, out_open = true, err_open = true; int status = 0; bool exited = false, status_valid = false; - u64 deadline = wasm_monotonic_ms() + timeout_ms; - while(out_open || err_open || !exited) - { - if(!exited) - { - pid_t w = waitpid(pid, &status, WNOHANG); - if(w == pid) { exited = true; status_valid = true; } - else if(w < 0 && errno == ECHILD) - { - u64 transfer_deadline = wasm_deadline_after_ms(50); - do { status_valid = child_exit_status_take(pid, status, child_status_snapshot); if(!status_valid) sched_yield(); } while(!status_valid && wasm_monotonic_ms() < transfer_deadline); - exited = true; - if(!status_valid) r["stderr"] = r["stderr"].to_string() + "lost child exit status"; - } - } - if(in_open) - { - if(input_off < input.size()) { ssize_t n=write(inpipe[1], input.data()+input_off, input.size()-input_off); if(n>0) input_off += (size_t)n; else if(n<0 && errno!=EINTR && errno!=EAGAIN && errno!=EWOULDBLOCK) { close(inpipe[1]); in_open=false; } } - else { close(inpipe[1]); in_open=false; } - } - char buf[4096]; - ssize_t n; - while((n=read(outpipe[0], buf, sizeof(buf))) > 0) r["stdout"] = r["stdout"].to_string() + String(buf, n); - if(n == 0 && out_open) { close(outpipe[0]); out_open=false; } - while((n=read(errpipe[0], buf, sizeof(buf))) > 0) r["stderr"] = r["stderr"].to_string() + String(buf, n); - if(n == 0 && err_open) { close(errpipe[0]); err_open=false; } - if((out_open || err_open || !exited) && wasm_monotonic_ms() >= deadline) - { - r["timed_out"].set_bool(true); - kill(-pid, SIGKILL); - kill(pid, SIGKILL); - if(!exited) - status_valid = waitpid(pid, &status, 0) == pid; - exited = true; - if(in_open) { close(inpipe[1]); in_open=false; } - if(out_open) { close(outpipe[0]); out_open=false; } - if(err_open) { close(errpipe[0]); err_open=false; } - } - if((out_open || err_open || !exited)) usleep(10000); - } - if(status_valid && WIFEXITED(status)) r["exit_code"] = (f64)WEXITSTATUS(status); - else if(status_valid && WIFSIGNALED(status)) r["exit_code"] = (f64)(128 + WTERMSIG(status)); - return(r); -} - static DValue uce_shell_exec_spec(const DValue& spec) { - return(uce_process_exec(spec.key("cmd") ? spec.key("cmd")->to_string() : String(""), spec.key("stdin") ? spec.key("stdin")->to_string() : String(""), spec.key("env") ? spec.key("env")->to_stringmap() : StringMap(), spec.key("timeout_ms") ? spec.key("timeout_ms")->to_u64(5000) : 5000)); + return(process_exec(spec.key("cmd") ? spec.key("cmd")->to_string() : String(""), spec.key("stdin") ? spec.key("stdin")->to_string() : String(""), spec.key("env") ? spec.key("env")->to_stringmap() : StringMap(), spec.key("timeout_ms") ? spec.key("timeout_ms")->to_u64(5000) : 5000)); } static void uce_job_finish(u64 id, DValue result, String final_state="done") @@ -1606,8 +1525,7 @@ public: return(UINT64_MAX); if(now >= invocation_deadline) return(0); - u64 remaining_us = (u64)std::chrono::duration_cast(invocation_deadline - now).count(); - return(remaining_us / 1000 + (remaining_us % 1000 != 0)); + return((u64)std::chrono::duration_cast(invocation_deadline - now).count()); } bool invocation_expired(InvocationClock::time_point now = InvocationClock::now()) const @@ -1659,7 +1577,7 @@ public: InvocationClock::time_point previous_deadline; u64 previous_budget_ms = 0; public: - InvocationScope(WasmWorkspace& workspace, u64 timeout_cap_ms = 0, bool force_new = false) : workspace(workspace) + InvocationScope(WasmWorkspace& workspace, u64 timeout_cap_ms = 0, bool force_new = false, u64 reported_budget_ms = 0) : workspace(workspace) { if(!workspace.invocation_active || force_new) { @@ -1668,10 +1586,10 @@ public: previous_deadline = workspace.invocation_deadline; previous_budget_ms = workspace.invocation_budget_ms; u64 budget_ms = workspace.worker.cfg.invocation_timeout_ms; - if(timeout_cap_ms > 0) + if(timeout_cap_ms != UINT64_MAX) budget_ms = std::min(budget_ms, timeout_cap_ms); workspace.invocation_active = true; - workspace.invocation_budget_ms = budget_ms; + workspace.invocation_budget_ms = reported_budget_ms > 0 ? reported_budget_ms : budget_ms; workspace.invocation_deadline = InvocationClock::now() + std::chrono::milliseconds(budget_ms); } workspace.arm_guest_deadline(workspace.ctx()); @@ -1786,6 +1704,8 @@ public: phase_start = now; return(elapsed); }; + if(invocation_expired()) + return(invocation_timeout_error()); auto cx = ctx(); store.limiter(worker.cfg.memory_limit, -1, -1, -1, -1); arm_guest_deadline(cx); @@ -2096,7 +2016,9 @@ private: String trap_text(const wasmtime::TrapError& error) { String result = wasm_trace_collapse(String(error.message())); - if(invocation_expired() && result.find("interrupt") != String::npos && result.find("UCE_INVOCATION_TIMEOUT:") == String::npos) + bool invocation_deadline_interrupt = invocation_active && + (invocation_expired() || invocation_remaining_ms() <= worker.cfg.epoch_period_ms); + if(invocation_deadline_interrupt && result.find("interrupt") != String::npos && result.find("UCE_INVOCATION_TIMEOUT:") == String::npos) result = invocation_timeout_error() + "\n" + result; struct Frame { @@ -2795,10 +2717,12 @@ private: // `handler` names the export ("render", "component:CARD", "cli", // "serve_http:named", "once") or is "exists" (probe only, loads nothing). int32_t component_resolve(const String& target, const String& handler, const String& current_unit, - String& resolved_out, int32_t* once_slot_out = 0) + String& resolved_out, int32_t* once_slot_out = 0, bool* compile_timed_out = 0) { if(once_slot_out) *once_slot_out = 0; + if(compile_timed_out) + *compile_timed_out = false; auto probe_start = std::chrono::steady_clock::now(); auto record_probe = [&]() { component_resolve_count += 1; @@ -2925,7 +2849,21 @@ private: } } if(!artifact_exists || (stale && !can_serve_stale)) - get_shared_unit(context, resolved); + { + u64 remaining_ms = invocation_remaining_ms(); + bool timed_out = false; + if(remaining_ms == 0) + timed_out = true; + else + get_shared_unit_bounded(context, resolved, remaining_ms, &timed_out); + if(timed_out) + { + if(compile_timed_out) + *compile_timed_out = true; + record_probe(); + return(0); + } + } component_artifact_total_us += (u64)std::chrono::duration_cast( std::chrono::steady_clock::now() - artifact_start).count(); @@ -3342,7 +3280,7 @@ private: { u64 remaining_ms = self->invocation_remaining_ms(); u64 timeout_ms = std::max(1, self->bounded_hostcall_timeout_ms(5000)); - DValue execution = uce_process_exec(cmd + " 2>&1", "", StringMap(), timeout_ms); + DValue execution = process_exec(cmd + " 2>&1", "", StringMap(), timeout_ms); if(execution["timed_out"].to_bool()) { String kind = self->invocation_expired() || remaining_ms <= 5000 ? @@ -3850,7 +3788,14 @@ private: } } else if(op == "compile") - response["ok"].set_bool(unit_compile(request["path"].to_string())); + { + u64 remaining_ms = self->invocation_remaining_ms(); + bool timed_out = false; + bool ok = remaining_ms > 0 && unit_compile_bounded(context, request["path"].to_string(), remaining_ms, &timed_out); + if(timed_out || remaining_ms == 0) + return(Trap(self->invocation_timeout_error())); + response["ok"].set_bool(ok); + } else if(op == "call") { DValue* param = request.key("param"); @@ -4292,7 +4237,10 @@ private: self->hostcall_read(args[2].i32(), args[3].i32(), handler); self->hostcall_read(args[4].i32(), args[5].i32(), current); int32_t once_slot = 0; - int32_t slot = self->component_resolve(target, handler, current, resolved, &once_slot); + bool compile_timed_out = false; + int32_t slot = self->component_resolve(target, handler, current, resolved, &once_slot, &compile_timed_out); + if(compile_timed_out) + return(Trap(self->invocation_timeout_error())); u32 cap = (u32)args[7].i32(); if(cap > 0) { @@ -4335,7 +4283,7 @@ inline String wasm_worker_prepare(WasmWorker& worker) // ---- public entry: one request through one workspace ----------------------- inline WasmResponse wasm_worker_serve(WasmWorker& worker, const Request& request, const String& entry_source_path, - const String& handler = "render") + const String& handler = "render", u64 timeout_cap_ms = UINT64_MAX) { WasmResponse response; f64 serve_started = time_precise(); @@ -4345,9 +4293,15 @@ inline WasmResponse wasm_worker_serve(WasmWorker& worker, const Request& request bool thread_runtime_profiled = worker.cfg.profile_thread_runtime && getrusage(RUSAGE_THREAD, &thread_runtime_start) == 0; int thread_cpu_start = thread_runtime_profiled ? sched_getcpu() : -1; WasmWorkspace workspace(worker); - f64 setup_cpu_finished = wasm_thread_cpu_time(); - workspace.workspace_setup_us = (u64)std::chrono::duration_cast( + u64 workspace_setup_us = (u64)std::chrono::duration_cast( std::chrono::steady_clock::now() - workspace_start).count(); + u64 workspace_setup_ms = workspace_setup_us / 1000 + (workspace_setup_us % 1000 != 0); + u64 workspace_budget_ms = timeout_cap_ms; + if(workspace_budget_ms != UINT64_MAX) + workspace_budget_ms = workspace_budget_ms > workspace_setup_ms ? workspace_budget_ms - workspace_setup_ms : 0; + WasmWorkspace::InvocationScope invocation(workspace, workspace_budget_ms, false, worker.cfg.invocation_timeout_ms); + f64 setup_cpu_finished = wasm_thread_cpu_time(); + workspace.workspace_setup_us = workspace_setup_us; workspace.workspace_setup_cpu_us = cpu_started > 0 && setup_cpu_finished > cpu_started ? (u64)((setup_cpu_finished - cpu_started) * 1000000.0) : 0; workspace.set_perf_snapshot(my_pid, (u64)parent_pid, request.server ? request.server->request_count : 0,