diff --git a/docs/setup.md b/docs/setup.md index 0833b9a..a4bdf59 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -252,6 +252,12 @@ scripts/systemd/manage-uce-service.sh restart scripts/systemd/manage-uce-service.sh logs 200 ``` +Managed restart builds and precompiles the complete candidate ABI generation +before it switches the service. Precompile uses two low-priority processes by +default; set `PRECOMPILE_JOBS` in `/etc/uce/settings.cfg` to tune the bounded +1–16 process count for the host. A failed worker, compile, serialization, or +result report aborts the switch and leaves the current service running. + Equivalent manual systemd service for a source checkout (`` = checkout root): ```ini diff --git a/docs/wasm-runtime-architecture.md b/docs/wasm-runtime-architecture.md index 833a7ee..5907ba4 100644 --- a/docs/wasm-runtime-architecture.md +++ b/docs/wasm-runtime-architecture.md @@ -238,7 +238,12 @@ is the runtime core ABI. Old and new service binaries therefore never publish or read the same unit path during an ABI transition. The managed restart builds and serializes the complete next generation before stopping the old service; failure aborts the switch and leaves the running generation intact. Old -generation directories are retained as an explicit rollback path. +generation directories are retained as an explicit rollback path. Two +low-priority precompile processes are used by default so independent units do +not serialize deployment time while live workers retain scheduler priority. +`PRECOMPILE_JOBS` or `UCE_PRECOMPILE_JOBS` may select 1–16 processes. Unit +publication and the shared generation PCH use separate advisory locks, and any +worker/reporting failure rejects the candidate generation. The proactive compiler and request workers coordinate through per-unit file locks and a lock-protected demand-priority queue under `BIN_DIRECTORY`. A small diff --git a/etc/uce/settings.cfg b/etc/uce/settings.cfg index b6121da..51401e6 100644 --- a/etc/uce/settings.cfg +++ b/etc/uce/settings.cfg @@ -27,6 +27,10 @@ WS_BROKER_OUTBOUND_TIMEOUT_SECONDS=30 # Leave empty to scan SITE_DIRECTORY relative to the runtime root. PRECOMPILE_FILES_IN= +# NUMBER OF LOW-PRIORITY PROCESSES USED BY MANAGED GENERATION PRECOMPILE +# Two keeps one half of a typical four-vCPU host available to live workers. +PRECOMPILE_JOBS=2 + # PUBLIC SITE DIRECTORY USED FOR STARTUP SCAN WHEN PRECOMPILE_FILES_IN IS EMPTY SITE_DIRECTORY=site diff --git a/scripts/compile_wasm_unit b/scripts/compile_wasm_unit index e0a1820..867449b 100755 --- a/scripts/compile_wasm_unit +++ b/scripts/compile_wasm_unit @@ -46,22 +46,29 @@ HEADER_HASH=$(find src/lib -maxdepth 1 -name '*.h' -type f -print0 | sort -z | x FLAGS_HASH=$(printf '%s\0' "${COMMON_FLAGS[@]}" -Isrc/lib | sha1sum | cut -c1-16) PCH_KEY=$(printf '%s\n%s\n%s\n%s\n' "$ABI_VERSION" "$TOOLCHAIN_ID" "$HEADER_HASH" "$FLAGS_HASH" | sha1sum | cut -c1-16) PCH_FN="$PCH_DIR/uce_lib-wasm-unit-$PCH_KEY.pch" +PCH_TMP="$PCH_FN.tmp.$$" mkdir -p "$DEST_DIR" >/dev/null 2>&1 -trap 'rm -f "$OBJ_FN" "$ABI_TMP" "$MODULE_TMP" "$WASM_TMP" "$MAP_TMP"' EXIT +trap 'rm -f "$OBJ_FN" "$ABI_TMP" "$MODULE_TMP" "$WASM_TMP" "$MAP_TMP" "$PCH_TMP"' EXIT build_pch_if_needed() { if [ "$PCH_ENABLED" = "0" ]; then return 0 fi mkdir -p "$PCH_DIR" + exec 9>"$PCH_FN.lock" + flock 9 if [ -s "$PCH_FN" ] && [ -z "$(find src/lib -maxdepth 1 -name '*.h' -type f -newer "$PCH_FN" -print -quit)" ]; then + flock -u 9 + exec 9>&- return 0 fi "$SDK/bin/clang++" "${COMMON_FLAGS[@]}" \ -Isrc/lib \ - -x c++-header src/lib/uce_lib.h -o "$PCH_FN.tmp" - mv "$PCH_FN.tmp" "$PCH_FN" + -x c++-header src/lib/uce_lib.h -o "$PCH_TMP" + mv "$PCH_TMP" "$PCH_FN" + flock -u 9 + exec 9>&- } cat > "$ABI_TMP" </dev/null 2>&1 || true + fi + if [[ -n "$artifact_dir" ]]; then + rm -rf "$artifact_dir" + fi +} +trap cleanup EXIT +mkdir -p "$source_dir" +absolute_source_dir=$(realpath "$source_dir") +artifact_dir="$(scripts/unit_cache_directory "$bin_directory")$absolute_source_dir" + +write_units() { + local version="$1" + for unit in 0 1 2 3; do + printf 'CLI(Request& context) { print("parallel-precompile-%s-%s"); }\n' "$version" "$unit" >"$source_dir/unit-$unit.uce" + done +} + +printf 'CLI(Request& context) { print("parallel-precompile-warmup"); }\n' >"$source_dir/warmup.uce" +UCE_PRECOMPILE_JOBS=1 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile >/dev/null +rm "$source_dir/warmup.uce" + +write_units serial +serial_start=$(date +%s%N) +serial_output=$(UCE_PRECOMPILE_JOBS=1 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile) +serial_ns=$(( $(date +%s%N) - serial_start )) +if ! grep -Eq 'with 1 job: .* 4 compiled, 0 failed' <<<"$serial_output"; then + echo "serial precompile did not compile the four controlled units" >&2 + echo "$serial_output" >&2 + exit 1 +fi + +write_units parallel +parallel_start=$(date +%s%N) +parallel_output=$(UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile) +parallel_ns=$(( $(date +%s%N) - parallel_start )) +if [[ $(grep -Ec '^Precompile worker [12]/2:' <<<"$parallel_output") -ne 2 ]]; then + echo "parallel precompile did not report both workers" >&2 + echo "$parallel_output" >&2 + exit 1 +fi +if ! grep -Eq 'with 2 jobs: .* 4 compiled, 0 failed' <<<"$parallel_output"; then + echo "parallel precompile did not compile the four controlled units" >&2 + echo "$parallel_output" >&2 + exit 1 +fi +for unit in 0 1 2 3; do + if [[ ! -s "$artifact_dir/unit-$unit.uce.wasm" || ! -s "$artifact_dir/unit-$unit.uce.cwasm" ]]; then + echo "parallel precompile did not publish wasm and serialized artifacts for unit $unit" >&2 + exit 1 + fi +done + +printf 'CLI(Request& context) { print("parallel-precompile-race-0"); }\n' >"$source_dir/race-0.uce" +printf 'CLI(Request& context) { print("parallel-precompile-race-1"); }\n' >"$source_dir/race-1.uce" +race_output=$(UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-race" bin/uce_fastcgi.linux.bin --precompile) +if ! grep -Eq 'with 2 jobs: .* 2 compiled, 0 failed' <<<"$race_output" || \ + [[ $(find "$artifact_dir/pch-race" -maxdepth 1 -type f -name '*.pch' | wc -l) -ne 1 ]] || \ + find "$artifact_dir/pch-race" -maxdepth 1 -type f -name '*.tmp.*' -print -quit | grep -q .; then + echo "parallel precompile did not publish exactly one clean shared PCH" >&2 + echo "$race_output" >&2 + exit 1 +fi +rm "$source_dir/race-0.uce" "$source_dir/race-1.uce" + +printf 'CLI(Request& context) { print(deliberate_parallel_precompile_failure); }\n' >"$source_dir/broken.uce" +set +e +failure_output=$(UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile 2>&1) +failure_rc=$? +set -e +if [[ $failure_rc -eq 0 ]] || ! grep -Eq 'with 2 jobs: .* 1 failed' <<<"$failure_output"; then + echo "parallel precompile did not aggregate a controlled worker failure" >&2 + echo "$failure_output" >&2 + exit 1 +fi +rm "$source_dir/broken.uce" + +printf 'parallel precompile passed: serial %.3fs, parallel %.3fs\n' \ + "$(awk -v ns="$serial_ns" 'BEGIN { print ns / 1000000000 }')" \ + "$(awk -v ns="$parallel_ns" 'BEGIN { print ns / 1000000000 }')" diff --git a/src/lib/sys.cpp b/src/lib/sys.cpp index 1c2ed3b..eb1c5b7 100644 --- a/src/lib/sys.cpp +++ b/src/lib/sys.cpp @@ -1700,6 +1700,7 @@ StringMap make_server_settings() cfg["SESSION_COOKIE_SECURE"] = "0"; cfg["COMPILER_SYS_PATH"] = "."; cfg["PRECOMPILE_FILES_IN"] = ""; + cfg["PRECOMPILE_JOBS"] = "2"; cfg["SITE_DIRECTORY"] = "site"; cfg["JIT_COMPILE_ON_REQUEST"] = "1"; cfg["PROACTIVE_COMPILE_ENABLED"] = "1"; diff --git a/src/linux_fastcgi.cpp b/src/linux_fastcgi.cpp index e98fadd..0c5e3bc 100644 --- a/src/linux_fastcgi.cpp +++ b/src/linux_fastcgi.cpp @@ -9,6 +9,7 @@ #include #include #include +#include ServerState server_state; @@ -1582,6 +1583,62 @@ void init_base_process() srand(time()); } +struct PrecompileWorkerResult +{ + u64 assigned = 0; + u64 compiled = 0; + u64 failed = 0; +}; + +PrecompileWorkerResult precompile_unit_range(Request& background_context, const StringList& files, u64 worker, u64 jobs) +{ + PrecompileWorkerResult result; + for(u64 i = worker; i < files.size(); i += jobs) + { + result.assigned++; + bool source_missing = false; + if(compiler_unit_needs_recompile(&background_context, files[i], &source_missing)) + result.compiled++; + if(proactive_compile_unit(background_context, files[i], source_missing)) + result.failed++; + } + return(result); +} + +bool precompile_write_result(int fd, const PrecompileWorkerResult& result) +{ + const char* data = (const char*)&result; + size_t remaining = sizeof(result); + while(remaining > 0) + { + ssize_t written = write(fd, data, remaining); + if(written < 0 && errno == EINTR) + continue; + if(written <= 0) + return(false); + data += written; + remaining -= (size_t)written; + } + return(true); +} + +bool precompile_read_result(int fd, PrecompileWorkerResult& result) +{ + char* data = (char*)&result; + size_t remaining = sizeof(result); + while(remaining > 0) + { + ssize_t received = read(fd, data, remaining); + if(received < 0 && errno == EINTR) + continue; + if(received <= 0) + return(false); + data += received; + remaining -= (size_t)received; + } + return(true); +} + int precompile_unit_generation() { f64 started_at = time_precise(); @@ -1595,21 +1652,77 @@ int precompile_unit_generation() set_active_request(background_context); auto files = compiler_scan_site_units(&background_context); compiler_set_known_units(&background_context, files); - u64 failed = 0; - u64 compiled = 0; - for(auto& file_name : files) + const char* jobs_env = getenv("UCE_PRECOMPILE_JOBS"); + u64 jobs = to_u64(jobs_env && jobs_env[0] != '\0' ? String(jobs_env) : server_state.config["PRECOMPILE_JOBS"], 2); + jobs = std::max(1, std::min(jobs, std::min(files.size() == 0 ? 1 : files.size(), 16))); + PrecompileWorkerResult total; + bool worker_error = false; + if(jobs == 1) + total = precompile_unit_range(background_context, files, 0, 1); + else { - bool source_missing = false; - if(compiler_unit_needs_recompile(&background_context, file_name, &source_missing)) - compiled++; - if(proactive_compile_unit(background_context, file_name, source_missing)) - failed++; + int result_pipe[2]; + if(pipe(result_pipe) != 0) + { + printf("(!) cannot create precompile result pipe: %s\n", std::strerror(errno)); + return(1); + } + std::vector workers; + fflush(0); + for(u64 worker = 0; worker < jobs; worker++) + { + pid_t pid = fork(); + if(pid < 0) + { + printf("(!) cannot fork precompile worker: %s\n", std::strerror(errno)); + worker_error = true; + break; + } + if(pid == 0) + { + close(result_pipe[0]); + PrecompileWorkerResult result = precompile_unit_range(background_context, files, worker, jobs); + printf("Precompile worker %llu/%llu: %llu units, %llu compiled, %llu failed\n", + (unsigned long long)(worker + 1), (unsigned long long)jobs, + (unsigned long long)result.assigned, (unsigned long long)result.compiled, + (unsigned long long)result.failed); + bool reported = precompile_write_result(result_pipe[1], result); + close(result_pipe[1]); + _exit(result.failed == 0 && reported ? 0 : 1); + } + workers.push_back(pid); + } + close(result_pipe[1]); + for(size_t i = 0; i < workers.size(); i++) + { + PrecompileWorkerResult result; + if(!precompile_read_result(result_pipe[0], result)) + { + worker_error = true; + break; + } + total.assigned += result.assigned; + total.compiled += result.compiled; + total.failed += result.failed; + } + close(result_pipe[0]); + for(auto pid : workers) + { + int status = 0; + pid_t waited; + do waited = waitpid(pid, &status, 0); while(waited < 0 && errno == EINTR); + if(waited != pid || !WIFEXITED(status) || WEXITSTATUS(status) != 0) + worker_error = true; + } + if(total.assigned != files.size()) + worker_error = true; } - printf("Precompiled unit generation %s: %zu units, %llu compiled, %llu failed in %.3f s\n", - compiler_unit_bin_directory(&background_context).c_str(), files.size(), - (unsigned long long)compiled, (unsigned long long)failed, + printf("Precompiled unit generation %s with %llu job%s: %zu units, %llu compiled, %llu failed in %.3f s\n", + compiler_unit_bin_directory(&background_context).c_str(), + (unsigned long long)jobs, jobs == 1 ? "" : "s", files.size(), + (unsigned long long)total.compiled, (unsigned long long)total.failed, time_precise() - started_at); - return(failed == 0 ? 0 : 1); + return(total.failed == 0 && !worker_error ? 0 : 1); } int main(int argc, char** argv)