From 33a7b00c78a0ca1a0e04b65228caa1d62480dbab Mon Sep 17 00:00:00 2001 From: udo Date: Sat, 18 Jul 2026 20:19:37 +0000 Subject: [PATCH] Parallelize proactive unit compilation --- docs/setup.md | 2 + docs/wasm-runtime-architecture.md | 18 ++- etc/uce/settings.cfg | 4 + scripts/run_cli_tests.sh | 1 + scripts/test_parallel_proactive_compile.sh | 168 ++++++++++++++++++++ src/lib/sys.cpp | 1 + src/linux_fastcgi.cpp | 173 ++++++++++++++------- 7 files changed, 303 insertions(+), 64 deletions(-) create mode 100755 scripts/test_parallel_proactive_compile.sh diff --git a/docs/setup.md b/docs/setup.md index 03a991a..55c9ee2 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -190,6 +190,7 @@ SITE_DIRECTORY=/var/www/html HTTP_DOCUMENT_ROOT=/var/www/html JIT_COMPILE_ON_REQUEST=1 PROACTIVE_COMPILE_ENABLED=1 +PROACTIVE_COMPILE_JOBS=2 PROACTIVE_COMPILE_CHECK_INTERVAL=60 WASM_COMPILE_SCRIPT=scripts/compile_wasm_unit @@ -224,6 +225,7 @@ Important settings: - `HTTP_PORT` is the built-in HTTP/WebSocket listener used for WebSocket upgrade traffic and direct local probes. Bind/firewall it for local access only; nginx/Apache should be the public entry point. - `WS_BROKER_OUTBOUND_TIMEOUT_SECONDS` controls how long a forwarded WS message can remain queued in the broker before being dropped (default `30`). Set to `0` to disable the timeout. - `WASM_COMPILE_SCRIPT` must point to `scripts/compile_wasm_unit` unless you provide an equivalent compiler. Relative paths are resolved from the runtime root/`COMPILER_SYS_PATH`. That script calls `scripts/check_unit_wasm.py` after linking each unit and uses the pinned WASI SDK on every deployment host. +- `PROACTIVE_COMPILE_JOBS` selects 1–16 low-priority full-site scanner processes (default `2`). Each canonical unit path has one scanner owner. The separate higher-priority demand compiler remains reserved for stale units requested over HTTP, so total background compile concurrency can reach this value plus one. - `WASM_CORE_PATH` must point at the built `core.wasm` file. After editing settings, restart UCE: diff --git a/docs/wasm-runtime-architecture.md b/docs/wasm-runtime-architecture.md index 5907ba4..a1d269c 100644 --- a/docs/wasm-runtime-architecture.md +++ b/docs/wasm-runtime-architecture.md @@ -246,14 +246,16 @@ publication and the shared generation PCH use separate advisory locks, and any worker/reporting failure rejects the candidate generation. The proactive compiler and request workers coordinate through per-unit file -locks and a lock-protected demand-priority queue under `BIN_DIRECTORY`. A small -priority-only compiler process drains that queue independently of the full-site -scanner. This matters when the scanner is already inside a long transitive C++ -compile: a requested stale component can rebuild immediately instead of waiting -for that unrelated compile to finish. Both compiler processes use the same -per-unit lock, so concurrent demand and scan discovery cannot publish duplicate -artifacts. The priority worker is idle when there is no demand and never scans -the site on its own. Unit +locks and a lock-protected demand-priority queue under `BIN_DIRECTORY`. The +full-site scan uses two low-priority processes by default, bounded from 1–16 by +`PROACTIVE_COMPILE_JOBS`; a stable canonical-path partition gives every unit one +scanner owner and keeps retry/backoff state local. A separate higher-priority +compiler process exclusively drains the demand queue. This matters when the +scanners are already inside long transitive C++ compiles: a requested stale +component can rebuild immediately instead of waiting for an unrelated compile. +All compiler processes use the same per-unit lock, so concurrent demand and scan +discovery cannot publish duplicate artifacts. The priority worker is idle when +there is no demand and never scans the site on its own. Unit compilation writes and validates a process-unique temporary wasm file, then publishes it with an atomic rename. When proactive compilation is enabled, read-only HTTP requests keep using the last complete artifact while requesting diff --git a/etc/uce/settings.cfg b/etc/uce/settings.cfg index 51401e6..2f832af 100644 --- a/etc/uce/settings.cfg +++ b/etc/uce/settings.cfg @@ -53,6 +53,10 @@ MYSQL_PERSISTENT_POOL_SIZE=8 # ENABLE THE BACKGROUND PROACTIVE COMPILER LOOP PROACTIVE_COMPILE_ENABLED=1 +# NUMBER OF LOW-PRIORITY FULL-SITE SCANNER PROCESSES (1-16) +# The separate demand-priority compiler is not counted here. +PROACTIVE_COMPILE_JOBS=2 + # AFTER A FAILED COMPILE, UCE NOW REUSES THE PERSISTED COMPILER OUTPUT UNTIL # THE SOURCE OR COMPILER INPUTS CHANGE. THIS SETTING IS KEPT FOR COMPATIBILITY. COMPILE_FAILURE_RETRY_SECONDS=10 diff --git a/scripts/run_cli_tests.sh b/scripts/run_cli_tests.sh index 07bc80a..286afad 100755 --- a/scripts/run_cli_tests.sh +++ b/scripts/run_cli_tests.sh @@ -69,6 +69,7 @@ if [[ "$action" == "run" ]]; then scripts/test_dependency_invalidation.sh scripts/test_abi_generation_rollout.sh scripts/test_parallel_precompile.sh + timeout --signal=TERM --kill-after=5s 175s scripts/test_parallel_proactive_compile.sh scripts/test_cold_component_deadline.sh scripts/test_nested_component_props.sh scripts/test_component_once_prefetch.sh diff --git a/scripts/test_parallel_proactive_compile.sh b/scripts/test_parallel_proactive_compile.sh new file mode 100755 index 0000000..6bdad56 --- /dev/null +++ b/scripts/test_parallel_proactive_compile.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +if [[ "${1:-}" != "--inside" ]]; then + exec timeout --signal=TERM --kill-after=5s 150s unshare --mount --fork --kill-child=TERM "$0" --inside +fi + +name="parallel-proactive-test-$$" +root="/tmp/$name" +site="$root/site" +work="$root/work" +settings="$root/settings.cfg" +log="$root/service.log" +shim_log="$root/compile.tsv" +server_pid="" + +cleanup() { + if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then + kill -TERM "$server_pid" 2>/dev/null || true + deadline=$((SECONDS + 10)) + while kill -0 "$server_pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.05; done + if kill -0 "$server_pid" 2>/dev/null; then kill -KILL "$server_pid" 2>/dev/null || true; fi + wait "$server_pid" 2>/dev/null || true + fi + rm -rf "$root" +} +trap cleanup EXIT +mkdir -p "$site" "$work" "$root/run" "$root/session" "$root/upload" +cp /etc/uce/settings.cfg "$settings" +cat >>"$settings" <"$root/compile" <>"\$root/counter.lock" +flock 9 +active=0 +if [[ -r "\$root/active" ]]; then read -r active <"\$root/active"; fi +active=\$((active + 1)) +printf '%s\n' "\$active" >"\$root/active" +maximum=0 +if [[ -r "\$root/maximum" ]]; then read -r maximum <"\$root/maximum"; fi +if (( active > maximum )); then printf '%s\n' "\$active" >"\$root/maximum"; fi +nice_value=\$(timeout 2s ps -o ni= -p "\$PPID" | tr -d ' ') +printf '%s\t%s\t%s\n' "\$(date +%s%N)" "\$nice_value" "\$source_file" >>"\$root/compile.tsv" +flock -u 9 +sleep 0.75 +"\$real" "\$@" +rc=\$? +flock 9 +active=1 +if [[ -r "\$root/active" ]]; then read -r active <"\$root/active"; fi +printf '%s\n' "\$((active - 1))" >"\$root/active" +flock -u 9 +exit "\$rc" +SHIM +chmod +x "$root/compile" + +printf '%s\n' 'String common_marker() { return("common-a"); }' >"$site/common.uce" +for unit in 0 1 2 3; do + printf '%s\n' '#load "common.uce"' "CLI(Request& context) { print(common_marker(), \"-$unit\"); }" >"$site/unit-$unit.uce" +done +printf '%s\n' 'CLI(Request& context) { print("removed"); }' >"$site/removed.uce" +printf '%s\n' 'CLI(Request& context) { print("baseline"); }' >"$site/broken.uce" + +timeout --signal=TERM --kill-after=5s 120s bin/uce_fastcgi.linux.bin >"$log" 2>&1 & +server_pid=$! +deadline=$((SECONDS + 20)) +while [[ ! -S "$root/run/cli.sock" ]] && (( SECONDS < deadline )); do sleep 0.05; done +[[ -S "$root/run/cli.sock" ]] || { echo "private UCE CLI socket was not ready" >&2; cat "$log" >&2; exit 1; } + +generation="$(scripts/unit_cache_directory "$work")" +artifacts="$generation$(realpath "$site")" +deadline=$((SECONDS + 60)) +while (( SECONDS < deadline )); do + ready=1 + for unit in 0 1 2 3; do + [[ -s "$artifacts/unit-$unit.uce.wasm" && -s "$artifacts/unit-$unit.uce.cwasm" ]] || ready=0 + done + (( ready == 1 )) && break + sleep 0.1 +done +(( ready == 1 )) || { echo "parallel scanners did not publish all valid units" >&2; cat "$log" >&2; exit 1; } +[[ "$(<"$root/maximum")" -ge 2 ]] || { echo "parallel scanners never overlapped" >&2; cat "$shim_log" >&2; exit 1; } +grep -q 'proactive compiler worker 1/2 ready' "$log" +grep -q 'proactive compiler worker 2/2 ready' "$log" +for unit in 0 1 2 3; do + [[ "$(grep -c "$site/unit-$unit.uce" "$shim_log")" -eq 1 ]] || { echo "unit-$unit was not compiled exactly once" >&2; cat "$shim_log" >&2; exit 1; } +done +deadline=$((SECONDS + 15)) +while { [[ ! -s "$artifacts/broken.uce.cwasm" || ! -s "$artifacts/removed.uce.cwasm" ]] || [[ "$(<"$root/active")" != "0" ]]; } && (( SECONDS < deadline )); do sleep 0.1; done +[[ -s "$artifacts/broken.uce.cwasm" && -s "$artifacts/removed.uce.cwasm" && "$(<"$root/active")" == "0" ]] || { echo "phase fixtures did not become idle" >&2; cat "$log" >&2; exit 1; } +exec 7>>"$root/counter.lock" +flock 7 +printf '0\n' >"$root/maximum" +flock -u 7 +sed -i 's/common-a/common-b/' "$site/common.uce" +deadline=$((SECONDS + 30)) +while (( SECONDS < deadline )); do + rebuilt=1 + for unit in 0 1 2 3; do + [[ "$(grep -c "$site/unit-$unit.uce" "$shim_log" || true)" -ge 2 ]] || rebuilt=0 + done + (( rebuilt == 1 )) && break + sleep 0.1 +done +(( rebuilt == 1 )) || { echo "common dependency edit did not rebuild all scanner shards" >&2; cat "$log" >&2; exit 1; } +for unit in 0 1 2 3; do + [[ "$(grep -c "$site/unit-$unit.uce" "$shim_log")" -eq 2 ]] || { echo "unit-$unit common-dependency rebuild was not exact" >&2; cat "$shim_log" >&2; exit 1; } +done +[[ "$(<"$root/maximum")" -ge 2 ]] || { echo "common dependency fanout did not overlap across scanner owners" >&2; cat "$shim_log" >&2; exit 1; } +marker=$(UCE_CLI_SOCKET="$root/run/cli.sock" timeout 20s scripts/uce-cli /unit-0.uce) +[[ "$marker" == *"common-b-0"* ]] || { echo "rebuilt parent did not execute the changed dependency: $marker" >&2; exit 1; } +rm "$site/removed.uce" +registry="$generation/known-uce-files.txt" +deadline=$((SECONDS + 5)) +while grep -q "$site/removed.uce" "$registry" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.1; done +if grep -q "$site/removed.uce" "$registry" 2>/dev/null; then echo "removed unit remained tracked" >&2; cat "$log" >&2; exit 1; fi + +printf '%s\n' 'CLI(Request& context) { print(deliberate_parallel_scanner_failure); }' >"$site/broken.uce" +deadline=$((SECONDS + 30)) +while [[ "$(grep -c "$site/broken.uce" "$shim_log" || true)" -lt 3 ]] && (( SECONDS < deadline )); do sleep 0.1; done +[[ "$(grep -c "$site/broken.uce" "$shim_log" || true)" -ge 3 ]] || { echo "persisted scanner failure was not retried" >&2; cat "$log" >&2; exit 1; } +mapfile -t broken_times < <(awk -F '\t' -v path="$site/broken.uce" '$3 == path { print $1 }' "$shim_log") +(( broken_times[2] - broken_times[1] >= 2000000000 )) || { echo "scanner failure retry ignored its backoff" >&2; cat "$shim_log" >&2; exit 1; } +[[ ! -e "$artifacts/broken.uce.wasm" ]] || { echo "failed scanner build published wasm" >&2; exit 1; } +printf '%s\n' 'CLI(Request& context) { print("recovered"); }' >"$site/broken.uce" +deadline=$((SECONDS + 8)) +while [[ ! -s "$artifacts/broken.uce.cwasm" ]] && (( SECONDS < deadline )); do sleep 0.1; done +[[ -s "$artifacts/broken.uce.cwasm" ]] || { echo "changed failed source waited for the unchanged-failure backoff" >&2; cat "$log" >&2; exit 1; } + +printf '%s\n' 'CLI(Request& context) { print("priority"); }' >"$site/priority.uce" +priority_file="$generation/proactive-priority.txt" +exec 8>"$priority_file.lock" +flock 8 +printf '%s\n' "$(realpath "$site/priority.uce")" >>"$priority_file" +flock -u 8 +deadline=$((SECONDS + 15)) +while [[ ! -s "$artifacts/priority.uce.cwasm" ]] && (( SECONDS < deadline )); do sleep 0.1; done +[[ -s "$artifacts/priority.uce.cwasm" ]] || { echo "priority compiler did not publish requested unit" >&2; cat "$log" >&2; exit 1; } +priority_nice=$(awk -F '\t' -v path="$site/priority.uce" '$3 == path { print $2; exit }' "$shim_log") +[[ "$priority_nice" == "5" ]] || { echo "priority queue was not owned by the nice-5 compiler: $priority_nice" >&2; cat "$shim_log" >&2; exit 1; } + +printf '%s\n' 'parallel proactive compile passed' diff --git a/src/lib/sys.cpp b/src/lib/sys.cpp index eb1c5b7..a6b31f9 100644 --- a/src/lib/sys.cpp +++ b/src/lib/sys.cpp @@ -1704,6 +1704,7 @@ StringMap make_server_settings() cfg["SITE_DIRECTORY"] = "site"; cfg["JIT_COMPILE_ON_REQUEST"] = "1"; cfg["PROACTIVE_COMPILE_ENABLED"] = "1"; + cfg["PROACTIVE_COMPILE_JOBS"] = "2"; cfg["COMPILE_FAILURE_RETRY_SECONDS"] = std::to_string(10); cfg["PROACTIVE_COMPILE_CHECK_INTERVAL"] = std::to_string(60); cfg["TRANSPORT_MAX_CLIENT_CONNECTIONS"] = std::to_string(256); diff --git a/src/linux_fastcgi.cpp b/src/linux_fastcgi.cpp index 871da8b..6c70dee 100644 --- a/src/linux_fastcgi.cpp +++ b/src/linux_fastcgi.cpp @@ -16,7 +16,7 @@ ServerState server_state; #include "fastcgi/src/fcgicc.cc" FastCGIServer server; -pid_t proactive_compiler_pid = 0; +std::vector proactive_compiler_pids; pid_t priority_compiler_pid = 0; // The central WS broker process: owns the WS port + every connection, forwards @@ -1212,6 +1212,32 @@ bool proactive_compile_queue_has(StringList& queue, String file_name) return(std::find(queue.begin(), queue.end(), file_name) != queue.end()); } +u64 bounded_compile_jobs(String value, u64 fallback = 2) +{ + value = trim(value); + u64 jobs = fallback; + if(value != "") + { + char* end = 0; + errno = 0; + long long parsed = strtoll(value.c_str(), &end, 10); + if(end != value.c_str() && end && *end == '\0' && errno != ERANGE) + jobs = parsed < 1 ? 1 : (u64)parsed; + } + return(std::max(1, std::min(jobs, 16))); +} + +bool proactive_compile_worker_owns(String file_name, u64 worker, u64 jobs) +{ + u64 hash = 1469598103934665603ull; + for(unsigned char c : file_name) + { + hash ^= c; + hash *= 1099511628211ull; + } + return(hash % jobs == worker); +} + void proactive_compile_queue_push(StringList& queue, String file_name) { if(file_name == "" || proactive_compile_queue_has(queue, file_name)) @@ -1252,7 +1278,7 @@ bool proactive_compile_unit(Request& context, String file_name, bool& source_mis return(failed); } -void run_proactive_compiler() +void run_proactive_compiler(u64 worker, u64 jobs) { Request background_context; StringList compile_queue; @@ -1281,6 +1307,8 @@ void run_proactive_compiler() signal(SIGFPE, on_segfault); signal(SIGPIPE, SIG_IGN); setpriority(PRIO_PROCESS, 0, 10); + printf("(P) proactive compiler worker %llu/%llu ready: PID %i\n", + (unsigned long long)(worker + 1), (unsigned long long)jobs, getpid()); try { @@ -1303,43 +1331,45 @@ void run_proactive_compiler() { try { - auto priority_units = compiler_take_priority_units(&background_context); - for(auto it = priority_units.rbegin(); it != priority_units.rend(); ++it) - { - compile_queue.erase(std::remove(compile_queue.begin(), compile_queue.end(), *it), compile_queue.end()); - compile_queue.insert(compile_queue.begin(), *it); - } if(compile_queue.size() == 0 && time_precise() >= next_scan_at) - { - auto tracked_units = compiler_list_known_units(&background_context); - StringList existing_units; - bool source_generation_changed = false; - - for(auto& file_name : tracked_units) { - bool source_missing = false; - auto retry_it = retry_after.find(file_name); - bool retry_allowed = (retry_it == retry_after.end() || time_precise() >= retry_it->second); - bool needs_compile = compiler_unit_needs_recompile(&background_context, file_name, &source_missing); - if(needs_compile && retry_allowed) - { - proactive_compile_queue_push(compile_queue, file_name); - source_generation_changed = true; - } - if(source_missing) - { - source_generation_changed = true; - printf("(i) proactive compiler forget removed unit %s\n", file_name.c_str()); - retry_after.erase(file_name); - continue; - } - existing_units.push_back(file_name); - } + auto tracked_units = compiler_list_known_units(&background_context); + bool source_generation_changed = false; - if(existing_units.size() != tracked_units.size()) - compiler_set_known_units(&background_context, existing_units); - if(source_generation_changed) - compiler_mark_source_generation(&background_context); + for(auto& file_name : tracked_units) + { + if(!proactive_compile_worker_owns(file_name, worker, jobs)) + continue; + bool source_missing = false; + auto retry_it = retry_after.find(file_name); + bool needs_compile = compiler_unit_needs_recompile(&background_context, file_name, &source_missing); + f64 now = time_precise(); + if(needs_compile && retry_it != retry_after.end()) + { + retry_after.erase(retry_it); + retry_it = retry_after.end(); + } + bool retry_due = retry_it != retry_after.end() && now >= retry_it->second; + if(!needs_compile && retry_it == retry_after.end() && !source_missing && + compiler_unit_needs_recompile(&background_context, file_name, 0, false, true, true)) + retry_after[file_name] = now + failure_retry_interval; + if(needs_compile || retry_due) + { + proactive_compile_queue_push(compile_queue, file_name); + source_generation_changed = true; + } + if(source_missing) + { + source_generation_changed = true; + printf("(i) proactive compiler forget removed unit %s\n", file_name.c_str()); + compiler_untrack_known_unit(&background_context, file_name); + retry_after.erase(file_name); + continue; + } + } + + if(source_generation_changed) + compiler_mark_source_generation(&background_context); next_scan_at = time_precise() + check_interval; } @@ -1352,7 +1382,8 @@ void run_proactive_compiler() auto retry_it = retry_after.find(file_name); if(retry_it != retry_after.end() && time_precise() < retry_it->second) continue; - bool failed = proactive_compile_unit(background_context, file_name, source_missing); + bool normal_compile = compiler_unit_needs_recompile(&background_context, file_name, &source_missing); + bool failed = proactive_compile_unit(background_context, file_name, source_missing, !normal_compile && !source_missing); if(source_missing) retry_after.erase(file_name); else if(failed) @@ -1412,9 +1443,9 @@ void run_priority_compiler() } } -bool proactive_compiler_alive() +bool proactive_compiler_alive(pid_t pid) { - return(proactive_compiler_pid > 0 && task_kill(proactive_compiler_pid, 0) == 0); + return(pid > 0 && task_kill(pid, 0) == 0); } pid_t spawn_compiler(const char* label, void (*runner)()) @@ -1435,6 +1466,25 @@ pid_t spawn_compiler(const char* label, void (*runner)()) return(p); } +pid_t spawn_proactive_compiler(u64 worker, u64 jobs) +{ + pid_t p = fork(); + if(p < 0) + { + perror("fork proactive compiler"); + return(0); + } + if(p == 0) + { + prctl(PR_SET_PDEATHSIG, SIGHUP); + run_proactive_compiler(worker, jobs); + exit(0); + } + printf("(P) proactive compiler worker %llu/%llu spawned: PID %i\n", + (unsigned long long)(worker + 1), (unsigned long long)jobs, p); + return(p); +} + void ensure_proactive_compiler() { if(!to_bool(server_state.config["PROACTIVE_COMPILE_ENABLED"], true)) @@ -1442,8 +1492,12 @@ void ensure_proactive_compiler() if(float_val(server_state.config["PROACTIVE_COMPILE_CHECK_INTERVAL"]) <= 0) return; - if(!proactive_compiler_alive()) - proactive_compiler_pid = spawn_compiler("proactive compiler", run_proactive_compiler); + u64 jobs = bounded_compile_jobs(server_state.config["PROACTIVE_COMPILE_JOBS"]); + if(proactive_compiler_pids.size() != jobs) + proactive_compiler_pids.resize(jobs, 0); + for(u64 worker = 0; worker < jobs; worker++) + if(!proactive_compiler_alive(proactive_compiler_pids[worker])) + proactive_compiler_pids[worker] = spawn_proactive_compiler(worker, jobs); if(priority_compiler_pid <= 0 || task_kill(priority_compiler_pid, 0) != 0) priority_compiler_pid = spawn_compiler("priority compiler", run_priority_compiler); } @@ -1661,16 +1715,7 @@ int precompile_unit_generation() compiler_set_known_units(&background_context, files); const char* jobs_env = getenv("UCE_PRECOMPILE_JOBS"); String jobs_text = trim(jobs_env && jobs_env[0] != '\0' ? String(jobs_env) : server_state.config["PRECOMPILE_JOBS"]); - u64 jobs = 2; - if(jobs_text != "") - { - char* end = 0; - errno = 0; - long long parsed = strtoll(jobs_text.c_str(), &end, 10); - if(end != jobs_text.c_str() && end && *end == '\0' && errno != ERANGE) - jobs = parsed < 1 ? 1 : (u64)parsed; - } - jobs = std::max(1, std::min(jobs, std::min(files.size() == 0 ? 1 : files.size(), 16))); + u64 jobs = std::min(bounded_compile_jobs(jobs_text), files.size() == 0 ? 1 : files.size()); PrecompileWorkerResult total; bool worker_error = false; if(jobs == 1) @@ -1784,8 +1829,9 @@ int main(int argc, char** argv) while(!termination_signal_received) { - if(!proactive_compiler_alive()) - proactive_compiler_pid = 0; + for(auto& pid : proactive_compiler_pids) + if(!proactive_compiler_alive(pid)) + pid = 0; if(priority_compiler_pid > 0 && task_kill(priority_compiler_pid, 0) != 0) priority_compiler_pid = 0; if(!termination_signal_received) @@ -1810,20 +1856,35 @@ int main(int argc, char** argv) printf("(P) draining %zu workers before shutdown\n", workers.size()); for(auto& worker : workers) kill(worker.first, SIGTERM); - if(proactive_compiler_pid > 0) - kill(proactive_compiler_pid, SIGTERM); + for(auto pid : proactive_compiler_pids) + if(pid > 0) + kill(pid, SIGTERM); if(priority_compiler_pid > 0) kill(priority_compiler_pid, SIGTERM); if(ws_broker_pid > 0) kill(ws_broker_pid, SIGTERM); f64 drain_deadline = time_precise() + (f64)to_u64(server_state.config["WORKER_DRAIN_TIMEOUT_SECONDS"], 10); - while(!workers.empty() && time_precise() < drain_deadline) + auto background_children_alive = [&]() { + bool alive = priority_compiler_pid > 0 && task_kill(priority_compiler_pid, 0) == 0; + alive = alive || (ws_broker_pid > 0 && task_kill(ws_broker_pid, 0) == 0); + for(auto pid : proactive_compiler_pids) + alive = alive || proactive_compiler_alive(pid); + return(alive); + }; + while((!workers.empty() || background_children_alive()) && time_precise() < drain_deadline) { on_child_exit(0); usleep(10000); } for(auto& worker : workers) kill(worker.first, SIGKILL); + for(auto pid : proactive_compiler_pids) + if(proactive_compiler_alive(pid)) + kill(pid, SIGKILL); + if(priority_compiler_pid > 0 && task_kill(priority_compiler_pid, 0) == 0) + kill(priority_compiler_pid, SIGKILL); + if(ws_broker_pid > 0 && task_kill(ws_broker_pid, 0) == 0) + kill(ws_broker_pid, SIGKILL); server.shutdown(); return 0;