Parallelize proactive unit compilation

docs/setup.md

@@ -190,6 +190,7 @@ SITE_DIRECTORY=/var/www/html
 HTTP_DOCUMENT_ROOT=/var/www/html
 JIT_COMPILE_ON_REQUEST=1
 PROACTIVE_COMPILE_ENABLED=1
+PROACTIVE_COMPILE_JOBS=2
 PROACTIVE_COMPILE_CHECK_INTERVAL=60
 
 WASM_COMPILE_SCRIPT=scripts/compile_wasm_unit
@@ -224,6 +225,7 @@ Important settings:
 - `HTTP_PORT` is the built-in HTTP/WebSocket listener used for WebSocket upgrade traffic and direct local probes. Bind/firewall it for local access only; nginx/Apache should be the public entry point.
 - `WS_BROKER_OUTBOUND_TIMEOUT_SECONDS` controls how long a forwarded WS message can remain queued in the broker before being dropped (default `30`). Set to `0` to disable the timeout.
 - `WASM_COMPILE_SCRIPT` must point to `scripts/compile_wasm_unit` unless you provide an equivalent compiler. Relative paths are resolved from the runtime root/`COMPILER_SYS_PATH`. That script calls `scripts/check_unit_wasm.py` after linking each unit and uses the pinned WASI SDK on every deployment host.
+- `PROACTIVE_COMPILE_JOBS` selects 1–16 low-priority full-site scanner processes (default `2`). Each canonical unit path has one scanner owner. The separate higher-priority demand compiler remains reserved for stale units requested over HTTP, so total background compile concurrency can reach this value plus one.
 - `WASM_CORE_PATH` must point at the built `core.wasm` file.
 
 After editing settings, restart UCE:

docs/wasm-runtime-architecture.md

@@ -246,14 +246,16 @@ publication and the shared generation PCH use separate advisory locks, and any
 worker/reporting failure rejects the candidate generation.
 
 The proactive compiler and request workers coordinate through per-unit file
-locks and a lock-protected demand-priority queue under `BIN_DIRECTORY`. A small
-priority-only compiler process drains that queue independently of the full-site
-scanner. This matters when the scanner is already inside a long transitive C++
-compile: a requested stale component can rebuild immediately instead of waiting
-for that unrelated compile to finish. Both compiler processes use the same
-per-unit lock, so concurrent demand and scan discovery cannot publish duplicate
-artifacts. The priority worker is idle when there is no demand and never scans
-the site on its own. Unit
+locks and a lock-protected demand-priority queue under `BIN_DIRECTORY`. The
+full-site scan uses two low-priority processes by default, bounded from 1–16 by
+`PROACTIVE_COMPILE_JOBS`; a stable canonical-path partition gives every unit one
+scanner owner and keeps retry/backoff state local. A separate higher-priority
+compiler process exclusively drains the demand queue. This matters when the
+scanners are already inside long transitive C++ compiles: a requested stale
+component can rebuild immediately instead of waiting for an unrelated compile.
+All compiler processes use the same per-unit lock, so concurrent demand and scan
+discovery cannot publish duplicate artifacts. The priority worker is idle when
+there is no demand and never scans the site on its own. Unit
 compilation writes and validates a process-unique temporary wasm file, then
 publishes it with an atomic rename. When proactive compilation is enabled,
 read-only HTTP requests keep using the last complete artifact while requesting

etc/uce/settings.cfg

@@ -53,6 +53,10 @@ MYSQL_PERSISTENT_POOL_SIZE=8
 # ENABLE THE BACKGROUND PROACTIVE COMPILER LOOP
 PROACTIVE_COMPILE_ENABLED=1
 
+# NUMBER OF LOW-PRIORITY FULL-SITE SCANNER PROCESSES (1-16)
+# The separate demand-priority compiler is not counted here.
+PROACTIVE_COMPILE_JOBS=2
+
 # AFTER A FAILED COMPILE, UCE NOW REUSES THE PERSISTED COMPILER OUTPUT UNTIL
 # THE SOURCE OR COMPILER INPUTS CHANGE. THIS SETTING IS KEPT FOR COMPATIBILITY.
 COMPILE_FAILURE_RETRY_SECONDS=10

scripts/run_cli_tests.sh

@@ -69,6 +69,7 @@ if [[ "$action" == "run" ]]; then
 	scripts/test_dependency_invalidation.sh
 	scripts/test_abi_generation_rollout.sh
 	scripts/test_parallel_precompile.sh
+	timeout --signal=TERM --kill-after=5s 175s scripts/test_parallel_proactive_compile.sh
 	scripts/test_cold_component_deadline.sh
 	scripts/test_nested_component_props.sh
 	scripts/test_component_once_prefetch.sh

scripts/test_parallel_proactive_compile.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [[ "${1:-}" != "--inside" ]]; then
+	exec timeout --signal=TERM --kill-after=5s 150s unshare --mount --fork --kill-child=TERM "$0" --inside
+fi
+
+name="parallel-proactive-test-$$"
+root="/tmp/$name"
+site="$root/site"
+work="$root/work"
+settings="$root/settings.cfg"
+log="$root/service.log"
+shim_log="$root/compile.tsv"
+server_pid=""
+
+cleanup() {
+	if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then
+		kill -TERM "$server_pid" 2>/dev/null || true
+		deadline=$((SECONDS + 10))
+		while kill -0 "$server_pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.05; done
+		if kill -0 "$server_pid" 2>/dev/null; then kill -KILL "$server_pid" 2>/dev/null || true; fi
+		wait "$server_pid" 2>/dev/null || true
+	fi
+	rm -rf "$root"
+}
+trap cleanup EXIT
+mkdir -p "$site" "$work" "$root/run" "$root/session" "$root/upload"
+cp /etc/uce/settings.cfg "$settings"
+cat >>"$settings" <<CFG
+BIN_DIRECTORY=$work
+PRECOMPILE_FILES_IN=$site
+SITE_DIRECTORY=$site
+FCGI_SOCKET_PATH=$root/run/fastcgi.sock
+FCGI_PORT=
+CLI_SOCKET_PATH=$root/run/cli.sock
+WS_BROKER_SOCKET_PATH=$root/run/ws.sock
+HTTP_PORT=
+HTTP_DOCUMENT_ROOT=$site
+SESSION_PATH=$root/session
+TMP_UPLOAD_PATH=$root/upload
+WASM_CORE_PATH=$(pwd)/bin/wasm/core.wasm
+WASM_COMPILE_SCRIPT=$root/compile
+WORKER_COUNT=1
+PROACTIVE_COMPILE_ENABLED=1
+PROACTIVE_COMPILE_JOBS=2
+PROACTIVE_COMPILE_CHECK_INTERVAL=1
+COMPILE_FAILURE_RETRY_SECONDS=2
+CFG
+mount --bind "$settings" /etc/uce/settings.cfg
+
+cat >"$root/compile" <<SHIM
+#!/usr/bin/env bash
+set -uo pipefail
+root='$root'
+real='$(pwd)/scripts/compile_wasm_unit'
+source_file="\$3"
+exec 9>>"\$root/counter.lock"
+flock 9
+active=0
+if [[ -r "\$root/active" ]]; then read -r active <"\$root/active"; fi
+active=\$((active + 1))
+printf '%s\n' "\$active" >"\$root/active"
+maximum=0
+if [[ -r "\$root/maximum" ]]; then read -r maximum <"\$root/maximum"; fi
+if (( active > maximum )); then printf '%s\n' "\$active" >"\$root/maximum"; fi
+nice_value=\$(timeout 2s ps -o ni= -p "\$PPID" | tr -d ' ')
+printf '%s\t%s\t%s\n' "\$(date +%s%N)" "\$nice_value" "\$source_file" >>"\$root/compile.tsv"
+flock -u 9
+sleep 0.75
+"\$real" "\$@"
+rc=\$?
+flock 9
+active=1
+if [[ -r "\$root/active" ]]; then read -r active <"\$root/active"; fi
+printf '%s\n' "\$((active - 1))" >"\$root/active"
+flock -u 9
+exit "\$rc"
+SHIM
+chmod +x "$root/compile"
+
+printf '%s\n' 'String common_marker() { return("common-a"); }' >"$site/common.uce"
+for unit in 0 1 2 3; do
+	printf '%s\n' '#load "common.uce"' "CLI(Request& context) { print(common_marker(), \"-$unit\"); }" >"$site/unit-$unit.uce"
+done
+printf '%s\n' 'CLI(Request& context) { print("removed"); }' >"$site/removed.uce"
+printf '%s\n' 'CLI(Request& context) { print("baseline"); }' >"$site/broken.uce"
+
+timeout --signal=TERM --kill-after=5s 120s bin/uce_fastcgi.linux.bin >"$log" 2>&1 &
+server_pid=$!
+deadline=$((SECONDS + 20))
+while [[ ! -S "$root/run/cli.sock" ]] && (( SECONDS < deadline )); do sleep 0.05; done
+[[ -S "$root/run/cli.sock" ]] || { echo "private UCE CLI socket was not ready" >&2; cat "$log" >&2; exit 1; }
+
+generation="$(scripts/unit_cache_directory "$work")"
+artifacts="$generation$(realpath "$site")"
+deadline=$((SECONDS + 60))
+while (( SECONDS < deadline )); do
+	ready=1
+	for unit in 0 1 2 3; do
+		[[ -s "$artifacts/unit-$unit.uce.wasm" && -s "$artifacts/unit-$unit.uce.cwasm" ]] || ready=0
+	done
+	(( ready == 1 )) && break
+	sleep 0.1
+done
+(( ready == 1 )) || { echo "parallel scanners did not publish all valid units" >&2; cat "$log" >&2; exit 1; }
+[[ "$(<"$root/maximum")" -ge 2 ]] || { echo "parallel scanners never overlapped" >&2; cat "$shim_log" >&2; exit 1; }
+grep -q 'proactive compiler worker 1/2 ready' "$log"
+grep -q 'proactive compiler worker 2/2 ready' "$log"
+for unit in 0 1 2 3; do
+	[[ "$(grep -c "$site/unit-$unit.uce" "$shim_log")" -eq 1 ]] || { echo "unit-$unit was not compiled exactly once" >&2; cat "$shim_log" >&2; exit 1; }
+done
+deadline=$((SECONDS + 15))
+while { [[ ! -s "$artifacts/broken.uce.cwasm" || ! -s "$artifacts/removed.uce.cwasm" ]] || [[ "$(<"$root/active")" != "0" ]]; } && (( SECONDS < deadline )); do sleep 0.1; done
+[[ -s "$artifacts/broken.uce.cwasm" && -s "$artifacts/removed.uce.cwasm" && "$(<"$root/active")" == "0" ]] || { echo "phase fixtures did not become idle" >&2; cat "$log" >&2; exit 1; }
+exec 7>>"$root/counter.lock"
+flock 7
+printf '0\n' >"$root/maximum"
+flock -u 7
+sed -i 's/common-a/common-b/' "$site/common.uce"
+deadline=$((SECONDS + 30))
+while (( SECONDS < deadline )); do
+	rebuilt=1
+	for unit in 0 1 2 3; do
+		[[ "$(grep -c "$site/unit-$unit.uce" "$shim_log" || true)" -ge 2 ]] || rebuilt=0
+	done
+	(( rebuilt == 1 )) && break
+	sleep 0.1
+done
+(( rebuilt == 1 )) || { echo "common dependency edit did not rebuild all scanner shards" >&2; cat "$log" >&2; exit 1; }
+for unit in 0 1 2 3; do
+	[[ "$(grep -c "$site/unit-$unit.uce" "$shim_log")" -eq 2 ]] || { echo "unit-$unit common-dependency rebuild was not exact" >&2; cat "$shim_log" >&2; exit 1; }
+done
+[[ "$(<"$root/maximum")" -ge 2 ]] || { echo "common dependency fanout did not overlap across scanner owners" >&2; cat "$shim_log" >&2; exit 1; }
+marker=$(UCE_CLI_SOCKET="$root/run/cli.sock" timeout 20s scripts/uce-cli /unit-0.uce)
+[[ "$marker" == *"common-b-0"* ]] || { echo "rebuilt parent did not execute the changed dependency: $marker" >&2; exit 1; }
+rm "$site/removed.uce"
+registry="$generation/known-uce-files.txt"
+deadline=$((SECONDS + 5))
+while grep -q "$site/removed.uce" "$registry" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.1; done
+if grep -q "$site/removed.uce" "$registry" 2>/dev/null; then echo "removed unit remained tracked" >&2; cat "$log" >&2; exit 1; fi
+
+printf '%s\n' 'CLI(Request& context) { print(deliberate_parallel_scanner_failure); }' >"$site/broken.uce"
+deadline=$((SECONDS + 30))
+while [[ "$(grep -c "$site/broken.uce" "$shim_log" || true)" -lt 3 ]] && (( SECONDS < deadline )); do sleep 0.1; done
+[[ "$(grep -c "$site/broken.uce" "$shim_log" || true)" -ge 3 ]] || { echo "persisted scanner failure was not retried" >&2; cat "$log" >&2; exit 1; }
+mapfile -t broken_times < <(awk -F '\t' -v path="$site/broken.uce" '$3 == path { print $1 }' "$shim_log")
+(( broken_times[2] - broken_times[1] >= 2000000000 )) || { echo "scanner failure retry ignored its backoff" >&2; cat "$shim_log" >&2; exit 1; }
+[[ ! -e "$artifacts/broken.uce.wasm" ]] || { echo "failed scanner build published wasm" >&2; exit 1; }
+printf '%s\n' 'CLI(Request& context) { print("recovered"); }' >"$site/broken.uce"
+deadline=$((SECONDS + 8))
+while [[ ! -s "$artifacts/broken.uce.cwasm" ]] && (( SECONDS < deadline )); do sleep 0.1; done
+[[ -s "$artifacts/broken.uce.cwasm" ]] || { echo "changed failed source waited for the unchanged-failure backoff" >&2; cat "$log" >&2; exit 1; }
+
+printf '%s\n' 'CLI(Request& context) { print("priority"); }' >"$site/priority.uce"
+priority_file="$generation/proactive-priority.txt"
+exec 8>"$priority_file.lock"
+flock 8
+printf '%s\n' "$(realpath "$site/priority.uce")" >>"$priority_file"
+flock -u 8
+deadline=$((SECONDS + 15))
+while [[ ! -s "$artifacts/priority.uce.cwasm" ]] && (( SECONDS < deadline )); do sleep 0.1; done
+[[ -s "$artifacts/priority.uce.cwasm" ]] || { echo "priority compiler did not publish requested unit" >&2; cat "$log" >&2; exit 1; }
+priority_nice=$(awk -F '\t' -v path="$site/priority.uce" '$3 == path { print $2; exit }' "$shim_log")
+[[ "$priority_nice" == "5" ]] || { echo "priority queue was not owned by the nice-5 compiler: $priority_nice" >&2; cat "$shim_log" >&2; exit 1; }
+
+printf '%s\n' 'parallel proactive compile passed'

src/lib/sys.cpp

@@ -1704,6 +1704,7 @@ StringMap make_server_settings()
 	cfg["SITE_DIRECTORY"] = "site";
 	cfg["JIT_COMPILE_ON_REQUEST"] = "1";
 	cfg["PROACTIVE_COMPILE_ENABLED"] = "1";
+	cfg["PROACTIVE_COMPILE_JOBS"] = "2";
 	cfg["COMPILE_FAILURE_RETRY_SECONDS"] = std::to_string(10);
 	cfg["PROACTIVE_COMPILE_CHECK_INTERVAL"] = std::to_string(60);
 	cfg["TRANSPORT_MAX_CLIENT_CONNECTIONS"] = std::to_string(256);

src/linux_fastcgi.cpp

@@ -16,7 +16,7 @@ ServerState server_state;
 #include "fastcgi/src/fcgicc.cc"
 
 FastCGIServer server;
-pid_t proactive_compiler_pid = 0;
+std::vector<pid_t> proactive_compiler_pids;
 pid_t priority_compiler_pid = 0;
 
 // The central WS broker process: owns the WS port + every connection, forwards
@@ -1212,6 +1212,32 @@ bool proactive_compile_queue_has(StringList& queue, String file_name)
 	return(std::find(queue.begin(), queue.end(), file_name) != queue.end());
 }
 
+u64 bounded_compile_jobs(String value, u64 fallback = 2)
+{
+	value = trim(value);
+	u64 jobs = fallback;
+	if(value != "")
+	{
+		char* end = 0;
+		errno = 0;
+		long long parsed = strtoll(value.c_str(), &end, 10);
+		if(end != value.c_str() && end && *end == '\0' && errno != ERANGE)
+			jobs = parsed < 1 ? 1 : (u64)parsed;
+	}
+	return(std::max<u64>(1, std::min<u64>(jobs, 16)));
+}
+
+bool proactive_compile_worker_owns(String file_name, u64 worker, u64 jobs)
+{
+	u64 hash = 1469598103934665603ull;
+	for(unsigned char c : file_name)
+	{
+		hash ^= c;
+		hash *= 1099511628211ull;
+	}
+	return(hash % jobs == worker);
+}
+
 void proactive_compile_queue_push(StringList& queue, String file_name)
 {
 	if(file_name == "" || proactive_compile_queue_has(queue, file_name))
@@ -1252,7 +1278,7 @@ bool proactive_compile_unit(Request& context, String file_name, bool& source_mis
 	return(failed);
 }
 
-void run_proactive_compiler()
+void run_proactive_compiler(u64 worker, u64 jobs)
 {
 	Request background_context;
 	StringList compile_queue;
@@ -1281,6 +1307,8 @@ void run_proactive_compiler()
 	signal(SIGFPE, on_segfault);
 	signal(SIGPIPE, SIG_IGN);
 	setpriority(PRIO_PROCESS, 0, 10);
+	printf("(P) proactive compiler worker %llu/%llu ready: PID %i\n",
+		(unsigned long long)(worker + 1), (unsigned long long)jobs, getpid());
 
 	try
 	{
@@ -1303,43 +1331,45 @@ void run_proactive_compiler()
 	{
 		try
 		{
-			auto priority_units = compiler_take_priority_units(&background_context);
-			for(auto it = priority_units.rbegin(); it != priority_units.rend(); ++it)
-			{
-				compile_queue.erase(std::remove(compile_queue.begin(), compile_queue.end(), *it), compile_queue.end());
-				compile_queue.insert(compile_queue.begin(), *it);
-			}
 			if(compile_queue.size() == 0 && time_precise() >= next_scan_at)
-		{
-			auto tracked_units = compiler_list_known_units(&background_context);
-			StringList existing_units;
-			bool source_generation_changed = false;
-
-			for(auto& file_name : tracked_units)
 			{
-				bool source_missing = false;
-				auto retry_it = retry_after.find(file_name);
-				bool retry_allowed = (retry_it == retry_after.end() || time_precise() >= retry_it->second);
-				bool needs_compile = compiler_unit_needs_recompile(&background_context, file_name, &source_missing);
-				if(needs_compile && retry_allowed)
-				{
-					proactive_compile_queue_push(compile_queue, file_name);
-					source_generation_changed = true;
-				}
-				if(source_missing)
-				{
-					source_generation_changed = true;
-					printf("(i) proactive compiler forget removed unit %s\n", file_name.c_str());
-					retry_after.erase(file_name);
-					continue;
-				}
-				existing_units.push_back(file_name);
-			}
+				auto tracked_units = compiler_list_known_units(&background_context);
+				bool source_generation_changed = false;
 
-			if(existing_units.size() != tracked_units.size())
-				compiler_set_known_units(&background_context, existing_units);
-			if(source_generation_changed)
-				compiler_mark_source_generation(&background_context);
+				for(auto& file_name : tracked_units)
+				{
+					if(!proactive_compile_worker_owns(file_name, worker, jobs))
+						continue;
+					bool source_missing = false;
+					auto retry_it = retry_after.find(file_name);
+					bool needs_compile = compiler_unit_needs_recompile(&background_context, file_name, &source_missing);
+					f64 now = time_precise();
+					if(needs_compile && retry_it != retry_after.end())
+					{
+						retry_after.erase(retry_it);
+						retry_it = retry_after.end();
+					}
+					bool retry_due = retry_it != retry_after.end() && now >= retry_it->second;
+					if(!needs_compile && retry_it == retry_after.end() && !source_missing &&
+						compiler_unit_needs_recompile(&background_context, file_name, 0, false, true, true))
+						retry_after[file_name] = now + failure_retry_interval;
+					if(needs_compile || retry_due)
+					{
+						proactive_compile_queue_push(compile_queue, file_name);
+						source_generation_changed = true;
+					}
+					if(source_missing)
+					{
+						source_generation_changed = true;
+						printf("(i) proactive compiler forget removed unit %s\n", file_name.c_str());
+						compiler_untrack_known_unit(&background_context, file_name);
+						retry_after.erase(file_name);
+						continue;
+					}
+				}
+
+				if(source_generation_changed)
+					compiler_mark_source_generation(&background_context);
 
 			next_scan_at = time_precise() + check_interval;
 		}
@@ -1352,7 +1382,8 @@ void run_proactive_compiler()
 			auto retry_it = retry_after.find(file_name);
 			if(retry_it != retry_after.end() && time_precise() < retry_it->second)
 				continue;
-			bool failed = proactive_compile_unit(background_context, file_name, source_missing);
+			bool normal_compile = compiler_unit_needs_recompile(&background_context, file_name, &source_missing);
+			bool failed = proactive_compile_unit(background_context, file_name, source_missing, !normal_compile && !source_missing);
 			if(source_missing)
 				retry_after.erase(file_name);
 			else if(failed)
@@ -1412,9 +1443,9 @@ void run_priority_compiler()
 	}
 }
 
-bool proactive_compiler_alive()
+bool proactive_compiler_alive(pid_t pid)
 {
-	return(proactive_compiler_pid > 0 && task_kill(proactive_compiler_pid, 0) == 0);
+	return(pid > 0 && task_kill(pid, 0) == 0);
 }
 
 pid_t spawn_compiler(const char* label, void (*runner)())
@@ -1435,6 +1466,25 @@ pid_t spawn_compiler(const char* label, void (*runner)())
 	return(p);
 }
 
+pid_t spawn_proactive_compiler(u64 worker, u64 jobs)
+{
+	pid_t p = fork();
+	if(p < 0)
+	{
+		perror("fork proactive compiler");
+		return(0);
+	}
+	if(p == 0)
+	{
+		prctl(PR_SET_PDEATHSIG, SIGHUP);
+		run_proactive_compiler(worker, jobs);
+		exit(0);
+	}
+	printf("(P) proactive compiler worker %llu/%llu spawned: PID %i\n",
+		(unsigned long long)(worker + 1), (unsigned long long)jobs, p);
+	return(p);
+}
+
 void ensure_proactive_compiler()
 {
 	if(!to_bool(server_state.config["PROACTIVE_COMPILE_ENABLED"], true))
@@ -1442,8 +1492,12 @@ void ensure_proactive_compiler()
 	if(float_val(server_state.config["PROACTIVE_COMPILE_CHECK_INTERVAL"]) <= 0)
 		return;
 
-	if(!proactive_compiler_alive())
-		proactive_compiler_pid = spawn_compiler("proactive compiler", run_proactive_compiler);
+	u64 jobs = bounded_compile_jobs(server_state.config["PROACTIVE_COMPILE_JOBS"]);
+	if(proactive_compiler_pids.size() != jobs)
+		proactive_compiler_pids.resize(jobs, 0);
+	for(u64 worker = 0; worker < jobs; worker++)
+		if(!proactive_compiler_alive(proactive_compiler_pids[worker]))
+			proactive_compiler_pids[worker] = spawn_proactive_compiler(worker, jobs);
 	if(priority_compiler_pid <= 0 || task_kill(priority_compiler_pid, 0) != 0)
 		priority_compiler_pid = spawn_compiler("priority compiler", run_priority_compiler);
 }
@@ -1661,16 +1715,7 @@ int precompile_unit_generation()
 	compiler_set_known_units(&background_context, files);
 	const char* jobs_env = getenv("UCE_PRECOMPILE_JOBS");
 	String jobs_text = trim(jobs_env && jobs_env[0] != '\0' ? String(jobs_env) : server_state.config["PRECOMPILE_JOBS"]);
-	u64 jobs = 2;
-	if(jobs_text != "")
-	{
-		char* end = 0;
-		errno = 0;
-		long long parsed = strtoll(jobs_text.c_str(), &end, 10);
-		if(end != jobs_text.c_str() && end && *end == '\0' && errno != ERANGE)
-			jobs = parsed < 1 ? 1 : (u64)parsed;
-	}
-	jobs = std::max<u64>(1, std::min<u64>(jobs, std::min<u64>(files.size() == 0 ? 1 : files.size(), 16)));
+	u64 jobs = std::min<u64>(bounded_compile_jobs(jobs_text), files.size() == 0 ? 1 : files.size());
 	PrecompileWorkerResult total;
 	bool worker_error = false;
 	if(jobs == 1)
@@ -1784,8 +1829,9 @@ int main(int argc, char** argv)
 
 	while(!termination_signal_received)
 	{
-		if(!proactive_compiler_alive())
-			proactive_compiler_pid = 0;
+		for(auto& pid : proactive_compiler_pids)
+			if(!proactive_compiler_alive(pid))
+				pid = 0;
 		if(priority_compiler_pid > 0 && task_kill(priority_compiler_pid, 0) != 0)
 			priority_compiler_pid = 0;
 		if(!termination_signal_received)
@@ -1810,20 +1856,35 @@ int main(int argc, char** argv)
 	printf("(P) draining %zu workers before shutdown\n", workers.size());
 	for(auto& worker : workers)
 		kill(worker.first, SIGTERM);
-	if(proactive_compiler_pid > 0)
-		kill(proactive_compiler_pid, SIGTERM);
+	for(auto pid : proactive_compiler_pids)
+		if(pid > 0)
+			kill(pid, SIGTERM);
 	if(priority_compiler_pid > 0)
 		kill(priority_compiler_pid, SIGTERM);
 	if(ws_broker_pid > 0)
 		kill(ws_broker_pid, SIGTERM);
 	f64 drain_deadline = time_precise() + (f64)to_u64(server_state.config["WORKER_DRAIN_TIMEOUT_SECONDS"], 10);
-	while(!workers.empty() && time_precise() < drain_deadline)
+	auto background_children_alive = [&]() {
+		bool alive = priority_compiler_pid > 0 && task_kill(priority_compiler_pid, 0) == 0;
+		alive = alive || (ws_broker_pid > 0 && task_kill(ws_broker_pid, 0) == 0);
+		for(auto pid : proactive_compiler_pids)
+			alive = alive || proactive_compiler_alive(pid);
+		return(alive);
+	};
+	while((!workers.empty() || background_children_alive()) && time_precise() < drain_deadline)
 	{
 		on_child_exit(0);
 		usleep(10000);
 	}
 	for(auto& worker : workers)
 		kill(worker.first, SIGKILL);
+	for(auto pid : proactive_compiler_pids)
+		if(proactive_compiler_alive(pid))
+			kill(pid, SIGKILL);
+	if(priority_compiler_pid > 0 && task_kill(priority_compiler_pid, 0) == 0)
+		kill(priority_compiler_pid, SIGKILL);
+	if(ws_broker_pid > 0 && task_kill(ws_broker_pid, 0) == 0)
+		kill(ws_broker_pid, SIGKILL);
 	server.shutdown();
 
 	return 0;