Parallelize proactive unit compilation

scripts/test_parallel_proactive_compile.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [[ "${1:-}" != "--inside" ]]; then
+	exec timeout --signal=TERM --kill-after=5s 150s unshare --mount --fork --kill-child=TERM "$0" --inside
+fi
+
+name="parallel-proactive-test-$$"
+root="/tmp/$name"
+site="$root/site"
+work="$root/work"
+settings="$root/settings.cfg"
+log="$root/service.log"
+shim_log="$root/compile.tsv"
+server_pid=""
+
+cleanup() {
+	if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then
+		kill -TERM "$server_pid" 2>/dev/null || true
+		deadline=$((SECONDS + 10))
+		while kill -0 "$server_pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.05; done
+		if kill -0 "$server_pid" 2>/dev/null; then kill -KILL "$server_pid" 2>/dev/null || true; fi
+		wait "$server_pid" 2>/dev/null || true
+	fi
+	rm -rf "$root"
+}
+trap cleanup EXIT
+mkdir -p "$site" "$work" "$root/run" "$root/session" "$root/upload"
+cp /etc/uce/settings.cfg "$settings"
+cat >>"$settings" <<CFG
+BIN_DIRECTORY=$work
+PRECOMPILE_FILES_IN=$site
+SITE_DIRECTORY=$site
+FCGI_SOCKET_PATH=$root/run/fastcgi.sock
+FCGI_PORT=
+CLI_SOCKET_PATH=$root/run/cli.sock
+WS_BROKER_SOCKET_PATH=$root/run/ws.sock
+HTTP_PORT=
+HTTP_DOCUMENT_ROOT=$site
+SESSION_PATH=$root/session
+TMP_UPLOAD_PATH=$root/upload
+WASM_CORE_PATH=$(pwd)/bin/wasm/core.wasm
+WASM_COMPILE_SCRIPT=$root/compile
+WORKER_COUNT=1
+PROACTIVE_COMPILE_ENABLED=1
+PROACTIVE_COMPILE_JOBS=2
+PROACTIVE_COMPILE_CHECK_INTERVAL=1
+COMPILE_FAILURE_RETRY_SECONDS=2
+CFG
+mount --bind "$settings" /etc/uce/settings.cfg
+
+cat >"$root/compile" <<SHIM
+#!/usr/bin/env bash
+set -uo pipefail
+root='$root'
+real='$(pwd)/scripts/compile_wasm_unit'
+source_file="\$3"
+exec 9>>"\$root/counter.lock"
+flock 9
+active=0
+if [[ -r "\$root/active" ]]; then read -r active <"\$root/active"; fi
+active=\$((active + 1))
+printf '%s\n' "\$active" >"\$root/active"
+maximum=0
+if [[ -r "\$root/maximum" ]]; then read -r maximum <"\$root/maximum"; fi
+if (( active > maximum )); then printf '%s\n' "\$active" >"\$root/maximum"; fi
+nice_value=\$(timeout 2s ps -o ni= -p "\$PPID" | tr -d ' ')
+printf '%s\t%s\t%s\n' "\$(date +%s%N)" "\$nice_value" "\$source_file" >>"\$root/compile.tsv"
+flock -u 9
+sleep 0.75
+"\$real" "\$@"
+rc=\$?
+flock 9
+active=1
+if [[ -r "\$root/active" ]]; then read -r active <"\$root/active"; fi
+printf '%s\n' "\$((active - 1))" >"\$root/active"
+flock -u 9
+exit "\$rc"
+SHIM
+chmod +x "$root/compile"
+
+printf '%s\n' 'String common_marker() { return("common-a"); }' >"$site/common.uce"
+for unit in 0 1 2 3; do
+	printf '%s\n' '#load "common.uce"' "CLI(Request& context) { print(common_marker(), \"-$unit\"); }" >"$site/unit-$unit.uce"
+done
+printf '%s\n' 'CLI(Request& context) { print("removed"); }' >"$site/removed.uce"
+printf '%s\n' 'CLI(Request& context) { print("baseline"); }' >"$site/broken.uce"
+
+timeout --signal=TERM --kill-after=5s 120s bin/uce_fastcgi.linux.bin >"$log" 2>&1 &
+server_pid=$!
+deadline=$((SECONDS + 20))
+while [[ ! -S "$root/run/cli.sock" ]] && (( SECONDS < deadline )); do sleep 0.05; done
+[[ -S "$root/run/cli.sock" ]] || { echo "private UCE CLI socket was not ready" >&2; cat "$log" >&2; exit 1; }
+
+generation="$(scripts/unit_cache_directory "$work")"
+artifacts="$generation$(realpath "$site")"
+deadline=$((SECONDS + 60))
+while (( SECONDS < deadline )); do
+	ready=1
+	for unit in 0 1 2 3; do
+		[[ -s "$artifacts/unit-$unit.uce.wasm" && -s "$artifacts/unit-$unit.uce.cwasm" ]] || ready=0
+	done
+	(( ready == 1 )) && break
+	sleep 0.1
+done
+(( ready == 1 )) || { echo "parallel scanners did not publish all valid units" >&2; cat "$log" >&2; exit 1; }
+[[ "$(<"$root/maximum")" -ge 2 ]] || { echo "parallel scanners never overlapped" >&2; cat "$shim_log" >&2; exit 1; }
+grep -q 'proactive compiler worker 1/2 ready' "$log"
+grep -q 'proactive compiler worker 2/2 ready' "$log"
+for unit in 0 1 2 3; do
+	[[ "$(grep -c "$site/unit-$unit.uce" "$shim_log")" -eq 1 ]] || { echo "unit-$unit was not compiled exactly once" >&2; cat "$shim_log" >&2; exit 1; }
+done
+deadline=$((SECONDS + 15))
+while { [[ ! -s "$artifacts/broken.uce.cwasm" || ! -s "$artifacts/removed.uce.cwasm" ]] || [[ "$(<"$root/active")" != "0" ]]; } && (( SECONDS < deadline )); do sleep 0.1; done
+[[ -s "$artifacts/broken.uce.cwasm" && -s "$artifacts/removed.uce.cwasm" && "$(<"$root/active")" == "0" ]] || { echo "phase fixtures did not become idle" >&2; cat "$log" >&2; exit 1; }
+exec 7>>"$root/counter.lock"
+flock 7
+printf '0\n' >"$root/maximum"
+flock -u 7
+sed -i 's/common-a/common-b/' "$site/common.uce"
+deadline=$((SECONDS + 30))
+while (( SECONDS < deadline )); do
+	rebuilt=1
+	for unit in 0 1 2 3; do
+		[[ "$(grep -c "$site/unit-$unit.uce" "$shim_log" || true)" -ge 2 ]] || rebuilt=0
+	done
+	(( rebuilt == 1 )) && break
+	sleep 0.1
+done
+(( rebuilt == 1 )) || { echo "common dependency edit did not rebuild all scanner shards" >&2; cat "$log" >&2; exit 1; }
+for unit in 0 1 2 3; do
+	[[ "$(grep -c "$site/unit-$unit.uce" "$shim_log")" -eq 2 ]] || { echo "unit-$unit common-dependency rebuild was not exact" >&2; cat "$shim_log" >&2; exit 1; }
+done
+[[ "$(<"$root/maximum")" -ge 2 ]] || { echo "common dependency fanout did not overlap across scanner owners" >&2; cat "$shim_log" >&2; exit 1; }
+marker=$(UCE_CLI_SOCKET="$root/run/cli.sock" timeout 20s scripts/uce-cli /unit-0.uce)
+[[ "$marker" == *"common-b-0"* ]] || { echo "rebuilt parent did not execute the changed dependency: $marker" >&2; exit 1; }
+rm "$site/removed.uce"
+registry="$generation/known-uce-files.txt"
+deadline=$((SECONDS + 5))
+while grep -q "$site/removed.uce" "$registry" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.1; done
+if grep -q "$site/removed.uce" "$registry" 2>/dev/null; then echo "removed unit remained tracked" >&2; cat "$log" >&2; exit 1; fi
+
+printf '%s\n' 'CLI(Request& context) { print(deliberate_parallel_scanner_failure); }' >"$site/broken.uce"
+deadline=$((SECONDS + 30))
+while [[ "$(grep -c "$site/broken.uce" "$shim_log" || true)" -lt 3 ]] && (( SECONDS < deadline )); do sleep 0.1; done
+[[ "$(grep -c "$site/broken.uce" "$shim_log" || true)" -ge 3 ]] || { echo "persisted scanner failure was not retried" >&2; cat "$log" >&2; exit 1; }
+mapfile -t broken_times < <(awk -F '\t' -v path="$site/broken.uce" '$3 == path { print $1 }' "$shim_log")
+(( broken_times[2] - broken_times[1] >= 2000000000 )) || { echo "scanner failure retry ignored its backoff" >&2; cat "$shim_log" >&2; exit 1; }
+[[ ! -e "$artifacts/broken.uce.wasm" ]] || { echo "failed scanner build published wasm" >&2; exit 1; }
+printf '%s\n' 'CLI(Request& context) { print("recovered"); }' >"$site/broken.uce"
+deadline=$((SECONDS + 8))
+while [[ ! -s "$artifacts/broken.uce.cwasm" ]] && (( SECONDS < deadline )); do sleep 0.1; done
+[[ -s "$artifacts/broken.uce.cwasm" ]] || { echo "changed failed source waited for the unchanged-failure backoff" >&2; cat "$log" >&2; exit 1; }
+
+printf '%s\n' 'CLI(Request& context) { print("priority"); }' >"$site/priority.uce"
+priority_file="$generation/proactive-priority.txt"
+exec 8>"$priority_file.lock"
+flock 8
+printf '%s\n' "$(realpath "$site/priority.uce")" >>"$priority_file"
+flock -u 8
+deadline=$((SECONDS + 15))
+while [[ ! -s "$artifacts/priority.uce.cwasm" ]] && (( SECONDS < deadline )); do sleep 0.1; done
+[[ -s "$artifacts/priority.uce.cwasm" ]] || { echo "priority compiler did not publish requested unit" >&2; cat "$log" >&2; exit 1; }
+priority_nice=$(awk -F '\t' -v path="$site/priority.uce" '$3 == path { print $2; exit }' "$shim_log")
+[[ "$priority_nice" == "5" ]] || { echo "priority queue was not owned by the nice-5 compiler: $priority_nice" >&2; cat "$shim_log" >&2; exit 1; }
+
+printf '%s\n' 'parallel proactive compile passed'