Isolate CLI workers and module serialization
This commit is contained in:
@@ -90,5 +90,6 @@ if [[ "$action" == "run" ]]; then
|
||||
timeout --signal=TERM --kill-after=5s 240s scripts/test_dynamic_compile_failures.sh
|
||||
scripts/test_wasm_source_locations.sh
|
||||
scripts/test_server_arguments.sh
|
||||
scripts/test_cli_worker_isolation.sh
|
||||
scripts/test_socket_activation.sh
|
||||
fi
|
||||
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [[ "${1:-}" != "--inside" ]]; then
|
||||
exec timeout --signal=TERM --kill-after=5s 120s unshare --mount --fork --kill-child=TERM "$0" --inside
|
||||
fi
|
||||
|
||||
root="/tmp/uce-cli-worker-isolation-$$"
|
||||
site="$root/site"
|
||||
work="$root/work"
|
||||
settings="$root/settings.cfg"
|
||||
log="$root/service.log"
|
||||
cli_socket="$root/run/cli.sock"
|
||||
fastcgi_socket="$root/run/fastcgi.sock"
|
||||
server_pid=""
|
||||
|
||||
cleanup() {
|
||||
status=$?
|
||||
if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then
|
||||
kill -TERM "$server_pid" 2>/dev/null || true
|
||||
deadline=$((SECONDS + 10))
|
||||
while kill -0 "$server_pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.05; done
|
||||
if kill -0 "$server_pid" 2>/dev/null; then kill -KILL "$server_pid" 2>/dev/null || true; fi
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
if (( status != 0 )) && [[ -r "$log" ]]; then cat "$log" >&2; fi
|
||||
rm -rf "$root"
|
||||
return "$status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir -p "$site" "$work" "$root/run" "$root/session" "$root/upload"
|
||||
cp /etc/uce/settings.cfg "$settings"
|
||||
cat >>"$settings" <<CFG
|
||||
BIN_DIRECTORY=$work
|
||||
PRECOMPILE_FILES_IN=$site
|
||||
SITE_DIRECTORY=$site
|
||||
FCGI_SOCKET_PATH=$fastcgi_socket
|
||||
FCGI_PORT=
|
||||
CLI_SOCKET_PATH=$cli_socket
|
||||
CLI_WORKER_COUNT=2
|
||||
CLI_WORKER_MAX_REQUESTS=3
|
||||
WS_BROKER_SOCKET_PATH=$root/run/ws.sock
|
||||
HTTP_PORT=
|
||||
HTTP_DOCUMENT_ROOT=$site
|
||||
SESSION_PATH=$root/session
|
||||
TMP_UPLOAD_PATH=$root/upload
|
||||
WASM_CORE_PATH=$(pwd)/bin/wasm/core.wasm
|
||||
WORKER_COUNT=2
|
||||
PROACTIVE_COMPILE_ENABLED=0
|
||||
CFG
|
||||
mount --bind "$settings" /etc/uce/settings.cfg
|
||||
|
||||
cat >"$site/isolation.uce" <<'UCE'
|
||||
RENDER(Request& context)
|
||||
{
|
||||
print(request_perf()["worker_pid"].to_string());
|
||||
}
|
||||
|
||||
CLI(Request& context)
|
||||
{
|
||||
print(request_perf()["worker_pid"].to_string());
|
||||
}
|
||||
UCE
|
||||
|
||||
cat >"$site/nested.uce" <<UCE
|
||||
CLI(Request& context)
|
||||
{
|
||||
print(shell_exec("cd $(pwd) && timeout 10s scripts/uce-cli --socket $cli_socket /isolation.uce"));
|
||||
}
|
||||
UCE
|
||||
|
||||
bin/uce_fastcgi.linux.bin >"$log" 2>&1 &
|
||||
server_pid=$!
|
||||
deadline=$((SECONDS + 20))
|
||||
while [[ ! -S "$cli_socket" ]] && (( SECONDS < deadline )); do sleep 0.05; done
|
||||
[[ -S "$cli_socket" ]] || { echo "private UCE CLI socket was not ready" >&2; exit 1; }
|
||||
timeout --signal=TERM --kill-after=1s 30s scripts/uce-cli --socket "$cli_socket" /isolation.uce >/dev/null
|
||||
timeout --signal=TERM --kill-after=1s 30s scripts/uce-cli --socket "$cli_socket" /nested.uce | grep -Eq '^[0-9]+([.]0+)?$'
|
||||
|
||||
fastcgi_request() {
|
||||
SCRIPT_FILENAME="$site/isolation.uce" SCRIPT_NAME=/isolation.uce REQUEST_URI=/isolation.uce REQUEST_METHOD=GET DOCUMENT_ROOT="$site" \
|
||||
timeout --signal=TERM --kill-after=1s 10s cgi-fcgi -bind -connect "$fastcgi_socket" | tr -d '\r' | awk 'NF{last=$0} END{print last}'
|
||||
}
|
||||
fastcgi_request >/dev/null
|
||||
|
||||
cli_request() {
|
||||
local output
|
||||
local deadline=$((SECONDS + 20))
|
||||
while (( SECONDS < deadline )); do
|
||||
if output=$(timeout --signal=TERM --kill-after=1s 10s scripts/uce-cli --socket "$cli_socket" /isolation.uce 2>/dev/null); then
|
||||
printf '%s\n' "$output"
|
||||
return 0
|
||||
fi
|
||||
sleep 0.05
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
cli_pids=()
|
||||
http_pids=()
|
||||
|
||||
# Concurrent completions must count individually and retire both initial CLI
|
||||
# workers despite neither connection pool necessarily becoming idle first.
|
||||
concurrent_jobs=()
|
||||
for n in $(seq 1 12); do
|
||||
cli_request >"$root/concurrent-$n" & concurrent_jobs+=("$!")
|
||||
done
|
||||
for job in "${concurrent_jobs[@]}"; do wait "$job"; done
|
||||
mapfile -t concurrent_cli < <(cat "$root"/concurrent-* | sort -u)
|
||||
cli_pids+=("${concurrent_cli[@]}")
|
||||
|
||||
for _ in $(seq 1 12); do
|
||||
cli_pids+=("$(cli_request)")
|
||||
http_pids+=("$(fastcgi_request)")
|
||||
done
|
||||
mapfile -t unique_cli < <(printf '%s\n' "${cli_pids[@]}" | sort -u)
|
||||
mapfile -t unique_http < <(printf '%s\n' "${http_pids[@]}" | sort -u)
|
||||
[[ ${#unique_cli[@]} -ge 4 ]] || { echo "CLI workers did not recycle after three requests: ${unique_cli[*]}" >&2; exit 1; }
|
||||
[[ ${#unique_http[@]} -ge 1 ]] || { echo "HTTP requests reached no public worker" >&2; exit 1; }
|
||||
for cli_pid in "${unique_cli[@]}"; do
|
||||
if printf '%s\n' "${unique_http[@]}" | grep -Fxq "$cli_pid"; then
|
||||
echo "CLI worker also served public HTTP: $cli_pid" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
[[ $(grep -c 'wasm FastCGI worker ready' "$log") -eq 2 ]]
|
||||
[[ $(grep -c 'wasm CLI worker ready' "$log") -ge 4 ]]
|
||||
|
||||
echo "CLI worker isolation and recycling passed: CLI ${unique_cli[*]}, public ${unique_http[*]}"
|
||||
|
||||
# A CLI-only runtime must not allocate a listener-less public Wasmtime worker.
|
||||
kill -TERM "$server_pid"
|
||||
wait "$server_pid"
|
||||
server_pid=""
|
||||
rm -f "$cli_socket" "$fastcgi_socket"
|
||||
cat >>"$settings" <<CFG
|
||||
FCGI_SOCKET_PATH=
|
||||
CLI_WORKER_COUNT=1
|
||||
CLI_WORKER_MAX_REQUESTS=0
|
||||
CFG
|
||||
: >"$log"
|
||||
bin/uce_fastcgi.linux.bin >"$log" 2>&1 &
|
||||
server_pid=$!
|
||||
deadline=$((SECONDS + 20))
|
||||
while [[ ! -S "$cli_socket" ]] && (( SECONDS < deadline )); do sleep 0.05; done
|
||||
[[ -S "$cli_socket" ]] || { echo "CLI-only socket was not ready" >&2; exit 1; }
|
||||
cli_request >/dev/null
|
||||
[[ $(grep -c 'wasm CLI worker ready' "$log") -eq 1 ]]
|
||||
[[ $(grep -c 'wasm FastCGI worker ready' "$log") -eq 0 ]]
|
||||
|
||||
echo "CLI-only worker allocation passed"
|
||||
@@ -450,14 +450,23 @@ fi
|
||||
|
||||
worker_count=$(awk -F= '/^[[:space:]]*WORKER_COUNT[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2; exit}' /etc/uce/settings.cfg 2>/dev/null || true)
|
||||
worker_count="${worker_count:-4}"
|
||||
cli_worker_count=$(awk -F= '/^[[:space:]]*CLI_WORKER_COUNT[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); value=$2} END{print value}' /etc/uce/settings.cfg 2>/dev/null || true)
|
||||
cli_worker_count="${cli_worker_count:-0}"
|
||||
cli_worker_max_requests=$(awk -F= '/^[[:space:]]*CLI_WORKER_MAX_REQUESTS[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); value=$2} END{print value}' /etc/uce/settings.cfg 2>/dev/null || true)
|
||||
cli_worker_max_requests="${cli_worker_max_requests:-8}"
|
||||
worker_pids=""
|
||||
for _ in {1..48}; do
|
||||
output=$(scripts/uce-cli "/$test_name/parent.uce")
|
||||
worker_pids+="${output##*:}"$'\n'
|
||||
done
|
||||
unique_workers=$(printf '%s' "$worker_pids" | sed '/^$/d' | sort -u | wc -l)
|
||||
if (( unique_workers > worker_count )); then
|
||||
echo "worker pool recycled during 48 requests: $unique_workers PIDs for $worker_count workers" >&2
|
||||
if (( cli_worker_count > 0 && cli_worker_max_requests > 0 )); then
|
||||
if (( unique_workers <= cli_worker_count )); then
|
||||
echo "dedicated CLI pool did not recycle during 48 requests: $unique_workers PIDs for $cli_worker_count workers" >&2
|
||||
exit 1
|
||||
fi
|
||||
elif (( unique_workers > worker_count )); then
|
||||
echo "shared worker pool recycled during 48 requests: $unique_workers PIDs for $worker_count workers" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -43,6 +43,8 @@ SITE_DIRECTORY=$site
|
||||
FCGI_SOCKET_PATH=$root/run/fastcgi.sock
|
||||
FCGI_PORT=
|
||||
CLI_SOCKET_PATH=$socket
|
||||
CLI_WORKER_COUNT=1
|
||||
CLI_WORKER_MAX_REQUESTS=0
|
||||
WS_BROKER_SOCKET_PATH=$root/run/ws.sock
|
||||
HTTP_PORT=
|
||||
HTTP_DOCUMENT_ROOT=$site
|
||||
|
||||
@@ -134,6 +134,15 @@ printf '0\n' >"$root/maximum"
|
||||
flock -u 7
|
||||
mapfile -t scanner_pids < <(awk -F '\t' -v site="$site/" '$3 ~ ("^" site "unit-[0-9]+[.]uce$") { count[$4]++ } END { for(pid in count) if(count[pid] >= 2) print pid }' "$shim_log" | sort -n)
|
||||
[[ "${#scanner_pids[@]}" -eq 2 ]] || { echo "controlled units were not split between both scanners" >&2; cat "$shim_log" >&2; exit 1; }
|
||||
assert_scanner_memory_bounded() {
|
||||
for scanner_pid in "${scanner_pids[@]}"; do
|
||||
retained_kb=$(awk '/^(VmRSS|VmSwap):/{total += $2} END{print total + 0}' "/proc/$scanner_pid/status")
|
||||
# Scanner bookkeeping is small; 64 MiB leaves ample headroom while still
|
||||
# catching the measured old 12-serialization retention (~73 MiB).
|
||||
(( retained_kb < 65536 )) || { echo "proactive scanner retained serialized-module arenas: pid=$scanner_pid retained_kb=$retained_kb" >&2; exit 1; }
|
||||
done
|
||||
}
|
||||
assert_scanner_memory_bounded
|
||||
blockers=()
|
||||
victims=()
|
||||
for scanner_pid in "${scanner_pids[@]}"; do
|
||||
@@ -202,4 +211,8 @@ while [[ ! -s "$artifacts/priority.uce.cwasm" ]] && (( SECONDS < deadline )); do
|
||||
priority_nice=$(awk -F '\t' -v path="$site/priority.uce" '$3 == path { print $2; exit }' "$shim_log")
|
||||
[[ "$priority_nice" == "5" ]] || { echo "priority queue was not owned by the nice-5 compiler: $priority_nice" >&2; cat "$shim_log" >&2; exit 1; }
|
||||
|
||||
# Recheck after dependency fanout, failure/retry, recovery, and priority work;
|
||||
# the historical leak accumulated in the long-lived scanners across phases.
|
||||
assert_scanner_memory_bounded
|
||||
|
||||
printf '%s\n' 'parallel proactive compile passed'
|
||||
|
||||
@@ -38,7 +38,8 @@ socket_inode=$(stat -c %i "$socket_path")
|
||||
invoke() {
|
||||
local output="$1"
|
||||
shift
|
||||
timeout --signal=TERM --kill-after=1s 2s unshare --mount --fork \
|
||||
local timeout_seconds="${INVOKE_TIMEOUT_SECONDS:-2}"
|
||||
timeout --signal=TERM --kill-after=1s "$timeout_seconds" unshare --mount --fork \
|
||||
bash -c 'mount --bind "$1" /etc/uce/settings.cfg; exec "$2" "${@:3}"' \
|
||||
_ "$cfg" "$binary" "$@" >"$output.stdout" 2>"$output.stderr"
|
||||
}
|
||||
@@ -47,10 +48,11 @@ for option in --help -h; do
|
||||
invoke "$root/help" "$option"
|
||||
grep -q '^Usage: uce_fastcgi' "$root/help.stdout"
|
||||
grep -q -- '--precompile' "$root/help.stdout"
|
||||
grep -q -- '--serialize-module' "$root/help.stdout"
|
||||
[[ ! -s "$root/help.stderr" ]]
|
||||
done
|
||||
|
||||
for arguments in '--unknown' '--precompile extra' '--help extra'; do
|
||||
for arguments in '--unknown' '--precompile extra' '--serialize-module' '--help extra'; do
|
||||
read -r -a argv <<<"$arguments"
|
||||
set +e
|
||||
invoke "$root/invalid" "${argv[@]}"
|
||||
@@ -62,6 +64,36 @@ for arguments in '--unknown' '--precompile extra' '--help extra'; do
|
||||
[[ ! -s "$root/invalid.stdout" ]]
|
||||
done
|
||||
|
||||
cp bin/wasm/core.wasm "$root/serialize.wasm"
|
||||
INVOKE_TIMEOUT_SECONDS=30 invoke "$root/serialize" --serialize-module "$root/serialize.wasm"
|
||||
[[ -s "$root/serialize.cwasm" ]]
|
||||
[[ ! -s "$root/serialize.stderr" ]]
|
||||
|
||||
# Serialization shares the unit compile lock. If the wasm path is replaced
|
||||
# while the serializer is waiting, it must read the replacement and must not
|
||||
# publish a stale native artifact for the old inode.
|
||||
cp bin/wasm/core.wasm "$root/race.wasm"
|
||||
exec 8>"$root/race.wasm.lock"
|
||||
flock 8
|
||||
INVOKE_TIMEOUT_SECONDS=30 invoke "$root/race" --serialize-module "$root/race.wasm" &
|
||||
race_pid=$!
|
||||
sleep 0.1
|
||||
kill -0 "$race_pid"
|
||||
printf 'not wasm\n' >"$root/race.wasm.next"
|
||||
mv "$root/race.wasm.next" "$root/race.wasm"
|
||||
flock -u 8
|
||||
set +e
|
||||
wait "$race_pid"
|
||||
race_rc=$?
|
||||
set -e
|
||||
[[ $race_rc -eq 1 ]]
|
||||
grep -Eqi 'wasm|magic|WebAssembly|compile' "$root/race.stderr"
|
||||
[[ ! -e "$root/race.cwasm" ]]
|
||||
if compgen -G "$root/race.cwasm.*.tmp" >/dev/null; then
|
||||
echo "failed serialization left a temporary artifact" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[[ -S "$socket_path" ]]
|
||||
[[ "$(stat -c %i "$socket_path")" == "$socket_inode" ]]
|
||||
kill -0 "$listener_pid"
|
||||
|
||||
@@ -31,7 +31,7 @@ cleanup() {
|
||||
}
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$site/components" "$work" "$root/run" "$root/session" "$root/upload"
|
||||
sed -E '/^[[:space:]]*(BIN_DIRECTORY|PRECOMPILE_FILES_IN|SITE_DIRECTORY|FCGI_SOCKET_PATH|FCGI_PORT|CLI_SOCKET_PATH|WS_BROKER_SOCKET_PATH|HTTP_PORT|HTTP_DOCUMENT_ROOT|SESSION_PATH|TMP_UPLOAD_PATH|WASM_CORE_PATH|WASM_COMPILE_SCRIPT|WASM_INVOCATION_TIMEOUT_MS|WASM_EPOCH_PERIOD_MS|PROACTIVE_COMPILE_ENABLED|WORKER_COUNT|COMPILE_FAILURE_RETRY_SECONDS)[[:space:]]*=/d' \
|
||||
sed -E '/^[[:space:]]*(BIN_DIRECTORY|PRECOMPILE_FILES_IN|SITE_DIRECTORY|FCGI_SOCKET_PATH|FCGI_PORT|CLI_SOCKET_PATH|CLI_WORKER_COUNT|CLI_WORKER_MAX_REQUESTS|WS_BROKER_SOCKET_PATH|HTTP_PORT|HTTP_DOCUMENT_ROOT|SESSION_PATH|TMP_UPLOAD_PATH|WASM_CORE_PATH|WASM_COMPILE_SCRIPT|WASM_INVOCATION_TIMEOUT_MS|WASM_EPOCH_PERIOD_MS|PROACTIVE_COMPILE_ENABLED|WORKER_COUNT|COMPILE_FAILURE_RETRY_SECONDS)[[:space:]]*=/d' \
|
||||
/etc/uce/settings.cfg >"$settings"
|
||||
cat >>"$settings" <<CFG
|
||||
BIN_DIRECTORY=$work
|
||||
@@ -40,6 +40,8 @@ SITE_DIRECTORY=$site
|
||||
FCGI_SOCKET_PATH=$root/run/fastcgi.sock
|
||||
FCGI_PORT=
|
||||
CLI_SOCKET_PATH=$root/run/cli.sock
|
||||
CLI_WORKER_COUNT=1
|
||||
CLI_WORKER_MAX_REQUESTS=0
|
||||
WS_BROKER_SOCKET_PATH=$root/run/ws.sock
|
||||
HTTP_PORT=
|
||||
HTTP_DOCUMENT_ROOT=$site
|
||||
|
||||
@@ -21,11 +21,20 @@ if [[ -r "$settings_file" ]]; then
|
||||
[[ -n "${UCE_CLI_SOCKET:-}" ]] || socket_path=$(awk -F= '/^[[:space:]]*CLI_SOCKET_PATH[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2; exit}' "$settings_file")
|
||||
[[ -n "${BIN_DIRECTORY:-}" ]] || bin_directory=$(awk -F= '/^[[:space:]]*BIN_DIRECTORY[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2; exit}' "$settings_file")
|
||||
invocation_ms=$(awk -F= '/^[[:space:]]*WASM_INVOCATION_TIMEOUT_MS[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2; exit}' "$settings_file")
|
||||
cli_worker_count=$(awk -F= '/^[[:space:]]*CLI_WORKER_COUNT[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); value=$2} END{print value}' "$settings_file")
|
||||
fi
|
||||
site_directory="${site_directory:-site}"
|
||||
socket_path="${socket_path:-/run/uce/cli.sock}"
|
||||
bin_directory="${bin_directory:-/tmp/uce/work}"
|
||||
invocation_ms="${invocation_ms:-30000}"
|
||||
cli_worker_count="${cli_worker_count:-0}"
|
||||
curl_transport=(--unix-socket "$socket_path")
|
||||
test_base_url="http://localhost"
|
||||
if (( cli_worker_count > 0 )); then
|
||||
test_http_host="${UCE_TEST_HTTP_HOST:-uce.openfu.com}"
|
||||
curl_transport=(--resolve "$test_http_host:80:127.0.0.1")
|
||||
test_base_url="http://$test_http_host"
|
||||
fi
|
||||
test_name="invocation-timeout-test-$$"
|
||||
source_dir="$site_directory/$test_name"
|
||||
pid_file="/tmp/uce-$test_name-worker"
|
||||
@@ -42,13 +51,15 @@ mkdir -p "$source_dir"
|
||||
cache_dir="$(scripts/unit_cache_directory "$bin_directory")$(realpath "$source_dir")"
|
||||
|
||||
printf '%s\n' \
|
||||
'CLI(Request& context) {' \
|
||||
'void timeout_test_run(Request& context) {' \
|
||||
' if(context.get["warm"] == "1") { print("warm"); return; }' \
|
||||
" file_put_contents(\"$pid_file\", std::to_string(request_perf()[\"worker_pid\"].to_u64()));" \
|
||||
' while(true) time_precise();' \
|
||||
'}' >"$source_dir/hostcall-loop.uce"
|
||||
'}' \
|
||||
'CLI(Request& context) { timeout_test_run(context); }' \
|
||||
'RENDER(Request& context) { timeout_test_run(context); }' >"$source_dir/hostcall-loop.uce"
|
||||
printf '%s\n' \
|
||||
'CLI(Request& context) {' \
|
||||
'void timeout_test_run(Request& context) {' \
|
||||
' if(context.get["warm"] == "1") { print("warm"); return; }' \
|
||||
' if(context.get["quick"] == "1") { print(shell_exec("printf quick")); return; }' \
|
||||
' if(context.get["status"] == "1") { DValue spec; spec["cmd"] = "exit 7"; spec["timeout_ms"] = (f64)500; print(shell_exec(spec)["exit_code"].to_u64()); return; }' \
|
||||
@@ -56,46 +67,50 @@ printf '%s\n' \
|
||||
' if(context.get["job"] == "1") { DValue spec; spec["cmd"] = "sleep 2"; spec["timeout_ms"] = (f64)5000; u64 job = shell_spawn(spec); f64 started = time_precise(); job_await(job, 300); u64 elapsed = (u64)((time_precise() - started) * 1000); job_cancel(job); print(elapsed); return; }' \
|
||||
" file_put_contents(\"$pid_file\", std::to_string(request_perf()[\"worker_pid\"].to_u64()));" \
|
||||
' print(shell_exec("printf shell-start; sleep 60 & printf shell-end"));' \
|
||||
'}' >"$source_dir/legacy-shell.uce"
|
||||
'}' \
|
||||
'CLI(Request& context) { timeout_test_run(context); }' \
|
||||
'RENDER(Request& context) { timeout_test_run(context); }' >"$source_dir/legacy-shell.uce"
|
||||
printf '%s\n' \
|
||||
'CLI(Request& context) {' \
|
||||
'void timeout_test_run(Request& context) {' \
|
||||
' if(context.get["warm"] == "1") { print("warm"); return; }' \
|
||||
" file_put_contents(\"$pid_file\", std::to_string(request_perf()[\"worker_pid\"].to_u64()));" \
|
||||
' while(true) sleep(60);' \
|
||||
' print("sleep-end");' \
|
||||
'}' >"$source_dir/sleep.uce"
|
||||
'}' \
|
||||
'CLI(Request& context) { timeout_test_run(context); }' \
|
||||
'RENDER(Request& context) { timeout_test_run(context); }' >"$source_dir/sleep.uce"
|
||||
printf '%s\n' \
|
||||
'CLI(Request& context) { print(request_perf()["worker_pid"].to_u64(), "|health"); }' >"$source_dir/health.uce"
|
||||
'void timeout_test_run(Request& context) { print(request_perf()["worker_pid"].to_u64(), "|health"); }' \
|
||||
'CLI(Request& context) { timeout_test_run(context); }' \
|
||||
'RENDER(Request& context) { timeout_test_run(context); }' >"$source_dir/health.uce"
|
||||
|
||||
max_seconds=$(( (invocation_ms + 15000) / 1000 ))
|
||||
(( max_seconds >= 10 )) || max_seconds=10
|
||||
|
||||
request() {
|
||||
local unit="$1"
|
||||
curl -sS --max-time "$max_seconds" -o "$body_file" -w '%{http_code}' --unix-socket "$socket_path" "http://localhost/$test_name/$unit"
|
||||
curl -sS --max-time "$max_seconds" -o "$body_file" -w '%{http_code}' "${curl_transport[@]}" "$test_base_url/$test_name/$unit"
|
||||
}
|
||||
|
||||
same_worker_health() {
|
||||
local expected_pid="$1"
|
||||
for _ in $(seq 1 32); do
|
||||
local health
|
||||
health=$(curl -sS --max-time 5 --fail-with-body --unix-socket "$socket_path" "http://localhost/$test_name/health.uce")
|
||||
[[ "$health" == "$expected_pid|health" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
kill -0 "$expected_pid" 2>/dev/null || return 1
|
||||
local health
|
||||
health=$(curl -sS --max-time 5 --fail-with-body "${curl_transport[@]}" "$test_base_url/$test_name/health.uce")
|
||||
[[ "$health" =~ ^[0-9]+\|health$ ]]
|
||||
}
|
||||
|
||||
curl -sS --max-time "$max_seconds" --fail-with-body --unix-socket "$socket_path" "http://localhost/$test_name/hostcall-loop.uce?warm=1" >/dev/null
|
||||
curl -sS --max-time "$max_seconds" --fail-with-body --unix-socket "$socket_path" "http://localhost/$test_name/legacy-shell.uce?warm=1" >/dev/null
|
||||
curl -sS --max-time "$max_seconds" --fail-with-body --unix-socket "$socket_path" "http://localhost/$test_name/sleep.uce?warm=1" >/dev/null
|
||||
curl -sS --max-time "$max_seconds" --fail-with-body --unix-socket "$socket_path" "http://localhost/$test_name/health.uce" >/dev/null
|
||||
quick=$(curl -sS --max-time 5 --fail-with-body --unix-socket "$socket_path" "http://localhost/$test_name/legacy-shell.uce?quick=1")
|
||||
curl -sS --max-time "$max_seconds" --fail-with-body "${curl_transport[@]}" "$test_base_url/$test_name/hostcall-loop.uce?warm=1" >/dev/null
|
||||
curl -sS --max-time "$max_seconds" --fail-with-body "${curl_transport[@]}" "$test_base_url/$test_name/legacy-shell.uce?warm=1" >/dev/null
|
||||
curl -sS --max-time "$max_seconds" --fail-with-body "${curl_transport[@]}" "$test_base_url/$test_name/sleep.uce?warm=1" >/dev/null
|
||||
curl -sS --max-time "$max_seconds" --fail-with-body "${curl_transport[@]}" "$test_base_url/$test_name/health.uce" >/dev/null
|
||||
quick=$(curl -sS --max-time 5 --fail-with-body "${curl_transport[@]}" "$test_base_url/$test_name/legacy-shell.uce?quick=1")
|
||||
[[ "$quick" == quick ]] || { echo "quick legacy shell returned: $quick" >&2; exit 1; }
|
||||
exit_status=$(curl -sS --max-time 5 --fail-with-body --unix-socket "$socket_path" "http://localhost/$test_name/legacy-shell.uce?status=1")
|
||||
exit_status=$(curl -sS --max-time 5 --fail-with-body "${curl_transport[@]}" "$test_base_url/$test_name/legacy-shell.uce?status=1")
|
||||
[[ "$exit_status" == 7 ]] || { echo "structured shell lost exit status: $exit_status" >&2; exit 1; }
|
||||
zero_timeout=$(curl -sS --max-time 5 --fail-with-body --unix-socket "$socket_path" "http://localhost/$test_name/legacy-shell.uce?zero=1")
|
||||
zero_timeout=$(curl -sS --max-time 5 --fail-with-body "${curl_transport[@]}" "$test_base_url/$test_name/legacy-shell.uce?zero=1")
|
||||
[[ "$zero_timeout" == zero ]] || { echo "structured shell zero-timeout default changed: $zero_timeout" >&2; exit 1; }
|
||||
job_elapsed=$(curl -sS --max-time 5 --fail-with-body --unix-socket "$socket_path" "http://localhost/$test_name/legacy-shell.uce?job=1")
|
||||
job_elapsed=$(curl -sS --max-time 5 --fail-with-body "${curl_transport[@]}" "$test_base_url/$test_name/legacy-shell.uce?job=1")
|
||||
[[ "$job_elapsed" =~ ^[0-9]+$ && "$job_elapsed" -ge 200 && "$job_elapsed" -lt 500 ]] || { echo "job_await two-call duration was ${job_elapsed}ms" >&2; exit 1; }
|
||||
|
||||
for unit in hostcall-loop.uce legacy-shell.uce sleep.uce; do
|
||||
|
||||
@@ -29,7 +29,7 @@ cleanup() {
|
||||
}
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$site/components" "$work" "$root/run" "$root/session" "$root/upload"
|
||||
sed -E '/^[[:space:]]*(BIN_DIRECTORY|PRECOMPILE_FILES_IN|SITE_DIRECTORY|FCGI_SOCKET_PATH|FCGI_PORT|CLI_SOCKET_PATH|WS_BROKER_SOCKET_PATH|HTTP_PORT|HTTP_DOCUMENT_ROOT|SESSION_PATH|TMP_UPLOAD_PATH|WASM_CORE_PATH|WASM_INVOCATION_TIMEOUT_MS|WASM_EPOCH_PERIOD_MS|PROACTIVE_COMPILE_ENABLED|SERVE_LAST_KNOWN_GOOD|SHOW_DYNAMIC_COMPILE_ERRORS|WORKER_COUNT)[[:space:]]*=/d' \
|
||||
sed -E '/^[[:space:]]*(BIN_DIRECTORY|PRECOMPILE_FILES_IN|SITE_DIRECTORY|FCGI_SOCKET_PATH|FCGI_PORT|CLI_SOCKET_PATH|CLI_WORKER_COUNT|CLI_WORKER_MAX_REQUESTS|WS_BROKER_SOCKET_PATH|HTTP_PORT|HTTP_DOCUMENT_ROOT|SESSION_PATH|TMP_UPLOAD_PATH|WASM_CORE_PATH|WASM_INVOCATION_TIMEOUT_MS|WASM_EPOCH_PERIOD_MS|PROACTIVE_COMPILE_ENABLED|SERVE_LAST_KNOWN_GOOD|SHOW_DYNAMIC_COMPILE_ERRORS|WORKER_COUNT)[[:space:]]*=/d' \
|
||||
/etc/uce/settings.cfg >"$settings"
|
||||
cat >>"$settings" <<CFG
|
||||
BIN_DIRECTORY=$work
|
||||
@@ -38,6 +38,8 @@ SITE_DIRECTORY=$site
|
||||
FCGI_SOCKET_PATH=$root/run/fastcgi.sock
|
||||
FCGI_PORT=
|
||||
CLI_SOCKET_PATH=$socket
|
||||
CLI_WORKER_COUNT=1
|
||||
CLI_WORKER_MAX_REQUESTS=0
|
||||
WS_BROKER_SOCKET_PATH=$root/run/ws.sock
|
||||
HTTP_PORT=
|
||||
HTTP_DOCUMENT_ROOT=$site
|
||||
|
||||
Reference in New Issue
Block a user