udo/uce
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Avoid duplicate proactive recompilation

Browse Source
This commit is contained in:
udo
2026-07-18 22:23:44 +00:00
parent 33a7b00c78
commit 14f4920ae8
3 changed files with 53 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test_parallel_proactive_compile.sh
+37 -1
View File
@@ -66,8 +66,21 @@ maximum=0
if [[ -r "\$root/maximum" ]]; then read -r maximum <"\$root/maximum"; fi
if (( active > maximum )); then printf '%s\n' "\$active" >"\$root/maximum"; fi
nice_value=\$(timeout 2s ps -o ni= -p "\$PPID" | tr -d ' ')
printf '%s\t%s\t%s\n' "\$(date +%s%N)" "\$nice_value" "\$source_file" >>"\$root/compile.tsv"
owner_pid=\$PPID
for _ in 1 2 3 4; do
owner_command=\$(timeout 2s ps -o comm= -p "\$owner_pid" | tr -d ' ')
[[ "\$owner_command" == uce_fastcgi.* ]] && break
owner_pid=\$(timeout 2s ps -o ppid= -p "\$owner_pid" | tr -d ' ')
[[ -n "\$owner_pid" && "\$owner_pid" != "1" ]] || break
done
printf '%s\t%s\t%s\t%s\n' "\$(date +%s%N)" "\$nice_value" "\$source_file" "\$owner_pid" >>"\$root/compile.tsv"
flock -u 9
if [[ -r "\$root/hold-paths" ]] && grep -Fxq "\$source_file" "\$root/hold-paths"; then
: >"\$root/held-\$owner_pid"
deadline=\$((SECONDS + 20))
while [[ ! -e "\$root/release" && \$SECONDS -lt \$deadline ]]; do sleep 0.05; done
[[ -e "\$root/release" ]] || exit 98
fi
sleep 0.75
"\$real" "\$@"
rc=\$?
@@ -118,7 +131,26 @@ exec 7>>"$root/counter.lock"
flock 7
printf '0\n' >"$root/maximum"
flock -u 7
mapfile -t scanner_pids < <(awk -F '\t' -v site="$site/" '$3 ~ ("^" site "unit-[0-9]+[.]uce$") { count[$4]++ } END { for(pid in count) if(count[pid] >= 2) print pid }' "$shim_log" | sort -n)
[[ "${#scanner_pids[@]}" -eq 2 ]] || { echo "controlled units were not split between both scanners" >&2; cat "$shim_log" >&2; exit 1; }
blockers=()
victims=()
for scanner_pid in "${scanner_pids[@]}"; do
mapfile -t scanner_units < <(awk -F '\t' -v pid="$scanner_pid" -v site="$site/" '$4 == pid && $3 ~ ("^" site "unit-[0-9]+[.]uce$") { print $3 }' "$shim_log")
blockers+=("${scanner_units[0]}")
victims+=("${scanner_units[1]}")
done
printf '%s\n' "${blockers[@]}" >"$root/hold-paths"
rm -f "$root"/held-* "$root/release"
sed -i 's/common-a/common-b/' "$site/common.uce"
deadline=$((SECONDS + 20))
while { [[ ! -e "$root/held-${scanner_pids[0]}" ]] || [[ ! -e "$root/held-${scanner_pids[1]}" ]]; } && (( SECONDS < deadline )); do sleep 0.05; done
[[ -e "$root/held-${scanner_pids[0]}" && -e "$root/held-${scanner_pids[1]}" ]] || { echo "both scanners did not reach their controlled queue barriers" >&2; cat "$log" >&2; exit 1; }
for victim in "${victims[@]}"; do
victim_marker=$(UCE_CLI_SOCKET="$root/run/cli.sock" timeout 20s scripts/uce-cli "/$(basename "$victim")")
[[ "$victim_marker" == *"common-b-"* ]] || { echo "request worker did not publish queued victim: $victim_marker" >&2; exit 1; }
done
touch "$root/release"
deadline=$((SECONDS + 30))
while (( SECONDS < deadline )); do
rebuilt=1
@@ -129,9 +161,13 @@ while (( SECONDS < deadline )); do
sleep 0.1
done
(( rebuilt == 1 )) || { echo "common dependency edit did not rebuild all scanner shards" >&2; cat "$log" >&2; exit 1; }
deadline=$((SECONDS + 5))
while [[ "$(<"$root/active")" != "0" ]] && (( SECONDS < deadline )); do sleep 0.1; done
sleep 1.25
for unit in 0 1 2 3; do
[[ "$(grep -c "$site/unit-$unit.uce" "$shim_log")" -eq 2 ]] || { echo "unit-$unit common-dependency rebuild was not exact" >&2; cat "$shim_log" >&2; exit 1; }
done
rm -f "$root/hold-paths" "$root"/held-* "$root/release"
[[ "$(<"$root/maximum")" -ge 2 ]] || { echo "common dependency fanout did not overlap across scanner owners" >&2; cat "$shim_log" >&2; exit 1; }
marker=$(UCE_CLI_SOCKET="$root/run/cli.sock" timeout 20s scripts/uce-cli /unit-0.uce)
[[ "$marker" == *"common-b-0"* ]] || { echo "rebuilt parent did not execute the changed dependency: $marker" >&2; exit 1; }
src/lib/compiler.cpp
+9 -8
View File
@@ -1039,20 +1039,21 @@ void compile_shared_unit(Request* context, SharedUnit* su)
compiler_mark_source_generation(context);
}
SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name, bool force_recompile)
SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name, bool force_recompile, bool retry_current_failure = false)
{
file_name = compiler_normalize_unit_path(context, file_name);
bool bypass_cached_result = force_recompile || retry_current_failure;
auto cached = compiler_reusable_cached_unit(context, file_name, force_recompile);
auto cached = compiler_reusable_cached_unit(context, file_name, bypass_cached_result);
if(cached)
return(cached);
compiler_release_cached_unit_if_needed(context, file_name, force_recompile);
compiler_release_cached_unit_if_needed(context, file_name, bypass_cached_result);
SharedUnit* su = new SharedUnit();
setup_unit_paths(context, su, file_name);
bool can_serve_stale = !force_recompile && compiler_unit_can_serve_stale_artifact(context, file_name);
bool can_serve_stale = !bypass_cached_result && compiler_unit_can_serve_stale_artifact(context, file_name);
int fdlock = compiler_open_lock_file(su->wasm_name + ".lock", "shared-unit:" + file_name, can_serve_stale);
if(fdlock == -2 && file_exists(su->wasm_name))
{
@@ -1076,7 +1077,7 @@ SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name
return(su);
}
cached = compiler_reusable_cached_unit(context, file_name, force_recompile);
cached = compiler_reusable_cached_unit(context, file_name, bypass_cached_result);
if(cached)
{
compiler_close_lock_file(fdlock);
@@ -1084,7 +1085,7 @@ SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name
return(cached);
}
compiler_release_cached_unit_if_needed(context, file_name, force_recompile);
compiler_release_cached_unit_if_needed(context, file_name, bypass_cached_result);
auto state = inspect_shared_unit_filesystem(context, su);
auto compile_check = shared_unit_compile_check(state);
@@ -1093,9 +1094,9 @@ SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name
bool do_recompile = force_recompile || compile_check.needs_compile;
if(do_recompile)
{
if(!force_recompile && retry_deferred)
if(!force_recompile && !retry_current_failure && retry_deferred)
compiler_restore_persisted_failure(su, state);
else if(!force_recompile && !jit_enabled)
else if(!force_recompile && !retry_current_failure && !jit_enabled)
{
compiler_restore_persisted_failure(su, state, "jit_compile_disabled");
if(trim(su->compiler_messages) == "")
src/linux_fastcgi.cpp
+7 -4
View File
@@ -1245,14 +1245,14 @@ void proactive_compile_queue_push(StringList& queue, String file_name)
queue.push_back(file_name);
}
bool proactive_compile_unit(Request& context, String file_name, bool& source_missing, bool force_compile = false)
bool proactive_compile_unit(Request& context, String file_name, bool& source_missing, bool retry_current_failure = false)
{
bool failed = false;
String wasm_path = compiler_unit_wasm_path(&context, file_name);
if(force_compile || compiler_unit_needs_recompile(&context, file_name, &source_missing))
if(retry_current_failure || compiler_unit_needs_recompile(&context, file_name, &source_missing))
{
printf("(i) proactive compile %s\n", file_name.c_str());
auto su = force_compile ? compiler_get_shared_unit_internal(&context, file_name, true) : get_shared_unit(&context, file_name);
auto su = retry_current_failure ? compiler_get_shared_unit_internal(&context, file_name, false, true) : get_shared_unit(&context, file_name);
failed = !su || su->compiler_messages != "";
if(su)
wasm_path = su->wasm_name;
@@ -1383,7 +1383,10 @@ void run_proactive_compiler(u64 worker, u64 jobs)
if(retry_it != retry_after.end() && time_precise() < retry_it->second)
continue;
bool normal_compile = compiler_unit_needs_recompile(&background_context, file_name, &source_missing);
bool failed = proactive_compile_unit(background_context, file_name, source_missing, !normal_compile && !source_missing);
bool retry_due = retry_it != retry_after.end();
bool retry_current_failure = retry_due && !normal_compile && !source_missing &&
compiler_unit_needs_recompile(&background_context, file_name, 0, false, true, true);
bool failed = proactive_compile_unit(background_context, file_name, source_missing, retry_current_failure);
if(source_missing)
retry_after.erase(file_name);
else if(failed)