udo/uce
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reject persisted failures during precompile

Browse Source
This commit is contained in:
udo 2026-07-18 15:44:44 +00:00
parent b9b70659a3
commit 22efea8ecd
4 changed files with 44 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
scripts/test_parallel_precompile.sh
View File

@ -46,7 +46,7 @@ write_units serial
serial_start=$(date +%s%N)
serial_output=$(UCE_PRECOMPILE_JOBS=1 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile)
serial_ns=$(( $(date +%s%N) - serial_start ))
if ! grep -Eq 'with 1 job: .* 4 compiled, 0 failed' <<<"$serial_output"; then
if ! grep -Eq 'with 1 job: .* 4 compiled, 0 failed, worker status ok' <<<"$serial_output"; then
echo "serial precompile did not compile the four controlled units" >&2
echo "$serial_output" >&2
exit 1
@ -61,7 +61,7 @@ if [[ $(grep -Ec '^Precompile worker [12]/2:' <<<"$parallel_output") -ne 2 ]]; t
echo "$parallel_output" >&2
exit 1
fi
if ! grep -Eq 'with 2 jobs: .* 4 compiled, 0 failed' <<<"$parallel_output"; then
if ! grep -Eq 'with 2 jobs: .* 4 compiled, 0 failed, worker status ok' <<<"$parallel_output"; then
echo "parallel precompile did not compile the four controlled units" >&2
echo "$parallel_output" >&2
exit 1
@ -76,7 +76,7 @@ done
printf 'CLI(Request& context) { print("parallel-precompile-race-0"); }\n' >"$source_dir/race-0.uce"
printf 'CLI(Request& context) { print("parallel-precompile-race-1"); }\n' >"$source_dir/race-1.uce"
race_output=$(UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-race" bin/uce_fastcgi.linux.bin --precompile)
if ! grep -Eq 'with 2 jobs: .* 2 compiled, 0 failed' <<<"$race_output" || \
if ! grep -Eq 'with 2 jobs: .* 2 compiled, 0 failed, worker status ok' <<<"$race_output" || \
[[ $(find "$artifact_dir/pch-race" -maxdepth 1 -type f -name '*.pch' | wc -l) -ne 1 ]] || \
find "$artifact_dir/pch-race" -maxdepth 1 -type f -name '*.tmp.*' -print -quit | grep -q .; then
echo "parallel precompile did not publish exactly one clean shared PCH" >&2
@ -90,13 +90,30 @@ set +e
failure_output=$(UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile 2>&1)
failure_rc=$?
set -e
if [[ $failure_rc -eq 0 ]] || ! grep -Eq 'with 2 jobs: .* 1 failed' <<<"$failure_output"; then
if [[ $failure_rc -eq 0 ]] || ! grep -Eq 'with 2 jobs: .* 1 failed, worker status failed' <<<"$failure_output"; then
echo "parallel precompile did not aggregate a controlled worker failure" >&2
echo "$failure_output" >&2
exit 1
fi
set +e
repeat_failure_output=$(UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile 2>&1)
repeat_failure_rc=$?
set -e
if [[ $repeat_failure_rc -eq 0 ]] || ! grep -Eq 'with 2 jobs: .* 1 failed, worker status failed' <<<"$repeat_failure_output"; then
echo "parallel precompile accepted a persisted current compile failure" >&2
echo "$repeat_failure_output" >&2
exit 1
fi
rm "$source_dir/broken.uce"
negative_jobs_output=$(UCE_PRECOMPILE_JOBS=-1 bin/uce_fastcgi.linux.bin --precompile)
malformed_jobs_output=$(UCE_PRECOMPILE_JOBS=invalid bin/uce_fastcgi.linux.bin --precompile)
oversized_jobs_output=$(UCE_PRECOMPILE_JOBS=17 bin/uce_fastcgi.linux.bin --precompile)
if ! grep -q 'with 1 job:' <<<"$negative_jobs_output" || ! grep -q 'with 2 jobs:' <<<"$malformed_jobs_output" || ! grep -q 'with 16 jobs:' <<<"$oversized_jobs_output"; then
echo "precompile job bounds did not map negative/malformed/oversized input to 1/2/16" >&2
exit 1
fi
printf 'parallel precompile passed: serial %.3fs, parallel %.3fs\n' \
"$(awk -v ns="$serial_ns" 'BEGIN { print ns / 1000000000 }')" \
"$(awk -v ns="$parallel_ns" 'BEGIN { print ns / 1000000000 }')"

4
src/lib/compiler.cpp
View File

@ -1447,7 +1447,7 @@ void compiler_untrack_known_unit(Request* context, String file_name)
});
}
bool compiler_unit_needs_recompile(Request* context, String file_name, bool* source_missing, bool allow_recent_source_stat, bool path_is_normalized)
bool compiler_unit_needs_recompile(Request* context, String file_name, bool* source_missing, bool allow_recent_source_stat, bool path_is_normalized, bool retry_current_failure)
{
if(!path_is_normalized)
file_name = compiler_normalize_unit_path(context, file_name);
@ -1459,7 +1459,7 @@ bool compiler_unit_needs_recompile(Request* context, String file_name, bool* sou
*source_missing = compile_check.source_missing;
if(compile_check.source_missing)
return(false);
if(compiler_failure_retry_deferred(context, &su, state))
if(!retry_current_failure && compiler_failure_retry_deferred(context, &su, state))
return(false);
return(compile_check.needs_compile);
}

2
src/lib/compiler.h
View File

@ -40,7 +40,7 @@ StringList compiler_list_known_units(Request* context);
void compiler_set_known_units(Request* context, StringList files);
void compiler_track_known_unit(Request* context, String file_name);
void compiler_untrack_known_unit(Request* context, String file_name);
bool compiler_unit_needs_recompile(Request* context, String file_name, bool* source_missing = 0, bool allow_recent_source_stat = false, bool path_is_normalized = false);
bool compiler_unit_needs_recompile(Request* context, String file_name, bool* source_missing = 0, bool allow_recent_source_stat = false, bool path_is_normalized = false, bool retry_current_failure = false);
DValue unit_info(String path = "");
StringList units_list();
bool unit_compile(String path = "");

27
src/linux_fastcgi.cpp
View File

@ -1219,14 +1219,14 @@ void proactive_compile_queue_push(StringList& queue, String file_name)
queue.push_back(file_name);
}
bool proactive_compile_unit(Request& context, String file_name, bool& source_missing)
bool proactive_compile_unit(Request& context, String file_name, bool& source_missing, bool force_compile = false)
{
bool failed = false;
String wasm_path = compiler_unit_wasm_path(&context, file_name);
if(compiler_unit_needs_recompile(&context, file_name, &source_missing))
if(force_compile || compiler_unit_needs_recompile(&context, file_name, &source_missing))
{
printf("(i) proactive compile %s\n", file_name.c_str());
auto su = get_shared_unit(&context, file_name);
auto su = force_compile ? compiler_get_shared_unit_internal(&context, file_name, true) : get_shared_unit(&context, file_name);
failed = !su || su->compiler_messages != "";
if(su)
wasm_path = su->wasm_name;
@ -1597,9 +1597,10 @@ PrecompileWorkerResult precompile_unit_range(Request& background_context, const
{
result.assigned++;
bool source_missing = false;
if(compiler_unit_needs_recompile(&background_context, files[i], &source_missing))
bool needs_compile = compiler_unit_needs_recompile(&background_context, files[i], &source_missing, false, true, true);
if(needs_compile)
result.compiled++;
if(proactive_compile_unit(background_context, files[i], source_missing))
if(proactive_compile_unit(background_context, files[i], source_missing, needs_compile))
result.failed++;
}
return(result);
@ -1653,7 +1654,16 @@ int precompile_unit_generation()
auto files = compiler_scan_site_units(&background_context);
compiler_set_known_units(&background_context, files);
const char* jobs_env = getenv("UCE_PRECOMPILE_JOBS");
u64 jobs = to_u64(jobs_env && jobs_env[0] != '\0' ? String(jobs_env) : server_state.config["PRECOMPILE_JOBS"], 2);
String jobs_text = trim(jobs_env && jobs_env[0] != '\0' ? String(jobs_env) : server_state.config["PRECOMPILE_JOBS"]);
u64 jobs = 2;
if(jobs_text != "")
{
char* end = 0;
errno = 0;
long long parsed = strtoll(jobs_text.c_str(), &end, 10);
if(end != jobs_text.c_str() && end && *end == '\0' && errno != ERANGE)
jobs = parsed < 1 ? 1 : (u64)parsed;
}
jobs = std::max<u64>(1, std::min<u64>(jobs, std::min<u64>(files.size() == 0 ? 1 : files.size(), 16)));
PrecompileWorkerResult total;
bool worker_error = false;
@ -1717,10 +1727,13 @@ int precompile_unit_generation()
if(total.assigned != files.size())
worker_error = true;
}
printf("Precompiled unit generation %s with %llu job%s: %zu units, %llu compiled, %llu failed in %.3f s\n",
if(worker_error)
printf("(!) precompile worker or result reporting failed; candidate generation rejected\n");
printf("Precompiled unit generation %s with %llu job%s: %zu units, %llu compiled, %llu failed, worker status %s in %.3f s\n",
compiler_unit_bin_directory(&background_context).c_str(),
(unsigned long long)jobs, jobs == 1 ? "" : "s", files.size(),
(unsigned long long)total.compiled, (unsigned long long)total.failed,
worker_error ? "failed" : "ok",
time_precise() - started_at);
return(total.failed == 0 && !worker_error ? 0 : 1);
}