Reject precompile generations changed in flight

scripts/test_parallel_precompile.sh

@@ -16,15 +16,24 @@ fi
 source_dir="$site_directory/$test_name"
 absolute_source_dir=""
 artifact_dir=""
+generation_log=""
+precompile_timeout="${UCE_TEST_PRECOMPILE_TIMEOUT:-120s}"
+
+precompile() {
+	timeout "$precompile_timeout" env "$@" bin/uce_fastcgi.linux.bin --precompile
+}
 
 cleanup() {
 	rm -rf "$source_dir"
 	if [[ -x bin/uce_fastcgi.linux.bin ]]; then
-		UCE_PRECOMPILE_JOBS=1 bin/uce_fastcgi.linux.bin --precompile >/dev/null 2>&1 || true
+		precompile UCE_PRECOMPILE_JOBS=1 >/dev/null 2>&1 || true
 	fi
 	if [[ -n "$artifact_dir" ]]; then
 		rm -rf "$artifact_dir"
 	fi
+	if [[ -n "$generation_log" ]]; then
+		rm -f "$generation_log"
+	fi
 }
 trap cleanup EXIT
 mkdir -p "$source_dir"
@@ -39,12 +48,12 @@ write_units() {
 }
 
 printf 'CLI(Request& context) { print("parallel-precompile-warmup"); }\n' >"$source_dir/warmup.uce"
-UCE_PRECOMPILE_JOBS=1 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile >/dev/null
+precompile UCE_PRECOMPILE_JOBS=1 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" >/dev/null
 rm "$source_dir/warmup.uce"
 
 write_units serial
 serial_start=$(date +%s%N)
-serial_output=$(UCE_PRECOMPILE_JOBS=1 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile)
+serial_output=$(precompile UCE_PRECOMPILE_JOBS=1 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared")
 serial_ns=$(( $(date +%s%N) - serial_start ))
 if ! grep -Eq 'with 1 job: .* 4 compiled, 0 failed, worker status ok' <<<"$serial_output"; then
 	echo "serial precompile did not compile the four controlled units" >&2
@@ -54,7 +63,7 @@ fi
 
 write_units parallel
 parallel_start=$(date +%s%N)
-parallel_output=$(UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile)
+parallel_output=$(precompile UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared")
 parallel_ns=$(( $(date +%s%N) - parallel_start ))
 if [[ $(grep -Ec '^Precompile worker [12]/2:' <<<"$parallel_output") -ne 2 ]]; then
 	echo "parallel precompile did not report both workers" >&2
@@ -75,7 +84,7 @@ done
 
 printf 'CLI(Request& context) { print("parallel-precompile-race-0"); }\n' >"$source_dir/race-0.uce"
 printf 'CLI(Request& context) { print("parallel-precompile-race-1"); }\n' >"$source_dir/race-1.uce"
-race_output=$(UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-race" bin/uce_fastcgi.linux.bin --precompile)
+race_output=$(precompile UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-race")
 if ! grep -Eq 'with 2 jobs: .* 2 compiled, 0 failed, worker status ok' <<<"$race_output" || \
 	[[ $(find "$artifact_dir/pch-race" -maxdepth 1 -type f -name '*.pch' | wc -l) -ne 1 ]] || \
 	find "$artifact_dir/pch-race" -maxdepth 1 -type f -name '*.tmp.*' -print -quit | grep -q .; then
@@ -85,9 +94,52 @@ if ! grep -Eq 'with 2 jobs: .* 2 compiled, 0 failed, worker status ok' <<<"$race
 fi
 rm "$source_dir/race-0.uce" "$source_dir/race-1.uce"
 
+for unit in $(seq -w 0 9); do
+	printf 'CLI(Request& context) { print("parallel-precompile-generation-%s"); }\n' "$unit" >"$source_dir/generation-$unit.uce"
+done
+generation_log="/tmp/$test_name-generation.log"
+precompile UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" >"$generation_log" 2>&1 &
+generation_pid=$!
+generation_deadline=$((SECONDS + 60))
+while [[ ! -s "$artifact_dir/generation-0.uce.meta.txt" ]]; do
+	if ! kill -0 "$generation_pid" 2>/dev/null; then
+		echo "parallel precompile exited before the controlled source mutation" >&2
+		wait "$generation_pid" || true
+		cat "$generation_log" >&2
+		exit 1
+	fi
+	if (( SECONDS >= generation_deadline )); then
+		echo "timed out waiting for the controlled precompile mutation boundary" >&2
+		exit 1
+	fi
+	sleep 0.05
+done
+printf 'CLI(Request& context) { print("parallel-precompile-generation-mutated"); }\n' >"$source_dir/generation-0.uce"
+printf 'CLI(Request& context) { print("parallel-precompile-generation-added"); }\n' >"$source_dir/generation-new.uce"
+if ! kill -0 "$generation_pid" 2>/dev/null; then
+	echo "parallel precompile exited at the controlled source mutation boundary" >&2
+	exit 1
+fi
+set +e
+wait "$generation_pid"
+generation_rc=$?
+set -e
+if [[ $generation_rc -eq 0 ]] || ! grep -q 'source generation changed during precompile; candidate generation rejected' "$generation_log"; then
+	echo "parallel precompile accepted a source generation that changed during compilation" >&2
+	cat "$generation_log" >&2
+	exit 1
+fi
+generation_retry_output=$(precompile UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared")
+if ! grep -Eq 'with 2 jobs: .* 2 compiled, 0 failed, worker status ok' <<<"$generation_retry_output"; then
+	echo "parallel precompile did not reconcile the changed and added units on retry" >&2
+	echo "$generation_retry_output" >&2
+	exit 1
+fi
+rm -f "$generation_log" "$source_dir"/generation-*.uce
+
 printf 'CLI(Request& context) { print(deliberate_parallel_precompile_failure); }\n' >"$source_dir/broken.uce"
 set +e
-failure_output=$(UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile 2>&1)
+failure_output=$(precompile UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" 2>&1)
 failure_rc=$?
 set -e
 if [[ $failure_rc -eq 0 ]] || ! grep -Eq 'with 2 jobs: .* 1 failed, worker status failed' <<<"$failure_output"; then
@@ -96,7 +148,7 @@ if [[ $failure_rc -eq 0 ]] || ! grep -Eq 'with 2 jobs: .* 1 failed, worker statu
 	exit 1
 fi
 set +e
-repeat_failure_output=$(UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" bin/uce_fastcgi.linux.bin --precompile 2>&1)
+repeat_failure_output=$(precompile UCE_PRECOMPILE_JOBS=2 UCE_WASM_PCH_DIR="$artifact_dir/pch-shared" 2>&1)
 repeat_failure_rc=$?
 set -e
 if [[ $repeat_failure_rc -eq 0 ]] || ! grep -Eq 'with 2 jobs: .* 1 failed, worker status failed' <<<"$repeat_failure_output"; then
@@ -106,9 +158,9 @@ if [[ $repeat_failure_rc -eq 0 ]] || ! grep -Eq 'with 2 jobs: .* 1 failed, worke
 fi
 rm "$source_dir/broken.uce"
 
-negative_jobs_output=$(UCE_PRECOMPILE_JOBS=-1 bin/uce_fastcgi.linux.bin --precompile)
+negative_jobs_output=$(precompile UCE_PRECOMPILE_JOBS=-1)
-malformed_jobs_output=$(UCE_PRECOMPILE_JOBS=invalid bin/uce_fastcgi.linux.bin --precompile)
+malformed_jobs_output=$(precompile UCE_PRECOMPILE_JOBS=invalid)
-oversized_jobs_output=$(UCE_PRECOMPILE_JOBS=17 bin/uce_fastcgi.linux.bin --precompile)
+oversized_jobs_output=$(precompile UCE_PRECOMPILE_JOBS=17)
 if ! grep -q 'with 1 job:' <<<"$negative_jobs_output" || ! grep -q 'with 2 jobs:' <<<"$malformed_jobs_output" || ! grep -q 'with 16 jobs:' <<<"$oversized_jobs_output"; then
 	echo "precompile job bounds did not map negative/malformed/oversized input to 1/2/16" >&2
 	exit 1

src/linux_fastcgi.cpp

@@ -1727,15 +1727,36 @@ int precompile_unit_generation()
 		if(total.assigned != files.size())
 			worker_error = true;
 	}
+	auto final_files = compiler_scan_site_units(&background_context);
+	bool source_generation_changed = final_files != files;
+	if(!source_generation_changed)
+	{
+		for(auto& file_name : final_files)
+		{
+			bool source_missing = false;
+			if(compiler_unit_needs_recompile(&background_context, file_name, &source_missing, false, true, true) || source_missing)
+			{
+				source_generation_changed = true;
+				break;
+			}
+		}
+	}
+	if(source_generation_changed)
+	{
+		compiler_set_known_units(&background_context, final_files);
+		compiler_mark_source_generation(&background_context);
+		printf("(!) source generation changed during precompile; candidate generation rejected\n");
+	}
 	if(worker_error)
 		printf("(!) precompile worker or result reporting failed; candidate generation rejected\n");
+	bool candidate_rejected = worker_error || source_generation_changed;
 	printf("Precompiled unit generation %s with %llu job%s: %zu units, %llu compiled, %llu failed, worker status %s in %.3f s\n",
 		compiler_unit_bin_directory(&background_context).c_str(),
 		(unsigned long long)jobs, jobs == 1 ? "" : "s", files.size(),
 		(unsigned long long)total.compiled, (unsigned long long)total.failed,
-		worker_error ? "failed" : "ok",
+		candidate_rejected ? "failed" : "ok",
 		time_precise() - started_at);
-	return(total.failed == 0 && !worker_error ? 0 : 1);
+	return(total.failed == 0 && !candidate_rejected ? 0 : 1);
 }
 
 int main(int argc, char** argv)