diff --git a/docs/wasm-runtime-architecture.md b/docs/wasm-runtime-architecture.md index c3e4b7d..5844e68 100644 --- a/docs/wasm-runtime-architecture.md +++ b/docs/wasm-runtime-architecture.md @@ -136,6 +136,15 @@ refreshes the epoch deadline before its first guest call. Otherwise a component whose compilation outlasted the guest CPU budget would immediately trap in the following allocator/relocation call even though no guest loop consumed it. +The proactive compiler and request workers coordinate through a per-unit file +lock. Unit compilation writes and validates a process-unique temporary wasm +file, then publishes it with an atomic rename. While another process holds the +lock for a stale unit, a request may therefore keep using the last complete +artifact instead of waiting across a transitive rebuild. Once the lock is +released, normal freshness checks require the new artifact; a failed rebuild +removes availability and surfaces the compiler error rather than serving the +old unit indefinitely. + --- ## 4. The workspace runtime @@ -346,7 +355,10 @@ header free-functions are `inline`. The wasm backend exposes only declarations recycling. `scripts/test_cold_component_deadline.sh` separately compiles a deliberately cold component that exceeds the development epoch window and proves the - parent request still renders it. + parent request still renders it. The dependency-invalidation gate also holds + parent and child compile locks across a transitive source edit, proves a + warmed request returns the last atomically published result without waiting, + and then proves the new dependency result appears after rebuild completion. - **WebSocket end-to-end**: a headless client performs a raw WS handshake to `:HTTP_PORT` with path `/site/tests/websockets.ws.uce` (self-resolving diff --git a/scripts/compile_wasm_unit b/scripts/compile_wasm_unit index b2f16dd..0f16d02 100755 --- a/scripts/compile_wasm_unit +++ b/scripts/compile_wasm_unit @@ -15,6 +15,7 @@ ABI_VERSION=${UCE_UNIT_ABI_VERSION:-6} ROOT=$(pwd) OBJ_FN="$DEST_DIR/$PP_FN.wasm.o" ABI_TMP="$DEST_DIR/$PP_FN.uce-abi.txt" +WASM_TMP="$DEST_DIR/$WASM_FN.tmp.$$" PCH_ENABLED=${UCE_WASM_UNIT_PCH:-1} PCH_DIR=${UCE_WASM_PCH_DIR:-/tmp/uce/wasm-w2/pch} COMMON_FLAGS=( @@ -44,6 +45,7 @@ PCH_KEY=$(printf '%s\n%s\n%s\n%s\n' "$ABI_VERSION" "$TOOLCHAIN_ID" "$HEADER_HASH PCH_FN="$PCH_DIR/uce_lib-wasm-unit-$PCH_KEY.pch" mkdir -p "$DEST_DIR" >/dev/null 2>&1 +trap 'rm -f "$OBJ_FN" "$ABI_TMP" "$WASM_TMP"' EXIT build_pch_if_needed() { if [ "$PCH_ENABLED" = "0" ]; then @@ -80,7 +82,7 @@ fi "$SDK/bin/wasm-ld" -shared --experimental-pic \ --unresolved-symbols=import-dynamic \ --Bsymbolic \ - "$OBJ_FN" -o "$DEST_DIR/$WASM_FN" \ + "$OBJ_FN" -o "$WASM_TMP" \ --export-if-defined=__uce_set_current_request \ --export-if-defined=__uce_render \ --export-if-defined=__uce_component \ @@ -90,8 +92,9 @@ fi --export-if-defined=__uce_once \ --export-if-defined=__uce_init -"$SDK/bin/llvm-objcopy" --add-section=uce.abi="$ABI_TMP" "$DEST_DIR/$WASM_FN" +"$SDK/bin/llvm-objcopy" --add-section=uce.abi="$ABI_TMP" "$WASM_TMP" -python3 scripts/check_unit_wasm.py "$DEST_DIR/$WASM_FN" --abi-version "$ABI_VERSION" --llvm-nm "$SDK/bin/llvm-nm" +python3 scripts/check_unit_wasm.py "$WASM_TMP" --abi-version "$ABI_VERSION" --llvm-nm "$SDK/bin/llvm-nm" +mv "$WASM_TMP" "$DEST_DIR/$WASM_FN" rm -f "$OBJ_FN" "$ABI_TMP" diff --git a/scripts/test_dependency_invalidation.sh b/scripts/test_dependency_invalidation.sh index 03a7e1e..afe2704 100755 --- a/scripts/test_dependency_invalidation.sh +++ b/scripts/test_dependency_invalidation.sh @@ -48,13 +48,37 @@ assert_marker parent dependency-marker-a sed -i 's/dependency-marker-a/dependency-marker-b/' "$source_dir/child.uce" assert_marker parent dependency-marker-b +# A proactive rebuild owns these same per-unit locks. While it publishes fresh +# artifacts, requests must use the last complete artifacts instead of waiting +# across the transitive graph. Atomic publication keeps those artifacts safe. +parent_wasm="$cache_dir/parent.uce.wasm" +child_wasm="$cache_dir/child.uce.wasm" +( + exec 8>"$parent_wasm.lock" + exec 9>"$child_wasm.lock" + flock 8 + flock 9 + sleep 3 +) & +rebuild_lock_pid=$! +sleep 0.2 +sed -i 's/dependency-marker-b/dependency-marker-d/' "$source_dir/child.uce" +started_at=$(date +%s%N) +assert_marker parent dependency-marker-b +elapsed_ms=$(( ($(date +%s%N) - started_at) / 1000000 )) +if (( elapsed_ms >= 2000 )); then + echo "request waited ${elapsed_ms}ms for an active transitive rebuild" >&2 + exit 1 +fi +wait "$rebuild_lock_pid" +assert_marker parent dependency-marker-d + # Warm every configured worker, then replace the artifact while retaining its # whole-second mtime. The worker cache must notice the nanosecond/ctime change. -for _ in {1..16}; do assert_marker parent dependency-marker-b; done -sed 's/dependency-marker-b/dependency-marker-c/' "$source_dir/child.uce" >"$source_dir/alternate-child.uce" +for _ in {1..16}; do assert_marker parent dependency-marker-d; done +sed 's/dependency-marker-d/dependency-marker-c/' "$source_dir/child.uce" >"$source_dir/alternate-child.uce" sed 's/child.uce/alternate-child.uce/' "$source_dir/parent.uce" >"$source_dir/alternate.uce" assert_marker alternate dependency-marker-c -parent_wasm="$cache_dir/parent.uce.wasm" alternate_wasm="$cache_dir/alternate.uce.wasm" parent_mtime=$(stat -c %Y "$parent_wasm") cp "$alternate_wasm" "$parent_wasm" diff --git a/src/lib/compiler.cpp b/src/lib/compiler.cpp index c89bdbe..e4bb23a 100644 --- a/src/lib/compiler.cpp +++ b/src/lib/compiler.cpp @@ -296,7 +296,7 @@ String compiler_registry_lock_file_name(Request* context) return(compiler_registry_file_name(context) + ".lock"); } -int compiler_open_lock_file(String file_name, String purpose) +int compiler_open_lock_file(String file_name, String purpose, bool nonblocking = false) { (void)purpose; auto lock_dir = dirname(file_name); @@ -309,8 +309,13 @@ int compiler_open_lock_file(String file_name, String purpose) return(fdlock); } fcntl(fdlock, F_SETFD, FD_CLOEXEC); - if(flock(fdlock, LOCK_EX) != 0) + if(flock(fdlock, LOCK_EX | (nonblocking ? LOCK_NB : 0)) != 0) { + if(nonblocking && (errno == EWOULDBLOCK || errno == EAGAIN)) + { + close(fdlock); + return(-2); + } close(fdlock); printf("(!) Could not lock file %s\n", file_name.c_str()); return(-1); @@ -845,7 +850,19 @@ SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name SharedUnit* su = new SharedUnit(); setup_unit_paths(context, su, file_name); - int fdlock = compiler_open_lock_file(su->wasm_name + ".lock", "shared-unit:" + file_name); + int fdlock = compiler_open_lock_file(su->wasm_name + ".lock", "shared-unit:" + file_name, !force_recompile); + if(fdlock == -2 && file_exists(su->wasm_name)) + { + auto state = inspect_shared_unit_filesystem(context, su); + su->api_declarations = split(file_get_contents(su->api_file_name), "\n"); + su->last_compiled = state.compiled_time; + su->compile_status = "rebuilding"; + compiler_record_observed_filesystem_state(su, state); + context->server->units[file_name] = su; + return(su); + } + if(fdlock == -2) + fdlock = compiler_open_lock_file(su->wasm_name + ".lock", "shared-unit:" + file_name); if(fdlock == -1) { su->compiler_messages = "could not open compile lock"; @@ -947,6 +964,17 @@ bool compiler_unit_compile_pending(Request* context, String file_name) return(true); } +bool compiler_unit_compile_in_progress(Request* context, String file_name) +{ + SharedUnit su; + setup_unit_paths(context, &su, compiler_normalize_unit_path(context, file_name)); + int fdlock = compiler_open_lock_file(su.wasm_name + ".lock", "compile-probe", true); + if(fdlock == -2) + return(true); + compiler_close_lock_file(fdlock); + return(false); +} + void unit_render(String file_name) { unit_render(file_name, *context); diff --git a/src/lib/compiler.h b/src/lib/compiler.h index 23dc6e2..ad88f1a 100644 --- a/src/lib/compiler.h +++ b/src/lib/compiler.h @@ -17,6 +17,7 @@ void compile_shared_unit(Request* context, SharedUnit* su); SharedUnit* get_shared_unit(Request* context, String file_name); String compiler_error_page_unit(Request* context, String config_key); bool compiler_unit_compile_pending(Request* context, String file_name); +bool compiler_unit_compile_in_progress(Request* context, String file_name); String compiler_site_directory(Request* context); StringList compiler_scan_site_units(Request* context); StringList compiler_list_known_units(Request* context); diff --git a/src/wasm/backend.cpp b/src/wasm/backend.cpp index 4edd5e7..f22d349 100644 --- a/src/wasm/backend.cpp +++ b/src/wasm/backend.cpp @@ -100,7 +100,7 @@ static bool wasm_artifact_exists(Request* context, const String& entry_unit) // metadata mismatches, which can leave stale wasm with old imports. bool source_missing = false; if(compiler_unit_needs_recompile(context, entry_unit, &source_missing)) - return(false); + return(compiler_unit_compile_in_progress(context, entry_unit)); if(source_missing) return(false); return(true); diff --git a/src/wasm/worker.cpp b/src/wasm/worker.cpp index 1c86c32..3489fec 100644 --- a/src/wasm/worker.cpp +++ b/src/wasm/worker.cpp @@ -1606,7 +1606,7 @@ private: return(1); } - if(!file_exists_host(worker.unit_wasm_path(resolved)) || compiler_unit_needs_recompile(context, resolved, 0)) + if(!file_exists_host(worker.unit_wasm_path(resolved)) || (compiler_unit_needs_recompile(context, resolved, 0) && !compiler_unit_compile_in_progress(context, resolved))) get_shared_unit(context, resolved); size_t unit_index = 0;