Stage ABI-scoped unit generations

src/lib/compiler.cpp

@@ -1,6 +1,7 @@
 #include "compiler.h"
 #include "compiler-parser.h"
 #include "hash.h"
+#include "../wasm/abi.h"
 #include <algorithm>
 #include <cerrno>
 #include <chrono>
@@ -16,7 +17,7 @@
 
 namespace {
 
-const u64 UCE_UNIT_ABI_VERSION = 11;
+const u64 UCE_UNIT_ABI_VERSION = UCE_COMPILER_UNIT_ABI_VERSION;
 
 struct SharedUnitFilesystemState
 {
@@ -35,6 +36,8 @@ struct SharedUnitFilesystemState
 	time_t required_time = 0;
 	u64 metadata_abi_version = 0;
 	u64 runtime_abi_version = UCE_UNIT_ABI_VERSION;
+	u64 metadata_wasm_core_abi_version = 0;
+	u64 runtime_wasm_core_abi_version = UCE_WASM_CORE_ABI_VERSION;
 	String metadata_content;
 	String compile_output_content;
 	String metadata_build_token;
@@ -262,7 +265,8 @@ String compiler_unit_input_signature(Request* context, SharedUnit* su, bool allo
 	return(
 		compiler_unit_source_signature(su->file_name, allow_recent_source_stat) + ":" +
 		gen_sha1(file_get_contents(setup_template)) + ":" +
-		std::to_string(UCE_UNIT_ABI_VERSION)
+		std::to_string(UCE_UNIT_ABI_VERSION) + ":" +
+		std::to_string(UCE_WASM_CORE_ABI_VERSION)
 	);
 }
 
@@ -281,6 +285,7 @@ String compiler_unit_metadata_text(Request* context, SharedUnit* su, String inpu
 	return(
 		"format=uce-unit-metadata-v1\n"
 		"unit_abi_version=" + std::to_string(UCE_UNIT_ABI_VERSION) + "\n"
+		"wasm_core_abi_version=" + std::to_string(UCE_WASM_CORE_ABI_VERSION) + "\n"
 		"source_path=" + su->file_name + "\n"
 		"input_signature=" + input_signature + "\n"
 		"build_token=" + compiler_unit_build_token() + "\n"
@@ -387,7 +392,7 @@ time_t compiler_runtime_abi_time(Request* context)
 
 String compiler_registry_file_name(Request* context)
 {
-	return(context->server->config["BIN_DIRECTORY"] + "/known-uce-files.txt");
+	return(compiler_unit_bin_directory(context) + "/known-uce-files.txt");
 }
 
 String compiler_registry_lock_file_name(Request* context)
@@ -397,12 +402,12 @@ String compiler_registry_lock_file_name(Request* context)
 
 String compiler_priority_file_name(Request* context)
 {
-	return(context->server->config["BIN_DIRECTORY"] + "/proactive-priority.txt");
+	return(compiler_unit_bin_directory(context) + "/proactive-priority.txt");
 }
 
 String compiler_source_generation_file_name(Request* context)
 {
-	return(context->server->config["BIN_DIRECTORY"] + "/source-generation.txt");
+	return(compiler_unit_bin_directory(context) + "/source-generation.txt");
 }
 
 int compiler_open_lock_file(String file_name, String purpose, bool nonblocking = false)
@@ -553,6 +558,9 @@ SharedUnitFilesystemState inspect_shared_unit_filesystem(Request* context, Share
 		auto input_it = metadata.find("input_signature");
 		if(input_it != metadata.end())
 			state.metadata_input_signature = input_it->second;
+		auto core_abi_it = metadata.find("wasm_core_abi_version");
+		if(core_abi_it != metadata.end() && compiler_is_u64_string(core_abi_it->second))
+			state.metadata_wasm_core_abi_version = (u64)atoll(core_abi_it->second.c_str());
 		auto source_it = metadata.find("source_path");
 		if(source_it != metadata.end())
 			state.metadata_source_path = source_it->second;
@@ -562,7 +570,11 @@ SharedUnitFilesystemState inspect_shared_unit_filesystem(Request* context, Share
 	}
 	if(state.compile_output_exists)
 		state.compile_output_content = file_get_contents(su->compile_output_file_name);
-	state.abi_compatible = (state.metadata_parsed && state.metadata_abi_version == state.runtime_abi_version);
+	state.abi_compatible = (
+		state.metadata_parsed &&
+		state.metadata_abi_version == state.runtime_abi_version &&
+		state.metadata_wasm_core_abi_version == state.runtime_wasm_core_abi_version
+	);
 	state.input_signature_matches = (
 		state.metadata_parsed &&
 		state.metadata_input_signature != "" &&
@@ -791,7 +803,23 @@ String compiler_generated_cpp_path(Request* context, String source_file)
 {
 	if(!context || !context->server || source_file == "")
 		return("");
-	return(context->server->config["BIN_DIRECTORY"] + dirname(source_file) + "/" + basename(source_file) + ".cpp");
+	return(compiler_unit_bin_directory(context) + dirname(source_file) + "/" + basename(source_file) + ".cpp");
+}
+
+String compiler_unit_bin_directory(Request* context)
+{
+	if(!context || !context->server)
+		return("");
+	return(path_join(
+		context->server->config["BIN_DIRECTORY"],
+		"units-c" + std::to_string(UCE_UNIT_ABI_VERSION) +
+		"-w" + std::to_string(UCE_WASM_CORE_ABI_VERSION)
+	));
+}
+
+String compiler_unit_wasm_path(Request* context, String source_file)
+{
+	return(compiler_unit_bin_directory(context) + source_file + ".wasm");
 }
 
 String compiler_generated_cpp_path(SharedUnit* su)
@@ -809,8 +837,8 @@ void setup_unit_paths(Request* context, SharedUnit* su, String file_name)
 		return;
 
 	su->src_path = dirname(file_name);
-	su->bin_path = context->server->config["BIN_DIRECTORY"] + su->src_path;
-	su->pre_path = context->server->config["BIN_DIRECTORY"] + su->src_path;
+	su->bin_path = compiler_unit_bin_directory(context) + su->src_path;
+	su->pre_path = compiler_unit_bin_directory(context) + su->src_path;
 
 	su->src_file_name = basename(file_name);
 	su->wasm_file_name = su->src_file_name + ".wasm";
@@ -1023,7 +1051,7 @@ SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name
 	SharedUnit* su = new SharedUnit();
 	setup_unit_paths(context, su, file_name);
 
-	bool can_serve_stale = !force_recompile && compiler_request_can_serve_stale_artifact(context);
+	bool can_serve_stale = !force_recompile && compiler_unit_can_serve_stale_artifact(context, file_name);
 	int fdlock = compiler_open_lock_file(su->wasm_name + ".lock", "shared-unit:" + file_name, can_serve_stale);
 	if(fdlock == -2 && file_exists(su->wasm_name))
 	{
@@ -1178,6 +1206,26 @@ bool compiler_request_can_serve_stale_artifact(Request* context)
 		float_val(context->server->config["PROACTIVE_COMPILE_CHECK_INTERVAL"]) > 0);
 }
 
+bool compiler_unit_can_serve_stale_artifact(Request* context, String file_name)
+{
+	if(!compiler_request_can_serve_stale_artifact(context))
+		return(false);
+	file_name = compiler_normalize_unit_path(context, file_name);
+	if(file_name == "")
+		return(false);
+	SharedUnit su;
+	setup_unit_paths(context, &su, file_name);
+	auto state = inspect_shared_unit_filesystem(context, &su, true);
+	return(
+		state.source_exists &&
+		state.compiled_time != 0 &&
+		state.metadata_exists &&
+		state.metadata_parsed &&
+		state.abi_compatible &&
+		state.metadata_source_path == file_name
+	);
+}
+
 void compiler_prioritize_unit(Request* context, String file_name)
 {
 	if(!compiler_request_can_serve_stale_artifact(context))
@@ -1493,6 +1541,8 @@ DValue unit_info(String path)
 	info["required_mtime"] = (f64)fs_state.required_time;
 	info["runtime_abi_version"] = (f64)fs_state.runtime_abi_version;
 	info["metadata_abi_version"] = (f64)fs_state.metadata_abi_version;
+	info["runtime_wasm_core_abi_version"] = (f64)fs_state.runtime_wasm_core_abi_version;
+	info["metadata_wasm_core_abi_version"] = (f64)fs_state.metadata_wasm_core_abi_version;
 	info["current_input_signature"] = fs_state.current_input_signature;
 	info["metadata_input_signature"] = fs_state.metadata_input_signature;
 	info["metadata_build_token"] = fs_state.metadata_build_token;

src/lib/compiler.h

@@ -18,6 +18,10 @@
 String preprocess_shared_unit(Request* context, SharedUnit* su);
 String compiler_generated_cpp_path(Request* context, String source_file);
 String compiler_generated_cpp_path(SharedUnit* su);
+#ifndef __UCE_WASM_UNIT__
+String compiler_unit_bin_directory(Request* context);
+String compiler_unit_wasm_path(Request* context, String source_file);
+#endif
 void setup_unit_paths(Request* context, SharedUnit* su, String file_name);
 void compile_shared_unit(Request* context, SharedUnit* su);
 SharedUnit* get_shared_unit(Request* context, String file_name);
@@ -25,6 +29,7 @@ String compiler_error_page_unit(Request* context, String config_key);
 bool compiler_unit_compile_pending(Request* context, String file_name);
 bool compiler_unit_compile_in_progress(Request* context, String file_name);
 bool compiler_request_can_serve_stale_artifact(Request* context);
+bool compiler_unit_can_serve_stale_artifact(Request* context, String file_name);
 void compiler_prioritize_unit(Request* context, String file_name);
 StringList compiler_take_priority_units(Request* context);
 String compiler_source_generation(Request* context);

src/linux_fastcgi.cpp

@@ -1221,7 +1221,7 @@ void proactive_compile_queue_push(StringList& queue, String file_name)
 bool proactive_compile_unit(Request& context, String file_name, bool& source_missing)
 {
 	bool failed = false;
-	String wasm_path = server_state.config["BIN_DIRECTORY"] + file_name + ".wasm";
+	String wasm_path = compiler_unit_wasm_path(&context, file_name);
 	if(compiler_unit_needs_recompile(&context, file_name, &source_missing))
 	{
 		printf("(i) proactive compile %s\n", file_name.c_str());
@@ -1582,6 +1582,36 @@ void init_base_process()
 	srand(time());
 }
 
+int precompile_unit_generation()
+{
+	f64 started_at = time_precise();
+	server_state.config = make_server_settings();
+	server_state.config["COMPILER_SYS_PATH"] = cwd_get();
+	Request background_context;
+	background_context.server = &server_state;
+	mkdir(server_state.config["BIN_DIRECTORY"]);
+	mkdir(compiler_unit_bin_directory(&background_context));
+
+	set_active_request(background_context);
+	auto files = compiler_scan_site_units(&background_context);
+	compiler_set_known_units(&background_context, files);
+	u64 failed = 0;
+	u64 compiled = 0;
+	for(auto& file_name : files)
+	{
+		bool source_missing = false;
+		if(compiler_unit_needs_recompile(&background_context, file_name, &source_missing))
+			compiled++;
+		if(proactive_compile_unit(background_context, file_name, source_missing))
+			failed++;
+	}
+	printf("Precompiled unit generation %s: %zu units, %llu compiled, %llu failed in %.3f s\n",
+		compiler_unit_bin_directory(&background_context).c_str(), files.size(),
+		(unsigned long long)compiled, (unsigned long long)failed,
+		time_precise() - started_at);
+	return(failed == 0 ? 0 : 1);
+}
+
 int main(int argc, char** argv)
 {
 	// systemd connects stdout to a pipe, which otherwise block-buffers child
@@ -1592,6 +1622,8 @@ int main(int argc, char** argv)
 	// after a fault does not allocate.
 	backtrace(request_fault_frames, 4);
 	process_start_directory();
+	if(argc == 2 && String(argv[1]) == "--precompile")
+		return(precompile_unit_generation());
 
 	init_base_process();
 	ensure_proactive_compiler();

src/wasm/abi.h

@@ -0,0 +1,4 @@
+#pragma once
+
+#define UCE_WASM_CORE_ABI_VERSION 7
+#define UCE_COMPILER_UNIT_ABI_VERSION 12

src/wasm/backend.cpp

@@ -42,7 +42,7 @@ static String wasm_backend_ensure_started(Request* context)
 	wc.core_wasm_path = first(cfg["WASM_CORE_PATH"],
 		path_join(cfg["COMPILER_SYS_PATH"], "bin/wasm/core.wasm"));
 	wc.site_root = path_join(cfg["COMPILER_SYS_PATH"], cfg["SITE_DIRECTORY"]);
-	wc.cache_root = cfg["BIN_DIRECTORY"];
+	wc.cache_root = compiler_unit_bin_directory(context);
 	// write membrane allowlist: the site tree plus the runtime scratch dirs
 	// pages legitimately write to (matches native reachable write targets).
 	wc.write_roots = { wc.site_root, "/tmp" };
@@ -114,7 +114,7 @@ static bool wasm_artifact_exists(Request* context, const String& entry_unit)
 {
 	if(entry_unit == "")
 		return(false);
-	String wasm_path = context->server->config["BIN_DIRECTORY"] + entry_unit + ".wasm";
+	String wasm_path = compiler_unit_wasm_path(context, entry_unit);
 	struct stat wasm_st;
 	f64 phase_started = time_precise();
 	context->stats.wasm_ready_check_count++;
@@ -133,7 +133,7 @@ static bool wasm_artifact_exists(Request* context, const String& entry_unit)
 	{
 		context->stats.wasm_ready_freshness_us += (u64)((time_precise() - phase_started) * 1000000.0);
 		compiler_prioritize_unit(context, entry_unit);
-		return(compiler_request_can_serve_stale_artifact(context));
+		return(compiler_unit_can_serve_stale_artifact(context, entry_unit));
 	}
 	context->stats.wasm_ready_freshness_us += (u64)((time_precise() - phase_started) * 1000000.0);
 	if(source_missing)

src/wasm/core.cpp

@@ -6,6 +6,7 @@
 // into core.wasm.
 
 #define __UCE_WASM_CORE__ 1
+#include "abi.h"
 #include "../lib/uce_lib.cpp"
 
 #include "../lib/mysql-connector.h"
@@ -809,7 +810,7 @@ void uce_free(void* ptr)
 
 u32 uce_wasm_core_abi_version()
 {
-	return(7);
+	return(UCE_WASM_CORE_ABI_VERSION);
 }
 
 int uce_wasm_core_init()

src/wasm/w1_smoke.cpp

@@ -1,6 +1,7 @@
 // W1 smoke driver for the production UCE core.wasm.
 
 #include <wasm.h>
+#include "abi.h"
 
 #include <cstdio>
 #include <cstdlib>
@@ -254,7 +255,7 @@ int main(int argc, char** argv)
 
 	CHECK(call_i32(core, "uce_wasm_core_init") == 0, "core init failed");
 	call_i32(core, "uce_wasm_core_reset_request");
-	CHECK(call_i32(core, "uce_wasm_core_abi_version") == 7, "unexpected ABI version");
+	CHECK(call_i32(core, "uce_wasm_core_abi_version") == UCE_WASM_CORE_ABI_VERSION, "unexpected ABI version");
 
 	wasm_memory_t* memory = core.memory();
 	int32_t root = call_i32(core, "uce_dv_root");
@@ -302,7 +303,7 @@ int main(int argc, char** argv)
 	int32_t output_ptr = call_i32(core, "uce_wasm_output_data");
 	CHECK(read_bytes(memory, output_ptr, output_len) == out, "output plumbing mismatch");
 
-	printf("W1 core.wasm smoke: abi=7 encoded=%d output=%d\n", encoded_len, output_len);
+	printf("W1 core.wasm smoke: abi=%d encoded=%d output=%d\n", UCE_WASM_CORE_ABI_VERSION, encoded_len, output_len);
 	printf("W1 EXIT CRITERION: PASS\n");
 	return(0);
 }

src/wasm/worker.cpp

@@ -2615,8 +2615,8 @@ private:
 		{
 			auto artifact_start = std::chrono::steady_clock::now();
 			bool artifact_exists = file_exists_host(worker.unit_wasm_path(resolved));
-			bool can_serve_stale = compiler_request_can_serve_stale_artifact(context);
-			if(can_serve_stale && read_request && !component_source_generation_checked)
+			bool stale_policy = compiler_request_can_serve_stale_artifact(context);
+			if(stale_policy && read_request && !component_source_generation_checked)
 			{
 				component_source_generation = compiler_source_generation(context);
 				component_source_generation_checked = true;
@@ -2629,7 +2629,7 @@ private:
 				// HTTP reads may serve a complete stale artifact, so bound their graph
 				// stat work; CLI and mutations always check the current graph.
 				bool generation_available = component_source_generation != "";
-				bool check_freshness = !can_serve_stale || !read_request ||
+				bool check_freshness = !stale_policy || !read_request ||
 					cached_freshness == worker.component_freshness.end() ||
 					(generation_available ?
 						cached_freshness->second.source_generation != component_source_generation :
@@ -2637,13 +2637,14 @@ private:
 							now - cached_freshness->second.checked_at).count() >= 1000);
 				if(check_freshness)
 				{
-					stale = compiler_unit_needs_recompile(context, resolved, 0, can_serve_stale && read_request, true);
+					stale = compiler_unit_needs_recompile(context, resolved, 0, stale_policy && read_request, true);
 					worker.component_freshness[resolved] = { now, stale, component_source_generation };
 				}
 				else
 					stale = cached_freshness->second.stale;
 			}
-			if(stale && can_serve_stale)
+			bool can_serve_stale = stale && compiler_unit_can_serve_stale_artifact(context, resolved);
+			if(stale && stale_policy)
 			{
 				compiler_prioritize_unit(context, resolved);
 				if(!read_request)