Parallelize managed unit precompile

src/lib/sys.cpp

@@ -1700,6 +1700,7 @@ StringMap make_server_settings()
 	cfg["SESSION_COOKIE_SECURE"] = "0";
 	cfg["COMPILER_SYS_PATH"] = ".";
 	cfg["PRECOMPILE_FILES_IN"] = "";
+	cfg["PRECOMPILE_JOBS"] = "2";
 	cfg["SITE_DIRECTORY"] = "site";
 	cfg["JIT_COMPILE_ON_REQUEST"] = "1";
 	cfg["PROACTIVE_COMPILE_ENABLED"] = "1";

src/linux_fastcgi.cpp

@@ -9,6 +9,7 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <sys/socket.h>
+#include <sys/wait.h>
 
 ServerState server_state;
 
@@ -1582,6 +1583,62 @@ void init_base_process()
 	srand(time());
 }
 
+struct PrecompileWorkerResult
+{
+	u64 assigned = 0;
+	u64 compiled = 0;
+	u64 failed = 0;
+};
+
+PrecompileWorkerResult precompile_unit_range(Request& background_context, const StringList& files, u64 worker, u64 jobs)
+{
+	PrecompileWorkerResult result;
+	for(u64 i = worker; i < files.size(); i += jobs)
+	{
+		result.assigned++;
+		bool source_missing = false;
+		if(compiler_unit_needs_recompile(&background_context, files[i], &source_missing))
+			result.compiled++;
+		if(proactive_compile_unit(background_context, files[i], source_missing))
+			result.failed++;
+	}
+	return(result);
+}
+
+bool precompile_write_result(int fd, const PrecompileWorkerResult& result)
+{
+	const char* data = (const char*)&result;
+	size_t remaining = sizeof(result);
+	while(remaining > 0)
+	{
+		ssize_t written = write(fd, data, remaining);
+		if(written < 0 && errno == EINTR)
+			continue;
+		if(written <= 0)
+			return(false);
+		data += written;
+		remaining -= (size_t)written;
+	}
+	return(true);
+}
+
+bool precompile_read_result(int fd, PrecompileWorkerResult& result)
+{
+	char* data = (char*)&result;
+	size_t remaining = sizeof(result);
+	while(remaining > 0)
+	{
+		ssize_t received = read(fd, data, remaining);
+		if(received < 0 && errno == EINTR)
+			continue;
+		if(received <= 0)
+			return(false);
+		data += received;
+		remaining -= (size_t)received;
+	}
+	return(true);
+}
+
 int precompile_unit_generation()
 {
 	f64 started_at = time_precise();
@@ -1595,21 +1652,77 @@ int precompile_unit_generation()
 	set_active_request(background_context);
 	auto files = compiler_scan_site_units(&background_context);
 	compiler_set_known_units(&background_context, files);
-	u64 failed = 0;
-	u64 compiled = 0;
-	for(auto& file_name : files)
+	const char* jobs_env = getenv("UCE_PRECOMPILE_JOBS");
+	u64 jobs = to_u64(jobs_env && jobs_env[0] != '\0' ? String(jobs_env) : server_state.config["PRECOMPILE_JOBS"], 2);
+	jobs = std::max<u64>(1, std::min<u64>(jobs, std::min<u64>(files.size() == 0 ? 1 : files.size(), 16)));
+	PrecompileWorkerResult total;
+	bool worker_error = false;
+	if(jobs == 1)
+		total = precompile_unit_range(background_context, files, 0, 1);
+	else
 	{
-		bool source_missing = false;
-		if(compiler_unit_needs_recompile(&background_context, file_name, &source_missing))
-			compiled++;
-		if(proactive_compile_unit(background_context, file_name, source_missing))
-			failed++;
+		int result_pipe[2];
+		if(pipe(result_pipe) != 0)
+		{
+			printf("(!) cannot create precompile result pipe: %s\n", std::strerror(errno));
+			return(1);
+		}
+		std::vector<pid_t> workers;
+		fflush(0);
+		for(u64 worker = 0; worker < jobs; worker++)
+		{
+			pid_t pid = fork();
+			if(pid < 0)
+			{
+				printf("(!) cannot fork precompile worker: %s\n", std::strerror(errno));
+				worker_error = true;
+				break;
+			}
+			if(pid == 0)
+			{
+				close(result_pipe[0]);
+				PrecompileWorkerResult result = precompile_unit_range(background_context, files, worker, jobs);
+				printf("Precompile worker %llu/%llu: %llu units, %llu compiled, %llu failed\n",
+					(unsigned long long)(worker + 1), (unsigned long long)jobs,
+					(unsigned long long)result.assigned, (unsigned long long)result.compiled,
+					(unsigned long long)result.failed);
+				bool reported = precompile_write_result(result_pipe[1], result);
+				close(result_pipe[1]);
+				_exit(result.failed == 0 && reported ? 0 : 1);
+			}
+			workers.push_back(pid);
+		}
+		close(result_pipe[1]);
+		for(size_t i = 0; i < workers.size(); i++)
+		{
+			PrecompileWorkerResult result;
+			if(!precompile_read_result(result_pipe[0], result))
+			{
+				worker_error = true;
+				break;
+			}
+			total.assigned += result.assigned;
+			total.compiled += result.compiled;
+			total.failed += result.failed;
+		}
+		close(result_pipe[0]);
+		for(auto pid : workers)
+		{
+			int status = 0;
+			pid_t waited;
+			do waited = waitpid(pid, &status, 0); while(waited < 0 && errno == EINTR);
+			if(waited != pid || !WIFEXITED(status) || WEXITSTATUS(status) != 0)
+				worker_error = true;
+		}
+		if(total.assigned != files.size())
+			worker_error = true;
 	}
-	printf("Precompiled unit generation %s: %zu units, %llu compiled, %llu failed in %.3f s\n",
-		compiler_unit_bin_directory(&background_context).c_str(), files.size(),
-		(unsigned long long)compiled, (unsigned long long)failed,
+	printf("Precompiled unit generation %s with %llu job%s: %zu units, %llu compiled, %llu failed in %.3f s\n",
+		compiler_unit_bin_directory(&background_context).c_str(),
+		(unsigned long long)jobs, jobs == 1 ? "" : "s", files.size(),
+		(unsigned long long)total.compiled, (unsigned long long)total.failed,
 		time_precise() - started_at);
-	return(failed == 0 ? 0 : 1);
+	return(total.failed == 0 && !worker_error ? 0 : 1);
 }
 
 int main(int argc, char** argv)