website with slop placeholders

src/lib/compiler.cpp

@@ -229,9 +229,13 @@ String compiler_registry_lock_file_name(Request* context)
 	return(compiler_registry_file_name(context) + ".lock");
 }
 
-int compiler_open_lock_file(String file_name)
+int compiler_open_lock_file(String file_name, String purpose)
 {
-	int fdlock = open(file_name.c_str(), O_RDWR | O_CREAT, 0666);
+	(void)purpose;
+	auto lock_dir = dirname(file_name);
+	if(lock_dir != "")
+		mkdir(lock_dir);
+	int fdlock = file_open_locked(file_name, O_RDWR | O_CREAT, LOCK_EX, 0666);
 	if(fdlock == -1)
 		printf("(!) Could not open lock file %s\n", file_name.c_str());
 	return(fdlock);
@@ -239,10 +243,7 @@ int compiler_open_lock_file(String file_name)
 
 void compiler_close_lock_file(int fdlock)
 {
-	if(fdlock == -1)
-		return;
-	flock(fdlock, LOCK_UN);
-	close(fdlock);
+	file_close_locked(fdlock);
 }
 
 String compiler_normalize_unit_path(Request* context, String file_name)
@@ -297,9 +298,7 @@ template <typename TCallback>
 auto compiler_with_registry_lock(Request* context, TCallback callback) -> decltype(callback())
 {
 	auto lock_file_name = compiler_registry_lock_file_name(context);
-	int fdlock = compiler_open_lock_file(lock_file_name);
-	if(fdlock != -1)
-		flock(fdlock, LOCK_EX);
+	int fdlock = compiler_open_lock_file(lock_file_name, "compiler-registry");
 	auto result = callback();
 	compiler_close_lock_file(fdlock);
 	return(result);
@@ -794,9 +793,16 @@ SharedUnit* compiler_get_shared_unit_internal(Request* context, String file_name
 	setup_unit_paths(context, su, file_name);
 	su->opt_so_optional = opt_so_optional;
 
-	int fdlock = compiler_open_lock_file(su->so_name + ".lock");
-	if(fdlock != -1)
-		flock(fdlock, LOCK_EX);
+	int fdlock = compiler_open_lock_file(su->so_name + ".lock", "shared-unit:" + file_name);
+	if(fdlock == -1)
+	{
+		su->compiler_messages = "could not open compile lock";
+		su->compile_status = "lock_error";
+		su->compile_error_status = su->compiler_messages;
+		su->last_error = time();
+		context->server->units[file_name] = su;
+		return(su);
+	}
 
 	cached = compiler_reusable_cached_unit(context, file_name, opt_so_optional, force_recompile);
 	if(cached)

src/lib/sys.cpp

@@ -1,4 +1,3 @@
-#include <errno.h>
 #include <string.h>
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -10,6 +9,12 @@
 #include <sys/file.h>
 #include "sys.h"
 
+namespace {
+
+constexpr f64 FILE_LOCK_WAIT_TIMEOUT_SECONDS = 3.0;
+
+}
+
 String capture_backtrace_string(u32 max_frames, u32 skip_frames)
 {
 	if(max_frames == 0)
@@ -188,12 +193,15 @@ bool file_exists(String path)
 	return(std::filesystem::exists(fp));
 }
 
-int file_open_locked(String file_name, int open_flags, int lock_type, int create_mode)
+int file_open_locked(String file_name, int open_flags, int lock_type, int create_mode, f64 wait_timeout_seconds, String purpose)
 {
+	(void)wait_timeout_seconds;
+	(void)purpose;
 	int fd = open(file_name.c_str(), open_flags, create_mode);
 	if(fd == -1)
 		return(-1);
-	if(flock(fd, lock_type) == -1)
+	fcntl(fd, F_SETFD, FD_CLOEXEC);
+	if(flock(fd, lock_type) != 0)
 	{
 		close(fd);
 		return(-1);
@@ -209,6 +217,11 @@ void file_close_locked(int fd)
 	close(fd);
 }
 
+void file_release_process_locks(String reason)
+{
+	(void)reason;
+}
+
 String file_get_contents_locked_fd(int fd)
 {
 	if(fd == -1)
@@ -254,7 +267,7 @@ bool file_put_contents_locked_fd(int fd, String content)
 
 String file_get_contents(String file_name)
 {
-	s32 fd = file_open_locked(file_name, O_RDONLY, LOCK_SH);
+	s32 fd = file_open_locked(file_name, O_RDONLY, LOCK_SH, 0644, FILE_LOCK_WAIT_TIMEOUT_SECONDS, "file_get_contents:" + file_name);
 	if(fd == -1)
 	{
 		printf("(!) Could not read %s\n", file_name.c_str());
@@ -267,7 +280,7 @@ String file_get_contents(String file_name)
 
 bool file_put_contents(String file_name, String content)
 {
-	s32 fd = file_open_locked(file_name, O_RDWR | O_CREAT, LOCK_EX, 0644);
+	s32 fd = file_open_locked(file_name, O_RDWR | O_CREAT, LOCK_EX, 0644, FILE_LOCK_WAIT_TIMEOUT_SECONDS, "file_put_contents:" + file_name);
 	if(fd == -1)
 	{
 		printf("(!) Could not write %s\n", file_name.c_str());
@@ -285,7 +298,7 @@ bool file_put_contents(String file_name, String content)
 
 bool file_append_contents(String file_name, String content)
 {
-	s32 fd = file_open_locked(file_name, O_RDWR | O_CREAT, LOCK_EX, 0644);
+	s32 fd = file_open_locked(file_name, O_RDWR | O_CREAT, LOCK_EX, 0644, FILE_LOCK_WAIT_TIMEOUT_SECONDS, "file_append:" + file_name);
 	if(fd == -1)
 	{
 		printf("(!) Could not append %s\n", file_name.c_str());
@@ -583,6 +596,7 @@ pid_t spawn_subprocess(std::function<void()> exec_after_spawn)
 	p = fork();
 	if(p == 0)
 	{
+		file_release_process_locks("fork child startup");
 		my_pid = getpid();
 		//printf("(C) child procress started, PID:%i\n", my_pid);
 		prctl(PR_SET_PDEATHSIG, SIGHUP);
@@ -629,7 +643,9 @@ pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout)
 {
 	String status_file_name = context->server->config["BIN_DIRECTORY"] + "/task-" + key;
 	String lock_file_name = status_file_name + ".lock";
-	int lock_fd = file_open_locked(lock_file_name, O_RDWR | O_CREAT, LOCK_EX, 0644);
+	int lock_fd = file_open_locked(lock_file_name, O_RDWR | O_CREAT, LOCK_EX, 0644, FILE_LOCK_WAIT_TIMEOUT_SECONDS, "task:" + key);
+	if(lock_fd == -1)
+		return(0);
 	String status_file = file_get_contents(status_file_name);
 	pid_t p;
 	if(status_file != "")
@@ -647,6 +663,7 @@ pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout)
 	p = fork();
 	if(p == 0)
 	{
+		file_release_process_locks("task child startup");
 		file_close_locked(lock_fd);
 		my_pid = getpid();
 
@@ -655,7 +672,7 @@ pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout)
 		//printf("(C) child procress started, PID:%i\n", my_pid);
 		//prctl(PR_SET_PDEATHSIG, SIGHUP);
 		exec_after_spawn();
-		int exit_lock_fd = file_open_locked(lock_file_name, O_RDWR | O_CREAT, LOCK_EX, 0644);
+		int exit_lock_fd = file_open_locked(lock_file_name, O_RDWR | O_CREAT, LOCK_EX, 0644, FILE_LOCK_WAIT_TIMEOUT_SECONDS, "task-exit:" + key);
 		file_unlink(status_file_name);
 		file_close_locked(exit_lock_fd);
 		printf("(P) worker process '%s' terminated: PID %i\n", key.c_str(), my_pid);

src/lib/sys.h

@@ -11,8 +11,9 @@ String dirname(String fn);
 String path_join(String base, String child);
 bool mkdir(String path);
 bool file_exists(String path);
-int file_open_locked(String file_name, int open_flags, int lock_type = LOCK_SH, int create_mode = 0644);
+int file_open_locked(String file_name, int open_flags, int lock_type = LOCK_SH, int create_mode = 0644, f64 wait_timeout_seconds = 3.0, String purpose = "");
 void file_close_locked(int fd);
+void file_release_process_locks(String reason = "");
 String file_get_contents_locked_fd(int fd);
 bool file_put_contents_locked_fd(int fd, String content);
 String file_get_contents(String file_name);

src/lib/types.cpp

@@ -98,11 +98,6 @@ Request::~Request()
 		delete stream;
 	ob_stack.clear();
 	ob = 0;
-	if(session_lock_fd != -1)
-	{
-		flock(session_lock_fd, LOCK_UN);
-		close(session_lock_fd);
-	}
 	for(auto& sockfd : resources.sockets)
 		close(sockfd);
 }

src/lib/types.h

@@ -155,9 +155,7 @@ struct Request {
 	StringMap post;
 	StringMap cookies;
 	StringMap session;
-	String session_file_name = "";
-	String session_serialized = "";
-	int session_lock_fd = -1;
+	String session_loaded_hash = "";
 
 	DTree var;
 	DTree cfg;
@@ -211,6 +209,9 @@ struct Request {
 		String websocket_connection_id = "";
 		String websocket_scope = "";
 		DTree* websocket_connection_state = 0;
+		StringList websocket_scope_connection_ids;
+		DTree websocket_dispatch_commands;
+		bool websocket_dispatch_capture = false;
 		u8 websocket_opcode = 0;
 		bool websocket_is_binary = false;
 		bool websocket_is_text = false;

src/lib/uri.cpp

@@ -544,15 +544,21 @@ String session_file_path(String session_id)
 
 namespace {
 
-void session_release_lock(Request* request)
+String session_serialize(const StringMap& data)
 {
-	if(!request)
-		return;
-	if(request->session_lock_fd != -1)
-		file_close_locked(request->session_lock_fd);
-	request->session_lock_fd = -1;
-	request->session_file_name = "";
-	request->session_serialized = "";
+	return(encode_query(data));
+}
+
+String session_hash_serialized(String serialized)
+{
+	return(gen_sha1(serialized));
+}
+
+String session_load_serialized(String session_path)
+{
+	if(session_path == "" || !file_exists(session_path))
+		return("");
+	return(file_get_contents(session_path));
 }
 
 }
@@ -562,47 +568,36 @@ StringMap load_session_data(String session_id)
 	String session_path = session_file_path(session_id);
 	if(session_path == "")
 		return(StringMap());
-	if(context && context->session_lock_fd != -1 && context->session_file_name == session_path)
-		return(parse_query(file_get_contents_locked_fd(context->session_lock_fd)));
-	return(parse_query(file_get_contents(session_path)));
+	return(parse_query(session_load_serialized(session_path)));
 }
 
 void save_session_data(String session_id, StringMap data)
 {
 	String session_path = session_file_path(session_id);
-	String encoded = encode_query(data);
+	String encoded = session_serialize(data);
+	String encoded_hash = session_hash_serialized(encoded);
 	if(session_path == "")
 	{
 		printf("(!) Refusing to save invalid session id\n");
 		return;
 	}
-	if(context && context->session_lock_fd != -1 && context->session_file_name == session_path)
+	if(context && encoded_hash == context->session_loaded_hash)
+		return;
+	if(!file_put_contents(session_path, encoded))
 	{
-		if(encoded == context->session_serialized)
-			return;
-		if(file_put_contents_locked_fd(context->session_lock_fd, encoded))
-			context->session_serialized = encoded;
+		printf("(!) Refusing to save session file %s\n", session_path.c_str());
 		return;
 	}
-	int fd = file_open_locked(session_path, O_RDWR | O_CREAT, LOCK_EX, 0644);
-	if(fd == -1)
-	{
-		printf("(!) Refusing to save unreadable session file %s\n", session_path.c_str());
-		return;
-	}
-	if(file_get_contents_locked_fd(fd) != encoded)
-		file_put_contents_locked_fd(fd, encoded);
-	file_close_locked(fd);
+	if(context)
+		context->session_loaded_hash = encoded_hash;
 }
 
 String session_start(String session_name)
 {
-	if(context->session_lock_fd != -1 && context->session_name == session_name && context->session_id != "")
+	if(context->session_name == session_name && context->session_id != "")
 		return(context->session_id);
-	session_release_lock(context);
 	context->session.clear();
-	context->session_serialized = "";
-	context->session_file_name = "";
+	context->session_loaded_hash = "";
 	context->session_id = "";
 	context->session_name = "";
 
@@ -617,15 +612,10 @@ String session_start(String session_name)
 	}
 	context->session_id = session_id;
 	context->session_name = session_name;
-	context->session_file_name = session_file_path(context->session_id);
-	if(context->session_file_name != "")
-	{
-		context->session_lock_fd = file_open_locked(context->session_file_name, O_RDWR | O_CREAT, LOCK_EX, 0644);
-		if(context->session_lock_fd == -1)
-			printf("(!) Could not lock session file %s\n", context->session_file_name.c_str());
-	}
-	context->session = load_session_data(context->session_id);
-	context->session_serialized = encode_query(context->session);
+	auto session_path = session_file_path(context->session_id);
+	auto serialized = session_load_serialized(session_path);
+	context->session_loaded_hash = session_hash_serialized(serialized);
+	context->session = parse_query(serialized);
 	return(context->session_id);
 }
 
@@ -636,7 +626,6 @@ void session_destroy(String session_name)
 		set_cookie(session_name, "", time() - int_val(context->server->config["SESSION_TIME"]));
 		context->session.clear();
 		save_session_data(context->session_id, context->session);
-		session_release_lock(context);
 		context->session_id = "";
 	}
 }

src/linux_fastcgi.cpp

@@ -1,5 +1,9 @@
 #include "lib/uce_lib.cpp"
 #include <csetjmp>
+#include <deque>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/socket.h>
 
 ServerState server_state;
 
@@ -8,12 +12,20 @@ ServerState server_state;
 FastCGIServer server;
 pid_t http_worker_pid = 0;
 pid_t proactive_compiler_pid = 0;
+pid_t websocket_exec_pid = 0;
 bool worker_accepts_http = false;
 static sigjmp_buf request_fault_jmp;
 static volatile sig_atomic_t request_fault_active = 0;
 static volatile sig_atomic_t request_fault_signal = 0;
 static Request* request_fault_request = 0;
 static String request_fault_trace = "";
+static int websocket_exec_fd = -1;
+static String websocket_exec_read_buffer = "";
+static std::deque<DTree> websocket_exec_pending_jobs;
+static DTree websocket_exec_inflight_job;
+static String websocket_exec_write_buffer = "";
+
+void close_inherited_server_sockets();
 
 Request* set_active_request(Request& request)
 {
@@ -119,6 +131,545 @@ void restore_request_fault_handlers()
 	signal(SIGFPE, on_segfault);
 }
 
+namespace {
+
+const char* websocket_ipc_base64_alphabet =
+	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+String websocket_ipc_base64_encode(String raw)
+{
+	String result;
+	size_t i = 0;
+	while(i < raw.length())
+	{
+		unsigned char input[3] = {0, 0, 0};
+		size_t chunk = 0;
+		for(; chunk < 3 && i < raw.length(); ++chunk, ++i)
+			input[chunk] = (unsigned char)raw[i];
+
+		result += websocket_ipc_base64_alphabet[input[0] >> 2];
+		result += websocket_ipc_base64_alphabet[((input[0] & 0x03) << 4) | (input[1] >> 4)];
+		result += (chunk > 1 ? websocket_ipc_base64_alphabet[((input[1] & 0x0F) << 2) | (input[2] >> 6)] : '=');
+		result += (chunk > 2 ? websocket_ipc_base64_alphabet[input[2] & 0x3F] : '=');
+	}
+	return(result);
+}
+
+int websocket_ipc_base64_value(char c)
+{
+	if(c >= 'A' && c <= 'Z')
+		return(c - 'A');
+	if(c >= 'a' && c <= 'z')
+		return(c - 'a' + 26);
+	if(c >= '0' && c <= '9')
+		return(c - '0' + 52);
+	if(c == '+')
+		return(62);
+	if(c == '/')
+		return(63);
+	return(-1);
+}
+
+String websocket_ipc_base64_decode(String raw, bool& ok)
+{
+	ok = false;
+	String filtered = "";
+	for(auto c : raw)
+	{
+		if(!isspace((unsigned char)c))
+			filtered.append(1, c);
+	}
+	if(filtered == "")
+	{
+		ok = true;
+		return("");
+	}
+	if(filtered.length() % 4 != 0)
+		return("");
+
+	String result;
+	for(size_t i = 0; i < filtered.length(); i += 4)
+	{
+		int values[4] = {0, 0, 0, 0};
+		int padding = 0;
+		for(int j = 0; j < 4; ++j)
+		{
+			char c = filtered[i + j];
+			if(c == '=')
+			{
+				values[j] = 0;
+				padding += 1;
+				continue;
+			}
+			values[j] = websocket_ipc_base64_value(c);
+			if(values[j] < 0)
+				return("");
+		}
+
+		result.append(1, (char)((values[0] << 2) | (values[1] >> 4)));
+		if(padding < 2)
+			result.append(1, (char)(((values[1] & 0x0F) << 4) | (values[2] >> 2)));
+		if(padding < 1)
+			result.append(1, (char)(((values[2] & 0x03) << 6) | values[3]));
+	}
+
+	ok = true;
+	return(result);
+}
+
+bool websocket_ipc_set_nonblocking(int fd)
+{
+	int flags = fcntl(fd, F_GETFL, 0);
+	if(flags == -1)
+		return(false);
+	return(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0);
+}
+
+bool websocket_exec_enabled_for_process()
+{
+	return(worker_accepts_http);
+}
+
+u64 websocket_exec_queue_limit_bytes()
+{
+	u64 configured = int_val(server_state.config["WEBSOCKET_EXEC_QUEUE_BYTES"]);
+	if(configured < 64 * 1024)
+		configured = 1024 * 1024;
+	return(configured);
+}
+
+bool websocket_exec_has_inflight_job()
+{
+	return(websocket_exec_inflight_job.to_bool());
+}
+
+FastCGIServer::Connection* websocket_find_connection(String connection_id)
+{
+	for(auto& item : server.client_sockets)
+	{
+		FastCGIServer::Connection* connection = item.second;
+		if(connection->is_websocket && connection->websocket_connection_id == connection_id)
+			return(connection);
+	}
+	return(0);
+}
+
+void websocket_exec_clear_ipc_state()
+{
+	if(websocket_exec_fd != -1)
+		close(websocket_exec_fd);
+	websocket_exec_fd = -1;
+	websocket_exec_read_buffer = "";
+	websocket_exec_write_buffer = "";
+	websocket_exec_inflight_job.clear();
+}
+
+void websocket_exec_close_connection(String connection_id, u16 status_code = 1011, String reason = "websocket handler unavailable")
+{
+	if(connection_id == "")
+		return;
+	server.websocket_close(connection_id, status_code, reason);
+}
+
+void websocket_exec_fail_inflight_job(String reason = "websocket handler unavailable")
+{
+	if(!websocket_exec_has_inflight_job())
+		return;
+	websocket_exec_close_connection(websocket_exec_inflight_job["connection_id"].to_string(), 1011, reason);
+	websocket_exec_inflight_job.clear();
+	websocket_exec_write_buffer = "";
+}
+
+void websocket_exec_queue_job(DTree job)
+{
+	if(job["connection_id"].to_string() == "")
+		return;
+
+	u64 queued_bytes = websocket_exec_write_buffer.length();
+	if(websocket_exec_has_inflight_job())
+		queued_bytes += websocket_exec_inflight_job["serialized"].to_string().length();
+	for(auto& pending : websocket_exec_pending_jobs)
+		queued_bytes += pending["serialized"].to_string().length();
+	queued_bytes += json_encode(job).length();
+
+	if(queued_bytes > websocket_exec_queue_limit_bytes())
+	{
+		printf("(!) websocket dispatch queue overflow for %s\n", job["connection_id"].to_string().c_str());
+		websocket_exec_close_connection(job["connection_id"].to_string(), 1013, "websocket server busy");
+		return;
+	}
+
+	job["serialized"] = json_encode(job) + "\n";
+	websocket_exec_pending_jobs.push_back(job);
+}
+
+StringList websocket_exec_snapshot_connections(String scope)
+{
+	return(server.websocket_connection_ids(scope));
+}
+
+void websocket_exec_append_command(DTree command)
+{
+	if(!context)
+		return;
+	context->resources.websocket_dispatch_commands.push(command);
+}
+
+void websocket_exec_apply_command(DTree command)
+{
+	String action = command["action"].to_string();
+	if(action == "broadcast")
+	{
+		bool ok = false;
+		String payload = websocket_ipc_base64_decode(command["message_b64"].to_string(), ok);
+		if(!ok)
+			return;
+		server.websocket_broadcast(command["scope"].to_string(), payload, command["binary"].to_bool());
+		return;
+	}
+	if(action == "send_to")
+	{
+		bool ok = false;
+		String payload = websocket_ipc_base64_decode(command["message_b64"].to_string(), ok);
+		if(!ok)
+			return;
+		server.websocket_send_to(command["connection_id"].to_string(), payload, command["binary"].to_bool());
+		return;
+	}
+	if(action == "close")
+	{
+		server.websocket_close(
+			command["connection_id"].to_string(),
+			(u16)command["status_code"].to_u64(),
+			command["reason"].to_string()
+		);
+	}
+}
+
+void websocket_exec_apply_result(DTree result)
+{
+	if(result["type"].to_string() != "result")
+		return;
+
+	String inflight_connection_id = websocket_exec_inflight_job["connection_id"].to_string();
+	String result_connection_id = result["connection_id"].to_string();
+	if(inflight_connection_id != "" && result_connection_id != "" && inflight_connection_id != result_connection_id)
+	{
+		printf("(!) websocket dispatch result mismatch: expected %s got %s\n",
+			inflight_connection_id.c_str(),
+			result_connection_id.c_str());
+	}
+
+	FastCGIServer::Connection* connection = websocket_find_connection(result_connection_id);
+	if(connection)
+		connection->websocket_state = result["connection_state"];
+
+	result["commands"].each([] (DTree command, String) {
+		websocket_exec_apply_command(command);
+	});
+
+	websocket_exec_inflight_job.clear();
+}
+
+void websocket_exec_handle_ipc_line(String line)
+{
+	line = trim(line);
+	if(line == "")
+		return;
+	DTree result = json_decode(line);
+	websocket_exec_apply_result(result);
+}
+
+void websocket_exec_read_results()
+{
+	if(websocket_exec_fd == -1)
+		return;
+
+	char buffer[4096];
+	for(;;)
+	{
+		ssize_t read_result = read(websocket_exec_fd, buffer, sizeof(buffer));
+		if(read_result == 0)
+		{
+			printf("(!) websocket executor disconnected\n");
+			websocket_exec_fail_inflight_job();
+			websocket_exec_clear_ipc_state();
+			return;
+		}
+		if(read_result < 0)
+		{
+			if(errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
+				break;
+			perror("websocket executor read");
+			websocket_exec_fail_inflight_job();
+			websocket_exec_clear_ipc_state();
+			return;
+		}
+
+		websocket_exec_read_buffer.append(buffer, read_result);
+		for(;;)
+		{
+			size_t line_end = websocket_exec_read_buffer.find('\n');
+			if(line_end == String::npos)
+				break;
+			String line = websocket_exec_read_buffer.substr(0, line_end);
+			websocket_exec_read_buffer.erase(0, line_end + 1);
+			websocket_exec_handle_ipc_line(line);
+		}
+	}
+}
+
+void websocket_exec_flush_queue()
+{
+	if(websocket_exec_fd == -1)
+		return;
+
+	if(websocket_exec_write_buffer == "" && !websocket_exec_has_inflight_job() && websocket_exec_pending_jobs.size() > 0)
+	{
+		websocket_exec_inflight_job = websocket_exec_pending_jobs.front();
+		websocket_exec_pending_jobs.pop_front();
+		websocket_exec_write_buffer = websocket_exec_inflight_job["serialized"].to_string();
+	}
+
+	while(websocket_exec_write_buffer != "")
+	{
+		ssize_t write_result = write(websocket_exec_fd, websocket_exec_write_buffer.data(), websocket_exec_write_buffer.length());
+		if(write_result < 0)
+		{
+			if(errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
+				return;
+			perror("websocket executor write");
+			websocket_exec_fail_inflight_job();
+			websocket_exec_clear_ipc_state();
+			return;
+		}
+		if(write_result == 0)
+			return;
+		websocket_exec_write_buffer.erase(0, write_result);
+	}
+}
+
+Request websocket_exec_build_event_request(DTree job, String message)
+{
+	Request event_request;
+	event_request.server = &server_state;
+	event_request.params = job["params"].to_stringmap();
+	event_request.params["REQUEST_METHOD"] = "WEBSOCKET";
+	event_request.get = parse_query(event_request.params["QUERY_STRING"]);
+	event_request.resources.is_websocket = true;
+	event_request.resources.websocket_connection_id = job["connection_id"].to_string();
+	event_request.resources.websocket_scope = job["scope"].to_string();
+	event_request.resources.websocket_opcode = (u8)job["opcode"].to_u64();
+	event_request.resources.websocket_is_binary = job["is_binary"].to_bool();
+	event_request.resources.websocket_is_text = job["is_text"].to_bool();
+	event_request.resources.websocket_dispatch_capture = true;
+	job["scope_connections"].each([&] (DTree item, String) {
+		event_request.resources.websocket_scope_connection_ids.push_back(item.to_string());
+	});
+	event_request.connection = job["connection_state"];
+	event_request.stats.time_init = time_precise();
+	event_request.stats.time_start = event_request.stats.time_init;
+	event_request.random_index = 0;
+	event_request.random_seed = gen_noise64(*reinterpret_cast<u64*>(&event_request.stats.time_start));
+	event_request.response_code = "WEBSOCKET";
+	event_request.header["Content-Type"] = server_state.config["CONTENT_TYPE"];
+	event_request.in = message;
+
+	if(event_request.params["HTTP_COOKIE"].length() > 0)
+		event_request.cookies = parse_cookies(event_request.params["HTTP_COOKIE"]);
+
+	event_request.var["ws"]["message"] = message;
+	event_request.var["ws"]["connection_id"] = event_request.resources.websocket_connection_id;
+	event_request.var["ws"]["scope"] = event_request.resources.websocket_scope;
+	event_request.var["ws"]["connection_count"] = (f64)event_request.resources.websocket_scope_connection_ids.size();
+	event_request.var["ws"]["opcode"] = (f64)event_request.resources.websocket_opcode;
+	event_request.var["ws"]["is_binary"].set_bool(event_request.resources.websocket_is_binary);
+	event_request.var["ws"]["is_text"].set_bool(event_request.resources.websocket_is_text);
+	event_request.var["ws"]["document_uri"] = first(
+		event_request.params["DOCUMENT_URI"],
+		event_request.params["REQUEST_URI"]
+	);
+
+	event_request.call["message"] = message;
+	event_request.call["connection_id"] = event_request.resources.websocket_connection_id;
+	event_request.call["scope"] = event_request.resources.websocket_scope;
+	event_request.call["opcode"] = (f64)event_request.resources.websocket_opcode;
+	event_request.call["document_uri"] = event_request.var["ws"]["document_uri"].to_string();
+	return(event_request);
+}
+
+bool websocket_exec_send_response(int fd, DTree response)
+{
+	String encoded = json_encode(response) + "\n";
+	size_t offset = 0;
+	while(offset < encoded.length())
+	{
+		ssize_t write_result = write(fd, encoded.data() + offset, encoded.length() - offset);
+		if(write_result < 0)
+		{
+			if(errno == EINTR)
+				continue;
+			return(false);
+		}
+		offset += (size_t)write_result;
+	}
+	return(true);
+}
+
+void websocket_exec_process_job_line(int fd, String line)
+{
+	line = trim(line);
+	if(line == "")
+		return;
+
+	DTree job = json_decode(line);
+	if(job["type"].to_string() != "dispatch")
+		return;
+
+	bool decoded = false;
+	String message = websocket_ipc_base64_decode(job["message_b64"].to_string(), decoded);
+	if(!decoded)
+	{
+		printf("(!) invalid websocket IPC payload for %s\n", job["connection_id"].to_string().c_str());
+		return;
+	}
+
+	Request event_request = websocket_exec_build_event_request(job, message);
+	Request* previous_context = set_active_request(event_request);
+	server_state.request_count += 1;
+
+	compiler_invoke_websocket(&event_request, event_request.params["SCRIPT_FILENAME"]);
+
+	if(event_request.session_id.length() > 0)
+		save_session_data(event_request.session_id, event_request.session);
+	cleanup_mysql_connections();
+
+	DTree response;
+	response["type"] = "result";
+	response["connection_id"] = event_request.resources.websocket_connection_id;
+	response["connection_state"] = event_request.connection;
+	response["commands"] = event_request.resources.websocket_dispatch_commands;
+
+	restore_active_request(previous_context);
+	if(!websocket_exec_send_response(fd, response))
+		exit(1);
+}
+
+void websocket_exec_child_loop(int fd)
+{
+	Request background_context;
+	my_pid = getpid();
+	context = &background_context;
+	close_inherited_server_sockets();
+	signal(SIGSEGV, on_segfault);
+	signal(SIGABRT, on_segfault);
+	signal(SIGBUS, on_segfault);
+	signal(SIGILL, on_segfault);
+	signal(SIGFPE, on_segfault);
+	signal(SIGPIPE, SIG_IGN);
+	setpriority(PRIO_PROCESS, 0, 5);
+
+	String read_buffer;
+	char buffer[4096];
+	for(;;)
+	{
+		ssize_t read_result = read(fd, buffer, sizeof(buffer));
+		if(read_result == 0)
+			exit(0);
+		if(read_result < 0)
+		{
+			if(errno == EINTR)
+				continue;
+			exit(1);
+		}
+
+		read_buffer.append(buffer, read_result);
+		for(;;)
+		{
+			size_t line_end = read_buffer.find('\n');
+			if(line_end == String::npos)
+				break;
+			String line = read_buffer.substr(0, line_end);
+			read_buffer.erase(0, line_end + 1);
+			websocket_exec_process_job_line(fd, line);
+		}
+	}
+}
+
+bool websocket_exec_alive()
+{
+	return(websocket_exec_pid > 0 && task_kill(websocket_exec_pid, 0) == 0 && websocket_exec_fd != -1);
+}
+
+void ensure_websocket_executor()
+{
+	if(!websocket_exec_enabled_for_process())
+		return;
+	if(websocket_exec_alive())
+		return;
+
+	if(websocket_exec_pid > 0 || websocket_exec_fd != -1)
+	{
+		websocket_exec_fail_inflight_job();
+		websocket_exec_clear_ipc_state();
+		websocket_exec_pid = 0;
+	}
+
+	int sockets[2] = {-1, -1};
+	if(socketpair(AF_UNIX, SOCK_STREAM, 0, sockets) != 0)
+	{
+		perror("socketpair");
+		return;
+	}
+
+	pid_t p = fork();
+	if(p < 0)
+	{
+		perror("fork");
+		close(sockets[0]);
+		close(sockets[1]);
+		return;
+	}
+	if(p == 0)
+	{
+		parent_pid = getppid();
+		file_release_process_locks("websocket executor fork");
+		prctl(PR_SET_PDEATHSIG, SIGHUP);
+		close(sockets[0]);
+		websocket_exec_child_loop(sockets[1]);
+		exit(0);
+	}
+
+	close(sockets[1]);
+	if(!websocket_ipc_set_nonblocking(sockets[0]))
+	{
+		printf("(!) failed to set websocket executor socket nonblocking\n");
+		close(sockets[0]);
+		return;
+	}
+
+	websocket_exec_fd = sockets[0];
+	websocket_exec_pid = p;
+	printf("(P) websocket executor spawned: PID %i\n", p);
+}
+
+void websocket_exec_tick()
+{
+	if(!websocket_exec_enabled_for_process())
+		return;
+	if(!websocket_exec_alive())
+	{
+		websocket_exec_fail_inflight_job();
+		websocket_exec_clear_ipc_state();
+		websocket_exec_pid = 0;
+		ensure_websocket_executor();
+	}
+	websocket_exec_read_results();
+	websocket_exec_flush_queue();
+}
+
+}
+
 String current_ws_scope()
 {
 	if(!context)
@@ -185,6 +736,13 @@ bool ws_is_binary()
 
 StringList ws_connections(String scope)
 {
+	if(context && context->resources.websocket_dispatch_capture)
+	{
+		String normalized_scope = normalize_ws_scope(scope);
+		if(normalized_scope == context->resources.websocket_scope)
+			return(context->resources.websocket_scope_connection_ids);
+		return(StringList());
+	}
 	return(server.websocket_connection_ids(normalize_ws_scope(scope)));
 }
 
@@ -195,11 +753,31 @@ u64 ws_connection_count(String scope)
 
 bool ws_send(String message, bool binary, String scope)
 {
+	if(context && context->resources.websocket_dispatch_capture)
+	{
+		DTree command;
+		command["action"] = "broadcast";
+		command["scope"] = normalize_ws_scope(scope);
+		command["binary"].set_bool(binary);
+		command["message_b64"] = websocket_ipc_base64_encode(message);
+		websocket_exec_append_command(command);
+		return(true);
+	}
 	return(server.websocket_broadcast(normalize_ws_scope(scope), message, binary) > 0);
 }
 
 bool ws_send_to(String connection_id, String message, bool binary)
 {
+	if(context && context->resources.websocket_dispatch_capture)
+	{
+		DTree command;
+		command["action"] = "send_to";
+		command["connection_id"] = connection_id;
+		command["binary"].set_bool(binary);
+		command["message_b64"] = websocket_ipc_base64_encode(message);
+		websocket_exec_append_command(command);
+		return(true);
+	}
 	return(server.websocket_send_to(connection_id, message, binary));
 }
 
@@ -209,6 +787,16 @@ bool ws_close(String connection_id)
 		connection_id = ws_connection_id();
 	if(connection_id == "")
 		return(false);
+	if(context && context->resources.websocket_dispatch_capture)
+	{
+		DTree command;
+		command["action"] = "close";
+		command["connection_id"] = connection_id;
+		command["status_code"] = (f64)1000;
+		command["reason"] = "";
+		websocket_exec_append_command(command);
+		return(true);
+	}
 	return(server.websocket_close(connection_id));
 }
 
@@ -334,55 +922,36 @@ int handle_complete(FastCGIRequest& request) {
 
 int handle_websocket_message(FastCGIRequest& request, const String& message, u8 opcode)
 {
-	Request event_request;
-	ByteStream ws_output;
+	ensure_websocket_executor();
+	if(!websocket_exec_alive())
+	{
+		printf("(!) websocket executor unavailable for %s\n", request.resources.websocket_connection_id.c_str());
+		server.websocket_close(request.resources.websocket_connection_id, 1011, "websocket handler unavailable");
+		return(0);
+	}
 
-	Request* previous_context = set_active_request(event_request);
-	server_state.request_count += 1;
-	event_request.server = &server_state;
-	event_request.params = request.params;
-	event_request.params["REQUEST_METHOD"] = "WEBSOCKET";
-	event_request.get = parse_query(event_request.params["QUERY_STRING"]);
-	event_request.resources = request.resources;
-	if(event_request.resources.websocket_connection_state)
-		event_request.connection.set_reference(event_request.resources.websocket_connection_state);
-	event_request.stats.time_init = time_precise();
-	event_request.stats.time_start = event_request.stats.time_init;
-	event_request.random_index = 0;
-	event_request.random_seed = gen_noise64(*reinterpret_cast<u64*>(&event_request.stats.time_start));
-	event_request.response_code = "WEBSOCKET";
-	event_request.header["Content-Type"] = context->server->config["CONTENT_TYPE"];
-	event_request.in = message;
-	event_request.ob = &ws_output;
+	DTree job;
+	job["type"] = "dispatch";
+	job["connection_id"] = request.resources.websocket_connection_id;
+	job["scope"] = request.resources.websocket_scope;
+	job["opcode"] = (f64)opcode;
+	job["is_binary"].set_bool(request.resources.websocket_is_binary);
+	job["is_text"].set_bool(request.resources.websocket_is_text);
+	job["message_b64"] = websocket_ipc_base64_encode(message);
+	if(request.resources.websocket_connection_state)
+		job["connection_state"] = *request.resources.websocket_connection_state;
 
-	if(event_request.params["HTTP_COOKIE"].length() > 0)
-		event_request.cookies = parse_cookies(event_request.params["HTTP_COOKIE"]);
+	for(auto& item : request.params)
+		job["params"][item.first] = item.second;
 
-	event_request.var["ws"]["message"] = message;
-	event_request.var["ws"]["connection_id"] = request.resources.websocket_connection_id;
-	event_request.var["ws"]["scope"] = request.resources.websocket_scope;
-	event_request.var["ws"]["connection_count"] = (f64)server.websocket_connection_ids(request.resources.websocket_scope).size();
-	event_request.var["ws"]["opcode"] = (f64)opcode;
-	event_request.var["ws"]["is_binary"].set_bool(request.resources.websocket_is_binary);
-	event_request.var["ws"]["is_text"].set_bool(request.resources.websocket_is_text);
-	event_request.var["ws"]["document_uri"] = first(
-		request.params["DOCUMENT_URI"],
-		request.params["REQUEST_URI"]
-	);
+	for(auto& connection_id : websocket_exec_snapshot_connections(request.resources.websocket_scope))
+	{
+		DTree snapshot_item;
+		snapshot_item = connection_id;
+		job["scope_connections"].push(snapshot_item);
+	}
 
-	event_request.call["message"] = message;
-	event_request.call["connection_id"] = request.resources.websocket_connection_id;
-	event_request.call["scope"] = request.resources.websocket_scope;
-	event_request.call["opcode"] = (f64)opcode;
-	event_request.call["document_uri"] = event_request.var["ws"]["document_uri"].to_string();
-
-	compiler_invoke_websocket(&event_request, request.params["SCRIPT_FILENAME"]);
-
-	if(event_request.session_id.length() > 0)
-		save_session_data(event_request.session_id, event_request.session);
-
-	cleanup_mysql_connections();
-	restore_active_request(previous_context);
+	websocket_exec_queue_job(job);
 	return 0;
 }
 
@@ -520,7 +1089,7 @@ void run_proactive_compiler()
 				retry_after.erase(file_name);
 			}
 			background_context.session.clear();
-			background_context.session_serialized = "";
+			background_context.session_loaded_hash = "";
 			clear_shared_unit_cache(server_state);
 			usleep(250000);
 			continue;
@@ -548,6 +1117,7 @@ void ensure_proactive_compiler()
 	pid_t p = fork();
 	if(p == 0)
 	{
+		file_release_process_locks("proactive compiler fork");
 		prctl(PR_SET_PDEATHSIG, SIGHUP);
 		run_proactive_compiler();
 		exit(0);
@@ -579,9 +1149,14 @@ void listen_for_connections()
 	server.on_data = &handle_data;
 	server.on_complete = &handle_complete;
 	server.on_websocket_message = &handle_websocket_message;
+	if(worker_accepts_http)
+		ensure_websocket_executor();
 	for(;;)
 	{
-		server.process();
+		file_release_process_locks("worker loop cleanup");
+		server.process(worker_accepts_http ? 50 : -1);
+		if(worker_accepts_http)
+			websocket_exec_tick();
 	}
 }