getting closer to full port of web app starter

src/fastcgi/src/fcgicc.cc

@@ -102,7 +102,7 @@ FastCGIServer::shutdown()
 
 	for (std::vector<std::string>::iterator it = listen_unlink.begin();
 		it != listen_unlink.end(); ++it)
-		unlink(it->c_str());
+		file_unlink(*it);
 
 	for (std::map<int, Connection*>::iterator it = client_sockets.begin();
 		it != client_sockets.end(); ++it)
@@ -188,7 +188,7 @@ FastCGIServer::listen(const std::string& local_path)
 
 		std::memcpy(sa.sun_path, local_path.data(), size);
 
-		unlink(local_path.c_str());
+		file_unlink(local_path);
 		try {
 			if (bind(server_socket, (struct sockaddr*)&sa,
 				sizeof(sa) - (sizeof(sa.sun_path) - size - 1)) == -1)
@@ -204,7 +204,7 @@ FastCGIServer::listen(const std::string& local_path)
 			printf("(P) listening to #%i socket %s\n", server_socket, local_path.c_str());
 
 		} catch (...) {
-			unlink(local_path.c_str());
+			file_unlink(local_path);
 			throw;
 		}
 
@@ -350,7 +350,7 @@ FastCGIServer::process(int timeout_ms)
 				FastCGIRequest* new_request = new FastCGIRequest();
 				new_request->resources.client_socket = client_socket;
 				new_request->resources.server_socket = socket_handle;
-				new_request->stats.time_init = microtime();
+				new_request->stats.time_init = time_precise();
 				client_sockets[client_socket]->requests[client_socket] = new_request;
 			}
 		}
@@ -467,7 +467,7 @@ FastCGIServer::process_http_request(FastCGIRequest& request, String& data)
 			request.params["SCRIPT_FILENAME"] = request.params["HTTP_SCRIPT_FILENAME"];
 		else if(request.params["SCRIPT_FILENAME"] == "" && request.params["DOCUMENT_URI"] != "")
 		{
-			String document_root = first(request.params["DOCUMENT_ROOT"], get_cwd());
+			String document_root = first(request.params["DOCUMENT_ROOT"], cwd_get());
 			if(document_root.length() > 1 && document_root[document_root.length()-1] == '/')
 				document_root.resize(document_root.length()-1);
 			request.params["DOCUMENT_ROOT"] = document_root;
@@ -899,7 +899,7 @@ FastCGIServer::read_fgci(Connection& connection)
 			FastCGIRequest* new_request = new FastCGIRequest();
 			new_request->resources.client_socket = connection.client_socket;
 			new_request->resources.server_socket = connection.server_socket;
-			new_request->stats.time_init = microtime();
+			new_request->stats.time_init = time_precise();
 			connection.requests[request_id] = new_request;
 
 			break;
@@ -1019,7 +1019,7 @@ FastCGIServer::assemble_output_buffer(FastCGIRequest& request, Connection* conne
 	}
 	request.ob_stack.clear();
 	request.flags.output_closed = true;
-	request.stats.time_end = microtime();
+	request.stats.time_end = time_precise();
 	if(request.flags.log_request)
 		printf("(r) pid:%i\t%s\t%0.6fs\tfps:%0.0f\tout:%0.1fkB\tmem:%0.0f/%0.0fkB\n",
 			my_pid,

src/lib/compiler.h

@@ -1,8 +1,8 @@
 #pragma once
 
-#define RENDER(X) extern "C" void render(Request& context)
-#define COMPONENT(X) extern "C" void component_render(Request& context)
-#define WS(X) extern "C" void websocket(Request& context)
+#define RENDER(X) extern "C" void __unit_render(Request& context)
+#define COMPONENT(X) extern "C" void __unit_component(Request& context)
+#define WS(X) extern "C" void __unit_websocket(Request& context)
 #define EXPORT extern "C"
 
 String process_html_literal(Request* context, SharedUnit* su, String content);
@@ -14,20 +14,28 @@ SharedUnit* get_shared_unit(Request* context, String file_name, bool opt_so_opti
 void compiler_invoke(Request* context, String file_name);
 void compiler_invoke_websocket(Request* context, String file_name);
 SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String current_path = "", bool opt_so_optional = false);
+String compiler_site_directory(Request* context);
+StringList compiler_scan_site_units(Request* context);
+StringList compiler_list_known_units(Request* context);
+void compiler_set_known_units(Request* context, StringList files);
+void compiler_track_known_unit(Request* context, String file_name);
+void compiler_untrack_known_unit(Request* context, String file_name);
+bool compiler_unit_needs_recompile(Request* context, String file_name, bool* source_missing = 0);
+DTree unit_info(String path = "");
+StringList units_list();
+bool unit_compile(String path = "");
 
-SharedUnit* load_file(String file_name);
-void render_file(String file_name);
-void render_file(String file_name, Request& context);
-DTree* call_file(String file_name, String function_name, DTree* call_param = 0);
+SharedUnit* unit_load(String file_name);
+void unit_render(String file_name);
+void unit_render(String file_name, Request& context);
+DTree* unit_call(String file_name, String function_name, DTree* call_param = 0);
 String component_resolve(String name);
 bool component_exists(String name);
-void render_component(String name);
-void render_component(String name, Request& context);
-void render_component(String name, DTree props);
-void render_component(String name, DTree props, Request& context);
+void component_render(String name);
+void component_render(String name, Request& context);
+void component_render(String name, DTree props);
+void component_render(String name, DTree props, Request& context);
 String component(String name);
 String component(String name, Request& context);
 String component(String name, DTree props);
 String component(String name, DTree props, Request& context);
-
-StringList precompile_jobs;

src/lib/dtree.cpp

@@ -16,6 +16,20 @@ TreePtr dtree_resolve_reference(TreePtr tree)
 	return(tree);
 }
 
+bool dtree_key_is_index(String key, s64 expected_index = -1)
+{
+	if(key == "")
+		return(false);
+	for(auto c : key)
+	{
+		if(!isdigit(c))
+			return(false);
+	}
+	if(expected_index >= 0)
+		return(key == std::to_string(expected_index));
+	return(true);
+}
+
 }
 
 void DTree::each(std::function <void (DTree t, String key)> f)
@@ -40,6 +54,23 @@ bool DTree::is_array()
 	return(deref().type == 'M');
 }
 
+bool DTree::is_list() const
+{
+	const DTree& target = deref();
+	if(target.type != 'M')
+		return(false);
+	if(target._map.size() == 0)
+		return(target._list_mode);
+	s64 expected_index = 0;
+	for(const auto& entry : target._map)
+	{
+		if(!dtree_key_is_index(entry.first, expected_index))
+			return(false);
+		expected_index += 1;
+	}
+	return(true);
+}
+
 String DTree::to_string()
 {
 	const DTree& target = deref();
@@ -213,6 +244,7 @@ void DTree::set_type(char t)
 			case('M'):
 				_map.clear();
 				_array_index = 0;
+				_list_mode = false;
 				break;
 		}
 	}
@@ -228,6 +260,7 @@ void DTree::set(String s)
 	}
 	set_type('S');
 	_String = s;
+	_list_mode = false;
 }
 
 void DTree::set(void* p)
@@ -240,6 +273,7 @@ void DTree::set(void* p)
 	}
 	set_type('P');
 	_ptr = p;
+	_list_mode = false;
 }
 
 void DTree::set(f64 f)
@@ -252,6 +286,7 @@ void DTree::set(f64 f)
 	}
 	set_type('F');
 	_float = f;
+	_list_mode = false;
 }
 
 void DTree::set_bool(bool b)
@@ -264,6 +299,7 @@ void DTree::set_bool(bool b)
 	}
 	set_type('B');
 	_bool = b;
+	_list_mode = false;
 }
 
 void DTree::set(DTree source)
@@ -279,21 +315,28 @@ void DTree::set(DTree source)
 	{
 		case('S'):
 			_String = source._String;
+			_list_mode = false;
 			break;
 		case('F'):
 			_float = source._float;
+			_list_mode = false;
 			break;
 		case('B'):
 			_bool = source._bool;
+			_list_mode = false;
 			break;
 		case('M'):
 			_map = source._map;
+			_array_index = source._array_index;
+			_list_mode = source._list_mode;
 			break;
 		case('P'):
 			_ptr = source._ptr;
+			_list_mode = false;
 			break;
 		case('R'):
 			_ptr = source._ptr;
+			_list_mode = false;
 			break;
 	}
 }
@@ -307,12 +350,28 @@ void DTree::set(StringMap source)
 		return;
 	}
 	set_type('M');
+	_array_index = 0;
+	_list_mode = false;
 	for (auto it = source.begin(); it != source.end(); ++it)
 	{
 		_map[it->first] = it->second;
 	}
 }
 
+void DTree::set_array()
+{
+	DTree* target = reference_target();
+	if(target)
+	{
+		target->set_array();
+		return;
+	}
+	type = 'M';
+	_map.clear();
+	_array_index = 0;
+	_list_mode = true;
+}
+
 void DTree::set_reference(DTree* target)
 {
 	type = 'R';
@@ -333,6 +392,8 @@ DTree& DTree::operator [] (String s) {
 	if(target)
 		return((*target)[s]);
 	set_type('M');
+	if(_list_mode && !dtree_key_is_index(s))
+		_list_mode = false;
 	return(_map[s]);
 }
 
@@ -351,6 +412,25 @@ void DTree::push(DTree& child)
 		return;
 	}
 	set_type('M');
+	if(_map.size() == 0)
+	{
+		_list_mode = true;
+		_array_index = 0;
+	}
+	else
+	{
+		if(is_list())
+		{
+			_list_mode = true;
+			_array_index = _map.size();
+		}
+		else
+		{
+			_list_mode = false;
+			while(_map.find(std::to_string(_array_index)) != _map.end())
+				_array_index += 1;
+		}
+	}
 	_map[std::to_string(_array_index)] = child;
 	_array_index += 1;
 }
@@ -364,6 +444,8 @@ DTree DTree::pop()
 	auto last = _map.rbegin();
 	DTree result = last->second;
 	_map.erase(last->first);
+	if(_list_mode)
+		_array_index = _map.size();
 	return(result);
 }
 
@@ -377,6 +459,8 @@ void DTree::remove(String s)
 	}
 	set_type('M');
 	_map.erase(s);
+	if(_map.size() == 0)
+		_array_index = 0;
 }
 
 void DTree::clear()
@@ -389,6 +473,7 @@ void DTree::clear()
 	}
 	set_type('M');
 	_map.clear();
+	_array_index = 0;
 }
 
 String to_String(DTree t)

src/lib/dtree.h

@@ -8,13 +8,15 @@ struct DTree {
 
 	String 	_String;
 	f64 	_float;
-	s64 	_array_index;
+	s64 	_array_index = 0;
 	bool	_bool;
+	bool	_list_mode = false;
 	void*	_ptr;
 	std::map<String, DTree> _map;
 
 	void each(std::function <void (DTree t, String key)> f);
 	bool is_array();
+	bool is_list() const;
 	String to_string();
 	String to_json(char quote_char = '"');
 	String get_type_name();
@@ -31,6 +33,7 @@ struct DTree {
 	void set_bool(bool b);
 	void set(DTree source);
 	void set(StringMap source);
+	void set_array();
 	void set_reference(DTree* target);
 	DTree* key(String s);
 	DTree& operator [] (String s);

src/lib/functionlib.cpp

@@ -396,19 +396,41 @@ String json_encode(DTree t, char quote_char)
 	String result = "";
 	if(t.is_array())
 	{
-		result += "{";
-		u32 count = 0;
-		t.each([&] (DTree item, String key) {
-			if(count > 0)
-				result += ", ";
-			count += 1;
-			result += json_escape(key, quote_char) + ": " + json_encode(item, quote_char);
-		});
-		result += "}";
+		if(t.is_list())
+		{
+			result += "[";
+			u32 count = 0;
+			for(u32 i = 0; i < t.deref()._map.size(); i += 1)
+			{
+				if(count > 0)
+					result += ", ";
+				count += 1;
+				auto it = t.deref()._map.find(std::to_string(i));
+				if(it == t.deref()._map.end())
+				{
+					result += "null";
+					continue;
+				}
+				result += json_encode(it->second, quote_char);
+			}
+			result += "]";
+		}
+		else
+		{
+			result += "{";
+			u32 count = 0;
+			t.each([&] (DTree item, String key) {
+				if(count > 0)
+					result += ", ";
+				count += 1;
+				result += json_escape(key, quote_char) + ": " + json_encode(item, quote_char);
+			});
+			result += "}";
+		}
 	}
 	else
 	{
-		result = t.to_json();
+		result = t.to_json(quote_char);
 	}
 	return(result);
 }
@@ -469,6 +491,7 @@ String json_decode_String(String s, u32& i, char termination_char)
 }
 
 DTree json_decode_map(String s, u32& i);
+DTree json_decode_array(String s, u32& i);
 
 void json_consume_space(String s, u32& i)
 {
@@ -542,6 +565,11 @@ DTree json_decode_value(String s, u32& i)
 		i += 1;
 		return(json_decode_map(s, i));
 	}
+	else if(c == '[')
+	{
+		i += 1;
+		return(json_decode_array(s, i));
+	}
 	else
 	{
 		value = json_decode_keyword(s, i);
@@ -599,6 +627,33 @@ DTree json_decode_map(String s, u32& i)
 	return(result);
 }
 
+DTree json_decode_array(String s, u32& i)
+{
+	DTree result;
+	result.set_array();
+	json_consume_space(s, i);
+	while(i < s.length())
+	{
+		char c = s[i];
+		if(c == ']')
+		{
+			i += 1;
+			return(result);
+		}
+		else if(c == ',')
+		{
+			i += 1;
+		}
+		else
+		{
+			DTree v = json_decode_value(s, i);
+			result.push(v);
+		}
+		json_consume_space(s, i);
+	}
+	return(result);
+}
+
 DTree json_decode(String s)
 {
 	u32 i = 0;

src/lib/sys.cpp

@@ -214,12 +214,12 @@ bool file_put_contents(String file_name, String content)
 	return(true);
 }
 
-String get_cwd()
+String cwd_get()
 {
 	return(std::filesystem::current_path());
 }
 
-void set_cwd(String path)
+void cwd_set(String path)
 {
 	chdir(path.c_str());
 }
@@ -237,7 +237,7 @@ time_t file_mtime(String file_name)
 	}
 }
 
-void unlink(String file_name)
+void file_unlink(String file_name)
 {
 	remove(file_name.c_str());
 }
@@ -247,7 +247,7 @@ String expand_path(String path, String relative_to_path)
 	String result;
 
 	if(relative_to_path == "")
-		relative_to_path = get_cwd();
+		relative_to_path = cwd_get();
 
 	auto base_path = split(relative_to_path, "/");
 	auto rel_path = split(path, "/");
@@ -271,7 +271,7 @@ String expand_path(String path, String relative_to_path)
 	return(join(base_path, "/"));
 }
 
-f64 microtime()
+f64 time_precise()
 {
 	return ((f64)std::chrono::duration_cast<std::chrono::microseconds>(
 		std::chrono::high_resolution_clock::now().time_since_epoch()).count()) / 1000000;
@@ -282,7 +282,7 @@ u64 time()
 	return(std::time(0));
 }
 
-String date(String format, u64 timestamp)
+String time_format_local(String format, u64 timestamp)
 {
 	String ts;
 	String fmt;
@@ -293,7 +293,7 @@ String date(String format, u64 timestamp)
 	return(trim(shell_exec("date "+ts+" "+fmt)));
 }
 
-String gmdate(String format, u64 timestamp)
+String time_format_utc(String format, u64 timestamp)
 {
 	String ts;
 	String fmt;
@@ -306,7 +306,7 @@ String gmdate(String format, u64 timestamp)
 	return(trim(shell_exec("date -u "+ts+" "+fmt)));
 }
 
-u64 parse_time(String time_String)
+u64 time_parse(String time_String)
 {
 	return(int_val(trim(shell_exec("date -u -d "+shell_escape(time_String)+" +'%s'"))));
 }
@@ -503,6 +503,11 @@ pid_t spawn_subprocess(std::function<void()> exec_after_spawn)
 	}
 }
 
+int task_kill(pid_t pid, int sig)
+{
+	return(kill(pid, sig));
+}
+
 pid_t task_pid(String key)
 {
 	String status_file_name = context->server->config["BIN_DIRECTORY"] + "/task-" + key;
@@ -511,9 +516,9 @@ pid_t task_pid(String key)
 	if(status_file != "")
 	{
 		p = int_val(status_file);
-		if(kill(p, 0) == 0) // process is still running
+		if(task_kill(p, 0) == 0) // process is still running
 			return(p);
-		unlink(status_file_name);
+		file_unlink(status_file_name);
 	}
 	return(p);
 }
@@ -526,13 +531,13 @@ pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout)
 	if(status_file != "")
 	{
 		p = int_val(status_file);
-		if(kill(p, 0) == 0) // process is still running
+		if(task_kill(p, 0) == 0) // process is still running
 		{
 			printf("(P) worker process '%s' already running: PID %i\n", key.c_str(), p);
 			return(p);
 		}
 		//printf("(P) worker process '%s' had crashed: PID %i\n", key.c_str(), p);
-		unlink(status_file_name);
+		file_unlink(status_file_name);
 	}
 	p = fork();
 	if(p == 0)
@@ -545,7 +550,7 @@ pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout)
 		//printf("(C) child procress started, PID:%i\n", my_pid);
 		//prctl(PR_SET_PDEATHSIG, SIGHUP);
 		exec_after_spawn();
-		unlink(status_file_name);
+		file_unlink(status_file_name);
 		printf("(P) worker process '%s' terminated: PID %i\n", key.c_str(), my_pid);
 		exit(0);
 	}
@@ -603,7 +608,9 @@ StringMap make_server_settings()
 	cfg["TMP_UPLOAD_PATH"] = "/tmp/uce/uploads";
 	cfg["SESSION_PATH"] = "/tmp/uce/sessions";
 	cfg["COMPILER_SYS_PATH"] = ".";
-	cfg["PRECOMPILE_FILES_IN"] = ".";
+	cfg["PRECOMPILE_FILES_IN"] = "";
+	cfg["SITE_DIRECTORY"] = "site";
+	cfg["PROACTIVE_COMPILE_CHECK_INTERVAL"] = std::to_string(60);
 
 	cfg["HTTP_PORT"] = std::to_string(8080);
 	cfg["SESSION_TIME"] = std::to_string(60*60*24*30);

src/lib/sys.h

@@ -21,18 +21,18 @@ bool file_append(String file_name, Ts... args)
 	fout.close();
 	return(true);
 }
-String get_cwd();
-void set_cwd(String path);
+String cwd_get();
+void cwd_set(String path);
 time_t file_mtime(String file_name);
-void unlink(String file_name);
+void file_unlink(String file_name);
 String expand_path(String path, String relative_to_path = "");
 StringList ls(String dir);
 
-f64 microtime();
+f64 time_precise();
 u64 time();
-String date(String format = "", u64 timestamp = 0);
-String gmdate(String format = "", u64 timestamp = 0);
-u64 parse_time(String time_String);
+String time_format_local(String format = "", u64 timestamp = 0);
+String time_format_utc(String format = "", u64 timestamp = 0);
+u64 time_parse(String time_String);
 
 u64 socket_connect(String host, short port);
 void socket_close(u64 sockfd);
@@ -66,6 +66,7 @@ pid_t parent_pid = 0;
 pid_t my_pid = 0;
 
 void on_segfault(int sig);
+int task_kill(pid_t pid, int sig = 0);
 
 pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout = 60*10);
 pid_t task_repeat(String key, f64 interval, std::function<void()> exec_after_spawn, u64 timeout = 60*10);

src/lib/types.h

@@ -76,15 +76,39 @@ struct SharedUnit {
 	String bin_file_name;
 	String pre_file_name;
 
-	void* so_handle;
+	void* so_handle = 0;
 
-	request_handler on_setup;
-	request_ref_handler on_render;
-	request_ref_handler on_component;
-	request_ref_handler on_websocket;
+	request_handler on_setup = 0;
+	request_ref_handler on_render = 0;
+	request_ref_handler on_component = 0;
+	request_ref_handler on_websocket = 0;
 
 	String compiler_messages;
-	time_t last_compiled;
+	String compile_status = "unknown";
+	String compile_error_status = "";
+	String runtime_error_status = "";
+	time_t last_compiled = 0;
+	time_t last_loaded = 0;
+	time_t last_rendered = 0;
+	time_t last_error = 0;
+
+	u64 request_count = 0;
+	u64 invoke_count = 0;
+	u64 runtime_error_count = 0;
+
+	u64 compile_count = 0;
+	u64 compile_success_count = 0;
+	u64 compile_failure_count = 0;
+
+	f64 last_compile_duration = 0;
+	f64 total_compile_duration = 0;
+	f64 best_compile_duration = 0;
+	f64 worst_compile_duration = 0;
+
+	f64 last_render_duration = 0;
+	f64 total_render_duration = 0;
+	f64 best_render_duration = 0;
+	f64 worst_render_duration = 0;
 
 	bool opt_so_optional = false;
 
@@ -100,6 +124,7 @@ struct UploadedFile {
 struct ServerState {
 
 	std::map<String, SharedUnit*> units;
+	std::map<String, bool> known_unit_files;
 	StringMap config;
 	u32 request_count = 0;
 
@@ -120,7 +145,7 @@ void compiler_invoke(Request* context, String file_name);
 
 struct Request {
 
-	ServerState* server;
+	ServerState* server = 0;
 
 	StringMap params;
 	StringMap get;
@@ -141,11 +166,11 @@ struct Request {
 	StringMap header;
 	StringList set_cookies;
 
-	u64 random_seed;
-	u64 random_index;
+	u64 random_seed = 0;
+	u64 random_index = 0;
 
 	std::vector<ByteStream*> ob_stack;
-	ByteStream* ob;
+	ByteStream* ob = 0;
 
 	String in;
 	String out;
@@ -161,10 +186,10 @@ struct Request {
 	} flags;
 
 	struct Stats {
-		u32 bytes_written;
-		f64 time_init;
-		f64 time_start;
-		f64 time_end;
+		u32 bytes_written = 0;
+		f64 time_init = 0;
+		f64 time_start = 0;
+		f64 time_end = 0;
 		u64 mem_high = 0;
 		u64 mem_alloc = 0;
 		u32 invoke_count = 0;

src/lib/uri.cpp

@@ -291,7 +291,7 @@ String make_upload_tmp_name()
 	String upload_path = context->server->config["TMP_UPLOAD_PATH"];
 	if(upload_path.length() > 0 && upload_path[upload_path.length()-1] != '/')
 		upload_path.append(1, '/');
-	return(upload_path + make_session_id());
+	return(upload_path + session_id_create());
 }
 
 StringMap parse_multipart(String q, String boundary, std::vector<UploadedFile>& uploaded_files)
@@ -495,7 +495,7 @@ void set_cookie(
 	String cookie = "Set-Cookie: ";
 	cookie.append(uri_encode(name) + "=" + uri_encode(value));
 	if(expires > 0)
-		cookie.append(String("; Expires=") + gmdate("RFC1123", expires));
+		cookie.append(String("; Expires=") + time_format_utc("RFC1123", expires));
 	context->set_cookies.push_back(cookie);
 	context->cookies[name] = value;
 }
@@ -512,7 +512,7 @@ StringMap parse_cookies(String cookie_String)
 	return(result);
 }
 
-String make_session_id()
+String session_id_create()
 {
 	return(to_hex(rand())+to_hex(rand())+to_hex(rand())+to_hex(rand()));
 }
@@ -563,7 +563,7 @@ String session_start(String session_name)
 
 	if(session_id.length() == 0)
 	{
-		session_id = make_session_id();
+		session_id = session_id_create();
 		set_cookie(session_name, session_id, time() + int_val(context->server->config["SESSION_TIME"]));
 	}
 	context->session_id = session_id;

src/lib/uri.h

@@ -14,7 +14,7 @@ void set_cookie(
 	u64 expires = 0, String path = "/", String domain = "",
 	bool secure = false, bool http_only = true);
 StringMap parse_cookies(String cookie_String);
-String make_session_id();
+String session_id_create();
 StringMap load_session_data(String session_id);
 void save_session_data(String session_id, StringMap data);
 String session_start(String session_name = "uce-session");

src/linux_fastcgi.cpp

@@ -7,6 +7,7 @@ ServerState server_state;
 
 FastCGIServer server;
 pid_t http_worker_pid = 0;
+pid_t proactive_compiler_pid = 0;
 bool worker_accepts_http = false;
 static sigjmp_buf request_fault_jmp;
 static volatile sig_atomic_t request_fault_active = 0;
@@ -134,7 +135,7 @@ String normalize_ws_scope(String scope)
 		return(current_ws_scope());
 	if(scope[0] == '/')
 		return(scope);
-	return(expand_path(scope, get_cwd()));
+	return(expand_path(scope, cwd_get()));
 }
 
 String ws_message()
@@ -235,7 +236,7 @@ int handle_complete(FastCGIRequest& request) {
 	Request* previous_context = set_active_request(request);
 	server_state.request_count += 1;
 	request.server = &server_state;
-	request.stats.time_start = microtime();
+	request.stats.time_start = time_precise();
 	//request.stats.mem_alloc = 0;
 	//request.stats.mem_high = 0;
     request.header["Content-Type"] = context->server->config["CONTENT_TYPE"];
@@ -307,7 +308,7 @@ int handle_complete(FastCGIRequest& request) {
 
 	for( auto &f : request.uploaded_files)
 	{
-		unlink(f.tmp_name);
+		file_unlink(f.tmp_name);
 	}
 
 	if(failure_title == "" && request.session_id.length() > 0)
@@ -333,7 +334,7 @@ int handle_websocket_message(FastCGIRequest& request, const String& message, u8
 	event_request.resources = request.resources;
 	if(event_request.resources.websocket_connection_state)
 		event_request.connection.set_reference(event_request.resources.websocket_connection_state);
-	event_request.stats.time_init = microtime();
+	event_request.stats.time_init = time_precise();
 	event_request.stats.time_start = event_request.stats.time_init;
 	event_request.random_index = 0;
 	event_request.random_seed = gen_noise64(*reinterpret_cast<u64*>(&event_request.stats.time_start));
@@ -387,19 +388,156 @@ void on_terminate(int sig)
 	exit(1);
 }
 
-void listen_for_connections()
+void clear_shared_unit_cache(ServerState& state)
 {
-	if(precompile_jobs.size() > 0)
+	for(auto& it : state.units)
+		delete it.second;
+	state.units.clear();
+}
+
+void close_inherited_server_sockets()
+{
+	for(auto socket_handle : server.server_sockets)
+		close(socket_handle);
+	server.server_sockets.clear();
+	server.server_socket_types.clear();
+}
+
+bool proactive_compile_queue_has(StringList& queue, String file_name)
+{
+	return(std::find(queue.begin(), queue.end(), file_name) != queue.end());
+}
+
+void proactive_compile_queue_push(StringList& queue, String file_name)
+{
+	if(file_name == "" || proactive_compile_queue_has(queue, file_name))
+		return;
+	queue.push_back(file_name);
+}
+
+void run_proactive_compiler()
+{
+	Request background_context;
+	StringList compile_queue;
+	f64 check_interval = float_val(server_state.config["PROACTIVE_COMPILE_CHECK_INTERVAL"]);
+	f64 failure_retry_interval = 0;
+	f64 next_scan_at = 0;
+	std::map<String, f64> retry_after;
+	if(check_interval < 1)
+		check_interval = 1;
+	failure_retry_interval = std::max(check_interval, 60.0);
+
+	my_pid = getpid();
+	context = &background_context;
+	background_context.server = &server_state;
+
+	close_inherited_server_sockets();
+	signal(SIGSEGV, on_segfault);
+	signal(SIGABRT, on_segfault);
+	signal(SIGBUS, on_segfault);
+	signal(SIGILL, on_segfault);
+	signal(SIGFPE, on_segfault);
+	signal(SIGPIPE, SIG_IGN);
+	setpriority(PRIO_PROCESS, 0, 10);
+
+	auto known_units = compiler_list_known_units(&background_context);
+	auto site_units = compiler_scan_site_units(&background_context);
+	known_units.insert(known_units.end(), site_units.begin(), site_units.end());
+	compiler_set_known_units(&background_context, known_units);
+	next_scan_at = time_precise();
+
+	for(;;)
 	{
-		context = new Request();
-		context->server = &server_state;
-		for(auto s : precompile_jobs)
+		if(compile_queue.size() == 0 && time_precise() >= next_scan_at)
 		{
-			printf("- worker [%i] precompile %s\n", getpid(), s.c_str());
-			get_shared_unit(context, s, false);
+			auto tracked_units = compiler_list_known_units(&background_context);
+			StringList existing_units;
+
+			for(auto& file_name : tracked_units)
+			{
+				bool source_missing = false;
+				auto retry_it = retry_after.find(file_name);
+				bool retry_allowed = (retry_it == retry_after.end() || time_precise() >= retry_it->second);
+				if(compiler_unit_needs_recompile(&background_context, file_name, &source_missing) && retry_allowed)
+					proactive_compile_queue_push(compile_queue, file_name);
+				if(source_missing)
+				{
+					printf("(i) proactive compiler forget removed unit %s\n", file_name.c_str());
+					retry_after.erase(file_name);
+					continue;
+				}
+				existing_units.push_back(file_name);
+			}
+
+			if(existing_units.size() != tracked_units.size())
+				compiler_set_known_units(&background_context, existing_units);
+
+			next_scan_at = time_precise() + check_interval;
 		}
+
+		if(compile_queue.size() > 0)
+		{
+			auto file_name = compile_queue.front();
+			compile_queue.erase(compile_queue.begin());
+			bool source_missing = false;
+			auto retry_it = retry_after.find(file_name);
+			if(retry_it != retry_after.end() && time_precise() < retry_it->second)
+				continue;
+			if(compiler_unit_needs_recompile(&background_context, file_name, &source_missing))
+			{
+				printf("(i) proactive compile %s\n", file_name.c_str());
+				auto su = get_shared_unit(&background_context, file_name, false);
+				if(su && su->compiler_messages == "")
+					retry_after.erase(file_name);
+				else
+					retry_after[file_name] = time_precise() + failure_retry_interval;
+			}
+			else if(source_missing)
+			{
+				printf("(i) proactive compiler forget removed unit %s\n", file_name.c_str());
+				compiler_untrack_known_unit(&background_context, file_name);
+				retry_after.erase(file_name);
+			}
+			else
+			{
+				retry_after.erase(file_name);
+			}
+			clear_shared_unit_cache(server_state);
+			usleep(250000);
+			continue;
+		}
+
+		usleep(250000);
+	}
+}
+
+bool proactive_compiler_alive()
+{
+	return(proactive_compiler_pid > 0 && task_kill(proactive_compiler_pid, 0) == 0);
+}
+
+void ensure_proactive_compiler()
+{
+	if(float_val(server_state.config["PROACTIVE_COMPILE_CHECK_INTERVAL"]) <= 0)
+		return;
+
+	if(proactive_compiler_alive())
+		return;
+
+	pid_t p = fork();
+	if(p == 0)
+	{
+		prctl(PR_SET_PDEATHSIG, SIGHUP);
+		run_proactive_compiler();
+		exit(0);
 	}
 
+	proactive_compiler_pid = p;
+	printf("(P) proactive compiler spawned: PID %i\n", p);
+}
+
+void listen_for_connections()
+{
 	signal(SIGSEGV, on_segfault);
 	signal(SIGABRT, on_segfault);
 	signal(SIGBUS, on_segfault);
@@ -431,7 +569,7 @@ void init_base_process()
 	printf("(P) Starting parent server PID:%i\n", getpid());
 
 	server_state.config = make_server_settings();
-	server_state.config["COMPILER_SYS_PATH"] = get_cwd();
+	server_state.config["COMPILER_SYS_PATH"] = cwd_get();
 	printf("Compiler base path: %s\n", server_state.config["COMPILER_SYS_PATH"].c_str());
 
 	server_state.config["COMPILE_SCRIPT"] =
@@ -461,52 +599,20 @@ void init_base_process()
 	srand(time());
 }
 
-StringList init_precompile(u32& precompile_jobs_per_worker)
-{
-	StringList precompile_jobs_pending;
-	if(server_state.config["PRECOMPILE_FILES_IN"] != "" && int_val(server_state.config["WORKER_COUNT"]) >= 2)
-	{
-		if(server_state.config["PRECOMPILE_FILES_IN"][0] != '/')
-			server_state.config["PRECOMPILE_FILES_IN"] = expand_path(server_state.config["PRECOMPILE_FILES_IN"]);
-		precompile_jobs_pending = split(trim(shell_exec(
-			"find " +
-			shell_escape(server_state.config["PRECOMPILE_FILES_IN"]) +
-			" -iname '*.uce' ")), "\n");
-		precompile_jobs_per_worker = 1 + (precompile_jobs_pending.size() / (int_val(server_state.config["WORKER_COUNT"]) -1));
-	}
-	return(precompile_jobs_pending);
-}
-
 int main(int argc, char** argv)
 {
-	StringList precompile_jobs_pending;
-	u32 precompile_jobs_per_worker = 1;
-
 	init_base_process();
-	precompile_jobs_pending = init_precompile(precompile_jobs_per_worker);
-
-	s32 worker_spawn_count = 0;
+	ensure_proactive_compiler();
 
 	for(;;)
 	{
+		if(!proactive_compiler_alive())
+			proactive_compiler_pid = 0;
+		if(!termination_signal_received)
+			ensure_proactive_compiler();
+
 		while(workers.size() < int_val(server_state.config["WORKER_COUNT"]))
 		{
-			worker_spawn_count++;
-			precompile_jobs.clear();
-			// spawn workers with precompile jobs if necessary but
-			// leave the first worker alone so it can start responding
-			// to requests right away
-			if(precompile_jobs_pending.size() > 0 && worker_spawn_count > 1)
-			{
-				for(u32 i = 0; i < precompile_jobs_per_worker; i++)
-				{
-					if(precompile_jobs_pending.size() > 0)
-					{
-						precompile_jobs.push_back(precompile_jobs_pending.back());
-						precompile_jobs_pending.pop_back();
-					}
-				}
-			}
 			if(!termination_signal_received)
 			{
 				worker_accepts_http = (http_worker_pid == 0 || workers.count(http_worker_pid) == 0);