fix: harden UCE runtime and starter

src/lib/compiler-parser.cpp

@@ -316,6 +316,118 @@ String compiler_process_text_literal(Request* context, SharedUnit* su, String co
 	return(parsed_content);
 }
 
+bool compiler_line_starts_entrypoint(String trimmed, String macro_name)
+{
+	if(trimmed.rfind(macro_name + "(", 0) == 0)
+		return(true);
+	return(trimmed.rfind(macro_name + ":", 0) == 0 && trimmed.find("(") != String::npos);
+}
+
+bool compiler_line_starts_fragmentable_entrypoint(String trimmed, String& kind)
+{
+	if(compiler_line_starts_entrypoint(trimmed, "ONCE"))
+	{
+		kind = "ONCE";
+		return(true);
+	}
+	if(compiler_line_starts_entrypoint(trimmed, "RENDER"))
+	{
+		kind = "RENDER";
+		return(true);
+	}
+	if(compiler_line_starts_entrypoint(trimmed, "COMPONENT"))
+	{
+		kind = "COMPONENT";
+		return(true);
+	}
+	return(false);
+}
+
+String compiler_fragment_capture_prelude(String slot)
+{
+	return("\nstruct __UceFragmentCapture { Request& context; String slot; __UceFragmentCapture(Request& c, String s) : context(c), slot(s) { ob_start(); } ~__UceFragmentCapture() { String html = ob_get_close(); if(html != \"\") context.call[\"fragments\"][slot] = context.call[\"fragments\"][slot].to_string() + html; } }; __UceFragmentCapture __uce_fragment_capture(context, " + compiler_cpp_string_literal(slot) + ");\n");
+}
+
+String compiler_rewrite_fragment_attributes(String content)
+{
+	StringList lines = split(content, "\n");
+	String result;
+	for(u32 i = 0; i < lines.size(); i++)
+	{
+		String line = lines[i];
+		String kind;
+		if(!compiler_line_starts_fragmentable_entrypoint(trim(line), kind))
+		{
+			result += line;
+			if(i + 1 < lines.size())
+				result += "\n";
+			continue;
+		}
+
+		String slot = (kind == "ONCE" ? "once" : "");
+		StringList pending;
+		bool has_fragment_attr = false;
+		u32 j = i + 1;
+		while(j < lines.size())
+		{
+			String attr_line = lines[j];
+			String attr_trimmed = trim(attr_line);
+			if(attr_trimmed.rfind("@fragment", 0) == 0 && (attr_trimmed.length() == 9 || isspace((unsigned char)attr_trimmed[9])))
+			{
+				slot = trim(attr_trimmed.substr(9));
+				has_fragment_attr = true;
+				j++;
+				continue;
+			}
+			break;
+		}
+
+		bool should_capture = (slot != "") && (kind == "ONCE" || has_fragment_attr);
+		if(!should_capture)
+		{
+			result += line;
+			if(j < lines.size())
+				result += "\n";
+			i = j - 1;
+			continue;
+		}
+
+		auto declaration_brace_pos = line.find("{");
+		if(declaration_brace_pos != String::npos)
+		{
+			result += line.substr(0, declaration_brace_pos + 1) + compiler_fragment_capture_prelude(slot) + line.substr(declaration_brace_pos + 1);
+			if(j < lines.size())
+				result += "\n";
+			i = j - 1;
+			continue;
+		}
+
+		result += line;
+		if(j < lines.size())
+			result += "\n";
+
+		while(j < lines.size())
+		{
+			String body_line = lines[j];
+			auto brace_pos = body_line.find("{");
+			if(brace_pos == String::npos)
+			{
+				result += body_line;
+				if(j + 1 < lines.size())
+					result += "\n";
+				j++;
+				continue;
+			}
+			result += body_line.substr(0, brace_pos + 1) + compiler_fragment_capture_prelude(slot) + body_line.substr(brace_pos + 1);
+			if(j + 1 < lines.size())
+				result += "\n";
+			i = j;
+			break;
+		}
+	}
+	return(result);
+}
+
 bool compiler_rewrite_named_entrypoint_line(String& line, String macro_prefix, String symbol_prefix)
 {
 	u32 indent_length = 0;
@@ -484,6 +596,7 @@ String compiler_preprocess_shared_unit_char_wise(Request* context, SharedUnit* s
 
 String compiler_preprocess_source(Request* context, SharedUnit* su, String content)
 {
+	content = compiler_rewrite_fragment_attributes(content);
 	content = compiler_rewrite_named_render_syntax(content);
 	return(compiler_preprocess_shared_unit_char_wise(context, su, content));
 }

src/lib/compiler.cpp

@@ -3,6 +3,7 @@
 #include "hash.h"
 #include <algorithm>
 #include <cstdlib>
+#include <cctype>
 #include <filesystem>
 #include <sys/file.h>
 #include <unistd.h>
@@ -17,7 +18,7 @@ const char* UCE_CLI_SYMBOL = "__uce_cli";
 const char* UCE_SERVE_HTTP_SYMBOL = "__uce_serve_http";
 const char* UCE_ONCE_SYMBOL = "__uce_once";
 const char* UCE_INIT_SYMBOL = "__uce_init";
-const u64 UCE_UNIT_ABI_VERSION = 1;
+const u64 UCE_UNIT_ABI_VERSION = 2;
 
 struct SharedUnitFilesystemState
 {
@@ -282,9 +283,19 @@ auto compiler_with_registry_lock(Request* context, TCallback callback) -> declty
 {
 	auto lock_file_name = compiler_registry_lock_file_name(context);
 	int fdlock = compiler_open_lock_file(lock_file_name, "compiler-registry");
-	auto result = callback();
-	compiler_close_lock_file(fdlock);
-	return(result);
+	if(fdlock == -1)
+		throw std::runtime_error("could not open compiler registry lock: " + lock_file_name);
+	try
+	{
+		auto result = callback();
+		compiler_close_lock_file(fdlock);
+		return(result);
+	}
+	catch(...)
+	{
+		compiler_close_lock_file(fdlock);
+		throw;
+	}
 }
 
 bool compiler_has_known_unit_cached(Request* context, String file_name)
@@ -609,6 +620,20 @@ String preprocess_shared_unit(Request* context, SharedUnit* su)
 	return(compiler_preprocess_source(context, su, content));
 }
 
+String compiler_generated_cpp_path(Request* context, String source_file)
+{
+	if(!context || !context->server || source_file == "")
+		return("");
+	return(context->server->config["BIN_DIRECTORY"] + dirname(source_file) + "/" + basename(source_file) + ".cpp");
+}
+
+String compiler_generated_cpp_path(SharedUnit* su)
+{
+	if(!su)
+		return("");
+	return(su->pre_path + "/" + su->pre_file_name);
+}
+
 void setup_unit_paths(Request* context, SharedUnit* su, String file_name)
 {
 	su->file_name = file_name;
@@ -722,6 +747,72 @@ void load_shared_unit(Request* context, SharedUnit* su)
 	return(result);
 }*/
 
+s64 compiler_first_error_line_for_path(String messages, String path)
+{
+	if(path == "")
+		return(-1);
+	String needle = path + ":";
+	auto pos = messages.find(needle);
+	if(pos == String::npos)
+		return(-1);
+	pos += needle.length();
+	String digits;
+	while(pos < messages.length() && isdigit((unsigned char)messages[pos]))
+	{
+		digits.append(1, messages[pos]);
+		pos += 1;
+	}
+	if(digits == "")
+		return(-1);
+	return(int_val(digits));
+}
+
+String compiler_source_excerpt(String file_name, s64 line_number, u64 radius = 3)
+{
+	if(file_name == "" || line_number <= 0 || !file_exists(file_name))
+		return("");
+	auto lines = split(file_get_contents(file_name), "\n");
+	if(lines.size() == 0)
+		return("");
+	s64 start = line_number - (s64)radius;
+	if(start < 1)
+		start = 1;
+	s64 end = line_number + (s64)radius;
+	if(end > (s64)lines.size())
+		end = lines.size();
+	String result;
+	for(s64 i = start; i <= end; i++)
+	{
+		String marker = (i == line_number ? "> " : "  ");
+		result += marker + std::to_string((u64)i) + " | " + lines[i - 1] + "\n";
+	}
+	return(result);
+}
+
+String compiler_format_compile_failure(SharedUnit* su, String raw_messages)
+{
+	String generated_file = compiler_generated_cpp_path(su);
+	String result = "UCE compile error\n";
+	result += "Source: " + su->file_name + "\n";
+	result += "Generated C++: " + generated_file + "\n";
+	result += "Compile output: " + su->compile_output_file_name + "\n";
+
+	s64 source_line = compiler_first_error_line_for_path(raw_messages, su->file_name);
+	String excerpt = compiler_source_excerpt(su->file_name, source_line);
+	if(excerpt != "")
+		result += "\nSource excerpt:\n" + excerpt;
+	else
+	{
+		s64 generated_line = compiler_first_error_line_for_path(raw_messages, generated_file);
+		String generated_excerpt = compiler_source_excerpt(generated_file, generated_line);
+		if(generated_excerpt != "")
+			result += "\nGenerated C++ excerpt:\n" + generated_excerpt;
+	}
+
+	result += "\nCompiler output:\n" + trim(raw_messages) + "\n";
+	return(result);
+}
+
 void compile_shared_unit(Request* context, SharedUnit* su)
 {
 	f64 comp_start = time_precise();
@@ -737,11 +828,10 @@ void compile_shared_unit(Request* context, SharedUnit* su)
 
 	shell_exec("mkdir -p " + shell_escape(su->pre_path));
 	file_put_contents(su->pre_path + "/" + su->pre_file_name, preprocess_shared_unit(context, su));
-	//file_put_contents(su->setup_file_name, compile_setup_file(context, su));
 	file_put_contents(su->api_file_name, join(su->api_declarations, "\n"));
 
 	if(!su->opt_so_optional)
-		su->compiler_messages = trim(shell_exec(context->server->config["COMPILE_SCRIPT"]+" "+
+		su->compiler_messages = trim(shell_exec(shell_escape(context->server->config["COMPILE_SCRIPT"])+" "+
 			shell_escape(su->src_path)+" "+
 			shell_escape(su->bin_path)+" "+
 			shell_escape(su->file_name)+" "+
@@ -751,9 +841,10 @@ void compile_shared_unit(Request* context, SharedUnit* su)
 
 	if(su->compiler_messages.length() > 0)
 	{
-		file_put_contents(su->compile_output_file_name, su->compiler_messages + "\n");
-		compiler_record_compile_result(su, time_precise() - comp_start, false, "compile_error", su->compiler_messages);
-		printf("%s \n", su->compiler_messages.c_str());
+		String raw_messages = su->compiler_messages;
+		file_put_contents(su->compile_output_file_name, raw_messages + "\n");
+		compiler_record_compile_result(su, time_precise() - comp_start, false, "compile_error", raw_messages);
+		printf("%s \n", compiler_format_compile_failure(su, raw_messages).c_str());
 	}
 	else
 	{
@@ -898,11 +989,17 @@ SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String
 	}
 	else if(su->compiler_messages.length() > 0)
 	{
-		printf("%s\n", su->compiler_messages.c_str());
+		String display_messages = su->compiler_messages;
+		if(su->compile_status == "compile_error")
+			display_messages = compiler_format_compile_failure(su, su->compiler_messages);
+		printf("%s\n", display_messages.c_str());
 		if(compiler_can_write_response(context) && context->stats.invoke_count == 1)
-			context->header["Content-Type"] = "text/plain";
+		{
+			context->set_status(500, "UCE Unit Compile Error");
+			context->header["Content-Type"] = "text/plain; charset=utf-8";
+		}
 		if(compiler_can_write_response(context))
-			print(su->compiler_messages);
+			print(display_messages);
 		return(0);
 	}
 	else

src/lib/compiler.h

@@ -10,6 +10,8 @@
 #define EXPORT extern "C"
 
 String preprocess_shared_unit(Request* context, SharedUnit* su);
+String compiler_generated_cpp_path(Request* context, String source_file);
+String compiler_generated_cpp_path(SharedUnit* su);
 void setup_unit_paths(Request* context, SharedUnit* su, String file_name);
 void load_shared_unit(Request* context, SharedUnit* su);
 void compile_shared_unit(Request* context, SharedUnit* su);

src/lib/dtree.cpp

@@ -140,7 +140,7 @@ u64 dtree_clamp_to_u64_range(long double value)
 
 }
 
-void DTree::each(std::function <void (DTree t, String key)> f)
+void DTree::each(std::function <void (const DTree& t, String key)> f)
 {
 	const DTree& target = deref();
 	switch(target.type)
@@ -695,7 +695,7 @@ void DTree::operator = (void* v) { set(v); }
 void DTree::operator = (DTree v) { set(v); }
 void DTree::operator = (StringMap v) { set(v); }
 
-void DTree::push(DTree& child)
+void DTree::push(const DTree& child)
 {
 	DTree* target = reference_target();
 	if(target)
@@ -733,6 +733,11 @@ DTree DTree::pop()
 	if(target)
 		return(target->pop());
 	set_type('M');
+	if(_map.empty())
+	{
+		_array_index = 0;
+		return(DTree());
+	}
 	auto last = _map.rbegin();
 	DTree result = last->second;
 	_map.erase(last->first);

src/lib/dtree.h

@@ -19,7 +19,7 @@ struct DTree {
 	void*	_ptr;
 	std::map<String, DTree> _map;
 
-	void each(std::function <void (DTree t, String key)> f);
+	void each(std::function <void (const DTree& t, String key)> f);
 	bool is_array();
 	bool is_list() const;
 	String to_string();
@@ -56,7 +56,7 @@ struct DTree {
 	void operator = (DTree v);
 	void operator = (StringMap v);
 
-	void push(DTree& child);
+	void push(const DTree& child);
 	DTree pop();
 	void remove(String s);
 	void clear();

src/lib/functionlib.cpp

@@ -4,6 +4,7 @@
 #include <pcre2.h>
 #include <cctype>
 #include <stdexcept>
+#include <algorithm>
 
 String var_dump(StringMap map, String prefix, String postfix)
 {
@@ -59,6 +60,145 @@ String to_upper(String s)
 	return(result);
 }
 
+StringList list_unique(StringList items)
+{
+	StringList result;
+	std::set<String> seen;
+	for(auto item : items)
+	{
+		if(seen.find(item) != seen.end())
+			continue;
+		seen.insert(item);
+		result.push_back(item);
+	}
+	return(result);
+}
+
+StringList list_sort(StringList items)
+{
+	std::sort(items.begin(), items.end());
+	return(items);
+}
+
+bool list_some(StringList items, std::function<bool (String)> f)
+{
+	for(auto item : items)
+	{
+		if(f(item))
+			return(true);
+	}
+	return(false);
+}
+
+bool list_every(StringList items, std::function<bool (String)> f)
+{
+	for(auto item : items)
+	{
+		if(!f(item))
+			return(false);
+	}
+	return(true);
+}
+
+String list_find(StringList items, std::function<bool (String)> f, String fallback)
+{
+	for(auto item : items)
+	{
+		if(f(item))
+			return(item);
+	}
+	return(fallback);
+}
+
+StringList dtree_keys(DTree tree)
+{
+	StringList result;
+	tree.each([&](const DTree& item, String key) {
+		if(key != "")
+			result.push_back(key);
+	});
+	return(result);
+}
+
+DTree dtree_values(DTree tree)
+{
+	DTree result;
+	result.set_array();
+	tree.each([&](const DTree& item, String key) {
+		result.push(item);
+	});
+	return(result);
+}
+
+DTree dtree_pick(DTree tree, StringList keys)
+{
+	DTree result;
+	for(auto key : keys)
+	{
+		DTree* item = tree.key(key);
+		if(item)
+			result[key] = *item;
+	}
+	return(result);
+}
+
+DTree dtree_omit(DTree tree, StringList keys)
+{
+	DTree result;
+	std::set<String> omitted(keys.begin(), keys.end());
+	tree.each([&](const DTree& item, String key) {
+		if(key != "" && omitted.find(key) == omitted.end())
+			result[key] = item;
+	});
+	return(result);
+}
+
+DTree dtree_map(DTree tree, std::function<DTree (const DTree&, String)> f)
+{
+	DTree result;
+	bool input_is_list = tree.is_list();
+	if(input_is_list)
+		result.set_array();
+	tree.each([&](const DTree& item, String key) {
+		DTree mapped = f(item, key);
+		if(key != "" && !input_is_list)
+			result[key] = mapped;
+		else
+			result.push(mapped);
+	});
+	return(result);
+}
+
+DTree dtree_filter(DTree tree, std::function<bool (const DTree&, String)> f)
+{
+	DTree result;
+	bool input_is_list = tree.is_list();
+	if(input_is_list)
+		result.set_array();
+	tree.each([&](const DTree& item, String key) {
+		if(!f(item, key))
+			return;
+		if(key != "" && !input_is_list)
+			result[key] = item;
+		else
+			result.push(item);
+	});
+	return(result);
+}
+
+DTree dtree_group_by(DTree tree, std::function<String (const DTree&, String)> f)
+{
+	DTree result;
+	tree.each([&](const DTree& item, String key) {
+		String group = f(item, key);
+		DTree* group_items = result.get_or_create(group);
+		if(!group_items->is_array())
+			group_items->set_array();
+		group_items->push(item);
+	});
+	return(result);
+}
+
 String substr(String s, s64 start_pos)
 {
 	s64 len = s.length();
@@ -502,11 +642,11 @@ String trim(String raw)
 	s64 len = raw.length();
 	s64 start_pos = 0;
 	s64 end_pos = len - 1;
-	if(len == 0 || (len == 1 && isspace(raw[0])))
+	if(len == 0 || (len == 1 && isspace((unsigned char)raw[0])))
 		return("");
-	while(start_pos < len && isspace(raw[start_pos]))
+	while(start_pos < len && isspace((unsigned char)raw[start_pos]))
 		start_pos++;
-	while(end_pos >= 0 && isspace(raw[end_pos]))
+	while(end_pos >= 0 && isspace((unsigned char)raw[end_pos]))
 		end_pos--;
 	if(end_pos < start_pos)
 		return("");
@@ -519,7 +659,7 @@ StringList split_space(String str)
 	String current_token = "";
 	for(auto c : str)
 	{
-		if(isspace(c))
+		if(isspace((unsigned char)c))
 		{
 			if(current_token != "")
 			{
@@ -540,6 +680,11 @@ StringList split_space(String str)
 StringList split(String str, String delim)
 {
 	StringList result;
+	if(delim == "")
+	{
+		result.push_back(str);
+		return(result);
+	}
 	int start = 0;
     int end = str.find(delim);
     while (end != String::npos)
@@ -562,14 +707,16 @@ StringMap split_kv(String s, char separator, bool trim_whitespace, bool uppercas
 		u8 mode = 0;
 		k = "";
 		v = "";
-		if(s[0] != '#') for(auto c : s)
+		if(s == "" || s[0] == '#')
+			continue;
+		for(auto c : s)
 		{
 			if(mode == 0)
 			{
 				if(c == separator)
 					mode = 1;
 				else
-					k.append(1, uppercase_keys ? toupper(c) : c);
+					k.append(1, uppercase_keys ? toupper((unsigned char)c) : c);
 			}
 			else
 			{
@@ -590,62 +737,43 @@ StringMap split_kv(String s, char separator, bool trim_whitespace, bool uppercas
 StringMap split_http_headers(String s)
 {
 	StringMap result;
-	String k;
-	String v;
-	String query_string;
-	String base_uri;
-	for(auto s : split(s, "\n"))
+	StringList lines = split(s, "\n");
+	if(lines.size() == 0)
+		return(result);
+
+	u64 header_start = 0;
+	while(header_start < lines.size() && trim(lines[header_start]) == "")
+		header_start++;
+
+	if(header_start < lines.size() && lines[header_start].find(':') == String::npos)
 	{
-		u8 mode = 0;
-		k = "";
-		v = "";
-		for(auto c : s)
-		{
-			if(mode == 0)
-			{
-				if(c == ':')
-					mode = 1;
-				else
-					k.append(1, c);
-			}
-			else
-			{
-				v.append(1, c);
-			}
-		}
-		if(k != "")
-		{
-			k = trim(k);
-			v = trim(v);
-			if(v == "")
-			{
-				if(result["REQUEST_METHOD"] == "")
-				{
-					result["REQUEST_METHOD"] = nibble(k, " ");
-					result["REQUEST_URI"] = nibble(k, " ");
-					String query_string = result["REQUEST_URI"];
-					String base_uri = nibble(query_string, "?");
-					result["SERVER_PROTOCOL"] = k;
-					result["SCRIPT_NAME"] = base_uri;
-					result["DOCUMENT_URI"] = base_uri;
-					result["QUERY_STRING"] = query_string;
-				}
-				else
-				{
-					if(k != "")
-						result["_"] = k;
-				}
-				}
-				else
-				{
-					String header_key = to_upper(k);
-					std::replace(header_key.begin(), header_key.end(), '-', '_');
-					result["HTTP_"+header_key] = v;
-					if(header_key == "CONTENT_TYPE" || header_key == "CONTENT_LENGTH")
-						result[header_key] = v;
-				}
-			}
-		}
+		String request_line = trim(lines[header_start]);
+		result["REQUEST_METHOD"] = nibble(request_line, " ");
+		result["REQUEST_URI"] = nibble(request_line, " ");
+		String query_string = result["REQUEST_URI"];
+		String base_uri = nibble(query_string, "?");
+		result["SERVER_PROTOCOL"] = trim(request_line);
+		result["SCRIPT_NAME"] = base_uri;
+		result["DOCUMENT_URI"] = base_uri;
+		result["QUERY_STRING"] = query_string;
+		header_start++;
+	}
+
+	for(u64 i = header_start; i < lines.size(); i++)
+	{
+		String line = lines[i];
+		size_t colon = line.find(':');
+		if(colon == String::npos)
+			continue;
+		String header_key = to_upper(trim(line.substr(0, colon)));
+		String value = trim(line.substr(colon + 1));
+		if(header_key == "")
+			continue;
+		std::replace(header_key.begin(), header_key.end(), '-', '_');
+		result["HTTP_" + header_key] = value;
+		if(header_key == "CONTENT_TYPE" || header_key == "CONTENT_LENGTH")
+			result[header_key] = value;
+	}
 	return(result);
 }
 
@@ -814,6 +942,9 @@ String html_escape(String s)
 			case('"'):
 				result.append("&quot;");
 				break;
+			case('\''):
+				result.append("&#39;");
+				break;
 			default:
 				result.append(1, c);
 				break;

src/lib/functionlib.h

@@ -65,8 +65,8 @@ String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1)
     return(rc);
 }
 
-template<typename T>
-std::vector<T> filter(std::vector<T> items, std::function<bool (T)> f)
+template<typename T, typename F>
+std::vector<T> filter(std::vector<T> items, F f)
 {
 	std::vector<T> new_items;
 	for(auto item : items)
@@ -77,6 +77,29 @@ std::vector<T> filter(std::vector<T> items, std::function<bool (T)> f)
 	return(new_items);
 }
 
+template<typename T, typename F>
+auto map(std::vector<T> items, F f)
+{
+	using ResultType = decltype(f(items[0]));
+	std::vector<ResultType> new_items;
+	for(auto item : items)
+		new_items.push_back(f(item));
+	return(new_items);
+}
+
+StringList list_unique(StringList items);
+StringList list_sort(StringList items);
+bool list_some(StringList items, std::function<bool (String)> f);
+bool list_every(StringList items, std::function<bool (String)> f);
+String list_find(StringList items, std::function<bool (String)> f, String fallback = "");
+StringList dtree_keys(DTree tree);
+DTree dtree_values(DTree tree);
+DTree dtree_pick(DTree tree, StringList keys);
+DTree dtree_omit(DTree tree, StringList keys);
+DTree dtree_map(DTree tree, std::function<DTree (const DTree&, String)> f);
+DTree dtree_filter(DTree tree, std::function<bool (const DTree&, String)> f);
+DTree dtree_group_by(DTree tree, std::function<String (const DTree&, String)> f);
+
 template <class ...Args>
 String first(Args... args)
 {

src/lib/mysql-connector.cpp

@@ -37,7 +37,8 @@ bool MySQL::connect(String host, String username, String password)
 	*/
 	//switch_to_arena(context->mem);
 	statement_info = String("connected");
-	context->resources.mysql_connections.push_back(connection);
+	if(context)
+		context->resources.mysql_connections.push_back(this);
 	return(true);
 }
 
@@ -181,8 +182,16 @@ DTree MySQL::get_pending_result()
     return(result_data);
 }
 
+static bool mysql_has_unquoted_positional_placeholder(String query);
+
 DTree MySQL::query(String q)
 {
+	if(mysql_has_unquoted_positional_placeholder(q))
+	{
+		_preload_next_error_code = CR_UNKNOWN_ERROR;
+		statement_info = "mysql positional ? placeholders are not supported; use named :name placeholders";
+		return(DTree());
+	}
 	_preload_next_error_code = mysql_query((MYSQL*)connection, q.c_str());
 	DTree result;
 	if(_preload_next_error_code == 0)
@@ -190,8 +199,50 @@ DTree MySQL::query(String q)
 	return(result);
 }
 
+static bool mysql_has_unquoted_positional_placeholder(String query)
+{
+	bool quoted = false;
+	char quote = 0;
+	bool escaped = false;
+	for(u32 i = 0; i < query.length(); i++)
+	{
+		char c = query[i];
+		if(quoted)
+		{
+			if(escaped)
+			{
+				escaped = false;
+				continue;
+			}
+			if(c == '\\')
+			{
+				escaped = true;
+				continue;
+			}
+			if(c == quote)
+				quoted = false;
+			continue;
+		}
+		if(c == '\'' || c == '"')
+		{
+			quoted = true;
+			quote = c;
+			continue;
+		}
+		if(c == '?')
+			return(true);
+	}
+	return(false);
+}
+
 DTree MySQL::query(String q, StringMap params)
 {
+	if(mysql_has_unquoted_positional_placeholder(q))
+	{
+		_preload_next_error_code = CR_UNKNOWN_ERROR;
+		statement_info = "mysql positional ? placeholders are not supported; use named :name placeholders";
+		return(DTree());
+	}
 	return(query(
 		parse_query_parameters(q, params).c_str()
 	));
@@ -255,12 +306,24 @@ void MySQL::disconnect()
 	if(connection)
 		mysql_close((MYSQL*)connection);
 	connection = NULL;
+	if(context)
+	{
+		auto& connections = context->resources.mysql_connections;
+		connections.erase(std::remove(connections.begin(), connections.end(), this), connections.end());
+	}
 }
 
 String MySQL::error()
 {
 	if(_preload_next_error_code)
 	{
+		if(statement_info != "")
+		{
+			String result = statement_info;
+			statement_info = "";
+			_preload_next_error_code = 0;
+			return(result);
+		}
 		String p = "Unknown error";
 		switch(_preload_next_error_code)
 		{
@@ -298,8 +361,16 @@ String MySQL::error()
 void cleanup_mysql_connections()
 {
 	//switch_to_system_alloc();
-	for(auto& con : context->resources.mysql_connections)
-		mysql_close((MYSQL*)con);
-	context->resources.mysql_connections.clear();
+	if(!context)
+		return;
+	while(!context->resources.mysql_connections.empty())
+	{
+		MySQL* db = (MySQL*)context->resources.mysql_connections.back();
+		context->resources.mysql_connections.pop_back();
+		bool should_delete = db->request_cleanup_delete;
+		db->disconnect();
+		if(should_delete)
+			delete db;
+	}
 	//switch_to_arena(context->mem);
 }

src/lib/mysql-connector.h

@@ -20,6 +20,7 @@ struct MySQL {
 	u32 row_count = 0;
 	u64 insert_id = 0;
 	String statement_info = ""; //
+	bool request_cleanup_delete = false;
 
 	std::vector<MySQLFieldInfo> field_info;
 
@@ -37,6 +38,7 @@ struct MySQL {
 MySQL* mysql_connect(String host = "localhost", String username = "root", String password = "")
 {
 	MySQL* m = new MySQL();
+	m->request_cleanup_delete = true;
 	m->connect(host, username, password);
 	return(m);
 }

src/lib/sqlite-connector.cpp

@@ -0,0 +1,353 @@
+#include "../3rdparty/sqlite/sqlite3.h"
+#include "sqlite-connector.h"
+
+namespace {
+
+std::map<String, SQLite*> sqlite_worker_connection_cache;
+
+void sqlite_register_request_connection(SQLite* db)
+{
+	if(!context || !db)
+		return;
+	auto& connections = context->resources.sqlite_connections;
+	if(std::find(connections.begin(), connections.end(), db) == connections.end())
+		connections.push_back(db);
+}
+
+void sqlite_unregister_request_connection(SQLite* db)
+{
+	if(!context || !db)
+		return;
+	auto& connections = context->resources.sqlite_connections;
+	connections.erase(std::remove(connections.begin(), connections.end(), db), connections.end());
+}
+
+void sqlite_forget_worker_cached_connection(SQLite* db)
+{
+	if(!db || db->path == "")
+		return;
+	auto found = sqlite_worker_connection_cache.find(db->path);
+	if(found != sqlite_worker_connection_cache.end() && found->second == db)
+		sqlite_worker_connection_cache.erase(found);
+}
+
+}
+
+void SQLite::set_error(s32 code, String info)
+{
+	error_code = code;
+	if(connection)
+		statement_info = sqlite3_errmsg((sqlite3*)connection);
+	else
+		statement_info = info;
+	if(info != "")
+		statement_info = info + (statement_info != "" ? ": " + statement_info : "");
+}
+
+bool SQLite::apply_default_pragmas()
+{
+	char* err = 0;
+	const char* pragmas =
+		"PRAGMA foreign_keys = ON;"
+		"PRAGMA journal_mode = WAL;"
+		"PRAGMA synchronous = NORMAL;";
+	s32 rc = sqlite3_exec((sqlite3*)connection, pragmas, 0, 0, &err);
+	if(rc != SQLITE_OK)
+	{
+		String message;
+		if(err)
+		{
+			message = err;
+			sqlite3_free(err);
+		}
+		set_error(rc, "sqlite default pragmas failed" + (message != "" ? ": " + message : ""));
+		return(false);
+	}
+	return(true);
+}
+
+bool SQLite::connect(String path)
+{
+	disconnect();
+	this->path = path;
+	s32 rc = sqlite3_open_v2(path.c_str(), (sqlite3**)&connection, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, 0);
+	if(rc != SQLITE_OK)
+	{
+		set_error(rc, "sqlite open failed for " + path);
+		disconnect();
+		return(false);
+	}
+	sqlite3_busy_timeout((sqlite3*)connection, 5000);
+	sqlite_register_request_connection(this);
+	if(!apply_default_pragmas())
+		return(false);
+	statement_info = "connected";
+	return(true);
+}
+
+void SQLite::disconnect()
+{
+	sqlite_forget_worker_cached_connection(this);
+	if(!connection)
+	{
+		sqlite_unregister_request_connection(this);
+		return;
+	}
+	sqlite3_close((sqlite3*)connection);
+	connection = 0;
+	sqlite_unregister_request_connection(this);
+}
+
+String SQLite::error()
+{
+	if(statement_info != "")
+		return(statement_info);
+	if(connection)
+		return(sqlite3_errmsg((sqlite3*)connection));
+	return("");
+}
+
+bool SQLite::bind_params(void* statement, const StringMap& params)
+{
+	sqlite3_stmt* stmt = (sqlite3_stmt*)statement;
+	s32 count = sqlite3_bind_parameter_count(stmt);
+	for(s32 i = 1; i <= count; i++)
+	{
+		const char* raw_name = sqlite3_bind_parameter_name(stmt, i);
+		if(!raw_name || raw_name[0] == '\0')
+		{
+			set_error(SQLITE_MISUSE, "sqlite positional ? placeholders are not supported; use named :name placeholders");
+			return(false);
+		}
+		String name = raw_name;
+		if(name[0] != ':')
+		{
+			set_error(SQLITE_MISUSE, "sqlite only supports :name placeholders; found " + name);
+			return(false);
+		}
+		String key = name.substr(1);
+		auto found = params.find(key);
+		s32 rc;
+		if(found == params.end())
+			rc = sqlite3_bind_null(stmt, i);
+		else
+			rc = sqlite3_bind_text(stmt, i, found->second.c_str(), found->second.length(), SQLITE_STATIC);
+		if(rc != SQLITE_OK)
+		{
+			set_error(rc, "sqlite bind failed for " + name);
+			return(false);
+		}
+	}
+	return(true);
+}
+
+DTree SQLite::collect_rows(void* statement)
+{
+	sqlite3_stmt* stmt = (sqlite3_stmt*)statement;
+	DTree result;
+	s32 column_count = sqlite3_column_count(stmt);
+	std::vector<String> column_names;
+	column_names.reserve(column_count);
+	for(s32 i = 0; i < column_count; i++)
+		column_names.push_back(sqlite3_column_name(stmt, i));
+	while(true)
+	{
+		s32 rc = sqlite3_step(stmt);
+		if(rc == SQLITE_ROW)
+		{
+			DTree row;
+			for(s32 i = 0; i < column_count; i++)
+			{
+				const String& name = column_names[i];
+				switch(sqlite3_column_type(stmt, i))
+				{
+					case SQLITE_INTEGER:
+						row[name].set((s64)sqlite3_column_int64(stmt, i));
+						break;
+					case SQLITE_FLOAT:
+						row[name].set((f64)sqlite3_column_double(stmt, i));
+						break;
+					case SQLITE_TEXT:
+					{
+						const unsigned char* text = sqlite3_column_text(stmt, i);
+						row[name].set(text ? String((const char*)text) : "");
+						break;
+					}
+					case SQLITE_BLOB:
+					{
+						const char* data = (const char*)sqlite3_column_blob(stmt, i);
+						s32 bytes = sqlite3_column_bytes(stmt, i);
+						row[name].set(data && bytes > 0 ? String(data, bytes) : "");
+						break;
+					}
+					case SQLITE_NULL:
+					default:
+						break;
+				}
+			}
+			result.push(row);
+			continue;
+		}
+		if(rc == SQLITE_DONE)
+		{
+			affected_rows = sqlite3_changes((sqlite3*)connection);
+			insert_id = sqlite3_last_insert_rowid((sqlite3*)connection);
+			return(result);
+		}
+		set_error(rc, "sqlite step failed");
+		return(DTree());
+	}
+}
+
+DTree SQLite::query(String q)
+{
+	StringMap params;
+	return(query(q, params));
+}
+
+DTree SQLite::query(String q, const StringMap& params)
+{
+	DTree result;
+	affected_rows = 0;
+	insert_id = 0;
+	error_code = SQLITE_OK;
+	statement_info = "";
+	if(!connection)
+	{
+		set_error(SQLITE_MISUSE, "sqlite query called without an open connection");
+		return(result);
+	}
+	sqlite3_stmt* stmt = 0;
+	const char* tail = 0;
+	s32 rc = sqlite3_prepare_v2((sqlite3*)connection, q.c_str(), q.length(), &stmt, &tail);
+	if(rc != SQLITE_OK)
+	{
+		set_error(rc, "sqlite prepare failed");
+		return(result);
+	}
+	if(!stmt)
+	{
+		statement_info = "ok";
+		return(result);
+	}
+	if(tail && trim(String(tail)) != "")
+	{
+		sqlite3_stmt* trailing_stmt = 0;
+		const char* trailing_tail = 0;
+		rc = sqlite3_prepare_v2((sqlite3*)connection, tail, -1, &trailing_stmt, &trailing_tail);
+		if(rc != SQLITE_OK)
+		{
+			sqlite3_finalize(stmt);
+			set_error(rc, "sqlite_query accepts exactly one SQL statement per call; trailing SQL after the first statement was rejected");
+			return(result);
+		}
+		if(trailing_stmt)
+		{
+			sqlite3_finalize(trailing_stmt);
+			sqlite3_finalize(stmt);
+			set_error(SQLITE_MISUSE, "sqlite_query accepts exactly one SQL statement per call; trailing SQL after the first statement was rejected");
+			return(result);
+		}
+	}
+	if(!bind_params(stmt, params))
+	{
+		sqlite3_finalize(stmt);
+		return(result);
+	}
+	result = collect_rows(stmt);
+	rc = sqlite3_finalize(stmt);
+	if(rc != SQLITE_OK && error_code == SQLITE_OK)
+		set_error(rc, "sqlite finalize failed");
+	if(statement_info == "")
+		statement_info = "ok";
+	return(result);
+}
+
+SQLite* sqlite_connect(String path)
+{
+	auto found = sqlite_worker_connection_cache.find(path);
+	if(found != sqlite_worker_connection_cache.end() && found->second && found->second->connection)
+	{
+		SQLite* cached = found->second;
+		cached->error_code = SQLITE_OK;
+		cached->affected_rows = 0;
+		cached->insert_id = 0;
+		cached->statement_info = "connected";
+		sqlite_register_request_connection(cached);
+		return(cached);
+	}
+
+	SQLite* db = new SQLite();
+	db->worker_cache = true;
+	if(db->connect(path) && db->connection)
+		sqlite_worker_connection_cache[path] = db;
+	else
+		db->request_cleanup_delete = true;
+	return(db);
+}
+
+void sqlite_disconnect(SQLite* db)
+{
+	if(!db)
+		return;
+	db->disconnect();
+	delete db;
+}
+
+String sqlite_error(SQLite* db)
+{
+	if(!db)
+		return("sqlite connection is null");
+	return(db->error());
+}
+
+DTree sqlite_query(SQLite* db, String q)
+{
+	if(!db)
+		return(DTree());
+	return(db->query(q));
+}
+
+DTree sqlite_query(SQLite* db, String q, const StringMap& params)
+{
+	if(!db)
+		return(DTree());
+	return(db->query(q, params));
+}
+
+u64 sqlite_insert_id(SQLite* db)
+{
+	if(!db)
+		return(0);
+	return(db->insert_id);
+}
+
+u32 sqlite_affected_rows(SQLite* db)
+{
+	if(!db)
+		return(0);
+	return(db->affected_rows);
+}
+
+void cleanup_sqlite_connections()
+{
+	if(!context)
+		return;
+	while(!context->resources.sqlite_connections.empty())
+	{
+		SQLite* db = (SQLite*)context->resources.sqlite_connections.back();
+		context->resources.sqlite_connections.pop_back();
+		if(db->worker_cache)
+		{
+			db->affected_rows = 0;
+			db->insert_id = 0;
+			db->error_code = SQLITE_OK;
+			db->statement_info = "";
+			continue;
+		}
+		bool should_delete = db->request_cleanup_delete;
+		db->disconnect();
+		if(should_delete)
+			delete db;
+	}
+}

src/lib/sqlite-connector.h

@@ -0,0 +1,34 @@
+#pragma once
+
+struct SQLite {
+
+	void* connection = 0;
+	s32 error_code = 0;
+	u32 affected_rows = 0;
+	u64 insert_id = 0;
+	String statement_info = "";
+	String path = "";
+	bool request_cleanup_delete = false;
+	bool worker_cache = false;
+
+	bool connect(String path);
+	void disconnect();
+	String error();
+	DTree query(String q);
+	DTree query(String q, const StringMap& params);
+
+private:
+	void set_error(s32 code, String info = "");
+	bool apply_default_pragmas();
+	bool bind_params(void* statement, const StringMap& params);
+	DTree collect_rows(void* statement);
+};
+
+SQLite* sqlite_connect(String path);
+void sqlite_disconnect(SQLite* db);
+String sqlite_error(SQLite* db);
+DTree sqlite_query(SQLite* db, String q);
+DTree sqlite_query(SQLite* db, String q, const StringMap& params);
+u64 sqlite_insert_id(SQLite* db);
+u32 sqlite_affected_rows(SQLite* db);
+void cleanup_sqlite_connections();

src/lib/types.h

@@ -208,6 +208,7 @@ struct Request {
 	struct Resources {
 		std::vector<u64> sockets;
 		std::vector<void*> mysql_connections;
+		std::vector<void*> sqlite_connections;
 		u64 client_socket = 0;
 		u64 server_socket = 0;
 		bool is_websocket = false;

src/lib/uce_lib.cpp

@@ -1,4 +1,8 @@
-
+// UCE runtime amalgamation include.
+//
+// The worker and generated units include this file to build the runtime in a
+// single translation unit. Do not compile the listed .cpp files separately
+// unless the build/compiler model is deliberately changed.
 
 #include "types.cpp"
 #include "dtree.cpp"
@@ -12,3 +16,4 @@
 #include "markdown.cpp"
 #include "zip.cpp"
 #include "mysql-connector.cpp"
+#include "sqlite-connector.cpp"

src/lib/uce_lib.h

@@ -11,3 +11,4 @@
 #include "markdown.h"
 #include "zip.h"
 #include "mysql-connector.h"
+#include "sqlite-connector.h"

src/lib/uri.cpp

@@ -220,38 +220,31 @@ String uri_encode(String q)
 
 StringMap parse_query(String q)
 {
-	StringMap result;
-	if(q.length() == 0)
-		return(result);
+	return(parse_query(q, 0));
+}
 
-	bool is_key = true;
-	String key = "";
-	String value = "";
-	for (char &c: q)
+StringMap parse_query(String q, String* first_keyless_path)
+{
+	StringMap result;
+	if(first_keyless_path)
+		*first_keyless_path = "";
+	for(String part : split(q, "&"))
 	{
-		if(c == '=')
+		if(part == "")
+			continue;
+		size_t equals = part.find('=');
+		if(equals == String::npos)
 		{
-			is_key = !is_key;
-		}
-		else if(c == '&')
-		{
-			result[uri_decode(key)] = uri_decode(value);
-			key = "";
-			value = "";
-			is_key = true;
-		}
-		else if(is_key)
-		{
-			key.append(1, c);
+			String key = uri_decode(part);
+			if(first_keyless_path && *first_keyless_path == "")
+				*first_keyless_path = key;
+			result[key] = "";
 		}
 		else
 		{
-			value.append(1, c);
+			result[uri_decode(part.substr(0, equals))] = uri_decode(part.substr(equals + 1));
 		}
 	}
-
-	result[uri_decode(key)] = uri_decode(value);
-
 	return(result);
 }
 
@@ -269,6 +262,132 @@ String encode_query(StringMap map)
 	return(result);
 }
 
+String route_path_normalize(String path)
+{
+	path = trim(path);
+	size_t start = path.find_first_not_of('/');
+	if(start == String::npos)
+		return("");
+	size_t end = path.find_last_not_of('/');
+	return(path.substr(start, end - start + 1));
+}
+
+bool route_path_normalized_is_safe(String path)
+{
+	if(path == "")
+		return(true);
+	for(String part : split(path, "/"))
+	{
+		if(part == "" || part == "." || part == "..")
+			return(false);
+		for(unsigned char c : part)
+		{
+			if(!(std::isalnum(c) || c == '-' || c == '_'))
+				return(false);
+		}
+	}
+	return(true);
+}
+
+bool route_path_is_safe(String path)
+{
+	return(route_path_normalized_is_safe(route_path_normalize(path)));
+}
+
+String route_path_sanitize(String path, String default_path)
+{
+	path = route_path_normalize(path);
+	if(path == "")
+		path = route_path_normalize(default_path);
+	if(!route_path_normalized_is_safe(path))
+		return("");
+	return(path);
+}
+
+String route_path_sanitize_normalized(String path, String default_path)
+{
+	if(path == "")
+		path = route_path_normalize(default_path);
+	if(!route_path_normalized_is_safe(path))
+		return("");
+	return(path);
+}
+
+String request_script_url(Request& context)
+{
+	String url = first(context.params["DOCUMENT_URI"], context.params["SCRIPT_NAME"]);
+	if(str_ends_with(url, "/index.uce"))
+		url = url.substr(0, url.length() - String("index.uce").length());
+	return(url);
+}
+
+String request_base_url_from_script_url(String script_url)
+{
+	String base = dirname(script_url);
+	if(base == "")
+		base = "/";
+	if(base[base.length() - 1] != '/')
+		base.append(1, '/');
+	return(base);
+}
+
+String request_base_url(Request& context)
+{
+	return(request_base_url_from_script_url(request_script_url(context)));
+}
+
+String request_query_path(Request& context, String default_path)
+{
+	return(request_query_route(context, default_path)["l_path"].to_string());
+}
+
+DTree request_query_route(Request& context, String default_path)
+{
+	String raw_path = "";
+	parse_query(context.params["QUERY_STRING"], &raw_path);
+	return(request_route_from_raw_path(raw_path, default_path));
+}
+
+DTree request_route_from_raw_path(String raw_path, String default_path)
+{
+	DTree route;
+	String normalized_path = route_path_normalize(raw_path);
+	String route_path = route_path_sanitize_normalized(normalized_path, default_path);
+	bool valid = route_path != "";
+	route["raw_path"] = normalized_path;
+	route["l_path"] = route_path;
+	String page = route_path;
+	route["page"] = valid ? nibble(page, "/") : "";
+	if(valid && route["page"].to_string() == "")
+		route["page"] = default_path;
+	route["valid"].set_bool(valid);
+	return(route);
+}
+
+void request_populate_context_params(Request& context, String default_path)
+{
+	DTree route = request_query_route(context, default_path);
+	String script_url = request_script_url(context);
+	context.params["SCRIPT_URL"] = script_url;
+	context.params["BASE_URL"] = request_base_url_from_script_url(script_url);
+	context.params["ROUTE_PATH"] = route["l_path"].to_string();
+	context.params["ROUTE_PAGE"] = route["page"].to_string();
+	context.params["ROUTE_PATH_RAW"] = route["raw_path"].to_string();
+	context.params["ROUTE_VALID"] = route["valid"].to_bool() ? "1" : "0";
+}
+
+void request_populate_context_params_from_route(Request& context, String raw_path, String default_path)
+{
+	DTree route = request_route_from_raw_path(raw_path, default_path);
+	String script_url = request_script_url(context);
+	context.params["SCRIPT_URL"] = script_url;
+	context.params["BASE_URL"] = request_base_url_from_script_url(script_url);
+	context.params["ROUTE_PATH"] = route["l_path"].to_string();
+	context.params["ROUTE_PAGE"] = route["page"].to_string();
+	context.params["ROUTE_PATH_RAW"] = route["raw_path"].to_string();
+	context.params["ROUTE_VALID"] = route["valid"].to_bool() ? "1" : "0";
+}
+
 bool http_header_name_valid(String name)
 {
 	if(name == "")

src/lib/uri.h

@@ -8,7 +8,18 @@ String base64_decode(String raw, bool& ok);
 String uri_decode(String q);
 String uri_encode(String q);
 StringMap parse_query(String q);
+StringMap parse_query(String q, String* first_keyless_path);
 String encode_query(StringMap map);
+String route_path_normalize(String path);
+bool route_path_is_safe(String path);
+String route_path_sanitize(String path, String default_path = "index");
+String request_script_url(Request& context);
+String request_base_url(Request& context);
+String request_query_path(Request& context, String default_path = "index");
+DTree request_query_route(Request& context, String default_path = "index");
+DTree request_route_from_raw_path(String raw_path, String default_path = "index");
+void request_populate_context_params(Request& context, String default_path = "index");
+void request_populate_context_params_from_route(Request& context, String raw_path, String default_path = "index");
 void redirect(String url, s32 code = 302);
 StringMap parse_multipart(String q, String boundary, std::vector<UploadedFile>& uploaded_files);
 URI parse_uri(String uri_String);