Add archive helpers and harden task runtime

src/lib/cli.cpp

@@ -0,0 +1,45 @@
+#include "cli.h"
+
+DTree cli_input(Request& context)
+{
+	DTree input;
+
+	for(auto& item : context.get)
+		input[item.first] = item.second;
+	for(auto& item : context.post)
+		input[item.first] = item.second;
+
+	String content_type_info = context.params["CONTENT_TYPE"];
+	String content_type = to_lower(trim(nibble(content_type_info, ";")));
+	if(content_type == "application/json" || str_ends_with(content_type, "+json"))
+	{
+		String body = trim(context.in);
+		if(body != "")
+		{
+			DTree parsed = json_decode(body);
+			if(parsed.type == 'M')
+			{
+				for(auto& item : parsed._map)
+					input[item.first] = item.second;
+			}
+			else
+			{
+				input["_"] = parsed;
+			}
+		}
+	}
+
+	return(input);
+}
+
+String cli_arg(Request& context, String key, String default_value)
+{
+	DTree input = cli_input(context);
+	DTree* value = input.key(key);
+	if(!value)
+		return(default_value);
+	String result = value->to_string();
+	if(result == "")
+		return(default_value);
+	return(result);
+}

src/lib/cli.h

@@ -0,0 +1,4 @@
+#pragma once
+
+DTree cli_input(Request& context);
+String cli_arg(Request& context, String key, String default_value = "");

src/lib/compiler.cpp

@@ -13,6 +13,7 @@ const char* UCE_SETUP_SYMBOL = "__uce_set_current_request";
 const char* UCE_RENDER_SYMBOL = "__uce_render";
 const char* UCE_COMPONENT_SYMBOL = "__uce_component";
 const char* UCE_WEBSOCKET_SYMBOL = "__uce_websocket";
+const char* UCE_CLI_SYMBOL = "__uce_cli";
 const char* UCE_ONCE_SYMBOL = "__uce_once";
 const char* UCE_INIT_SYMBOL = "__uce_init";
 const u64 UCE_UNIT_ABI_VERSION = 1;
@@ -690,6 +691,8 @@ void load_shared_unit(Request* context, SharedUnit* su)
 		dlerror();
 		su->on_websocket = (request_ref_handler)dlsym(su->so_handle, UCE_WEBSOCKET_SYMBOL);
 		dlerror();
+		su->on_cli = (request_ref_handler)dlsym(su->so_handle, UCE_CLI_SYMBOL);
+		dlerror();
 		su->on_once = (request_ref_handler)dlsym(su->so_handle, UCE_ONCE_SYMBOL);
 		dlerror();
 		su->on_init = (request_ref_handler)dlsym(su->so_handle, UCE_INIT_SYMBOL);
@@ -1191,7 +1194,8 @@ enum class UnitCallMacroKind
 	render,
 	component,
 	once,
-	init
+	init,
+	cli
 };
 
 struct UnitCallMacroTarget
@@ -1234,6 +1238,7 @@ void compiler_unload_failed_shared_unit(SharedUnit* su)
 	su->on_render = 0;
 	su->on_component = 0;
 	su->on_websocket = 0;
+	su->on_cli = 0;
 	su->on_once = 0;
 	su->on_init = 0;
 }
@@ -1358,6 +1363,11 @@ UnitCallMacroTarget unit_call_macro_target(String function_name)
 		target.kind = UnitCallMacroKind::init;
 		return(target);
 	}
+	if(function_name == "CLI")
+	{
+		target.kind = UnitCallMacroKind::cli;
+		return(target);
+	}
 
 	return(target);
 }
@@ -1465,6 +1475,8 @@ String compiler_missing_request_handler_message(UnitCallMacroKind kind, String h
 	}
 	if(kind == UnitCallMacroKind::once)
 		return("no ONCE() entry point");
+	if(kind == UnitCallMacroKind::cli)
+		return("no CLI() entry point");
 	if(kind == UnitCallMacroKind::init)
 		return("no INIT() entry point");
 	return("request handler not found");
@@ -1544,20 +1556,24 @@ request_ref_handler get_component_handler(SharedUnit* su, String render_name)
 	return(handler);
 }
 
-bool compiler_invoke_render(Request* context, String file_name, String render_name, String* error_out = 0)
+bool compiler_invoke_loaded_request_handler(
+	Request* context,
+	SharedUnit* su,
+	request_ref_handler handler,
+	UnitCallMacroKind kind,
+	String handler_name,
+	bool count_request,
+	String runtime_error_status,
+	String* error_out = 0
+)
 {
-	auto su = compiler_load_shared_unit(context, file_name, "", false);
-	if(!su)
-		return(false);
-
 	if(!compiler_prepare_request_handler(context, su, error_out, true))
 		return(false);
 
-	auto handler = get_page_render_handler(su, render_name);
 	if(!handler)
 	{
 		if(error_out)
-			*error_out = compiler_missing_request_handler_message(UnitCallMacroKind::render, render_name);
+			*error_out = compiler_missing_request_handler_message(kind, handler_name);
 		return(false);
 	}
 
@@ -1565,37 +1581,46 @@ bool compiler_invoke_render(Request* context, String file_name, String render_na
 		context,
 		su,
 		handler,
-		compiler_is_request_entry_unit(context, su),
-		"uncaught exception during render"
+		count_request,
+		runtime_error_status
 	);
 	return(true);
 }
 
+bool compiler_invoke_render(Request* context, String file_name, String render_name, String* error_out = 0)
+{
+	auto su = compiler_load_shared_unit(context, file_name, "", false);
+	if(!su)
+		return(false);
+
+	return(compiler_invoke_loaded_request_handler(
+		context,
+		su,
+		get_page_render_handler(su, render_name),
+		UnitCallMacroKind::render,
+		render_name,
+		compiler_is_request_entry_unit(context, su),
+		"uncaught exception during render",
+		error_out
+	));
+}
+
 bool compiler_invoke_component(Request* context, String file_name, String render_name, String* error_out = 0)
 {
 	auto su = compiler_load_shared_unit(context, file_name, "", false);
 	if(!su)
 		return(false);
 
-	if(!compiler_prepare_request_handler(context, su, error_out, true))
-		return(false);
-
-	auto handler = get_component_handler(su, render_name);
-	if(!handler)
-	{
-		if(error_out)
-			*error_out = compiler_missing_request_handler_message(UnitCallMacroKind::component, render_name);
-		return(false);
-	}
-
-	compiler_execute_request_handler(
+	return(compiler_invoke_loaded_request_handler(
 		context,
 		su,
-		handler,
+		get_component_handler(su, render_name),
+		UnitCallMacroKind::component,
+		render_name,
 		false,
-		"uncaught exception during component render"
-	);
-	return(true);
+		"uncaught exception during component render",
+		error_out
+	));
 }
 
 void compiler_invoke(Request* context, String file_name)
@@ -1610,6 +1635,33 @@ void compiler_invoke(Request* context, String file_name)
 	}
 }
 
+void compiler_invoke_cli(Request* context, String file_name)
+{
+	auto su = compiler_load_shared_unit(context, file_name, "", false);
+	if(!su)
+		return;
+
+	String error_message = "";
+	if(!compiler_invoke_loaded_request_handler(
+		context,
+		su,
+		su->on_cli,
+		UnitCallMacroKind::cli,
+		"",
+		compiler_is_request_entry_unit(context, su),
+		"uncaught exception during cli handler",
+		&error_message
+	))
+	{
+		if(!su->on_cli)
+			context->set_status(404, "CLI Entry Point Not Found");
+		else
+			context->set_status(500, "CLI Unit Error");
+		if(error_message != "")
+			print(error_message);
+	}
+}
+
 void compiler_invoke_websocket(Request* context, String file_name)
 {
 	auto su = compiler_load_shared_unit(context, file_name, "", false);
@@ -1767,19 +1819,31 @@ DTree* unit_call(String file_name, String function_name, DTree* call_param)
 				}
 				else
 				{
-					UnitInvocationScope invoke_scope(context, su);
-					su->on_setup(context);
 					request_ref_handler handler = 0;
 
 					if(macro_target.kind == UnitCallMacroKind::once)
 						handler = su->on_once;
 					else if(macro_target.kind == UnitCallMacroKind::init)
 						handler = su->on_init;
+					else if(macro_target.kind == UnitCallMacroKind::cli)
+						handler = su->on_cli;
 
 					if(!handler)
 						print("Error: unit_call() ", compiler_missing_request_handler_message(macro_target.kind, macro_target.handler_name));
+					else if(macro_target.kind == UnitCallMacroKind::cli)
+					{
+						String prepare_error = "";
+						if(!compiler_prepare_request_handler(context, su, &prepare_error, true))
+							print("Error: unit_call() ", prepare_error);
+						else
+							compiler_execute_request_handler(context, su, handler, false, "uncaught exception during cli handler");
+					}
 					else
+					{
+						UnitInvocationScope invoke_scope(context, su);
+						su->on_setup(context);
 						handler(*context);
+					}
 				}
 			}
 			else

src/lib/compiler.h

@@ -5,6 +5,7 @@
 #define ONCE(X) extern "C" void __uce_once(X)
 #define INIT(X) extern "C" void __uce_init(X)
 #define WS(X) extern "C" void __uce_websocket(X)
+#define CLI(X) extern "C" void __uce_cli(X)
 #define EXPORT extern "C"
 
 String preprocess_shared_unit(Request* context, SharedUnit* su);
@@ -14,6 +15,7 @@ void compile_shared_unit(Request* context, SharedUnit* su);
 SharedUnit* get_shared_unit(Request* context, String file_name, bool opt_so_optional = false);
 void compiler_invoke(Request* context, String file_name);
 void compiler_invoke_websocket(Request* context, String file_name);
+void compiler_invoke_cli(Request* context, String file_name);
 SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String current_path = "", bool opt_so_optional = false);
 String compiler_site_directory(Request* context);
 StringList compiler_scan_site_units(Request* context);

src/lib/functionlib.h

@@ -94,6 +94,10 @@ String html_escape(f64 a);
 String json_encode(String s, char quote_char = '"');
 String json_encode(DTree t, char quote_char = '"');
 DTree json_decode(String s);
+String xml_encode(DTree t, String root_name = "root");
+DTree xml_decode(String s);
+String yaml_encode(DTree t);
+DTree yaml_decode(String s);
 
 String var_dump(StringMap map, String prefix = "", String postfix = "\n");
 String var_dump(StringList slist, String prefix = "", String postfix = "\n");

src/lib/sys.cpp

@@ -613,89 +613,201 @@ pid_t spawn_subprocess(std::function<void()> exec_after_spawn)
 	}
 }
 
+String task_safe_key(String key)
+{
+	key = trim(key);
+	if(key == "")
+		throw std::runtime_error("task key cannot be empty");
+	return(gen_sha1(key));
+}
+
+String task_file_prefix(String key)
+{
+	return(path_join(context->server->config["BIN_DIRECTORY"], "task-" + task_safe_key(key)));
+}
+
+struct TaskStatus {
+	pid_t pid = 0;
+	String process_start_ticks = "";
+};
+
+TaskStatus task_status_parse(String status_file)
+{
+	TaskStatus status;
+	auto lines = split(trim(status_file), "\n");
+	if(lines.size() > 0)
+		status.pid = (pid_t)int_val(trim(lines[0]));
+	if(lines.size() > 1)
+		status.process_start_ticks = trim(lines[1]);
+	return(status);
+}
+
+String task_process_start_ticks(pid_t pid)
+{
+	if(pid <= 0)
+		return("");
+	String stat_file_name = "/proc/" + std::to_string(pid) + "/stat";
+	int fd = open(stat_file_name.c_str(), O_RDONLY);
+	if(fd == -1)
+		return("");
+	char buffer[4096];
+	ssize_t bytes_read = read(fd, buffer, sizeof(buffer) - 1);
+	close(fd);
+	if(bytes_read <= 0)
+		return("");
+	buffer[bytes_read] = '\0';
+	String stat = buffer;
+	size_t command_end = stat.rfind(") ");
+	if(command_end == String::npos)
+		return("");
+	String after_command = stat.substr(command_end + 2);
+	auto fields = split_space(after_command);
+	if(fields.size() <= 19)
+		return("");
+	return(fields[19]);
+}
+
+String task_status_content(pid_t pid)
+{
+	return(std::to_string(pid) + "\n" + task_process_start_ticks(pid) + "\n");
+}
+
+bool task_status_is_alive(TaskStatus status)
+{
+	if(status.pid <= 0)
+		return(false);
+	if(kill(status.pid, 0) != 0)
+		return(false);
+	if(status.process_start_ticks == "")
+		return(true);
+	return(task_process_start_ticks(status.pid) == status.process_start_ticks);
+}
+
 int task_kill(pid_t pid, int sig)
 {
+	if(pid <= 0)
+	{
+		errno = EINVAL;
+		return(-1);
+	}
 	return(kill(pid, sig));
 }
 
 pid_t task_pid(String key)
 {
-	String status_file_name = context->server->config["BIN_DIRECTORY"] + "/task-" + key;
+	String status_file_name = task_file_prefix(key);
 	String lock_file_name = status_file_name + ".lock";
-	int lock_fd = file_open_locked(lock_file_name, O_RDWR | O_CREAT, LOCK_EX, 0644);
+	int lock_fd = file_open_locked(lock_file_name, O_RDWR | O_CREAT, LOCK_EX, 0644, FILE_LOCK_WAIT_TIMEOUT_SECONDS, "task-pid:" + key);
+	if(lock_fd == -1)
+	{
+		fprintf(stderr, "task_pid(): could not lock task key '%s'\n", key.c_str());
+		return(0);
+	}
 	String status_file = file_get_contents(status_file_name);
-	pid_t p = 0;
 	if(status_file != "")
 	{
-		p = int_val(status_file);
-		if(task_kill(p, 0) == 0) // process is still running
+		TaskStatus status = task_status_parse(status_file);
+		if(task_status_is_alive(status))
 		{
 			file_close_locked(lock_fd);
-			return(p);
+			return(status.pid);
 		}
 		file_unlink(status_file_name);
 	}
 	file_close_locked(lock_fd);
-	return(p);
+	return(0);
 }
 
 pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout)
 {
-	String status_file_name = context->server->config["BIN_DIRECTORY"] + "/task-" + key;
+	String status_file_name = task_file_prefix(key);
 	String lock_file_name = status_file_name + ".lock";
 	int lock_fd = file_open_locked(lock_file_name, O_RDWR | O_CREAT, LOCK_EX, 0644, FILE_LOCK_WAIT_TIMEOUT_SECONDS, "task:" + key);
 	if(lock_fd == -1)
+	{
+		fprintf(stderr, "task(): could not lock task key '%s'\n", key.c_str());
 		return(0);
+	}
 	String status_file = file_get_contents(status_file_name);
-	pid_t p;
+	pid_t p = 0;
 	if(status_file != "")
 	{
-		p = int_val(status_file);
-		if(task_kill(p, 0) == 0) // process is still running
+		TaskStatus status = task_status_parse(status_file);
+		if(task_status_is_alive(status))
 		{
-			printf("(P) worker process '%s' already running: PID %i\n", key.c_str(), p);
+			printf("(P) worker process '%s' already running: PID %i\n", key.c_str(), status.pid);
 			file_close_locked(lock_fd);
-			return(p);
+			return(status.pid);
 		}
-		//printf("(P) worker process '%s' had crashed: PID %i\n", key.c_str(), p);
 		file_unlink(status_file_name);
 	}
 	p = fork();
+	if(p < 0)
+	{
+		fprintf(stderr, "task(): fork failed for key '%s': %s\n", key.c_str(), strerror(errno));
+		file_close_locked(lock_fd);
+		return(0);
+	}
 	if(p == 0)
 	{
 		file_release_process_locks("task child startup");
 		file_close_locked(lock_fd);
 		my_pid = getpid();
+		prctl(PR_SET_PDEATHSIG, SIGHUP);
+		if(timeout > 0)
+			alarm(timeout);
 
-		close(context->resources.client_socket);
-		context->resources.client_socket = 0;
-		//printf("(C) child procress started, PID:%i\n", my_pid);
-		//prctl(PR_SET_PDEATHSIG, SIGHUP);
+		if(context->resources.client_socket > 0)
+		{
+			close(context->resources.client_socket);
+			context->resources.client_socket = 0;
+		}
 		exec_after_spawn();
 		int exit_lock_fd = file_open_locked(lock_file_name, O_RDWR | O_CREAT, LOCK_EX, 0644, FILE_LOCK_WAIT_TIMEOUT_SECONDS, "task-exit:" + key);
-		file_unlink(status_file_name);
-		file_close_locked(exit_lock_fd);
+		if(exit_lock_fd != -1)
+		{
+			file_unlink(status_file_name);
+			file_close_locked(exit_lock_fd);
+		}
+		else
+		{
+			fprintf(stderr, "task(): could not lock task key '%s' during child cleanup\n", key.c_str());
+		}
 		printf("(P) worker process '%s' terminated: PID %i\n", key.c_str(), my_pid);
 		exit(0);
 	}
-	else
+
+	if(!file_put_contents(status_file_name, task_status_content(p)))
 	{
-		file_put_contents(status_file_name, std::to_string(p));
+		fprintf(stderr, "task(): could not write status file for key '%s'; terminating child PID %i\n", key.c_str(), p);
+		kill(p, SIGTERM);
 		file_close_locked(lock_fd);
-		printf("(P) worker process '%s' spawned: PID %i\n", key.c_str(), p);
-		return(p);
+		return(0);
 	}
+	file_close_locked(lock_fd);
+	printf("(P) worker process '%s' spawned: PID %i\n", key.c_str(), p);
+	return(p);
 }
 
 #include <unistd.h>
 pid_t task_repeat(String key, f64 interval, std::function<void()> exec_after_spawn, u64 timeout)
 {
-	auto repeater_function = [&]() {
-		while (true)
+	if(interval <= 0)
+		throw std::runtime_error("task_repeat(): interval must be greater than zero");
+	auto repeater_function = [key, interval, exec_after_spawn, timeout]() {
+		f64 started_at = time_precise();
+		while (timeout == 0 || time_precise() - started_at < (f64)timeout)
 		{
 			exec_after_spawn();
+			f64 elapsed = time_precise() - started_at;
+			if(timeout > 0 && elapsed >= (f64)timeout)
+				break;
+			f64 sleep_seconds = interval;
+			if(timeout > 0 && elapsed + sleep_seconds > (f64)timeout)
+				sleep_seconds = (f64)timeout - elapsed;
 			printf("(P) worker process '%s' sleeping\n", key.c_str());
-			usleep((s64)(interval*1000000));
+			if(sleep_seconds > 0)
+				usleep((useconds_t)(sleep_seconds * 1000000.0));
 		}
 	};
 	return(task(key, repeater_function, timeout));
@@ -703,17 +815,21 @@ pid_t task_repeat(String key, f64 interval, std::function<void()> exec_after_spa
 
 void on_child_exit(int sig)
 {
-    pid_t pid;
-    int   status;
-    if ((pid = waitpid(-1, &status, WNOHANG)) != -1)
-    {
+	(void)sig;
+	pid_t pid;
+	int status;
+	while((pid = waitpid(-1, &status, WNOHANG)) > 0)
+	{
 		if(workers.count(pid) > 0)
 		{
 			workers.erase(pid);
 			printf("(P) child terminated (PID:%i)\n", pid);
-			//spawn_subprocess();
 		}
-    }
+		else
+		{
+			printf("(P) task child reaped (PID:%i)\n", pid);
+		}
+	}
 }
 
 StringList ls(String dir)
@@ -731,6 +847,7 @@ StringMap make_server_settings()
 	cfg["LIT_ESC"] = "3d5b5_1";
 	cfg["CONTENT_TYPE"] = "text/html; charset=utf-8";
 	cfg["FCGI_SOCKET_PATH"] = "/run/uce.sock";
+	cfg["CLI_SOCKET_PATH"] = "/run/uce/cli.sock";
 	cfg["TMP_UPLOAD_PATH"] = "/tmp/uce/uploads";
 	cfg["SESSION_PATH"] = "/tmp/uce/sessions";
 	cfg["COMPILER_SYS_PATH"] = ".";

src/lib/types.h

@@ -85,6 +85,7 @@ struct SharedUnit {
 	request_ref_handler on_render = 0;
 	request_ref_handler on_component = 0;
 	request_ref_handler on_websocket = 0;
+	request_ref_handler on_cli = 0;
 	request_ref_handler on_once = 0;
 	request_ref_handler on_init = 0;
 
@@ -210,6 +211,7 @@ struct Request {
 		u64 client_socket = 0;
 		u64 server_socket = 0;
 		bool is_websocket = false;
+		bool is_cli = false;
 		String websocket_connection_id = "";
 		String websocket_scope = "";
 		DTree* websocket_connection_state = 0;

src/lib/uce_lib.cpp

@@ -6,7 +6,9 @@
 #include "hash.cpp"
 #include "sys.cpp"
 #include "uri.cpp"
+#include "cli.cpp"
 #include "compiler-parser.cpp"
 #include "compiler.cpp"
 #include "markdown.cpp"
+#include "zip.cpp"
 #include "mysql-connector.cpp"

src/lib/uce_lib.h

@@ -6,6 +6,8 @@
 #include "functionlib.h"
 #include "sys.h"
 #include "uri.h"
+#include "cli.h"
 #include "compiler.h"
 #include "markdown.h"
+#include "zip.h"
 #include "mysql-connector.h"

src/lib/uri.cpp

@@ -707,6 +707,11 @@ bool WSFrame::parse(const String& buffer, String& error)
 		if(buffer.length() < 4)
 			return(false);
 		payload_length = ((u64)raw[2] << 8) | (u64)raw[3];
+		if(payload_length < 126)
+		{
+			error = "non-minimal websocket frame length";
+			return(false);
+		}
 		header_length = 4;
 	}
 	else if(payload_length == 127)
@@ -721,6 +726,11 @@ bool WSFrame::parse(const String& buffer, String& error)
 		payload_length = 0;
 		for(u32 i = 0; i < 8; i++)
 			payload_length = (payload_length << 8) | (u64)raw[2 + i];
+		if(payload_length <= 0xFFFF)
+		{
+			error = "non-minimal websocket frame length";
+			return(false);
+		}
 		header_length = 10;
 	}
 

src/lib/zip.cpp

@@ -0,0 +1,350 @@
+#include "zip.h"
+
+#include <cstring>
+
+extern "C" {
+#include "../../vendor/miniz/miniz.c"
+#include "../../vendor/miniz/miniz_tdef.c"
+#include "../../vendor/miniz/miniz_tinfl.c"
+#include "../../vendor/miniz/miniz_zip.c"
+}
+
+String zip_error(String api, String detail)
+{
+	return(api + "(): " + detail);
+}
+
+bool zip_entry_name_safe(String name)
+{
+	if(name == "")
+		return(false);
+	if(name[0] == '/' || name[0] == '\\')
+		return(false);
+	if(name.find(":") != String::npos)
+		return(false);
+
+	auto parts = split(replace(name, "\\", "/"), "/");
+	for(auto part : parts)
+	{
+		if(part == "..")
+			return(false);
+	}
+	return(true);
+}
+
+String zip_normalize_entry_name(String name)
+{
+	name = replace(name, "\\", "/");
+	while(name.find("//") != String::npos)
+		name = replace(name, "//", "/");
+	return(name);
+}
+
+bool zip_ensure_parent_directory(String path)
+{
+	String parent = dirname(path);
+	if(parent == "" || parent == "." || parent == "/")
+		return(true);
+	if(file_exists(parent))
+		return(true);
+	return(mkdir(parent));
+}
+
+void zip_add_file_info(DTree& files, mz_zip_archive* archive, mz_uint index)
+{
+	mz_zip_archive_file_stat stat;
+	std::memset(&stat, 0, sizeof(stat));
+	if(!mz_zip_reader_file_stat(archive, index, &stat))
+		throw std::runtime_error(zip_error("zip_list", "could not read file metadata at index " + std::to_string((u64)index)));
+
+	DTree item;
+	item["name"] = String(stat.m_filename);
+	item["index"] = (f64)index;
+	item["size"] = (f64)stat.m_uncomp_size;
+	item["compressed_size"] = (f64)stat.m_comp_size;
+	item["is_directory"].set_bool(mz_zip_reader_is_file_a_directory(archive, index));
+	item["method"] = (f64)stat.m_method;
+	files.push(item);
+}
+
+DTree zip_list(String zip_file_name)
+{
+	mz_zip_archive archive;
+	std::memset(&archive, 0, sizeof(archive));
+	if(!mz_zip_reader_init_file(&archive, zip_file_name.c_str(), 0))
+		throw std::runtime_error(zip_error("zip_list", "could not open " + zip_file_name));
+
+	DTree result;
+	try
+	{
+		mz_uint count = mz_zip_reader_get_num_files(&archive);
+		result["file"] = zip_file_name;
+		result["count"] = (f64)count;
+		result["entries"].set_array();
+		for(mz_uint i = 0; i < count; i++)
+			zip_add_file_info(result["entries"], &archive, i);
+		mz_zip_reader_end(&archive);
+		return(result);
+	}
+	catch(...)
+	{
+		mz_zip_reader_end(&archive);
+		throw;
+	}
+}
+
+String zip_read(String zip_file_name, String entry_name)
+{
+	entry_name = zip_normalize_entry_name(entry_name);
+	if(!zip_entry_name_safe(entry_name))
+		throw std::runtime_error(zip_error("zip_read", "unsafe or empty entry name '" + entry_name + "'"));
+
+	mz_zip_archive archive;
+	std::memset(&archive, 0, sizeof(archive));
+	if(!mz_zip_reader_init_file(&archive, zip_file_name.c_str(), 0))
+		throw std::runtime_error(zip_error("zip_read", "could not open " + zip_file_name));
+
+	size_t size = 0;
+	void* data = mz_zip_reader_extract_file_to_heap(&archive, entry_name.c_str(), &size, 0);
+	if(!data)
+	{
+		mz_zip_reader_end(&archive);
+		throw std::runtime_error(zip_error("zip_read", "entry not found or not readable: " + entry_name));
+	}
+	String result((char*)data, size);
+	mz_free(data);
+	mz_zip_reader_end(&archive);
+	return(result);
+}
+
+String zip_entry_content(DTree item)
+{
+	DTree* content = item.key("content");
+	if(content)
+		return(content->to_string());
+	DTree* file_name = item.key("file");
+	if(file_name)
+		return(file_get_contents(file_name->to_string()));
+	return(item.to_string());
+}
+
+String zip_entry_name(String key, DTree item)
+{
+	DTree* name = item.key("name");
+	if(name)
+		return(name->to_string());
+	return(key);
+}
+
+void zip_add_entry(mz_zip_archive* archive, String name, String content)
+{
+	name = zip_normalize_entry_name(name);
+	if(!zip_entry_name_safe(name))
+		throw std::runtime_error(zip_error("zip_create", "unsafe or empty entry name '" + name + "'"));
+	if(!mz_zip_writer_add_mem(archive, name.c_str(), content.data(), content.size(), MZ_BEST_COMPRESSION))
+		throw std::runtime_error(zip_error("zip_create", "could not add entry " + name));
+}
+
+bool zip_create(String zip_file_name, DTree entries)
+{
+	if(!zip_ensure_parent_directory(zip_file_name))
+		throw std::runtime_error(zip_error("zip_create", "could not create parent directory for " + zip_file_name));
+
+	mz_zip_archive archive;
+	std::memset(&archive, 0, sizeof(archive));
+	if(!mz_zip_writer_init_file(&archive, zip_file_name.c_str(), 0))
+		throw std::runtime_error(zip_error("zip_create", "could not open " + zip_file_name + " for writing"));
+
+	try
+	{
+		entries.each([&](DTree item, String key)
+		{
+			String name = zip_entry_name(key, item);
+			String content = zip_entry_content(item);
+			zip_add_entry(&archive, name, content);
+		});
+		if(!mz_zip_writer_finalize_archive(&archive))
+			throw std::runtime_error(zip_error("zip_create", "could not finalize " + zip_file_name));
+		mz_zip_writer_end(&archive);
+		return(true);
+	}
+	catch(...)
+	{
+		mz_zip_writer_end(&archive);
+		throw;
+	}
+}
+
+void gz_append_u32_le(String& out, u32 value)
+{
+	out.push_back((char)(value & 0xff));
+	out.push_back((char)((value >> 8) & 0xff));
+	out.push_back((char)((value >> 16) & 0xff));
+	out.push_back((char)((value >> 24) & 0xff));
+}
+
+u32 gz_read_u32_le(String src, size_t offset)
+{
+	return(
+		((u32)(u8)src[offset]) |
+		((u32)(u8)src[offset + 1] << 8) |
+		((u32)(u8)src[offset + 2] << 16) |
+		((u32)(u8)src[offset + 3] << 24)
+	);
+}
+
+void gz_skip_zero_terminated(String compressed, size_t& offset)
+{
+	while(offset < compressed.size() && compressed[offset] != '\0')
+		offset++;
+	if(offset >= compressed.size())
+		throw std::runtime_error("gz_uncompress(): truncated gzip header");
+	offset++;
+}
+
+size_t gz_deflate_offset(String compressed)
+{
+	if(compressed.size() < 18)
+		throw std::runtime_error("gz_uncompress(): compressed data is too short");
+	if((u8)compressed[0] != 0x1f || (u8)compressed[1] != 0x8b)
+		throw std::runtime_error("gz_uncompress(): missing gzip header");
+	if((u8)compressed[2] != 8)
+		throw std::runtime_error("gz_uncompress(): unsupported gzip compression method");
+
+	u8 flags = (u8)compressed[3];
+	if(flags & 0xe0)
+		throw std::runtime_error("gz_uncompress(): gzip header uses reserved flags");
+
+	size_t offset = 10;
+	if(flags & 0x04)
+	{
+		if(offset + 2 > compressed.size())
+			throw std::runtime_error("gz_uncompress(): truncated gzip extra header");
+		u16 extra_len = ((u16)(u8)compressed[offset]) | ((u16)(u8)compressed[offset + 1] << 8);
+		offset += 2 + extra_len;
+		if(offset > compressed.size())
+			throw std::runtime_error("gz_uncompress(): truncated gzip extra data");
+	}
+	if(flags & 0x08)
+		gz_skip_zero_terminated(compressed, offset);
+	if(flags & 0x10)
+		gz_skip_zero_terminated(compressed, offset);
+	if(flags & 0x02)
+	{
+		offset += 2;
+		if(offset > compressed.size())
+			throw std::runtime_error("gz_uncompress(): truncated gzip header crc");
+	}
+	if(offset + 8 > compressed.size())
+		throw std::runtime_error("gz_uncompress(): missing gzip footer");
+	return(offset);
+}
+
+String gz_compress(String src)
+{
+	mz_stream stream;
+	std::memset(&stream, 0, sizeof(stream));
+	int status = mz_deflateInit2(&stream, MZ_DEFAULT_COMPRESSION, MZ_DEFLATED, -MZ_DEFAULT_WINDOW_BITS, 9, MZ_DEFAULT_STRATEGY);
+	if(status != MZ_OK)
+		throw std::runtime_error("gz_compress(): could not initialize compressor");
+
+	mz_ulong bound = mz_deflateBound(&stream, src.size());
+	String deflated;
+	deflated.resize(bound);
+	stream.next_in = (const unsigned char*)src.data();
+	stream.avail_in = src.size();
+	stream.next_out = (unsigned char*)deflated.data();
+	stream.avail_out = deflated.size();
+	status = mz_deflate(&stream, MZ_FINISH);
+	if(status != MZ_STREAM_END)
+	{
+		mz_deflateEnd(&stream);
+		throw std::runtime_error("gz_compress(): compression failed");
+	}
+	deflated.resize(stream.total_out);
+	mz_deflateEnd(&stream);
+
+	String result;
+	result.reserve(10 + deflated.size() + 8);
+	result.push_back((char)0x1f);
+	result.push_back((char)0x8b);
+	result.push_back((char)0x08);
+	result.push_back((char)0x00);
+	gz_append_u32_le(result, 0);
+	result.push_back((char)0x00);
+	result.push_back((char)0xff);
+	result.append(deflated);
+	gz_append_u32_le(result, (u32)mz_crc32(MZ_CRC32_INIT, (const unsigned char*)src.data(), src.size()));
+	gz_append_u32_le(result, (u32)src.size());
+	return(result);
+}
+
+String gz_uncompress(String compressed)
+{
+	size_t deflate_offset = gz_deflate_offset(compressed);
+	size_t footer_offset = compressed.size() - 8;
+	size_t deflate_size = footer_offset - deflate_offset;
+	size_t out_len = 0;
+	void* out = tinfl_decompress_mem_to_heap(compressed.data() + deflate_offset, deflate_size, &out_len, 0);
+	if(!out)
+		throw std::runtime_error("gz_uncompress(): decompression failed");
+
+	String result((char*)out, out_len);
+	mz_free(out);
+
+	u32 expected_crc = gz_read_u32_le(compressed, footer_offset);
+	u32 expected_size = gz_read_u32_le(compressed, footer_offset + 4);
+	u32 actual_crc = (u32)mz_crc32(MZ_CRC32_INIT, (const unsigned char*)result.data(), result.size());
+	if(actual_crc != expected_crc)
+		throw std::runtime_error("gz_uncompress(): crc check failed");
+	if((u32)result.size() != expected_size)
+		throw std::runtime_error("gz_uncompress(): size check failed");
+	return(result);
+}
+
+bool zip_extract(String zip_file_name, String destination_directory)
+{
+	if(destination_directory == "")
+		throw std::runtime_error(zip_error("zip_extract", "destination directory is empty"));
+	if(!file_exists(destination_directory) && !mkdir(destination_directory))
+		throw std::runtime_error(zip_error("zip_extract", "could not create destination directory " + destination_directory));
+
+	mz_zip_archive archive;
+	std::memset(&archive, 0, sizeof(archive));
+	if(!mz_zip_reader_init_file(&archive, zip_file_name.c_str(), 0))
+		throw std::runtime_error(zip_error("zip_extract", "could not open " + zip_file_name));
+
+	try
+	{
+		mz_uint count = mz_zip_reader_get_num_files(&archive);
+		for(mz_uint i = 0; i < count; i++)
+		{
+			mz_zip_archive_file_stat stat;
+			std::memset(&stat, 0, sizeof(stat));
+			if(!mz_zip_reader_file_stat(&archive, i, &stat))
+				throw std::runtime_error(zip_error("zip_extract", "could not read file metadata at index " + std::to_string((u64)i)));
+			String name = zip_normalize_entry_name(stat.m_filename);
+			if(!zip_entry_name_safe(name))
+				throw std::runtime_error(zip_error("zip_extract", "refusing unsafe entry name '" + name + "'"));
+
+			String target = path_join(destination_directory, name);
+			if(mz_zip_reader_is_file_a_directory(&archive, i))
+			{
+				if(!file_exists(target) && !mkdir(target))
+					throw std::runtime_error(zip_error("zip_extract", "could not create directory " + target));
+				continue;
+			}
+			if(!zip_ensure_parent_directory(target))
+				throw std::runtime_error(zip_error("zip_extract", "could not create parent directory for " + target));
+			if(!mz_zip_reader_extract_to_file(&archive, i, target.c_str(), 0))
+				throw std::runtime_error(zip_error("zip_extract", "could not extract " + name));
+		}
+		mz_zip_reader_end(&archive);
+		return(true);
+	}
+	catch(...)
+	{
+		mz_zip_reader_end(&archive);
+		throw;
+	}
+}

src/lib/zip.h

@@ -0,0 +1,8 @@
+#pragma once
+
+DTree zip_list(String zip_file_name);
+String zip_read(String zip_file_name, String entry_name);
+bool zip_create(String zip_file_name, DTree entries);
+bool zip_extract(String zip_file_name, String destination_directory);
+String gz_compress(String src);
+String gz_uncompress(String compressed);