W7 done done

README.md

@@ -293,7 +293,7 @@ The runtime reads its server settings from:
 /etc/uce/settings.cfg
 ```
 
-The shipped example contains the important filesystem and FastCGI settings:
+The example contains the filesystem and FastCGI settings:
 
 ```ini
 BIN_DIRECTORY=/var/cache/uce/work
@@ -473,7 +473,7 @@ Important details:
 - `SCRIPT_FILENAME` should resolve to the actual `.uce` file on disk
 - `proxy_http_version 1.1` and the `Upgrade` / `Connection` headers are required for WebSockets
 
-The `location /` block above is intentionally conservative and only serves real files from `site/`. If your app uses a front-controller pattern such as routing everything through `/index.uce`, change that block accordingly.
+The `location /` block only serves files from `site/`. If your app uses a front-controller pattern such as routing everything through `/index.uce`, change that block accordingly.
 
 ### 6. Think about document root and private files
 

docs/patches/miniz-3.0.2-cxx-single-translation-unit.patch

@@ -1,6 +1,6 @@
-diff --git a/vendor/miniz/miniz_tdef.c b/vendor/miniz/miniz_tdef.c
---- a/vendor/miniz/miniz_tdef.c
-+++ b/vendor/miniz/miniz_tdef.c
+diff --git a/src/3rdparty/miniz/miniz_tdef.c b/src/3rdparty/miniz/miniz_tdef.c
+--- a/src/3rdparty/miniz/miniz_tdef.c
++++ b/src/3rdparty/miniz/miniz_tdef.c
 @@
 -static const mz_uint s_tdefl_num_probes[11];
 +static const mz_uint s_tdefl_num_probes[11] = { 0, 1, 6, 32, 16, 32, 128, 256, 512, 768, 1500 };

scripts/api_coverage_manifest.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Guard the hand-maintained unit-facing API coverage manifest.
+
+This intentionally avoids network/external services. It checks that public API
+names we expose to wasm units are either mentioned by a site test or explicitly
+marked internal/integration-only, and that active docs exist for doc-required
+APIs. The manifest is deliberately source-controlled so a new public function
+requires an explicit coverage decision.
+"""
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+TEST_DIR = ROOT / "site" / "tests"
+DOC_DIR = ROOT / "site" / "doc" / "pages"
+
+# name, needs_doc, status. status: public | internal | integration
+PUBLIC_APIS = [
+    ("shell_exec", True, "public"), ("shell_escape", True, "public"),
+    ("basename", True, "public"), ("dirname", True, "public"), ("path_join", True, "public"),
+    ("path_real", True, "public"), ("path_is_within", True, "public"),
+    ("file_open_locked", True, "public"), ("file_close_locked", True, "public"),
+    ("file_release_process_locks", True, "public"), ("file_get_contents_locked_fd", True, "public"),
+    ("file_put_contents_locked_fd", True, "public"), ("file_get_contents", True, "public"),
+    ("file_put_contents", True, "public"), ("file_append_contents", False, "public"),
+    ("cwd_get", True, "public"), ("cwd_set", True, "public"), ("process_start_directory", True, "public"),
+    ("file_mtime", True, "public"), ("file_unlink", True, "public"), ("expand_path", True, "public"),
+    ("ls", True, "public"), ("config_map_u64", True, "public"), ("config_map_f64", True, "public"),
+    ("config_bool_value", True, "public"), ("config_map_bool", True, "public"),
+    ("config_u64", True, "public"), ("config_f64", True, "public"), ("config_bool", True, "public"),
+    ("request_perf", True, "public"), ("time_format_local", True, "public"),
+    ("time_format_relative", True, "public"), ("time_parse", True, "public"),
+    ("backtrace_frames_string", False, "public"), ("capture_backtrace_string", False, "public"),
+    ("signal_name", False, "public"), ("memcache_escape_key", True, "public"),
+    ("memcache_escape_keys", True, "public"), ("memcache_command", True, "public"),
+    ("memcache_get_multiple", True, "public"), ("runtime_safe_key", True, "public"),
+    ("float_val", True, "public"), ("nibble", True, "public"), ("json_consume_space", False, "public"),
+    ("array_merge", True, "public"), ("safe_name", True, "public"), ("ascii_safe_name", True, "public"),
+    ("to_json", False, "public"), ("remove", False, "public"), ("clear", False, "public"),
+    ("gen_sha1", True, "public"), ("gen_noise32", True, "public"), ("gen_noise64", True, "public"),
+    ("gen_noise01", True, "public"), ("gen_int", True, "public"), ("gen_float", True, "public"),
+    ("draw_int", True, "public"), ("draw_float", True, "public"),
+    ("encode_query", True, "public"), ("request_script_url", True, "public"),
+    ("request_base_url", True, "public"), ("request_route_from_raw_path", True, "public"),
+    ("cli_arg", True, "public"), ("unit_compile", True, "public"),
+    ("cleanup_sqlite_connections", False, "internal"), ("cleanup_mysql_connections", False, "internal"),
+    ("mysql_connect", True, "integration"), ("mysql_query", True, "integration"),
+]
+
+REMOVED_APIS = ["unit_load", "concat"]
+
+
+def all_test_text() -> str:
+    parts = []
+    for path in TEST_DIR.glob("*.uce"):
+        parts.append(path.read_text(errors="ignore"))
+    return "\n".join(parts)
+
+
+def doc_exists(name: str) -> bool:
+    path = DOC_DIR / f"{name}.txt"
+    return path.exists() and "Removed" not in path.read_text(errors="ignore")[:200]
+
+
+def has_call(text: str, name: str) -> bool:
+    return f"{name}(" in text or f".{name}(" in text or f'"{name}"' in text
+
+
+def main() -> int:
+    tests = all_test_text()
+    errors = []
+    for name, needs_doc, status in PUBLIC_APIS:
+        if status == "public" and not has_call(tests, name):
+            errors.append(f"missing test coverage: {name}")
+        if needs_doc and status in {"public", "integration"} and not doc_exists(name):
+            errors.append(f"missing active doc page: {name}")
+    compiler_h = (ROOT / "src" / "lib" / "compiler.h").read_text(errors="ignore")
+    if "#ifndef __UCE_WASM_UNIT__\nSharedUnit* unit_load" not in compiler_h:
+        errors.append("unit_load is not guarded out of wasm-unit exposure")
+    for name in REMOVED_APIS:
+        page = DOC_DIR / f"{name}.txt"
+        if name == "concat" and page.exists() and "Removed" not in page.read_text(errors="ignore")[:300]:
+            errors.append("concat doc is not tombstoned")
+        if name == "unit_load" and page.exists() and "native-only" not in page.read_text(errors="ignore"):
+            errors.append("unit_load doc is not native-only/tombstoned")
+    if errors:
+        print("API coverage manifest FAILED")
+        for error in errors:
+            print("- " + error)
+        return 1
+    print(f"API coverage manifest ok: {len(PUBLIC_APIS)} entries checked")
+    return 0
+
+if __name__ == "__main__":
+    raise SystemExit(main())

site/tests/api_coverage.uce

@@ -0,0 +1,28 @@
+#include "testlib.h"
+
+RENDER(Request& context)
+{
+	if(!test_demo_request_allowed(context))
+	{
+		site_tests_restricted(context, "API Coverage Manifest", "scan repository headers, tests, and docs with the local manifest script");
+		return;
+	}
+
+	u64 passed = 0;
+	u64 failed = 0;
+	u64 skipped = 0;
+	auto check = [&](String name, bool ok, String detail)
+	{
+		site_tests_case(name, ok ? "pass" : "fail", detail);
+		if(ok)
+			passed++;
+		else
+			failed++;
+	};
+
+	site_tests_page_start("API Coverage Manifest", "Repository-level guard that public wasm-unit APIs have an explicit test/doc/internal decision.");
+	String out = shell_exec("cd /Code/uce.openfu.com/uce && python3 scripts/api_coverage_manifest.py");
+	check("api coverage manifest", contains(out, "API coverage manifest ok"), out);
+	site_tests_summary(passed, failed, skipped, "This page intentionally runs only on trusted/local test requests.");
+	site_tests_page_end();
+}

site/tests/cli.uce

@@ -14,6 +14,12 @@ CLI(Request& context)
 		print(input["message"].to_string(), "\n");
 		return;
 	}
+	if(action == "arg")
+	{
+		print(cli_arg(context, "message", "fallback"), "\n");
+		print(cli_arg(context, "missing", "fallback"), "\n");
+		return;
+	}
 	context.set_status(400, "CLI Error");
 	print("unknown cli action: ", action, "\n");
 }

site/tests/cli_runner.uce

@@ -325,6 +325,8 @@ CLI(Request& context)
 	print("[GROUP] starter ", std::to_string((u64)((time_precise() - group_start) * 1000)), " ms\n");
 	group_start = time_precise();
 	cli_run_tcp_smoke();
+	String cli_arg_out = shell_exec("cd /Code/uce.openfu.com/uce && bash scripts/uce-cli /tests/cli.uce action=arg message=hello-cli 2>&1");
+	cli_test_case("uce_cli_fixture:cli_arg", contains(cli_arg_out, "hello-cli") && contains(cli_arg_out, "fallback"), cli_truncate(cli_arg_out));
 	print("[GROUP] tcp ", std::to_string((u64)((time_precise() - group_start) * 1000)), " ms\n");
 	group_start = time_precise();
 	cli_run_wasm_kill(include_kill);

site/tests/core.uce

@@ -236,6 +236,30 @@ RENDER(Request& context)
 	tree["nested"]["ok"].set_bool(true);
 	check("DValue map access", tree["suite"].to_string() == "core" && tree["nested"]["api"].to_string() == "dvalue", json_encode(tree));
 
+	String nibble_source = "alpha/beta/gamma";
+	String nibble_first = nibble(nibble_source, "/");
+	StringMap merge_a; merge_a["a"] = "one"; merge_a["shared"] = "old";
+	StringMap merge_b; merge_b["b"] = "two"; merge_b["shared"] = "new";
+	StringMap merged_map = array_merge(merge_a, merge_b);
+	DValue merge_dv_a; merge_dv_a["a"] = "one"; merge_dv_a["shared"] = "old";
+	DValue merge_dv_b; merge_dv_b["b"] = "two"; merge_dv_b["shared"] = "new";
+	DValue merged_dv = array_merge(merge_dv_a, merge_dv_b);
+	u32 json_space_i = 0;
+	json_consume_space(" \n\tvalue", json_space_i);
+	check("float_val() / nibble() / array_merge() / json_consume_space()", float_val("12.5") > 12.49 && float_val("12.5") < 12.51 && nibble_first == "alpha" && nibble_source == "beta/gamma" && merged_map["shared"] == "new" && merged_map["a"] == "one" && merged_dv["b"].to_string() == "two" && json_space_i == 3, "nibble=" + nibble_first + " rest=" + nibble_source + " merged=" + var_dump(merged_map) + " json_space_i=" + std::to_string(json_space_i));
+
+	String unsafe = "Hello, 世界 / ../../ name!";
+	check("safe_name() / ascii_safe_name()", safe_name(unsafe) != "" && ascii_safe_name("Hello world! 42") == "Hello_world_42", safe_name(unsafe) + " / " + ascii_safe_name("Hello world! 42"));
+
+	DValue mutable_tree;
+	mutable_tree["keep"] = "yes";
+	mutable_tree["remove"] = "no";
+	String mutable_json = mutable_tree.to_json();
+	mutable_tree.remove("remove");
+	bool removed_ok = !mutable_tree.has("remove") && mutable_tree["keep"].to_string() == "yes";
+	mutable_tree.clear();
+	check("DValue to_json() / remove() / clear()", contains(mutable_json, "keep") && removed_ok && !mutable_tree.has("keep") && mutable_tree.get_type_name() == "empty", mutable_json);
+
 	// wasm_trace.h (WASM-PROPOSAL Phase 4): canned Wasmtime-format messages —
 	// the live trap path is gated by the phase-4 spike runner
 	String trace_msg = "error while executing at wasm backtrace:\n    0:     0x9f - units!_Z6renderR7Request\n";
@@ -255,6 +279,16 @@ RENDER(Request& context)
 
 	check("wasm_trace_collapse() raw passthrough", wasm_trace_collapse("plain host error") == "plain host error", wasm_trace_collapse("plain host error"));
 
+	u32 noise32_a = gen_noise32(7, 11);
+	u32 noise32_b = gen_noise32(7, 11);
+	u64 noise64 = gen_noise64(7, 11);
+	f64 noise01 = gen_noise01(7, 11);
+	u64 ranged_int = gen_int(10, 20, 3, 4);
+	f64 ranged_float = gen_float(1.0, 2.0, 3, 4);
+	u64 drawn_int = draw_int(1, 3);
+	f64 drawn_float = draw_float(1.0, 2.0);
+	check("gen_sha1() / gen_*() / draw_*()", gen_sha1("uce") == gen_sha1("uce") && gen_sha1("uce") != gen_sha1("UCE") && noise32_a == noise32_b && noise64 > 0 && noise01 >= 0.0 && noise01 <= 1.0 && ranged_int >= 10 && ranged_int <= 20 && ranged_float >= 1.0 && ranged_float <= 2.0 && drawn_int >= 1 && drawn_int <= 3 && drawn_float >= 1.0 && drawn_float <= 2.0, "noise32=" + std::to_string((u64)noise32_a) + " noise64=" + std::to_string(noise64) + " noise01=" + std::to_string(noise01) + " draw=" + std::to_string(drawn_int));
+
 	site_tests_summary(passed, failed, skipped, "These assertions intentionally stay pure and side-effect free so they remain safe on the public site.");
 	site_tests_page_end();
 }

site/tests/http.uce

@@ -44,6 +44,12 @@ RENDER(Request& context)
 	check("uri_decode() malformed percent literals", malformed_percent == "% %A %GG ok done", malformed_percent);
 	check("parse_uri() empty input", empty_uri.parts["raw"] == "", var_dump(empty_uri));
 	check("parse_query()", query["alpha"] == "1" && query["beta"] == "two words" && query["token"] == "a=b" && query["keyless"] == "" && query["empty"] == "" && query["trailing"] == "ok" && query.count("") == 0, query_dump);
+	StringMap encoded_query_map;
+	encoded_query_map["alpha"] = "one two";
+	encoded_query_map["token"] = "a=b";
+	String encoded_query = encode_query(encoded_query_map);
+	DValue raw_route = request_route_from_raw_path("/workspace/projects/", "index");
+	check("encode_query() / request route helpers", contains(encoded_query, "alpha=one%20two") && contains(encoded_query, "token=a%3Db") && request_script_url(context) != "" && request_base_url(context) != "" && raw_route["path"].to_string() == "workspace/projects" && raw_route["valid"].to_bool(), encoded_query + " script=" + request_script_url(context) + " base=" + request_base_url(context) + " route=" + json_encode(raw_route));
 	check("set_cookie()", set_cookie_dump.find("site-tests-cookie") != String::npos && set_cookie_dump.find("cookie-value") != String::npos && set_cookie_dump.find("HttpOnly") != String::npos && set_cookie_dump.find("SameSite=Lax") != String::npos, set_cookie_dump);
 	check("response header mutation", context.header["X-Site-Tests"] == "http-suite", header_dump);
 	check("session_start()", session_id != "", "session_id=" + session_id);

site/tests/io.uce

@@ -29,9 +29,57 @@ RENDER(Request& context)
 	site_tests_page_start("Filesystem", "Local-only file helper coverage using a temporary file under /tmp.");
 
 	check("path_join()", file_name == "/tmp/uce-site-tests-io.txt", file_name);
+	check("basename() / dirname() / expand_path()", basename(file_name) == "uce-site-tests-io.txt" && dirname(file_name) == "/tmp" && expand_path("child", "/tmp/base") == "/tmp/base/child" && expand_path("../sibling", "/tmp/base") == "/tmp/sibling", basename(file_name) + " / " + dirname(file_name) + " / " + expand_path("../sibling", "/tmp/base"));
 	check("file_put_contents()", write_ok, file_name);
-	check("file_append()", file_text.find("|beta") != String::npos, file_text);
+	check("file_append() / file_append_contents()", file_append_contents(file_name, "|gamma") && contains(file_get_contents(file_name), "|gamma"), file_get_contents(file_name));
+	file_put_contents(file_name, "alpha|beta");
 	check("file_get_contents()", file_text == "alpha|beta", file_text);
+	check("file_mtime()", file_mtime(file_name) > 0, std::to_string((u64)file_mtime(file_name)));
+
+	String real_tmp = path_real("/tmp");
+	String real_file = path_real(file_name);
+	check("path_real() / path_is_within() canonical containment", real_tmp != "" && real_file != "" && path_is_within(file_name, "/tmp") && path_is_within("/tmp/../tmp/uce-site-tests-io.txt", "/tmp") && !path_is_within("/tmp", file_name) && !path_is_within("/tmp", "/tmp/uce-site-tests-io.txt"), real_tmp + " / " + real_file);
+
+	String shell_escaped = shell_escape("a'b; echo injected");
+	String shell_out = shell_exec("printf %s " + shell_escaped);
+	check("shell_escape() / shell_exec()", shell_out == "a'b; echo injected" && contains(shell_escaped, "'\\''"), shell_escaped + " => " + shell_out);
+
+	String lock_file = "/tmp/uce-site-tests-lock.txt";
+	int lock_fd = file_open_locked(lock_file, 66, LOCK_EX, 0644, 3.0, "site tests");
+	bool lock_write = file_put_contents_locked_fd(lock_fd, "locked-content");
+	String lock_read = file_get_contents_locked_fd(lock_fd);
+	file_close_locked(lock_fd);
+	check("file_open_locked() / locked fd read-write / close", lock_fd > 0 && lock_write && lock_read == "locked-content" && file_get_contents(lock_file) == "locked-content", "fd=" + std::to_string(lock_fd) + " read=" + lock_read);
+	int release_fd = file_open_locked(lock_file, 66, LOCK_EX, 0644, 3.0, "release test");
+	file_release_process_locks("site tests release");
+	check("file_release_process_locks()", release_fd > 0, "fd=" + std::to_string(release_fd));
+
+	String start_dir = process_start_directory();
+	String old_cwd = cwd_get();
+	cwd_set("/tmp");
+	String tmp_cwd = cwd_get();
+	cwd_set(old_cwd);
+	check("cwd_get() / cwd_set() / process_start_directory()", old_cwd != "" && tmp_cwd == "/tmp" && process_start_directory() == start_dir && start_dir != "", "old=" + old_cwd + " tmp=" + tmp_cwd + " start=" + start_dir);
+
+	StringMap cfg;
+	cfg["u64"] = "42";
+	cfg["f64"] = "3.5";
+	cfg["yes"] = "yes";
+	cfg["no"] = "off";
+	check("config_map_*() / config_*() helpers", config_map_u64(cfg, "u64", 7) == 42 && config_map_u64(cfg, "bad", 7) == 7 && config_map_f64(cfg, "f64", 1.0) > 3.49 && !config_map_bool(cfg, "no") && config_bool_value("false") == false && config_u64("NO_SUCH_UCE_TEST_KEY", 99) == 99 && config_f64("NO_SUCH_UCE_TEST_KEY", 1.25) == 1.25 && !config_bool("NO_SUCH_UCE_TEST_KEY", false), var_dump(cfg));
+
+	DValue perf = request_perf();
+	check("request_perf()", perf["worker_pid"].to_u64() > 0 && perf["request_count"].to_u64() > 0, json_encode(perf));
+
+	u64 parsed_epoch = time_parse("1970-01-01 00:00:05 UTC");
+	String utc_year = time_format_utc("%Y", parsed_epoch);
+	String local_year = time_format_local("%Y", parsed_epoch);
+	String relative = time_format_relative(time() - 120, "recent %deltaS", 1, "medium %deltaM", 3600, "old %deltaH");
+	check("time_format_local() / time_format_relative() / time_parse()", parsed_epoch == 5 && utc_year == "1970" && local_year != "" && contains(relative, "medium"), "parsed=" + std::to_string(parsed_epoch) + " utc=" + utc_year + " local=" + local_year + " rel=" + relative);
+
+	StringList escaped_keys = memcache_escape_keys({"a b", "line\nkey"});
+	check("memcache_escape_key() / memcache_escape_keys()", memcache_escape_key("a b") == "a_b" && escaped_keys.size() == 2 && !contains(escaped_keys[1], "\n"), join(escaped_keys, ","));
+	check("signal_name() / runtime_safe_key() / capture_backtrace_string()", signal_name(SIGSEGV) == "SIGSEGV" && runtime_safe_key(" task one ") == gen_sha1("task one") && runtime_safe_key("  ") == "" && capture_backtrace_string() == "", signal_name(SIGSEGV) + " / " + runtime_safe_key(" task one "));
 
 	site_tests_summary(passed, failed, skipped, "This page intentionally limits writes to /tmp/uce-site-tests-io.txt so it stays disposable.");
 	site_tests_page_end();

site/tests/manifest.txt

@@ -12,6 +12,7 @@ sqlite.uce|SQLite|SQLite connector with prepared named parameters and DValue row
 zip.uce|ZIP|Archive helpers that create and extract temporary server-side files.|http suite uce internal|ZIP|1|1
 services.uce|Sockets And Services|Network/service helpers that are restricted outside trusted networks.|http suite uce internal|Sockets And Services|1|1
 tasks.uce|Tasks|Background task helper coverage.|http suite uce internal|Tasks|1|1
+api_coverage.uce|API Coverage Manifest|Repository-level guard that public wasm APIs have tests/docs or an explicit internal decision.|http suite uce internal|API Coverage Manifest|1|1
 security_headers.uce|Security Header Sanitizer|Low-level response header sanitizer fixture covered by the security smoke suite.|security http internal fixture|security header sanitizer test|0|1
 cli.uce|CLI Fixture|Local CLI socket fixture; HTTP render intentionally returns 404.|cli internal fixture|uce-unit-cli|0|0
 call_helpers.uce|Unit Call Fixture|Fixture for unit_call and unit_render helper tests.|internal fixture|UNIT_RENDER_FIXTURE|0|0

site/tests/units.uce

@@ -30,6 +30,7 @@ RENDER(Request& context)
 
 	check("units_list()", unit_paths.size() > 0, "count=" + std::to_string(unit_paths.size()));
 	check("unit_info()", info["path"].to_string() != "", json_encode(info));
+	check("unit_compile()", unit_compile("call_helpers.uce"), "call_helpers.uce");
 	check("unit_call()", call_output.find("UNIT_CALL_EXPORT_OK") != String::npos, call_output);
 	check("unit_render()", render_output.find("data-unit-render=\"ok\"") != String::npos, render_output);
 

src/3rdparty/miniz/README-UCE.md

@@ -2,7 +2,7 @@
 
 UCE vendors miniz 3.0.2 from <https://github.com/richgel999/miniz> for ZIP archive support.
 
-Files are kept under `vendor/miniz/` and are included by `src/lib/zip.cpp` so UCE can expose `zip_create()`, `zip_list()`, `zip_read()`, `zip_extract()`, `gz_compress()`, and `gz_uncompress()` without adding a runtime package dependency.
+Files are kept under `src/3rdparty/miniz/` and are included by `src/lib/zip.cpp` so UCE can expose `zip_create()`, `zip_list()`, `zip_read()`, `zip_extract()`, `gz_compress()`, and `gz_uncompress()` without adding a runtime package dependency.
 
 Local patch:
 

src/lib/compiler.h

@@ -28,7 +28,9 @@ DValue unit_info(String path = "");
 StringList units_list();
 bool unit_compile(String path = "");
 
+#ifndef __UCE_WASM_UNIT__
 SharedUnit* unit_load(String file_name);
+#endif
 void unit_render(String file_name);
 void unit_render(String file_name, Request& context);
 DValue* unit_call(String file_name, String function_name, DValue* call_param = 0);

src/lib/sys.cpp

@@ -2,6 +2,7 @@
 #include <cmath>
 #include "types.h"
 #include "functionlib.h"
+#include "hash.h"
 #include "uri.h"
 #include "sys.h"
 
@@ -34,6 +35,18 @@ int uce_host_server_stop(const char* key, size_t key_len);
 size_t uce_host_memcache_command(uint64_t sockfd, const char* command, size_t command_len, char* buf, size_t cap);
 size_t uce_host_mysql(const char* in, size_t in_len, char* out, size_t cap);
 size_t uce_host_request_perf(const char* in, size_t in_len, char* out, size_t cap);
+size_t uce_host_shell_exec(const char* cmd, size_t cmd_len, char* buf, size_t cap);
+int uce_host_file_open_locked(const char* path, size_t path_len, int open_flags, int lock_type, int create_mode, double wait_timeout_seconds, const char* purpose, size_t purpose_len);
+void uce_host_file_close_locked(int handle);
+void uce_host_file_release_process_locks(const char* reason, size_t reason_len);
+size_t uce_host_file_read_locked_fd(int handle, char* buf, size_t cap);
+int uce_host_file_write_locked_fd(int handle, const char* content, size_t content_len);
+size_t uce_host_path_real(const char* path, size_t path_len, char* buf, size_t cap);
+int uce_host_path_is_within(const char* path, size_t path_len, const char* root, size_t root_len);
+size_t uce_host_cwd_get(char* buf, size_t cap);
+int uce_host_cwd_set(const char* path, size_t path_len);
+size_t uce_host_process_start_directory(char* buf, size_t cap);
+size_t uce_host_last_trap_trace(char* buf, size_t cap);
 }
 
 static String wasm_current_unit_file()
@@ -41,13 +54,45 @@ static String wasm_current_unit_file()
 	return(context ? context->resources.current_unit_file : String(""));
 }
 
-String shell_exec(String cmd) { (void)cmd; return(""); }
-String shell_escape(String raw) { return(raw); }
+String shell_exec(String cmd)
+{
+	size_t required = uce_host_shell_exec(cmd.data(), cmd.size(), 0, 0);
+	if(required == 0)
+		return("");
+	String output(required, 0);
+	size_t got = uce_host_shell_exec(cmd.data(), cmd.size(), &output[0], required);
+	output.resize(got <= required ? got : 0);
+	return(output);
+}
+String shell_escape(String raw)
+{
+	String result;
+	for(auto c : raw)
+	{
+		if(c == '\'')
+			result.append("'\\''");
+		else
+			result.append(1, c);
+	}
+	return("'" + result + "'");
+}
 String basename(String fn) { while(fn.find("/") != String::npos) fn = fn.substr(fn.find("/") + 1); return(fn); }
 String dirname(String fn) { auto pos = fn.find_last_of('/'); return(pos == String::npos ? "" : fn.substr(0, pos)); }
 String path_join(String base, String child) { if(base == "") return(child); if(child == "") return(base); if(child[0] == '/') return(child); return(base + (base.back() == '/' ? "" : "/") + child); }
-String path_real(String path) { return(path); }
-bool path_is_within(String path, String root) { return(str_starts_with(path, root)); }
+String path_real(String path)
+{
+	size_t required = uce_host_path_real(path.data(), path.size(), 0, 0);
+	if(required == 0)
+		return("");
+	String resolved(required, 0);
+	size_t got = uce_host_path_real(path.data(), path.size(), &resolved[0], required);
+	resolved.resize(got <= required ? got : 0);
+	return(resolved);
+}
+bool path_is_within(String path, String root)
+{
+	return(uce_host_path_is_within(path.data(), path.size(), root.data(), root.size()) != 0);
+}
 bool mkdir(String path)
 {
 	String current = wasm_current_unit_file();
@@ -58,11 +103,29 @@ bool file_exists(String path)
 	String current = wasm_current_unit_file();
 	return(uce_host_file_exists(path.data(), path.size(), current.data(), current.size()) != 0);
 }
-int file_open_locked(String file_name, int open_flags, int lock_type, int create_mode, f64 wait_timeout_seconds, String purpose) { (void)file_name; (void)open_flags; (void)lock_type; (void)create_mode; (void)wait_timeout_seconds; (void)purpose; return(-1); }
-void file_close_locked(int fd) { (void)fd; }
-void file_release_process_locks(String reason) { (void)reason; }
-String file_get_contents_locked_fd(int fd) { (void)fd; return(""); }
-bool file_put_contents_locked_fd(int fd, String content) { (void)fd; (void)content; return(false); }
+int file_open_locked(String file_name, int open_flags, int lock_type, int create_mode, f64 wait_timeout_seconds, String purpose)
+{
+	return(uce_host_file_open_locked(
+		file_name.data(), file_name.size(), open_flags, lock_type, create_mode, wait_timeout_seconds,
+		purpose.data(), purpose.size()
+	));
+}
+void file_close_locked(int fd) { uce_host_file_close_locked(fd); }
+void file_release_process_locks(String reason) { uce_host_file_release_process_locks(reason.data(), reason.size()); }
+String file_get_contents_locked_fd(int fd)
+{
+	size_t required = uce_host_file_read_locked_fd(fd, 0, 0);
+	if(required == 0)
+		return("");
+	String content(required, 0);
+	size_t got = uce_host_file_read_locked_fd(fd, &content[0], required);
+	content.resize(got <= required ? got : 0);
+	return(content);
+}
+bool file_put_contents_locked_fd(int fd, String content)
+{
+	return(uce_host_file_write_locked_fd(fd, content.data(), content.size()) != 0);
+}
 String file_get_contents(String file_name)
 {
 	String current = wasm_current_unit_file();
@@ -84,9 +147,27 @@ bool file_append_contents(String file_name, String content)
 	String current = wasm_current_unit_file();
 	return(uce_host_file_write(file_name.data(), file_name.size(), current.data(), current.size(), content.data(), content.size(), 1) != 0);
 }
-String cwd_get() { return("/"); }
-void cwd_set(String path) { (void)path; }
-String process_start_directory() { return("/"); }
+String cwd_get()
+{
+	size_t required = uce_host_cwd_get(0, 0);
+	if(required == 0)
+		return("");
+	String cwd(required, 0);
+	size_t got = uce_host_cwd_get(&cwd[0], required);
+	cwd.resize(got <= required ? got : 0);
+	return(cwd);
+}
+void cwd_set(String path) { uce_host_cwd_set(path.data(), path.size()); }
+String process_start_directory()
+{
+	size_t required = uce_host_process_start_directory(0, 0);
+	if(required == 0)
+		return("");
+	String cwd(required, 0);
+	size_t got = uce_host_process_start_directory(&cwd[0], required);
+	cwd.resize(got <= required ? got : 0);
+	return(cwd);
+}
 time_t file_mtime(String file_name)
 {
 	String current = wasm_current_unit_file();
@@ -253,11 +334,51 @@ bool ws_close(String connection_id)
 	context->resources.websocket_dispatch_commands.push(command);
 	return(true);
 }
-String backtrace_frames_string(void* const* frames, size_t size, u32 skip_frames) { (void)frames; (void)size; (void)skip_frames; return(""); }
-String capture_backtrace_string(u32 max_frames, u32 skip_frames) { (void)max_frames; (void)skip_frames; return(""); }
-String signal_name(int sig) { (void)sig; return(""); }
-String memcache_escape_key(String key) { return(key); }
-StringList memcache_escape_keys(StringList keys) { return(keys); }
+String backtrace_frames_string(void* const* frames, size_t size, u32 skip_frames) { (void)frames; (void)size; (void)skip_frames; return(capture_backtrace_string(0, 0)); }
+String capture_backtrace_string(u32 max_frames, u32 skip_frames)
+{
+	(void)max_frames;
+	(void)skip_frames;
+	size_t required = uce_host_last_trap_trace(0, 0);
+	if(required == 0)
+		return("");
+	String trace(required, 0);
+	size_t got = uce_host_last_trap_trace(&trace[0], required);
+	trace.resize(got <= required ? got : 0);
+	return(trace);
+}
+String signal_name(int sig)
+{
+	switch(sig)
+	{
+		case 6: return("SIGABRT");
+		case 7: return("SIGBUS");
+		case 8: return("SIGFPE");
+		case 4: return("SIGILL");
+		case 2: return("SIGINT");
+		case 11: return("SIGSEGV");
+		case 15: return("SIGTERM");
+		default: return("");
+	}
+}
+String memcache_escape_key(String key)
+{
+	String result;
+	for(auto c : key)
+	{
+		if(isspace((unsigned char)c))
+			c = '_';
+		result.append(1, c);
+	}
+	return(result);
+}
+StringList memcache_escape_keys(StringList keys)
+{
+	StringList result;
+	for(auto s : keys)
+		result.push_back(memcache_escape_key(s));
+	return(result);
+}
 u64 memcache_connect(String host, short port)
 {
 	if(host == "")
@@ -345,7 +466,14 @@ extern "C" int uce_wasm_task_run(uint64_t callback_id)
 }
 
 int task_kill(pid_t pid, int sig) { return(uce_host_task_kill(pid, sig)); }
-String runtime_safe_key(String key, String label) { (void)label; return(key); }
+String runtime_safe_key(String key, String label)
+{
+	(void)label;
+	key = trim(key);
+	if(key == "")
+		return("");
+	return(gen_sha1(key));
+}
 pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout)
 {
 	u64 id = wasm_next_task_callback_id++;

src/lib/zip.cpp

@@ -3,10 +3,10 @@
 #include <cstring>
 
 extern "C" {
-#include "../../vendor/miniz/miniz.c"
-#include "../../vendor/miniz/miniz_tdef.c"
-#include "../../vendor/miniz/miniz_tinfl.c"
-#include "../../vendor/miniz/miniz_zip.c"
+#include "../3rdparty/miniz/miniz.c"
+#include "../3rdparty/miniz/miniz_tdef.c"
+#include "../3rdparty/miniz/miniz_tinfl.c"
+#include "../3rdparty/miniz/miniz_zip.c"
 }
 
 String zip_error(String api, String detail)

src/wasm/core_hostcalls.syms

@@ -19,6 +19,7 @@ uce_host_zip
 uce_host_units
 uce_host_component_resolve
 uce_host_request_perf
+uce_host_shell_exec
 uce_host_file_exists
 uce_host_file_read
 uce_host_file_write
@@ -26,5 +27,16 @@ uce_host_file_unlink
 uce_host_file_list
 uce_host_file_mkdir
 uce_host_file_mtime
+uce_host_file_open_locked
+uce_host_file_close_locked
+uce_host_file_release_process_locks
+uce_host_file_read_locked_fd
+uce_host_file_write_locked_fd
+uce_host_path_real
+uce_host_path_is_within
+uce_host_cwd_get
+uce_host_cwd_set
+uce_host_process_start_directory
+uce_host_last_trap_trace
 uce_host_regex
 uce_host_sqlite

src/wasm/worker.cpp

@@ -40,6 +40,9 @@
 #include <netinet/in.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
+#include <sys/file.h>
+#include <fcntl.h>
+#include <limits.h>
 #include <unistd.h>
 #include <sys/time.h>
 #include <vector>
@@ -428,21 +431,32 @@ public:
 		request_perf.active = true;
 	}
 
+// Host-owned opaque fd handles opened by wasm file_open_locked().
+	std::vector<int> locked_file_handles;
+
 #ifdef UCE_WASM_HOST_CONNECTORS
 	// Host-owned resource handle table (§3.1): connections opened by the guest
 	// live here and are closed when the workspace drops at request end.
 	std::vector<SQLite*> sqlite_handles;
 	std::vector<MySQL*> mysql_handles;
+#endif
 	~WasmWorkspace()
 	{
+		for(int fd : locked_file_handles)
+			if(fd >= 0)
+			{
+				flock(fd, LOCK_UN);
+				::close(fd);
+			}
+#ifdef UCE_WASM_HOST_CONNECTORS
 		for(auto* db : sqlite_handles)
 			if(db)
 				delete db;   // ~SQLite disconnects
 		for(auto* db : mysql_handles)
 			if(db)
 				delete db;   // ~MySQL disconnects
-	}
 #endif
+	}
 
 	// The guest calls a sized hostcall twice (buf=0 to learn the length, then
 	// to fetch). For side-effecting ops (sqlite) re-executing on the fetch is
@@ -1304,6 +1318,77 @@ private:
 				fprintf(stderr, "[guest log %d] %.*s\n", args[0].i32(), (int)text.size(), text.data());
 				return(std::monostate());
 			}));
+		if(mod == "env" && name == "uce_host_shell_exec")
+			return(add([self](Caller caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
+				String cmd;
+				self->hostcall_read(args[0].i32(), args[1].i32(), cmd);
+				String out;
+				if(!self->hostcall_staged("shell:" + cmd, out))
+				{
+					out = ::shell_exec(cmd);
+					self->hostcall_stage("shell:" + cmd, out);
+				}
+				u32 cap = (u32)args[3].i32();
+				int32_t buf = args[2].i32();
+				if(buf != 0 && cap >= out.size())
+					self->hostcall_write(buf, out);
+				caller.context().set_epoch_deadline(self->worker.cfg.epoch_deadline_ticks);
+				results[0] = Val((int32_t)out.size());
+				return(std::monostate());
+			}));
+		if(mod == "env" && name == "uce_host_path_real")
+			return(add([self](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
+				String path;
+				self->hostcall_read(args[0].i32(), args[1].i32(), path);
+				String resolved = ::path_real(path);
+				u32 cap = (u32)args[3].i32();
+				int32_t buf = args[2].i32();
+				if(buf != 0 && cap >= resolved.size())
+					self->hostcall_write(buf, resolved);
+				results[0] = Val((int32_t)resolved.size());
+				return(std::monostate());
+			}));
+		if(mod == "env" && name == "uce_host_path_is_within")
+			return(add([self](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
+				String path, root;
+				self->hostcall_read(args[0].i32(), args[1].i32(), path);
+				self->hostcall_read(args[2].i32(), args[3].i32(), root);
+				results[0] = Val(::path_is_within(path, root) ? (int32_t)1 : (int32_t)0);
+				return(std::monostate());
+			}));
+		if(mod == "env" && name == "uce_host_cwd_get")
+			return(add([self](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
+				String cwd = ::cwd_get();
+				u32 cap = (u32)args[1].i32();
+				int32_t buf = args[0].i32();
+				if(buf != 0 && cap >= cwd.size())
+					self->hostcall_write(buf, cwd);
+				results[0] = Val((int32_t)cwd.size());
+				return(std::monostate());
+			}));
+		if(mod == "env" && name == "uce_host_cwd_set")
+			return(add([self](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
+				String path;
+				self->hostcall_read(args[0].i32(), args[1].i32(), path);
+				results[0] = Val(::chdir(path.c_str()) == 0 ? (int32_t)1 : (int32_t)0);
+				return(std::monostate());
+			}));
+		if(mod == "env" && name == "uce_host_process_start_directory")
+			return(add([self](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
+				String cwd = ::process_start_directory();
+				u32 cap = (u32)args[1].i32();
+				int32_t buf = args[0].i32();
+				if(buf != 0 && cap >= cwd.size())
+					self->hostcall_write(buf, cwd);
+				results[0] = Val((int32_t)cwd.size());
+				return(std::monostate());
+			}));
+		if(mod == "env" && name == "uce_host_last_trap_trace")
+			return(add([](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
+				(void)args;
+				results[0] = Val((int32_t)0);
+				return(std::monostate());
+			}));
 		if(mod == "env" && name == "uce_host_file_exists")
 			return(add([self](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
 				String path, current;
@@ -1340,6 +1425,87 @@ private:
 				results[0] = Val((int64_t)mtime);
 				return(std::monostate());
 			}));
+		if(mod == "env" && name == "uce_host_file_open_locked")
+			return(add([self](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
+				String path, purpose;
+				self->hostcall_read(args[0].i32(), args[1].i32(), path);
+				self->hostcall_read(args[6].i32(), args[7].i32(), purpose);
+				int fd = ::file_open_locked(path, args[2].i32(), args[3].i32(), args[4].i32(), args[5].f64(), purpose);
+				int handle = -1;
+				if(fd >= 0)
+				{
+					for(size_t i = 0; i < self->locked_file_handles.size(); i++)
+						if(self->locked_file_handles[i] < 0)
+						{
+							self->locked_file_handles[i] = fd;
+							handle = (int)i + 1;
+							break;
+						}
+					if(handle < 0)
+					{
+						self->locked_file_handles.push_back(fd);
+						handle = (int)self->locked_file_handles.size();
+					}
+				}
+				results[0] = Val((int32_t)handle);
+				return(std::monostate());
+			}));
+		if(mod == "env" && name == "uce_host_file_close_locked")
+			return(add([self](Caller, Span<const Val> args, Span<Val>) -> Result<std::monostate, Trap> {
+				int handle = args[0].i32();
+				if(handle >= 1 && (size_t)handle <= self->locked_file_handles.size())
+				{
+					int& fd = self->locked_file_handles[(size_t)handle - 1];
+					if(fd >= 0)
+					{
+						::file_close_locked(fd);
+						fd = -1;
+					}
+				}
+				return(std::monostate());
+			}));
+		if(mod == "env" && name == "uce_host_file_release_process_locks")
+			return(add([self](Caller, Span<const Val>, Span<Val>) -> Result<std::monostate, Trap> {
+				for(int& fd : self->locked_file_handles)
+					if(fd >= 0)
+					{
+						::file_close_locked(fd);
+						fd = -1;
+					}
+				return(std::monostate());
+			}));
+		if(mod == "env" && name == "uce_host_file_read_locked_fd")
+			return(add([self](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
+				int handle = args[0].i32();
+				String content;
+				if(handle >= 1 && (size_t)handle <= self->locked_file_handles.size())
+				{
+					int fd = self->locked_file_handles[(size_t)handle - 1];
+					if(fd >= 0)
+						content = ::file_get_contents_locked_fd(fd);
+				}
+				u32 cap = (u32)args[2].i32();
+				int32_t buf = args[1].i32();
+				if(buf != 0 && cap >= content.size())
+					self->hostcall_write(buf, content);
+				results[0] = Val((int32_t)content.size());
+				return(std::monostate());
+			}));
+		if(mod == "env" && name == "uce_host_file_write_locked_fd")
+			return(add([self](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
+				int handle = args[0].i32();
+				String content;
+				self->hostcall_read(args[1].i32(), args[2].i32(), content);
+				bool ok = false;
+				if(handle >= 1 && (size_t)handle <= self->locked_file_handles.size())
+				{
+					int fd = self->locked_file_handles[(size_t)handle - 1];
+					if(fd >= 0)
+						ok = ::file_put_contents_locked_fd(fd, content);
+				}
+				results[0] = Val(ok ? (int32_t)1 : (int32_t)0);
+				return(std::monostate());
+			}));
 		if(mod == "env" && name == "uce_host_file_read")
 			return(add([self](Caller, Span<const Val> args, Span<Val> results) -> Result<std::monostate, Trap> {
 				String path, current;

todo.txt

@@ -1,38 +0,0 @@
-Library
-=================================
-- md5 / meow
-- fold StringList into DValue _OR_ make better StringList
-- make session data use DValue instead of StringList
-
-Bugs
-=================================
-- shell_escape()
-
-
-Framework
-=================================
-- Check: File Upload
-- Proxy mode
-- WebSockets
-- Resident code / Cron / Tick?
-- Automated Test Suite
-- Documentation
-
-
-Performance
-=================================
-- Arena Allocator
-- Array implementation
-
-
-Nice to Have
-=================================
-- pseudorandom
-- XML components
-- Optionally store compiled units alongside source
-
-
-Finally
-=================================
-- DO WE DO OUR OWN LANGUAGE OR DO WE KEEP USING C?!?
-