Retire idle persistent MySQL connections

docs/setup.md

@@ -203,6 +203,7 @@ WASM_EPOCH_DEADLINE_TICKS=200
 WASM_EPOCH_PERIOD_MS=50
 WASM_INVOCATION_TIMEOUT_MS=30000
 MYSQL_PERSISTENT_POOL_SIZE=8
+MYSQL_PERSISTENT_POOL_IDLE_TIMEOUT_SECONDS=300
 
 WORKER_COUNT=4
 MAX_MEMORY=16777216
@@ -225,6 +226,7 @@ Important settings:
 - `TMP_UPLOAD_PATH` and `SESSION_PATH` must be writable by the runtime.
 - `SESSION_COOKIE_SECURE=1` adds the `Secure` attribute to UCE-managed session cookies and should be used for HTTPS-only deployments. Leave it `0` only for local/plain-HTTP development.
 - `MYSQL_PERSISTENT_POOL_SIZE` caps credential-keyed connections retained by each Wasm worker. The default `8` is clamped to `64`; set it to `0` to restore request-lifetime connections. Cached sessions are reset before reuse.
+- `MYSQL_PERSISTENT_POOL_IDLE_TIMEOUT_SECONDS` retires a pooled connection at the first request boundary after it has been idle for this interval (default `300`, maximum `86400`). Set it to `0` to retain idle connections until capacity eviction, reset failure, worker exit, or database-side closure. Keep a positive value below the database server's idle timeout to avoid attempting stale sessions after traffic resumes. A completely idle UCE worker has no request boundary and therefore does not proactively close sockets.
 - `HTTP_PORT` is the built-in HTTP/WebSocket listener used for WebSocket upgrade traffic and direct local probes. Bind/firewall it for local access only; nginx/Apache should be the public entry point.
 - `WS_BROKER_OUTBOUND_TIMEOUT_SECONDS` controls how long a forwarded WS message can remain queued in the broker before being dropped (default `30`). Set to `0` to disable the timeout.
 - `WASM_COMPILE_SCRIPT` must point to `scripts/compile_wasm_unit` unless you provide an equivalent compiler. Relative paths are resolved from the runtime root/`COMPILER_SYS_PATH`. That script calls `scripts/check_unit_wasm.py` after linking each unit and uses the pinned WASI SDK on every deployment host.

etc/uce/settings.cfg

@@ -60,6 +60,7 @@ WASM_EPOCH_DEADLINE_TICKS=200
 WASM_EPOCH_PERIOD_MS=50
 WASM_INVOCATION_TIMEOUT_MS=30000
 MYSQL_PERSISTENT_POOL_SIZE=8
+MYSQL_PERSISTENT_POOL_IDLE_TIMEOUT_SECONDS=300
 
 # ENABLE THE BACKGROUND PROACTIVE COMPILER LOOP
 PROACTIVE_COMPILE_ENABLED=1

scripts/run_cli_tests.sh

@@ -79,6 +79,7 @@ if [[ "$action" == "run" ]]; then
 	scripts/test_password_hashing.sh
 	scripts/test_mysql_epoch_refresh.sh
 	scripts/test_mysql_persistent_pool.sh
+	scripts/test_mysql_persistent_pool_idle.sh
 	scripts/test_log_timeliness.sh
 	scripts/test_raw_http_request_log.sh
 	scripts/test_component_resolution_ttl.sh

scripts/test_mysql_persistent_pool_idle.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [[ "${1:-}" != "--inside" ]]; then
+	exec timeout --signal=TERM --kill-after=5s 150s unshare --mount --fork --kill-child=TERM "$0" --inside
+fi
+
+name="mysql-pool-idle-test-$$"
+root="/tmp/$name"
+site="$root/site"
+work="$root/work"
+settings="$root/settings.cfg"
+log="$root/service.log"
+socket="$root/run/cli.sock"
+test_user="uce_pool_idle_$$"
+test_database="uce_pool_idle_$$"
+test_password=$(printf '%s' "$name-$(date +%s%N)" | sha256sum | cut -c1-32)
+server_pid=""
+
+cleanup() {
+	status=$?
+	if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then
+		kill -TERM "$server_pid" 2>/dev/null || true
+		deadline=$((SECONDS + 10))
+		while kill -0 "$server_pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.05; done
+		if kill -0 "$server_pid" 2>/dev/null; then kill -KILL "$server_pid" 2>/dev/null || true; fi
+		wait "$server_pid" 2>/dev/null || true
+	fi
+	mariadb -e "DROP DATABASE IF EXISTS \`$test_database\`; DROP USER IF EXISTS '$test_user'@'127.0.0.1'" >/dev/null 2>&1 || true
+	if (( status != 0 )) && [[ -r "$log" ]]; then cat "$log" >&2; fi
+	rm -rf "$root"
+	return "$status"
+}
+trap cleanup EXIT
+
+mkdir -p "$site" "$work" "$root/run" "$root/session" "$root/upload"
+cp /etc/uce/settings.cfg "$settings"
+cat >>"$settings" <<CFG
+BIN_DIRECTORY=$work
+PRECOMPILE_FILES_IN=$site
+SITE_DIRECTORY=$site
+FCGI_SOCKET_PATH=$root/run/fastcgi.sock
+FCGI_PORT=
+CLI_SOCKET_PATH=$socket
+WS_BROKER_SOCKET_PATH=$root/run/ws.sock
+HTTP_PORT=
+HTTP_DOCUMENT_ROOT=$site
+SESSION_PATH=$root/session
+TMP_UPLOAD_PATH=$root/upload
+WASM_CORE_PATH=$(pwd)/bin/wasm/core.wasm
+WORKER_COUNT=1
+PROACTIVE_COMPILE_ENABLED=0
+MYSQL_PERSISTENT_POOL_SIZE=8
+MYSQL_PERSISTENT_POOL_IDLE_TIMEOUT_SECONDS=2
+CFG
+mount --bind "$settings" /etc/uce/settings.cfg
+
+mariadb -e "DROP DATABASE IF EXISTS \`$test_database\`; DROP USER IF EXISTS '$test_user'@'127.0.0.1'; CREATE DATABASE \`$test_database\`; CREATE USER '$test_user'@'127.0.0.1' IDENTIFIED BY '$test_password'; GRANT ALL ON \`$test_database\`.* TO '$test_user'@'127.0.0.1'"
+
+cat >"$site/database.uce" <<UCE
+CLI(Request& context)
+{
+	MySQL* db = mysql_connect("127.0.0.1", "$test_user", "$test_password", "$test_database");
+	if(!mysql_connected(db)) { print("connect-failed"); return; }
+	String connection_id;
+	mysql_query(db, "SELECT CONNECTION_ID() AS id").each([&](DValue row, String key) { connection_id = row["id"].to_string(); });
+	String source;
+	DValue perf = request_perf();
+	perf["mysql_operations"].each([&](DValue operation, String key) {
+		if(operation["op"].to_string() == "connect") source = operation["source"].to_string();
+	});
+	print(perf["worker_pid"].to_string(), "|", connection_id, "|", source);
+	mysql_disconnect(db);
+}
+UCE
+printf '%s\n' 'CLI(Request& context) { print(request_perf()["worker_pid"].to_string(), "|plain"); }' >"$site/plain.uce"
+
+timeout --signal=TERM --kill-after=5s 120s bin/uce_fastcgi.linux.bin >"$log" 2>&1 &
+server_pid=$!
+deadline=$((SECONDS + 20))
+while [[ ! -S "$socket" ]] && (( SECONDS < deadline )); do sleep 0.05; done
+[[ -S "$socket" ]] || { echo "private UCE CLI socket was not ready" >&2; exit 1; }
+
+request() {
+	timeout --signal=TERM --kill-after=1s 30s scripts/uce-cli --socket "$socket" "$1"
+}
+
+first=$(request /database.uce)
+second=$(request /database.uce)
+IFS='|' read -r first_pid first_id first_source <<<"$first"
+IFS='|' read -r second_pid second_id second_source <<<"$second"
+[[ "$first_source" == "new" && "$second_source" == "worker" ]]
+[[ "$first_pid" == "$second_pid" && "$first_id" == "$second_id" ]]
+
+sleep 3
+plain=$(request /plain.uce)
+[[ "$plain" == "$first_pid|plain" ]]
+deadline=$((SECONDS + 5))
+while (( SECONDS < deadline )); do
+	connections=$(mariadb --batch --skip-column-names -e "SELECT COUNT(*) FROM information_schema.PROCESSLIST WHERE USER='$test_user'")
+	[[ "$connections" == "0" ]] && break
+	sleep 0.05
+done
+[[ "$connections" == "0" ]] || { echo "expired pooled connection remained after a request boundary" >&2; exit 1; }
+
+third=$(request /database.uce)
+IFS='|' read -r third_pid third_id third_source <<<"$third"
+[[ "$third_pid" == "$first_pid" && "$third_source" == "new" && "$third_id" != "$first_id" ]]
+
+echo "Persistent MySQL idle eviction preserved hot reuse and retired the expired connection"

site/doc/pages/mysql_connect.txt

@@ -16,7 +16,7 @@ Establishes a connection to a MySQL server and returns a request-owned pointer t
 
 This connection handle is then used with helpers such as `mysql_query()`, `mysql_error()`, and `mysql_disconnect()`.
 
-MySQL handles are request-scoped framework resources. Repeated `mysql_connect()` calls with the same host, credentials, and database reuse one server connection within the current request. Database identity is part of both request-local and worker-persistent pool keys. Each call creates a lease and `mysql_disconnect()` releases that lease. At request cleanup UCE returns the server connection to the current worker's persistent pool, then resets it before cross-request reuse. The pool holds up to `MYSQL_PERSISTENT_POOL_SIZE` connections per worker (default 8); set the size to 0 to disable cross-request reuse. Never store a `MySQL*` in globals, sessions, or other state that can outlive the current request.
+MySQL handles are request-scoped framework resources. Repeated `mysql_connect()` calls with the same host, credentials, and database reuse one server connection within the current request. Database identity is part of both request-local and worker-persistent pool keys. Each call creates a lease and `mysql_disconnect()` releases that lease. At request cleanup UCE returns the server connection to the current worker's persistent pool, then resets it before cross-request reuse. The pool holds up to `MYSQL_PERSISTENT_POOL_SIZE` connections per worker (default 8); set the size to 0 to disable cross-request reuse. `MYSQL_PERSISTENT_POOL_IDLE_TIMEOUT_SECONDS` retires idle entries at the first later request boundary (default 300, 0 keeps unlimited idle retention). A completely idle worker has no boundary and does not proactively close sockets. Never store a `MySQL*` in globals, sessions, or other state that can outlive the current request.
 
 :example
 MySQL* db = mysql_connect();

site/doc/pages/request_perf.txt

@@ -27,7 +27,7 @@ The request log's `wasm-ready`, `wasm`, `workspace`, `invoke`, `collect`, and `p
 
 Workspace birth is subdivided into `birth_policy_us`, `birth_import_us`, `birth_instantiate_us`, `birth_exports_us`, and `birth_initialize_us`. Request-context transfer reports `context_bytes`, `context_encode_us`, `context_allocate_us`, `context_write_us`, `context_guest_apply_us`, and `context_free_us`. These bounded aggregate fields expose sizes and timing only, never request values.
 
-Wasm FastCGI workers retain up to `MYSQL_PERSISTENT_POOL_SIZE` credential-keyed MySQL connections (default `8`; set `0` to disable). UCE calls the client library's connection-reset operation before another request receives a cached connection, clearing transactions, temporary tables, session variables, and selected databases while avoiding a new authentication handshake. Same-request leases continue to share state until request cleanup.
+Wasm FastCGI workers retain up to `MYSQL_PERSISTENT_POOL_SIZE` credential-keyed MySQL connections (default `8`; set `0` to disable). `MYSQL_PERSISTENT_POOL_IDLE_TIMEOUT_SECONDS` retires an idle entry at the first later request boundary (default `300`; `0` disables age eviction). UCE calls the client library's connection-reset operation before another request receives a cached connection, clearing transactions, temporary tables, session variables, and selected databases while avoiding a new authentication handshake. Same-request leases continue to share state until request cleanup.
 
 :example
 DValue perf = request_perf();

src/lib/sys.cpp

@@ -1904,6 +1904,8 @@ StringMap make_server_settings()
 	cfg["WASM_MEMORY_LIMIT_BYTES"] = std::to_string(512ull * 1024 * 1024);
 	cfg["WASM_EPOCH_DEADLINE_TICKS"] = "200";
 	cfg["WASM_EPOCH_PERIOD_MS"] = "50";
+	cfg["MYSQL_PERSISTENT_POOL_SIZE"] = "8";
+	cfg["MYSQL_PERSISTENT_POOL_IDLE_TIMEOUT_SECONDS"] = "300";
 	cfg["SETUP_TEMPLATE"] = "scripts/setup.h.template";
 	cfg["LIT_ESC"] = "3d5b5_1";
 	cfg["CONTENT_TYPE"] = "text/html; charset=utf-8";

src/wasm/backend.cpp

@@ -121,6 +121,13 @@ static String wasm_backend_ensure_started(Request* context)
 		return(g_wasm_init_error);
 	}
 	wc.mysql_persistent_pool_size = std::min<u64>(to_u64(cfg["MYSQL_PERSISTENT_POOL_SIZE"], 8), 64);
+	u64 mysql_idle_timeout = to_u64(first(cfg["MYSQL_PERSISTENT_POOL_IDLE_TIMEOUT_SECONDS"], "300"), UINT64_MAX);
+	if(mysql_idle_timeout == UINT64_MAX || mysql_idle_timeout > 86400)
+	{
+		g_wasm_init_error = "MYSQL_PERSISTENT_POOL_IDLE_TIMEOUT_SECONDS must be an integer no greater than 86400";
+		return(g_wasm_init_error);
+	}
+	wc.mysql_persistent_pool_idle_timeout_seconds = mysql_idle_timeout;
 	wc.profile_hostcall_cpu = to_bool(cfg["WASM_PROFILE_HOSTCALL_CPU"], false);
 	wc.profile_thread_runtime = to_bool(cfg["WASM_PROFILE_THREAD_RUNTIME"], false);
 	wc.verbose = to_bool(cfg["WASM_BACKEND_VERBOSE"], false);

src/wasm/worker.cpp

@@ -153,6 +153,7 @@ struct WasmWorkerConfig
 	u64 epoch_period_ms = 50;
 	u64 invocation_timeout_ms = 30000;
 	u64 mysql_persistent_pool_size = 8;
+	u64 mysql_persistent_pool_idle_timeout_seconds = 300;
 	bool profile_hostcall_cpu = false;
 	bool profile_thread_runtime = false;
 	bool verbose = false;
@@ -1271,6 +1272,14 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
 
 class WasmWorkspace;
 
+#ifdef UCE_WASM_HOST_CONNECTORS
+struct WasmMySQLPersistentConnection
+{
+	MySQL* db = 0;
+	std::chrono::steady_clock::time_point idle_since;
+};
+#endif
+
 class WasmWorker
 {
 public:
@@ -1283,25 +1292,59 @@ public:
 #ifdef UCE_WASM_HOST_CONNECTORS
 	~WasmWorker()
 	{
-		for(auto* db : mysql_persistent_pool)
+		for(auto& entry : mysql_persistent_pool)
-			delete db;
+			delete entry.db;
+	}
+
+	void mysql_evict_idle(std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now())
+	{
+		if(cfg.mysql_persistent_pool_idle_timeout_seconds == 0)
+			return;
+		auto timeout = std::chrono::seconds(cfg.mysql_persistent_pool_idle_timeout_seconds);
+		for(size_t i = 0; i < mysql_persistent_pool.size();)
+		{
+			auto& entry = mysql_persistent_pool[i];
+			if(entry.idle_since != std::chrono::steady_clock::time_point() && now - entry.idle_since >= timeout)
+			{
+				delete entry.db;
+				mysql_persistent_pool.erase(mysql_persistent_pool.begin() + i);
+				continue;
+			}
+			i++;
+		}
+	}
+
+	void mysql_release(MySQL* db)
+	{
+		if(!db)
+			return;
+		db->request_leases = 0;
+		for(auto& entry : mysql_persistent_pool)
+			if(entry.db == db)
+			{
+				entry.idle_since = std::chrono::steady_clock::now();
+				return;
+			}
 	}
 
 	MySQL* mysql_checkout(const String& host, const String& username, const String& password, const String& database, bool& reused, bool& persistent)
 	{
+		mysql_evict_idle();
 		reused = false;
 		persistent = false;
 		for(size_t i = 0; i < mysql_persistent_pool.size(); i++)
 		{
-			MySQL* db = mysql_persistent_pool[i];
+			MySQL* db = mysql_persistent_pool[i].db;
 			if(!db || !db->connection || db->request_host != host || db->request_username != username || db->request_password != password || db->request_database != database)
 				continue;
 			if(db->reset_connection())
 			{
+				mysql_persistent_pool[i].idle_since = std::chrono::steady_clock::time_point();
 				if(i + 1 < mysql_persistent_pool.size())
 				{
+					auto entry = mysql_persistent_pool[i];
 					mysql_persistent_pool.erase(mysql_persistent_pool.begin() + i);
-					mysql_persistent_pool.push_back(db);
+					mysql_persistent_pool.push_back(entry);
 				}
 				reused = true;
 				persistent = true;
@@ -1326,15 +1369,17 @@ public:
 		{
 			while(mysql_persistent_pool.size() >= cfg.mysql_persistent_pool_size)
 			{
-				delete mysql_persistent_pool.front();
+				delete mysql_persistent_pool.front().db;
 				mysql_persistent_pool.erase(mysql_persistent_pool.begin());
 			}
-			mysql_persistent_pool.push_back(db);
+			WasmMySQLPersistentConnection entry;
+			entry.db = db;
+			mysql_persistent_pool.push_back(entry);
 		}
 		return(db);
 	}
 
-	std::vector<MySQL*> mysql_persistent_pool;
+	std::vector<WasmMySQLPersistentConnection> mysql_persistent_pool;
 #endif
 
 	String init()
@@ -1931,8 +1976,7 @@ public:
 			if(db)
 				delete db;   // ~SQLite disconnects
 		for(auto* db : mysql_request_pool)
-			if(db)
+			worker.mysql_release(db);
-				db->request_leases = 0;
 		for(auto* db : mysql_request_owned)
 			if(db)
 				delete db;   // ~MySQL disconnects
@@ -4597,6 +4641,9 @@ inline WasmResponse wasm_worker_serve(WasmWorker& worker, const Request& request
 	WasmResponse response;
 	f64 serve_started = time_precise();
 	auto workspace_start = std::chrono::steady_clock::now();
+#ifdef UCE_WASM_HOST_CONNECTORS
+	worker.mysql_evict_idle(workspace_start);
+#endif
 	f64 cpu_started = wasm_thread_cpu_time();
 	struct rusage thread_runtime_start = {};
 	bool thread_runtime_profiled = worker.cfg.profile_thread_runtime && getrusage(RUSAGE_THREAD, &thread_runtime_start) == 0;