diff --git a/README.md b/README.md index d231bc8..b61c994 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Useful helpers for that data model include: - `path_join(base, child)` for filesystem-style path assembly - `zip_create()`, `zip_list()`, `zip_read()`, and `zip_extract()` for minimal ZIP archive workflows - `gz_compress()` and `gz_uncompress()` for gzip-format byte strings +- `server_start_http()` / `server_stop()` for runtime-managed custom HTTP listeners backed by `SERVE_HTTP` handlers Named component handlers are also supported: diff --git a/etc/uce/settings.cfg b/etc/uce/settings.cfg index b5438ed..da76ca0 100644 --- a/etc/uce/settings.cfg +++ b/etc/uce/settings.cfg @@ -37,5 +37,31 @@ WORKER_COUNT=4 # MAX MEMORY PER REQUEST MAX_MEMORY=16777216 +# TRANSPORT / DOS LIMITS +TRANSPORT_MAX_CLIENT_CONNECTIONS=256 +TRANSPORT_MAX_HTTP_HEADER_BYTES=16384 +TRANSPORT_MAX_HTTP_BODY_BYTES=1048576 +TRANSPORT_MAX_WEBSOCKET_FRAME_BYTES=1048576 +TRANSPORT_MAX_WEBSOCKET_MESSAGE_BYTES=1048576 +TRANSPORT_MAX_WEBSOCKET_OUTPUT_BYTES=4194304 +TRANSPORT_MAX_RESPONSE_BYTES=8388608 +TRANSPORT_HTTP_REQUEST_TIMEOUT_SECONDS=15 +TRANSPORT_CONNECTION_IDLE_TIMEOUT_SECONDS=120 + +# CUSTOM SERVER LIMITS +CUSTOM_SERVER_MAX_SERVERS=16 +CUSTOM_SERVER_MIN_PORT=1024 +CUSTOM_SERVER_MAX_PORT=65535 +CUSTOM_SERVER_ALLOW_PUBLIC_BIND=0 +CUSTOM_SERVER_UNIX_SOCKET_PREFIX=/tmp/uce/custom-servers/ +CUSTOM_SERVER_HANDLER_TIMEOUT_SECONDS=30 +# Empty means COMPILER_SYS_PATH/SITE_DIRECTORY. +CUSTOM_SERVER_UCE_ROOT= + +# ARCHIVE HELPER LIMITS +ARCHIVE_MAX_INPUT_BYTES=67108864 +ARCHIVE_MAX_OUTPUT_BYTES=67108864 +ARCHIVE_MAX_ZIP_ENTRIES=4096 + # LIFETIME OF SESSION COOKIES IN SECONDS SESSION_TIME=2592000 diff --git a/site/doc/areas/sys.txt b/site/doc/areas/sys.txt index f6b6bd9..1bb2a26 100644 --- a/site/doc/areas/sys.txt +++ b/site/doc/areas/sys.txt @@ -22,3 +22,5 @@ zip_read zip_extract gz_compress gz_uncompress +server_start_http +server_stop diff --git a/site/doc/pages/server_start_http.txt b/site/doc/pages/server_start_http.txt new file mode 100644 index 0000000..36aa2fa --- /dev/null +++ b/site/doc/pages/server_start_http.txt @@ -0,0 +1,38 @@ +:sig +pid_t server_start_http(String key, String socket_fn_or_port, String call_uce_filename, String call_function = "") + +:params +key : runtime-wide server key +socket_fn_or_port : TCP port number or Unix socket path to listen on +call_uce_filename : `.uce` file containing `SERVE_HTTP` handlers +call_function : optional named `SERVE_HTTP:name` handler +return value : listener process PID, or `0` when it could not be started + +:see +>sys +server_stop +task + +:content +Starts or updates a custom HTTP listener backed by a UCE handler file. + +The server key is runtime-wide. Calling `server_start_http()` again with the same key and the same bind target updates the mapped UCE file/function without restarting the listener. If the bind target changes, UCE stops the old listener and starts a replacement. + +`socket_fn_or_port` may be a numeric TCP port such as `"19091"` or a Unix socket path. + +Handler files use `SERVE_HTTP`: + +```uce +SERVE_HTTP(Request* req) +{ + req->header["Content-Type"] = "text/plain; charset=utf-8"; + print("hello from custom HTTP\n"); +} + +SERVE_HTTP:admin(Request* req) +{ + print("named handler\n"); +} +``` + +The custom server dispatcher uses the same nonblocking HTTP listener machinery as the built-in HTTP/WebSocket path, and hands request/response data through the normal `Request` object. Handler code may block in the first implementation; future worker dispatch can be added without changing this API. diff --git a/site/doc/pages/server_stop.txt b/site/doc/pages/server_stop.txt new file mode 100644 index 0000000..50c77d4 --- /dev/null +++ b/site/doc/pages/server_stop.txt @@ -0,0 +1,19 @@ +:sig +bool server_stop(String key) + +:params +key : runtime-wide server key used with `server_start_http()` +return value : `true` when no listener remains or a stop signal was sent + +:see +>sys +server_start_http + +:content +Stops a custom server listener by key and removes its registry config. + +```uce +server_stop("site-tests-http"); +``` + +`server_stop()` currently targets custom servers started through `server_start_http()`. The same key model is intended to extend to future `server_start_tcp()` and `server_start_udp()` helpers. diff --git a/site/tests/servers/http.uce b/site/tests/servers/http.uce new file mode 100644 index 0000000..7cc3dfb --- /dev/null +++ b/site/tests/servers/http.uce @@ -0,0 +1,16 @@ +SERVE_HTTP(Request* req) +{ + req->header["Content-Type"] = "text/plain; charset=utf-8"; + print("custom-http-default\n"); + print("method=", req->params["REQUEST_METHOD"], "\n"); + print("uri=", req->params["REQUEST_URI"], "\n"); + print("body=", req->in, "\n"); +} + +SERVE_HTTP:named(Request* req) +{ + req->header["Content-Type"] = "text/plain; charset=utf-8"; + print("custom-http-named\n"); + print("function=", req->params["UCE_SERVE_HTTP_FUNCTION"], "\n"); + print("query=", req->params["QUERY_STRING"], "\n"); +} diff --git a/site/tests/services.uce b/site/tests/services.uce index ab40f05..94604f7 100644 --- a/site/tests/services.uce +++ b/site/tests/services.uce @@ -25,7 +25,7 @@ RENDER(Request& context) failed++; }; - u64 sockfd = socket_connect("localhost", 80); + u64 sockfd = socket_connect("127.0.0.1", 80); if(sockfd != 0) { bool write_ok = socket_write(sockfd, "GET /tests/index.uce HTTP/1.0\r\nHost: uce.openfu.com\r\n\r\n"); @@ -39,7 +39,28 @@ RENDER(Request& context) } else { - mark("socket_connect() / socket_write() / socket_read()", "fail", "socket_connect(localhost, 80) returned 0"); + mark("socket_connect() / socket_write() / socket_read()", "fail", "socket_connect(127.0.0.1, 80) returned 0"); + } + + pid_t custom_http_pid = server_start_http("site-tests-http", "19091", "servers/http.uce", "named"); + usleep(200000); + u64 custom_http_sockfd = socket_connect("127.0.0.1", 19091); + if(custom_http_sockfd != 0) + { + bool write_ok = socket_write(custom_http_sockfd, "GET /custom-test?alpha=1 HTTP/1.0\r\nHost: localhost\r\n\r\n"); + String response = socket_read(custom_http_sockfd, 4096, 2); + socket_close(custom_http_sockfd); + server_stop("site-tests-http"); + mark( + "server_start_http() / SERVE_HTTP:named", + (custom_http_pid != 0 && write_ok && response.find("custom-http-named") != String::npos && response.find("query=alpha=1") != String::npos) ? "pass" : "fail", + response.substr(0, response.length() > 260 ? 260 : response.length()) + ); + } + else + { + server_stop("site-tests-http"); + mark("server_start_http() / SERVE_HTTP:named", "fail", "socket_connect(127.0.0.1, 19091) returned 0; pid=" + std::to_string(custom_http_pid)); } u64 memfd = memcache_connect(); @@ -60,7 +81,7 @@ RENDER(Request& context) } MySQL mysql; - mysql.connect("localhost", "root", ""); + mysql.connect("127.0.0.1", "root", ""); String mysql_error_text = trim(mysql.error()); if(mysql_error_text != "") { diff --git a/src/fastcgi/src/fcgicc.cc b/src/fastcgi/src/fcgicc.cc index 8407108..8930952 100644 --- a/src/fastcgi/src/fcgicc.cc +++ b/src/fastcgi/src/fcgicc.cc @@ -58,6 +58,7 @@ struct TransportLimits { u64 max_websocket_frame_bytes = 1024 * 1024; u64 max_websocket_message_bytes = 1024 * 1024; u64 max_websocket_output_bytes = 4 * 1024 * 1024; + u64 max_response_bytes = 8 * 1024 * 1024; f64 http_request_timeout_seconds = 15.0; f64 connection_idle_timeout_seconds = 120.0; @@ -72,9 +73,38 @@ struct TransportLimits { } }; -const TransportLimits& transport_limits() +u64 transport_config_u64(String key, u64 fallback) { - static const TransportLimits limits; + if(!context || !context->server) + return(fallback); + String raw = trim(context->server->config[key]); + if(raw == "") + return(fallback); + return(int_val(raw)); +} + +f64 transport_config_f64(String key, f64 fallback) +{ + if(!context || !context->server) + return(fallback); + String raw = trim(context->server->config[key]); + if(raw == "") + return(fallback); + return(float_val(raw)); +} + +TransportLimits transport_limits() +{ + TransportLimits limits; + limits.max_client_connections = transport_config_u64("TRANSPORT_MAX_CLIENT_CONNECTIONS", limits.max_client_connections); + limits.max_http_header_bytes = transport_config_u64("TRANSPORT_MAX_HTTP_HEADER_BYTES", limits.max_http_header_bytes); + limits.max_http_body_bytes = transport_config_u64("TRANSPORT_MAX_HTTP_BODY_BYTES", limits.max_http_body_bytes); + limits.max_websocket_frame_bytes = transport_config_u64("TRANSPORT_MAX_WEBSOCKET_FRAME_BYTES", limits.max_websocket_frame_bytes); + limits.max_websocket_message_bytes = transport_config_u64("TRANSPORT_MAX_WEBSOCKET_MESSAGE_BYTES", limits.max_websocket_message_bytes); + limits.max_websocket_output_bytes = transport_config_u64("TRANSPORT_MAX_WEBSOCKET_OUTPUT_BYTES", limits.max_websocket_output_bytes); + limits.max_response_bytes = transport_config_u64("TRANSPORT_MAX_RESPONSE_BYTES", limits.max_response_bytes); + limits.http_request_timeout_seconds = transport_config_f64("TRANSPORT_HTTP_REQUEST_TIMEOUT_SECONDS", limits.http_request_timeout_seconds); + limits.connection_idle_timeout_seconds = transport_config_f64("TRANSPORT_CONNECTION_IDLE_TIMEOUT_SECONDS", limits.connection_idle_timeout_seconds); return(limits); } @@ -185,6 +215,14 @@ FastCGIServer::listen_http(unsigned tcp_port) return server_socket; } +int +FastCGIServer::listen_http(unsigned tcp_port, const std::string& bind_address) +{ + int server_socket = listen(tcp_port, bind_address); + server_socket_types[server_socket] = 'H'; + return server_socket; +} + int FastCGIServer::listen_cli(const std::string& local_path) { @@ -196,6 +234,12 @@ FastCGIServer::listen_cli(const std::string& local_path) int FastCGIServer::listen(unsigned tcp_port) +{ + return(listen(tcp_port, "0.0.0.0")); +} + +int +FastCGIServer::listen(unsigned tcp_port, const std::string& bind_address) { int server_socket = socket(PF_INET, SOCK_STREAM, 0); if (server_socket == -1) @@ -211,7 +255,10 @@ FastCGIServer::listen(unsigned tcp_port) bzero(&sa, sizeof(sa)); sa.sin_family = AF_INET; sa.sin_port = htons(tcp_port); - sa.sin_addr.s_addr = htonl(INADDR_ANY); + if(bind_address == "" || bind_address == "0.0.0.0" || bind_address == "*") + sa.sin_addr.s_addr = htonl(INADDR_ANY); + else if(inet_pton(AF_INET, bind_address.c_str(), &sa.sin_addr) != 1) + throw std::runtime_error("invalid bind address"); if (bind(server_socket, (struct sockaddr*)&sa, sizeof(sa)) == -1) throw std::runtime_error("bind() failed"); @@ -226,7 +273,7 @@ FastCGIServer::listen(unsigned tcp_port) } server_socket_types[server_socket] = 'F'; - printf("(P) listening to #%i port %i\n", server_socket, tcp_port); + printf("(P) listening to #%i %s:%i\n", server_socket, bind_address.c_str(), tcp_port); return server_socket; } @@ -358,7 +405,7 @@ FastCGIServer::enforce_connection_timeouts(Connection& connection) if(connection.close_socket) return; - const TransportLimits& limits = transport_limits(); + TransportLimits limits = transport_limits(); f64 now = time_precise(); if(is_http_like_type(connection.type) && !connection.is_websocket && !connection.requests.empty()) { @@ -603,7 +650,7 @@ bool FastCGIServer::parse_http_message(FastCGIRequest& request, String& data) { Connection* connection = client_sockets[request.resources.client_socket]; - const TransportLimits& limits = transport_limits(); + TransportLimits limits = transport_limits(); auto header_end = data.find("\r\n\r\n"); if(header_end == String::npos) { @@ -772,7 +819,7 @@ FastCGIServer::process_http_request(FastCGIRequest& request, String& data) void FastCGIServer::process_websocket_input(Connection& connection) { - const TransportLimits& limits = transport_limits(); + TransportLimits limits = transport_limits(); if(connection.input_buffer.length() > limits.max_websocket_buffer_bytes()) { fail_websocket_connection(connection, 1009, "websocket input buffer is too large"); @@ -1121,8 +1168,15 @@ FastCGIServer::read_fgci(Connection& connection) FastCGIRequest& request = *it->second; //switch_to_arena(it->second->mem); if (!request.flags.params_closed) - if (content_length != 0) + if (content_length != 0) { + if(request.resources.params_buffer.size() + content_length > transport_limits().max_http_header_bytes) + { + request.flags.status = 1; + connection.close_socket = true; + break; + } request.resources.params_buffer.append(content, content_length); + } else { request.params = parse_pairs_fcgi( request.resources.params_buffer.data(), @@ -1152,6 +1206,12 @@ FastCGIServer::read_fgci(Connection& connection) //switch_to_arena(it->second->mem); if (!request.flags.input_closed) if (content_length != 0) { + if(request.in.size() + content_length > transport_limits().max_http_body_bytes) + { + request.flags.status = 1; + connection.close_socket = true; + break; + } request.in.append(content, content_length); if (request.flags.params_closed && request.flags.status == 0) { @@ -1203,6 +1263,13 @@ FastCGIServer::assemble_output_buffer(FastCGIRequest& request, Connection* conne request.out += obs->str(); delete obs; } + if(request.out.length() > transport_limits().max_response_bytes) + { + request.set_status(500, "Response Too Large"); + request.header.clear(); + request.header["Content-Type"] = "text/plain; charset=utf-8"; + request.out = request.response_code + "\r\n" + var_dump(request.header, "", "\r\n") + "\r\nresponse exceeded configured output limit\n"; + } request.ob_stack.clear(); request.flags.output_closed = true; request.stats.time_end = time_precise(); diff --git a/src/fastcgi/src/fcgicc.h b/src/fastcgi/src/fcgicc.h index ad2ddc3..172d031 100644 --- a/src/fastcgi/src/fcgicc.h +++ b/src/fastcgi/src/fcgicc.h @@ -52,7 +52,9 @@ public: std::function on_websocket_message = 0; int listen(unsigned tcp_port); + int listen(unsigned tcp_port, const std::string& bind_address); int listen_http(unsigned tcp_port); + int listen_http(unsigned tcp_port, const std::string& bind_address); int listen_cli(const std::string& local_path); int listen(const std::string& local_path); diff --git a/src/lib/compiler-parser.cpp b/src/lib/compiler-parser.cpp index 3de7332..9f508ca 100644 --- a/src/lib/compiler-parser.cpp +++ b/src/lib/compiler-parser.cpp @@ -6,6 +6,7 @@ namespace { const char* UCE_NAMED_RENDER_SYMBOL = "__uce_render"; const char* UCE_NAMED_COMPONENT_SYMBOL = "__uce_component"; +const char* UCE_NAMED_SERVE_HTTP_SYMBOL = "__uce_serve_http"; struct CompilerCodeState { @@ -315,6 +316,31 @@ String compiler_process_text_literal(Request* context, SharedUnit* su, String co return(parsed_content); } +bool compiler_rewrite_named_entrypoint_line(String& line, String macro_prefix, String symbol_prefix) +{ + u32 indent_length = 0; + while(indent_length < line.length() && isspace(line[indent_length])) + indent_length += 1; + + String indent = line.substr(0, indent_length); + String trimmed = trim(line); + if(trimmed.rfind(macro_prefix, 0) != 0) + return(false); + + String signature = trimmed.substr(macro_prefix.length()); + auto open_paren_pos = signature.find("("); + if(open_paren_pos == String::npos) + return(false); + + String render_name = trim(signature.substr(0, open_paren_pos)); + String render_signature = signature.substr(open_paren_pos); + if(render_name == "") + return(false); + + line = indent + "EXPORT void " + symbol_prefix + "_" + safe_name(render_name) + render_signature; + return(true); +} + String compiler_rewrite_named_render_syntax(String content) { String result = ""; @@ -332,36 +358,9 @@ String compiler_rewrite_named_render_syntax(String content) line.pop_back(); } - u32 indent_length = 0; - while(indent_length < line.length() && isspace(line[indent_length])) - indent_length += 1; - - String indent = line.substr(0, indent_length); - String trimmed = trim(line); - if(trimmed.rfind("RENDER:", 0) == 0) - { - String signature = trimmed.substr(7); - auto open_paren_pos = signature.find("("); - if(open_paren_pos != String::npos) - { - String render_name = trim(signature.substr(0, open_paren_pos)); - String render_signature = signature.substr(open_paren_pos); - if(render_name != "") - line = indent + "EXPORT void " + String(UCE_NAMED_RENDER_SYMBOL) + "_" + safe_name(render_name) + render_signature; - } - } - else if(trimmed.rfind("COMPONENT:", 0) == 0) - { - String signature = trimmed.substr(10); - auto open_paren_pos = signature.find("("); - if(open_paren_pos != String::npos) - { - String render_name = trim(signature.substr(0, open_paren_pos)); - String render_signature = signature.substr(open_paren_pos); - if(render_name != "") - line = indent + "EXPORT void " + String(UCE_NAMED_COMPONENT_SYMBOL) + "_" + safe_name(render_name) + render_signature; - } - } + compiler_rewrite_named_entrypoint_line(line, "RENDER:", UCE_NAMED_RENDER_SYMBOL) || + compiler_rewrite_named_entrypoint_line(line, "COMPONENT:", UCE_NAMED_COMPONENT_SYMBOL) || + compiler_rewrite_named_entrypoint_line(line, "SERVE_HTTP:", UCE_NAMED_SERVE_HTTP_SYMBOL); result += line + line_break; current_line = ""; diff --git a/src/lib/compiler.cpp b/src/lib/compiler.cpp index 1c772a1..fb714f6 100644 --- a/src/lib/compiler.cpp +++ b/src/lib/compiler.cpp @@ -14,6 +14,7 @@ const char* UCE_RENDER_SYMBOL = "__uce_render"; const char* UCE_COMPONENT_SYMBOL = "__uce_component"; const char* UCE_WEBSOCKET_SYMBOL = "__uce_websocket"; const char* UCE_CLI_SYMBOL = "__uce_cli"; +const char* UCE_SERVE_HTTP_SYMBOL = "__uce_serve_http"; const char* UCE_ONCE_SYMBOL = "__uce_once"; const char* UCE_INIT_SYMBOL = "__uce_init"; const u64 UCE_UNIT_ABI_VERSION = 1; @@ -1195,7 +1196,8 @@ enum class UnitCallMacroKind component, once, init, - cli + cli, + serve_http }; struct UnitCallMacroTarget @@ -1368,6 +1370,17 @@ UnitCallMacroTarget unit_call_macro_target(String function_name) target.kind = UnitCallMacroKind::cli; return(target); } + if(function_name == "SERVE_HTTP") + { + target.kind = UnitCallMacroKind::serve_http; + return(target); + } + if(function_name.rfind("SERVE_HTTP:", 0) == 0) + { + target.kind = UnitCallMacroKind::serve_http; + target.handler_name = trim(function_name.substr(11)); + return(target); + } return(target); } @@ -1458,6 +1471,14 @@ String component_handler_symbol(String render_name) return(String(UCE_COMPONENT_SYMBOL) + "_" + safe_name(render_name)); } +String serve_http_handler_symbol(String handler_name) +{ + handler_name = trim(handler_name); + if(handler_name == "") + return(UCE_SERVE_HTTP_SYMBOL); + return(String(UCE_SERVE_HTTP_SYMBOL) + "_" + safe_name(handler_name)); +} + String compiler_missing_request_handler_message(UnitCallMacroKind kind, String handler_name) { handler_name = trim(handler_name); @@ -1477,6 +1498,12 @@ String compiler_missing_request_handler_message(UnitCallMacroKind kind, String h return("no ONCE() entry point"); if(kind == UnitCallMacroKind::cli) return("no CLI() entry point"); + if(kind == UnitCallMacroKind::serve_http) + { + if(handler_name == "") + return("no SERVE_HTTP() entry point"); + return("no SERVE_HTTP:" + handler_name + "() entry point"); + } if(kind == UnitCallMacroKind::init) return("no INIT() entry point"); return("request handler not found"); @@ -1524,11 +1551,10 @@ void compiler_execute_request_handler( } } -request_ref_handler get_page_render_handler(SharedUnit* su, String render_name) +request_ref_handler get_request_handler_symbol(SharedUnit* su, String symbol, request_ref_handler default_handler = 0) { - String symbol = page_render_handler_symbol(render_name); - if(symbol == UCE_RENDER_SYMBOL) - return(su->on_render); + if(default_handler && symbol == "") + return(default_handler); auto it = su->api_functions.find(symbol); if(it != su->api_functions.end()) @@ -1540,20 +1566,25 @@ request_ref_handler get_page_render_handler(SharedUnit* su, String render_name) return(handler); } +request_ref_handler get_page_render_handler(SharedUnit* su, String render_name) +{ + String symbol = page_render_handler_symbol(render_name); + if(symbol == UCE_RENDER_SYMBOL) + return(su->on_render); + return(get_request_handler_symbol(su, symbol)); +} + request_ref_handler get_component_handler(SharedUnit* su, String render_name) { String symbol = component_handler_symbol(render_name); if(symbol == UCE_COMPONENT_SYMBOL) return(su->on_component); + return(get_request_handler_symbol(su, symbol)); +} - auto it = su->api_functions.find(symbol); - if(it != su->api_functions.end()) - return((request_ref_handler)it->second); - - auto handler = (request_ref_handler)dlsym(su->so_handle, symbol.c_str()); - dlerror(); - su->api_functions[symbol] = (void*)handler; - return(handler); +request_ref_handler get_serve_http_handler(SharedUnit* su, String handler_name) +{ + return(get_request_handler_symbol(su, serve_http_handler_symbol(handler_name))); } bool compiler_invoke_loaded_request_handler( @@ -1662,6 +1693,30 @@ void compiler_invoke_cli(Request* context, String file_name) } } +void compiler_invoke_serve_http(Request* context, String file_name, String handler_name) +{ + auto su = compiler_load_shared_unit(context, file_name, "", false); + if(!su) + return; + + String error_message = ""; + if(!compiler_invoke_loaded_request_handler( + context, + su, + get_serve_http_handler(su, handler_name), + UnitCallMacroKind::serve_http, + handler_name, + true, + "uncaught exception during HTTP server handler", + &error_message + )) + { + context->set_status(500, "HTTP Server Handler Error"); + if(error_message != "") + print(error_message); + } +} + void compiler_invoke_websocket(Request* context, String file_name) { auto su = compiler_load_shared_unit(context, file_name, "", false); diff --git a/src/lib/compiler.h b/src/lib/compiler.h index 1eb4b3e..f7b3f94 100644 --- a/src/lib/compiler.h +++ b/src/lib/compiler.h @@ -6,6 +6,7 @@ #define INIT(X) extern "C" void __uce_init(X) #define WS(X) extern "C" void __uce_websocket(X) #define CLI(X) extern "C" void __uce_cli(X) +#define SERVE_HTTP(X) extern "C" void __uce_serve_http(X) #define EXPORT extern "C" String preprocess_shared_unit(Request* context, SharedUnit* su); @@ -16,6 +17,7 @@ SharedUnit* get_shared_unit(Request* context, String file_name, bool opt_so_opti void compiler_invoke(Request* context, String file_name); void compiler_invoke_websocket(Request* context, String file_name); void compiler_invoke_cli(Request* context, String file_name); +void compiler_invoke_serve_http(Request* context, String file_name, String handler_name = ""); SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String current_path = "", bool opt_so_optional = false); String compiler_site_directory(Request* context); StringList compiler_scan_site_units(Request* context); diff --git a/src/lib/mysql-connector.cpp b/src/lib/mysql-connector.cpp index 67588c3..476b456 100644 --- a/src/lib/mysql-connector.cpp +++ b/src/lib/mysql-connector.cpp @@ -300,5 +300,6 @@ void cleanup_mysql_connections() //switch_to_system_alloc(); for(auto& con : context->resources.mysql_connections) mysql_close((MYSQL*)con); + context->resources.mysql_connections.clear(); //switch_to_arena(context->mem); } diff --git a/src/lib/mysql-connector.h b/src/lib/mysql-connector.h index 832294c..0d2197d 100644 --- a/src/lib/mysql-connector.h +++ b/src/lib/mysql-connector.h @@ -43,7 +43,10 @@ MySQL* mysql_connect(String host = "localhost", String username = "root", String void mysql_disconnect(MySQL* m) { + if(!m) + return; m->disconnect(); + delete m; } String mysql_error(MySQL* m) diff --git a/src/lib/sys.cpp b/src/lib/sys.cpp index efe8abd..a098453 100644 --- a/src/lib/sys.cpp +++ b/src/lib/sys.cpp @@ -8,6 +8,7 @@ #include #include #include "sys.h" +#include "hash.h" namespace { @@ -613,17 +614,17 @@ pid_t spawn_subprocess(std::function exec_after_spawn) } } -String task_safe_key(String key) +String runtime_safe_key(String key, String label) { key = trim(key); if(key == "") - throw std::runtime_error("task key cannot be empty"); + throw std::runtime_error(label + " cannot be empty"); return(gen_sha1(key)); } String task_file_prefix(String key) { - return(path_join(context->server->config["BIN_DIRECTORY"], "task-" + task_safe_key(key))); + return(path_join(context->server->config["BIN_DIRECTORY"], "task-" + runtime_safe_key(key, "task key"))); } struct TaskStatus { @@ -683,6 +684,15 @@ bool task_status_is_alive(TaskStatus status) return(task_process_start_ticks(status.pid) == status.process_start_ticks); } +void task_close_inherited_fds() +{ + long max_fd = sysconf(_SC_OPEN_MAX); + if(max_fd < 0 || max_fd > 65536) + max_fd = 4096; + for(int fd = 3; fd < max_fd; fd++) + close(fd); +} + int task_kill(pid_t pid, int sig) { if(pid <= 0) @@ -703,7 +713,7 @@ pid_t task_pid(String key) fprintf(stderr, "task_pid(): could not lock task key '%s'\n", key.c_str()); return(0); } - String status_file = file_get_contents(status_file_name); + String status_file = file_exists(status_file_name) ? file_get_contents(status_file_name) : ""; if(status_file != "") { TaskStatus status = task_status_parse(status_file); @@ -728,7 +738,7 @@ pid_t task(String key, std::function exec_after_spawn, u64 timeout) fprintf(stderr, "task(): could not lock task key '%s'\n", key.c_str()); return(0); } - String status_file = file_get_contents(status_file_name); + String status_file = file_exists(status_file_name) ? file_get_contents(status_file_name) : ""; pid_t p = 0; if(status_file != "") { @@ -753,7 +763,7 @@ pid_t task(String key, std::function exec_after_spawn, u64 timeout) file_release_process_locks("task child startup"); file_close_locked(lock_fd); my_pid = getpid(); - prctl(PR_SET_PDEATHSIG, SIGHUP); + signal(SIGALRM, SIG_DFL); if(timeout > 0) alarm(timeout); @@ -762,6 +772,7 @@ pid_t task(String key, std::function exec_after_spawn, u64 timeout) close(context->resources.client_socket); context->resources.client_socket = 0; } + task_close_inherited_fds(); exec_after_spawn(); int exit_lock_fd = file_open_locked(lock_file_name, O_RDWR | O_CREAT, LOCK_EX, 0644, FILE_LOCK_WAIT_TIMEOUT_SECONDS, "task-exit:" + key); if(exit_lock_fd != -1) @@ -857,6 +868,25 @@ StringMap make_server_settings() cfg["PROACTIVE_COMPILE_ENABLED"] = "1"; cfg["COMPILE_FAILURE_RETRY_SECONDS"] = std::to_string(10); cfg["PROACTIVE_COMPILE_CHECK_INTERVAL"] = std::to_string(60); + cfg["TRANSPORT_MAX_CLIENT_CONNECTIONS"] = std::to_string(256); + cfg["TRANSPORT_MAX_HTTP_HEADER_BYTES"] = std::to_string(16 * 1024); + cfg["TRANSPORT_MAX_HTTP_BODY_BYTES"] = std::to_string(1024 * 1024); + cfg["TRANSPORT_MAX_WEBSOCKET_FRAME_BYTES"] = std::to_string(1024 * 1024); + cfg["TRANSPORT_MAX_WEBSOCKET_MESSAGE_BYTES"] = std::to_string(1024 * 1024); + cfg["TRANSPORT_MAX_WEBSOCKET_OUTPUT_BYTES"] = std::to_string(4 * 1024 * 1024); + cfg["TRANSPORT_MAX_RESPONSE_BYTES"] = std::to_string(8 * 1024 * 1024); + cfg["TRANSPORT_HTTP_REQUEST_TIMEOUT_SECONDS"] = "15"; + cfg["TRANSPORT_CONNECTION_IDLE_TIMEOUT_SECONDS"] = "120"; + cfg["CUSTOM_SERVER_MAX_SERVERS"] = "16"; + cfg["CUSTOM_SERVER_MIN_PORT"] = "1024"; + cfg["CUSTOM_SERVER_MAX_PORT"] = "65535"; + cfg["CUSTOM_SERVER_ALLOW_PUBLIC_BIND"] = "0"; + cfg["CUSTOM_SERVER_UNIX_SOCKET_PREFIX"] = "/tmp/uce/custom-servers/"; + cfg["CUSTOM_SERVER_HANDLER_TIMEOUT_SECONDS"] = "30"; + cfg["CUSTOM_SERVER_UCE_ROOT"] = ""; + cfg["ARCHIVE_MAX_INPUT_BYTES"] = std::to_string(64 * 1024 * 1024); + cfg["ARCHIVE_MAX_OUTPUT_BYTES"] = std::to_string(64 * 1024 * 1024); + cfg["ARCHIVE_MAX_ZIP_ENTRIES"] = "4096"; cfg["HTTP_PORT"] = std::to_string(8080); cfg["SESSION_TIME"] = std::to_string(60*60*24*30); diff --git a/src/lib/sys.h b/src/lib/sys.h index 81aa52b..a61b182 100644 --- a/src/lib/sys.h +++ b/src/lib/sys.h @@ -74,6 +74,9 @@ pid_t my_pid = 0; void on_segfault(int sig); int task_kill(pid_t pid, int sig = 0); +String runtime_safe_key(String key, String label = "runtime key"); pid_t task(String key, std::function exec_after_spawn, u64 timeout = 60*10); pid_t task_repeat(String key, f64 interval, std::function exec_after_spawn, u64 timeout = 60*10); pid_t task_pid(String key); +pid_t server_start_http(String key, String socket_fn_or_port, String call_uce_filename, String call_function = ""); +bool server_stop(String key); diff --git a/src/lib/zip.cpp b/src/lib/zip.cpp index 5f5f1a6..d29761f 100644 --- a/src/lib/zip.cpp +++ b/src/lib/zip.cpp @@ -14,6 +14,23 @@ String zip_error(String api, String detail) return(api + "(): " + detail); } +u64 archive_config_u64(String key, u64 fallback) +{ + if(!context || !context->server) + return(fallback); + String raw = trim(context->server->config[key]); + if(raw == "") + return(fallback); + return(int_val(raw)); +} + +void archive_check_size(String api, String label, u64 size, String config_key, u64 fallback) +{ + u64 limit = archive_config_u64(config_key, fallback); + if(limit > 0 && size > limit) + throw std::runtime_error(zip_error(api, label + " exceeds configured limit")); +} + bool zip_entry_name_safe(String name) { if(name == "") @@ -78,6 +95,7 @@ DTree zip_list(String zip_file_name) try { mz_uint count = mz_zip_reader_get_num_files(&archive); + archive_check_size("zip_list", "entry count", count, "ARCHIVE_MAX_ZIP_ENTRIES", 4096); result["file"] = zip_file_name; result["count"] = (f64)count; result["entries"].set_array(); @@ -111,6 +129,7 @@ String zip_read(String zip_file_name, String entry_name) mz_zip_reader_end(&archive); throw std::runtime_error(zip_error("zip_read", "entry not found or not readable: " + entry_name)); } + archive_check_size("zip_read", "uncompressed entry", size, "ARCHIVE_MAX_OUTPUT_BYTES", 64 * 1024 * 1024); String result((char*)data, size); mz_free(data); mz_zip_reader_end(&archive); @@ -138,6 +157,7 @@ String zip_entry_name(String key, DTree item) void zip_add_entry(mz_zip_archive* archive, String name, String content) { + archive_check_size("zip_create", "entry content", content.size(), "ARCHIVE_MAX_INPUT_BYTES", 64 * 1024 * 1024); name = zip_normalize_entry_name(name); if(!zip_entry_name_safe(name)) throw std::runtime_error(zip_error("zip_create", "unsafe or empty entry name '" + name + "'")); @@ -157,8 +177,11 @@ bool zip_create(String zip_file_name, DTree entries) try { + u64 entry_count = 0; entries.each([&](DTree item, String key) { + entry_count++; + archive_check_size("zip_create", "entry count", entry_count, "ARCHIVE_MAX_ZIP_ENTRIES", 4096); String name = zip_entry_name(key, item); String content = zip_entry_content(item); zip_add_entry(&archive, name, content); @@ -242,6 +265,7 @@ size_t gz_deflate_offset(String compressed) String gz_compress(String src) { + archive_check_size("gz_compress", "input", src.size(), "ARCHIVE_MAX_INPUT_BYTES", 64 * 1024 * 1024); mz_stream stream; std::memset(&stream, 0, sizeof(stream)); int status = mz_deflateInit2(&stream, MZ_DEFAULT_COMPRESSION, MZ_DEFLATED, -MZ_DEFAULT_WINDOW_BITS, 9, MZ_DEFAULT_STRATEGY); @@ -289,6 +313,7 @@ String gz_uncompress(String compressed) if(!out) throw std::runtime_error("gz_uncompress(): decompression failed"); + archive_check_size("gz_uncompress", "output", out_len, "ARCHIVE_MAX_OUTPUT_BYTES", 64 * 1024 * 1024); String result((char*)out, out_len); mz_free(out); @@ -317,12 +342,14 @@ bool zip_extract(String zip_file_name, String destination_directory) try { mz_uint count = mz_zip_reader_get_num_files(&archive); + archive_check_size("zip_extract", "entry count", count, "ARCHIVE_MAX_ZIP_ENTRIES", 4096); for(mz_uint i = 0; i < count; i++) { mz_zip_archive_file_stat stat; std::memset(&stat, 0, sizeof(stat)); if(!mz_zip_reader_file_stat(&archive, i, &stat)) throw std::runtime_error(zip_error("zip_extract", "could not read file metadata at index " + std::to_string((u64)i))); + archive_check_size("zip_extract", "uncompressed entry", stat.m_uncomp_size, "ARCHIVE_MAX_OUTPUT_BYTES", 64 * 1024 * 1024); String name = zip_normalize_entry_name(stat.m_filename); if(!zip_entry_name_safe(name)) throw std::runtime_error(zip_error("zip_extract", "refusing unsafe entry name '" + name + "'")); diff --git a/src/linux_fastcgi.cpp b/src/linux_fastcgi.cpp index edf277b..0f2b02c 100644 --- a/src/linux_fastcgi.cpp +++ b/src/linux_fastcgi.cpp @@ -121,6 +121,7 @@ void install_request_fault_handlers() signal(SIGBUS, on_request_fault_signal); signal(SIGILL, on_request_fault_signal); signal(SIGFPE, on_request_fault_signal); + signal(SIGALRM, on_request_fault_signal); } void restore_request_fault_handlers() @@ -130,6 +131,7 @@ void restore_request_fault_handlers() signal(SIGBUS, on_segfault); signal(SIGILL, on_segfault); signal(SIGFPE, on_segfault); + signal(SIGALRM, on_segfault); } namespace { @@ -1005,6 +1007,8 @@ int handle_complete(FastCGIRequest& request) { request.props = DTree(); if(request.resources.is_cli) compiler_invoke_cli(&request, request.params["SCRIPT_FILENAME"]); + else if(request.params["UCE_SERVE_HTTP"] == "1") + compiler_invoke_serve_http(&request, request.params["SCRIPT_FILENAME"], request.params["UCE_SERVE_HTTP_FUNCTION"]); else compiler_invoke(&request, request.params["SCRIPT_FILENAME"]); } @@ -1106,6 +1110,270 @@ void close_inherited_server_sockets() server.server_socket_types.clear(); } +void install_process_fault_handlers() +{ + signal(SIGSEGV, on_segfault); + signal(SIGABRT, on_segfault); + signal(SIGBUS, on_segfault); + signal(SIGILL, on_segfault); + signal(SIGFPE, on_segfault); + signal(SIGPIPE, SIG_IGN); +} + +String custom_server_safe_key(String key) +{ + return(runtime_safe_key(key, "server key")); +} + +String custom_server_config_file(String key) +{ + return(path_join(server_state.config["BIN_DIRECTORY"], "server-" + custom_server_safe_key(key) + ".cfg")); +} + +String custom_server_task_key(String key) +{ + return("server:" + custom_server_safe_key(key)); +} + +String custom_server_config_encode(String key, String type, String bind, String file_name, String function_name) +{ + DTree config; + config["key"] = key; + config["type"] = type; + config["bind"] = bind; + config["file"] = file_name; + config["function"] = function_name; + return(json_encode(config)); +} + +StringMap custom_server_config_decode(String content) +{ + StringMap result; + content = trim(content); + if(content == "") + return(result); + + if(content[0] == '{') + return(json_decode(content).to_stringmap()); + + // Compatibility for early newline/key=value config files from development. + for(auto line : split(content, "\n")) + { + line = trim(line); + if(line == "") + continue; + String key = trim(nibble(line, "=")); + if(key != "") + result[key] = json_decode(line).to_string(); + } + return(result); +} + +u64 custom_server_config_u64(String key, u64 fallback) +{ + String raw = trim(server_state.config[key]); + if(raw == "") + return(fallback); + return(int_val(raw)); +} + +bool custom_server_config_truthy(String key, bool fallback) +{ + String raw = to_lower(trim(server_state.config[key])); + if(raw == "") + return(fallback); + return(raw == "1" || raw == "true" || raw == "yes" || raw == "on"); +} + +bool custom_server_is_numeric_port(String value) +{ + value = trim(value); + if(value == "") + return(false); + for(char c : value) + { + if(c < '0' || c > '9') + return(false); + } + return(true); +} + +u64 custom_server_registry_count() +{ + u64 count = 0; + for(auto file_name : ls(server_state.config["BIN_DIRECTORY"])) + { + if(str_starts_with(file_name, "server-") && str_ends_with(file_name, ".cfg")) + count++; + } + return(count); +} + +bool custom_server_wait_for_stop(String task_key, f64 timeout_seconds = 2.0) +{ + f64 deadline = time_precise() + timeout_seconds; + while(time_precise() < deadline) + { + if(task_pid(task_key) == 0) + return(true); + usleep(50000); + } + return(task_pid(task_key) == 0); +} + +int custom_server_bind_http(FastCGIServer& dispatcher, String bind) +{ + bind = trim(bind); + if(custom_server_is_numeric_port(bind)) + { + u64 port = int_val(bind); + u64 min_port = custom_server_config_u64("CUSTOM_SERVER_MIN_PORT", 1024); + u64 max_port = custom_server_config_u64("CUSTOM_SERVER_MAX_PORT", 65535); + if(port < min_port || port > max_port) + throw std::runtime_error("server_start_http(): TCP port is outside configured custom server range"); + String bind_address = custom_server_config_truthy("CUSTOM_SERVER_ALLOW_PUBLIC_BIND", false) ? "0.0.0.0" : "127.0.0.1"; + return(dispatcher.listen_http((unsigned)port, bind_address)); + } + String socket_prefix = first(server_state.config["CUSTOM_SERVER_UNIX_SOCKET_PREFIX"], "/tmp/uce/custom-servers/"); + if(!str_starts_with(bind, socket_prefix)) + throw std::runtime_error("server_start_http(): Unix socket path is outside configured custom server prefix"); + int socket_handle = dispatcher.listen(bind); + dispatcher.server_socket_types[socket_handle] = 'H'; + return(socket_handle); +} + +StringMap custom_server_read_config(String key) +{ + String config_file = custom_server_config_file(key); + if(!file_exists(config_file)) + return(StringMap()); + return(custom_server_config_decode(file_get_contents(config_file))); +} + +static String custom_server_dispatcher_key = ""; + +int custom_server_http_complete(FastCGIRequest& request) +{ + String key = first(request.params["UCE_SERVER_KEY"], custom_server_dispatcher_key); + StringMap cfg = custom_server_read_config(key); + if(cfg["type"] != "http" || cfg["file"] == "") + { + request.set_status(503, "Service Unavailable"); + request.header["Content-Type"] = "text/plain; charset=utf-8"; + request.ob_start(); + (*request.ob) << "custom HTTP server is not configured\n"; + return(request.flags.status); + } + + request.params["UCE_SERVE_HTTP"] = "1"; + request.params["UCE_SERVE_HTTP_KEY"] = key; + request.params["UCE_SERVE_HTTP_BIND"] = cfg["bind"]; + request.params["UCE_SERVE_HTTP_FUNCTION"] = cfg["function"]; + request.params["SCRIPT_FILENAME"] = cfg["file"]; + u64 timeout = custom_server_config_u64("CUSTOM_SERVER_HANDLER_TIMEOUT_SECONDS", 30); + if(timeout > 0) + alarm(timeout); + int status = handle_complete(request); + if(timeout > 0) + alarm(0); + return(status); +} + +void custom_server_http_dispatcher_loop(String key) +{ + my_pid = getpid(); + custom_server_dispatcher_key = key; + close_inherited_server_sockets(); + install_process_fault_handlers(); + + StringMap cfg = custom_server_read_config(key); + if(cfg["type"] != "http" || cfg["bind"] == "") + { + fprintf(stderr, "server_start_http(): missing config for key '%s'\n", key.c_str()); + exit(1); + } + + try + { + FastCGIServer dispatcher; + custom_server_bind_http(dispatcher, cfg["bind"]); + dispatcher.calls_until_termination = -1; + dispatcher.on_complete = &custom_server_http_complete; + for(;;) + dispatcher.process(-1); + } + catch(const std::exception& e) + { + fprintf(stderr, "server_start_http(): dispatcher for key '%s' stopped: %s\n", key.c_str(), e.what()); + exit(1); + } + catch(...) + { + fprintf(stderr, "server_start_http(): dispatcher for key '%s' stopped with unknown error\n", key.c_str()); + exit(1); + } +} + +pid_t server_start_http(String key, String socket_fn_or_port, String call_uce_filename, String call_function) +{ + key = trim(key); + socket_fn_or_port = trim(socket_fn_or_port); + call_uce_filename = trim(call_uce_filename); + call_function = trim(call_function); + if(key == "") + throw std::runtime_error("server_start_http(): key cannot be empty"); + if(socket_fn_or_port == "") + throw std::runtime_error("server_start_http(): socket_fn_or_port cannot be empty"); + if(call_uce_filename == "") + throw std::runtime_error("server_start_http(): call_uce_filename cannot be empty"); + if(call_uce_filename[0] != '/') + call_uce_filename = expand_path(call_uce_filename, cwd_get()); + String allowed_root = first(server_state.config["CUSTOM_SERVER_UCE_ROOT"], path_join(server_state.config["COMPILER_SYS_PATH"], server_state.config["SITE_DIRECTORY"])); + if(allowed_root[allowed_root.length() - 1] != '/') + allowed_root += "/"; + if(!str_starts_with(call_uce_filename, allowed_root)) + throw std::runtime_error("server_start_http(): call_uce_filename is outside configured custom server UCE root"); + + String config_file = custom_server_config_file(key); + StringMap previous_config; + if(file_exists(config_file)) + previous_config = custom_server_config_decode(file_get_contents(config_file)); + String new_config = custom_server_config_encode(key, "http", socket_fn_or_port, call_uce_filename, call_function); + String task_key = custom_server_task_key(key); + u64 max_servers = custom_server_config_u64("CUSTOM_SERVER_MAX_SERVERS", 16); + if(!file_exists(config_file) && max_servers > 0 && custom_server_registry_count() >= max_servers) + throw std::runtime_error("server_start_http(): custom server quota exceeded"); + pid_t existing_pid = task_pid(task_key); + if(existing_pid != 0 && previous_config["bind"] != "" && previous_config["bind"] != socket_fn_or_port) + { + task_kill(existing_pid, SIGTERM); + custom_server_wait_for_stop(task_key); + existing_pid = 0; + } + if(!file_put_contents(config_file, new_config)) + throw std::runtime_error("server_start_http(): could not write server config"); + if(existing_pid != 0) + return(existing_pid); + return(task(task_key, [key]() { + custom_server_http_dispatcher_loop(key); + }, 0)); +} + +bool server_stop(String key) +{ + key = trim(key); + if(key == "") + return(false); + String task_key = custom_server_task_key(key); + pid_t pid = task_pid(task_key); + file_unlink(custom_server_config_file(key)); + if(pid == 0) + return(true); + if(task_kill(pid, SIGTERM) != 0) + return(false); + return(custom_server_wait_for_stop(task_key)); +} + bool proactive_compile_queue_has(StringList& queue, String file_name) { return(std::find(queue.begin(), queue.end(), file_name) != queue.end()); @@ -1251,12 +1519,7 @@ void ensure_proactive_compiler() void listen_for_connections() { - signal(SIGSEGV, on_segfault); - signal(SIGABRT, on_segfault); - signal(SIGBUS, on_segfault); - signal(SIGILL, on_segfault); - signal(SIGFPE, on_segfault); - signal(SIGPIPE, SIG_IGN); + install_process_fault_handlers(); if(worker_accepts_http) { // Keep the dedicated HTTP/WebSocket worker alive. If it ages out like a