update sigh

src/fastcgi/src/fcgicc.cc

@@ -261,6 +261,12 @@ FastCGIServer::process(int timeout_ms)
 			Connection* connection = it->second;
 			read_sockets.erase(it++);
 			delete connection;
+			if(calls_until_termination != -1 && read_sockets.size() == 0)
+			{
+				calls_until_termination -= 1;
+				if(calls_until_termination <= 0)
+					exit(0);
+			}
 		} else
 			++it;
 	}
@@ -278,9 +284,9 @@ int
 FastCGIServer::call_completion_handler(FastCGIRequest& request)
 {
 	//printf("- request complete\n");
-	switch_to_arena(request.mem);
+	////switch_to_arena(request.mem);
 	auto result = on_complete(request);
-	switch_to_system_alloc();
+	////switch_to_system_alloc();
 	return(result);
 }
 
@@ -364,9 +370,9 @@ FastCGIServer::process_connection_read(Connection& connection)
 				if (it != connection.requests.end())
 				{
 					//printf("- delete request object\n");
-					switch_to_arena(it->second->mem);
+					//switch_to_arena(it->second->mem);
 					delete it->second;
-					switch_to_system_alloc();
+					//switch_to_system_alloc();
 					connection.requests.erase(it);
 				}
 			}
@@ -376,14 +382,12 @@ FastCGIServer::process_connection_read(Connection& connection)
 				printf("(!) %i requests in flight at the same time!\n", connection.requests.size());
 			}
 
-			//auto arena = new MemoryArena(server_state.config.MAX_MEMORY, "request");
-			request_arena->clear();
-			switch_to_arena(request_arena);
+			//switch_to_arena(request_arena);
 			RequestInfo* new_request = new RequestInfo();
 			new_request->resources.fcgi_socket = connection.posix_con;
-		    new_request->mem = request_arena;
+		    //new_request->mem = request_arena;
 			new_request->stats.time_init = microtime();
-			switch_to_system_alloc();
+			//switch_to_system_alloc();
 			connection.requests[request_id] = new_request;
 
 			break;
@@ -417,7 +421,7 @@ FastCGIServer::process_connection_read(Connection& connection)
 				break;
 
 			RequestInfo& request = *it->second;
-			switch_to_arena(it->second->mem);
+			//switch_to_arena(it->second->mem);
 			if (!request.params_closed)
 				if (content_length != 0)
 				request.params_buffer.append(content, content_length);
@@ -436,7 +440,7 @@ FastCGIServer::process_connection_read(Connection& connection)
 				}
 				process_write_request(connection, request_id, request);
 				}
-			switch_to_system_alloc();
+			//switch_to_system_alloc();
 			break;
 		}
 		case FCGI_STDIN:
@@ -446,7 +450,7 @@ FastCGIServer::process_connection_read(Connection& connection)
 				break;
 
 			RequestInfo& request = *it->second;
-			switch_to_arena(it->second->mem);
+			//switch_to_arena(it->second->mem);
 			if (!request.in_closed)
 				if (content_length != 0) {
 				request.in.append(content, content_length);
@@ -462,7 +466,7 @@ FastCGIServer::process_connection_read(Connection& connection)
 							process_write_request(connection, request_id, request);
 				}
 				}
-			switch_to_system_alloc();
+			//switch_to_system_alloc();
 			break;
 		}
 		case FCGI_DATA:
@@ -494,21 +498,21 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
 	if (!request.out.empty())
 	{
 		write_data(connection.output_buffer, id, request.out, FCGI_STDOUT);
-		switch_to_arena(request.mem);
+		//switch_to_arena(request.mem);
 		request.out.clear();
-		switch_to_system_alloc();
+		//switch_to_system_alloc();
 	}
 	if (!request.err.empty())
 	{
 		write_data(connection.output_buffer, id, request.err, FCGI_STDERR);
-		switch_to_arena(request.mem);
+		//switch_to_arena(request.mem);
 		request.err.clear();
-		switch_to_system_alloc();
+		//switch_to_system_alloc();
 	}
 	if ((request.in_closed || request.status != 0) &&
 				!request.output_closed)
 	{
-		switch_to_arena(request.mem);
+		//switch_to_arena(request.mem);
 		request.out =
 			var_dump(request.header, "", "\r\n") +
 			var_dump(request.set_cookies, "", "\r\n") +
@@ -521,7 +525,7 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
 		}
 		request.ob_stack.clear();
 
-		switch_to_system_alloc();
+		//switch_to_system_alloc();
 		write_data(connection.output_buffer, id, request.out, FCGI_STDOUT);
 		write_data(connection.output_buffer, id, request.err, FCGI_STDERR);
 
@@ -533,8 +537,8 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
 				request.stats.time_end - request.stats.time_start,
 				1.0 / (request.stats.time_end - request.stats.time_start),
 				(f32)(request.out.length()/1024),
-				(f32)(request.mem->size/1024),
-				(f32)(request.mem->capacity/1024)
+				(f32)(request.stats.mem_high/1024),
+				(f32)(request.stats.mem_alloc/1024)
 			);
 
 		FCGI_EndRequestRecord complete;
@@ -557,7 +561,7 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
 		request.output_closed = true;
 		//printf("- output done\n");
 	}
-	switch_to_system_alloc();
+	//switch_to_system_alloc();
 }
 
 
@@ -570,10 +574,10 @@ FastCGIServer::process_connection_write(Connection& connection)
 		process_write_request(connection, it->first, *it->second);
 		if (it->second->params_closed && it->second->in_closed)
 		{
-			switch_to_arena(it->second->mem);
+			//switch_to_arena(it->second->mem);
 			//printf("- process_connection_write close\n");
 			delete it->second;
-			switch_to_system_alloc();
+			//switch_to_system_alloc();
 			connection.requests.erase(it++);
 		}
 		else

src/fastcgi/src/fcgicc.h

@@ -55,6 +55,7 @@ public:
 
 	void process(int timeout_ms = -1); // timeout_ms<0 blocks forever
 	void process_forever();
+	int calls_until_termination = 8; // set this to -1 to never terminate
 
 	int call_completion_handler(FastCGIRequest& request);
 

src/lib/_scratchpad.cpp

@@ -0,0 +1,90 @@
+#define NO_GLOBAL_ARENA_ALLOCATOR
+
+struct MemoryArena {
+
+	u8* data;
+	u64 size = 0;
+	u64 capacity = 0;
+	String name = "unnamed";
+
+	MemoryArena(u64 cap, String _name = "unnamed")
+	{
+		name = _name;
+		capacity = cap;
+		printf("(i) memory arena '%s' created with capacity of %llu bytes\n", name.c_str(), capacity);
+		data = (u8*)malloc(cap);
+	}
+
+	~MemoryArena()
+	{
+		free(data);
+	}
+
+	void clear()
+	{
+		#ifdef DEBUG_MEMORY
+		printf("(i) memory arena '%s' cleared after high mark of %llu bytes\n", name.c_str(), size);
+		#endif
+		size = 0;
+	}
+
+	void* get(u64 size_needed)
+	{
+		u64 size_aligned = 8 + (8 * ((size_needed) / 8));
+		u8* result = data + size;
+		if(size_aligned + size >= capacity)
+		{
+			printf("(!) memory arena '%s' capacity (%llu) exceeded %llu/%llu + %llu >= %llu\n",
+				name.c_str(), capacity, size_needed, size_aligned, size, capacity);
+			return(0);
+		}
+		size += size_aligned;
+		#ifdef DEBUG_MEMORY_DETAILED
+		printf("(i) memory arena '%s' [+%llu]:%p alloc %llu/%llu bytes\n", name.c_str(), size, result, size_needed, size_aligned);
+		#endif
+		return(result);
+	}
+
+};
+
+MemoryArena* current_memory_arena = 0;
+
+void switch_to_system_alloc()
+{
+#ifdef GLOBAL_ARENA_ALLOCATOR
+	current_memory_arena = 0;
+#endif
+}
+
+void switch_to_arena(MemoryArena* a)
+{
+#ifdef GLOBAL_ARENA_ALLOCATOR
+	current_memory_arena = a;
+#endif
+}
+
+#ifdef GLOBAL_ARENA_ALLOCATOR
+void * operator new(decltype(sizeof(0)) n) noexcept(false)
+{
+	if(current_memory_arena)
+	{
+		return(current_memory_arena->get(n));
+	}
+	else
+	{
+		return(malloc(n));
+	}
+}
+
+void operator delete(void * p) throw()
+{
+	if(current_memory_arena)
+	{
+
+	}
+	else
+	{
+		free(p);
+	}
+}
+#endif

src/lib/compiler.cpp

@@ -89,7 +89,7 @@ String process_html_literal(Request* context, SharedUnit* su, String content)
 
 String preprocess_shared_unit_char_wise(Request* context, SharedUnit* su, String content)
 {
-	String pc = "#include \"" + su->src_file_name + ".setup.h" + "\"\n";
+	String pc = "#include \"" + su->src_file_name + ".setup.h" + "\"\n#line 1\n";
 	String token = "";
 	String html_buffer = "";
 	u8 mode = 0;
@@ -235,6 +235,7 @@ void load_shared_unit(Request* context, SharedUnit* su, String file_name)
 		if ((error = dlerror()) != NULL)
 			printf("Error - %s in %s\n", error, su->file_name.c_str());
 		su->on_render = (call_handler)dlsym(su->so_handle, "render");
+		su->api_declarations = split(file_get_contents(su->api_file_name), "\n");
 		//else
 		//	printf("(i) loaded unit %s\n", su->file_name.c_str());
 	}
@@ -250,7 +251,8 @@ String compile_setup_file(Request* context, SharedUnit* su)
 		String("#ifndef UCE_LIB_INCLUDED\n") +
 		"#define UCE_LIB_INCLUDED\n" +
 		("#include \"")+context->server->config["COMPILER_SYS_PATH"] +"/src/lib/uce_lib.h\" \n"+
-		file_get_contents(context->server->config["COMPILER_SYS_PATH"] + "/" + context->server->config["SETUP_TEMPLATE"]) +
+		file_get_contents(
+			context->server->config["COMPILER_SYS_PATH"] + "/" + context->server->config["SETUP_TEMPLATE"]) +
 		"#endif \n";
 	StringList declarations;
 	StringList load_units;
@@ -363,9 +365,9 @@ SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String
 
 	//printf("(i) load '%s'\n", file_name.c_str());
 
-	switch_to_system_alloc();
+	//switch_to_system_alloc();
 	auto su = get_shared_unit(context, file_name, opt_so_optional);
-	switch_to_arena(context->mem);
+	//switch_to_arena(context->mem);
 	if(!su)
 	{
 		printf("Error loading unit %s\n", file_name.c_str());

src/lib/compiler.h

@@ -8,7 +8,7 @@ void load_shared_unit(Request* context, SharedUnit* su, String file_name);
 void compile_shared_unit(Request* context, SharedUnit* su, String file_name);
 SharedUnit* get_shared_unit(Request* context, String file_name, bool opt_so_optional = false);
 void compiler_invoke(Request* context, String file_name, DTree& call_param);
-SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String current_path, bool opt_so_optional = false);
+SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String current_path = "", bool opt_so_optional = false);
 
 SharedUnit* load_file(String file_name);
 void render_file(String file_name);

src/lib/functionlib.h

@@ -16,6 +16,31 @@ String join(StringList l, String delim = "\n");
 String nibble(String& haystack, String delim);
 void json_consume_space(String s, u32& i);
 
+String to_string(StringList l) {
+	String result;
+	u32 i = 0;
+	for(auto& s : l)
+	{
+		if(i > 0)
+			result.append("\n");
+		result.append(s);
+		i += 1;
+	}
+	return(result);
+}
+
+String to_string(SharedUnit* u) {
+	String result;
+
+	result += String("SharedUnit( \n")+
+		"Source:"+(u->file_name)+"\n"+
+		"SharedObject:"+(u->so_name)+"\n"+
+		"API:"+(u->api_file_name)+"\n"+
+		to_string(u->api_declarations);
+
+	return(result);
+}
+
 template <typename ITYPE>
 String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1)
 {

src/lib/mysql-connector.cpp

@@ -5,13 +5,13 @@
 
 bool MySQL::connect(String host, String username, String password)
 {
-	switch_to_system_alloc();
+	//switch_to_system_alloc();
 	connection = mysql_init(NULL);
 	if (connection == NULL)
 	{
 		auto e = mysql_error((MYSQL*)connection);
 		fprintf(stderr, "%s\n", e);
-		switch_to_arena(context->mem);
+		//switch_to_arena(context->mem);
 		statement_info.assign(e);
 		return(false);
 	}
@@ -22,7 +22,7 @@ bool MySQL::connect(String host, String username, String password)
 		auto e = mysql_error((MYSQL*)connection);
 		fprintf(stderr, "%s\n", e);
 		mysql_close((MYSQL*)connection);
-		switch_to_arena(context->mem);
+		//switch_to_arena(context->mem);
 		statement_info.assign(e);
 		return(false);
 	}
@@ -35,7 +35,7 @@ bool MySQL::connect(String host, String username, String password)
 		exit(1);
 	}
 	*/
-	switch_to_arena(context->mem);
+	//switch_to_arena(context->mem);
 	statement_info = String("connected");
 	context->resources.mysql_connections.push_back(connection);
 	return(true);
@@ -297,8 +297,8 @@ String MySQL::error()
 
 void cleanup_mysql_connections()
 {
-	switch_to_system_alloc();
+	//switch_to_system_alloc();
 	for(auto& con : context->resources.mysql_connections)
 		mysql_close((MYSQL*)con);
-	switch_to_arena(context->mem);
+	//switch_to_arena(context->mem);
 }

src/lib/sys.cpp

@@ -491,6 +491,20 @@ pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout)
 	}
 }
 
+#include <unistd.h>
+pid_t task_repeat(String key, u64 interval, std::function<void()> exec_after_spawn, u64 timeout)
+{
+	auto repeater_function = [&]() {
+		while (true)
+		{
+			exec_after_spawn();
+			printf("(P) worker process '%s' sleeping\n", key.c_str());
+			usleep((s64)(interval*1000000));
+		}
+	};
+	return(task(key, repeater_function, timeout));
+}
+
 void on_child_exit(int sig)
 {
     pid_t pid;

src/lib/sys.h

@@ -51,4 +51,5 @@ pid_t my_pid = 0;
 void on_segfault(int sig);
 
 pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout = 60*10);
+pid_t task_repeat(String key, f64 interval, std::function<void()> exec_after_spawn, u64 timeout = 60*10);
 pid_t task_pid(String key);

src/lib/types.h

@@ -4,6 +4,7 @@
 #include <vector>
 #include <functional>
 #include <sstream>
+#include <atomic>
 
 typedef unsigned char u8;
 typedef signed char s8;
@@ -43,96 +44,6 @@ String operator+(String lhs, f32 rhs) {
 }
 
 #define DEBUG_MEMORY_OFF
-#define NO_GLOBAL_ARENA_ALLOCATOR
-
-struct MemoryArena {
-
-	u8* data;
-	u64 size = 0;
-	u64 capacity = 0;
-	String name = "unnamed";
-
-	MemoryArena(u64 cap, String _name = "unnamed")
-	{
-		name = _name;
-		capacity = cap;
-		printf("(i) memory arena '%s' created with capacity of %llu bytes\n", name.c_str(), capacity);
-		data = (u8*)malloc(cap);
-	}
-
-	~MemoryArena()
-	{
-		free(data);
-	}
-
-	void clear()
-	{
-		#ifdef DEBUG_MEMORY
-		printf("(i) memory arena '%s' cleared after high mark of %llu bytes\n", name.c_str(), size);
-		#endif
-		size = 0;
-	}
-
-	void* get(u64 size_needed)
-	{
-		u64 size_aligned = 8 + (8 * ((size_needed) / 8));
-		u8* result = data + size;
-		if(size_aligned + size >= capacity)
-		{
-			printf("(!) memory arena '%s' capacity (%llu) exceeded %llu/%llu + %llu >= %llu\n",
-				name.c_str(), capacity, size_needed, size_aligned, size, capacity);
-			return(0);
-		}
-		size += size_aligned;
-		#ifdef DEBUG_MEMORY_DETAILED
-		printf("(i) memory arena '%s' [+%llu]:%p alloc %llu/%llu bytes\n", name.c_str(), size, result, size_needed, size_aligned);
-		#endif
-		return(result);
-	}
-
-};
-
-MemoryArena* current_memory_arena = 0;
-
-void switch_to_system_alloc()
-{
-#ifdef GLOBAL_ARENA_ALLOCATOR
-	current_memory_arena = 0;
-#endif
-}
-
-void switch_to_arena(MemoryArena* a)
-{
-#ifdef GLOBAL_ARENA_ALLOCATOR
-	current_memory_arena = a;
-#endif
-}
-
-#ifdef GLOBAL_ARENA_ALLOCATOR
-void * operator new(decltype(sizeof(0)) n) noexcept(false)
-{
-	if(current_memory_arena)
-	{
-		return(current_memory_arena->get(n));
-	}
-	else
-	{
-		return(malloc(n));
-	}
-}
-
-void operator delete(void * p) throw()
-{
-	if(current_memory_arena)
-	{
-
-	}
-	else
-	{
-		free(p);
-	}
-}
-#endif
 
 typedef std::map<String, String> StringMap;
 typedef std::vector<String> StringList;
@@ -146,26 +57,6 @@ typedef void (*request_handler)(Request* request);
 
 String to_string(s64 v) { return(std::to_string(v)); }
 
-/*struct ServerSettings {
-
-	String BIN_DIRECTORY = "/tmp/uce/work";
-	String COMPILE_SCRIPT = "scripts/compile";
-	String SETUP_TEMPLATE = "scripts/setup.h.template";
-	String LIT_ESC = "3d5b5_1";
-	String CONTENT_TYPE = "text/html; charset=utf-8";
-	String SOCKET_PATH = "/run/uce.sock";
-	String TMP_UPLOAD_PATH = "/tmp/uce/uploads";
-	String SESSION_PATH = "/tmp/uce/sessions";
-	String COMPILER_SYS_PATH = ".";
-	String PRECOMPILE_FILES_IN = ".";
-
-	u32 LISTEN_PORT = 9993;
-	u64 SESSION_TIME = 60*60*24*30;
-	u32 WORKER_COUNT = 4;
-	u32 MAX_MEMORY = 1024*1024*16;
-
-};*/
-
 struct SharedUnit {
 
 	String file_name;
@@ -244,8 +135,6 @@ struct Request {
 	u64 random_seed;
 	u64 random_index;
 
-	MemoryArena* mem;
-
 	String in;
 	std::vector<std::ostringstream*> ob_stack;
 	std::ostringstream* ob;
@@ -263,6 +152,8 @@ struct Request {
 		f64 time_init;
 		f64 time_start;
 		f64 time_end;
+		u64 mem_high;
+		u64 mem_alloc;
 		u32 invoke_count = 0;
 	} stats;
 
@@ -301,3 +192,20 @@ String concat(Ts... args)
     return(out.str());
 }
 
+void * operator new(decltype(sizeof(0)) n) noexcept(false)
+{
+	void* ptr = malloc(n);
+	if(context)
+	{
+		context->stats.mem_alloc += n;
+		if(context->stats.mem_alloc > context->stats.mem_high)
+			context->stats.mem_high = context->stats.mem_alloc;
+	}
+	return(ptr);
+}
+
+void operator delete(void * p) throw()
+{
+	//TO DO: track deallocations
+	free(p);
+}

src/linux_fastcgi.cpp

@@ -1,7 +1,6 @@
 #include "lib/uce_lib.cpp"
 
 ServerState server_state;
-MemoryArena* request_arena = 0;
 
 #include "fastcgi/src/fcgicc.cc"
 
@@ -44,6 +43,8 @@ int handle_complete(FastCGIRequest& request) {
 	server_state.request_count += 1;
 	request.server = &server_state;
 	request.stats.time_start = microtime();
+	//request.stats.mem_alloc = 0;
+	//request.stats.mem_high = 0;
     request.header["Content-Type"] = context->server->config["CONTENT_TYPE"];
     request.get = parse_query(request.params["QUERY_STRING"]);
     request.random_index = 0;
@@ -102,15 +103,8 @@ void listen_for_connections()
 	server.on_request = &handle_request;
 	server.on_data = &handle_data;
 	server.on_complete = &handle_complete;
-	/*
-	server.request_handler(&handle_request);
-	server.data_handler(&handle_data);
-	server.complete_handler(&handle_complete);
-	*/
 	for(;;)
 	{
-		//if(request_arena) request_arena->clear();
-		//current_memory_arena = request_arena;
 		server.process();
 	}
 }
@@ -143,7 +137,6 @@ void init_base_process()
 	mkdir(server_state.config["TMP_UPLOAD_PATH"]);
 	mkdir(server_state.config["SESSION_PATH"]);
 
-	request_arena = new MemoryArena(int_val(server_state.config["MAX_MEMORY"]), "request");
 	signal(SIGCHLD, on_child_exit);
 	srand(time());
 }