update sigh

.gitignore

@@ -1,3 +1,4 @@
+.fuse*
 # Prerequisites
 *.d
 

scripts/build_linux.sh

@@ -12,7 +12,7 @@ mkdir bin/assets > /dev/null 2>&1
 mkdir work > /dev/null 2>&1
 
 COMPILER="clang++"
-FLAGS="-g -rdynamic -w -Wall -$OPT_FLAG -std=c++17 -fpermissive -ffast-math"
+FLAGS="-g -rdynamic -w -Wall -$OPT_FLAG -std=c++20 -fpermissive -ffast-math"
 
 LIBS="-ldl -lm -lpthread `mysql_config --cflags --libs`"
 SRCFLAGS="-D EXEC_NAME=\"$GF\" -D PLATFORM_NAME=\"linux\""

scripts/compile

@@ -26,7 +26,7 @@ OPT_FLAG="O0"
 
 COMPILER="clang++"
 #COMPILER="g++"
-FLAGS="-shared -g -rdynamic -w -Wall -$OPT_FLAG -std=c++14 -fpermissive -ffast-math -fPIC"
+FLAGS="-shared -g -rdynamic -w -Wall -$OPT_FLAG -std=c++20 -fpermissive -ffast-math -fPIC"
 
 LIBS="-ldl -lm -lpthread"
 SRCFLAGS="-D PLATFORM_NAME=\"linux\""

src/fastcgi/src/fcgicc.cc

@@ -261,6 +261,12 @@ FastCGIServer::process(int timeout_ms)
 			Connection* connection = it->second;
 			read_sockets.erase(it++);
 			delete connection;
+			if(calls_until_termination != -1 && read_sockets.size() == 0)
+			{
+				calls_until_termination -= 1;
+				if(calls_until_termination <= 0)
+					exit(0);
+			}
 		} else
 			++it;
 	}
@@ -278,9 +284,9 @@ int
 FastCGIServer::call_completion_handler(FastCGIRequest& request)
 {
 	//printf("- request complete\n");
-	switch_to_arena(request.mem);
+	////switch_to_arena(request.mem);
 	auto result = on_complete(request);
-	switch_to_system_alloc();
+	////switch_to_system_alloc();
 	return(result);
 }
 
@@ -364,9 +370,9 @@ FastCGIServer::process_connection_read(Connection& connection)
 				if (it != connection.requests.end())
 				{
 					//printf("- delete request object\n");
-					switch_to_arena(it->second->mem);
+					//switch_to_arena(it->second->mem);
 					delete it->second;
-					switch_to_system_alloc();
+					//switch_to_system_alloc();
 					connection.requests.erase(it);
 				}
 			}
@@ -376,14 +382,12 @@ FastCGIServer::process_connection_read(Connection& connection)
 				printf("(!) %i requests in flight at the same time!\n", connection.requests.size());
 			}
 
-			//auto arena = new MemoryArena(server_state.config.MAX_MEMORY, "request");
+			//switch_to_arena(request_arena);
-			request_arena->clear();
-			switch_to_arena(request_arena);
 			RequestInfo* new_request = new RequestInfo();
 			new_request->resources.fcgi_socket = connection.posix_con;
-		    new_request->mem = request_arena;
+		    //new_request->mem = request_arena;
 			new_request->stats.time_init = microtime();
-			switch_to_system_alloc();
+			//switch_to_system_alloc();
 			connection.requests[request_id] = new_request;
 
 			break;
@@ -417,7 +421,7 @@ FastCGIServer::process_connection_read(Connection& connection)
 				break;
 
 			RequestInfo& request = *it->second;
-			switch_to_arena(it->second->mem);
+			//switch_to_arena(it->second->mem);
 			if (!request.params_closed)
 				if (content_length != 0)
 				request.params_buffer.append(content, content_length);
@@ -436,7 +440,7 @@ FastCGIServer::process_connection_read(Connection& connection)
 				}
 				process_write_request(connection, request_id, request);
 				}
-			switch_to_system_alloc();
+			//switch_to_system_alloc();
 			break;
 		}
 		case FCGI_STDIN:
@@ -446,7 +450,7 @@ FastCGIServer::process_connection_read(Connection& connection)
 				break;
 
 			RequestInfo& request = *it->second;
-			switch_to_arena(it->second->mem);
+			//switch_to_arena(it->second->mem);
 			if (!request.in_closed)
 				if (content_length != 0) {
 				request.in.append(content, content_length);
@@ -462,7 +466,7 @@ FastCGIServer::process_connection_read(Connection& connection)
 							process_write_request(connection, request_id, request);
 				}
 				}
-			switch_to_system_alloc();
+			//switch_to_system_alloc();
 			break;
 		}
 		case FCGI_DATA:
@@ -494,21 +498,21 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
 	if (!request.out.empty())
 	{
 		write_data(connection.output_buffer, id, request.out, FCGI_STDOUT);
-		switch_to_arena(request.mem);
+		//switch_to_arena(request.mem);
 		request.out.clear();
-		switch_to_system_alloc();
+		//switch_to_system_alloc();
 	}
 	if (!request.err.empty())
 	{
 		write_data(connection.output_buffer, id, request.err, FCGI_STDERR);
-		switch_to_arena(request.mem);
+		//switch_to_arena(request.mem);
 		request.err.clear();
-		switch_to_system_alloc();
+		//switch_to_system_alloc();
 	}
 	if ((request.in_closed || request.status != 0) &&
 				!request.output_closed)
 	{
-		switch_to_arena(request.mem);
+		//switch_to_arena(request.mem);
 		request.out =
 			var_dump(request.header, "", "\r\n") +
 			var_dump(request.set_cookies, "", "\r\n") +
@@ -521,7 +525,7 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
 		}
 		request.ob_stack.clear();
 
-		switch_to_system_alloc();
+		//switch_to_system_alloc();
 		write_data(connection.output_buffer, id, request.out, FCGI_STDOUT);
 		write_data(connection.output_buffer, id, request.err, FCGI_STDERR);
 
@@ -533,8 +537,8 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
 				request.stats.time_end - request.stats.time_start,
 				1.0 / (request.stats.time_end - request.stats.time_start),
 				(f32)(request.out.length()/1024),
-				(f32)(request.mem->size/1024),
+				(f32)(request.stats.mem_high/1024),
-				(f32)(request.mem->capacity/1024)
+				(f32)(request.stats.mem_alloc/1024)
 			);
 
 		FCGI_EndRequestRecord complete;
@@ -557,7 +561,7 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
 		request.output_closed = true;
 		//printf("- output done\n");
 	}
-	switch_to_system_alloc();
+	//switch_to_system_alloc();
 }
 
 
@@ -570,10 +574,10 @@ FastCGIServer::process_connection_write(Connection& connection)
 		process_write_request(connection, it->first, *it->second);
 		if (it->second->params_closed && it->second->in_closed)
 		{
-			switch_to_arena(it->second->mem);
+			//switch_to_arena(it->second->mem);
 			//printf("- process_connection_write close\n");
 			delete it->second;
-			switch_to_system_alloc();
+			//switch_to_system_alloc();
 			connection.requests.erase(it++);
 		}
 		else

src/fastcgi/src/fcgicc.h

@@ -55,6 +55,7 @@ public:
 
 	void process(int timeout_ms = -1); // timeout_ms<0 blocks forever
 	void process_forever();
+	int calls_until_termination = 8; // set this to -1 to never terminate
 
 	int call_completion_handler(FastCGIRequest& request);
 

src/lib/_scratchpad.cpp

@@ -0,0 +1,90 @@
+#define NO_GLOBAL_ARENA_ALLOCATOR
+
+struct MemoryArena {
+
+	u8* data;
+	u64 size = 0;
+	u64 capacity = 0;
+	String name = "unnamed";
+
+	MemoryArena(u64 cap, String _name = "unnamed")
+	{
+		name = _name;
+		capacity = cap;
+		printf("(i) memory arena '%s' created with capacity of %llu bytes\n", name.c_str(), capacity);
+		data = (u8*)malloc(cap);
+	}
+
+	~MemoryArena()
+	{
+		free(data);
+	}
+
+	void clear()
+	{
+		#ifdef DEBUG_MEMORY
+		printf("(i) memory arena '%s' cleared after high mark of %llu bytes\n", name.c_str(), size);
+		#endif
+		size = 0;
+	}
+
+	void* get(u64 size_needed)
+	{
+		u64 size_aligned = 8 + (8 * ((size_needed) / 8));
+		u8* result = data + size;
+		if(size_aligned + size >= capacity)
+		{
+			printf("(!) memory arena '%s' capacity (%llu) exceeded %llu/%llu + %llu >= %llu\n",
+				name.c_str(), capacity, size_needed, size_aligned, size, capacity);
+			return(0);
+		}
+		size += size_aligned;
+		#ifdef DEBUG_MEMORY_DETAILED
+		printf("(i) memory arena '%s' [+%llu]:%p alloc %llu/%llu bytes\n", name.c_str(), size, result, size_needed, size_aligned);
+		#endif
+		return(result);
+	}
+
+};
+
+MemoryArena* current_memory_arena = 0;
+
+void switch_to_system_alloc()
+{
+#ifdef GLOBAL_ARENA_ALLOCATOR
+	current_memory_arena = 0;
+#endif
+}
+
+void switch_to_arena(MemoryArena* a)
+{
+#ifdef GLOBAL_ARENA_ALLOCATOR
+	current_memory_arena = a;
+#endif
+}
+
+#ifdef GLOBAL_ARENA_ALLOCATOR
+void * operator new(decltype(sizeof(0)) n) noexcept(false)
+{
+	if(current_memory_arena)
+	{
+		return(current_memory_arena->get(n));
+	}
+	else
+	{
+		return(malloc(n));
+	}
+}
+
+void operator delete(void * p) throw()
+{
+	if(current_memory_arena)
+	{
+
+	}
+	else
+	{
+		free(p);
+	}
+}
+#endif

src/lib/compiler.cpp

@@ -89,7 +89,7 @@ String process_html_literal(Request* context, SharedUnit* su, String content)
 
 String preprocess_shared_unit_char_wise(Request* context, SharedUnit* su, String content)
 {
-	String pc = "#include \"" + su->src_file_name + ".setup.h" + "\"\n";
+	String pc = "#include \"" + su->src_file_name + ".setup.h" + "\"\n#line 1\n";
 	String token = "";
 	String html_buffer = "";
 	u8 mode = 0;
@@ -235,6 +235,7 @@ void load_shared_unit(Request* context, SharedUnit* su, String file_name)
 		if ((error = dlerror()) != NULL)
 			printf("Error - %s in %s\n", error, su->file_name.c_str());
 		su->on_render = (call_handler)dlsym(su->so_handle, "render");
+		su->api_declarations = split(file_get_contents(su->api_file_name), "\n");
 		//else
 		//	printf("(i) loaded unit %s\n", su->file_name.c_str());
 	}
@@ -250,7 +251,8 @@ String compile_setup_file(Request* context, SharedUnit* su)
 		String("#ifndef UCE_LIB_INCLUDED\n") +
 		"#define UCE_LIB_INCLUDED\n" +
 		("#include \"")+context->server->config["COMPILER_SYS_PATH"] +"/src/lib/uce_lib.h\" \n"+
-		file_get_contents(context->server->config["COMPILER_SYS_PATH"] + "/" + context->server->config["SETUP_TEMPLATE"]) +
+		file_get_contents(
+			context->server->config["COMPILER_SYS_PATH"] + "/" + context->server->config["SETUP_TEMPLATE"]) +
 		"#endif \n";
 	StringList declarations;
 	StringList load_units;
@@ -363,9 +365,9 @@ SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String
 
 	//printf("(i) load '%s'\n", file_name.c_str());
 
-	switch_to_system_alloc();
+	//switch_to_system_alloc();
 	auto su = get_shared_unit(context, file_name, opt_so_optional);
-	switch_to_arena(context->mem);
+	//switch_to_arena(context->mem);
 	if(!su)
 	{
 		printf("Error loading unit %s\n", file_name.c_str());

src/lib/compiler.h

@@ -8,7 +8,7 @@ void load_shared_unit(Request* context, SharedUnit* su, String file_name);
 void compile_shared_unit(Request* context, SharedUnit* su, String file_name);
 SharedUnit* get_shared_unit(Request* context, String file_name, bool opt_so_optional = false);
 void compiler_invoke(Request* context, String file_name, DTree& call_param);
-SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String current_path, bool opt_so_optional = false);
+SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String current_path = "", bool opt_so_optional = false);
 
 SharedUnit* load_file(String file_name);
 void render_file(String file_name);

src/lib/functionlib.h

@@ -16,6 +16,31 @@ String join(StringList l, String delim = "\n");
 String nibble(String& haystack, String delim);
 void json_consume_space(String s, u32& i);
 
+String to_string(StringList l) {
+	String result;
+	u32 i = 0;
+	for(auto& s : l)
+	{
+		if(i > 0)
+			result.append("\n");
+		result.append(s);
+		i += 1;
+	}
+	return(result);
+}
+
+String to_string(SharedUnit* u) {
+	String result;
+
+	result += String("SharedUnit( \n")+
+		"Source:"+(u->file_name)+"\n"+
+		"SharedObject:"+(u->so_name)+"\n"+
+		"API:"+(u->api_file_name)+"\n"+
+		to_string(u->api_declarations);
+
+	return(result);
+}
+
 template <typename ITYPE>
 String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1)
 {

src/lib/mysql-connector.cpp

@@ -5,13 +5,13 @@
 
 bool MySQL::connect(String host, String username, String password)
 {
-	switch_to_system_alloc();
+	//switch_to_system_alloc();
 	connection = mysql_init(NULL);
 	if (connection == NULL)
 	{
 		auto e = mysql_error((MYSQL*)connection);
 		fprintf(stderr, "%s\n", e);
-		switch_to_arena(context->mem);
+		//switch_to_arena(context->mem);
 		statement_info.assign(e);
 		return(false);
 	}
@@ -22,7 +22,7 @@ bool MySQL::connect(String host, String username, String password)
 		auto e = mysql_error((MYSQL*)connection);
 		fprintf(stderr, "%s\n", e);
 		mysql_close((MYSQL*)connection);
-		switch_to_arena(context->mem);
+		//switch_to_arena(context->mem);
 		statement_info.assign(e);
 		return(false);
 	}
@@ -35,7 +35,7 @@ bool MySQL::connect(String host, String username, String password)
 		exit(1);
 	}
 	*/
-	switch_to_arena(context->mem);
+	//switch_to_arena(context->mem);
 	statement_info = String("connected");
 	context->resources.mysql_connections.push_back(connection);
 	return(true);
@@ -297,8 +297,8 @@ String MySQL::error()
 
 void cleanup_mysql_connections()
 {
-	switch_to_system_alloc();
+	//switch_to_system_alloc();
 	for(auto& con : context->resources.mysql_connections)
 		mysql_close((MYSQL*)con);
-	switch_to_arena(context->mem);
+	//switch_to_arena(context->mem);
 }

src/lib/sys.cpp

@@ -491,6 +491,20 @@ pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout)
 	}
 }
 
+#include <unistd.h>
+pid_t task_repeat(String key, u64 interval, std::function<void()> exec_after_spawn, u64 timeout)
+{
+	auto repeater_function = [&]() {
+		while (true)
+		{
+			exec_after_spawn();
+			printf("(P) worker process '%s' sleeping\n", key.c_str());
+			usleep((s64)(interval*1000000));
+		}
+	};
+	return(task(key, repeater_function, timeout));
+}
+
 void on_child_exit(int sig)
 {
     pid_t pid;

src/lib/sys.h

@@ -51,4 +51,5 @@ pid_t my_pid = 0;
 void on_segfault(int sig);
 
 pid_t task(String key, std::function<void()> exec_after_spawn, u64 timeout = 60*10);
+pid_t task_repeat(String key, f64 interval, std::function<void()> exec_after_spawn, u64 timeout = 60*10);
 pid_t task_pid(String key);

src/lib/types.h

@@ -4,6 +4,7 @@
 #include <vector>
 #include <functional>
 #include <sstream>
+#include <atomic>
 
 typedef unsigned char u8;
 typedef signed char s8;
@@ -43,96 +44,6 @@ String operator+(String lhs, f32 rhs) {
 }
 
 #define DEBUG_MEMORY_OFF
-#define NO_GLOBAL_ARENA_ALLOCATOR
-
-struct MemoryArena {
-
-	u8* data;
-	u64 size = 0;
-	u64 capacity = 0;
-	String name = "unnamed";
-
-	MemoryArena(u64 cap, String _name = "unnamed")
-	{
-		name = _name;
-		capacity = cap;
-		printf("(i) memory arena '%s' created with capacity of %llu bytes\n", name.c_str(), capacity);
-		data = (u8*)malloc(cap);
-	}
-
-	~MemoryArena()
-	{
-		free(data);
-	}
-
-	void clear()
-	{
-		#ifdef DEBUG_MEMORY
-		printf("(i) memory arena '%s' cleared after high mark of %llu bytes\n", name.c_str(), size);
-		#endif
-		size = 0;
-	}
-
-	void* get(u64 size_needed)
-	{
-		u64 size_aligned = 8 + (8 * ((size_needed) / 8));
-		u8* result = data + size;
-		if(size_aligned + size >= capacity)
-		{
-			printf("(!) memory arena '%s' capacity (%llu) exceeded %llu/%llu + %llu >= %llu\n",
-				name.c_str(), capacity, size_needed, size_aligned, size, capacity);
-			return(0);
-		}
-		size += size_aligned;
-		#ifdef DEBUG_MEMORY_DETAILED
-		printf("(i) memory arena '%s' [+%llu]:%p alloc %llu/%llu bytes\n", name.c_str(), size, result, size_needed, size_aligned);
-		#endif
-		return(result);
-	}
-
-};
-
-MemoryArena* current_memory_arena = 0;
-
-void switch_to_system_alloc()
-{
-#ifdef GLOBAL_ARENA_ALLOCATOR
-	current_memory_arena = 0;
-#endif
-}
-
-void switch_to_arena(MemoryArena* a)
-{
-#ifdef GLOBAL_ARENA_ALLOCATOR
-	current_memory_arena = a;
-#endif
-}
-
-#ifdef GLOBAL_ARENA_ALLOCATOR
-void * operator new(decltype(sizeof(0)) n) noexcept(false)
-{
-	if(current_memory_arena)
-	{
-		return(current_memory_arena->get(n));
-	}
-	else
-	{
-		return(malloc(n));
-	}
-}
-
-void operator delete(void * p) throw()
-{
-	if(current_memory_arena)
-	{
-
-	}
-	else
-	{
-		free(p);
-	}
-}
-#endif
 
 typedef std::map<String, String> StringMap;
 typedef std::vector<String> StringList;
@@ -146,26 +57,6 @@ typedef void (*request_handler)(Request* request);
 
 String to_string(s64 v) { return(std::to_string(v)); }
 
-/*struct ServerSettings {
-
-	String BIN_DIRECTORY = "/tmp/uce/work";
-	String COMPILE_SCRIPT = "scripts/compile";
-	String SETUP_TEMPLATE = "scripts/setup.h.template";
-	String LIT_ESC = "3d5b5_1";
-	String CONTENT_TYPE = "text/html; charset=utf-8";
-	String SOCKET_PATH = "/run/uce.sock";
-	String TMP_UPLOAD_PATH = "/tmp/uce/uploads";
-	String SESSION_PATH = "/tmp/uce/sessions";
-	String COMPILER_SYS_PATH = ".";
-	String PRECOMPILE_FILES_IN = ".";
-
-	u32 LISTEN_PORT = 9993;
-	u64 SESSION_TIME = 60*60*24*30;
-	u32 WORKER_COUNT = 4;
-	u32 MAX_MEMORY = 1024*1024*16;
-
-};*/
-
 struct SharedUnit {
 
 	String file_name;
@@ -244,8 +135,6 @@ struct Request {
 	u64 random_seed;
 	u64 random_index;
 
-	MemoryArena* mem;
-
 	String in;
 	std::vector<std::ostringstream*> ob_stack;
 	std::ostringstream* ob;
@@ -263,6 +152,8 @@ struct Request {
 		f64 time_init;
 		f64 time_start;
 		f64 time_end;
+		u64 mem_high;
+		u64 mem_alloc;
 		u32 invoke_count = 0;
 	} stats;
 
@@ -301,3 +192,20 @@ String concat(Ts... args)
     return(out.str());
 }
 
+void * operator new(decltype(sizeof(0)) n) noexcept(false)
+{
+	void* ptr = malloc(n);
+	if(context)
+	{
+		context->stats.mem_alloc += n;
+		if(context->stats.mem_alloc > context->stats.mem_high)
+			context->stats.mem_high = context->stats.mem_alloc;
+	}
+	return(ptr);
+}
+
+void operator delete(void * p) throw()
+{
+	//TO DO: track deallocations
+	free(p);
+}

src/linux_fastcgi.cpp

@@ -1,7 +1,6 @@
 #include "lib/uce_lib.cpp"
 
 ServerState server_state;
-MemoryArena* request_arena = 0;
 
 #include "fastcgi/src/fcgicc.cc"
 
@@ -44,6 +43,8 @@ int handle_complete(FastCGIRequest& request) {
 	server_state.request_count += 1;
 	request.server = &server_state;
 	request.stats.time_start = microtime();
+	//request.stats.mem_alloc = 0;
+	//request.stats.mem_high = 0;
     request.header["Content-Type"] = context->server->config["CONTENT_TYPE"];
     request.get = parse_query(request.params["QUERY_STRING"]);
     request.random_index = 0;
@@ -102,15 +103,8 @@ void listen_for_connections()
 	server.on_request = &handle_request;
 	server.on_data = &handle_data;
 	server.on_complete = &handle_complete;
-	/*
-	server.request_handler(&handle_request);
-	server.data_handler(&handle_data);
-	server.complete_handler(&handle_complete);
-	*/
 	for(;;)
 	{
-		//if(request_arena) request_arena->clear();
-		//current_memory_arena = request_arena;
 		server.process();
 	}
 }
@@ -143,7 +137,6 @@ void init_base_process()
 	mkdir(server_state.config["TMP_UPLOAD_PATH"]);
 	mkdir(server_state.config["SESSION_PATH"]);
 
-	request_arena = new MemoryArena(int_val(server_state.config["MAX_MEMORY"]), "request");
 	signal(SIGCHLD, on_child_exit);
 	srand(time());
 }

test/index.uce

@@ -29,7 +29,8 @@ RENDER()
 			<li><a href="shell.uce">Shell</a></li>
 			<li><a href="file_append.uce">File Append</a></li>
 			<li><a href="random.uce">RNG/Noise</a></li>
-			<li><a href="task.uce">Task API</a></li>
+			<li><a href="task.uce">Task</a></li>
+			<li><a href="task_repeat.uce">Task repeat</a></li>
 			<li><a href="utf8.uce">UTF-8</a></li>
 			<li><a href="call_file.uce">call_file()</a></li>
 		</ul>

test/post.uce

@@ -1,5 +1,5 @@
 
-void show_form()
+EXPORT void show_form()
 {
 
 	<><form action="?" method="post">

test/script-example1.usp

@@ -1,6 +1,17 @@
 v1 : string = "";
 
-v2 = 'BLA';
+v2 := 'BLA';
+i1 := 123.2;
 
-for(v3 => c);
+for(v3 => c)
 	print(c);
+
+v1 = "This is an interpolated string {=i1}
+	which also contains line breaks";
+
+?><div>
+	It's now time for some HTML!
+	</div>
+<?
+
+print("And, we're back!");

test/script.h

@@ -0,0 +1 @@
+

test/script.uce

@@ -1,332 +1,114 @@
+enum TokenType{ TT_UNKNOWN, TT_STRING, TT_INT, TT_FLOAT, TT_IDENTIFIER, TT_OPERATOR };
+String TokenTypeName[] = { "TT_UNKNOWN", "TT_STRING", "TT_INT", "TT_FLOAT", "TT_IDENTIFIER", "TT_OPERATOR" };
 
-String add_token_type(String tokenstr)
+struct Token
 {
-	char c = tokenstr[0];
+	String literal = "";
-	if(c == 34) // quote character, as number here because of HTML parser bug
+	TokenType type = TT_UNKNOWN;
-		return(String("Q") + tokenstr);
+	char delim = ' ';
-	if(isalpha(c))
+};
-		return(String("I") + tokenstr);
-	if(isdigit(c))
-		return(String("N") + tokenstr);
-	if(ispunct(c))
-		return(String("O") + tokenstr);
-	return(String("?") + tokenstr);
-}
 
-StringList script_tokenize(String code)
+#define NEW_TOKEN { result.push_back(ctok); ctok = new Token(); goto eval_char; }
+
+std::vector<Token*> script_tokenize(String code)
 {
-	StringList result;
+	std::vector<Token*> result;
-	String current_token = "";
+	auto ctok = new Token();
-	char mode = 'N';
+
-	char quote_character = 0;
-	char last_character = 0;
 	for(auto c : code)
 	{
-		if(mode == 'Q')
+		eval_char:
-		{
+		if(ctok->type == TT_UNKNOWN)
-			if(c == '\\')
-			{
-
-			}
-			else if(quote_character == c && last_character != '\\')
-			{
-				mode = 'N';
-				result.push_back(add_token_type(current_token));
-				current_token = "";
-			}
-			else
-			{
-				current_token.append(1, c);
-			}
-		}
-		else
 		{
 			if(isspace(c))
 			{
-				if(current_token != "")
+
-					result.push_back(add_token_type(current_token));
+			}
-				current_token = "";
+			else if(isdigit(c))
+			{
+				ctok->type = TT_INT;
+			}
+			else if(isalpha(c))
+			{
+				ctok->type = TT_IDENTIFIER;
 			}
 			else if(c == '"' || c == '\'')
 			{
-				mode = 'Q';
+				ctok->type = TT_STRING;
-				quote_character = c;
+				ctok->delim = c;
-				if(current_token != "")
+				continue;
-					result.push_back(add_token_type(current_token));
-				current_token = "\"";
 			}
-			else if(ispunct(c) && c != '_')
+			else if(ispunct(c))
 			{
-				if(current_token != "")
+				ctok->type = TT_OPERATOR;
-					result.push_back(add_token_type(current_token));
-				current_token = "";
-				result.push_back(add_token_type(String("").append(1, c)));
 			}
-			else
+		}
+		switch(ctok->type)
+		{
+			case(TT_IDENTIFIER):
 			{
-				current_token.append(1, c);
+				if(isalnum(c) || c == '_')
+					ctok->literal += c;
+				else
+					NEW_TOKEN
+				break;
 			}
-		}
+			case(TT_INT):
-		last_character = c;
-	}
-	if(current_token != "")
-		result.push_back(add_token_type(current_token));
-	return(result);
-}
-
-struct ASTNode
-{
-	String identifier = "";
-	String type_name = "";
-	String string_literal = "";
-	char type = 0;
-	std::vector<ASTNode*> assign;
-	std::vector<ASTNode*> call_params;
-	std::vector<ASTNode*> children;
-
-	ASTNode(char _type)
-	{
-		identifier = "";
-		type_name = "";
-		string_literal = "";
-		type = _type;
-	}
-
-	String to_string(String indent = "")
-	{
-		String result = "";
-		result += indent + String().append(1, type);
-		if(identifier != "")
-			result += " ident=" + identifier;
-		if(type_name != "")
-			result += " type_name=" + type_name;
-		if(string_literal != "")
-			result += " " + string_literal;
-		result += "\n";
-		for(auto callp : call_params)
-			result += indent + "  CALLP " + callp->to_string(indent + "  ");
-		for(auto assign_expr : assign)
-			result += indent + "  ASSIGN VALUE " + assign_expr->to_string(indent + "  ");
-		for(auto child : children)
-			result += indent + "  " + child->to_string(indent + "  ");
-		return(result);
-	}
-
-};
-
-struct ParserState
-{
-
-	ASTNode* root;
-	StringList tokens;
-	s32 token_index = 0;
-	String error = "";
-	u64 error_location = 0;
-
-	String consume(char token_kind = 0)
-	{
-		if(error_location > 0) return("");
-		if(token_kind && look_ahead(0)[0] != token_kind)
-		{
-			unexpected_token();
-		}
-		printf("consume #%i %s -> %s \n", token_index, look_ahead(0).c_str(), look_ahead(1).c_str());
-		String tk = look_ahead(0);
-		token_index++;
-		return(tk);
-	}
-
-	String look_ahead(s32 by)
-	{
-		if(error_location > 0)
-			return("_");
-		if(token_index+by < 0 || token_index+by >= tokens.size())
-			return("_");
-		return(tokens[token_index+by]);
-	}
-
-	bool more()
-	{
-		if(error_location > 0)
-			return(false);
-		return(token_index < tokens.size());
-	}
-
-	void unexpected_token(s32 offset = 0)
-	{
-		if(error_location) return;
-		String found = look_ahead(offset);
-		error_location = token_index;
-		token_index = tokens.size() + 1;
-		error = String("unexpected '") + found.substr(1) + String("' found");
-	}
-
-	bool expect(String token_str, String in_mode)
-	{
-		if(error_location) return(false);
-		String found = look_ahead(0);
-		if(found == token_str)
-		{
-			consume();
-			return(true);
-		}
-		error_location = token_index;
-		token_index = tokens.size() + 1;
-		error = String("'") + token_str.substr(1) + "' expected but '" + found.substr(1) + "' found in "+in_mode.substr(1);
-		return(false);
-	}
-
-};
-
-ASTNode* script_parse_expression(ParserState* state, String delim1, String delim2);
-ASTNode* script_parse_declaration(ParserState* state, String delim1, String delim2);
-
-ASTNode* script_parse_declaration(ParserState* state, String delim1, String delim2)
-{
-	ASTNode* result = new ASTNode('D');
-
-	String ident = state->consume();
-	if(ident[0] != 'I')
-	{
-		state->unexpected_token(-1);
-		return(result);
-	}
-	result->identifier = ident.substr(1);
-
-	if(!state->expect("O:", "declaration"))
-		return(result);
-
-	String n0 = state->look_ahead(0);
-	if(n0 == "O=")
-	{
-		result->type_name = "auto";
-		result->assign.push_back(script_parse_expression(state, delim1, delim2));
-		return(result);
-	}
-	else
-	{
-		result->type_name = state->consume('I').substr(1);
-		String n2 = state->look_ahead(0);
-		if(n2 == delim1 || n2 == delim2)
-		{
-			state->consume();
-			return(result);
-		}
-		else if(n2 == "O=")
-		{
-			state->expect("O=", "declaration");
-			result->assign.push_back(script_parse_expression(state, delim1, delim2));
-			return(result);
-		}
-		else
-		{
-			state->unexpected_token();
-			return(result);
-		}
-	}
-
-	return(result);
-}
-
-ASTNode* parse_single_token(ParserState* state)
-{
-	ASTNode* result = new ASTNode('?');
-
-	String n = state->consume();
-	if(n[0] == 'Q')
-	{
-		result->type = 'S';
-		result->string_literal = n.substr(1);
-	}
-	else if(n[0] == 'N')
-	{
-		result->type = 'N';
-		result->string_literal = n.substr(1);
-	}
-	else if(n[0] == 'I')
-	{
-		result->type = 'I';
-		result->identifier = n.substr(1);
-	}
-	else
-	{
-		state->unexpected_token(-1);
-	}
-	return(result);
-}
-
-ASTNode* script_parse_assignment(ParserState* state, String delim1, String delim2)
-{
-	ASTNode* result = new ASTNode('A');
-	result->identifier = state->consume('I').substr(1);
-	state->expect("O=", "assignment");
-	result->assign.push_back(script_parse_expression(state, delim1, delim2));
-	return(result);
-}
-
-ASTNode* script_parse_expression(ParserState* state, String delim1, String delim2)
-{
-	ASTNode* result = new ASTNode('E');
-
-	String n = state->look_ahead(0);
-
-	if(n == "O(")
-	{
-		state->consume();
-		result->children.push_back(script_parse_expression(state, "O)", ""));
-	}
-	else if(n == delim1 || n == delim2)
-	{
-		state->consume();
-		return(result);
-	}
-	else
-	{
-		String n2 = state->look_ahead(1);
-		if(n[0] == 'I' && n2 == "O:")
-		{
-			return(script_parse_declaration(state, "O;", ""));
-		}
-		else if(n2 == delim1 || n2 == delim2)
-		{
-			// single component 'expression'
-			result = parse_single_token(state);
-			state->consume();
-			return(result);
-		}
-		else if(n[0] == 'I' && n2 == "O=")
-		{
-			result = script_parse_assignment(state, delim1, delim2);
-			n2 = state->look_ahead(0);
-			if(n2 != delim1 && n2 != delim2)
 			{
-				state->unexpected_token();
+				if(isdigit(c))
+					ctok->literal += c;
+				else if(c == '.')
+				{
+					ctok->type = TT_FLOAT;
+					ctok->literal += c;
+				}
+				else
+					NEW_TOKEN
+				break;
+			}
+			case(TT_FLOAT):
+			{
+				if(isdigit(c))
+					ctok->literal += c;
+				else
+					NEW_TOKEN
+				break;
+			}
+			case(TT_OPERATOR):
+			{
+				ctok->literal += c;
+				c = ' ';
+				NEW_TOKEN
+				break;
+			}
+			case(TT_STRING):
+			{
+				if(c == ctok->delim)
+				{
+					c = ' ';
+					NEW_TOKEN
+				}
+				else
+					ctok->literal += c;
+				break;
 			}
-			return(result);
-		}
-		else
-		{
-			state->unexpected_token();
-			state->error += " in expression";
 		}
 	}
 
+	result.push_back(ctok);
 	return(result);
 }
 
-ParserState* script_parse(StringList tokens)
+String to_string(std::vector<Token*> t)
 {
-	ParserState* state = new ParserState();
+	String result = "";
-	state->tokens = tokens;
+	for(auto& ti : t)
-	state->root = new ASTNode('M');
-
-	while(state->token_index < state->tokens.size())
 	{
-		state->root->children.push_back(script_parse_expression(state, "O;", ""));
+		result += concat(TokenTypeName[ti->type], "(", ti->literal, ") ");
 	}
-
+	return(result);
-	return(state);
 }
+// "
 
 RENDER()
 {
@@ -349,17 +131,9 @@ RENDER()
 		</form>
 		<pre><?
 
-			StringList tokens = script_tokenize(script_src);
+			auto tokens = script_tokenize(script_src);
-			//print(var_dump(tokens));
+			print(html_escape(to_string(tokens)));
 
-			ParserState* ps = script_parse(tokens);
-
-			if(ps->error_location > 0)
-			{
-				print(ps->error, " @ token ", 1+ps->error_location, "\n");
-			}
-
-			print(ps->root->to_string());
 
 		?></pre>
 		Params

test/sharedunit.uce

@@ -0,0 +1,7 @@
+
+RENDER()
+{
+	auto p = compiler_load_shared_unit(context, "post.uce");
+	if(p)
+		print(to_string(p));
+}

test/task.uce

@@ -34,11 +34,11 @@ RENDER()
 
 		<script>
 
-		setInterval(() => {
+		/*setInterval(() => {
 
 			load(document.getElementById('task-status'), 'task-status.uce?task-name=<?= uri_encode(task_name) ?>');
 
-		}, 200);
+		}, 200);*/
 
 		</script>
 

test/task_repeat.uce

@@ -0,0 +1,80 @@
+
+RENDER()
+{
+	DTree t;
+
+	String task_name = first(context->get["task-name"], "example-task");
+
+	<>
+		<link rel="stylesheet" href='style.css'></link>
+		<h1>
+			<a href="index.uce">UCE Test</a>:
+			Task repeat
+		</h1>
+
+		<script>
+
+		function load(target, url) {
+			var r = new XMLHttpRequest();
+			r.open("GET", url, true);
+			r.onreadystatechange = function () {
+				if (r.readyState != 4 || r.status != 200) return;
+				target.innerHTML = r.responseText;
+			};
+			r.send();
+		}
+		</script>
+
+		<form action="?">
+			Task name
+			<input type="text" name="task-name" value="<?= task_name ?>"/>
+			<input type="submit" name="cmd" value="Get Info"/>
+			<input type="submit" name="cmd" value="Execute"/>
+		</form>
+
+		<script>
+
+		/*setInterval(() => {
+
+			load(document.getElementById('task-status'), 'task-status.uce?task-name=<?= uri_encode(task_name) ?>');
+
+		}, 200);*/
+
+		</script>
+
+		Task Status
+
+		<pre id="task-status" style="white-space: pre-wrap"><?
+
+			print("Task Name: ", task_name, "\n");
+			print("Task ID: ", task_pid(task_name), "\n");
+			print("Task Running: ", task_pid(task_name) == 0 ? "no" : "yes", "\n");
+
+
+		?></pre>
+
+		<?
+		if(context->get["cmd"] == "Execute")
+		{
+			?>
+			Task Start
+
+			<pre style="white-space: pre-wrap"><?
+
+				print("New Task ID: ", task_repeat("example-task", 5, []() {
+
+					sleep(1);
+
+				}), "\n");
+
+				print("Task interval: 5 seconds");
+
+			?></pre><?
+		}
+		?>
+
+		Params
+		<pre><?= var_dump(context->get) ?></pre>
+	</>
+
+}