shell stuff, preprocessor directive

doc/areas/string.txt

@@ -5,6 +5,7 @@ first
 join
 nibble
 split
-str_to_lower
+replace
-str_to_upper
+to_lower
+to_upper
 trim

doc/index.uce

@@ -43,18 +43,26 @@ RENDER()
 			for(auto file_name : ls("pages/"))
 			{
 				String ft = nibble(file_name, ".");
-				if(ft[1] == '_')
+				if(ft.substr(0, 2) == "0_")
 				{
 					String fn = ft;
 					String pre = nibble(fn, "_");
 					?>
-					<div><a href="?p=<?= ft ?>"><?= fn ?></a><span style="opacity:0.5"> : struct</span></div>
+					<div><a href="?p=<?= uri_encode(ft) ?>"><?= fn ?></a><span style="opacity:0.5"> : struct</span></div>
+					<?
+				}
+				else if(ft.substr(0, 2) == "1_")
+				{
+					String fn = ft;
+					String pre = nibble(fn, "_");
+					?>
+					<div><a href="?p=<?= uri_encode(ft) ?>"><?= fn ?></a><span style="opacity:0.5"> : directive</span></div>
 					<?
 				}
 				else
 				{
 					?>
-					<div><a href="?p=<?= ft ?>"><?= ft ?><span style="opacity:0.5">()</span></a></div>
+					<div><a href="?p=<?= uri_encode(ft) ?>"><?= ft ?><span style="opacity:0.5">()</span></a></div>
 					<?
 				}
 			}

doc/pages/1_#load.txt

@@ -0,0 +1,9 @@
+:sig
+#load "myfile.uce"
+
+:params
+file name : name of an UCE file that should be included
+
+:desc
+Includes another UCE file
+

doc/pages/replace.txt

@@ -0,0 +1,14 @@
+:sig
+String replace(String s, String search, String replace_with)
+
+:params
+s : the string where replacements should happen
+search : the string that should be searched for
+replace_with : the string that should appear in places where 'search' occurs
+return value : a version of 's' where all instances of 'search' have been replaced with 'replace_with'
+
+:desc
+Replace all occurrences of 'search' with the string defined in 'replace_with'.
+
+:see
+>string

doc/pages/split.txt

@@ -1,9 +1,11 @@
 :sig
-StringList split(String str, String delim = "\n")
+StringList split(String str, String delim)
+StringList split(String str)
+
 
 :params
 str : string to be split
-delim : delimiter (defaults to newline character)
+delim : optional, delimiter (if omitted, any sequence of whitespace characters will count as a delimiter)
 return value : a list of strings
 
 :desc

doc/pages/to_lower.txt

@@ -1,5 +1,5 @@
 :sig
-String str_to_lower(String s)
+String to_lower(String s)
 
 :params
 s : the string to be converted

doc/pages/to_upper.txt

@@ -1,5 +1,5 @@
 :sig
-String str_to_upper(String s)
+String to_upper(String s)
 
 :params
 s : the string to be converted

examples/blog/index.uce

@@ -0,0 +1,9 @@
+#load "lib.uce"
+
+RENDER()
+{
+	<>
+	Hello world2
+	<?= bla("!x") ?>
+	</>
+}

examples/blog/lib.uce

@@ -0,0 +1,5 @@
+
+String bla(String x)
+{
+	return(x + "!");
+}

scripts/setup.h.template

@@ -0,0 +1,8 @@
+/*load_declarations*/
+
+extern "C" void set_current_request(Request* _request)
+{
+	context = _request;
+	signal(SIGSEGV, on_segfault);
+	/*load_units*/
+}

src/lib/compiler.cpp

@@ -87,18 +87,19 @@ String process_html_literal(Request* context, SharedUnit* su, String content)
 	return(HT_START + pc + HT_END);
 }
 
-String preprocess_shared_unit(Request* context, SharedUnit* su)
+String preprocess_shared_unit_char_wise(Request* context, SharedUnit* su, String content)
 {
-	String content = file_get_contents(su->file_name);
+	String pc = "#include \"" + su->src_file_name + ".setup.h" + "\"\n";
-	printf("(c) compiling with root dir %s\n", context->server->config.COMPILER_SYS_PATH.c_str());
-	String pc = ("#include \"")+context->server->config.COMPILER_SYS_PATH +"/src/lib/uce_lib.h\" \n";
 	String token = "";
 	String html_buffer = "";
 	u8 mode = 0;
 	bool inside_quote = false;
-	for(u32 i = 0; i < content.length(); i++)
+	u32 source_length = content.length();
+	String current_line = "";
+	for(u32 i = 0; i < source_length; i++)
 	{
 		char c = content[i];
+		current_line.append(1, c);
 		if(mode == 2)
 		{
 			auto end_pos = content.find(String("</")+token+">", i);
@@ -142,14 +143,54 @@ String preprocess_shared_unit(Request* context, SharedUnit* su)
 			inside_quote = !inside_quote;
 			pc.append(1, c);
 		}
-		else
+		else // we're in C++ code here
 		{
 			pc.append(1, c);
+			if(c == 10 && current_line.substr(0, 6) == "#load ")
+			{
+				//printf("#load directive\n");
+				pc.resize(pc.length() - current_line.length());
+				nibble(current_line, "\"");
+				String unit_name = nibble(current_line, "\"");
+				SharedUnit* sub_su = compiler_load_shared_unit(context, unit_name, su->src_path, true);
+				if(sub_su)
+				{
+					pc.append("#include \"" + sub_su->pre_file_name + "\"");
+				}
+			}
+			else if(current_line.length() == 4 && current_line.substr(0, 4) == "API ")
+			{
+				auto end_declaration_pos = content.find("{", i);
+				if(end_declaration_pos != std::string::npos)
+				{
+					// remove string "API " from output
+					pc.resize(pc.length() - 4);
+					String declaration = trim(content.substr(i, end_declaration_pos - i));
+					String return_type_and_name = nibble(declaration, "(");
+					StringList rtn_list = split(return_type_and_name);
+					String fn_name = rtn_list.back(); rtn_list.pop_back();
+					su->api_declarations.push_back(fn_name + ":" + join(rtn_list, " ") + ":(" + declaration + "\n");
+					printf("declaration found: %s\n", declaration.c_str());
+				}
+			}
 		}
+		if(c == 10)
+			current_line = "";
 	}
 	return(pc);
 }
 
+String preprocess_shared_unit(Request* context, SharedUnit* su)
+{
+	return(
+		preprocess_shared_unit_char_wise(
+			context,
+			su,
+			file_get_contents(su->file_name)
+		)
+	);
+}
+
 void setup_unit_paths(Request* context, SharedUnit* su, String file_name)
 {
 	su->file_name = file_name;
@@ -166,6 +207,8 @@ void setup_unit_paths(Request* context, SharedUnit* su, String file_name)
 	su->pre_file_name = su->src_file_name + ".cpp";
 
 	su->so_name = su->bin_path + "/" + su->bin_file_name;
+	su->api_file_name = su->bin_path + "/" + su->src_file_name + ".api.txt";
+	su->setup_file_name = su->bin_path + "/" + su->src_file_name + ".setup.h";
 }
 
 void load_shared_unit(Request* context, SharedUnit* su, String file_name)
@@ -178,6 +221,8 @@ void load_shared_unit(Request* context, SharedUnit* su, String file_name)
 
 	if(!file_exists(su->so_name))
 	{
+		if(su->opt_so_optional)
+			return;
 		printf("(i) unit file not found: %s\n", su->so_name.c_str());
 		su->compiler_messages = "unit file not found";
 		return;
@@ -201,6 +246,22 @@ void load_shared_unit(Request* context, SharedUnit* su, String file_name)
 	}
 }
 
+String compile_setup_file(Request* context, SharedUnit* su)
+{
+	String result =
+		String("#ifndef UCE_LIB_INCLUDED\n") +
+		"#define UCE_LIB_INCLUDED\n" +
+		("#include \"")+context->server->config.COMPILER_SYS_PATH +"/src/lib/uce_lib.h\" \n"+
+		file_get_contents(context->server->config.SETUP_TEMPLATE) +
+		"#endif \n";
+	StringList declarations;
+	StringList load_units;
+
+	result = replace(result, "/*load_declarations*/", join(declarations, "\n"));
+	result = replace(result, "/*load_units*/", join(load_units, "\n"));
+	return(result);
+}
+
 void compile_shared_unit(Request* context, SharedUnit* su, String file_name)
 {
 	//setup_unit_paths(context, su, file_name);
@@ -213,16 +274,19 @@ void compile_shared_unit(Request* context, SharedUnit* su, String file_name)
 
 	shell_exec("mkdir -p " + shell_escape(su->pre_path));
 	file_put_contents(su->pre_path + "/" + su->pre_file_name, preprocess_shared_unit(context, su));
+	file_put_contents(su->setup_file_name, compile_setup_file(context, su));
+	file_put_contents(su->api_file_name, join(su->api_declarations, "\n"));
 
-	printf("Config.COMPILE_SCRIPT %s\n", String(su->pre_path + "/" + su->pre_file_name).c_str());
+	//printf("Config.COMPILE_SCRIPT %s\n", String(su->pre_path + "/" + su->pre_file_name).c_str());
 
-	su->compiler_messages = trim(shell_exec(context->server->config.COMPILE_SCRIPT+" "+
+	if(!su->opt_so_optional)
-		shell_escape(su->src_path)+" "+
+		su->compiler_messages = trim(shell_exec(context->server->config.COMPILE_SCRIPT+" "+
-		shell_escape(su->bin_path)+" "+
+			shell_escape(su->src_path)+" "+
-		shell_escape(su->file_name)+" "+
+			shell_escape(su->bin_path)+" "+
-		shell_escape(su->pre_file_name)+" "+
+			shell_escape(su->file_name)+" "+
-		shell_escape(su->bin_file_name)
+			shell_escape(su->pre_file_name)+" "+
-	));
+			shell_escape(su->bin_file_name)
+		));
 
 	if(su->compiler_messages.length() > 0)
 	{
@@ -235,27 +299,35 @@ void compile_shared_unit(Request* context, SharedUnit* su, String file_name)
 	}
 }
 
-SharedUnit* get_shared_unit(Request* context, String file_name)
+SharedUnit* get_shared_unit(Request* context, String file_name, bool opt_so_optional)
 {
 	SharedUnit* su = context->server->units[file_name];
 	auto mod_time = file_mtime(file_name);
+	auto compiled_time = su ? file_mtime(su->so_name) : 0;
 	bool do_recompile = false;
-	if(su && (su->last_compiled < mod_time || mod_time == 0))
+	if(su && (compiled_time < mod_time || mod_time == 0))
 	{
 		delete su;
 		su = 0;
 		do_recompile = true;
 	}
+	else if(su && (su->last_compiled < mod_time))
+	{
+		delete su;
+		su = 0;
+		do_recompile = false;
+	}
 	if(!su)
 	{
 		su = new SharedUnit();
 		setup_unit_paths(context, su, file_name);
-		if(!do_recompile)
+		su->opt_so_optional = opt_so_optional;
+		if(compiled_time != 0)
 		{
 			// if we didn't decide to force recompile yet, we need to check
 			// (this case should only happen if the SU cache for that entry is cold
 			//  but the SO itself exists _AND_ is stale)
-			su->last_compiled = file_mtime(su->so_name);
+			su->last_compiled = compiled_time;
 			if(su->last_compiled < mod_time || mod_time == 0)
 				do_recompile = true;
 		}
@@ -301,25 +373,53 @@ void compiler_invoke(Request* context, String file_name, DTree& call_param)
 		printf("Error loading unit %s\n", file_name.c_str());
 		print("Error loading unit: "+file_name);
 	}
+	else if(su->compiler_messages.length() > 0)
+	{
+		context->header["Content-Type"] = "text/plain";
+		print(su->compiler_messages);
+	}
 	else if(!su->on_render)
 	{
 		context->header["Content-Type"] = "text/plain";
-		print("Compiler error: "+su->compiler_messages);
+		print("no RENDER() entry point");
 	}
 	else
 	{
-		if(su->compiler_messages.length() > 0)
+		String prev_wd = get_cwd();
-			print(su->compiler_messages);
+		set_cwd(su->src_path);
-		else
+		su->on_setup(context);
-		{
+		su->on_render(call_param);
-			String prev_wd = get_cwd();
+		set_cwd(prev_wd);
-			set_cwd(su->src_path);
-			su->on_setup(context);
-			su->on_render(call_param);
-			set_cwd(prev_wd);
-		}
 	}
 }
 
+SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String current_path, bool opt_so_optional)
+{
 
+	if(file_name[0] != '/')
+	{
+		file_name = expand_path(file_name, current_path);
+	}
+
+	switch_to_system_alloc();
+	auto su = get_shared_unit(context, file_name, opt_so_optional);
+	switch_to_arena(context->mem);
+	if(!su)
+	{
+		printf("Error loading unit %s\n", file_name.c_str());
+		print("Error loading unit: "+file_name);
+		return(0);
+	}
+	else if(su->compiler_messages.length() > 0)
+	{
+		context->header["Content-Type"] = "text/plain";
+		print(su->compiler_messages);
+		return(0);
+	}
+	else
+	{
+		return(su);
+	}
+
+}
 

src/lib/compiler.h

@@ -5,6 +5,6 @@ String preprocess_shared_unit(Request* context, SharedUnit* su);
 void setup_unit_paths(Request* context, SharedUnit* su, String file_name);
 void load_shared_unit(Request* context, SharedUnit* su, String file_name);
 void compile_shared_unit(Request* context, SharedUnit* su, String file_name);
-SharedUnit* get_shared_unit(Request* context, String file_name);
+SharedUnit* get_shared_unit(Request* context, String file_name, bool opt_so_optional = false);
 void compiler_invoke(Request* context, String file_name, DTree& call_param);
-
+SharedUnit* compiler_load_shared_unit(Request* context, String file_name, String current_path, bool opt_so_optional = false);

src/lib/functionlib.cpp

@@ -40,7 +40,7 @@ u8 hex_to_u8(String src)
 	return(char_to_u8(src[0])*16 + char_to_u8(src[1]));
 }
 
-String str_to_lower(String s)
+String to_lower(String s)
 {
 	String result = "";
 	for(auto c : s)
@@ -52,7 +52,7 @@ String str_to_lower(String s)
 	return(result);
 }
 
-String str_to_upper(String s)
+String to_upper(String s)
 {
 	String result = "";
 	for(auto c : s)
@@ -64,6 +64,27 @@ String str_to_upper(String s)
 	return(result);
 }
 
+String replace(String s, String search, String replace_with)
+{
+	s64 last_spos = 0;
+	auto spos = s.find(search);
+	if(spos == std::string::npos)
+		return(s);
+	String result = "";
+	auto slen = search.length();
+	while(spos != std::string::npos)
+	{
+		if(spos - last_spos > 0)
+			result.append(s.substr(last_spos, spos - last_spos));
+		result.append(replace_with);
+		last_spos = spos + slen;
+		spos = s.find(search, last_spos);
+	}
+	if(last_spos < s.length())
+		result.append(s.substr(last_spos));
+	return(result);
+}
+
 String trim(String raw)
 {
 	u32 len = raw.length();
@@ -80,6 +101,30 @@ String trim(String raw)
 	return(raw.substr(start_pos, 1 + end_pos - start_pos));
 }
 
+StringList split(String str)
+{
+	StringList result;
+	String current_token = "";
+	for(auto c : str)
+	{
+		if(isspace(c))
+		{
+			if(current_token != "")
+			{
+				result.push_back(current_token);
+				current_token = "";
+			}
+		}
+		else
+		{
+			current_token.append(1, c);
+		}
+	}
+	if(current_token != "")
+		result.push_back(current_token);
+	return(result);
+}
+
 StringList split(String str, String delim)
 {
 	StringList result;

src/lib/functionlib.h

@@ -2,11 +2,13 @@
 u8 char_to_u8(char input);
 u8 hex_to_u8(String src);
 u64 int_val(String s, u32 base = 10);
-String str_to_lower(String s);
+String to_lower(String s);
-String str_to_upper(String s);
+String to_upper(String s);
+String replace(String s, String search, String replace_with);
 
 String trim(String raw);
-StringList split(String str, String delim = "\n");
+StringList split(String str);
+StringList split(String str, String delim);
 String join(StringList l, String delim = "\n");
 String nibble(String& haystack, String delim);
 void json_consume_space(String s, u32& i);

src/lib/sys.cpp

@@ -32,7 +32,19 @@ String shell_exec(String cmd)
 String shell_escape(String raw)
 {
 	// FIXME
-	return("\"" + raw + "\"");
+	String result = "";
+	for(auto c : raw)
+	{
+		if(c == '\'')
+		{
+			result.append("'\\''");
+		}
+		else
+		{
+			result.append(1, c);
+		}
+	}
+	return("\'" + result + "\'");
 	/*
 	`	U+0060 (Grave Accent)	Backtick	Command substitution
 ~	U+007E	Tilde	Tilde expansion
@@ -164,11 +176,14 @@ void unlink(String file_name)
 	remove(file_name.c_str());
 }
 
-String expand_path(String path)
+String expand_path(String path, String relative_to_path)
 {
 	String result;
 
-	auto base_path = split(get_cwd(), "/");
+	if(relative_to_path == "")
+		relative_to_path = get_cwd();
+
+	auto base_path = split(relative_to_path, "/");
 	auto rel_path = split(path, "/");
 
 	for(auto& s : rel_path)

src/lib/sys.h

@@ -22,7 +22,7 @@ String get_cwd();
 void set_cwd(String path);
 time_t file_mtime(String file_name);
 void unlink(String file_name);
-String expand_path(String path);
+String expand_path(String path, String relative_to_path = "");
 StringList ls(String dir);
 
 f64 microtime();

src/lib/types.h

@@ -149,6 +149,7 @@ struct ServerSettings {
 
 	String BIN_DIRECTORY = "/tmp/uce/work";
 	String COMPILE_SCRIPT = "scripts/compile";
+	String SETUP_TEMPLATE = "scripts/setup.h.template";
 	String LIT_ESC = "3d5b5_1";
 	String CONTENT_TYPE = "text/html; charset=utf-8";
 	String SOCKET_PATH = "/run/uce.sock";
@@ -166,6 +167,10 @@ struct SharedUnit {
 
 	String file_name;
 	String so_name;
+	String api_file_name;
+	String setup_file_name;
+	StringList api_declarations;
+	std::map<String, void*> api_functions;
 
 	String src_path;
 	String bin_path;
@@ -182,6 +187,8 @@ struct SharedUnit {
 	String compiler_messages;
 	time_t last_compiled;
 
+	bool opt_so_optional = false;
+
 	~SharedUnit();
 };
 

src/lib/uce_lib.h

@@ -7,8 +7,4 @@
 #include "compiler.h"
 #include "mysql-connector.h"
 
-extern "C" void set_current_request(Request* _request)
+
-{
-	context = _request;
-	signal(SIGSEGV, on_segfault);
-}

test/index.uce

@@ -20,11 +20,13 @@ RENDER()
 			<li><a href="uri.uce">URI</a></li>
 			<li><a href="cookie.uce">Cookies</a></li>
 			<li><a href="session.uce">Session</a></li>
+			<li><a href="str_replace.uce">String replace</a></li>
 			<li><a href="dtree.uce">DTree</a></li>
 			<li><a href="json.uce">JSON</a></li>
 			<li><a href="memcached.uce">Memcached</a></li>
 			<li><a href="mysql.uce">MySQL Connector</a></li>
 			<li><a href="fileio.uce">File I/O</a></li>
+			<li><a href="shell.uce">Shell Stuff</a></li>
 			<li><a href="file_append.uce">File Append</a></li>
 			<li><a href="random.uce">RNG/Noise</a></li>
 			<li><a href="task.uce">Task API</a></li>

test/shell.uce

@@ -0,0 +1,32 @@
+
+
+RENDER()
+{
+
+	<><html>
+		<link rel="stylesheet" href='style.css?v=1'></link>
+		<h1>
+			<a href="index.uce">UCE Test</a>:
+			Shell Stuff
+		</h1>
+
+		<div>shell_exec()</div>
+
+		<pre><?
+
+			print(shell_exec("ls -l"));
+
+		?></pre>
+
+		<div>shell_escape()</div>
+
+		<pre><?
+
+			print(shell_exec("echo "+shell_escape("kasjdf 1ölkasj öflkjasdö\\lkfjsöa'ldkfj 23487692\"bla83746")));
+
+		?></pre>
+
+		<pre><?= var_dump(context->params) ?></pre>
+	</html></>
+
+}

test/str_replace.uce

@@ -0,0 +1,33 @@
+
+void str_replace_test(String s, String sr, String rp)
+{
+	print("in '", s, "' replace all '", sr, "' with '", rp, "' => ");
+	print(replace(s, sr, rp));
+}
+
+RENDER()
+{
+
+	<><html>
+		<link rel="stylesheet" href='style.css?v=1'></link>
+		<h1>
+			<a href="index.uce">UCE Test</a>:
+			String Replace
+		</h1>
+
+		<div>
+			<? str_replace_test("abcdefgh", "c", "C"); ?>
+		</div>
+
+		<div>
+			<? str_replace_test("abcccdefgh", "c", "C"); ?>
+		</div>
+
+		<div>
+			<? str_replace_test("abcccdefgh", "ccc", "C"); ?>
+		</div>
+
+		<pre><?= var_dump(context->params) ?></pre>
+	</html></>
+
+}