Unicode stuff

src/lib/compiler.cpp

@@ -167,7 +167,7 @@ String preprocess_shared_unit_char_wise(Request* context, SharedUnit* su, String
 					pc.resize(pc.length() - 4);
 					String declaration = trim(content.substr(i, end_declaration_pos - i));
 					String return_type_and_name = nibble(declaration, "(");
-					StringList rtn_list = split(return_type_and_name);
+					StringList rtn_list = split_space(return_type_and_name);
 					String fn_name = rtn_list.back(); rtn_list.pop_back();
 					su->api_declarations.push_back(fn_name + ":" + join(rtn_list, " ") + ":(" + declaration + "\n");
 					printf("declaration found: %s\n", declaration.c_str());

src/lib/functionlib.cpp

@@ -101,7 +101,7 @@ String trim(String raw)
 	return(raw.substr(start_pos, 1 + end_pos - start_pos));
 }
 
-StringList split(String str)
+StringList split_space(String str)
 {
 	StringList result;
 	String current_token = "";
@@ -154,7 +154,7 @@ String join(StringList l, String delim)
 	return(result);
 }
 
-StringList split_utf8(String s)
+StringList split_utf8(String s, bool compound_characters)
 {
 	StringList result;
 	auto len = s.size();
@@ -185,6 +185,50 @@ StringList split_utf8(String s)
 			result.push_back(String().append(1, c));
 		}
 	}
+	if(compound_characters)
+	{
+		StringList compound_result;
+		bool join_next = false;
+		bool last_was_regional = false;
+		for(auto& s : result)
+		{
+			if(join_next)
+			{
+				compound_result[compound_result.size()-1] += s;
+				join_next = false;
+			}
+			else if(s == "\xE2" "\x80" "\x8D") // ZWJ
+			{
+				compound_result[compound_result.size()-1] += s;
+				join_next = true;
+				last_was_regional = false;
+			}
+			else if(s[0] == '\xF0' && s[1] == '\x9F' && s[2] == '\x87' && s[3] >= '\xA6' && s[3] <= '\xBF') // Regional indicator letters
+			{
+				if(last_was_regional)
+				{
+					compound_result[compound_result.size()-1] += s;
+					last_was_regional = false;
+				}
+				else
+				{
+					compound_result.push_back(s);
+					last_was_regional = true;
+				}
+			}
+			else if(s[0] == '\xEF' && s[1] == '\xB8' && s[2] >= '\x80' && s[2] <= '\x8F') // Variation selector
+			{
+				compound_result[compound_result.size()-1] += s;
+				last_was_regional = false;
+			}
+			else
+			{
+				compound_result.push_back(s);
+				last_was_regional = false;
+			}
+		}
+		return(compound_result);
+	}
 	return(result);
 }
 

src/lib/functionlib.h

@@ -7,12 +7,22 @@ String to_upper(String s);
 String replace(String s, String search, String replace_with);
 
 String trim(String raw);
-StringList split(String str);
+StringList split_space(String str);
 StringList split(String str, String delim);
+StringList split_utf8(String s, bool compound_characters = false);
 String join(StringList l, String delim = "\n");
 String nibble(String& haystack, String delim);
 void json_consume_space(String s, u32& i);
-StringList split_utf8(String s);
+
+template <typename ITYPE>
+String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1)
+{
+    static const char* digits = "0123456789ABCDEF";
+    String rc(hex_len,'0');
+    for (size_t i=0, j=(hex_len-1)*4 ; i<hex_len; ++i,j-=4)
+        rc[i] = digits[(w>>j) & 0x0f];
+    return(rc);
+}
 
 template<typename T>
 std::vector<T> filter(std::vector<T> items, std::function<bool (T)> f)

src/lib/types.h

@@ -215,16 +215,6 @@ struct URI {
 
 String nibble(String div, String& haystack);
 
-template <typename ITYPE>
-String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1)
-{
-    static const char* digits = "0123456789ABCDEF";
-    String rc(hex_len,'0');
-    for (size_t i=0, j=(hex_len-1)*4 ; i<hex_len; ++i,j-=4)
-        rc[i] = digits[(w>>j) & 0x0f];
-    return(rc);
-}
-
 #include "dtree.h"
 
 void compiler_invoke(Request* context, String file_name, DTree& call_param);

src/lib/uce_lib.cpp

@@ -1,9 +1,9 @@
 
 
 #include "types.cpp"
-#include "hash.cpp"
 #include "dtree.cpp"
 #include "functionlib.cpp"
+#include "hash.cpp"
 #include "sys.cpp"
 #include "uri.cpp"
 #include "compiler.cpp"