Unicode stuff

doc/index.uce

@@ -145,7 +145,7 @@ RENDER()
 					}
 					else
 					{
-						?><div><?= (s) ?></div><?
+						?><div><? print(s); ?></div><?
 					}
 				}
 			}

doc/pages/split_utf8.txt

@@ -1,15 +1,20 @@
 :sig
-StringList split_utf8(String str)
+StringList split_utf8(String str, bool compound_characters = false)
 
 
 :params
 str : string to be split
+compound_characters : optional, if true tries to combine compound characters
 return value : a list of Unicode characters
 
 :desc
 Splits the string 'str' into its constituent Unicode code points.
 
-This currently does not honor compound characters such as flags or composite emojis.
+If 'compound_characters' is true, split_utf8 will attempt to combine compound characters based on very simple rules:
+<li>combine characters if they're connected by a Zero-Width Joiner (ZWJ) character</li>
+<li>combine two characters if they're both a Regional Indicator Symbol Letter</li>
+<li>if a character is a Variation Selector, append it to the previous character</li>
+<li>in all other cases, characters remain on their own</li>
 
 :see
 >string

src/lib/compiler.cpp

@@ -167,7 +167,7 @@ String preprocess_shared_unit_char_wise(Request* context, SharedUnit* su, String
 					pc.resize(pc.length() - 4);
 					String declaration = trim(content.substr(i, end_declaration_pos - i));
 					String return_type_and_name = nibble(declaration, "(");
-					StringList rtn_list = split(return_type_and_name);
+					StringList rtn_list = split_space(return_type_and_name);
 					String fn_name = rtn_list.back(); rtn_list.pop_back();
 					su->api_declarations.push_back(fn_name + ":" + join(rtn_list, " ") + ":(" + declaration + "\n");
 					printf("declaration found: %s\n", declaration.c_str());

src/lib/functionlib.cpp

@@ -101,7 +101,7 @@ String trim(String raw)
 	return(raw.substr(start_pos, 1 + end_pos - start_pos));
 }
 
-StringList split(String str)
+StringList split_space(String str)
 {
 	StringList result;
 	String current_token = "";
@@ -154,7 +154,7 @@ String join(StringList l, String delim)
 	return(result);
 }
 
-StringList split_utf8(String s)
+StringList split_utf8(String s, bool compound_characters)
 {
 	StringList result;
 	auto len = s.size();
@@ -185,6 +185,50 @@ StringList split_utf8(String s)
 			result.push_back(String().append(1, c));
 		}
 	}
+	if(compound_characters)
+	{
+		StringList compound_result;
+		bool join_next = false;
+		bool last_was_regional = false;
+		for(auto& s : result)
+		{
+			if(join_next)
+			{
+				compound_result[compound_result.size()-1] += s;
+				join_next = false;
+			}
+			else if(s == "\xE2" "\x80" "\x8D") // ZWJ
+			{
+				compound_result[compound_result.size()-1] += s;
+				join_next = true;
+				last_was_regional = false;
+			}
+			else if(s[0] == '\xF0' && s[1] == '\x9F' && s[2] == '\x87' && s[3] >= '\xA6' && s[3] <= '\xBF') // Regional indicator letters
+			{
+				if(last_was_regional)
+				{
+					compound_result[compound_result.size()-1] += s;
+					last_was_regional = false;
+				}
+				else
+				{
+					compound_result.push_back(s);
+					last_was_regional = true;
+				}
+			}
+			else if(s[0] == '\xEF' && s[1] == '\xB8' && s[2] >= '\x80' && s[2] <= '\x8F') // Variation selector
+			{
+				compound_result[compound_result.size()-1] += s;
+				last_was_regional = false;
+			}
+			else
+			{
+				compound_result.push_back(s);
+				last_was_regional = false;
+			}
+		}
+		return(compound_result);
+	}
 	return(result);
 }
 

src/lib/functionlib.h

@@ -7,12 +7,22 @@ String to_upper(String s);
 String replace(String s, String search, String replace_with);
 
 String trim(String raw);
-StringList split(String str);
+StringList split_space(String str);
 StringList split(String str, String delim);
+StringList split_utf8(String s, bool compound_characters = false);
 String join(StringList l, String delim = "\n");
 String nibble(String& haystack, String delim);
 void json_consume_space(String s, u32& i);
-StringList split_utf8(String s);
+
+template <typename ITYPE>
+String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1)
+{
+    static const char* digits = "0123456789ABCDEF";
+    String rc(hex_len,'0');
+    for (size_t i=0, j=(hex_len-1)*4 ; i<hex_len; ++i,j-=4)
+        rc[i] = digits[(w>>j) & 0x0f];
+    return(rc);
+}
 
 template<typename T>
 std::vector<T> filter(std::vector<T> items, std::function<bool (T)> f)

src/lib/types.h

@@ -215,16 +215,6 @@ struct URI {
 
 String nibble(String div, String& haystack);
 
-template <typename ITYPE>
-String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1)
-{
-    static const char* digits = "0123456789ABCDEF";
-    String rc(hex_len,'0');
-    for (size_t i=0, j=(hex_len-1)*4 ; i<hex_len; ++i,j-=4)
-        rc[i] = digits[(w>>j) & 0x0f];
-    return(rc);
-}
-
 #include "dtree.h"
 
 void compiler_invoke(Request* context, String file_name, DTree& call_param);

src/lib/uce_lib.cpp

@@ -1,9 +1,9 @@
 
 
 #include "types.cpp"
-#include "hash.cpp"
 #include "dtree.cpp"
 #include "functionlib.cpp"
+#include "hash.cpp"
 #include "sys.cpp"
 #include "uri.cpp"
 #include "compiler.cpp"

test/utf8.uce

@@ -1,8 +1,18 @@
+String string_to_hex(String s)
+{
+	String result = "";
+	for(auto c : s)
+	{
+		result += to_hex(c);
+	}
+	return(result);
+}
+
 
 RENDER()
 {
-	
-	String raw = first(context->post["raw"], "■▧▲😂😆😏😱🇦🇽");
+
+	String raw = first(context->post["raw"], "■👪▧▲🏳️‍🌈😂👋🏽😆😏😱🇦🇽Udø島リZ̸̢̧̡̧̗̰̪͉̤͖͉̪̝̦͎̮̑͜a̸̧̝̱̹̲̗̪̰̦͒̃͋̿̿̃̈͑̐͑͗̚̕̚͝͠l͚̜͕̠̣ģ̸̧̘̜͇͚͈͙̓̌̅͑͊͊͋̓́͌̈́̿̈́͗͘̚ͅͅȏ̶̗̤̳͎̫̥͕̣͔̥̙̜̰̂͌̍͊͂́̅̇̒̕̕ルイ社もなく");
 
 	<>
 		<link rel="stylesheet" href='style.css'></link>
@@ -19,12 +29,29 @@ RENDER()
 				<input type="submit" value="Submit Form"/>
 			</div>
 		</form>
+		Simple Characters
 		<pre><?
 
 		u32 item_idx = 0;
 		for(auto seg : split_utf8(raw))
 		{
-			print(item_idx++, " : ", seg, "\n");
+			item_idx++;
+			print(string_to_hex(seg), " ", seg, "\t");
+			if(item_idx % 4 == 0)
+				print("\n");
+		}
+
+		?></pre>
+		Compound Characters
+		<pre><?
+
+		item_idx = 0;
+		for(auto seg : split_utf8(raw, true))
+		{
+			item_idx++;
+			print(string_to_hex(seg), " ", seg, "\t");
+			if(item_idx % 4 == 0)
+				print("\n");
 		}
 
 		?></pre>