diff --git a/doc/index.uce b/doc/index.uce
index 482ae4b..70baf47 100644
--- a/doc/index.uce
+++ b/doc/index.uce
@@ -145,7 +145,7 @@ RENDER()
}
else
{
- ?>combine characters if they're connected by a Zero-Width Joiner (ZWJ) character
+combine two characters if they're both a Regional Indicator Symbol Letter
+if a character is a Variation Selector, append it to the previous character
+in all other cases, characters remain on their own
:see
>string
diff --git a/src/lib/compiler.cpp b/src/lib/compiler.cpp
index f877300..1ce7295 100644
--- a/src/lib/compiler.cpp
+++ b/src/lib/compiler.cpp
@@ -167,7 +167,7 @@ String preprocess_shared_unit_char_wise(Request* context, SharedUnit* su, String
pc.resize(pc.length() - 4);
String declaration = trim(content.substr(i, end_declaration_pos - i));
String return_type_and_name = nibble(declaration, "(");
- StringList rtn_list = split(return_type_and_name);
+ StringList rtn_list = split_space(return_type_and_name);
String fn_name = rtn_list.back(); rtn_list.pop_back();
su->api_declarations.push_back(fn_name + ":" + join(rtn_list, " ") + ":(" + declaration + "\n");
printf("declaration found: %s\n", declaration.c_str());
diff --git a/src/lib/functionlib.cpp b/src/lib/functionlib.cpp
index 1f5604f..e986b36 100644
--- a/src/lib/functionlib.cpp
+++ b/src/lib/functionlib.cpp
@@ -101,7 +101,7 @@ String trim(String raw)
return(raw.substr(start_pos, 1 + end_pos - start_pos));
}
-StringList split(String str)
+StringList split_space(String str)
{
StringList result;
String current_token = "";
@@ -154,7 +154,7 @@ String join(StringList l, String delim)
return(result);
}
-StringList split_utf8(String s)
+StringList split_utf8(String s, bool compound_characters)
{
StringList result;
auto len = s.size();
@@ -185,6 +185,50 @@ StringList split_utf8(String s)
result.push_back(String().append(1, c));
}
}
+ if(compound_characters)
+ {
+ StringList compound_result;
+ bool join_next = false;
+ bool last_was_regional = false;
+ for(auto& s : result)
+ {
+ if(join_next)
+ {
+ compound_result[compound_result.size()-1] += s;
+ join_next = false;
+ }
+ else if(s == "\xE2" "\x80" "\x8D") // ZWJ
+ {
+ compound_result[compound_result.size()-1] += s;
+ join_next = true;
+ last_was_regional = false;
+ }
+ else if(s[0] == '\xF0' && s[1] == '\x9F' && s[2] == '\x87' && s[3] >= '\xA6' && s[3] <= '\xBF') // Regional indicator letters
+ {
+ if(last_was_regional)
+ {
+ compound_result[compound_result.size()-1] += s;
+ last_was_regional = false;
+ }
+ else
+ {
+ compound_result.push_back(s);
+ last_was_regional = true;
+ }
+ }
+ else if(s[0] == '\xEF' && s[1] == '\xB8' && s[2] >= '\x80' && s[2] <= '\x8F') // Variation selector
+ {
+ compound_result[compound_result.size()-1] += s;
+ last_was_regional = false;
+ }
+ else
+ {
+ compound_result.push_back(s);
+ last_was_regional = false;
+ }
+ }
+ return(compound_result);
+ }
return(result);
}
diff --git a/src/lib/functionlib.h b/src/lib/functionlib.h
index 8f7f70a..edd0de8 100644
--- a/src/lib/functionlib.h
+++ b/src/lib/functionlib.h
@@ -7,12 +7,22 @@ String to_upper(String s);
String replace(String s, String search, String replace_with);
String trim(String raw);
-StringList split(String str);
+StringList split_space(String str);
StringList split(String str, String delim);
+StringList split_utf8(String s, bool compound_characters = false);
String join(StringList l, String delim = "\n");
String nibble(String& haystack, String delim);
void json_consume_space(String s, u32& i);
-StringList split_utf8(String s);
+
+template
+String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1)
+{
+ static const char* digits = "0123456789ABCDEF";
+ String rc(hex_len,'0');
+ for (size_t i=0, j=(hex_len-1)*4 ; i>j) & 0x0f];
+ return(rc);
+}
template
std::vector filter(std::vector items, std::function f)
diff --git a/src/lib/types.h b/src/lib/types.h
index da5fcb1..272fa9e 100644
--- a/src/lib/types.h
+++ b/src/lib/types.h
@@ -215,16 +215,6 @@ struct URI {
String nibble(String div, String& haystack);
-template
-String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1)
-{
- static const char* digits = "0123456789ABCDEF";
- String rc(hex_len,'0');
- for (size_t i=0, j=(hex_len-1)*4 ; i>j) & 0x0f];
- return(rc);
-}
-
#include "dtree.h"
void compiler_invoke(Request* context, String file_name, DTree& call_param);
diff --git a/src/lib/uce_lib.cpp b/src/lib/uce_lib.cpp
index 66d1a56..6e27be3 100644
--- a/src/lib/uce_lib.cpp
+++ b/src/lib/uce_lib.cpp
@@ -1,9 +1,9 @@
#include "types.cpp"
-#include "hash.cpp"
#include "dtree.cpp"
#include "functionlib.cpp"
+#include "hash.cpp"
#include "sys.cpp"
#include "uri.cpp"
#include "compiler.cpp"
diff --git a/test/utf8.uce b/test/utf8.uce
index 92935d8..252ce55 100644
--- a/test/utf8.uce
+++ b/test/utf8.uce
@@ -1,8 +1,18 @@
+String string_to_hex(String s)
+{
+ String result = "";
+ for(auto c : s)
+ {
+ result += to_hex(c);
+ }
+ return(result);
+}
+
RENDER()
{
-
- String raw = first(context->post["raw"], "■▧▲😂😆😏😱🇦🇽");
+
+ String raw = first(context->post["raw"], "■👪▧▲🏳️🌈😂👋🏽😆😏😱🇦🇽Udø島リZ̸̢̧̡̧̗̰̪͉̤͖͉̪̝̦͎̮̑͜a̸̧̝̱̹̲̗̪̰̦͒̃͋̿̿̃̈͑̐͑͗̚̕̚͝͠l͚̜͕̠̣ģ̸̧̘̜͇͚͈͙̓̌̅͑͊͊͋̓́͌̈́̿̈́͗͘̚ͅͅȏ̶̗̤̳͎̫̥͕̣͔̥̙̜̰̂͌̍͊͂́̅̇̒̕̕ルイ社もなく");
<>
@@ -19,12 +29,29 @@ RENDER()
+ Simple Characters
+ Compound Characters
+