From 17336fe649c52c162a4541968c8d24c0c2ab3eb9 Mon Sep 17 00:00:00 2001 From: Udo Date: Sun, 30 Jan 2022 12:16:09 +0000 Subject: [PATCH] Unicode stuff --- doc/index.uce | 2 +- doc/pages/split_utf8.txt | 9 ++++++-- src/lib/compiler.cpp | 2 +- src/lib/functionlib.cpp | 48 ++++++++++++++++++++++++++++++++++++++-- src/lib/functionlib.h | 14 ++++++++++-- src/lib/types.h | 10 --------- src/lib/uce_lib.cpp | 2 +- test/utf8.uce | 33 ++++++++++++++++++++++++--- 8 files changed, 98 insertions(+), 22 deletions(-) diff --git a/doc/index.uce b/doc/index.uce index 482ae4b..70baf47 100644 --- a/doc/index.uce +++ b/doc/index.uce @@ -145,7 +145,7 @@ RENDER() } else { - ?>
combine characters if they're connected by a Zero-Width Joiner (ZWJ) character +
  • combine two characters if they're both a Regional Indicator Symbol Letter
    • +
  • if a character is a Variation Selector, append it to the previous character
    • +
  • in all other cases, characters remain on their own
    • :see >string diff --git a/src/lib/compiler.cpp b/src/lib/compiler.cpp index f877300..1ce7295 100644 --- a/src/lib/compiler.cpp +++ b/src/lib/compiler.cpp @@ -167,7 +167,7 @@ String preprocess_shared_unit_char_wise(Request* context, SharedUnit* su, String pc.resize(pc.length() - 4); String declaration = trim(content.substr(i, end_declaration_pos - i)); String return_type_and_name = nibble(declaration, "("); - StringList rtn_list = split(return_type_and_name); + StringList rtn_list = split_space(return_type_and_name); String fn_name = rtn_list.back(); rtn_list.pop_back(); su->api_declarations.push_back(fn_name + ":" + join(rtn_list, " ") + ":(" + declaration + "\n"); printf("declaration found: %s\n", declaration.c_str()); diff --git a/src/lib/functionlib.cpp b/src/lib/functionlib.cpp index 1f5604f..e986b36 100644 --- a/src/lib/functionlib.cpp +++ b/src/lib/functionlib.cpp @@ -101,7 +101,7 @@ String trim(String raw) return(raw.substr(start_pos, 1 + end_pos - start_pos)); } -StringList split(String str) +StringList split_space(String str) { StringList result; String current_token = ""; @@ -154,7 +154,7 @@ String join(StringList l, String delim) return(result); } -StringList split_utf8(String s) +StringList split_utf8(String s, bool compound_characters) { StringList result; auto len = s.size(); @@ -185,6 +185,50 @@ StringList split_utf8(String s) result.push_back(String().append(1, c)); } } + if(compound_characters) + { + StringList compound_result; + bool join_next = false; + bool last_was_regional = false; + for(auto& s : result) + { + if(join_next) + { + compound_result[compound_result.size()-1] += s; + join_next = false; + } + else if(s == "\xE2" "\x80" "\x8D") // ZWJ + { + compound_result[compound_result.size()-1] += s; + join_next = true; + last_was_regional = false; + } + else if(s[0] == '\xF0' && s[1] == '\x9F' && s[2] == '\x87' && s[3] >= '\xA6' && s[3] <= '\xBF') // Regional indicator letters + { + if(last_was_regional) + { + compound_result[compound_result.size()-1] += s; + last_was_regional = false; + } + else + { + compound_result.push_back(s); + last_was_regional = true; + } + } + else if(s[0] == '\xEF' && s[1] == '\xB8' && s[2] >= '\x80' && s[2] <= '\x8F') // Variation selector + { + compound_result[compound_result.size()-1] += s; + last_was_regional = false; + } + else + { + compound_result.push_back(s); + last_was_regional = false; + } + } + return(compound_result); + } return(result); } diff --git a/src/lib/functionlib.h b/src/lib/functionlib.h index 8f7f70a..edd0de8 100644 --- a/src/lib/functionlib.h +++ b/src/lib/functionlib.h @@ -7,12 +7,22 @@ String to_upper(String s); String replace(String s, String search, String replace_with); String trim(String raw); -StringList split(String str); +StringList split_space(String str); StringList split(String str, String delim); +StringList split_utf8(String s, bool compound_characters = false); String join(StringList l, String delim = "\n"); String nibble(String& haystack, String delim); void json_consume_space(String s, u32& i); -StringList split_utf8(String s); + +template +String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1) +{ + static const char* digits = "0123456789ABCDEF"; + String rc(hex_len,'0'); + for (size_t i=0, j=(hex_len-1)*4 ; i>j) & 0x0f]; + return(rc); +} template std::vector filter(std::vector items, std::function f) diff --git a/src/lib/types.h b/src/lib/types.h index da5fcb1..272fa9e 100644 --- a/src/lib/types.h +++ b/src/lib/types.h @@ -215,16 +215,6 @@ struct URI { String nibble(String div, String& haystack); -template -String to_hex(ITYPE w, size_t hex_len = sizeof(ITYPE)<<1) -{ - static const char* digits = "0123456789ABCDEF"; - String rc(hex_len,'0'); - for (size_t i=0, j=(hex_len-1)*4 ; i>j) & 0x0f]; - return(rc); -} - #include "dtree.h" void compiler_invoke(Request* context, String file_name, DTree& call_param); diff --git a/src/lib/uce_lib.cpp b/src/lib/uce_lib.cpp index 66d1a56..6e27be3 100644 --- a/src/lib/uce_lib.cpp +++ b/src/lib/uce_lib.cpp @@ -1,9 +1,9 @@ #include "types.cpp" -#include "hash.cpp" #include "dtree.cpp" #include "functionlib.cpp" +#include "hash.cpp" #include "sys.cpp" #include "uri.cpp" #include "compiler.cpp" diff --git a/test/utf8.uce b/test/utf8.uce index 92935d8..252ce55 100644 --- a/test/utf8.uce +++ b/test/utf8.uce @@ -1,8 +1,18 @@ +String string_to_hex(String s) +{ + String result = ""; + for(auto c : s) + { + result += to_hex(c); + } + return(result); +} + RENDER() { - - String raw = first(context->post["raw"], "■▧▲😂😆😏😱🇦🇽"); + + String raw = first(context->post["raw"], "■👪▧▲🏳️‍🌈😂👋🏽😆😏😱🇦🇽Udø島リZ̸̢̧̡̧̗̰̪͉̤͖͉̪̝̦͎̮̑͜a̸̧̝̱̹̲̗̪̰̦͒̃͋̿̿̃̈͑̐͑͗̚̕̚͝͠l͚̜͕̠̣ģ̸̧̘̜͇͚͈͙̓̌̅͑͊͊͋̓́͌̈́̿̈́͗͘̚ͅͅȏ̶̗̤̳͎̫̥͕̣͔̥̙̜̰̂͌̍͊͂́̅̇̒̕̕ルイ社もなく"); <> @@ -19,12 +29,29 @@ RENDER() + Simple Characters
    + Compound Characters +