diff --git a/doc/areas/string.txt b/doc/areas/string.txt index 711a9dc..a2c6e21 100644 --- a/doc/areas/string.txt +++ b/doc/areas/string.txt @@ -5,6 +5,7 @@ first join nibble split +split_utf8 replace to_lower to_upper diff --git a/doc/pages/expand_path.txt b/doc/pages/expand_path.txt index 0a82d22..b77e7c3 100644 --- a/doc/pages/expand_path.txt +++ b/doc/pages/expand_path.txt @@ -1,8 +1,9 @@ :sig -String expand_path(String path) +String expand_path(String path, String relative_to_path = "") :params path : a relative path +relative_to_path : optional, expand relative to this path (if not given, the current path is used) return value : expanded version of the 'path' :desc diff --git a/doc/pages/split_utf8.txt b/doc/pages/split_utf8.txt new file mode 100644 index 0000000..c7922ee --- /dev/null +++ b/doc/pages/split_utf8.txt @@ -0,0 +1,15 @@ +:sig +StringList split_utf8(String str) + + +:params +str : string to be split +return value : a list of Unicode characters + +:desc +Splits the string 'str' into its constituent Unicode code points. + +This currently does not honor compound characters such as flags or composite emojis. + +:see +>string diff --git a/scripts/compile b/scripts/compile index 93fd790..64f194d 100755 --- a/scripts/compile +++ b/scripts/compile @@ -24,7 +24,7 @@ OPT_FLAG="O0" COMPILER="clang++" #COMPILER="g++" -FLAGS="-shared -g -rdynamic -w -Wall -$OPT_FLAG -std=c++17 -fpermissive -ffast-math -fPIC" +FLAGS="-shared -g -rdynamic -w -Wall -$OPT_FLAG -std=c++14 -fpermissive -ffast-math -fPIC" LIBS="-ldl -lm -lpthread " SRCFLAGS="-D PLATFORM_NAME=\"linux\"" diff --git a/src/lib/functionlib.cpp b/src/lib/functionlib.cpp index 8b12e01..1f5604f 100644 --- a/src/lib/functionlib.cpp +++ b/src/lib/functionlib.cpp @@ -154,6 +154,40 @@ String join(StringList l, String delim) return(result); } +StringList split_utf8(String s) +{ + StringList result; + auto len = s.size(); + String codepoint = ""; + for(s64 i = 0; i < len; i++) + { + u8 c = s[i]; + if(is_bit_set(c, 7)) + { + codepoint = ""; + codepoint.append(1, c); + if(is_bit_set(c, 6)) + { + codepoint.append(1, s[++i]); + if(is_bit_set(c, 5)) + { + codepoint.append(1, s[++i]); + if(is_bit_set(c, 4)) + { + codepoint.append(1, s[++i]); + } + } + } + result.push_back(codepoint); + } + else + { + result.push_back(String().append(1, c)); + } + } + return(result); +} + String html_escape(String s) { String result; diff --git a/src/lib/functionlib.h b/src/lib/functionlib.h index d941f37..8f7f70a 100644 --- a/src/lib/functionlib.h +++ b/src/lib/functionlib.h @@ -12,6 +12,7 @@ StringList split(String str, String delim); String join(StringList l, String delim = "\n"); String nibble(String& haystack, String delim); void json_consume_space(String s, u32& i); +StringList split_utf8(String s); template std::vector filter(std::vector items, std::function f) @@ -50,3 +51,4 @@ void ob_clear(); String ob_get_clear(); String ob_get(); +#define is_bit_set(var,pos) ((var) & (1<<(pos))) diff --git a/test/index.uce b/test/index.uce index 9dbe440..dad8973 100644 --- a/test/index.uce +++ b/test/index.uce @@ -30,6 +30,7 @@ RENDER()
  • File Append
  • RNG/Noise
  • Task API
  • +
  • UTF-8
  • type);
    +			if(identifier != "") result += " ident=" + identifier;
    +			if(type_name != "")  result += " type=" + type_name;
    +			result += "\n";
    +			if(current->call_params)
    +				result += current->call_params->to_string(indent + "--CALL ");
    +			if(current->assign)
    +				result += current->assign->to_string(indent + "--ASSIGN ");
    +			if(current->children)
    +				result += current->children->to_string(indent + "  ");
    +			current = current->next;
    +		}
    +		return(result);
    +	}
    +
    +};
    +
    +struct ParserState
    +{
    +
    +	ASTNode* root = 0;
    +	StringList tokens;
    +	s32 token_index = 0;
    +	String error = "";
    +
    +	String next()
    +	{
    +		return(tokens[token_index++]);
    +	}
    +
    +	String look_ahead(u32 by)
    +	{
    +		return(tokens[token_index+by]);
    +	}
    +
    +	bool expect(String token_str)
    +	{
    +		String found = look_ahead(0);
    +		if(found == token_str)
    +			return(true);
    +		token_index = tokens.size() + 1;
    +		error = String("'") + token_str + "' expected but '" + found + "' found";
    +		return(false);
    +	}
    +
    +	void consume(String token_str)
    +	{
    +		if(expect(token_str))
    +			next();
    +	}
    +
    +};
    +
    +ASTNode* script_parse_expression(ParserState* state, String accept_delim);
    +
    +ASTNode* script_parse_call_params(ParserState* state)
    +{
    +	ASTNode* result = new ASTNode('P');
    +	while(state->look_ahead(0) != ")")
    +	{
    +		state->next();
    +	}
    +	state->consume(")");
    +	return(result);
    +}
    +
    +ASTNode* script_parse_expression(ParserState* state, String accept_delim)
    +{
    +	ASTNode* result = new ASTNode('E');
    +	String tname = state->next();
    +	String tnx = state->look_ahead(0);
    +	if(tnx == ":")
    +	{
    +		// declaration
    +		result->type = 'D';
    +		result->identifier = tname;
    +		state->consume(":");
    +		if(state->look_ahead(0) != "=")
    +			result->type_name = state->next();
    +		if(state->look_ahead(0) == "=")
    +		{
    +			state->consume("=");
    +			result->assign = script_parse_expression(state, accept_delim);
    +		}
    +		else
    +		{
    +			state->consume(accept_delim);
    +		}
    +		return(result);
    +	}
    +	else if(tnx == "=")
    +	{
    +		// assignment
    +		result->type = 'A';
    +		result->identifier = tname;
    +		state->consume("=");
    +		result->assign = script_parse_expression(state, accept_delim);
    +		return(result);
    +	}
    +	else
    +	{
    +		// who knows
    +		result->type = 'E';
    +		result->identifier = tname;
    +		tnx = state->look_ahead(0);
    +		if(tnx == "(")
    +		{
    +			result->type = 'C';
    +			result->call_params = script_parse_call_params(state);
    +		}
    +		state->consume(accept_delim);
    +		return(result);
    +	}
    +	return(result);
    +}
    +
    +ParserState* script_parse(StringList tokens)
    +{
    +	ParserState* state = new ParserState();
    +	state->tokens = tokens;
    +	state->root = new ASTNode('R');
    +	ASTNode* current_node = 0;
    +	while(state->token_index < state->tokens.size())
    +	{
    +		if(!current_node)
    +		{
    +			current_node = script_parse_expression(state, ";");
    +			state->root->children = current_node;
    +		}
    +		else
    +		{
    +			current_node->next = script_parse_expression(state, ";");
    +			current_node = current_node->next;
    +		}
    +	}
    +	return(state);
    +}
    +
    +RENDER()
    +{
    +
    +	<>
    +		
    +		

    + UCE Test: + Script +

    + + + + <> +

    Code

    +
    + + +
    +
    post["code"]);
    +
    +			ParserState* ps = script_parse(tokens);
    +
    +			print(ps->error+"\n");
    +
    +			print(ps->root->to_string());
    +
    +		?>
    + Params +
    params) ?>
    + + +} diff --git a/test/utf8.uce b/test/utf8.uce new file mode 100644 index 0000000..92935d8 --- /dev/null +++ b/test/utf8.uce @@ -0,0 +1,34 @@ + +RENDER() +{ + + String raw = first(context->post["raw"], "โ– โ–งโ–ฒ๐Ÿ˜‚๐Ÿ˜†๐Ÿ˜๐Ÿ˜ฑ๐Ÿ‡ฆ๐Ÿ‡ฝ"); + + <> + +

    + UCE Test: + UTF-8 +

    +
    +
    + + +
    +
    + +
    +
    +
    +
    params) ?>
    + + +}