370 lines
7.6 KiB
Plaintext
370 lines
7.6 KiB
Plaintext
|
|
String add_token_type(String tokenstr)
|
|
{
|
|
char c = tokenstr[0];
|
|
if(c == 34) // quote character, as number here because of HTML parser bug
|
|
return(String("Q") + tokenstr);
|
|
if(isalpha(c))
|
|
return(String("I") + tokenstr);
|
|
if(isdigit(c))
|
|
return(String("N") + tokenstr);
|
|
if(ispunct(c))
|
|
return(String("O") + tokenstr);
|
|
return(String("?") + tokenstr);
|
|
}
|
|
|
|
StringList script_tokenize(String code)
|
|
{
|
|
StringList result;
|
|
String current_token = "";
|
|
char mode = 'N';
|
|
char quote_character = 0;
|
|
char last_character = 0;
|
|
for(auto c : code)
|
|
{
|
|
if(mode == 'Q')
|
|
{
|
|
if(c == '\\')
|
|
{
|
|
|
|
}
|
|
else if(quote_character == c && last_character != '\\')
|
|
{
|
|
mode = 'N';
|
|
result.push_back(add_token_type(current_token));
|
|
current_token = "";
|
|
}
|
|
else
|
|
{
|
|
current_token.append(1, c);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(isspace(c))
|
|
{
|
|
if(current_token != "")
|
|
result.push_back(add_token_type(current_token));
|
|
current_token = "";
|
|
}
|
|
else if(c == '"' || c == '\'')
|
|
{
|
|
mode = 'Q';
|
|
quote_character = c;
|
|
if(current_token != "")
|
|
result.push_back(add_token_type(current_token));
|
|
current_token = "\"";
|
|
}
|
|
else if(ispunct(c) && c != '_')
|
|
{
|
|
if(current_token != "")
|
|
result.push_back(add_token_type(current_token));
|
|
current_token = "";
|
|
result.push_back(add_token_type(String("").append(1, c)));
|
|
}
|
|
else
|
|
{
|
|
current_token.append(1, c);
|
|
}
|
|
}
|
|
last_character = c;
|
|
}
|
|
if(current_token != "")
|
|
result.push_back(add_token_type(current_token));
|
|
return(result);
|
|
}
|
|
|
|
struct ASTNode
|
|
{
|
|
String identifier = "";
|
|
String type_name = "";
|
|
String string_literal = "";
|
|
char type = 0;
|
|
std::vector<ASTNode*> assign;
|
|
std::vector<ASTNode*> call_params;
|
|
std::vector<ASTNode*> children;
|
|
|
|
ASTNode(char _type)
|
|
{
|
|
identifier = "";
|
|
type_name = "";
|
|
string_literal = "";
|
|
type = _type;
|
|
}
|
|
|
|
String to_string(String indent = "")
|
|
{
|
|
String result = "";
|
|
result += indent + String().append(1, type);
|
|
if(identifier != "")
|
|
result += " ident=" + identifier;
|
|
if(type_name != "")
|
|
result += " type_name=" + type_name;
|
|
if(string_literal != "")
|
|
result += " " + string_literal;
|
|
result += "\n";
|
|
for(auto callp : call_params)
|
|
result += indent + " CALLP " + callp->to_string(indent + " ");
|
|
for(auto assign_expr : assign)
|
|
result += indent + " ASSIGN VALUE " + assign_expr->to_string(indent + " ");
|
|
for(auto child : children)
|
|
result += indent + " " + child->to_string(indent + " ");
|
|
return(result);
|
|
}
|
|
|
|
};
|
|
|
|
struct ParserState
|
|
{
|
|
|
|
ASTNode* root;
|
|
StringList tokens;
|
|
s32 token_index = 0;
|
|
String error = "";
|
|
u64 error_location = 0;
|
|
|
|
String consume(char token_kind = 0)
|
|
{
|
|
if(error_location > 0) return("");
|
|
if(token_kind && look_ahead(0)[0] != token_kind)
|
|
{
|
|
unexpected_token();
|
|
}
|
|
printf("consume #%i %s -> %s \n", token_index, look_ahead(0).c_str(), look_ahead(1).c_str());
|
|
String tk = look_ahead(0);
|
|
token_index++;
|
|
return(tk);
|
|
}
|
|
|
|
String look_ahead(s32 by)
|
|
{
|
|
if(error_location > 0)
|
|
return("_");
|
|
if(token_index+by < 0 || token_index+by >= tokens.size())
|
|
return("_");
|
|
return(tokens[token_index+by]);
|
|
}
|
|
|
|
bool more()
|
|
{
|
|
if(error_location > 0)
|
|
return(false);
|
|
return(token_index < tokens.size());
|
|
}
|
|
|
|
void unexpected_token(s32 offset = 0)
|
|
{
|
|
if(error_location) return;
|
|
String found = look_ahead(offset);
|
|
error_location = token_index;
|
|
token_index = tokens.size() + 1;
|
|
error = String("unexpected '") + found.substr(1) + String("' found");
|
|
}
|
|
|
|
bool expect(String token_str, String in_mode)
|
|
{
|
|
if(error_location) return(false);
|
|
String found = look_ahead(0);
|
|
if(found == token_str)
|
|
{
|
|
consume();
|
|
return(true);
|
|
}
|
|
error_location = token_index;
|
|
token_index = tokens.size() + 1;
|
|
error = String("'") + token_str.substr(1) + "' expected but '" + found.substr(1) + "' found in "+in_mode.substr(1);
|
|
return(false);
|
|
}
|
|
|
|
};
|
|
|
|
ASTNode* script_parse_expression(ParserState* state, String delim1, String delim2);
|
|
ASTNode* script_parse_declaration(ParserState* state, String delim1, String delim2);
|
|
|
|
ASTNode* script_parse_declaration(ParserState* state, String delim1, String delim2)
|
|
{
|
|
ASTNode* result = new ASTNode('D');
|
|
|
|
String ident = state->consume();
|
|
if(ident[0] != 'I')
|
|
{
|
|
state->unexpected_token(-1);
|
|
return(result);
|
|
}
|
|
result->identifier = ident.substr(1);
|
|
|
|
if(!state->expect("O:", "declaration"))
|
|
return(result);
|
|
|
|
String n0 = state->look_ahead(0);
|
|
if(n0 == "O=")
|
|
{
|
|
result->type_name = "auto";
|
|
result->assign.push_back(script_parse_expression(state, delim1, delim2));
|
|
return(result);
|
|
}
|
|
else
|
|
{
|
|
result->type_name = state->consume('I').substr(1);
|
|
String n2 = state->look_ahead(0);
|
|
if(n2 == delim1 || n2 == delim2)
|
|
{
|
|
state->consume();
|
|
return(result);
|
|
}
|
|
else if(n2 == "O=")
|
|
{
|
|
state->expect("O=", "declaration");
|
|
result->assign.push_back(script_parse_expression(state, delim1, delim2));
|
|
return(result);
|
|
}
|
|
else
|
|
{
|
|
state->unexpected_token();
|
|
return(result);
|
|
}
|
|
}
|
|
|
|
return(result);
|
|
}
|
|
|
|
ASTNode* parse_single_token(ParserState* state)
|
|
{
|
|
ASTNode* result = new ASTNode('?');
|
|
|
|
String n = state->consume();
|
|
if(n[0] == 'Q')
|
|
{
|
|
result->type = 'S';
|
|
result->string_literal = n.substr(1);
|
|
}
|
|
else if(n[0] == 'N')
|
|
{
|
|
result->type = 'N';
|
|
result->string_literal = n.substr(1);
|
|
}
|
|
else if(n[0] == 'I')
|
|
{
|
|
result->type = 'I';
|
|
result->identifier = n.substr(1);
|
|
}
|
|
else
|
|
{
|
|
state->unexpected_token(-1);
|
|
}
|
|
return(result);
|
|
}
|
|
|
|
ASTNode* script_parse_assignment(ParserState* state, String delim1, String delim2)
|
|
{
|
|
ASTNode* result = new ASTNode('A');
|
|
result->identifier = state->consume('I').substr(1);
|
|
state->expect("O=", "assignment");
|
|
result->assign.push_back(script_parse_expression(state, delim1, delim2));
|
|
return(result);
|
|
}
|
|
|
|
ASTNode* script_parse_expression(ParserState* state, String delim1, String delim2)
|
|
{
|
|
ASTNode* result = new ASTNode('E');
|
|
|
|
String n = state->look_ahead(0);
|
|
|
|
if(n == "O(")
|
|
{
|
|
state->consume();
|
|
result->children.push_back(script_parse_expression(state, "O)", ""));
|
|
}
|
|
else if(n == delim1 || n == delim2)
|
|
{
|
|
state->consume();
|
|
return(result);
|
|
}
|
|
else
|
|
{
|
|
String n2 = state->look_ahead(1);
|
|
if(n[0] == 'I' && n2 == "O:")
|
|
{
|
|
return(script_parse_declaration(state, "O;", ""));
|
|
}
|
|
else if(n2 == delim1 || n2 == delim2)
|
|
{
|
|
// single component 'expression'
|
|
result = parse_single_token(state);
|
|
state->consume();
|
|
return(result);
|
|
}
|
|
else if(n[0] == 'I' && n2 == "O=")
|
|
{
|
|
result = script_parse_assignment(state, delim1, delim2);
|
|
n2 = state->look_ahead(0);
|
|
if(n2 != delim1 && n2 != delim2)
|
|
{
|
|
state->unexpected_token();
|
|
}
|
|
return(result);
|
|
}
|
|
else
|
|
{
|
|
state->unexpected_token();
|
|
state->error += " in expression";
|
|
}
|
|
}
|
|
|
|
return(result);
|
|
}
|
|
|
|
ParserState* script_parse(StringList tokens)
|
|
{
|
|
ParserState* state = new ParserState();
|
|
state->tokens = tokens;
|
|
state->root = new ASTNode('M');
|
|
|
|
while(state->token_index < state->tokens.size())
|
|
{
|
|
state->root->children.push_back(script_parse_expression(state, "O;", ""));
|
|
}
|
|
|
|
return(state);
|
|
}
|
|
|
|
RENDER()
|
|
{
|
|
|
|
<>
|
|
<link rel="stylesheet" href='style.css'></link>
|
|
<h1>
|
|
<a href="index.uce">UCE Test</a>:
|
|
Script
|
|
</h1>
|
|
</>
|
|
|
|
String script_src = first(context->post["code"], file_get_contents("script-example1.usp"));
|
|
|
|
<>
|
|
<h3>Code</h3>
|
|
<form action="?" method="post">
|
|
<textarea name="code"><?= script_src ?></textarea>
|
|
<input type="submit" value="parse"/>
|
|
</form>
|
|
<pre><?
|
|
|
|
StringList tokens = script_tokenize(script_src);
|
|
//print(var_dump(tokens));
|
|
|
|
ParserState* ps = script_parse(tokens);
|
|
|
|
if(ps->error_location > 0)
|
|
{
|
|
print(ps->error, " @ token ", 1+ps->error_location, "\n");
|
|
}
|
|
|
|
print(ps->root->to_string());
|
|
|
|
?></pre>
|
|
Params
|
|
<pre><?= var_dump(context->params) ?></pre>
|
|
</>
|
|
|
|
}
|