Enhance template parser to handle C++ comments and refactor preprocessing logic

- Updated parser to correctly interpret C++ `//` and `/* ... */` comments within template code.
- Split preprocessing implementation into separate files for better organization.
- Added regression test for comment parsing in templates.
- Adjusted CSS styles for improved layout and readability in documentation.
This commit is contained in:
udo
2026-04-20 21:41:23 +00:00
parent dc05f9faa5
commit b53eb6e4f1
11 changed files with 568 additions and 390 deletions
+373
View File
@@ -0,0 +1,373 @@
#include "compiler-parser.h"
#include "compiler.h"
namespace {
const char* UCE_NAMED_RENDER_SYMBOL = "__uce_render";
const char* UCE_NAMED_COMPONENT_SYMBOL = "__uce_component";
struct CompilerCodeState
{
char quote_char = '\0';
bool inside_quote = false;
bool inside_line_comment = false;
bool inside_block_comment = false;
};
bool compiler_code_state_is_neutral(const CompilerCodeState& state)
{
return(!state.inside_quote && !state.inside_line_comment && !state.inside_block_comment);
}
void compiler_code_state_consume(CompilerCodeState& state, String& buffer, const String& content, u32& i)
{
char c = content[i];
char c1 = (i + 1 < content.length()) ? content[i + 1] : '\0';
buffer.append(1, c);
if(state.inside_quote)
{
if(state.quote_char == c && (i == 0 || content[i-1] != '\\'))
state.inside_quote = false;
return;
}
if(state.inside_line_comment)
{
if(c == '\n')
state.inside_line_comment = false;
return;
}
if(state.inside_block_comment)
{
if(c == '*' && c1 == '/')
{
buffer.append(1, c1);
state.inside_block_comment = false;
i += 1;
}
return;
}
if(c == '/' && c1 == '/')
{
buffer.append(1, c1);
state.inside_line_comment = true;
i += 1;
return;
}
if(c == '/' && c1 == '*')
{
buffer.append(1, c1);
state.inside_block_comment = true;
i += 1;
return;
}
if(c == '"' || c == '\'')
{
state.inside_quote = true;
state.quote_char = c;
}
}
String compiler_capture_markup_literal(const String& content, u32& i)
{
String buffer = "";
u32 depth = 0;
bool inside_code = false;
CompilerCodeState code_state;
u32 j = i + 2;
while(j < content.length())
{
char c = content[j];
char c1 = (j + 1 < content.length()) ? content[j + 1] : '\0';
char c2 = (j + 2 < content.length()) ? content[j + 2] : '\0';
if(inside_code)
{
if(compiler_code_state_is_neutral(code_state) && c == '?' && c1 == '>')
{
buffer.append(1, c);
buffer.append(1, c1);
inside_code = false;
j += 2;
continue;
}
compiler_code_state_consume(code_state, buffer, content, j);
j += 1;
continue;
}
if(c == '<' && c1 == '?')
{
inside_code = true;
code_state = CompilerCodeState();
buffer.append(1, c);
buffer.append(1, c1);
j += 2;
continue;
}
if(c == '<' && c1 == '/' && c2 == '>')
{
if(depth > 0)
{
depth -= 1;
buffer.append("</>");
j += 3;
continue;
}
i = j + 2;
return(buffer);
}
if(c == '<' && c1 == '>')
{
depth += 1;
buffer.append("<>");
j += 2;
continue;
}
buffer.append(1, c);
j += 1;
}
i = content.length();
return(buffer);
}
String compiler_process_text_literal(Request* context, SharedUnit* su, String content)
{
String parsed_content;
String html_output_start = "print(R\"(";
String html_output_end = ")\");";
String code_buffer = "";
CompilerCodeState code_state;
bool inside_code = false;
bool is_field = false;
bool escape_field = false;
for(u32 i = 0; i < content.length(); i++)
{
char c = content[i];
char c1 = (i + 1 < content.length()) ? content[i + 1] : '\0';
char c2 = (i + 2 < content.length()) ? content[i + 2] : '\0';
if(!inside_code)
{
if(c == '<' && c1 == '?')
{
inside_code = true;
code_buffer = "";
code_state = CompilerCodeState();
if(c2 == '=')
{
is_field = true;
escape_field = true;
i += 2;
}
else if(c2 == ':')
{
is_field = true;
escape_field = false;
i += 2;
}
else
{
is_field = false;
escape_field = false;
i += 1;
}
continue;
}
parsed_content.append(1, c);
continue;
}
if(compiler_code_state_is_neutral(code_state) && c == '?' && c1 == '>')
{
inside_code = false;
i += 1;
if(is_field)
{
if(escape_field)
{
parsed_content.append(
html_output_end +
"print(html_escape( " +
code_buffer +
" )); " +
html_output_start
);
}
else
{
parsed_content.append(
html_output_end +
"print( " +
code_buffer +
" ); " +
html_output_start
);
}
}
else
{
parsed_content.append(html_output_end + code_buffer + html_output_start);
}
continue;
}
if(compiler_code_state_is_neutral(code_state) && c == '<' && c1 == '>')
{
String nested_markup = compiler_capture_markup_literal(content, i);
code_buffer.append(compiler_process_text_literal(context, su, nested_markup));
continue;
}
compiler_code_state_consume(code_state, code_buffer, content, i);
}
return(html_output_start + parsed_content + html_output_end);
}
String compiler_rewrite_named_render_syntax(String content)
{
String result = "";
String current_line = "";
auto flush_line = [&]() {
if(current_line.length() == 0)
return;
String line = current_line;
String line_break = "";
if(line.length() > 0 && line.back() == '\n')
{
line_break = "\n";
line.pop_back();
}
u32 indent_length = 0;
while(indent_length < line.length() && isspace(line[indent_length]))
indent_length += 1;
String indent = line.substr(0, indent_length);
String trimmed = trim(line);
if(trimmed.rfind("RENDER:", 0) == 0)
{
String signature = trimmed.substr(7);
auto open_paren_pos = signature.find("(");
if(open_paren_pos != String::npos)
{
String render_name = trim(signature.substr(0, open_paren_pos));
String render_signature = signature.substr(open_paren_pos);
if(render_name != "")
line = indent + "EXPORT void " + String(UCE_NAMED_RENDER_SYMBOL) + "_" + safe_name(render_name) + render_signature;
}
}
else if(trimmed.rfind("COMPONENT:", 0) == 0)
{
String signature = trimmed.substr(10);
auto open_paren_pos = signature.find("(");
if(open_paren_pos != String::npos)
{
String render_name = trim(signature.substr(0, open_paren_pos));
String render_signature = signature.substr(open_paren_pos);
if(render_name != "")
line = indent + "EXPORT void " + String(UCE_NAMED_COMPONENT_SYMBOL) + "_" + safe_name(render_name) + render_signature;
}
}
result += line + line_break;
current_line = "";
};
for(auto c : content)
{
current_line.append(1, c);
if(c == '\n')
flush_line();
}
flush_line();
return(result);
}
String compiler_preprocess_shared_unit_char_wise(Request* context, SharedUnit* su, String content)
{
String parsed_content =
("#include \"")+context->server->config["COMPILER_SYS_PATH"] +"/src/lib/uce_lib.h\" \n"+
file_get_contents(
context->server->config["COMPILER_SYS_PATH"] + "/" + context->server->config["SETUP_TEMPLATE"]
)+
"#line 1\n";
CompilerCodeState code_state;
String current_line = "";
for(u32 i = 0; i < content.length(); i++)
{
char c = content[i];
char c1 = (i + 1 < content.length()) ? content[i + 1] : '\0';
current_line.append(1, c);
if(compiler_code_state_is_neutral(code_state) && c == '<' && c1 == '>')
{
String markup_literal = compiler_capture_markup_literal(content, i);
parsed_content.append(compiler_process_text_literal(context, su, markup_literal));
if(i < content.length() && content[i] == '\n')
current_line = "\n";
continue;
}
compiler_code_state_consume(code_state, parsed_content, content, i);
if(c == 10 && current_line.substr(0, 6) == "#load ")
{
parsed_content.resize(parsed_content.length() - current_line.length());
nibble(current_line, "\"");
String unit_name = nibble(current_line, "\"");
SharedUnit* sub_su = compiler_load_shared_unit(context, unit_name, su->src_path, true);
if(sub_su)
parsed_content.append("#include \"" + sub_su->bin_path + "/" + sub_su->pre_file_name + "\"\n");
}
else
{
String trimmed_line = trim(current_line);
if(c == 10 && trimmed_line.length() > 7 && trimmed_line.substr(0, 6) == "EXPORT" && isspace(trimmed_line[6]))
{
current_line = "";
auto end_declaration_pos = content.find("{", i);
if(end_declaration_pos != std::string::npos)
{
parsed_content.append(1, '\n');
String declaration = trim(content.substr(i, end_declaration_pos - i));
su->api_declarations.push_back(declaration+";\n");
}
}
}
if(c == 10)
current_line = "";
}
return(parsed_content);
}
}
String compiler_preprocess_source(Request* context, SharedUnit* su, String content)
{
content = compiler_rewrite_named_render_syntax(content);
return(compiler_preprocess_shared_unit_char_wise(context, su, content));
}
+5
View File
@@ -0,0 +1,5 @@
#pragma once
#include "types.h"
String compiler_preprocess_source(Request* context, SharedUnit* su, String content);
+6 -325
View File
@@ -1,4 +1,5 @@
#include "compiler.h"
#include "compiler-parser.h"
#include <algorithm>
#include <cstdlib>
#include <filesystem>
@@ -103,6 +104,8 @@ time_t compiler_runtime_abi_time(Request* context)
return(std::max({
file_mtime(root + "/src/lib/compiler.h"),
file_mtime(root + "/src/lib/compiler.cpp"),
file_mtime(root + "/src/lib/compiler-parser.h"),
file_mtime(root + "/src/lib/compiler-parser.cpp"),
file_mtime(root + "/bin/uce_fastcgi.linux.bin")
}));
}
@@ -201,7 +204,8 @@ SharedUnitFilesystemState inspect_shared_unit_filesystem(Request* context, Share
state.source_time = file_mtime(su->file_name);
state.setup_template_time = file_mtime(
context->server->config["COMPILER_SYS_PATH"] + "/" +
context->server->config["SETUP_TEMPLATE"]);
context->server->config["SETUP_TEMPLATE"]
);
state.compiler_header_time = file_mtime(context->server->config["COMPILER_SYS_PATH"] + "/src/lib/compiler.h");
state.compiler_source_time = file_mtime(context->server->config["COMPILER_SYS_PATH"] + "/src/lib/compiler.cpp");
state.runtime_binary_time = file_mtime(context->server->config["COMPILER_SYS_PATH"] + "/bin/uce_fastcgi.linux.bin");
@@ -425,333 +429,10 @@ bool compiler_can_write_response(Request* context)
}
String process_text_literal(Request* context, SharedUnit* su, String content)
{
String pc;
String HT_START = "print(R\"(";
String HT_END = ")\");";
u8 mode = 0;
char quote_char;
bool inside_quote = false;
String code_buffer = "";
bool is_field = false;
bool escape_field = false;
for(u32 i = 0; i < content.length(); i++)
{
char c = content[i];
char c1 = (i + 1 < content.length()) ? content[i + 1] : '\0';
char c2 = (i + 2 < content.length()) ? content[i + 2] : '\0';
switch(mode)
{
case(0):
if(c == '<' && c1 == '?')
{
code_buffer = "";
if(c2 == '=')
{
is_field = true;
escape_field = true;
i += 2;
}
else if(c2 == ':')
{
is_field = true;
escape_field = false;
i += 2;
}
else
{
is_field = false;
escape_field = false;
i += 1;
}
mode = 1; // code-parsing mode
}
else
{
pc.append(1, c);
}
break;
case(1):
if(inside_quote)
{
if(quote_char == c && (i == 0 || content[i-1] != '\\'))
inside_quote = false;
code_buffer.append(1, c);
}
else
{
if(c == '\"' || c == '\'')
{
inside_quote = true;
quote_char = c;
code_buffer.append(1, c);
}
else if(c == '?' && c1 == '>')
{
mode = 0;
i += 1;
if(is_field)
{
if(escape_field)
{
pc.append(
HT_END +
"print(html_escape( " +
code_buffer +
" )); " +
HT_START
);
}
else
{
pc.append(
HT_END +
"print( " +
code_buffer +
" ); " +
HT_START
);
}
}
else
{
pc.append(HT_END + code_buffer + HT_START);
}
}
else if(c == '<' && c1 == '>')
{
// Nested <> markup block inside code
String sub_buffer = "";
u32 sub_depth = 0;
u32 j = i + 2;
while(j < content.length())
{
char jc = content[j];
char jc1 = (j + 1 < content.length()) ? content[j + 1] : '\0';
char jc2 = (j + 2 < content.length()) ? content[j + 2] : '\0';
if(jc == '<' && jc1 == '/' && jc2 == '>')
{
if(sub_depth > 0)
{
sub_depth--;
sub_buffer.append("</>");
j += 3;
}
else
{
j += 3;
break;
}
}
else if(jc == '<' && jc1 == '>')
{
sub_depth++;
sub_buffer.append("<>");
j += 2;
}
else
{
sub_buffer.append(1, jc);
j++;
}
}
i = j - 1; // -1 because outer for loop increments
code_buffer.append(process_text_literal(context, su, sub_buffer));
}
else
{
code_buffer.append(1, c);
}
}
break;
}
}
return(HT_START + pc + HT_END);
}
String preprocess_named_render_syntax(String content)
{
String result = "";
String current_line = "";
auto flush_line = [&]() {
if(current_line.length() == 0)
return;
String line = current_line;
String line_break = "";
if(line.length() > 0 && line.back() == '\n')
{
line_break = "\n";
line.pop_back();
}
u32 indent_length = 0;
while(indent_length < line.length() && isspace(line[indent_length]))
indent_length += 1;
String indent = line.substr(0, indent_length);
String trimmed = trim(line);
if(trimmed.rfind("RENDER:", 0) == 0)
{
String signature = trimmed.substr(7);
auto open_paren_pos = signature.find("(");
if(open_paren_pos != String::npos)
{
String render_name = trim(signature.substr(0, open_paren_pos));
String render_signature = signature.substr(open_paren_pos);
if(render_name != "")
line = indent + "EXPORT void " + String(UCE_RENDER_SYMBOL) + "_" + safe_name(render_name) + render_signature;
}
}
else if(trimmed.rfind("COMPONENT:", 0) == 0)
{
String signature = trimmed.substr(10);
auto open_paren_pos = signature.find("(");
if(open_paren_pos != String::npos)
{
String render_name = trim(signature.substr(0, open_paren_pos));
String render_signature = signature.substr(open_paren_pos);
if(render_name != "")
line = indent + "EXPORT void " + String(UCE_COMPONENT_SYMBOL) + "_" + safe_name(render_name) + render_signature;
}
}
result += line + line_break;
current_line = "";
};
for(auto c : content)
{
current_line.append(1, c);
if(c == '\n')
flush_line();
}
flush_line();
return(result);
}
String preprocess_shared_unit_char_wise(Request* context, SharedUnit* su, String content)
{
String pc =
("#include \"")+context->server->config["COMPILER_SYS_PATH"] +"/src/lib/uce_lib.h\" \n"+
file_get_contents(
context->server->config["COMPILER_SYS_PATH"] + "/" + context->server->config["SETUP_TEMPLATE"]
)+
"#line 1\n";
String token = "";
String html_buffer = "";
u8 mode = 0;
u32 depth = 0;
bool inside_quote = false;
u32 source_length = content.length();
String current_line = "";
for(u32 i = 0; i < source_length; i++)
{
char c = content[i];
char c1 = (i + 1 < source_length) ? content[i + 1] : '\0';
char c2 = (i + 2 < source_length) ? content[i + 2] : '\0';
current_line.append(1, c);
if(mode == 1)
{
if(c == '<' && c1 == '/' && c2 == '>')
{
if(depth > 0)
{
depth -= 1;
token.append("</>");
html_buffer.append("</>");
i += 2;
}
else
{
i += 2;
pc.append(process_text_literal(context, su, html_buffer));
mode = 0;
}
}
else if(c == '<' && c1 == '>')
{
depth += 1;
token.append("<>");
html_buffer.append("<>");
i += 1;
}
else
{
token.append(1, c);
html_buffer.append(1, c);
}
}
else if(!inside_quote && c == '<' && c1 == '>')
{
mode = 1;
token = "";
html_buffer = "";
i += 1;
}
else if(c == '\"')
{
inside_quote = !inside_quote;
pc.append(1, c);
}
else // we're in C++ code here
{
pc.append(1, c);
if(c == 10 && current_line.substr(0, 6) == "#load ")
{
//printf("#load directive\n");
pc.resize(pc.length() - current_line.length());
nibble(current_line, "\"");
String unit_name = nibble(current_line, "\"");
//printf("(i) #load %s\n", unit_name.c_str());
SharedUnit* sub_su = compiler_load_shared_unit(context, unit_name, su->src_path, true);
if(sub_su)
{
pc.append("#include \"" + sub_su->bin_path + "/" + sub_su->pre_file_name + "\"\n");
}
}
else
{
String trimmed_line = trim(current_line);
if(c == 10 && trimmed_line.length() > 7 && trimmed_line.substr(0, 6) == "EXPORT" && isspace(trimmed_line[6]))
{
current_line = "";
auto end_declaration_pos = content.find("{", i);
if(end_declaration_pos != std::string::npos)
{
pc.append(1, '\n');
String declaration = trim(content.substr(i, end_declaration_pos - i));
su->api_declarations.push_back(declaration+";\n");
//printf("declaration found: %s\n", declaration.c_str());
}
}
}
}
if(c == 10)
current_line = "";
}
return(pc);
}
String preprocess_shared_unit(Request* context, SharedUnit* su)
{
String content = file_get_contents(su->file_name);
content = preprocess_named_render_syntax(content);
return(
preprocess_shared_unit_char_wise(
context,
su,
content
)
);
return(compiler_preprocess_source(context, su, content));
}
void setup_unit_paths(Request* context, SharedUnit* su, String file_name)
-1
View File
@@ -5,7 +5,6 @@
#define WS(X) extern "C" void __uce_websocket(Request& context)
#define EXPORT extern "C"
String process_html_literal(Request* context, SharedUnit* su, String content);
String preprocess_shared_unit(Request* context, SharedUnit* su);
void setup_unit_paths(Request* context, SharedUnit* su, String file_name);
void load_shared_unit(Request* context, SharedUnit* su, String file_name);
+1
View File
@@ -6,6 +6,7 @@
#include "hash.cpp"
#include "sys.cpp"
#include "uri.cpp"
#include "compiler-parser.cpp"
#include "compiler.cpp"
#include "markdown.cpp"
#include "mysql-connector.cpp"