Add a canonical token-preserving Capy formatter.

scripts/test_capy_native_frontend.cpp

@@ -1,4 +1,5 @@
 #include "frontend.h"
+#include <algorithm>
 #include <cassert>
 #include <fstream>
 #include <iostream>
@@ -334,6 +335,31 @@ int main(int argc, char** argv)
 		auto tokens = lexer.tokens();
 		assert(tokens[0].location.column == 1 && tokens[1].location.column == 3 && tokens[1].location.offset == 2);
 	}
+	{
+		const std::string source = "#import   \"child.capy\" as child\nfunction CLI(request : dval) { var text := \"héllo\\nworld\" // keep\nif true { print(\"this string makes the call exceed the formatter line limit and proves that every map entry receives one extra tab\", page: 1, limit: 2) } else { return }\n}\n";
+		const std::string formatted = format_source(source);
+		assert(formatted == "#import   \"child.capy\" as child\nfunction CLI(request: dval)\n{\n\tvar text := \"héllo\\nworld\" // keep\n\tif true\n\t{\n\t\tprint(\n\t\t\t\"this string makes the call exceed the formatter line limit and proves that every map entry receives one extra tab\",\n\t\t\tpage: 1,\n\t\t\tlimit: 2\n\t\t)\n\t}\n\telse\n\t{\n\t\treturn\n\t}\n}\n");
+		assert(format_source(formatted) == formatted);
+		assert(parse(source).items.size() == parse(formatted).items.size());
+		auto input_tokens = Lexer(source).tokens(), output_tokens = Lexer(formatted).tokens();
+		input_tokens.erase(std::remove_if(input_tokens.begin(), input_tokens.end(), [](const Token& token) { return token.kind == TokenKind::newline; }), input_tokens.end());
+		output_tokens.erase(std::remove_if(output_tokens.begin(), output_tokens.end(), [](const Token& token) { return token.kind == TokenKind::newline; }), output_tokens.end());
+		assert(input_tokens.size() == output_tokens.size());
+		for (std::size_t index = 0; index < input_tokens.size(); ++index)
+			assert(input_tokens[index].kind == output_tokens[index].kind && input_tokens[index].text == output_tokens[index].text);
+		assert(format_source("function CLI(request : dval) { <>  raw markup <?= string(\"x\") ?>  </> }\n") == "function CLI(request: dval)\n{\n\t<>  raw markup <?= string(\"x\") ?>  </>\n}\n");
+	}
+	{
+		const std::string source = "#exports CLI  \n\n\nfunction CLI(request : dval) { if !bool(false) { print(\":\", \")\", \",\", \"}\", \"function\") } }\n// last comment  \n";
+		const std::string formatted = format_source(source);
+		assert(formatted.starts_with("#exports CLI  \n\nfunction"));
+		assert(formatted.find("if !bool(false)") != std::string::npos);
+		assert(formatted.ends_with("// last comment  \n"));
+		assert(format_source(formatted) == formatted);
+		assert(parse(source).items.size() == parse(formatted).items.size());
+		const std::string host = "host function example(value : s32) s32 // host  \nfunction next() {}\n";
+		assert(parse(format_source(host)).items.size() == 2);
+	}
 	for (int index = 1; index < argc; ++index)
 	{
 		std::ifstream input(argv[index], std::ios::binary);