udo/uce
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

memory arenas (sort of)

Browse Source
This commit is contained in:
Udo 2021-12-27 23:54:15 +00:00
parent 3bc6fe7f90
commit 0889c69582
10 changed files with 240 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
bin/uce_fastcgi.debug.linux.bin
View File

Binary file not shown.

386
src/fastcgi/src/fcgicc.cc
View File

@ -61,24 +61,14 @@ FastCGIServer::Connection::Connection() :
close_responsibility(false),
close_socket(false)
{
}
int
FastCGIServer::HandlerBase::operator()(FastCGIRequest&)
FastCGIServer::FastCGIServer()
{
return 0;
}
FastCGIServer::FastCGIServer() :
handle_request(new HandlerBase),
handle_data(new HandlerBase),
handle_complete(new HandlerBase)
{
}
FastCGIServer::~FastCGIServer()
{
if(my_pid != parent_pid) // if we're a child process, we must not close the handles
@ -101,38 +91,6 @@ FastCGIServer::~FastCGIServer()
delete it->second;
}
delete handle_request;
delete handle_data;
delete handle_complete;
}
void
FastCGIServer::request_handler(int (* function)(FastCGIRequest&))
{
handle_request = new StaticHandler(function);
}
void
FastCGIServer::data_handler(int (* function)(FastCGIRequest&))
{
handle_data = new StaticHandler(function);
}
void
FastCGIServer::complete_handler(int (* function)(FastCGIRequest&))
{
handle_complete = new StaticHandler(function);
}
void
FastCGIServer::set_handler(HandlerBase*& handler, HandlerBase* new_handler)
{
delete handler;
handler = new_handler;
}
@ -313,6 +271,15 @@ FastCGIServer::process_forever()
process();
}
int
FastCGIServer::call_completion_handler(FastCGIRequest& request)
{
//printf("- request complete\n");
switch_to_arena(request.mem);
auto result = on_complete(request);
switch_to_system_alloc();
return(result);
}
void
FastCGIServer::process_connection_read(Connection& connection)
@ -336,153 +303,172 @@ FastCGIServer::process_connection_read(Connection& connection)
RequestID request_id = (header.requestIdB1 << 8) + header.requestIdB0;
switch (header.type) {
case FCGI_GET_VALUES: {
Pairs pairs = parse_pairs(content, content_length);
switch (header.type)
{
case FCGI_GET_VALUES:
{
Pairs pairs = parse_pairs(content, content_length);
std::string::size_type base = connection.output_buffer.size();
connection.output_buffer.push_back(FCGI_VERSION_1);
connection.output_buffer.push_back(FCGI_GET_VALUES_RESULT);
connection.output_buffer.append(FCGI_HEADER_LEN - 2, 0);
std::string::size_type base = connection.output_buffer.size();
connection.output_buffer.push_back(FCGI_VERSION_1);
connection.output_buffer.push_back(FCGI_GET_VALUES_RESULT);
connection.output_buffer.append(FCGI_HEADER_LEN - 2, 0);
for (Pairs::iterator it = pairs.begin(); it != pairs.end(); ++it)
if (it->first == FCGI_MAX_CONNS)
write_pair(connection.output_buffer,
it->first, std::string("100"));
else if (it->first == FCGI_MAX_REQS)
write_pair(connection.output_buffer,
it->first, std::string("1000"));
else if (it->first == FCGI_MPXS_CONNS)
write_pair(connection.output_buffer,
it->first, std::string("1"));
for (Pairs::iterator it = pairs.begin(); it != pairs.end(); ++it)
if (it->first == FCGI_MAX_CONNS)
write_pair(connection.output_buffer,
it->first, std::string("100"));
else if (it->first == FCGI_MAX_REQS)
write_pair(connection.output_buffer,
it->first, std::string("1000"));
else if (it->first == FCGI_MPXS_CONNS)
write_pair(connection.output_buffer,
it->first, std::string("1"));
std::string::size_type len = connection.output_buffer.size() - base;
connection.output_buffer[base + 4] = (len >> 8) & 0xff;
connection.output_buffer[base + 5] = len & 0xff;
std::string::size_type len = connection.output_buffer.size() - base;
connection.output_buffer[base + 4] = (len >> 8) & 0xff;
connection.output_buffer[base + 5] = len & 0xff;
break;
}
case FCGI_BEGIN_REQUEST:
{
if (content_length < sizeof(FCGI_BeginRequestBody))
break;
}
case FCGI_BEGIN_REQUEST: {
if (content_length < sizeof(FCGI_BeginRequestBody))
break;
const FCGI_BeginRequestBody& body =
*reinterpret_cast<const FCGI_BeginRequestBody*>(content);
const FCGI_BeginRequestBody& body =
*reinterpret_cast<const FCGI_BeginRequestBody*>(content);
if (!(body.flags & FCGI_KEEP_CONN))
connection.close_responsibility = true;
if (!(body.flags & FCGI_KEEP_CONN))
connection.close_responsibility = true;
unsigned role = (body.roleB1 << 8) + body.roleB0;
if (role != FCGI_RESPONDER) {
FCGI_EndRequestRecord unknown;
bzero(&unknown, sizeof(unknown));
unknown.header.version = FCGI_VERSION_1;
unknown.header.type = FCGI_END_REQUEST;
unknown.header.contentLengthB0 = sizeof(unknown.body);
unknown.body.protocolStatus = FCGI_UNKNOWN_ROLE;
connection.output_buffer.append(
reinterpret_cast<const char*>(&unknown), sizeof(unknown));
if (connection.close_responsibility)
connection.close_socket = true;
break;
}
{
RequestList::iterator it = connection.requests.find(request_id);
if (it != connection.requests.end()) {
delete it->second;
connection.requests.erase(it);
}
}
RequestInfo* new_request = new RequestInfo;
try {
connection.requests.insert(RequestList::value_type(
request_id, new_request));
} catch (...) {
delete new_request;
throw;
}
break;
}
case FCGI_ABORT_REQUEST: {
RequestList::iterator it = connection.requests.find(request_id);
if (it == connection.requests.end())
break;
FCGI_EndRequestRecord aborted;
bzero(&aborted, sizeof(aborted));
aborted.header.version = FCGI_VERSION_1;
aborted.header.type = FCGI_END_REQUEST;
aborted.header.contentLengthB0 = sizeof(aborted.body);
aborted.body.appStatusB0 = 1;
aborted.body.protocolStatus = FCGI_REQUEST_COMPLETE;
connection.output_buffer.append(
reinterpret_cast<const char*>(&aborted), sizeof(aborted));
if (connection.close_responsibility)
connection.close_socket = true;
delete it->second;
connection.requests.erase(it);
break;
}
case FCGI_PARAMS: {
RequestList::iterator it = connection.requests.find(request_id);
if (it == connection.requests.end())
break;
RequestInfo& request = *it->second;
if (!request.params_closed)
if (content_length != 0)
request.params_buffer.append(content, content_length);
else {
request.params = parse_pairs(request.params_buffer.data(),
request.params_buffer.size());
request.params_buffer.clear();
request.params_closed = true;
request.status = (*handle_request)(request);
if (request.status == 0 && !request.in.empty()) {
request.status = (*handle_data)(request);
if (request.status == 0 && request.in_closed)
request.status = (*handle_complete)(request);
}
process_write_request(connection, request_id, request);
}
break;
}
case FCGI_STDIN: {
RequestList::iterator it = connection.requests.find(request_id);
if (it == connection.requests.end())
break;
RequestInfo& request = *it->second;
if (!request.in_closed)
if (content_length != 0) {
request.in.append(content, content_length);
if (request.params_closed && request.status == 0) {
request.status = (*handle_data)(request);
process_write_request(connection, request_id, request);
}
} else {
request.in_closed = true;
if (request.params_closed && request.status == 0) {
request.status = (*handle_complete)(request);
process_write_request(connection, request_id, request);
}
}
break;
}
case FCGI_DATA:
break;
default: {
FCGI_UnknownTypeRecord unknown;
unsigned role = (body.roleB1 << 8) + body.roleB0;
if (role != FCGI_RESPONDER)
{
FCGI_EndRequestRecord unknown;
bzero(&unknown, sizeof(unknown));
unknown.header.version = FCGI_VERSION_1;
unknown.header.type = FCGI_UNKNOWN_TYPE;
unknown.header.type = FCGI_END_REQUEST;
unknown.header.contentLengthB0 = sizeof(unknown.body);
unknown.body.type = header.type;
unknown.body.protocolStatus = FCGI_UNKNOWN_ROLE;
connection.output_buffer.append(
reinterpret_cast<const char*>(&unknown), sizeof(unknown));
}
reinterpret_cast<const char*>(&unknown), sizeof(unknown));
if (connection.close_responsibility)
connection.close_socket = true;
break;
}
{
RequestList::iterator it = connection.requests.find(request_id);
if (it != connection.requests.end())
{
//printf("- delete request object\n");
switch_to_arena(it->second->mem);
delete it->second;
switch_to_system_alloc();
connection.requests.erase(it);
}
}
//auto arena = new MemoryArena(server_state.config.MAX_MEMORY, "request");
request_arena->clear();
switch_to_arena(request_arena);
RequestInfo* new_request = new RequestInfo();
new_request->mem = request_arena;
new_request->stats.time_init = microtime();
switch_to_system_alloc();
connection.requests[request_id] = new_request;
break;
}
case FCGI_ABORT_REQUEST:
{
RequestList::iterator it = connection.requests.find(request_id);
if (it == connection.requests.end())
break;
FCGI_EndRequestRecord aborted;
bzero(&aborted, sizeof(aborted));
aborted.header.version = FCGI_VERSION_1;
aborted.header.type = FCGI_END_REQUEST;
aborted.header.contentLengthB0 = sizeof(aborted.body);
aborted.body.appStatusB0 = 1;
aborted.body.protocolStatus = FCGI_REQUEST_COMPLETE;
connection.output_buffer.append(
reinterpret_cast<const char*>(&aborted), sizeof(aborted));
if (connection.close_responsibility)
connection.close_socket = true;
delete it->second;
connection.requests.erase(it);
break;
}
case FCGI_PARAMS:
{
RequestList::iterator it = connection.requests.find(request_id);
if (it == connection.requests.end())
break;
RequestInfo& request = *it->second;
switch_to_arena(it->second->mem);
if (!request.params_closed)
if (content_length != 0)
request.params_buffer.append(content, content_length);
else {
request.params = parse_pairs(request.params_buffer.data(),
request.params_buffer.size());
request.params_buffer.clear();
request.params_closed = true;
request.status = on_request(request);
if (request.status == 0 && !request.in.empty())
{
request.status = on_data(request);
if (request.status == 0 && request.in_closed)
request.status = call_completion_handler(request);
}
process_write_request(connection, request_id, request);
}
switch_to_system_alloc();
break;
}
case FCGI_STDIN:
{
RequestList::iterator it = connection.requests.find(request_id);
if (it == connection.requests.end())
break;
RequestInfo& request = *it->second;
switch_to_arena(it->second->mem);
if (!request.in_closed)
if (content_length != 0) {
request.in.append(content, content_length);
if (request.params_closed && request.status == 0)
{
request.status = on_data(request);
process_write_request(connection, request_id, request);
}
} else {
request.in_closed = true;
if (request.params_closed && request.status == 0) {
request.status = call_completion_handler(request);
process_write_request(connection, request_id, request);
}
}
switch_to_system_alloc();
break;
}
case FCGI_DATA:
break;
default:
{
FCGI_UnknownTypeRecord unknown;
bzero(&unknown, sizeof(unknown));
unknown.header.version = FCGI_VERSION_1;
unknown.header.type = FCGI_UNKNOWN_TYPE;
unknown.header.contentLengthB0 = sizeof(unknown.body);
unknown.body.type = header.type;
connection.output_buffer.append(
reinterpret_cast<const char*>(&unknown), sizeof(unknown));
}
}
n += FCGI_HEADER_LEN + content_length + header.paddingLength;
@ -499,16 +485,21 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
if (!request.out.empty())
{
write_data(connection.output_buffer, id, request.out, FCGI_STDOUT);
switch_to_arena(request.mem);
request.out.clear();
switch_to_system_alloc();
}
if (!request.err.empty())
{
write_data(connection.output_buffer, id, request.err, FCGI_STDERR);
switch_to_arena(request.mem);
request.err.clear();
switch_to_system_alloc();
}
if ((request.in_closed || request.status != 0) &&
!request.output_closed)
{
switch_to_arena(request.mem);
request.out =
var_dump(request.header, "", "\r\n") +
var_dump(request.set_cookies, "", "\r\n") +
@ -521,15 +512,19 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
}
request.ob_stack.clear();
switch_to_system_alloc();
write_data(connection.output_buffer, id, request.out, FCGI_STDOUT);
write_data(connection.output_buffer, id, request.err, FCGI_STDERR);
request.stats.time_end = microtime();
printf("(r) pid:%i\t%s\t%0.6fs\t%0.1fkB\n",
printf("(r) pid:%i\t%s\t%0.6fs\tfps:%0.0f\tout:%0.1fkB\tmem:%0.0f/%0.0fkB\n",
my_pid,
request.params["REQUEST_URI"].c_str(),
request.stats.time_end - request.stats.time_start,
(f32)(request.out.length()/1024)
1.0 / (request.stats.time_end - request.stats.time_start),
(f32)(request.out.length()/1024),
(f32)(request.mem->size/1024),
(f32)(request.mem->capacity/1024)
);
FCGI_EndRequestRecord complete;
@ -550,7 +545,9 @@ FastCGIServer::process_write_request(Connection& connection, RequestID id,
connection.close_socket = true;
request.output_closed = true;
//printf("- output done\n");
}
switch_to_system_alloc();
}
@ -558,14 +555,19 @@ void
FastCGIServer::process_connection_write(Connection& connection)
{
for (RequestList::iterator it = connection.requests.begin();
it != connection.requests.end();) {
process_write_request(connection, it->first, *it->second);
if (it->second->params_closed && it->second->in_closed) {
RequestInfo* request = it->second;
connection.requests.erase(it++);
delete request;
} else
++it;
it != connection.requests.end();)
{
process_write_request(connection, it->first, *it->second);
if (it->second->params_closed && it->second->in_closed)
{
switch_to_arena(it->second->mem);
//printf("- process_connection_write close\n");
delete it->second;
switch_to_system_alloc();
connection.requests.erase(it++);
}
else
++it;
}
}

56
src/fastcgi/src/fcgicc.h
View File

@ -45,25 +45,9 @@ public:
~FastCGIServer();
// called when the parameters and standard input have been receieved
void request_handler(int (* function)(FastCGIRequest&));
template<class C>
void request_handler(C& object, int (C::* function)(FastCGIRequest&)) {
set_handler(handle_request, new Handler<C>(object, function));
}
// called when new data appears on stdin
void data_handler(int (* function)(FastCGIRequest&));
template<class C>
void data_handler(C& object, int (C::* function)(FastCGIRequest&)) {
set_handler(handle_data, new Handler<C>(object, function));
}
// called when the complete request has been received
void complete_handler(int (* function)(FastCGIRequest&));
template<class C>
void complete_handler(C& object, int (C::* function)(FastCGIRequest&)) {
set_handler(handle_complete, new Handler<C>(object, function));
}
std::function<int(FastCGIRequest&)> on_request = 0;
std::function<int(FastCGIRequest&)> on_data = 0;
std::function<int(FastCGIRequest&)> on_complete = 0;
void listen(unsigned tcp_port);
void listen(const std::string& local_path);
@ -72,6 +56,8 @@ public:
void process(int timeout_ms = -1); // timeout_ms<0 blocks forever
void process_forever();
int call_completion_handler(FastCGIRequest& request);
protected:
struct RequestInfo : FastCGIRequest {
RequestInfo();
@ -113,38 +99,6 @@ protected:
static void write_data(std::string& buffer, RequestID id,
const std::string& input, unsigned char type);
struct HandlerBase {
virtual int operator()(FastCGIRequest&);
};
struct StaticHandler : public HandlerBase {
StaticHandler(int (* p_function)(FastCGIRequest&)) :
function(p_function) {}
int operator()(FastCGIRequest& request) {
return function(request);
}
int (* function)(FastCGIRequest&);
};
template<class C>
struct Handler : public HandlerBase {
Handler(C& p_object, int (C::* p_function)(FastCGIRequest&)) :
object(p_object), function(p_function) {}
int operator()(FastCGIRequest& request) {
return (object.*function)(request);
}
C& object;
int (C::* function)(FastCGIRequest&);
};
void set_handler(HandlerBase*&, HandlerBase*);
HandlerBase* handle_request;
HandlerBase* handle_data;
HandlerBase* handle_complete;
};
#endif // !FCGICC_H

2
src/lib/compiler.cpp
View File

@ -295,7 +295,7 @@ void compiler_invoke(Request* context, String file_name, DTree& call_param)
//printf("(i) invoke(%s)\n", file_name.c_str());
switch_to_system_alloc();
auto su = get_shared_unit(context, file_name);
switch_to_arena(context->request_arena);
switch_to_arena(context->mem);
if(!su)
{
printf("Error loading unit %s\n", file_name.c_str());

16
src/lib/mysql-connector.cpp
View File

@ -5,22 +5,28 @@
bool MySQL::connect(String host, String username, String password)
{
switch_to_system_alloc();
connection = mysql_init(NULL);
if (connection == NULL)
{
fprintf(stderr, "%s\n", mysql_error((MYSQL*)connection));
auto e = mysql_error((MYSQL*)connection);
fprintf(stderr, "%s\n", e);
switch_to_arena(context->mem);
statement_info.assign(e);
return(false);
}
if (mysql_real_connect((MYSQL*)connection, host.c_str(), username.c_str(), password.c_str(),
NULL, 0, NULL, 0) == NULL)
{
fprintf(stderr, "%s\n", mysql_error((MYSQL*)connection));
auto e = mysql_error((MYSQL*)connection);
fprintf(stderr, "%s\n", e);
mysql_close((MYSQL*)connection);
switch_to_arena(context->mem);
statement_info.assign(e);
return(false);
}
printf("(i) MySQL Connection established to %s\n", host.c_str());
/*
if (mysql_query(con, "CREATE DATABASE testdb"))
{
@ -29,6 +35,8 @@ bool MySQL::connect(String host, String username, String password)
exit(1);
}
*/
switch_to_arena(context->mem);
statement_info = String("connected");
context->resources.mysql_connections.push_back(connection);
return(true);
}
@ -282,6 +290,8 @@ String MySQL::error()
void cleanup_mysql_connections()
{
switch_to_system_alloc();
for(auto& con : context->resources.mysql_connections)
mysql_close((MYSQL*)con);
switch_to_arena(context->mem);
}

12
src/lib/types.h
View File

@ -42,8 +42,8 @@ String operator+(String lhs, f32 rhs) {
return(lhs + std::to_string(rhs));
}
#define DEBUG_MEMORY_OFF;
#define NO_GLOBAL_ARENA_ALLOCATOR;
#define DEBUG_MEMORY_OFF
#define GLOBAL_ARENA_ALLOCATOR
struct MemoryArena {
@ -84,7 +84,7 @@ struct MemoryArena {
return(0);
}
size += size_aligned;
#ifdef DEBUG_MEMORY
#ifdef DEBUG_MEMORY_DETAILED
printf("(i) memory arena '%s' [+%llu]:%p alloc %llu/%llu bytes\n", name.c_str(), size, result, size_needed, size_aligned);
#endif
return(result);
@ -96,12 +96,16 @@ MemoryArena* current_memory_arena = 0;
void switch_to_system_alloc()
{
#ifdef GLOBAL_ARENA_ALLOCATOR
current_memory_arena = 0;
#endif
}
void switch_to_arena(MemoryArena* a)
{
#ifdef GLOBAL_ARENA_ALLOCATOR
current_memory_arena = a;
#endif
}
#ifdef GLOBAL_ARENA_ALLOCATOR
@ -239,7 +243,7 @@ struct Request {
u64 random_seed;
u64 random_index;
MemoryArena* request_arena;
MemoryArena* mem;
String in;
std::vector<std::ostringstream*> ob_stack;

17
src/linux_fastcgi.cpp
View File

@ -1,19 +1,16 @@
#include "lib/uce_lib.cpp"
ServerState server_state;
MemoryArena* request_arena = new MemoryArena(server_state.config.MAX_MEMORY, "request");
#include "fastcgi/src/fcgicc.cc"
FastCGIServer server;
ServerState server_state;
MemoryArena* request_arena = 0;
int handle_request(FastCGIRequest& request) {
// This is always the first event to occur. It occurs when the
// server receives all parameters. There may be more data coming on the
// standard input stream.
request.request_arena = request_arena;
request.stats.time_init = microtime();
request.ob_start(); // this one is for the headers
request.ob_start(); // this one is for the content
if (request.params.count("REQUEST_URI"))
return 0; // OK, continue processing
else
@ -50,6 +47,7 @@ int handle_complete(FastCGIRequest& request) {
request.get = parse_query(request.params["QUERY_String"]);
request.random_index = 0;
request.random_seed = noise64(*reinterpret_cast<u64*>(&request.stats.time_start));
request.ob_start();
if(request.params["HTTP_COOKIE"].length() > 0)
request.cookies = parse_cookies(request.params["HTTP_COOKIE"]);
@ -72,7 +70,6 @@ int handle_complete(FastCGIRequest& request) {
// printf("(i) request ready\n");
request.invoke(request.params["SCRIPT_FILENAME"]);
switch_to_system_alloc();
for( auto &f : request.uploaded_files)
{
@ -89,11 +86,15 @@ int handle_complete(FastCGIRequest& request) {
void listen_for_connections()
{
//request_arena = new MemoryArena(server_state.config.MAX_MEMORY, "request");
signal(SIGSEGV, on_segfault);
server.on_request = &handle_request;
server.on_data = &handle_data;
server.on_complete = &handle_complete;
/*
server.request_handler(&handle_request);
server.data_handler(&handle_data);
server.complete_handler(&handle_complete);
*/
for(;;)
{
//if(request_arena) request_arena->clear();

4
test/index.uce
View File

@ -29,8 +29,12 @@ RENDER()
<pre><?
print("Worker PID: ", my_pid, "\n");
print("Parent PID: ", parent_pid, " \n");
print("Output buffer size: ", context->ob->str().length(), " \n");
?></pre>
<pre><?= (var_dump(p)) ?></pre>
<div><?
print("Output buffer size: ", context->ob->str().length(), " \n");
?></div>
</>
String test;

6
test/mysql.uce
View File

@ -21,7 +21,6 @@ RENDER()
MySQL con;
if(con.connect("localhost", "root", ""))
{
print("connection established\n");
print("error: "+con.error()+"\n");
print("quotation test: "+con.escape("‘\"' or 1=1;–.")+"\n");
print("query parse: "+con.parse_query_parameters(query, params)+"\n");
@ -38,8 +37,13 @@ RENDER()
}
?></pre>
<div><?
print("connection status: ", con.statement_info, "\n");
?></div>
<label>CGI Params</label>
<pre><?= var_dump(context->params) ?></pre>
</html></>

2
test/test2/working-dir-test.uce
View File

@ -2,5 +2,5 @@
RENDER()
{
echo("Sub-Invoke Working dir: " + get_cwd() + "\n");
print("Sub-Invoke Working dir: ", get_cwd(), "\n");
}