udo/uce
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bound Wasm artifact metadata loading

Browse Source
This commit is contained in:
udo
2026-07-19 15:43:52 +00:00
parent 15fdfe725d
commit 881dba078a
6 changed files with 733 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/wasm/worker.cpp
+235 -12
View File
@@ -123,6 +123,7 @@ struct WasmUnitModuleLoadProfile
{
bool cache_hit = false;
bool serialized_cache_hit = false;
bool timed_out = false;
u64 lookup_us = 0;
u64 read_us = 0;
u64 read_bytes = 0;
@@ -725,6 +726,8 @@ static bool wasm_read_uleb(const std::vector<u8>& buf, size_t& pos, size_t end,
if(pos >= end || shift >= 64)
return(false);
u8 byte = buf[pos++];
if(shift == 63 && (byte & 0x7e) != 0)
return(false);
out |= ((u64)(byte & 0x7f)) << shift;
if((byte & 0x80) == 0)
return(true);
@@ -732,8 +735,19 @@ static bool wasm_read_uleb(const std::vector<u8>& buf, size_t& pos, size_t end,
}
}
static bool wasm_parse_sections(const std::vector<u8>& bytes, WasmDylinkInfo& dylink, WasmAbiInfo& abi, String& error)
static bool wasm_parse_sections(const std::vector<u8>& bytes, WasmDylinkInfo& dylink, WasmAbiInfo& abi, String& error,
bool deadline_active = false, std::chrono::steady_clock::time_point deadline = {}, bool* timed_out = 0)
{
auto deadline_expired = [&]() {
if(!deadline_active || std::chrono::steady_clock::now() < deadline)
return(false);
if(timed_out)
*timed_out = true;
error = "wasm metadata parse timed out";
return(true);
};
if(timed_out)
*timed_out = false;
if(bytes.size() < 8 || memcmp(bytes.data(), "\0asm", 4) != 0)
{
error = "not a wasm module";
@@ -745,8 +759,13 @@ static bool wasm_parse_sections(const std::vector<u8>& bytes, WasmDylinkInfo& dy
return(false);
}
size_t pos = 8;
bool dylink_seen = false;
bool abi_seen = false;
bool module_seen = false;
while(pos < bytes.size())
{
if(deadline_expired())
return(false);
u8 section_id = bytes[pos++];
u64 size = 0;
if(!wasm_read_uleb(bytes, pos, bytes.size(), size) || size > bytes.size() - pos)
@@ -764,12 +783,30 @@ static bool wasm_parse_sections(const std::vector<u8>& bytes, WasmDylinkInfo& dy
error = "malformed custom section name";
return(false);
}
if(name_len > 64)
{
pos = end;
continue;
}
String name((const char*)bytes.data() + cursor, (size_t)name_len);
cursor += (size_t)name_len;
if((name == "dylink.0" || name == "uce.abi" || name == "uce.module") && size > 1024 * 1024)
{
error = "oversized wasm metadata section";
return(false);
}
if(name == "dylink.0")
{
if(dylink_seen)
{
error = "duplicate dylink.0 metadata section";
return(false);
}
dylink_seen = true;
while(cursor < end)
{
if(deadline_expired())
return(false);
u8 sub = bytes[cursor++];
u64 sub_len = 0;
if(!wasm_read_uleb(bytes, cursor, end, sub_len) || sub_len > end - cursor)
@@ -808,11 +845,19 @@ static bool wasm_parse_sections(const std::vector<u8>& bytes, WasmDylinkInfo& dy
}
else if(name == "uce.abi")
{
if(abi_seen)
{
error = "duplicate uce.abi metadata section";
return(false);
}
abi_seen = true;
String text((const char*)bytes.data() + cursor, end - cursor);
abi.found = true;
size_t line_start = 0;
while(line_start < text.size())
{
if(deadline_expired())
return(false);
size_t line_end = text.find('\n', line_start);
if(line_end == String::npos)
line_end = text.size();
@@ -825,7 +870,15 @@ static bool wasm_parse_sections(const std::vector<u8>& bytes, WasmDylinkInfo& dy
}
}
else if(name == "uce.module")
{
if(module_seen)
{
error = "duplicate uce.module metadata section";
return(false);
}
module_seen = true;
abi.module_name.assign((const char*)bytes.data() + cursor, end - cursor);
}
}
pos = end;
}
@@ -849,6 +902,98 @@ static bool wasm_read_file(const String& path, std::vector<u8>& out)
return((bool)in);
}
static bool wasm_read_file_deadline(const String& path, std::vector<u8>& out, String& error, u64& bytes_read, u64& read_count,
bool& timed_out, bool deadline_active, std::chrono::steady_clock::time_point deadline, const struct stat& expected_st)
{
bytes_read = 0;
read_count = 0;
timed_out = false;
auto expired = [&]() {
if(!deadline_active || std::chrono::steady_clock::now() < deadline)
return(false);
timed_out = true;
error = "wasm artifact read timed out";
return(true);
};
int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if(fd < 0)
return(false);
struct stat st;
u64 expected_modified_ns = (u64)expected_st.st_mtim.tv_sec * 1000000000ull + (u64)expected_st.st_mtim.tv_nsec;
u64 expected_changed_ns = (u64)expected_st.st_ctim.tv_sec * 1000000000ull + (u64)expected_st.st_ctim.tv_nsec;
if(fstat(fd, &st) != 0 || !S_ISREG(st.st_mode))
{
close(fd);
error = "wasm artifact is not a regular file";
return(false);
}
u64 modified_ns = (u64)st.st_mtim.tv_sec * 1000000000ull + (u64)st.st_mtim.tv_nsec;
u64 changed_ns = (u64)st.st_ctim.tv_sec * 1000000000ull + (u64)st.st_ctim.tv_nsec;
if(st.st_dev != expected_st.st_dev || st.st_ino != expected_st.st_ino || st.st_mode != expected_st.st_mode ||
modified_ns != expected_modified_ns || changed_ns != expected_changed_ns || st.st_size != expected_st.st_size)
{
close(fd);
error = "wasm artifact changed while loading";
return(false);
}
out.clear();
u64 offset = 0;
while(offset < (u64)st.st_size)
{
if(expired())
break;
size_t wanted = (size_t)std::min<u64>(64 * 1024, (u64)st.st_size - offset);
if(offset > out.max_size() || wanted > out.max_size() - (size_t)offset)
{
error = "wasm artifact is too large";
break;
}
try
{
out.resize((size_t)offset + wanted);
}
catch(const std::bad_alloc&)
{
error = "cannot allocate wasm artifact buffer";
break;
}
catch(const std::length_error&)
{
error = "wasm artifact is too large";
break;
}
if(expired())
break;
ssize_t count = pread(fd, out.data() + offset, wanted, (off_t)offset);
if(count < 0 && errno == EINTR)
continue;
if(count <= 0)
{
error = "cannot read wasm artifact";
break;
}
offset += (u64)count;
out.resize((size_t)offset);
bytes_read += (u64)count;
read_count++;
if(expired())
break;
}
struct stat final_st;
if(error == "" && (fstat(fd, &final_st) != 0 || final_st.st_dev != st.st_dev || final_st.st_ino != st.st_ino ||
final_st.st_mode != st.st_mode || final_st.st_mtim.tv_sec != st.st_mtim.tv_sec ||
final_st.st_mtim.tv_nsec != st.st_mtim.tv_nsec || final_st.st_ctim.tv_sec != st.st_ctim.tv_sec ||
final_st.st_ctim.tv_nsec != st.st_ctim.tv_nsec || final_st.st_size != st.st_size))
error = "wasm artifact changed while loading";
close(fd);
if(error != "")
{
out.clear();
return(false);
}
return(true);
}
static bool wasm_source_map_load(const String& path, WasmSourceMap& map)
{
std::ifstream input(path);
@@ -905,6 +1050,17 @@ struct WasmMetadataReader
u64 buffer_offset = 0;
size_t buffer_size = 0;
u8 buffer[4096];
bool deadline_active = false;
std::chrono::steady_clock::time_point deadline;
bool timed_out = false;
bool expired()
{
if(!deadline_active || std::chrono::steady_clock::now() < deadline)
return(false);
timed_out = true;
return(true);
}
bool read(u64 offset, u8* out, size_t size)
{
@@ -912,6 +1068,8 @@ struct WasmMetadataReader
return(false);
while(size > 0)
{
if(expired())
return(false);
if(offset < buffer_offset || offset >= buffer_offset + buffer_size)
{
buffer_offset = offset;
@@ -919,6 +1077,8 @@ struct WasmMetadataReader
size_t wanted = (size_t)std::min<u64>(sizeof(buffer), file_size - offset);
while(true)
{
if(expired())
return(false);
ssize_t count = pread(fd, buffer, wanted, (off_t)offset);
if(count < 0 && errno == EINTR)
continue;
@@ -927,6 +1087,8 @@ struct WasmMetadataReader
buffer_size = (size_t)count;
bytes_read += (u64)count;
read_count++;
if(expired())
return(false);
break;
}
}
@@ -949,6 +1111,8 @@ struct WasmMetadataReader
u8 byte = 0;
if(!read(pos++, &byte, 1))
return(false);
if(shift == 63 && (byte & 0x7e) != 0)
return(false);
out |= ((u64)(byte & 0x7f)) << shift;
if((byte & 0x80) == 0)
return(true);
@@ -970,15 +1134,16 @@ static void wasm_write_uleb(std::vector<u8>& out, u64 value)
}
static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadata, String& error, u64& bytes_read, u64& read_count,
u64 expected_modified_ns, u64 expected_changed_ns, u64 expected_size)
bool& timed_out, bool deadline_active, std::chrono::steady_clock::time_point deadline, const struct stat& expected_st)
{
bytes_read = 0;
read_count = 0;
timed_out = false;
int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if(fd < 0)
return(false);
struct stat st;
if(fstat(fd, &st) != 0 || st.st_size < 8)
if(fstat(fd, &st) != 0 || !S_ISREG(st.st_mode) || st.st_size < 8)
{
close(fd);
error = "not a wasm module";
@@ -986,7 +1151,10 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
}
u64 modified_ns = (u64)st.st_mtim.tv_sec * 1000000000ull + (u64)st.st_mtim.tv_nsec;
u64 changed_ns = (u64)st.st_ctim.tv_sec * 1000000000ull + (u64)st.st_ctim.tv_nsec;
if(modified_ns != expected_modified_ns || changed_ns != expected_changed_ns || (u64)st.st_size != expected_size)
u64 expected_modified_ns = (u64)expected_st.st_mtim.tv_sec * 1000000000ull + (u64)expected_st.st_mtim.tv_nsec;
u64 expected_changed_ns = (u64)expected_st.st_ctim.tv_sec * 1000000000ull + (u64)expected_st.st_ctim.tv_nsec;
if(st.st_dev != expected_st.st_dev || st.st_ino != expected_st.st_ino || st.st_mode != expected_st.st_mode ||
modified_ns != expected_modified_ns || changed_ns != expected_changed_ns || st.st_size != expected_st.st_size)
{
close(fd);
error = "wasm artifact changed while loading metadata";
@@ -996,19 +1164,30 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
WasmMetadataReader reader;
reader.fd = fd;
reader.file_size = file_size;
reader.deadline_active = deadline_active;
reader.deadline = deadline;
u8 header[8];
if(!reader.read(0, header, sizeof(header)) || memcmp(header, "\0asm\1\0\0\0", sizeof(header)) != 0)
{
bytes_read = reader.bytes_read;
read_count = reader.read_count;
timed_out = reader.timed_out;
close(fd);
error = "not a supported wasm module";
error = timed_out ? "wasm metadata read timed out" : "not a supported wasm module";
return(false);
}
metadata.assign(header, header + sizeof(header));
u64 pos = sizeof(header);
bool dylink_seen = false;
bool abi_seen = false;
bool module_seen = false;
while(pos < file_size)
{
if(reader.expired())
{
error = "wasm metadata read timed out";
break;
}
u8 section_id = 0;
if(!reader.read(pos++, &section_id, 1))
{
@@ -1043,6 +1222,16 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
}
if(name == "dylink.0" || name == "uce.abi" || name == "uce.module")
{
bool duplicate = (name == "dylink.0" && dylink_seen) || (name == "uce.abi" && abi_seen) ||
(name == "uce.module" && module_seen);
if(duplicate)
{
error = "duplicate " + name + " metadata section";
break;
}
if(name == "dylink.0") dylink_seen = true;
else if(name == "uce.abi") abi_seen = true;
else module_seen = true;
if(section_size > 1024 * 1024)
{
error = "oversized wasm metadata section";
@@ -1063,6 +1252,9 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
}
bytes_read = reader.bytes_read;
read_count = reader.read_count;
timed_out = reader.timed_out || reader.expired();
if(timed_out)
error = "wasm metadata read timed out";
struct stat final_st;
if(error == "" && (fstat(fd, &final_st) != 0 || final_st.st_dev != st.st_dev || final_st.st_ino != st.st_ino ||
final_st.st_mtim.tv_sec != st.st_mtim.tv_sec || final_st.st_mtim.tv_nsec != st.st_mtim.tv_nsec ||
@@ -1195,8 +1387,16 @@ public:
return(cfg.cache_root + source_path + ".wasm");
}
std::shared_ptr<WasmUnitModule> unit_module(const String& source_path, String& error, WasmUnitModuleLoadProfile& profile)
std::shared_ptr<WasmUnitModule> unit_module(const String& source_path, String& error, WasmUnitModuleLoadProfile& profile,
bool deadline_active, std::chrono::steady_clock::time_point deadline)
{
auto deadline_expired = [&]() {
if(!deadline_active || std::chrono::steady_clock::now() < deadline)
return(false);
profile.timed_out = true;
error = "wasm module load timed out";
return(true);
};
auto lookup_start = std::chrono::steady_clock::now();
String wasm_path = unit_wasm_path(source_path);
struct stat st;
@@ -1207,11 +1407,20 @@ public:
error = "no wasm artifact for " + source_path + " (expected " + wasm_path + ")";
return(nullptr);
}
if(!S_ISREG(st.st_mode))
{
profile.lookup_us = (u64)std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - lookup_start).count();
error = "wasm artifact is not a regular file: " + wasm_path;
return(nullptr);
}
auto cached = module_cache.find(wasm_path);
u64 modified_ns = (u64)st.st_mtim.tv_sec * 1000000000ull + (u64)st.st_mtim.tv_nsec;
u64 changed_ns = (u64)st.st_ctim.tv_sec * 1000000000ull + (u64)st.st_ctim.tv_nsec;
profile.lookup_us = (u64)std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - lookup_start).count();
if(deadline_expired())
return(nullptr);
if(cached != module_cache.end() && cached->second->modified_ns == modified_ns && cached->second->changed_ns == changed_ns && cached->second->size == (u64)st.st_size)
{
profile.cache_hit = true;
@@ -1230,13 +1439,15 @@ public:
profile.compile_us = (u64)std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - compile_start).count();
profile.serialized_cache_hit = compiled_or_cached.has_value();
if(deadline_expired())
return(nullptr);
std::vector<u8> bytes;
auto read_start = std::chrono::steady_clock::now();
bool read_ok = profile.serialized_cache_hit
? wasm_read_metadata_file(wasm_path, bytes, error, profile.read_bytes, profile.read_count, modified_ns, changed_ns, unit->size)
: wasm_read_file(wasm_path, bytes);
if(!profile.serialized_cache_hit)
profile.read_bytes = bytes.size();
? wasm_read_metadata_file(wasm_path, bytes, error, profile.read_bytes, profile.read_count, profile.timed_out,
deadline_active, deadline, st)
: wasm_read_file_deadline(wasm_path, bytes, error, profile.read_bytes, profile.read_count, profile.timed_out,
deadline_active, deadline, st);
if(!read_ok)
{
profile.read_us = (u64)std::chrono::duration_cast<std::chrono::microseconds>(
@@ -1247,8 +1458,10 @@ public:
}
profile.read_us = (u64)std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - read_start).count();
if(deadline_expired())
return(nullptr);
auto parse_start = std::chrono::steady_clock::now();
if(!wasm_parse_sections(bytes, unit->dylink, unit->abi, error))
if(!wasm_parse_sections(bytes, unit->dylink, unit->abi, error, deadline_active, deadline, &profile.timed_out))
{
profile.parse_us = (u64)std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - parse_start).count();
@@ -1257,6 +1470,8 @@ public:
}
profile.parse_us = (u64)std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - parse_start).count();
if(deadline_expired())
return(nullptr);
if(!unit->dylink.found)
{
error = wasm_path + ": missing dylink.0 mem_info (not a PIC side module)";
@@ -1275,6 +1490,8 @@ public:
profile.compile_us += (u64)std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - compile_start).count();
}
if(deadline_expired())
return(nullptr);
if(!compiled_or_cached)
{
error = wasm_path + ": compile failed: " + compile_error;
@@ -1284,6 +1501,8 @@ public:
auto classify_start = std::chrono::steady_clock::now();
for(auto import_type : unit->module->imports())
{
if(deadline_expired())
return(nullptr);
String mod_name(import_type.module());
String name(import_type.name());
auto extern_type = wasmtime::ExternType::from_import(import_type);
@@ -1317,6 +1536,8 @@ public:
}
profile.classify_us = (u64)std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - classify_start).count();
if(deadline_expired())
return(nullptr);
module_cache[wasm_path] = unit;
return(unit);
}
@@ -2274,7 +2495,7 @@ private:
String error;
auto module_start = std::chrono::steady_clock::now();
WasmUnitModuleLoadProfile module_profile;
auto mod = worker.unit_module(source_path, error, module_profile);
auto mod = worker.unit_module(source_path, error, module_profile, invocation_active, invocation_deadline);
u64 module_total_us = (u64)std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - module_start).count();
unit_module_total_us += module_total_us;
@@ -2319,6 +2540,8 @@ private:
module_operation_index = 32;
unit_module_operations_dropped++;
}
if(module_profile.timed_out || invocation_expired())
return(invocation_timeout_error());
if(!mod)
return(error);
// Compiling/deserializing a cold module is host work. Refresh the guest