udo/uce
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Buffer Wasm metadata reads

Browse Source
This commit is contained in:
udo
2026-07-19 06:43:19 +00:00
parent a9e24cffc4
commit 15fdfe725d
6 changed files with 182 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/wasm/worker.cpp
+85 -35
View File
@@ -126,6 +126,7 @@ struct WasmUnitModuleLoadProfile
u64 lookup_us = 0;
u64 read_us = 0;
u64 read_bytes = 0;
u64 read_count = 0;
u64 parse_us = 0;
u64 compile_us = 0;
u64 classify_us = 0;
@@ -177,6 +178,7 @@ struct WasmUnitModuleOperation
u64 lookup_us = 0;
u64 read_us = 0;
u64 read_bytes = 0;
u64 read_count = 0;
u64 parse_us = 0;
u64 build_us = 0;
u64 classify_us = 0;
@@ -237,6 +239,7 @@ struct WasmRequestProfile
u64 unit_module_lookup_total_us = 0;
u64 unit_module_read_total_us = 0;
u64 unit_module_read_bytes = 0;
u64 unit_module_read_count = 0;
u64 unit_module_parse_total_us = 0;
u64 unit_module_compile_total_us = 0;
u64 unit_module_classify_total_us = 0;
@@ -893,38 +896,67 @@ static String wasm_source_map_lookup(const WasmSourceMap& map, u64 address)
return(result);
}
static bool wasm_pread_all(int fd, u64 offset, u8* out, size_t size, u64& bytes_read)
struct WasmMetadataReader
{
size_t done = 0;
while(done < size)
{
ssize_t n = pread(fd, out + done, size - done, (off_t)(offset + done));
if(n < 0 && errno == EINTR)
continue;
if(n <= 0)
return(false);
done += (size_t)n;
bytes_read += (u64)n;
}
return(true);
}
int fd = -1;
u64 file_size = 0;
u64 bytes_read = 0;
u64 read_count = 0;
u64 buffer_offset = 0;
size_t buffer_size = 0;
u8 buffer[4096];
static bool wasm_read_uleb_fd(int fd, u64& pos, u64 end, u64& out, u64& bytes_read)
{
out = 0;
u32 shift = 0;
while(pos < end && shift < 64)
bool read(u64 offset, u8* out, size_t size)
{
u8 byte = 0;
if(!wasm_pread_all(fd, pos++, &byte, 1, bytes_read))
if(offset > file_size || size > file_size - offset)
return(false);
out |= ((u64)(byte & 0x7f)) << shift;
if((byte & 0x80) == 0)
return(true);
shift += 7;
while(size > 0)
{
if(offset < buffer_offset || offset >= buffer_offset + buffer_size)
{
buffer_offset = offset;
buffer_size = 0;
size_t wanted = (size_t)std::min<u64>(sizeof(buffer), file_size - offset);
while(true)
{
ssize_t count = pread(fd, buffer, wanted, (off_t)offset);
if(count < 0 && errno == EINTR)
continue;
if(count <= 0)
return(false);
buffer_size = (size_t)count;
bytes_read += (u64)count;
read_count++;
break;
}
}
size_t available = buffer_size - (size_t)(offset - buffer_offset);
size_t copied = std::min(size, available);
memcpy(out, buffer + (size_t)(offset - buffer_offset), copied);
offset += copied;
out += copied;
size -= copied;
}
return(true);
}
return(false);
}
bool read_uleb(u64& pos, u64 end, u64& out)
{
out = 0;
u32 shift = 0;
while(pos < end && shift < 64)
{
u8 byte = 0;
if(!read(pos++, &byte, 1))
return(false);
out |= ((u64)(byte & 0x7f)) << shift;
if((byte & 0x80) == 0)
return(true);
shift += 7;
}
return(false);
}
};
static void wasm_write_uleb(std::vector<u8>& out, u64 value)
{
@@ -937,10 +969,11 @@ static void wasm_write_uleb(std::vector<u8>& out, u64 value)
while(value);
}
static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadata, String& error, u64& bytes_read,
static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadata, String& error, u64& bytes_read, u64& read_count,
u64 expected_modified_ns, u64 expected_changed_ns, u64 expected_size)
{
bytes_read = 0;
read_count = 0;
int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if(fd < 0)
return(false);
@@ -960,9 +993,14 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
return(false);
}
u64 file_size = (u64)st.st_size;
WasmMetadataReader reader;
reader.fd = fd;
reader.file_size = file_size;
u8 header[8];
if(!wasm_pread_all(fd, 0, header, sizeof(header), bytes_read) || memcmp(header, "\0asm\1\0\0\0", sizeof(header)) != 0)
if(!reader.read(0, header, sizeof(header)) || memcmp(header, "\0asm\1\0\0\0", sizeof(header)) != 0)
{
bytes_read = reader.bytes_read;
read_count = reader.read_count;
close(fd);
error = "not a supported wasm module";
return(false);
@@ -972,13 +1010,13 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
while(pos < file_size)
{
u8 section_id = 0;
if(!wasm_pread_all(fd, pos++, &section_id, 1, bytes_read))
if(!reader.read(pos++, &section_id, 1))
{
error = "malformed wasm section header";
break;
}
u64 section_size = 0;
if(!wasm_read_uleb_fd(fd, pos, file_size, section_size, bytes_read) || section_size > file_size - pos)
if(!reader.read_uleb(pos, file_size, section_size) || section_size > file_size - pos)
{
error = "malformed wasm section header";
break;
@@ -988,7 +1026,7 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
{
u64 cursor = pos;
u64 name_len = 0;
if(!wasm_read_uleb_fd(fd, cursor, section_end, name_len, bytes_read) || name_len > section_end - cursor)
if(!reader.read_uleb(cursor, section_end, name_len) || name_len > section_end - cursor)
{
error = "malformed custom section name";
break;
@@ -997,7 +1035,7 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
if(name_len <= 64)
{
name.resize((size_t)name_len);
if(name_len && !wasm_pread_all(fd, cursor, (u8*)&name[0], (size_t)name_len, bytes_read))
if(name_len && !reader.read(cursor, (u8*)&name[0], (size_t)name_len))
{
error = "malformed custom section name";
break;
@@ -1011,7 +1049,7 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
break;
}
std::vector<u8> section((size_t)section_size);
if(section_size && !wasm_pread_all(fd, pos, section.data(), section.size(), bytes_read))
if(section_size && !reader.read(pos, section.data(), section.size()))
{
error = "cannot read wasm metadata section";
break;
@@ -1023,6 +1061,14 @@ static bool wasm_read_metadata_file(const String& path, std::vector<u8>& metadat
}
pos = section_end;
}
bytes_read = reader.bytes_read;
read_count = reader.read_count;
struct stat final_st;
if(error == "" && (fstat(fd, &final_st) != 0 || final_st.st_dev != st.st_dev || final_st.st_ino != st.st_ino ||
final_st.st_mtim.tv_sec != st.st_mtim.tv_sec || final_st.st_mtim.tv_nsec != st.st_mtim.tv_nsec ||
final_st.st_ctim.tv_sec != st.st_ctim.tv_sec || final_st.st_ctim.tv_nsec != st.st_ctim.tv_nsec ||
final_st.st_size != st.st_size))
error = "wasm artifact changed while loading metadata";
close(fd);
if(error != "")
return(false);
@@ -1187,7 +1233,7 @@ public:
std::vector<u8> bytes;
auto read_start = std::chrono::steady_clock::now();
bool read_ok = profile.serialized_cache_hit
? wasm_read_metadata_file(wasm_path, bytes, error, profile.read_bytes, modified_ns, changed_ns, unit->size)
? wasm_read_metadata_file(wasm_path, bytes, error, profile.read_bytes, profile.read_count, modified_ns, changed_ns, unit->size)
: wasm_read_file(wasm_path, bytes);
if(!profile.serialized_cache_hit)
profile.read_bytes = bytes.size();
@@ -2239,6 +2285,7 @@ private:
unit_module_lookup_total_us += module_profile.lookup_us;
unit_module_read_total_us += module_profile.read_us;
unit_module_read_bytes += module_profile.read_bytes;
unit_module_read_count += module_profile.read_count;
unit_module_parse_total_us += module_profile.parse_us;
unit_module_compile_total_us += module_profile.compile_us;
unit_module_classify_total_us += module_profile.classify_us;
@@ -2261,6 +2308,7 @@ private:
operation.lookup_us = module_profile.lookup_us;
operation.read_us = module_profile.read_us;
operation.read_bytes = module_profile.read_bytes;
operation.read_count = module_profile.read_count;
operation.parse_us = module_profile.parse_us;
operation.build_us = module_profile.compile_us;
operation.classify_us = module_profile.classify_us;
@@ -3176,6 +3224,7 @@ private:
response["unit_module_lookup_us"] = (f64)self->unit_module_lookup_total_us;
response["unit_module_read_us"] = (f64)self->unit_module_read_total_us;
response["unit_module_read_bytes"] = (f64)self->unit_module_read_bytes;
response["unit_module_read_count"] = (f64)self->unit_module_read_count;
response["unit_module_parse_us"] = (f64)self->unit_module_parse_total_us;
response["unit_module_compile_us"] = (f64)self->unit_module_compile_total_us;
response["unit_module_classify_us"] = (f64)self->unit_module_classify_total_us;
@@ -3192,6 +3241,7 @@ private:
item["lookup_us"] = (f64)operation.lookup_us;
item["read_us"] = (f64)operation.read_us;
item["read_bytes"] = (f64)operation.read_bytes;
item["read_count"] = (f64)operation.read_count;
item["parse_us"] = (f64)operation.parse_us;
item["build_us"] = (f64)operation.build_us;
item["classify_us"] = (f64)operation.classify_us;