From c86551b1e911984f495f3d4bd8cff6cf1f51b652 Mon Sep 17 00:00:00 2001 From: Yang Liu Date: Thu, 11 Jun 2026 17:31:26 +0800 Subject: [PATCH] [DYNACACHE] Refined cache validation --- src/tools/dynacache.c | 293 +++++++++++++++++++++++++++++++----------- 1 file changed, 218 insertions(+), 75 deletions(-) diff --git a/src/tools/dynacache.c b/src/tools/dynacache.c index e1b3f03fa4..352a74a931 100644 --- a/src/tools/dynacache.c +++ b/src/tools/dynacache.c @@ -5,6 +5,7 @@ #include #include #include +#include #if defined(DYNAREC) && !defined(WIN32) #include #include @@ -23,6 +24,7 @@ #include "fileutils.h" #include "dynacache.h" #include "dynacache_compress.h" +#include "dynablock.h" #include "env_private.h" #ifdef DYNAREC #include "dynacache_hashes.h" @@ -89,7 +91,7 @@ const char* GetDynacacheFolder(mapping_t* mapping) `box64 --dynacache-clean` can be used from command line to purge obsolete DyaCache files */ -#define FILE_VERSION 3 +#define FILE_VERSION 4 #define HEADER_SIGN "DynaCache" typedef struct DynaCacheHeader_s { @@ -104,6 +106,10 @@ typedef struct DynaCacheHeader_s { uintptr_t map_addr; size_t map_len; size_t file_length; + size_t cache_header_size; + size_t cache_file_size; + uint64_t header_hash; + uint64_t payload_hash; uint32_t filename_length; uint32_t nblocks; uint32_t nLockAddresses; @@ -212,6 +218,30 @@ int ReadDynaCache(const char* folder, const char* name, mapping_t* mapping, int #endif #define DYNACACHE_TMP_STALE_SECONDS 60 +#define DYNACACHE_HASH_INIT UINT64_C(0) + +static uint64_t DynaCacheHash(uint64_t hash, const void* data, size_t size) +{ + const uint8_t* p = data; + while(size) { + int len = (size>INT_MAX)?INT_MAX:(int)size; + uint32_t crc = X31_hash_code((void*)p, len); + hash ^= (uint64_t)crc + UINT64_C(0x9e3779b97f4a7c15) + (hash << 6) + (hash >> 2) + (uint64_t)len; + p += len; + size -= len; + } + return hash; +} + +static size_t DynaCacheHeaderSize(const DynaCacheHeader_t* header) +{ + size_t total = sizeof(DynaCacheHeader_t); + total += (size_t)header->filename_length + 1; + total += header->nblocks*sizeof(CompressedDynaCacheBlock_t); + total += header->nLockAddresses*sizeof(uintptr_t); + total += header->nUnalignedAddresses*sizeof(uintptr_t); + return ALIGN(total); +} static void SyncDynaCacheFolder(const char* folder) { @@ -355,6 +385,10 @@ static void PruneDynaCacheFolder(const char* folder, uint64_t max_size) void SerializeMmaplist(mapping_t* mapping) { + #ifdef WIN32 + (void)mapping; + return; + #else if(!DYNAREC_VERSION) return; if(mapping->env && mapping->env->is_dynacache_overridden && (mapping->env->dynacache!=1)) @@ -392,10 +426,14 @@ void SerializeMmaplist(mapping_t* mapping) size_t map_len = SizeFileMapped(mapping->start); size_t nLockAddresses = nLockAddressRange(mapping->start, map_len); size_t nUnaligned = nUnalignedRange(mapping->start, map_len); - size_t total = sizeof(DynaCacheHeader_t) + strlen(mapping->fullname) + 1 + nblocks*sizeof(CompressedDynaCacheBlock_t) + nLockAddresses*sizeof(uintptr_t) + nUnaligned*sizeof(uintptr_t);; - total = ALIGN(total); // align on pagesize - uint8_t all_header[total]; - memset(all_header, 0, total); + DynaCacheHeader_t header_info = {0}; + header_info.filename_length = strlen(mapping->fullname); + header_info.nblocks = nblocks; + header_info.nLockAddresses = nLockAddresses; + header_info.nUnalignedAddresses = nUnaligned; + size_t total = DynaCacheHeaderSize(&header_info); + uint8_t* all_header = box_calloc(1, total); + if(!all_header) return; void* p = all_header; DynaCacheHeader_t* header = p; strcpy(header->sign, HEADER_SIGN); @@ -408,6 +446,7 @@ void SerializeMmaplist(mapping_t* mapping) header->codesize = MmaplistTotalAlloc(mapping->mmaplist); header->map_addr = mapping->start; header->file_length = filesize; + header->cache_header_size = total; header->filename_length = strlen(mapping->fullname); header->nblocks = MmaplistNBlocks(mapping->mmaplist); header->map_len = map_len; @@ -418,6 +457,7 @@ void SerializeMmaplist(mapping_t* mapping) dynacache_min = mapping->env->dynacache_min; if(dynacache_min*1024>header->codesize) { dynarec_log(LOG_INFO, "DynaCache will not serialize cache for %s because there is not enough usefull code (%s)\n", mapping->fullname, NicePrintSize(header->codesize)); + box_free(all_header); return; // not enugh code, do no write } p += sizeof(DynaCacheHeader_t); // fullname @@ -439,6 +479,7 @@ void SerializeMmaplist(mapping_t* mapping) int tmpfd = DynaCacheOpenTempFile(mapname, tmpname, sizeof(tmpname)); if(tmpfd<0) { dynarec_log(LOG_INFO, "Cannot create cache file %s\n", mapname); + box_free(all_header); return; } FILE* f = fdopen(tmpfd, "wb"); @@ -446,49 +487,61 @@ void SerializeMmaplist(mapping_t* mapping) close(tmpfd); unlink(tmpname); dynarec_log(LOG_INFO, "Cannot create cache file %s\n", mapname); + box_free(all_header); return; } int write_error = 0; + uint64_t payload_hash = DYNACACHE_HASH_INIT; if(fwrite(all_header, total, 1, f)!=1) write_error = 1; - int rewrite_header = 0; int type = BOX64ENV(dynacache_compress); if(type!=COMP_NONE) { for(int i=0; icache_file_size = cache_file_size; + header->payload_hash = payload_hash; + header->header_hash = 0; + header->header_hash = DynaCacheHash(DYNACACHE_HASH_INIT, all_header, total); + if(fseeko(f, 0, SEEK_SET) || fwrite(all_header, total, 1, f)!=1) + write_error = 1; + } } if(!write_error && fflush(f)) write_error = 1; @@ -496,9 +549,11 @@ void SerializeMmaplist(mapping_t* mapping) write_error = 1; if(fclose(f)) write_error = 1; + tmpfd = -1; if(write_error) { dynarec_log(LOG_INFO, "Error writing Cache file (disk full?)\n"); unlink(tmpname); + box_free(all_header); return; } int dynacache_limit = box64env.dynacache_limit; @@ -508,6 +563,7 @@ void SerializeMmaplist(mapping_t* mapping) if(rename(tmpname, mapname)) { dynarec_log(LOG_INFO, "Cannot publish cache file %s\n", mapname); unlink(tmpname); + box_free(all_header); return; } SyncDynaCacheFolder(folder); @@ -515,9 +571,12 @@ void SerializeMmaplist(mapping_t* mapping) PruneDynaCacheFolder(folder, (uint64_t)dynacache_limit*1024*1024); SyncDynaCacheFolder(folder); } + box_free(all_header); #else + box_free(all_header); // TODO? #endif + #endif } #define DCERR_OK 0 @@ -534,10 +593,13 @@ void SerializeMmaplist(mapping_t* mapping) #define DCERR_RELOC 11 #define DCERR_BADNAME 12 #define DCERR_CPUEXT 13 +#define DCERR_CHECKSUM 14 #ifndef WIN32 int ReadDynaCache(const char* folder, const char* name, mapping_t* mapping, int verbose) { + int ret = DCERR_OK; + uint8_t* all_header = NULL; char filename[strlen(folder)+strlen(name)+1]; strcpy(filename, folder); strcat(filename, name); @@ -549,7 +611,8 @@ int ReadDynaCache(const char* folder, const char* name, mapping_t* mapping, int return exists?DCERR_FERROR:DCERR_NEXIST; } struct stat st = {0}; - if(fstat(fileno(f), &st) || !S_ISREG(st.st_mode) || st.st_size<0) { + int fd = fileno(f); + if(fstat(fd, &st) || !S_ISREG(st.st_mode) || st.st_size<0) { if(verbose) printf_log_prefix(0, LOG_NONE, "Cannot stat file\n"); fclose(f); return DCERR_FERROR; @@ -566,7 +629,7 @@ int ReadDynaCache(const char* folder, const char* name, mapping_t* mapping, int fclose(f); return DCERR_FERROR; } - if(strcmp(header.sign, HEADER_SIGN)) { + if(memcmp(header.sign, HEADER_SIGN, sizeof(header.sign))) { if(verbose) printf_log_prefix(0, LOG_NONE, "Bad header\n"); fclose(f); return DCERR_BADHEADER; @@ -596,79 +659,157 @@ int ReadDynaCache(const char* folder, const char* name, mapping_t* mapping, int fclose(f); return DCERR_CPUEXT; } - char map_filename[header.filename_length+1]; - if(fread(map_filename, header.filename_length+1, 1, f)!=1) { - if(verbose) printf_log_prefix(0, LOG_NONE, "Cannot read filename\n"); + if(!header.filename_length || !header.nblocks) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Invalid header\n"); + fclose(f); + return DCERR_BADHEADER; + } + size_t header_size = DynaCacheHeaderSize(&header); + if(header.cache_header_size!=header_size + || header.cache_header_size>filesize + || header.cache_file_size!=filesize) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Invalid cache size\n"); + fclose(f); + return DCERR_BADHEADER; + } + all_header = box_malloc(header.cache_header_size); + if(!all_header) { fclose(f); return DCERR_FERROR; } + if(fseeko(f, 0, SEEK_SET) || fread(all_header, header.cache_header_size, 1, f)!=1) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Cannot read header\n"); + ret = DCERR_FERROR; + goto done; + } + DynaCacheHeader_t* file_header = (DynaCacheHeader_t*)all_header; + if(file_header->cache_header_size!=header.cache_header_size) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Invalid cache size\n"); + ret = DCERR_BADHEADER; + goto done; + } + uint64_t expected_header_hash = file_header->header_hash; + file_header->header_hash = 0; + uint64_t actual_header_hash = DynaCacheHash(DYNACACHE_HASH_INIT, all_header, header.cache_header_size); + file_header->header_hash = expected_header_hash; + if(actual_header_hash!=expected_header_hash) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Bad header checksum\n"); + ret = DCERR_CHECKSUM; + goto done; + } + char* map_filename = (char*)all_header + sizeof(DynaCacheHeader_t); + if(map_filename[file_header->filename_length] || strlen(map_filename)!=file_header->filename_length) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Invalid filename\n"); + ret = DCERR_BADHEADER; + goto done; + } if(!FileExist(map_filename, IS_FILE)) { if(verbose) printf_log_prefix(0, LOG_NONE, "Mapfiled does not exists\n"); - fclose(f); - return DCERR_MAPNEXIST; + ret = DCERR_MAPNEXIST; + goto done; } - if(FileSize(map_filename)!=header.file_length) { + if(FileSize(map_filename)!=file_header->file_length) { if(verbose) printf_log_prefix(0, LOG_NONE, "File changed\n"); - fclose(f); - return DCERR_MAPCHG; - } - CompressedDynaCacheBlock_t blocks[header.nblocks]; - if(fread(blocks, sizeof(CompressedDynaCacheBlock_t), header.nblocks, f)!=header.nblocks) { - if(verbose) printf_log_prefix(0, LOG_NONE, "Cannot read blocks\n"); - fclose(f); - return DCERR_FERROR; + ret = DCERR_MAPCHG; + goto done; } - uintptr_t lockAddresses[header.nLockAddresses]; - if(fread(lockAddresses, sizeof(uintptr_t), header.nLockAddresses, f)!=header.nLockAddresses) { - if(verbose) printf_log_prefix(0, LOG_NONE, "Cannot read lockAddresses\n"); - fclose(f); - return DCERR_FERROR; + if(mapping && strcmp(map_filename, mapping->fullname)) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Invalid cache name\n"); + ret = DCERR_BADNAME; + goto done; } - uintptr_t unalignedAddresses[header.nUnalignedAddresses]; - if(fread(unalignedAddresses, sizeof(uintptr_t), header.nUnalignedAddresses, f)!=header.nUnalignedAddresses) { - if(verbose) printf_log_prefix(0, LOG_NONE, "Cannot read unalignedAddresses\n"); - fclose(f); - return DCERR_FERROR; + if(mapping && file_header->dynarec_settings!=GetDynSetting(mapping)) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Incompatible Dynarec Settings\n"); + ret = DCERR_BADNAME; + goto done; } - off_t p = ftell(f); - p = ALIGN(p);; - if(fseek(f, p, SEEK_SET)<0) { + + CompressedDynaCacheBlock_t* blocks = (CompressedDynaCacheBlock_t*)(map_filename + file_header->filename_length + 1); + uintptr_t* lockAddresses = (uintptr_t*)(blocks + file_header->nblocks); + uintptr_t* unalignedAddresses = lockAddresses + file_header->nLockAddresses; + + off_t p = file_header->cache_header_size; + uint64_t payload_hash = DYNACACHE_HASH_INIT; + if(fseeko(f, p, SEEK_SET)) { if(verbose) printf_log_prefix(0, LOG_NONE, "Error reading a block\n"); - fclose(f); - return DCERR_FERROR; - } - if(!mapping) { - // check the blocks can be read without reading... - for(int i=0; inblocks; ++i) { + size_t file_size = 0; + size_t payload_size = 0; + int valid_block = 0; + if(blocks[i].block.block && blocks[i].block.size) { + if(blocks[i].type==COMP_NONE) { + valid_block = !blocks[i].compsize; + payload_size = blocks[i].block.size; + file_size = blocks[i].block.size; + } else if((blocks[i].type==COMP_Z || blocks[i].type==COMP_Z_MAX) + && blocks[i].compsize) { + valid_block = 1; + payload_size = blocks[i].compsize; + file_size = ALIGN(blocks[i].compsize); + } + } + if(!valid_block || p<0 || (uintmax_t)p>filesize || file_size>filesize-(size_t)p) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Invalid block table\n"); + ret = DCERR_BADHEADER; + goto done; + } + void* payload = box_malloc(payload_size); + if(!payload || fread(payload, payload_size, 1, f)!=1) { + box_free(payload); + if(verbose) printf_log_prefix(0, LOG_NONE, "Error reading a block\n"); + ret = DCERR_FERROR; + goto done; + } + payload_hash = DynaCacheHash(payload_hash, payload, payload_size); + box_free(payload); + if(file_size>payload_size) { + size_t padding_size = file_size-payload_size; + uint8_t padding[padding_size]; + if(fread(padding, padding_size, 1, f)!=1) { if(verbose) printf_log_prefix(0, LOG_NONE, "Error reading a block\n"); - fclose(f); - return DCERR_FERROR; + ret = DCERR_FERROR; + goto done; } + payload_hash = DynaCacheHash(payload_hash, padding, padding_size); } + p += file_size; + } + if(p<0 || (uintmax_t)p!=filesize) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Invalid cache size\n"); + ret = DCERR_BADHEADER; + goto done; + } + if(payload_hash!=file_header->payload_hash) { + if(verbose) printf_log_prefix(0, LOG_NONE, "Bad payload checksum\n"); + ret = DCERR_CHECKSUM; + goto done; + } + if(!mapping) { char* short_name = strrchr(map_filename, '/'); if(short_name) ++short_name; else short_name = map_filename; short_name = LowerCase(short_name); - const char* file_name = MmaplistName(short_name, header.dynarec_settings, map_filename); + const char* file_name = MmaplistName(short_name, file_header->dynarec_settings, map_filename); box_free(short_name); if(strcmp(file_name, name)) { if(verbose) printf_log_prefix(0, LOG_NONE, "Invalid cache name\n"); - fclose(f); - return DCERR_BADNAME; + ret = DCERR_BADNAME; + goto done; } if(verbose) { // check if name is coherent // file is valid, gives informations: printf_log_prefix(0, LOG_NONE, "%s (%s)\n", map_filename, NicePrintSize(filesize)); printf_log_prefix(0, LOG_NONE, "\tDynarec Settings:\n"); - PrintDynfSettings(LOG_NONE, header.dynarec_settings); + PrintDynfSettings(LOG_NONE, file_header->dynarec_settings); size_t total_blocks = 0, total_free = 0, total_compressed = 0, total_uncompressed = 0; - size_t total_code = header.codesize; - for(int i=0; icodesize; + for(int i=0; inblocks; ++i) { total_blocks += blocks[i].block.size; total_free += blocks[i].block.free_size; if(blocks[i].compsize) @@ -676,9 +817,9 @@ int ReadDynaCache(const char* folder, const char* name, mapping_t* mapping, int else total_uncompressed += blocks[i].block.size; } - printf_log_prefix(0, LOG_NONE, "\tHas %d blocks for a total of %s", header.nblocks, NicePrintSize(total_blocks)); + printf_log_prefix(0, LOG_NONE, "\tHas %d blocks for a total of %s", file_header->nblocks, NicePrintSize(total_blocks)); printf_log_prefix(0, LOG_NONE, " with %s still free", NicePrintSize(total_free)); - printf_log_prefix(0, LOG_NONE, " and %s non-canceled blocks (mapped at %p-%p, with %zu lock and %zu unaligned addresses)", NicePrintSize(total_code), (void*)header.map_addr, (void*)header.map_addr+header.map_len, header.nLockAddresses, header.nUnalignedAddresses); + printf_log_prefix(0, LOG_NONE, " and %s non-canceled blocks (mapped at %p-%p, with %zu lock and %zu unaligned addresses)", NicePrintSize(total_code), (void*)file_header->map_addr, (void*)file_header->map_addr+file_header->map_len, file_header->nLockAddresses, file_header->nUnalignedAddresses); if(total_compressed) { printf_log_prefix(0, LOG_NONE, " with %s compressed blocks", NicePrintSize(total_compressed)); if(total_uncompressed) @@ -688,42 +829,44 @@ int ReadDynaCache(const char* folder, const char* name, mapping_t* mapping, int } } else { // actually reading! - int fd = fileno(f); - intptr_t delta_map = mapping->start - header.map_addr; + intptr_t delta_map = mapping->start - file_header->map_addr; dynarec_log(LOG_INFO, "Trying to load DynaCache for %s, with a delta_map=%zx\n", mapping->fullname, delta_map); if(!mapping->mmaplist) mapping->mmaplist = NewMmaplist(); - MmaplistAddNBlocks(mapping->mmaplist, header.nblocks); - for(int i=0; immaplist, file_header->nblocks); + p = file_header->cache_header_size; + for(int i=0; inblocks; ++i) { + int reloc_ret = 0; + size_t file_size = (blocks[i].type==COMP_NONE)?blocks[i].block.size:ALIGN(blocks[i].compsize); if(blocks[i].type==COMP_NONE) - ret = MmaplistAddBlock(mapping->mmaplist, fd, p, blocks[i].block.block, blocks[i].block.size, delta_map, mapping->start); + reloc_ret = MmaplistAddBlock(mapping->mmaplist, fd, p, blocks[i].block.block, blocks[i].block.size, delta_map, mapping->start); else { void* src = box_malloc(blocks[i].compsize); - fseek(f, p, SEEK_SET); - if(!src || (fread(src, blocks[i].compsize, 1, f)!=1)) { - ret = -10; + if(!src || fseeko(f, p, SEEK_SET) || fread(src, blocks[i].compsize, 1, f)!=1) { + reloc_ret = -10; } else - ret = MmaplistAddCompressedBlock(mapping->mmaplist, blocks[i].type, src, blocks[i].compsize, blocks[i].block.block, blocks[i].block.size, delta_map, mapping->start); + reloc_ret = MmaplistAddCompressedBlock(mapping->mmaplist, blocks[i].type, src, blocks[i].compsize, blocks[i].block.block, blocks[i].block.size, delta_map, mapping->start); box_free(src); } - if(ret) { + if(reloc_ret) { printf_log(LOG_NONE, "Error while doing relocation on a DynaCache (block %d)\n", i); - fclose(f); - return DCERR_RELOC; + ret = DCERR_RELOC; + goto done; } - p+=(blocks[i].type==COMP_NONE)?blocks[i].block.size:ALIGN(blocks[i].compsize); + p += file_size; } - for(size_t i=0; inLockAddresses; ++i) addLockAddress(lockAddresses[i]+delta_map); - for(size_t i=0; inUnalignedAddresses; ++i) add_unaligned_address(unalignedAddresses[i]+delta_map); - dynarec_log(LOG_INFO, "Loaded DynaCache for %s, with %d blocks\n", mapping->fullname, header.nblocks); + dynarec_log(LOG_INFO, "Loaded DynaCache for %s, with %d blocks\n", mapping->fullname, file_header->nblocks); // try to update mtime for used cache file, so that it is less likely to be pruned utime(filename, NULL); } +done: + box_free(all_header); fclose(f); - return DCERR_OK; + return ret; } #endif