From 060acc2945a2a50ea26b4a833ad20b7d63202ffe Mon Sep 17 00:00:00 2001 From: Eremey Valetov Date: Sat, 7 Nov 2026 06:02:42 -0500 Subject: [PATCH] Fix Windows snapshot timestamps and validate Turbo master caches --- lib/src/decompress.c | 49 ++++++++++++++++++----------- python/tests/test_snapshot.py | 18 +++++++++++ python/uc2/snapshot.py | 14 +++++++-- tests/src/fixture_archive.h | 26 ++++++++++++--- tests/src/test_turbo.c | 59 ++++++++++++++++++++++++++++++++++- 5 files changed, 139 insertions(+), 27 deletions(-) diff --git a/lib/src/decompress.c b/lib/src/decompress.c index bebacea..b370d8a 100644 --- a/lib/src/decompress.c +++ b/lib/src/decompress.c @@ -557,14 +557,15 @@ static struct master_info *find_master(struct uc2_context *uc2, unsigned id) static int resolve_master(struct uc2_context *uc2, unsigned master) { + int ret = UC2_Damaged; if (uc2->pcp) { uc2->message = "PCP not implemented"; return UC2_Unimplemented; } if (!uc2->supermaster) { - uc2->supermaster = u_alloc(uc2, 49152); - if (!uc2->supermaster) + u8 *data = u_alloc(uc2, 49152); + if (!data) return UC2_UserFault; #ifdef UC2_SUPER_C @@ -575,15 +576,16 @@ static int resolve_master(struct uc2_context *uc2, unsigned master) extern u8 uc2_supermaster_compressed[], uc2_supermaster_compressed_end[]; struct range br = {.ptr = uc2_supermaster_compressed, .end = uc2_supermaster_compressed_end}; #endif - struct range bw = {.ptr = uc2->supermaster, .end = uc2->supermaster + 49152}; + struct range bw = {.ptr = data, .end = data + 49152}; struct reader rd = {.read = buf_read, .context = &br}; struct writer wr = {.write = buf_write, .context = &bw}; - u16 csum; + u16 csum = 0; int r = decompressor(uc2, 4, &rd, &wr, NoMaster, 49152, &csum); - if (r < 0) - return r; - if (csum != 0x1E55) - return UC2_InternalError; + if (r < 0 || range_len(&bw) || csum != 0x1E55) { + u_free(uc2, data); + return r < 0 ? r : UC2_InternalError; + } + uc2->supermaster = data; } if (master < FirstMaster) @@ -594,13 +596,13 @@ static int resolve_master(struct uc2_context *uc2, unsigned master) struct master_info *m = find_master(uc2, master); if (!m) { diag("Master %X missing\n", master); - return UC2_Damaged; + goto fail; } if (m->size > 0xffff) - return UC2_Damaged; + goto fail; if (m->needed_by) { diag("%X: Circular dependency\n", master); - return UC2_Damaged; + goto fail; } if (m->data) break; @@ -612,23 +614,34 @@ static int resolve_master(struct uc2_context *uc2, unsigned master) while (mi) { diag("Decompressing master %X size:%u master:%X method:%u\n", mi->id, mi->size, mi->com.master, mi->com.method); - mi->data = u_alloc(uc2, mi->size); - if (!mi->data) - return UC2_UserFault; + u8 *data = u_alloc(uc2, mi->size ? mi->size : 1); + if (!data) { ret = UC2_UserFault; goto fail; } struct archive_ctx ar = {.offset = mi->offset, .uc2 = uc2}; struct reader rd = {.read = archive_read, .context = &ar}; - struct range bw = {.ptr = mi->data, .end = mi->data + mi->size}; + struct range bw = {.ptr = data, .end = data + mi->size}; struct writer wr = {.write = buf_write, .context = &bw}; int r = decompressor(uc2, mi->com.method, &rd, &wr, mi->com.master, mi->size, 0); diag("Decompressed master %u left:%u\n", mi->id, range_len(&bw)); - if (r < 0) - return r; + if (r < 0 || range_len(&bw)) { + u_free(uc2, data); + ret = r < 0 ? r : UC2_Truncated; + goto fail; + } + mi->data = data; struct master_info *m = mi; mi = mi->needed_by; m->needed_by = 0; } return 0; + +fail: + /* A caller may retry extraction on this handle after an allocation or + input error. Only complete dictionaries are cached, and no traversal + links may survive a failed resolution (including dependency cycles). */ + for (struct list *l = uc2->masters.next; l != &uc2->masters; l = l->next) + list_item(l, struct master_info, list)->needed_by = 0; + return ret; } static int use_master(struct uc2_context *uc2, u8 buffer[65535], u32 id) @@ -1141,7 +1154,7 @@ static int decompressor_turbo(struct uc2_context *uc2, unsigned master, ret = turbo_read(rd, header, 2); if (ret < 0) break; unsigned count = header[0] | (unsigned)header[1] << 8; - if (!count) { *csum = csum_get(&cs); break; } + if (!count) { if (csum) *csum = csum_get(&cs); break; } if (count > sizeof t->input) { ret = UC2_Damaged; break; } ret = turbo_read(rd, t->input, count); if (ret < 0) break; diff --git a/python/tests/test_snapshot.py b/python/tests/test_snapshot.py index e304213..9508334 100644 --- a/python/tests/test_snapshot.py +++ b/python/tests/test_snapshot.py @@ -131,6 +131,24 @@ def test_source_change_and_failed_commit(tmp_path, monkeypatch): assert not (repo/'.writer-lock').exists() +def test_path_and_handle_ctime_can_differ(tmp_path, monkeypatch): + # CPython on Windows can report creation time from lstat, but change + # time from fstat. Reproduce the discrepancy independently of the OS. + original = os.fstat + class HandleStat: + def __init__(self, fd): + self.value = original(fd) + self.st_ctime_ns = self.value.st_ctime_ns + 10**9 + def __getattr__(self, name): + return getattr(self.value, name) + monkeypatch.setattr(s.os, 'fstat', HandleStat) + source, repo = tmp_path/'source', tmp_path/'repo' + source.mkdir(); (source/'data').write_bytes(b'updated file') + result = s.create(source, repo) + s.restore(repo, result['snapshot'], tmp_path/'restored') + assert tree(tmp_path/'restored') == tree(source) + + @pytest.mark.skipif(os.name == 'nt', reason='requires POSIX symlink/FIFO privileges') def test_symlinks_special_files_and_collisions(tmp_path): src, repo = tmp_path/'src', tmp_path/'repo' diff --git a/python/uc2/snapshot.py b/python/uc2/snapshot.py index 6da6730..66ccaad 100644 --- a/python/uc2/snapshot.py +++ b/python/uc2/snapshot.py @@ -297,13 +297,14 @@ def create(source, repository, *, label="", max_total_size=1 << 40): keys.add(key) if len(keys) > MAX_ENTRIES: raise ValueError("snapshot entry limit exceeded") - mode = path.lstat().st_mode + path_before = path.lstat() + mode = path_before.st_mode if stat.S_ISDIR(mode): entry = dict(path=relative) entries.append(entry) visit(path, entry) elif stat.S_ISREG(mode): - entry = dict(path=relative, **_metadata(path, "file"), chunks=[]) + entry = dict(path=relative, **_metadata(path, "file", path_before), chunks=[]) sha, size = hashlib.sha256(), 0 with _regular(path) as stream: before = os.fstat(stream.fileno()) @@ -323,7 +324,14 @@ def create(source, repository, *, label="", max_total_size=1 << 40): after = os.fstat(stream.fileno()) current = path.lstat() signature = lambda s: (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns) - if signature(before) != signature(after) or signature(before) != signature(current) or size != before.st_size: + # Windows fstat and lstat can give ctime different + # meanings (change time versus creation time). Compare + # timestamps only within the same API, and identities + # across APIs. Keep both change checks on every OS. + if (signature(before) != signature(after) or + signature(path_before) != signature(current) or + (before.st_dev, before.st_ino) != (path_before.st_dev, path_before.st_ino) or + size != before.st_size): raise ValueError(f"source changed during snapshot: {relative}") entry.update(size=size, sha256=sha.hexdigest(), mtime_ns=before.st_mtime_ns, mode=stat.S_IMODE(before.st_mode) & 0o777) diff --git a/tests/src/fixture_archive.h b/tests/src/fixture_archive.h index fdba21d..0da7e60 100644 --- a/tests/src/fixture_archive.h +++ b/tests/src/fixture_archive.h @@ -125,9 +125,11 @@ static int compress_data(const unsigned char *data, unsigned len, int level, return ret; } -static int build_archive(const unsigned char *file_compressed, unsigned file_csize, +static int build_archive_parts(const unsigned char *file_compressed, unsigned file_csize, unsigned file_orig_size, unsigned short file_csum, - unsigned master_id, int level, struct membuf *archive) + unsigned master_id, int level, struct membuf *archive, + const unsigned char *master_records, unsigned records_size, + const unsigned char *master_data, unsigned masters_size) { /* * Archive layout: @@ -192,21 +194,26 @@ static int build_archive(const unsigned char *file_compressed, unsigned file_csi put_u32le(p, 0); p += 4; unsigned raw_cdir_len = (unsigned)(p - raw_cdir); + struct membuf directory; + membuf_init(&directory); + if (records_size) membuf_append(&directory, master_records, records_size); + membuf_append(&directory, raw_cdir, raw_cdir_len); /* Compute Fletcher checksum of raw cdir */ - unsigned short cdir_csum = fletcher_checksum(raw_cdir, raw_cdir_len); + unsigned short cdir_csum = fletcher_checksum(directory.data, directory.len); /* Compress the cdir */ struct membuf cdir_compressed; unsigned short cdir_compress_csum = 0; - int ret = compress_data(raw_cdir, raw_cdir_len, level == 80 ? 4 : level, NULL, 0, + int ret = compress_data(directory.data, directory.len, level == 80 ? 4 : level, NULL, 0, &cdir_compressed, &cdir_compress_csum); + membuf_free(&directory); if (ret < 0) { membuf_free(&cdir_compressed); return ret; } - unsigned cdir_compress_offset = 29 + file_csize; + unsigned cdir_compress_offset = 29 + file_csize + masters_size; unsigned total_size = cdir_compress_offset + 10 + cdir_compressed.len; /* Assemble archive */ @@ -233,6 +240,7 @@ static int build_archive(const unsigned char *file_compressed, unsigned file_csi /* File compressed bitstream */ membuf_append(archive, file_compressed, file_csize); + if (masters_size) membuf_append(archive, master_data, masters_size); /* COMPRESS record for cdir (10 bytes) */ unsigned char cdir_compress_rec[10]; @@ -248,3 +256,11 @@ static int build_archive(const unsigned char *file_compressed, unsigned file_csi return 0; } +static int build_archive(const unsigned char *file_compressed, unsigned file_csize, + unsigned file_orig_size, unsigned short file_csum, + unsigned master_id, int level, struct membuf *archive) +{ + return build_archive_parts(file_compressed, file_csize, file_orig_size, + file_csum, master_id, level, archive, NULL, 0, NULL, 0); +} + diff --git a/tests/src/test_turbo.c b/tests/src/test_turbo.c index 015239f..b902963 100644 --- a/tests/src/test_turbo.c +++ b/tests/src/test_turbo.c @@ -56,6 +56,61 @@ static int extract(struct membuf *archive, struct membuf *out) return ret; } +static unsigned fail_allocation, allocation_count; +static void *retry_alloc(void *ctx, unsigned size) +{ + (void)ctx; + if (++allocation_count == fail_allocation) return NULL; + return malloc(size); +} + +/* Two Turbo-compressed master dictionaries exercise the NULL-checksum + * decoder path and recovery on the SAME handle after failed resolution. */ +static void master_case(unsigned mode, unsigned fail_at) +{ + unsigned char zero[32] = {0}, records[78] = {0}; + struct membuf compressed, archive, out; + encode(zero, sizeof zero, 1, &compressed); + for (unsigned i = 0; i < 2; i++) { + unsigned char *p = records + i * 39; + p[0] = 3; /* MasterEntry */ + put_u32le(p + 1, i + 2); /* id */ + put_u16le(p + 17, 32 + (mode == 1 && i == 1)); + put_u16le(p + 19, 0xdede); /* unused checksum */ + put_u32le(p + 21, compressed.len); + put_u16le(p + 25, 80); + put_u32le(p + 27, i == 0 ? 3 : mode == 2 ? 2 : mode == 3 ? 99 : 1); + put_u32le(p + 31, 1); /* volume */ + put_u32le(p + 35, 29 + compressed.len); + } + const unsigned char bad[] = {1, 0, 0, 0, 0}; + int ret = build_archive_parts(compressed.data, compressed.len, sizeof zero, + fletcher_checksum(zero, sizeof zero), 2, 80, &archive, + records, sizeof records, mode == 4 ? bad : compressed.data, + mode == 4 ? sizeof bad : compressed.len); + assert(ret >= 0); + struct uc2_io io = {archive_read, retry_alloc, my_free, NULL}; + fail_allocation = 0; allocation_count = 0; + uc2_handle h = uc2_open(&io, &archive); assert(h); + struct uc2_entry entry; + ret = uc2_read_cdir(h, &entry); assert(ret == UC2_BareEntry); + ret = uc2_read_cdir(h, NULL); assert(ret == UC2_End); + fail_allocation = fail_at; allocation_count = 0; + membuf_init(&out); + ret = uc2_extract(h, &entry.xi, entry.size, membuf_write, &out); + if (mode || (fail_at && allocation_count >= fail_at)) assert(ret < 0); + else assert(ret >= 0 && out.len == sizeof zero && !memcmp(out.data, zero, sizeof zero)); + membuf_free(&out); + fail_allocation = 0; + membuf_init(&out); + ret = uc2_extract(h, &entry.xi, entry.size, membuf_write, &out); + if (mode) assert(ret < 0 && out.len == 0); + else assert(ret >= 0 && out.len == sizeof zero && !memcmp(out.data, zero, sizeof zero)); + membuf_free(&out); + uc2_close(h); + membuf_free(&archive); membuf_free(&compressed); +} + int main(int argc, char **argv) { static unsigned char data[70001]; @@ -113,6 +168,8 @@ int main(int argc, char **argv) ret = extract(&archive, &out); assert(ret < 0); membuf_free(&out); membuf_free(&archive); } - puts("Turbo: 18 size/master cases and malformed stream checks passed"); + for (unsigned mode = 0; mode < 5; mode++) master_case(mode, 0); + for (unsigned fail_at = 1; fail_at <= 12; fail_at++) master_case(0, fail_at); + puts("Turbo: size/master, malformed streams and dictionary retry checks passed"); return 0; }