Add optional shared chunks across named archive entries and harden reader limits

This commit is contained in:
Eremey Valetov committed 2026-11-07 12:43:36 -05:00
1 parent fdbdd539b5
commit 07bb648e34
43 files changed
+2239 -101

No files matched your search

+6 -1
View File
@@ -136,7 +136,7 @@ through EOF without seeking and leaves the input open. A supplied size must
match exactly; detecting excess input can consume one extra byte.
Inputs spool to anonymous temporary files as they are added. Compression runs
at close, using CDC-selected custom master dictionaries. Payload buffers have
at close, using CDC-selected custom master dictionaries by default. Payload buffers have
fixed sizes; metadata grows with entries and chunks. Temporary disk space
covers the uncompressed inputs and completed archive, plus the destination's
sibling temporary file during the final copy. Duplicate file paths store
@@ -212,3 +212,8 @@ Filesystem paths now resolve SAS and DOS UC2SEA sets without executing input.
Keep all volumes together. Snapshot maintenance adds explicit `prune`, `repair`
and `compact`; packed repositories require alpha10 or later. See the
[repository guides](https://github.com/evvaletov/uc2/tree/main/docs).
Optional `UC2Writer(..., chunked=True)` shares payload chunks across partly
changed files and stored versions. Use the full 3.0.0a11 or newer reader.
For method 11, `UC2Info.compress_size` counts the recipe only; shared payloads
are excluded. See the [chunk-sharing guide](../docs/chunk-sharing.md).
+207
View File
@@ -0,0 +1,207 @@
"""Shared payloads preserve named files; malformed recipes fail explicitly."""
import hashlib
import io
import random
import struct
import subprocess
import tarfile
import pytest
from uc2 import UC2File, UC2Writer, BadArchiveError
from uc2._cli import executable
MAGIC = b"UC2CHN1\0"
HEADER, RECORD = 88, 48
@pytest.mark.parametrize("level", range(2, 10))
def test_named_chunks_versions_and_codecs(tmp_path, level):
base = random.Random(7).randbytes(180000) + b"ABCD" * 22000
changed = b"prefix change" + base[:160000] + b"X" + base[160000:]
path = tmp_path / "shared.uc2"
with UC2Writer(path, level=level, chunked=True) as writer:
writer.writestr("folder/café.bin", base)
writer.writestr("folder/café.bin", changed)
writer.writestr("empty", b"")
writer.writestr("copy", base)
with UC2File(io.BytesIO(path.read_bytes())) as archive:
files = [i for i in archive.infolist(all_versions=True) if not i.is_dir()]
assert len(files) == 4
assert all(i.method == 11 for i in files)
assert archive.read("folder/café.bin", version=1) == base
assert archive.read("folder/café.bin") == changed
assert archive.read("copy") == base
assert archive.read("empty") == b""
assert writer.stats["chunk_references"] > writer.stats["unique_chunks"]
assert writer.stats["shared_bytes"] > len(base)
assert writer.stats["masters"] == 0
def test_partial_edits_share_beyond_dictionary_window(tmp_path):
base = random.Random(341).randbytes(512 * 1024)
changed = base[:200000] + b"an inserted line\n" + base[200000:]
sizes = {}
for chunked in (False, True):
path = tmp_path / f"{chunked}.uc2"
with UC2Writer(path, chunked=chunked) as writer:
writer.writestr("old.bin", base)
writer.writestr("new.bin", changed)
sizes[chunked] = path.stat().st_size
with UC2File(path) as archive:
assert archive.namelist() == ["old.bin", "new.bin"]
assert archive.read("old.bin") == base
assert archive.read("new.bin") == changed
if chunked:
assert writer.stats["shared_bytes"] > 400000
else:
assert writer.stats["chunk_references"] == 0
assert sizes[True] < sizes[False] * 0.75
def recipe_digest(data, start):
count = struct.unpack_from("<I", data, start + 8)[0]
data[start + 56:start + 88] = hashlib.sha256(
data[start:start + 56] + data[start + HEADER:start + HEADER + count * RECORD]
).digest()
@pytest.mark.parametrize("fault", [
"magic", "flags", "count", "size", "table_size", "table_digest",
"offset_header", "offset_self", "offset_overflow", "zero_size", "large_size",
"large_compressed", "method", "chunk_checksum", "chunk_digest", "file_digest",
"payload_same_xor", "reordered",
])
def test_bad_chunk_recipes(tmp_path, fault):
payload = random.Random(91).randbytes(70001)
path = tmp_path / "bad.uc2"
with UC2Writer(path, chunked=True) as writer:
writer.writestr("data", payload)
data = bytearray(path.read_bytes())
start = data.index(MAGIC)
row = start + HEADER
value = {
"flags": (start + 20, 1), "count": (start + 8, 1048577),
"size": (start + 12, 70002), "table_size": (start + 16, 0),
"offset_header": (row, 0), "offset_self": (row, start),
"offset_overflow": (row, 0xFFFFFFFF), "zero_size": (row + 8, 0),
"large_size": (row + 8, 32769), "large_compressed": (row + 4, 65537),
}
if fault in value:
struct.pack_into("<I", data, *value[fault])
elif fault == "method":
struct.pack_into("<H", data, row + 12, 11) # nested recipes are forbidden
elif fault == "magic":
data[start] ^= 1
elif fault == "table_digest":
data[start + 56] ^= 1
elif fault == "chunk_checksum":
data[row + 14] ^= 1
elif fault == "chunk_digest":
data[row + 16] ^= 1
elif fault == "file_digest":
data[start + 24] ^= 1
elif fault == "payload_same_xor":
assert struct.unpack_from("<H", data, row + 12)[0] == 0
offset = struct.unpack_from("<I", data, row)[0]
data[offset] ^= 1
data[offset + 2] ^= 1 # preserve the weak historical XOR checksum
elif fault == "reordered":
first = bytes(data[row:row + RECORD])
data[row:row + RECORD] = data[row + RECORD:row + 2 * RECORD]
data[row + RECORD:row + 2 * RECORD] = first
if fault not in {"magic", "count", "table_digest"}:
recipe_digest(data, start)
path.write_bytes(data)
with pytest.raises(BadArchiveError):
with UC2File(path) as archive:
archive.read("data")
@pytest.mark.parametrize("level", [4, 8])
def test_damaged_compressed_chunks_preserve_destination(tmp_path, level):
path = tmp_path / "compressed.uc2"
payload = b"compressible data\n" * 5000
with UC2Writer(path, level=level, chunked=True) as writer:
writer.writestr("data", payload)
data = bytearray(path.read_bytes())
start = data.index(MAGIC)
row = start + HEADER
assert struct.unpack_from("<H", data, row + 12)[0] != 0
offset = struct.unpack_from("<I", data, row)[0]
data[offset] ^= 0xFF
path.write_bytes(data)
target = tmp_path / "output"
target.mkdir()
(target / "data").write_bytes(b"keep existing")
with UC2File(path) as archive:
with pytest.raises(BadArchiveError):
archive.extractall(target, overwrite=True)
assert (target / "data").read_bytes() == b"keep existing"
def test_chunked_cli_tar_and_stream(tmp_path):
base = random.Random(71).randbytes(180000)
changed = base[:70000] + b"new" + base[70000:]
tar = io.BytesIO()
with tarfile.open(fileobj=tar, mode="w") as archive:
for name, data in [("old", base), ("new", changed)]:
info = tarfile.TarInfo(name)
info.size = len(data)
archive.addfile(info, io.BytesIO(data))
path = tmp_path / "tar.uc2"
subprocess.run([executable(), "--chunked", "--ingest-tar", str(path)],
input=tar.getvalue(), check=True, capture_output=True)
with UC2File(path) as archive:
assert archive.namelist() == ["old", "new"]
assert archive.read("old") == base
assert archive.read("new") == changed
stream = base + base + changed
path = tmp_path / "stream.uc2"
subprocess.run([executable(), "--chunked", "--ingest", str(path)],
input=stream, check=True, capture_output=True)
with UC2File(path) as archive:
assert archive.namelist() == ["data"]
assert archive.read("data") == stream
result = subprocess.run([executable(), "--ingest-restore", str(path)],
check=True, capture_output=True)
assert result.stdout == stream
def test_chunked_option_validation(tmp_path):
with pytest.raises(TypeError, match="chunked"):
UC2Writer(tmp_path / "invalid.uc2", chunked="yes")
with UC2Writer(tmp_path / "empty.uc2", chunked=True):
pass
with UC2File(tmp_path / "empty.uc2") as archive:
assert archive.namelist() == []
@pytest.mark.parametrize("level", [4, 8])
@pytest.mark.parametrize("delta", [-2, 2])
def test_compressed_chunk_must_consume_exact_word_range(tmp_path, level, delta):
path = tmp_path / "word-bound.uc2"
with UC2Writer(path, level=level, chunked=True) as writer:
writer.writestr("data", b"compressed payload\n" * 12000)
data = bytearray(path.read_bytes())
start = data.index(MAGIC)
row = start + HEADER
assert struct.unpack_from("<H", data, row + 12)[0] != 0
size = struct.unpack_from("<I", data, row + 4)[0]
struct.pack_into("<I", data, row + 4, size + delta)
recipe_digest(data, start)
path.write_bytes(data)
with pytest.raises(BadArchiveError):
with UC2File(path) as archive:
archive.read("data")
@pytest.mark.parametrize("chunked", [False, True])
def test_many_repeated_version_names_are_not_a_ratio_bomb(tmp_path, chunked):
path = tmp_path / "many-versions.uc2"
name = "a" * 240
with UC2Writer(path, chunked=chunked) as writer:
for _ in range(4000):
writer.writestr(name, b"a")
with UC2File(path) as archive:
assert len(archive.infolist(all_versions=True)) == 4000
assert archive.read(name) == archive.read(name, version=3999) == b"a"
+4 -2
View File
@@ -68,7 +68,7 @@ class UC2Writer:
"""
def __init__(self, file, *, level: int = 4, max_entries: int = 100_000,
max_total_size: int = 0xFFFFFFFF):
max_total_size: int = 0xFFFFFFFF, chunked: bool = False):
self._writer = None
self._busy = False
self._temporary = None
@@ -83,9 +83,11 @@ class UC2Writer:
_integer("max_total_size", max_total_size, 0xFFFFFFFF)
if not max_entries or not max_total_size:
raise ValueError("writer limits must be positive")
if not isinstance(chunked, bool):
raise TypeError("chunked must be a bool")
self._directories = {(): 0}
self._files = set()
self._writer = _native.Writer(level, max_entries, max_total_size)
self._writer = _native.Writer(level, max_entries, max_total_size, chunked)
@property
def closed(self) -> bool:
+8 -6
View File
@@ -30,14 +30,14 @@ static void set_error(Writer *self, int result)
}
static int writer_init(Writer *self, PyObject *args, PyObject *kwargs)
{
int level=4; unsigned maximum=100000; long long total=UINT32_MAX;
static char *names[]={"level","max_entries","max_total_size",NULL};
int level=4,chunked=0; unsigned maximum=100000; long long total=UINT32_MAX;
static char *names[]={"level","max_entries","max_total_size","chunked",NULL};
if (self->initialized) { PyErr_SetString(PyExc_RuntimeError,"writer cannot be initialized twice"); return -1; }
if (!PyArg_ParseTupleAndKeywords(args,kwargs,"|iIL",names,&level,&maximum,&total)) return -1;
if (!PyArg_ParseTupleAndKeywords(args,kwargs,"|iILp",names,&level,&maximum,&total,&chunked)) return -1;
if (level<2 || level>9 || maximum<1 || maximum>1000000 || total<1 || total>UINT32_MAX) {
PyErr_SetString(PyExc_ValueError,"invalid writer level or limits"); return -1;
}
struct uc2_writer_options options={level,maximum,(uint64_t)total};
struct uc2_writer_options options={level,maximum,(uint64_t)total,chunked};
self->initialized=1; self->writer=uc2_writer_create(&options);
if (!self->writer) { PyErr_SetString(PyExc_OSError,"cannot allocate writer or create temporary input file"); return -1; }
return 0;
@@ -108,8 +108,10 @@ static PyObject *writer_finish(Writer *self, PyObject *destination)
if (r<0 || PyErr_Occurred()) { set_error(self,r); return NULL; }
struct uc2_writer_stats stats; uc2_writer_get_stats(self->writer,&stats);
uc2_writer_free(self->writer); self->writer=NULL;
return Py_BuildValue("{sI,sI,sI,sI}","files",stats.files,"directories",stats.directories,
"masters",stats.masters,"archive_size",stats.archive_size);
return Py_BuildValue("{sI,sI,sI,sI,sI,sI,sK}","files",stats.files,"directories",stats.directories,
"masters",stats.masters,"archive_size",stats.archive_size,
"unique_chunks",stats.unique_chunks,"chunk_references",stats.chunk_references,
"shared_bytes",(unsigned long long)stats.shared_bytes);
}
static PyMethodDef methods[]={
{"abort",(PyCFunction)writer_abort,METH_NOARGS,NULL},