Add optional shared chunks across named archive entries and harden reader limits
This commit is contained in:
1 parent
fdbdd539b5
commit
07bb648e34
43 files changed
+2239
-101
No files matched your search
+6
-1
@@ -136,7 +136,7 @@ through EOF without seeking and leaves the input open. A supplied size must
|
||||
match exactly; detecting excess input can consume one extra byte.
|
||||
|
||||
Inputs spool to anonymous temporary files as they are added. Compression runs
|
||||
at close, using CDC-selected custom master dictionaries. Payload buffers have
|
||||
at close, using CDC-selected custom master dictionaries by default. Payload buffers have
|
||||
fixed sizes; metadata grows with entries and chunks. Temporary disk space
|
||||
covers the uncompressed inputs and completed archive, plus the destination's
|
||||
sibling temporary file during the final copy. Duplicate file paths store
|
||||
@@ -212,3 +212,8 @@ Filesystem paths now resolve SAS and DOS UC2SEA sets without executing input.
|
||||
Keep all volumes together. Snapshot maintenance adds explicit `prune`, `repair`
|
||||
and `compact`; packed repositories require alpha10 or later. See the
|
||||
[repository guides](https://github.com/evvaletov/uc2/tree/main/docs).
|
||||
|
||||
Optional `UC2Writer(..., chunked=True)` shares payload chunks across partly
|
||||
changed files and stored versions. Use the full 3.0.0a11 or newer reader.
|
||||
For method 11, `UC2Info.compress_size` counts the recipe only; shared payloads
|
||||
are excluded. See the [chunk-sharing guide](../docs/chunk-sharing.md).
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Shared payloads preserve named files; malformed recipes fail explicitly."""
|
||||
import hashlib
|
||||
import io
|
||||
import random
|
||||
import struct
|
||||
import subprocess
|
||||
import tarfile
|
||||
|
||||
import pytest
|
||||
from uc2 import UC2File, UC2Writer, BadArchiveError
|
||||
from uc2._cli import executable
|
||||
|
||||
MAGIC = b"UC2CHN1\0"
|
||||
HEADER, RECORD = 88, 48
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", range(2, 10))
|
||||
def test_named_chunks_versions_and_codecs(tmp_path, level):
|
||||
base = random.Random(7).randbytes(180000) + b"ABCD" * 22000
|
||||
changed = b"prefix change" + base[:160000] + b"X" + base[160000:]
|
||||
path = tmp_path / "shared.uc2"
|
||||
with UC2Writer(path, level=level, chunked=True) as writer:
|
||||
writer.writestr("folder/café.bin", base)
|
||||
writer.writestr("folder/café.bin", changed)
|
||||
writer.writestr("empty", b"")
|
||||
writer.writestr("copy", base)
|
||||
with UC2File(io.BytesIO(path.read_bytes())) as archive:
|
||||
files = [i for i in archive.infolist(all_versions=True) if not i.is_dir()]
|
||||
assert len(files) == 4
|
||||
assert all(i.method == 11 for i in files)
|
||||
assert archive.read("folder/café.bin", version=1) == base
|
||||
assert archive.read("folder/café.bin") == changed
|
||||
assert archive.read("copy") == base
|
||||
assert archive.read("empty") == b""
|
||||
assert writer.stats["chunk_references"] > writer.stats["unique_chunks"]
|
||||
assert writer.stats["shared_bytes"] > len(base)
|
||||
assert writer.stats["masters"] == 0
|
||||
|
||||
|
||||
def test_partial_edits_share_beyond_dictionary_window(tmp_path):
|
||||
base = random.Random(341).randbytes(512 * 1024)
|
||||
changed = base[:200000] + b"an inserted line\n" + base[200000:]
|
||||
sizes = {}
|
||||
for chunked in (False, True):
|
||||
path = tmp_path / f"{chunked}.uc2"
|
||||
with UC2Writer(path, chunked=chunked) as writer:
|
||||
writer.writestr("old.bin", base)
|
||||
writer.writestr("new.bin", changed)
|
||||
sizes[chunked] = path.stat().st_size
|
||||
with UC2File(path) as archive:
|
||||
assert archive.namelist() == ["old.bin", "new.bin"]
|
||||
assert archive.read("old.bin") == base
|
||||
assert archive.read("new.bin") == changed
|
||||
if chunked:
|
||||
assert writer.stats["shared_bytes"] > 400000
|
||||
else:
|
||||
assert writer.stats["chunk_references"] == 0
|
||||
assert sizes[True] < sizes[False] * 0.75
|
||||
|
||||
|
||||
def recipe_digest(data, start):
|
||||
count = struct.unpack_from("<I", data, start + 8)[0]
|
||||
data[start + 56:start + 88] = hashlib.sha256(
|
||||
data[start:start + 56] + data[start + HEADER:start + HEADER + count * RECORD]
|
||||
).digest()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fault", [
|
||||
"magic", "flags", "count", "size", "table_size", "table_digest",
|
||||
"offset_header", "offset_self", "offset_overflow", "zero_size", "large_size",
|
||||
"large_compressed", "method", "chunk_checksum", "chunk_digest", "file_digest",
|
||||
"payload_same_xor", "reordered",
|
||||
])
|
||||
def test_bad_chunk_recipes(tmp_path, fault):
|
||||
payload = random.Random(91).randbytes(70001)
|
||||
path = tmp_path / "bad.uc2"
|
||||
with UC2Writer(path, chunked=True) as writer:
|
||||
writer.writestr("data", payload)
|
||||
data = bytearray(path.read_bytes())
|
||||
start = data.index(MAGIC)
|
||||
row = start + HEADER
|
||||
value = {
|
||||
"flags": (start + 20, 1), "count": (start + 8, 1048577),
|
||||
"size": (start + 12, 70002), "table_size": (start + 16, 0),
|
||||
"offset_header": (row, 0), "offset_self": (row, start),
|
||||
"offset_overflow": (row, 0xFFFFFFFF), "zero_size": (row + 8, 0),
|
||||
"large_size": (row + 8, 32769), "large_compressed": (row + 4, 65537),
|
||||
}
|
||||
if fault in value:
|
||||
struct.pack_into("<I", data, *value[fault])
|
||||
elif fault == "method":
|
||||
struct.pack_into("<H", data, row + 12, 11) # nested recipes are forbidden
|
||||
elif fault == "magic":
|
||||
data[start] ^= 1
|
||||
elif fault == "table_digest":
|
||||
data[start + 56] ^= 1
|
||||
elif fault == "chunk_checksum":
|
||||
data[row + 14] ^= 1
|
||||
elif fault == "chunk_digest":
|
||||
data[row + 16] ^= 1
|
||||
elif fault == "file_digest":
|
||||
data[start + 24] ^= 1
|
||||
elif fault == "payload_same_xor":
|
||||
assert struct.unpack_from("<H", data, row + 12)[0] == 0
|
||||
offset = struct.unpack_from("<I", data, row)[0]
|
||||
data[offset] ^= 1
|
||||
data[offset + 2] ^= 1 # preserve the weak historical XOR checksum
|
||||
elif fault == "reordered":
|
||||
first = bytes(data[row:row + RECORD])
|
||||
data[row:row + RECORD] = data[row + RECORD:row + 2 * RECORD]
|
||||
data[row + RECORD:row + 2 * RECORD] = first
|
||||
if fault not in {"magic", "count", "table_digest"}:
|
||||
recipe_digest(data, start)
|
||||
path.write_bytes(data)
|
||||
with pytest.raises(BadArchiveError):
|
||||
with UC2File(path) as archive:
|
||||
archive.read("data")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", [4, 8])
|
||||
def test_damaged_compressed_chunks_preserve_destination(tmp_path, level):
|
||||
path = tmp_path / "compressed.uc2"
|
||||
payload = b"compressible data\n" * 5000
|
||||
with UC2Writer(path, level=level, chunked=True) as writer:
|
||||
writer.writestr("data", payload)
|
||||
data = bytearray(path.read_bytes())
|
||||
start = data.index(MAGIC)
|
||||
row = start + HEADER
|
||||
assert struct.unpack_from("<H", data, row + 12)[0] != 0
|
||||
offset = struct.unpack_from("<I", data, row)[0]
|
||||
data[offset] ^= 0xFF
|
||||
path.write_bytes(data)
|
||||
target = tmp_path / "output"
|
||||
target.mkdir()
|
||||
(target / "data").write_bytes(b"keep existing")
|
||||
with UC2File(path) as archive:
|
||||
with pytest.raises(BadArchiveError):
|
||||
archive.extractall(target, overwrite=True)
|
||||
assert (target / "data").read_bytes() == b"keep existing"
|
||||
|
||||
|
||||
def test_chunked_cli_tar_and_stream(tmp_path):
|
||||
base = random.Random(71).randbytes(180000)
|
||||
changed = base[:70000] + b"new" + base[70000:]
|
||||
tar = io.BytesIO()
|
||||
with tarfile.open(fileobj=tar, mode="w") as archive:
|
||||
for name, data in [("old", base), ("new", changed)]:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(data)
|
||||
archive.addfile(info, io.BytesIO(data))
|
||||
path = tmp_path / "tar.uc2"
|
||||
subprocess.run([executable(), "--chunked", "--ingest-tar", str(path)],
|
||||
input=tar.getvalue(), check=True, capture_output=True)
|
||||
with UC2File(path) as archive:
|
||||
assert archive.namelist() == ["old", "new"]
|
||||
assert archive.read("old") == base
|
||||
assert archive.read("new") == changed
|
||||
stream = base + base + changed
|
||||
path = tmp_path / "stream.uc2"
|
||||
subprocess.run([executable(), "--chunked", "--ingest", str(path)],
|
||||
input=stream, check=True, capture_output=True)
|
||||
with UC2File(path) as archive:
|
||||
assert archive.namelist() == ["data"]
|
||||
assert archive.read("data") == stream
|
||||
result = subprocess.run([executable(), "--ingest-restore", str(path)],
|
||||
check=True, capture_output=True)
|
||||
assert result.stdout == stream
|
||||
|
||||
|
||||
def test_chunked_option_validation(tmp_path):
|
||||
with pytest.raises(TypeError, match="chunked"):
|
||||
UC2Writer(tmp_path / "invalid.uc2", chunked="yes")
|
||||
with UC2Writer(tmp_path / "empty.uc2", chunked=True):
|
||||
pass
|
||||
with UC2File(tmp_path / "empty.uc2") as archive:
|
||||
assert archive.namelist() == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", [4, 8])
|
||||
@pytest.mark.parametrize("delta", [-2, 2])
|
||||
def test_compressed_chunk_must_consume_exact_word_range(tmp_path, level, delta):
|
||||
path = tmp_path / "word-bound.uc2"
|
||||
with UC2Writer(path, level=level, chunked=True) as writer:
|
||||
writer.writestr("data", b"compressed payload\n" * 12000)
|
||||
data = bytearray(path.read_bytes())
|
||||
start = data.index(MAGIC)
|
||||
row = start + HEADER
|
||||
assert struct.unpack_from("<H", data, row + 12)[0] != 0
|
||||
size = struct.unpack_from("<I", data, row + 4)[0]
|
||||
struct.pack_into("<I", data, row + 4, size + delta)
|
||||
recipe_digest(data, start)
|
||||
path.write_bytes(data)
|
||||
with pytest.raises(BadArchiveError):
|
||||
with UC2File(path) as archive:
|
||||
archive.read("data")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunked", [False, True])
|
||||
def test_many_repeated_version_names_are_not_a_ratio_bomb(tmp_path, chunked):
|
||||
path = tmp_path / "many-versions.uc2"
|
||||
name = "a" * 240
|
||||
with UC2Writer(path, chunked=chunked) as writer:
|
||||
for _ in range(4000):
|
||||
writer.writestr(name, b"a")
|
||||
with UC2File(path) as archive:
|
||||
assert len(archive.infolist(all_versions=True)) == 4000
|
||||
assert archive.read(name) == archive.read(name, version=3999) == b"a"
|
||||
@@ -68,7 +68,7 @@ class UC2Writer:
|
||||
"""
|
||||
|
||||
def __init__(self, file, *, level: int = 4, max_entries: int = 100_000,
|
||||
max_total_size: int = 0xFFFFFFFF):
|
||||
max_total_size: int = 0xFFFFFFFF, chunked: bool = False):
|
||||
self._writer = None
|
||||
self._busy = False
|
||||
self._temporary = None
|
||||
@@ -83,9 +83,11 @@ class UC2Writer:
|
||||
_integer("max_total_size", max_total_size, 0xFFFFFFFF)
|
||||
if not max_entries or not max_total_size:
|
||||
raise ValueError("writer limits must be positive")
|
||||
if not isinstance(chunked, bool):
|
||||
raise TypeError("chunked must be a bool")
|
||||
self._directories = {(): 0}
|
||||
self._files = set()
|
||||
self._writer = _native.Writer(level, max_entries, max_total_size)
|
||||
self._writer = _native.Writer(level, max_entries, max_total_size, chunked)
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
|
||||
+8
-6
@@ -30,14 +30,14 @@ static void set_error(Writer *self, int result)
|
||||
}
|
||||
static int writer_init(Writer *self, PyObject *args, PyObject *kwargs)
|
||||
{
|
||||
int level=4; unsigned maximum=100000; long long total=UINT32_MAX;
|
||||
static char *names[]={"level","max_entries","max_total_size",NULL};
|
||||
int level=4,chunked=0; unsigned maximum=100000; long long total=UINT32_MAX;
|
||||
static char *names[]={"level","max_entries","max_total_size","chunked",NULL};
|
||||
if (self->initialized) { PyErr_SetString(PyExc_RuntimeError,"writer cannot be initialized twice"); return -1; }
|
||||
if (!PyArg_ParseTupleAndKeywords(args,kwargs,"|iIL",names,&level,&maximum,&total)) return -1;
|
||||
if (!PyArg_ParseTupleAndKeywords(args,kwargs,"|iILp",names,&level,&maximum,&total,&chunked)) return -1;
|
||||
if (level<2 || level>9 || maximum<1 || maximum>1000000 || total<1 || total>UINT32_MAX) {
|
||||
PyErr_SetString(PyExc_ValueError,"invalid writer level or limits"); return -1;
|
||||
}
|
||||
struct uc2_writer_options options={level,maximum,(uint64_t)total};
|
||||
struct uc2_writer_options options={level,maximum,(uint64_t)total,chunked};
|
||||
self->initialized=1; self->writer=uc2_writer_create(&options);
|
||||
if (!self->writer) { PyErr_SetString(PyExc_OSError,"cannot allocate writer or create temporary input file"); return -1; }
|
||||
return 0;
|
||||
@@ -108,8 +108,10 @@ static PyObject *writer_finish(Writer *self, PyObject *destination)
|
||||
if (r<0 || PyErr_Occurred()) { set_error(self,r); return NULL; }
|
||||
struct uc2_writer_stats stats; uc2_writer_get_stats(self->writer,&stats);
|
||||
uc2_writer_free(self->writer); self->writer=NULL;
|
||||
return Py_BuildValue("{sI,sI,sI,sI}","files",stats.files,"directories",stats.directories,
|
||||
"masters",stats.masters,"archive_size",stats.archive_size);
|
||||
return Py_BuildValue("{sI,sI,sI,sI,sI,sI,sK}","files",stats.files,"directories",stats.directories,
|
||||
"masters",stats.masters,"archive_size",stats.archive_size,
|
||||
"unique_chunks",stats.unique_chunks,"chunk_references",stats.chunk_references,
|
||||
"shared_bytes",(unsigned long long)stats.shared_bytes);
|
||||
}
|
||||
static PyMethodDef methods[]={
|
||||
{"abort",(PyCFunction)writer_abort,METH_NOARGS,NULL},
|
||||
|
||||
Reference in new issue
Block a user