Add incremental snapshots, Turbo decoding and bounded libarchive reader

This commit is contained in:
Eremey Valetov committed 2026-11-07 05:48:43 -05:00
1 parent 3d55f36019
commit 8e1c22aa28
46 files changed
+9469 -527

No files matched your search

+5 -1
View File
@@ -7,7 +7,7 @@ or compiler is needed when a wheel is available.
This is an alpha release. It supports the implemented legacy Huffman methods,
master dictionaries, directories, stored versions, and UC2 v3 rANS entries.
Turbo/method 80 and multi-volume archives remain unsupported. Use `UC2File` for reading and `UC2Writer` for creation.
Turbo/method 80 is supported for reading. Multi-volume sets require reassembly. Use `UC2File` for reading and `UC2Writer` for creation.
## Install and read
@@ -203,3 +203,7 @@ The distribution contains GPL-3.0-or-later and LGPL-3.0-only code. Independently
written Python integration code also has a BSD-2-Clause option. This does not
change the licenses of the bundled core or third-party code. License texts,
credits, and the per-file audit are included in the distribution.
Incremental directory versions are available through `uc2 snapshot` in the
Python package, with shared SHA-256-addressed UC2 chunks and verified restores.
See [the snapshot guide](https://github.com/evvaletov/uc2/blob/main/docs/snapshots.md).
+20 -1
View File
@@ -279,6 +279,25 @@ static int module_clear(PyObject *module)
Py_CLEAR(state->bad_archive); Py_CLEAR(state->resource_limit); return 0;
}
int uc2_python_writer_init(PyObject *module);
/* A bounded buffer contains at least a maximum-sized chunk unless it is
* the final buffer. Reuse the archive ingestion Gear boundaries exactly. */
#include "uc2/uc2_cdc.h"
static PyObject *chunk_length(PyObject *module, PyObject *args)
{
(void)module;
const char *data;
Py_ssize_t length;
if (!PyArg_ParseTuple(args, "y#", &data, &length)) return NULL;
struct uc2_chunker chunker;
size_t offset, size;
uc2_chunker_init(&chunker, 13, 0, 0);
uc2_chunker_next(&chunker, (const uint8_t *)data, (size_t)length, &offset, &size);
return PyLong_FromSize_t(size);
}
static PyMethodDef module_methods[] = {
{"chunk_length", chunk_length, METH_VARARGS, "First Gear chunk length (8 KiB target)."},
{NULL, NULL, 0, NULL}
};
static int module_exec(PyObject *module)
{
struct module_state *state = PyModule_GetState(module);
@@ -295,7 +314,7 @@ static int module_exec(PyObject *module)
}
static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {0, NULL} };
static struct PyModuleDef module_def = {
PyModuleDef_HEAD_INIT, "_native", NULL, sizeof(struct module_state), NULL,
PyModuleDef_HEAD_INIT, "_native", NULL, sizeof(struct module_state), module_methods,
module_slots, module_traverse, module_clear, NULL
};
PyMODINIT_FUNC PyInit__native(void) { return PyModuleDef_Init(&module_def); }
+2 -1
View File
@@ -76,7 +76,8 @@ def test_migration_retains_versions_metadata_and_bytes(collection, tmp_path, for
@pytest.mark.parametrize('format', ['zip', 'tar'])
def test_original_encoder_corpus(tmp_path, format):
report = preserve([ROOT / 'tests/archives'], tmp_path / format, format=format)
report = preserve([ROOT / 'tests/archives' / item['path'] for item in
json.loads((ROOT / 'tests/archives/manifest.json').read_text())['archives']], tmp_path / format, format=format)
assert report['status'] == 'ok'
assert len(report['archives']) == 5
for record in report['archives']:
+210
View File
@@ -0,0 +1,210 @@
"""Restorable versions, deduplication, corruption and failed publication."""
import hashlib
import json
import os
from pathlib import Path
import random
import subprocess
import sys
import pytest
from uc2 import snapshot as s
def tree(root):
return {p.relative_to(root).as_posix(): p.read_bytes() for p in root.rglob('*') if p.is_file()}
def manifest(repo, address):
return json.loads((repo / 'snapshots' / (address + '.json')).read_bytes())
def forged(repo, value):
data = json.dumps(value).encode()
digest = hashlib.sha256(data).hexdigest()
(repo / 'snapshots' / (digest + '.json')).write_bytes(data)
return digest
def test_incremental_versions_and_recovery(tmp_path):
src, repo = tmp_path/'src', tmp_path/'repo'
src.mkdir(); (src/'emptydir').mkdir(); (src/'empty').touch()
data = random.Random(77).randbytes(512*1024)
(src/'large.bin').write_bytes(data)
(src/'duplicate.bin').write_bytes(data)
(src/'old.txt').write_bytes(b'first version\n')
os.utime(src/'old.txt', ns=(1600000000123456789,)*2)
initial = tree(src)
first = s.create(src, repo, label='first')
assert first['new_chunks'] < len(manifest(repo, first['snapshot'])['entries'][0].get('chunks', [])) + 100
same = s.create(src, repo)
assert same['new_chunks'] == same['new_chunk_bytes'] == 0
(src/'old.txt').unlink()
(src/'new.txt').write_bytes(b'second version\n')
(src/'large.bin').write_bytes(data[:100000] + b'insertion' * 127 + data[100000:])
second_tree = tree(src)
second = s.create(src, repo, label='second')
assert second['new_chunk_bytes'] < first['new_chunk_bytes'] // 4
for result, expected in [(first, initial), (second, second_tree)]:
address = result['snapshot']
assert s.verify(repo, address)['verified_files'] == len(expected)
out = tmp_path/address
s.restore(repo, address, out)
assert tree(out) == expected
assert (out/'emptydir').is_dir()
assert (tmp_path/first['snapshot']/'old.txt').stat().st_mtime_ns // 10**6 == 1600000000123
assert len(s.snapshots(repo)) == 3
# Damage a shared chunk. A failed restore must expose no partial directory.
chunk = next((repo/'chunks').rglob('*.uc2'))
original = chunk.read_bytes()
chunk.write_bytes(b'damaged')
with pytest.raises((ValueError, OSError)):
s.restore(repo, first['snapshot'], tmp_path/'failed')
assert not (tmp_path/'failed').exists()
assert not list(tmp_path.glob('.uc2-restore-*'))
chunk.write_bytes(original)
s.restore(repo, first['snapshot'], tmp_path/'recovered')
assert tree(tmp_path/'recovered') == initial
def test_empty_limits_lock_and_cli(tmp_path):
src, repo = tmp_path/'src', tmp_path/'repo'
src.mkdir()
result = s.create(src, repo, max_total_size=0)
assert s.verify(repo, result['snapshot'], max_total_size=0)['verified_files'] == 0
out = subprocess.check_output([sys.executable, '-m', 'uc2', 'snapshot', 'list', str(repo)])
assert json.loads(out)[0]['snapshot'] == result['snapshot']
(src/'a').write_bytes(b'a')
with pytest.raises(ValueError, match='total size'):
s.create(src, repo, max_total_size=0)
assert len(s.snapshots(repo)) == 1
(repo/'.writer-lock').write_text('active')
with pytest.raises(FileExistsError):
s.create(src, repo)
assert (repo/'.writer-lock').read_text() == 'active'
with pytest.raises(ValueError, match='outside'):
s.create(src, src/'inside')
with pytest.raises(FileExistsError):
s.restore(repo, result['snapshot'], src)
@pytest.mark.parametrize('name', ['../escape', '/absolute', 'C:drive', 'con', 'a.', 'a\\b', 'a//b'])
def test_unsafe_manifest_paths(tmp_path, name):
src, repo = tmp_path/'src', tmp_path/'repo'
src.mkdir(); (src/'a').write_bytes(b'payload')
result = s.create(src, repo)
m = manifest(repo, result['snapshot'])
m['entries'][0]['path'] = name
address = forged(repo, m)
with pytest.raises(ValueError):
s.restore(repo, address, tmp_path/'out')
assert not (tmp_path/'out').exists()
def test_manifest_and_chunk_integrity(tmp_path):
src, repo = tmp_path/'src', tmp_path/'repo'
src.mkdir(); (src/'a').write_bytes(b'payload')
result = s.create(src, repo)
address = result['snapshot']
m = manifest(repo, address)
m['entries'][0]['sha256'] = '0'*64
with pytest.raises(ValueError, match='file SHA'):
s.restore(repo, forged(repo, m), tmp_path/'out')
path = repo/'snapshots'/(address+'.json')
path.write_bytes(path.read_bytes()+b' ')
with pytest.raises(ValueError, match='manifest SHA'):
s.verify(repo, address)
def test_source_change_and_failed_commit(tmp_path, monkeypatch):
src, repo = tmp_path/'src', tmp_path/'repo'
src.mkdir(); file = src/'a'; file.write_bytes(b'a'*100)
store = s._store
def changing(*args):
result = store(*args)
file.write_bytes(b'b'*100)
return result
monkeypatch.setattr(s, '_store', changing)
with pytest.raises(ValueError, match='source changed'):
s.create(src, repo)
assert not list((repo/'snapshots').iterdir())
assert not (repo/'.writer-lock').exists()
@pytest.mark.skipif(os.name == 'nt', reason='requires POSIX symlink/FIFO privileges')
def test_symlinks_special_files_and_collisions(tmp_path):
src, repo = tmp_path/'src', tmp_path/'repo'
src.mkdir(); (src/'link').symlink_to(tmp_path, target_is_directory=True)
with pytest.raises(ValueError, match='unsupported'):
s.create(src, repo)
(src/'link').unlink(); os.mkfifo(src/'fifo')
with pytest.raises(ValueError, match='unsupported'):
s.create(src, repo)
(src/'fifo').unlink()
(src/'A').write_bytes(b'a'); (src/'a').write_bytes(b'b')
if (src/'A').stat().st_ino == (src/'a').stat().st_ino:
pytest.skip('filesystem cannot represent a case collision')
with pytest.raises(ValueError, match='collision'):
s.create(src, repo)
def test_filesystem_without_hardlinks_and_repo_overlap(tmp_path, monkeypatch):
import errno
def unavailable(*args):
raise OSError(errno.ENOTSUP, 'no hardlinks')
monkeypatch.setattr(s.os, 'link', unavailable)
src, repo = tmp_path/'src', tmp_path/'repo'
src.mkdir(); (src/'a').write_bytes(b'portable storage')
result = s.create(src, repo)
s.restore(repo, result['snapshot'], tmp_path/'out')
assert tree(tmp_path/'out') == tree(src)
with pytest.raises(ValueError, match='outside'):
s.create(repo/'chunks', repo)
def test_identical_manifest_is_idempotent(tmp_path, monkeypatch):
from datetime import datetime, timezone
class Clock:
@staticmethod
def now(tz):
return datetime(2026, 11, 7, tzinfo=timezone.utc)
monkeypatch.setattr(s, 'datetime', Clock)
source, repo = tmp_path/'src', tmp_path/'repo'
source.mkdir()
first = s.create(source, repo)
second = s.create(source, repo)
assert first['snapshot'] == second['snapshot']
assert second['growth_bytes'] == second['new_manifest_bytes'] == 0
assert len(s.snapshots(repo)) == 1
@pytest.mark.skipif(os.name == 'nt', reason='POSIX directory permissions')
def test_failed_publish_cleans_restricted_directories(tmp_path, monkeypatch):
source, repo = tmp_path/'src', tmp_path/'repo'
(source/'d'/'nested').mkdir(parents=True)
(source/'d'/'nested'/'data').write_bytes(b'payload')
result = s.create(source, repo)
m = manifest(repo, result['snapshot'])
for entry in m['entries']:
entry['mode'] = 0
address = forged(repo, m)
def fail(*args):
raise OSError('injected publication failure')
monkeypatch.setattr(s.Path, 'rename', fail)
with pytest.raises(OSError, match='injected'):
s.restore(repo, address, tmp_path/'out')
assert not (tmp_path/'out').exists()
assert not list(tmp_path.glob('.uc2-restore-*'))
@pytest.mark.parametrize('code', [1, 50])
def test_windows_no_hardlink_error_mapping(tmp_path, monkeypatch, code):
import errno
def unavailable(*args):
error = OSError(errno.EINVAL, 'Win32 hardlinks unsupported')
error.winerror = code
raise error
monkeypatch.setattr(s.os, 'link', unavailable)
source = tmp_path/'source'; source.mkdir()
result = s.create(source, tmp_path/'repo')
assert s.verify(tmp_path/'repo', result['snapshot'])['verified_files'] == 0
+27
View File
@@ -0,0 +1,27 @@
"""Pinned Turbo outputs cross-checked with original DOS and independent Go."""
from pathlib import Path
import hashlib
import json
import pytest
from uc2 import UC2File
CORPUS = Path(__file__).resolve().parents[2] / 'tests/archives'
@pytest.mark.parametrize('case', json.loads((CORPUS/'turbo.json').read_text())['archives'])
def test_turbo_original_and_independent_hashes(case):
path = CORPUS/case['path']
assert hashlib.sha256(path.read_bytes()).hexdigest() == case['sha256']
with UC2File(path) as archive:
info = archive.getinfo(case['payload_path'])
assert info.method == 80 and info.file_size == case['payload_size']
assert hashlib.sha256(archive.read(info)).hexdigest() == case['payload_sha256']
def test_original_sas_parts_are_explicitly_unsupported():
from uc2 import BadArchiveError
manifest = json.loads((CORPUS/'sas/manifest.json').read_text())
for item in manifest['parts']:
path = CORPUS/'sas'/item['path']
assert hashlib.sha256(path.read_bytes()).hexdigest() == item['sha256']
with pytest.raises(BadArchiveError, match='SAS split archive'):
UC2File(path)
+4 -1
View File
@@ -60,7 +60,10 @@ def test_read_versions_and_roundtrip(archive, tmp_path):
def test_reference_archives():
for path in (ROOT / "tests/archives").glob("*.uc2"):
import json
corpus = json.loads((ROOT / "tests/archives/manifest.json").read_text())
for item in corpus["archives"]:
path = ROOT / "tests/archives" / item["path"]
with uc2.UC2File(path) as reader:
reader.verify()
for info in reader.infolist():
+3
View File
@@ -11,6 +11,9 @@ def executable() -> Path:
def main() -> int:
if sys.argv[1:2] == ["snapshot"]:
from .snapshot import main as snapshot_main
return snapshot_main(sys.argv[2:])
binary = executable()
if os.name == "posix":
os.execv(str(binary), [str(binary), *sys.argv[1:]])
+534
View File
@@ -0,0 +1,534 @@
# SPDX-License-Identifier: GPL-3.0-or-later OR BSD-2-Clause
"""Incremental directory snapshots using shared, verified UC2 chunks.
The repository is a directory, not a legacy monolithic UC2 archive. Source,
repository and destination parents must be trusted and remain stable while an
operation runs. Symlinks and special files are rejected. No pruning is implicit.
"""
from __future__ import annotations
import argparse
from contextlib import contextmanager
from datetime import datetime, timezone
import errno
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import stat
import tempfile
import unicodedata
from . import UC2File, UC2Writer, _native
FORMAT = {"format": "uc2-snapshot", "version": 1, "chunker": "gear-13-2048-32768",
"address": "sha256", "storage": "uc2-level-4"}
MAX_CHUNK = 32768
MAX_MANIFEST = 64 * 1024 * 1024
MAX_ENTRIES = 100_000
MAX_REFS = 500_000
DIGEST = re.compile(r"[0-9a-f]{64}\Z")
RESERVED = re.compile(r"(?:con|prn|aux|nul|com[1-9¹²³]|lpt[1-9¹²³])(?:\.|$)", re.I)
def _json(value):
return (json.dumps(value, ensure_ascii=True, sort_keys=True,
separators=(",", ":")) + "\n").encode("ascii")
def _hash(data):
return hashlib.sha256(data).hexdigest()
def _digest(value):
if not isinstance(value, str) or not DIGEST.fullmatch(value):
raise ValueError("invalid SHA-256 address")
return value
def _directory(path):
if not stat.S_ISDIR(path.lstat().st_mode):
raise ValueError(f"expected a real directory: {path}")
@contextmanager
def _regular(path):
before = path.lstat()
if not stat.S_ISREG(before.st_mode):
raise ValueError(f"expected a regular file: {path}")
flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
flags |= getattr(os, "O_NONBLOCK", 0)
fd = os.open(path, flags)
with os.fdopen(fd, "rb") as source:
opened = os.fstat(source.fileno())
if not stat.S_ISREG(opened.st_mode) or (before.st_dev, before.st_ino) != (opened.st_dev, opened.st_ino):
raise ValueError(f"source changed while opening: {path}")
yield source
def _read(path, limit):
with _regular(path) as source:
if os.fstat(source.fileno()).st_size > limit:
raise ValueError(f"file exceeds size limit: {path}")
data = source.read(limit + 1)
if len(data) > limit:
raise ValueError(f"file exceeds size limit: {path}")
return data
def _sync_directory(path):
# Windows has no portable directory fsync. File data is flushed on both.
if os.name == "posix":
fd = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
try:
os.fsync(fd)
finally:
os.close(fd)
def _publish_file(temporary, destination):
with temporary.open("r+b") as source:
os.fsync(source.fileno())
# Atomic, exclusive publication, including on NTFS; never overwrite a chunk.
try:
os.link(temporary, destination)
except OSError as error:
if (error.errno not in (errno.EPERM, errno.ENOTSUP, errno.EOPNOTSUPP, errno.ENOSYS)
and getattr(error, "winerror", None) not in (1, 50)):
raise
# FAT/exFAT and some network mounts have no hardlinks. All callers
# own the writer lock (or a new repository during initialization),
# and parents are trusted. Readers see only the completed rename.
if destination.exists() or destination.is_symlink():
raise FileExistsError(destination) from error
temporary.rename(destination)
else:
temporary.unlink()
_sync_directory(destination.parent)
def _save(destination, data):
fd, name = tempfile.mkstemp(prefix=".pending-", dir=destination.parent)
temporary = Path(name)
try:
with os.fdopen(fd, "wb") as output:
output.write(data)
output.flush()
os.fsync(output.fileno())
try:
_publish_file(temporary, destination)
except FileExistsError:
if _read(destination, len(data)) != data:
raise ValueError("existing content-addressed file differs")
return False
return True
finally:
temporary.unlink(missing_ok=True)
def _repository(repository, *, create=False):
root = Path(repository).absolute()
if create:
root.mkdir(exist_ok=True)
_directory(root)
config = root / "config.json"
if create and not config.exists():
# Refuse to adopt an unrelated or incomplete directory.
if any(root.iterdir()):
raise ValueError("repository has no config and is not empty")
for name in ("chunks", "snapshots"):
(root / name).mkdir()
_save(config, _json(FORMAT))
if json.loads(_read(config, 4096)) != FORMAT:
raise ValueError("unsupported snapshot repository format")
for name in ("chunks", "snapshots"):
_directory(root / name)
return root
@contextmanager
def _lock(root):
path = root / ".writer-lock"
# O_EXCL also rejects symlinks. A killed writer leaves a visible lock;
# remove it only after confirming that no writer is running.
with path.open("x", encoding="ascii") as lock:
lock.write(str(os.getpid()) + "\n")
try:
yield
finally:
path.unlink()
def _path(value):
if not isinstance(value, str) or not value or len(value.encode("utf-8")) > 2048:
raise ValueError("invalid snapshot path")
parts = value.split("/")
if len(parts) > 64:
raise ValueError("snapshot path exceeds 64 components")
for part in parts:
if (part in ("", ".", "..") or part[-1:] in (".", " ") or
RESERVED.match(part) or len(part.encode("utf-8")) > 240 or
any(ord(c) < 32 or ord(c) == 127 or c in '\\:< >"|?*'.replace(" ", "") for c in part)):
raise ValueError(f"path cannot be restored portably: {value!r}")
return tuple(parts)
def _key(value):
return unicodedata.normalize("NFC", value).casefold()
def _integer(value, low, high):
if type(value) is not int or not low <= value <= high:
raise ValueError("invalid snapshot integer or resource limit")
return value
def _metadata(path, kind, info=None):
s = path.lstat() if info is None else info
return {"type": kind, "mode": stat.S_IMODE(s.st_mode) & 0o777, "mtime_ns": s.st_mtime_ns}
def _chunk_path(root, digest, *, create=False):
_digest(digest)
directory = root / "chunks" / digest[:2]
if create:
try:
directory.mkdir()
except FileExistsError:
pass
else:
_sync_directory(root / "chunks")
_directory(directory)
return directory / (digest + ".uc2")
def _chunk(root, digest, size):
path = _chunk_path(root, digest)
with _regular(path) as source:
if os.fstat(source.fileno()).st_size > 1024 * 1024:
raise ValueError("chunk archive exceeds 1 MiB")
with UC2File(source, max_entries=1, max_decoder_memory=8 * 1024 * 1024, max_extract_size=MAX_CHUNK) as archive:
entries = archive.infolist(all_versions=True)
if len(entries) != 1 or entries[0].is_dir() or entries[0].file_size != size:
raise ValueError(f"invalid chunk archive: {digest}")
data = archive.read(entries[0])
if _hash(data) != digest:
raise ValueError(f"chunk SHA-256 mismatch: {digest}")
return data
def _store(root, data):
digest = _hash(data)
path = _chunk_path(root, digest, create=True)
if path.exists() or path.is_symlink():
if _chunk(root, digest, len(data)) != data:
raise ValueError("chunk collision")
return digest, 0
fd, name = tempfile.mkstemp(prefix=".pending-", dir=path.parent)
os.close(fd)
temporary = Path(name)
try:
with UC2Writer(temporary, level=4, max_entries=1) as archive:
archive.writestr("chunk", data)
# Verify the writer result before the chunk becomes reusable.
with UC2File(temporary, max_entries=1, max_extract_size=MAX_CHUNK) as archive:
if archive.read("chunk") != data:
raise ValueError("new chunk did not round trip")
size = temporary.stat().st_size
_publish_file(temporary, path)
return digest, size
finally:
temporary.unlink(missing_ok=True)
def _chunks(source):
pending = b""
while True:
more = source.read(MAX_CHUNK - len(pending))
pending += more
if not pending:
return
if len(pending) == MAX_CHUNK or not more:
length = _native.chunk_length(pending)
yield pending[:length]
pending = pending[length:]
def create(source, repository, *, label="", max_total_size=1 << 40):
"""Commit a directory version and return its address and measured growth.
All files are read and hashed, including unchanged files. Reused chunks are
decoded and checked. A failed run may leave unreferenced valid chunks, but
never publishes an incomplete snapshot. Concurrent writers are refused.
"""
_integer(max_total_size, 0, (1 << 63) - 1)
if not isinstance(label, str) or len(label.encode("utf-8")) > 1024:
raise ValueError("snapshot label must be at most 1024 UTF-8 bytes")
source = Path(source).absolute()
_directory(source)
dest = Path(repository).resolve()
if (dest == source.resolve() or source.resolve() in dest.parents or
dest in source.resolve().parents):
raise ValueError("repository and source must be outside each other")
root = _repository(repository, create=True)
with _lock(root):
entries, keys = [], set()
total = added = new_chunks = refs = 0
directories = []
def visit(directory, entry=None):
nonlocal total, added, new_chunks, refs
initial = directory.lstat()
if not stat.S_ISDIR(initial.st_mode):
raise ValueError(f"expected a real directory: {directory}")
if entry is not None:
entry.update(_metadata(directory, "directory", initial))
directories.append((directory, initial))
with os.scandir(directory) as scan:
children = sorted(scan, key=lambda e: e.name)
for child in children:
path = Path(child.path)
relative = path.relative_to(source).as_posix()
_path(relative)
key = _key(relative)
if key in keys:
raise ValueError(f"case or Unicode path collision: {relative}")
keys.add(key)
if len(keys) > MAX_ENTRIES:
raise ValueError("snapshot entry limit exceeded")
mode = path.lstat().st_mode
if stat.S_ISDIR(mode):
entry = dict(path=relative)
entries.append(entry)
visit(path, entry)
elif stat.S_ISREG(mode):
entry = dict(path=relative, **_metadata(path, "file"), chunks=[])
sha, size = hashlib.sha256(), 0
with _regular(path) as stream:
before = os.fstat(stream.fileno())
for data in _chunks(stream):
total += len(data)
if total > max_total_size:
raise ValueError("snapshot total size limit exceeded")
refs += 1
if refs > MAX_REFS:
raise ValueError("snapshot chunk-reference limit exceeded")
digest, growth = _store(root, data)
added += growth
new_chunks += bool(growth)
entry["chunks"].append([digest, len(data)])
sha.update(data)
size += len(data)
after = os.fstat(stream.fileno())
current = path.lstat()
signature = lambda s: (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
if signature(before) != signature(after) or signature(before) != signature(current) or size != before.st_size:
raise ValueError(f"source changed during snapshot: {relative}")
entry.update(size=size, sha256=sha.hexdigest(), mtime_ns=before.st_mtime_ns,
mode=stat.S_IMODE(before.st_mode) & 0o777)
entries.append(entry)
else:
raise ValueError(f"symlink or special file is unsupported: {relative}")
visit(source)
for directory, before in directories:
after = directory.lstat()
if (before.st_dev, before.st_ino, before.st_mtime_ns, before.st_ctime_ns) != (after.st_dev, after.st_ino, after.st_mtime_ns, after.st_ctime_ns):
raise ValueError(f"source directory changed during snapshot: {directory}")
manifest = {"format": FORMAT, "created": datetime.now(timezone.utc).isoformat(),
"label": label, "root": _metadata(source, "directory", directories[0][1]),
"entries": entries, "total_size": total}
data = _json(manifest)
if len(data) > MAX_MANIFEST:
raise ValueError("snapshot manifest exceeds 64 MiB")
address = _hash(data)
published = _save(root / "snapshots" / (address + ".json"), data)
return {"snapshot": address, "files": sum(e["type"] == "file" for e in entries),
"total_size": total, "new_chunks": new_chunks, "new_chunk_bytes": added,
"manifest_bytes": len(data), "new_manifest_bytes": len(data) if published else 0,
"growth_bytes": added + (len(data) if published else 0)}
def _manifest(root, address, max_total_size):
_digest(address)
data = _read(root / "snapshots" / (address + ".json"), MAX_MANIFEST)
if _hash(data) != address:
raise ValueError("manifest SHA-256 mismatch")
m = json.loads(data)
if not isinstance(m, dict) or m.get("format") != FORMAT:
raise ValueError("unsupported snapshot manifest")
if not isinstance(m.get("created"), str) or not isinstance(m.get("label"), str):
raise ValueError("invalid snapshot description")
entries = m.get("entries")
if not isinstance(entries, list) or len(entries) > MAX_ENTRIES:
raise ValueError("invalid snapshot entries")
paths, total, refs = {"": "directory"}, 0, 0
exact_paths = {""}
def metadata(e):
if not isinstance(e, dict) or e.get("type") not in ("file", "directory"):
raise ValueError("invalid snapshot entry")
_integer(e.get("mode"), 0, 0o777)
_integer(e.get("mtime_ns"), -(1 << 63), (1 << 63) - 1)
metadata(m.get("root"))
if m["root"]["type"] != "directory":
raise ValueError("invalid snapshot root")
for e in entries:
metadata(e)
parts = _path(e.get("path"))
key = _key(e["path"])
parent = _key("/".join(parts[:-1]))
if key in paths or paths.get(parent) != "directory" or "/".join(parts[:-1]) not in exact_paths:
raise ValueError("duplicate path or missing/invalid parent")
paths[key] = e["type"]
exact_paths.add(e["path"])
if e["type"] == "file":
_integer(e.get("size"), 0, max_total_size)
_digest(e.get("sha256"))
chunks = e.get("chunks")
if not isinstance(chunks, list):
raise ValueError("invalid chunk list")
refs += len(chunks)
if refs > MAX_REFS:
raise ValueError("snapshot chunk-reference limit exceeded")
size = 0
for c in chunks:
if not isinstance(c, list) or len(c) != 2:
raise ValueError("invalid chunk reference")
_digest(c[0])
size += _integer(c[1], 1, MAX_CHUNK)
if size != e["size"]:
raise ValueError("file and chunk sizes differ")
total += size
if total > max_total_size:
raise ValueError("snapshot total size limit exceeded")
if total != _integer(m.get("total_size"), 0, max_total_size):
raise ValueError("invalid snapshot total size")
return m
def _files(root, manifest, destination=None):
checked = 0
for e in manifest["entries"]:
if e["type"] == "directory":
if destination is not None:
(destination / e["path"]).mkdir()
continue
sha = hashlib.sha256()
output = None
try:
if destination is not None:
output = (destination / e["path"]).open("xb")
for digest, size in e["chunks"]:
data = _chunk(root, digest, size)
sha.update(data)
if output is not None:
output.write(data)
if sha.hexdigest() != e["sha256"]:
raise ValueError(f"file SHA-256 mismatch: {e['path']}")
if output is not None:
output.flush()
os.fsync(output.fileno())
checked += 1
finally:
if output is not None:
output.close()
return checked
def verify(repository, address, *, max_total_size=1 << 40):
"""Verify one manifest, all referenced chunks, and complete file hashes."""
_integer(max_total_size, 0, (1 << 63) - 1)
root = _repository(repository)
manifest = _manifest(root, address, max_total_size)
return {"snapshot": address, "verified_files": _files(root, manifest),
"total_size": manifest["total_size"]}
def restore(repository, address, destination, *, max_total_size=1 << 40):
"""Verify and restore into a NEW directory; failures publish no destination."""
_integer(max_total_size, 0, (1 << 63) - 1)
root = _repository(repository)
manifest = _manifest(root, address, max_total_size)
destination = Path(destination).absolute()
if destination.exists() or destination.is_symlink():
raise FileExistsError(destination)
_directory(destination.parent)
if root.resolve() == destination.resolve() or root.resolve() in destination.resolve().parents:
raise ValueError("restore destination must be outside the repository")
stage = Path(tempfile.mkdtemp(prefix=".uc2-restore-", dir=destination.parent))
reserved = False
try:
checked = _files(root, manifest, stage)
# Children first: readonly directories must not prevent restoration.
for e in reversed(manifest["entries"]):
path = stage / e["path"]
os.utime(path, ns=(e["mtime_ns"], e["mtime_ns"]))
os.chmod(path, e["mode"])
os.utime(stage, ns=(manifest["root"]["mtime_ns"],) * 2)
os.chmod(stage, manifest["root"]["mode"])
if os.name == "posix":
# Reserve the name exclusively; parents are trusted during restore.
destination.mkdir()
reserved = True
stage.rename(destination)
reserved = False
_sync_directory(destination.parent)
return {"snapshot": address, "restored_files": checked,
"total_size": manifest["total_size"]}
finally:
if reserved and stage.exists():
destination.rmdir()
if stage.exists():
# A metadata failure may have made directories readonly.
os.chmod(stage, 0o700)
for directory, dirs, files in os.walk(stage):
for name in dirs:
os.chmod(Path(directory) / name, 0o700)
for name in files:
os.chmod(Path(directory) / name, 0o600)
shutil.rmtree(stage)
def snapshots(repository):
"""List committed snapshot IDs with validated manifests."""
root = _repository(repository)
result = []
for path in sorted((root / "snapshots").glob("*.json")):
m = _manifest(root, path.stem, (1 << 63) - 1)
result.append({"snapshot": path.stem, "created": m["created"],
"label": m["label"], "total_size": m["total_size"]})
return result
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
new = commands.add_parser("create", help="snapshot a directory")
new.add_argument("source")
new.add_argument("repository")
new.add_argument("--label", default="")
for name in ("list", "verify", "restore"):
command = commands.add_parser(name)
command.add_argument("repository")
if name != "list":
command.add_argument("snapshot")
if name == "restore":
command.add_argument("destination")
if name != "list":
command.add_argument("--max-total-size", type=int, default=1 << 40)
new.add_argument("--max-total-size", type=int, default=1 << 40)
args = vars(parser.parse_args(argv))
command = args.pop("command")
if "snapshot" in args:
args["address"] = args.pop("snapshot")
try:
result = {"create": create, "list": snapshots, "verify": verify,
"restore": restore}[command](**args)
except (OSError, ValueError, TypeError, KeyError) as error:
parser.exit(1, f"uc2-snapshot: {error}\n")
print(json.dumps(result, ensure_ascii=True, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())