Add incremental snapshots, Turbo decoding and bounded libarchive reader
This commit is contained in:
1 parent
3d55f36019
commit
8e1c22aa28
46 files changed
+9469
-527
No files matched your search
+5
-1
@@ -7,7 +7,7 @@ or compiler is needed when a wheel is available.
|
||||
|
||||
This is an alpha release. It supports the implemented legacy Huffman methods,
|
||||
master dictionaries, directories, stored versions, and UC2 v3 rANS entries.
|
||||
Turbo/method 80 and multi-volume archives remain unsupported. Use `UC2File` for reading and `UC2Writer` for creation.
|
||||
Turbo/method 80 is supported for reading. Multi-volume sets require reassembly. Use `UC2File` for reading and `UC2Writer` for creation.
|
||||
|
||||
## Install and read
|
||||
|
||||
@@ -203,3 +203,7 @@ The distribution contains GPL-3.0-or-later and LGPL-3.0-only code. Independently
|
||||
written Python integration code also has a BSD-2-Clause option. This does not
|
||||
change the licenses of the bundled core or third-party code. License texts,
|
||||
credits, and the per-file audit are included in the distribution.
|
||||
|
||||
Incremental directory versions are available through `uc2 snapshot` in the
|
||||
Python package, with shared SHA-256-addressed UC2 chunks and verified restores.
|
||||
See [the snapshot guide](https://github.com/evvaletov/uc2/blob/main/docs/snapshots.md).
|
||||
+20
-1
@@ -279,6 +279,25 @@ static int module_clear(PyObject *module)
|
||||
Py_CLEAR(state->bad_archive); Py_CLEAR(state->resource_limit); return 0;
|
||||
}
|
||||
int uc2_python_writer_init(PyObject *module);
|
||||
/* A bounded buffer contains at least a maximum-sized chunk unless it is
|
||||
* the final buffer. Reuse the archive ingestion Gear boundaries exactly. */
|
||||
#include "uc2/uc2_cdc.h"
|
||||
static PyObject *chunk_length(PyObject *module, PyObject *args)
|
||||
{
|
||||
(void)module;
|
||||
const char *data;
|
||||
Py_ssize_t length;
|
||||
if (!PyArg_ParseTuple(args, "y#", &data, &length)) return NULL;
|
||||
struct uc2_chunker chunker;
|
||||
size_t offset, size;
|
||||
uc2_chunker_init(&chunker, 13, 0, 0);
|
||||
uc2_chunker_next(&chunker, (const uint8_t *)data, (size_t)length, &offset, &size);
|
||||
return PyLong_FromSize_t(size);
|
||||
}
|
||||
static PyMethodDef module_methods[] = {
|
||||
{"chunk_length", chunk_length, METH_VARARGS, "First Gear chunk length (8 KiB target)."},
|
||||
{NULL, NULL, 0, NULL}
|
||||
};
|
||||
static int module_exec(PyObject *module)
|
||||
{
|
||||
struct module_state *state = PyModule_GetState(module);
|
||||
@@ -295,7 +314,7 @@ static int module_exec(PyObject *module)
|
||||
}
|
||||
static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {0, NULL} };
|
||||
static struct PyModuleDef module_def = {
|
||||
PyModuleDef_HEAD_INIT, "_native", NULL, sizeof(struct module_state), NULL,
|
||||
PyModuleDef_HEAD_INIT, "_native", NULL, sizeof(struct module_state), module_methods,
|
||||
module_slots, module_traverse, module_clear, NULL
|
||||
};
|
||||
PyMODINIT_FUNC PyInit__native(void) { return PyModuleDef_Init(&module_def); }
|
||||
@@ -76,7 +76,8 @@ def test_migration_retains_versions_metadata_and_bytes(collection, tmp_path, for
|
||||
|
||||
@pytest.mark.parametrize('format', ['zip', 'tar'])
|
||||
def test_original_encoder_corpus(tmp_path, format):
|
||||
report = preserve([ROOT / 'tests/archives'], tmp_path / format, format=format)
|
||||
report = preserve([ROOT / 'tests/archives' / item['path'] for item in
|
||||
json.loads((ROOT / 'tests/archives/manifest.json').read_text())['archives']], tmp_path / format, format=format)
|
||||
assert report['status'] == 'ok'
|
||||
assert len(report['archives']) == 5
|
||||
for record in report['archives']:
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Restorable versions, deduplication, corruption and failed publication."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from uc2 import snapshot as s
|
||||
|
||||
|
||||
def tree(root):
|
||||
return {p.relative_to(root).as_posix(): p.read_bytes() for p in root.rglob('*') if p.is_file()}
|
||||
|
||||
|
||||
def manifest(repo, address):
|
||||
return json.loads((repo / 'snapshots' / (address + '.json')).read_bytes())
|
||||
|
||||
|
||||
def forged(repo, value):
|
||||
data = json.dumps(value).encode()
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
(repo / 'snapshots' / (digest + '.json')).write_bytes(data)
|
||||
return digest
|
||||
|
||||
|
||||
def test_incremental_versions_and_recovery(tmp_path):
|
||||
src, repo = tmp_path/'src', tmp_path/'repo'
|
||||
src.mkdir(); (src/'emptydir').mkdir(); (src/'empty').touch()
|
||||
data = random.Random(77).randbytes(512*1024)
|
||||
(src/'large.bin').write_bytes(data)
|
||||
(src/'duplicate.bin').write_bytes(data)
|
||||
(src/'old.txt').write_bytes(b'first version\n')
|
||||
os.utime(src/'old.txt', ns=(1600000000123456789,)*2)
|
||||
initial = tree(src)
|
||||
first = s.create(src, repo, label='first')
|
||||
assert first['new_chunks'] < len(manifest(repo, first['snapshot'])['entries'][0].get('chunks', [])) + 100
|
||||
same = s.create(src, repo)
|
||||
assert same['new_chunks'] == same['new_chunk_bytes'] == 0
|
||||
(src/'old.txt').unlink()
|
||||
(src/'new.txt').write_bytes(b'second version\n')
|
||||
(src/'large.bin').write_bytes(data[:100000] + b'insertion' * 127 + data[100000:])
|
||||
second_tree = tree(src)
|
||||
second = s.create(src, repo, label='second')
|
||||
assert second['new_chunk_bytes'] < first['new_chunk_bytes'] // 4
|
||||
for result, expected in [(first, initial), (second, second_tree)]:
|
||||
address = result['snapshot']
|
||||
assert s.verify(repo, address)['verified_files'] == len(expected)
|
||||
out = tmp_path/address
|
||||
s.restore(repo, address, out)
|
||||
assert tree(out) == expected
|
||||
assert (out/'emptydir').is_dir()
|
||||
assert (tmp_path/first['snapshot']/'old.txt').stat().st_mtime_ns // 10**6 == 1600000000123
|
||||
assert len(s.snapshots(repo)) == 3
|
||||
# Damage a shared chunk. A failed restore must expose no partial directory.
|
||||
chunk = next((repo/'chunks').rglob('*.uc2'))
|
||||
original = chunk.read_bytes()
|
||||
chunk.write_bytes(b'damaged')
|
||||
with pytest.raises((ValueError, OSError)):
|
||||
s.restore(repo, first['snapshot'], tmp_path/'failed')
|
||||
assert not (tmp_path/'failed').exists()
|
||||
assert not list(tmp_path.glob('.uc2-restore-*'))
|
||||
chunk.write_bytes(original)
|
||||
s.restore(repo, first['snapshot'], tmp_path/'recovered')
|
||||
assert tree(tmp_path/'recovered') == initial
|
||||
|
||||
|
||||
def test_empty_limits_lock_and_cli(tmp_path):
|
||||
src, repo = tmp_path/'src', tmp_path/'repo'
|
||||
src.mkdir()
|
||||
result = s.create(src, repo, max_total_size=0)
|
||||
assert s.verify(repo, result['snapshot'], max_total_size=0)['verified_files'] == 0
|
||||
out = subprocess.check_output([sys.executable, '-m', 'uc2', 'snapshot', 'list', str(repo)])
|
||||
assert json.loads(out)[0]['snapshot'] == result['snapshot']
|
||||
(src/'a').write_bytes(b'a')
|
||||
with pytest.raises(ValueError, match='total size'):
|
||||
s.create(src, repo, max_total_size=0)
|
||||
assert len(s.snapshots(repo)) == 1
|
||||
(repo/'.writer-lock').write_text('active')
|
||||
with pytest.raises(FileExistsError):
|
||||
s.create(src, repo)
|
||||
assert (repo/'.writer-lock').read_text() == 'active'
|
||||
with pytest.raises(ValueError, match='outside'):
|
||||
s.create(src, src/'inside')
|
||||
with pytest.raises(FileExistsError):
|
||||
s.restore(repo, result['snapshot'], src)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('name', ['../escape', '/absolute', 'C:drive', 'con', 'a.', 'a\\b', 'a//b'])
|
||||
def test_unsafe_manifest_paths(tmp_path, name):
|
||||
src, repo = tmp_path/'src', tmp_path/'repo'
|
||||
src.mkdir(); (src/'a').write_bytes(b'payload')
|
||||
result = s.create(src, repo)
|
||||
m = manifest(repo, result['snapshot'])
|
||||
m['entries'][0]['path'] = name
|
||||
address = forged(repo, m)
|
||||
with pytest.raises(ValueError):
|
||||
s.restore(repo, address, tmp_path/'out')
|
||||
assert not (tmp_path/'out').exists()
|
||||
|
||||
|
||||
def test_manifest_and_chunk_integrity(tmp_path):
|
||||
src, repo = tmp_path/'src', tmp_path/'repo'
|
||||
src.mkdir(); (src/'a').write_bytes(b'payload')
|
||||
result = s.create(src, repo)
|
||||
address = result['snapshot']
|
||||
m = manifest(repo, address)
|
||||
m['entries'][0]['sha256'] = '0'*64
|
||||
with pytest.raises(ValueError, match='file SHA'):
|
||||
s.restore(repo, forged(repo, m), tmp_path/'out')
|
||||
path = repo/'snapshots'/(address+'.json')
|
||||
path.write_bytes(path.read_bytes()+b' ')
|
||||
with pytest.raises(ValueError, match='manifest SHA'):
|
||||
s.verify(repo, address)
|
||||
|
||||
|
||||
def test_source_change_and_failed_commit(tmp_path, monkeypatch):
|
||||
src, repo = tmp_path/'src', tmp_path/'repo'
|
||||
src.mkdir(); file = src/'a'; file.write_bytes(b'a'*100)
|
||||
store = s._store
|
||||
def changing(*args):
|
||||
result = store(*args)
|
||||
file.write_bytes(b'b'*100)
|
||||
return result
|
||||
monkeypatch.setattr(s, '_store', changing)
|
||||
with pytest.raises(ValueError, match='source changed'):
|
||||
s.create(src, repo)
|
||||
assert not list((repo/'snapshots').iterdir())
|
||||
assert not (repo/'.writer-lock').exists()
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == 'nt', reason='requires POSIX symlink/FIFO privileges')
|
||||
def test_symlinks_special_files_and_collisions(tmp_path):
|
||||
src, repo = tmp_path/'src', tmp_path/'repo'
|
||||
src.mkdir(); (src/'link').symlink_to(tmp_path, target_is_directory=True)
|
||||
with pytest.raises(ValueError, match='unsupported'):
|
||||
s.create(src, repo)
|
||||
(src/'link').unlink(); os.mkfifo(src/'fifo')
|
||||
with pytest.raises(ValueError, match='unsupported'):
|
||||
s.create(src, repo)
|
||||
(src/'fifo').unlink()
|
||||
(src/'A').write_bytes(b'a'); (src/'a').write_bytes(b'b')
|
||||
if (src/'A').stat().st_ino == (src/'a').stat().st_ino:
|
||||
pytest.skip('filesystem cannot represent a case collision')
|
||||
with pytest.raises(ValueError, match='collision'):
|
||||
s.create(src, repo)
|
||||
|
||||
|
||||
def test_filesystem_without_hardlinks_and_repo_overlap(tmp_path, monkeypatch):
|
||||
import errno
|
||||
def unavailable(*args):
|
||||
raise OSError(errno.ENOTSUP, 'no hardlinks')
|
||||
monkeypatch.setattr(s.os, 'link', unavailable)
|
||||
src, repo = tmp_path/'src', tmp_path/'repo'
|
||||
src.mkdir(); (src/'a').write_bytes(b'portable storage')
|
||||
result = s.create(src, repo)
|
||||
s.restore(repo, result['snapshot'], tmp_path/'out')
|
||||
assert tree(tmp_path/'out') == tree(src)
|
||||
with pytest.raises(ValueError, match='outside'):
|
||||
s.create(repo/'chunks', repo)
|
||||
|
||||
|
||||
def test_identical_manifest_is_idempotent(tmp_path, monkeypatch):
|
||||
from datetime import datetime, timezone
|
||||
class Clock:
|
||||
@staticmethod
|
||||
def now(tz):
|
||||
return datetime(2026, 11, 7, tzinfo=timezone.utc)
|
||||
monkeypatch.setattr(s, 'datetime', Clock)
|
||||
source, repo = tmp_path/'src', tmp_path/'repo'
|
||||
source.mkdir()
|
||||
first = s.create(source, repo)
|
||||
second = s.create(source, repo)
|
||||
assert first['snapshot'] == second['snapshot']
|
||||
assert second['growth_bytes'] == second['new_manifest_bytes'] == 0
|
||||
assert len(s.snapshots(repo)) == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == 'nt', reason='POSIX directory permissions')
|
||||
def test_failed_publish_cleans_restricted_directories(tmp_path, monkeypatch):
|
||||
source, repo = tmp_path/'src', tmp_path/'repo'
|
||||
(source/'d'/'nested').mkdir(parents=True)
|
||||
(source/'d'/'nested'/'data').write_bytes(b'payload')
|
||||
result = s.create(source, repo)
|
||||
m = manifest(repo, result['snapshot'])
|
||||
for entry in m['entries']:
|
||||
entry['mode'] = 0
|
||||
address = forged(repo, m)
|
||||
def fail(*args):
|
||||
raise OSError('injected publication failure')
|
||||
monkeypatch.setattr(s.Path, 'rename', fail)
|
||||
with pytest.raises(OSError, match='injected'):
|
||||
s.restore(repo, address, tmp_path/'out')
|
||||
assert not (tmp_path/'out').exists()
|
||||
assert not list(tmp_path.glob('.uc2-restore-*'))
|
||||
|
||||
|
||||
@pytest.mark.parametrize('code', [1, 50])
|
||||
def test_windows_no_hardlink_error_mapping(tmp_path, monkeypatch, code):
|
||||
import errno
|
||||
def unavailable(*args):
|
||||
error = OSError(errno.EINVAL, 'Win32 hardlinks unsupported')
|
||||
error.winerror = code
|
||||
raise error
|
||||
monkeypatch.setattr(s.os, 'link', unavailable)
|
||||
source = tmp_path/'source'; source.mkdir()
|
||||
result = s.create(source, tmp_path/'repo')
|
||||
assert s.verify(tmp_path/'repo', result['snapshot'])['verified_files'] == 0
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Pinned Turbo outputs cross-checked with original DOS and independent Go."""
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import json
|
||||
import pytest
|
||||
from uc2 import UC2File
|
||||
|
||||
CORPUS = Path(__file__).resolve().parents[2] / 'tests/archives'
|
||||
|
||||
@pytest.mark.parametrize('case', json.loads((CORPUS/'turbo.json').read_text())['archives'])
|
||||
def test_turbo_original_and_independent_hashes(case):
|
||||
path = CORPUS/case['path']
|
||||
assert hashlib.sha256(path.read_bytes()).hexdigest() == case['sha256']
|
||||
with UC2File(path) as archive:
|
||||
info = archive.getinfo(case['payload_path'])
|
||||
assert info.method == 80 and info.file_size == case['payload_size']
|
||||
assert hashlib.sha256(archive.read(info)).hexdigest() == case['payload_sha256']
|
||||
|
||||
|
||||
def test_original_sas_parts_are_explicitly_unsupported():
|
||||
from uc2 import BadArchiveError
|
||||
manifest = json.loads((CORPUS/'sas/manifest.json').read_text())
|
||||
for item in manifest['parts']:
|
||||
path = CORPUS/'sas'/item['path']
|
||||
assert hashlib.sha256(path.read_bytes()).hexdigest() == item['sha256']
|
||||
with pytest.raises(BadArchiveError, match='SAS split archive'):
|
||||
UC2File(path)
|
||||
@@ -60,7 +60,10 @@ def test_read_versions_and_roundtrip(archive, tmp_path):
|
||||
|
||||
|
||||
def test_reference_archives():
|
||||
for path in (ROOT / "tests/archives").glob("*.uc2"):
|
||||
import json
|
||||
corpus = json.loads((ROOT / "tests/archives/manifest.json").read_text())
|
||||
for item in corpus["archives"]:
|
||||
path = ROOT / "tests/archives" / item["path"]
|
||||
with uc2.UC2File(path) as reader:
|
||||
reader.verify()
|
||||
for info in reader.infolist():
|
||||
|
||||
@@ -11,6 +11,9 @@ def executable() -> Path:
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if sys.argv[1:2] == ["snapshot"]:
|
||||
from .snapshot import main as snapshot_main
|
||||
return snapshot_main(sys.argv[2:])
|
||||
binary = executable()
|
||||
if os.name == "posix":
|
||||
os.execv(str(binary), [str(binary), *sys.argv[1:]])
|
||||
|
||||
@@ -0,0 +1,534 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later OR BSD-2-Clause
|
||||
"""Incremental directory snapshots using shared, verified UC2 chunks.
|
||||
|
||||
The repository is a directory, not a legacy monolithic UC2 archive. Source,
|
||||
repository and destination parents must be trusted and remain stable while an
|
||||
operation runs. Symlinks and special files are rejected. No pruning is implicit.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import unicodedata
|
||||
|
||||
from . import UC2File, UC2Writer, _native
|
||||
|
||||
FORMAT = {"format": "uc2-snapshot", "version": 1, "chunker": "gear-13-2048-32768",
|
||||
"address": "sha256", "storage": "uc2-level-4"}
|
||||
MAX_CHUNK = 32768
|
||||
MAX_MANIFEST = 64 * 1024 * 1024
|
||||
MAX_ENTRIES = 100_000
|
||||
MAX_REFS = 500_000
|
||||
DIGEST = re.compile(r"[0-9a-f]{64}\Z")
|
||||
RESERVED = re.compile(r"(?:con|prn|aux|nul|com[1-9¹²³]|lpt[1-9¹²³])(?:\.|$)", re.I)
|
||||
|
||||
|
||||
def _json(value):
|
||||
return (json.dumps(value, ensure_ascii=True, sort_keys=True,
|
||||
separators=(",", ":")) + "\n").encode("ascii")
|
||||
|
||||
|
||||
def _hash(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _digest(value):
|
||||
if not isinstance(value, str) or not DIGEST.fullmatch(value):
|
||||
raise ValueError("invalid SHA-256 address")
|
||||
return value
|
||||
|
||||
|
||||
def _directory(path):
|
||||
if not stat.S_ISDIR(path.lstat().st_mode):
|
||||
raise ValueError(f"expected a real directory: {path}")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _regular(path):
|
||||
before = path.lstat()
|
||||
if not stat.S_ISREG(before.st_mode):
|
||||
raise ValueError(f"expected a regular file: {path}")
|
||||
flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||
flags |= getattr(os, "O_NONBLOCK", 0)
|
||||
fd = os.open(path, flags)
|
||||
with os.fdopen(fd, "rb") as source:
|
||||
opened = os.fstat(source.fileno())
|
||||
if not stat.S_ISREG(opened.st_mode) or (before.st_dev, before.st_ino) != (opened.st_dev, opened.st_ino):
|
||||
raise ValueError(f"source changed while opening: {path}")
|
||||
yield source
|
||||
|
||||
|
||||
def _read(path, limit):
|
||||
with _regular(path) as source:
|
||||
if os.fstat(source.fileno()).st_size > limit:
|
||||
raise ValueError(f"file exceeds size limit: {path}")
|
||||
data = source.read(limit + 1)
|
||||
if len(data) > limit:
|
||||
raise ValueError(f"file exceeds size limit: {path}")
|
||||
return data
|
||||
|
||||
|
||||
def _sync_directory(path):
|
||||
# Windows has no portable directory fsync. File data is flushed on both.
|
||||
if os.name == "posix":
|
||||
fd = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _publish_file(temporary, destination):
|
||||
with temporary.open("r+b") as source:
|
||||
os.fsync(source.fileno())
|
||||
# Atomic, exclusive publication, including on NTFS; never overwrite a chunk.
|
||||
try:
|
||||
os.link(temporary, destination)
|
||||
except OSError as error:
|
||||
if (error.errno not in (errno.EPERM, errno.ENOTSUP, errno.EOPNOTSUPP, errno.ENOSYS)
|
||||
and getattr(error, "winerror", None) not in (1, 50)):
|
||||
raise
|
||||
# FAT/exFAT and some network mounts have no hardlinks. All callers
|
||||
# own the writer lock (or a new repository during initialization),
|
||||
# and parents are trusted. Readers see only the completed rename.
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise FileExistsError(destination) from error
|
||||
temporary.rename(destination)
|
||||
else:
|
||||
temporary.unlink()
|
||||
_sync_directory(destination.parent)
|
||||
|
||||
|
||||
def _save(destination, data):
|
||||
fd, name = tempfile.mkstemp(prefix=".pending-", dir=destination.parent)
|
||||
temporary = Path(name)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as output:
|
||||
output.write(data)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
try:
|
||||
_publish_file(temporary, destination)
|
||||
except FileExistsError:
|
||||
if _read(destination, len(data)) != data:
|
||||
raise ValueError("existing content-addressed file differs")
|
||||
return False
|
||||
return True
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _repository(repository, *, create=False):
|
||||
root = Path(repository).absolute()
|
||||
if create:
|
||||
root.mkdir(exist_ok=True)
|
||||
_directory(root)
|
||||
config = root / "config.json"
|
||||
if create and not config.exists():
|
||||
# Refuse to adopt an unrelated or incomplete directory.
|
||||
if any(root.iterdir()):
|
||||
raise ValueError("repository has no config and is not empty")
|
||||
for name in ("chunks", "snapshots"):
|
||||
(root / name).mkdir()
|
||||
_save(config, _json(FORMAT))
|
||||
if json.loads(_read(config, 4096)) != FORMAT:
|
||||
raise ValueError("unsupported snapshot repository format")
|
||||
for name in ("chunks", "snapshots"):
|
||||
_directory(root / name)
|
||||
return root
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _lock(root):
|
||||
path = root / ".writer-lock"
|
||||
# O_EXCL also rejects symlinks. A killed writer leaves a visible lock;
|
||||
# remove it only after confirming that no writer is running.
|
||||
with path.open("x", encoding="ascii") as lock:
|
||||
lock.write(str(os.getpid()) + "\n")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
path.unlink()
|
||||
|
||||
|
||||
def _path(value):
|
||||
if not isinstance(value, str) or not value or len(value.encode("utf-8")) > 2048:
|
||||
raise ValueError("invalid snapshot path")
|
||||
parts = value.split("/")
|
||||
if len(parts) > 64:
|
||||
raise ValueError("snapshot path exceeds 64 components")
|
||||
for part in parts:
|
||||
if (part in ("", ".", "..") or part[-1:] in (".", " ") or
|
||||
RESERVED.match(part) or len(part.encode("utf-8")) > 240 or
|
||||
any(ord(c) < 32 or ord(c) == 127 or c in '\\:< >"|?*'.replace(" ", "") for c in part)):
|
||||
raise ValueError(f"path cannot be restored portably: {value!r}")
|
||||
return tuple(parts)
|
||||
|
||||
|
||||
def _key(value):
|
||||
return unicodedata.normalize("NFC", value).casefold()
|
||||
|
||||
|
||||
def _integer(value, low, high):
|
||||
if type(value) is not int or not low <= value <= high:
|
||||
raise ValueError("invalid snapshot integer or resource limit")
|
||||
return value
|
||||
|
||||
|
||||
def _metadata(path, kind, info=None):
|
||||
s = path.lstat() if info is None else info
|
||||
return {"type": kind, "mode": stat.S_IMODE(s.st_mode) & 0o777, "mtime_ns": s.st_mtime_ns}
|
||||
|
||||
|
||||
def _chunk_path(root, digest, *, create=False):
|
||||
_digest(digest)
|
||||
directory = root / "chunks" / digest[:2]
|
||||
if create:
|
||||
try:
|
||||
directory.mkdir()
|
||||
except FileExistsError:
|
||||
pass
|
||||
else:
|
||||
_sync_directory(root / "chunks")
|
||||
_directory(directory)
|
||||
return directory / (digest + ".uc2")
|
||||
|
||||
|
||||
def _chunk(root, digest, size):
|
||||
path = _chunk_path(root, digest)
|
||||
with _regular(path) as source:
|
||||
if os.fstat(source.fileno()).st_size > 1024 * 1024:
|
||||
raise ValueError("chunk archive exceeds 1 MiB")
|
||||
with UC2File(source, max_entries=1, max_decoder_memory=8 * 1024 * 1024, max_extract_size=MAX_CHUNK) as archive:
|
||||
entries = archive.infolist(all_versions=True)
|
||||
if len(entries) != 1 or entries[0].is_dir() or entries[0].file_size != size:
|
||||
raise ValueError(f"invalid chunk archive: {digest}")
|
||||
data = archive.read(entries[0])
|
||||
if _hash(data) != digest:
|
||||
raise ValueError(f"chunk SHA-256 mismatch: {digest}")
|
||||
return data
|
||||
|
||||
|
||||
def _store(root, data):
|
||||
digest = _hash(data)
|
||||
path = _chunk_path(root, digest, create=True)
|
||||
if path.exists() or path.is_symlink():
|
||||
if _chunk(root, digest, len(data)) != data:
|
||||
raise ValueError("chunk collision")
|
||||
return digest, 0
|
||||
fd, name = tempfile.mkstemp(prefix=".pending-", dir=path.parent)
|
||||
os.close(fd)
|
||||
temporary = Path(name)
|
||||
try:
|
||||
with UC2Writer(temporary, level=4, max_entries=1) as archive:
|
||||
archive.writestr("chunk", data)
|
||||
# Verify the writer result before the chunk becomes reusable.
|
||||
with UC2File(temporary, max_entries=1, max_extract_size=MAX_CHUNK) as archive:
|
||||
if archive.read("chunk") != data:
|
||||
raise ValueError("new chunk did not round trip")
|
||||
size = temporary.stat().st_size
|
||||
_publish_file(temporary, path)
|
||||
return digest, size
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _chunks(source):
|
||||
pending = b""
|
||||
while True:
|
||||
more = source.read(MAX_CHUNK - len(pending))
|
||||
pending += more
|
||||
if not pending:
|
||||
return
|
||||
if len(pending) == MAX_CHUNK or not more:
|
||||
length = _native.chunk_length(pending)
|
||||
yield pending[:length]
|
||||
pending = pending[length:]
|
||||
|
||||
|
||||
def create(source, repository, *, label="", max_total_size=1 << 40):
|
||||
"""Commit a directory version and return its address and measured growth.
|
||||
|
||||
All files are read and hashed, including unchanged files. Reused chunks are
|
||||
decoded and checked. A failed run may leave unreferenced valid chunks, but
|
||||
never publishes an incomplete snapshot. Concurrent writers are refused.
|
||||
"""
|
||||
_integer(max_total_size, 0, (1 << 63) - 1)
|
||||
if not isinstance(label, str) or len(label.encode("utf-8")) > 1024:
|
||||
raise ValueError("snapshot label must be at most 1024 UTF-8 bytes")
|
||||
source = Path(source).absolute()
|
||||
_directory(source)
|
||||
dest = Path(repository).resolve()
|
||||
if (dest == source.resolve() or source.resolve() in dest.parents or
|
||||
dest in source.resolve().parents):
|
||||
raise ValueError("repository and source must be outside each other")
|
||||
root = _repository(repository, create=True)
|
||||
with _lock(root):
|
||||
entries, keys = [], set()
|
||||
total = added = new_chunks = refs = 0
|
||||
directories = []
|
||||
def visit(directory, entry=None):
|
||||
nonlocal total, added, new_chunks, refs
|
||||
initial = directory.lstat()
|
||||
if not stat.S_ISDIR(initial.st_mode):
|
||||
raise ValueError(f"expected a real directory: {directory}")
|
||||
if entry is not None:
|
||||
entry.update(_metadata(directory, "directory", initial))
|
||||
directories.append((directory, initial))
|
||||
with os.scandir(directory) as scan:
|
||||
children = sorted(scan, key=lambda e: e.name)
|
||||
for child in children:
|
||||
path = Path(child.path)
|
||||
relative = path.relative_to(source).as_posix()
|
||||
_path(relative)
|
||||
key = _key(relative)
|
||||
if key in keys:
|
||||
raise ValueError(f"case or Unicode path collision: {relative}")
|
||||
keys.add(key)
|
||||
if len(keys) > MAX_ENTRIES:
|
||||
raise ValueError("snapshot entry limit exceeded")
|
||||
mode = path.lstat().st_mode
|
||||
if stat.S_ISDIR(mode):
|
||||
entry = dict(path=relative)
|
||||
entries.append(entry)
|
||||
visit(path, entry)
|
||||
elif stat.S_ISREG(mode):
|
||||
entry = dict(path=relative, **_metadata(path, "file"), chunks=[])
|
||||
sha, size = hashlib.sha256(), 0
|
||||
with _regular(path) as stream:
|
||||
before = os.fstat(stream.fileno())
|
||||
for data in _chunks(stream):
|
||||
total += len(data)
|
||||
if total > max_total_size:
|
||||
raise ValueError("snapshot total size limit exceeded")
|
||||
refs += 1
|
||||
if refs > MAX_REFS:
|
||||
raise ValueError("snapshot chunk-reference limit exceeded")
|
||||
digest, growth = _store(root, data)
|
||||
added += growth
|
||||
new_chunks += bool(growth)
|
||||
entry["chunks"].append([digest, len(data)])
|
||||
sha.update(data)
|
||||
size += len(data)
|
||||
after = os.fstat(stream.fileno())
|
||||
current = path.lstat()
|
||||
signature = lambda s: (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
|
||||
if signature(before) != signature(after) or signature(before) != signature(current) or size != before.st_size:
|
||||
raise ValueError(f"source changed during snapshot: {relative}")
|
||||
entry.update(size=size, sha256=sha.hexdigest(), mtime_ns=before.st_mtime_ns,
|
||||
mode=stat.S_IMODE(before.st_mode) & 0o777)
|
||||
entries.append(entry)
|
||||
else:
|
||||
raise ValueError(f"symlink or special file is unsupported: {relative}")
|
||||
visit(source)
|
||||
for directory, before in directories:
|
||||
after = directory.lstat()
|
||||
if (before.st_dev, before.st_ino, before.st_mtime_ns, before.st_ctime_ns) != (after.st_dev, after.st_ino, after.st_mtime_ns, after.st_ctime_ns):
|
||||
raise ValueError(f"source directory changed during snapshot: {directory}")
|
||||
manifest = {"format": FORMAT, "created": datetime.now(timezone.utc).isoformat(),
|
||||
"label": label, "root": _metadata(source, "directory", directories[0][1]),
|
||||
"entries": entries, "total_size": total}
|
||||
data = _json(manifest)
|
||||
if len(data) > MAX_MANIFEST:
|
||||
raise ValueError("snapshot manifest exceeds 64 MiB")
|
||||
address = _hash(data)
|
||||
published = _save(root / "snapshots" / (address + ".json"), data)
|
||||
return {"snapshot": address, "files": sum(e["type"] == "file" for e in entries),
|
||||
"total_size": total, "new_chunks": new_chunks, "new_chunk_bytes": added,
|
||||
"manifest_bytes": len(data), "new_manifest_bytes": len(data) if published else 0,
|
||||
"growth_bytes": added + (len(data) if published else 0)}
|
||||
|
||||
|
||||
def _manifest(root, address, max_total_size):
|
||||
_digest(address)
|
||||
data = _read(root / "snapshots" / (address + ".json"), MAX_MANIFEST)
|
||||
if _hash(data) != address:
|
||||
raise ValueError("manifest SHA-256 mismatch")
|
||||
m = json.loads(data)
|
||||
if not isinstance(m, dict) or m.get("format") != FORMAT:
|
||||
raise ValueError("unsupported snapshot manifest")
|
||||
if not isinstance(m.get("created"), str) or not isinstance(m.get("label"), str):
|
||||
raise ValueError("invalid snapshot description")
|
||||
entries = m.get("entries")
|
||||
if not isinstance(entries, list) or len(entries) > MAX_ENTRIES:
|
||||
raise ValueError("invalid snapshot entries")
|
||||
paths, total, refs = {"": "directory"}, 0, 0
|
||||
exact_paths = {""}
|
||||
def metadata(e):
|
||||
if not isinstance(e, dict) or e.get("type") not in ("file", "directory"):
|
||||
raise ValueError("invalid snapshot entry")
|
||||
_integer(e.get("mode"), 0, 0o777)
|
||||
_integer(e.get("mtime_ns"), -(1 << 63), (1 << 63) - 1)
|
||||
metadata(m.get("root"))
|
||||
if m["root"]["type"] != "directory":
|
||||
raise ValueError("invalid snapshot root")
|
||||
for e in entries:
|
||||
metadata(e)
|
||||
parts = _path(e.get("path"))
|
||||
key = _key(e["path"])
|
||||
parent = _key("/".join(parts[:-1]))
|
||||
if key in paths or paths.get(parent) != "directory" or "/".join(parts[:-1]) not in exact_paths:
|
||||
raise ValueError("duplicate path or missing/invalid parent")
|
||||
paths[key] = e["type"]
|
||||
exact_paths.add(e["path"])
|
||||
if e["type"] == "file":
|
||||
_integer(e.get("size"), 0, max_total_size)
|
||||
_digest(e.get("sha256"))
|
||||
chunks = e.get("chunks")
|
||||
if not isinstance(chunks, list):
|
||||
raise ValueError("invalid chunk list")
|
||||
refs += len(chunks)
|
||||
if refs > MAX_REFS:
|
||||
raise ValueError("snapshot chunk-reference limit exceeded")
|
||||
size = 0
|
||||
for c in chunks:
|
||||
if not isinstance(c, list) or len(c) != 2:
|
||||
raise ValueError("invalid chunk reference")
|
||||
_digest(c[0])
|
||||
size += _integer(c[1], 1, MAX_CHUNK)
|
||||
if size != e["size"]:
|
||||
raise ValueError("file and chunk sizes differ")
|
||||
total += size
|
||||
if total > max_total_size:
|
||||
raise ValueError("snapshot total size limit exceeded")
|
||||
if total != _integer(m.get("total_size"), 0, max_total_size):
|
||||
raise ValueError("invalid snapshot total size")
|
||||
return m
|
||||
|
||||
|
||||
def _files(root, manifest, destination=None):
|
||||
checked = 0
|
||||
for e in manifest["entries"]:
|
||||
if e["type"] == "directory":
|
||||
if destination is not None:
|
||||
(destination / e["path"]).mkdir()
|
||||
continue
|
||||
sha = hashlib.sha256()
|
||||
output = None
|
||||
try:
|
||||
if destination is not None:
|
||||
output = (destination / e["path"]).open("xb")
|
||||
for digest, size in e["chunks"]:
|
||||
data = _chunk(root, digest, size)
|
||||
sha.update(data)
|
||||
if output is not None:
|
||||
output.write(data)
|
||||
if sha.hexdigest() != e["sha256"]:
|
||||
raise ValueError(f"file SHA-256 mismatch: {e['path']}")
|
||||
if output is not None:
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
checked += 1
|
||||
finally:
|
||||
if output is not None:
|
||||
output.close()
|
||||
return checked
|
||||
|
||||
|
||||
def verify(repository, address, *, max_total_size=1 << 40):
|
||||
"""Verify one manifest, all referenced chunks, and complete file hashes."""
|
||||
_integer(max_total_size, 0, (1 << 63) - 1)
|
||||
root = _repository(repository)
|
||||
manifest = _manifest(root, address, max_total_size)
|
||||
return {"snapshot": address, "verified_files": _files(root, manifest),
|
||||
"total_size": manifest["total_size"]}
|
||||
|
||||
|
||||
def restore(repository, address, destination, *, max_total_size=1 << 40):
|
||||
"""Verify and restore into a NEW directory; failures publish no destination."""
|
||||
_integer(max_total_size, 0, (1 << 63) - 1)
|
||||
root = _repository(repository)
|
||||
manifest = _manifest(root, address, max_total_size)
|
||||
destination = Path(destination).absolute()
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise FileExistsError(destination)
|
||||
_directory(destination.parent)
|
||||
if root.resolve() == destination.resolve() or root.resolve() in destination.resolve().parents:
|
||||
raise ValueError("restore destination must be outside the repository")
|
||||
stage = Path(tempfile.mkdtemp(prefix=".uc2-restore-", dir=destination.parent))
|
||||
reserved = False
|
||||
try:
|
||||
checked = _files(root, manifest, stage)
|
||||
# Children first: readonly directories must not prevent restoration.
|
||||
for e in reversed(manifest["entries"]):
|
||||
path = stage / e["path"]
|
||||
os.utime(path, ns=(e["mtime_ns"], e["mtime_ns"]))
|
||||
os.chmod(path, e["mode"])
|
||||
os.utime(stage, ns=(manifest["root"]["mtime_ns"],) * 2)
|
||||
os.chmod(stage, manifest["root"]["mode"])
|
||||
if os.name == "posix":
|
||||
# Reserve the name exclusively; parents are trusted during restore.
|
||||
destination.mkdir()
|
||||
reserved = True
|
||||
stage.rename(destination)
|
||||
reserved = False
|
||||
_sync_directory(destination.parent)
|
||||
return {"snapshot": address, "restored_files": checked,
|
||||
"total_size": manifest["total_size"]}
|
||||
finally:
|
||||
if reserved and stage.exists():
|
||||
destination.rmdir()
|
||||
if stage.exists():
|
||||
# A metadata failure may have made directories readonly.
|
||||
os.chmod(stage, 0o700)
|
||||
for directory, dirs, files in os.walk(stage):
|
||||
for name in dirs:
|
||||
os.chmod(Path(directory) / name, 0o700)
|
||||
for name in files:
|
||||
os.chmod(Path(directory) / name, 0o600)
|
||||
shutil.rmtree(stage)
|
||||
|
||||
|
||||
def snapshots(repository):
|
||||
"""List committed snapshot IDs with validated manifests."""
|
||||
root = _repository(repository)
|
||||
result = []
|
||||
for path in sorted((root / "snapshots").glob("*.json")):
|
||||
m = _manifest(root, path.stem, (1 << 63) - 1)
|
||||
result.append({"snapshot": path.stem, "created": m["created"],
|
||||
"label": m["label"], "total_size": m["total_size"]})
|
||||
return result
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
new = commands.add_parser("create", help="snapshot a directory")
|
||||
new.add_argument("source")
|
||||
new.add_argument("repository")
|
||||
new.add_argument("--label", default="")
|
||||
for name in ("list", "verify", "restore"):
|
||||
command = commands.add_parser(name)
|
||||
command.add_argument("repository")
|
||||
if name != "list":
|
||||
command.add_argument("snapshot")
|
||||
if name == "restore":
|
||||
command.add_argument("destination")
|
||||
if name != "list":
|
||||
command.add_argument("--max-total-size", type=int, default=1 << 40)
|
||||
new.add_argument("--max-total-size", type=int, default=1 << 40)
|
||||
args = vars(parser.parse_args(argv))
|
||||
command = args.pop("command")
|
||||
if "snapshot" in args:
|
||||
args["address"] = args.pop("snapshot")
|
||||
try:
|
||||
result = {"create": create, "list": snapshots, "verify": verify,
|
||||
"restore": restore}[command](**args)
|
||||
except (OSError, ValueError, TypeError, KeyError) as error:
|
||||
parser.exit(1, f"uc2-snapshot: {error}\n")
|
||||
print(json.dumps(result, ensure_ascii=True, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user