Harden migration failures and add historical interoperability checks
This commit is contained in:
1 parent
ebce333244
commit
3d2e3de854
10 files changed
+1187
-59
No files matched your search
@@ -182,7 +182,7 @@ def test_special_file_is_rejected_without_waiting(tmp_path):
|
||||
assert 'regular file' in manifest.read_text()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('failure', ['spool', 'member', 'manifest'])
|
||||
@pytest.mark.parametrize('failure', ['spool', 'member', 'manifest', 'export-hash', 'bad-zip', 'bad-tar'])
|
||||
def test_output_failure_aborts_without_publishing(tmp_path, monkeypatch, failure):
|
||||
import importlib
|
||||
module = importlib.import_module('uc2.preserve')
|
||||
@@ -192,8 +192,15 @@ def test_output_failure_aborts_without_publishing(tmp_path, monkeypatch, failure
|
||||
monkeypatch.setattr(module._HashedWriter, 'write', lambda self, data: module._output_call(fail))
|
||||
elif failure == 'member':
|
||||
monkeypatch.setattr(module, '_write_member', fail)
|
||||
else:
|
||||
elif failure == 'manifest':
|
||||
monkeypatch.setattr(Path, 'write_text', fail)
|
||||
elif failure == 'export-hash':
|
||||
monkeypatch.setattr(module, '_hash_path', fail)
|
||||
else:
|
||||
def bad(*args, **kwargs):
|
||||
error = zipfile.BadZipFile if failure == 'bad-zip' else tarfile.ReadError
|
||||
raise error('simulated destination corruption')
|
||||
monkeypatch.setattr(module, '_verify_export', bad)
|
||||
with pytest.raises(OSError, match='simulated destination'):
|
||||
preserve([ROOT / 'tests/archives/single.uc2'], tmp_path / 'out', format='zip')
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
@@ -212,3 +219,30 @@ def test_publish_does_not_replace_concurrently_created_directory(tmp_path, monke
|
||||
assert (tmp_path / 'out').is_dir()
|
||||
assert list((tmp_path / 'out').iterdir()) == []
|
||||
assert list(tmp_path.iterdir()) == [tmp_path / 'out']
|
||||
|
||||
|
||||
def test_walk_error_is_not_silently_skipped(tmp_path, monkeypatch):
|
||||
def broken_walk(path, *, followlinks, onerror):
|
||||
onerror(PermissionError('unreadable archive directory'))
|
||||
return iter(())
|
||||
monkeypatch.setattr(os, 'walk', broken_walk)
|
||||
with pytest.raises(PermissionError, match='unreadable'):
|
||||
preserve([tmp_path], tmp_path / 'out')
|
||||
assert not (tmp_path / 'out').exists()
|
||||
|
||||
|
||||
def test_source_change_invalidates_manifest_hashes(tmp_path, monkeypatch):
|
||||
path = tmp_path / 'changing.uc2'
|
||||
path.write_bytes((ROOT / 'tests/archives/single.uc2').read_bytes())
|
||||
copy = UC2File.copy_to
|
||||
def changing(self, *args, **kwargs):
|
||||
result = copy(self, *args, **kwargs)
|
||||
with path.open('ab') as source:
|
||||
source.write(b'changed')
|
||||
return result
|
||||
monkeypatch.setattr(UC2File, 'copy_to', changing)
|
||||
report = preserve([path], tmp_path / 'out', format='zip')
|
||||
manifest = json.loads((tmp_path / 'out' / report['archives'][0]['manifest']).read_text())
|
||||
assert report['status'] == 'partial'
|
||||
assert manifest['status'] == 'error' and manifest['export'] is None
|
||||
assert all(e['sha256'] is None and e['status'] == 'error' for e in manifest['entries'])
|
||||
+21
-6
@@ -16,6 +16,7 @@ import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
import zlib
|
||||
|
||||
from . import UC2File, ResourceLimitError, __version__
|
||||
|
||||
@@ -30,7 +31,7 @@ class _OutputError(OSError):
|
||||
def _output_call(function, *args, **kwargs):
|
||||
try:
|
||||
return function(*args, **kwargs)
|
||||
except (OSError, ValueError, RuntimeError) as error:
|
||||
except (OSError, ValueError, RuntimeError, EOFError, zipfile.BadZipFile, tarfile.TarError, zlib.error) as error:
|
||||
raise _OutputError(str(error)) from error
|
||||
|
||||
|
||||
@@ -50,6 +51,11 @@ def _hash(source):
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _hash_path(path):
|
||||
with path.open('rb') as source:
|
||||
return _hash(source)
|
||||
|
||||
|
||||
def _leaf(name):
|
||||
# A numbered parent makes versions and case-folding collisions independent.
|
||||
leaf = re.sub(r'[^A-Za-z0-9._-]', '_', name.rsplit('/', 1)[-1])[:100]
|
||||
@@ -203,9 +209,8 @@ def _one(path, output, format, limits):
|
||||
_output_call(container.close)
|
||||
container = None
|
||||
_output_call(_verify_export, export, manifest['entries'], format)
|
||||
with export.open('rb') as content:
|
||||
manifest['export'] = {'path': export.name, 'format': format,
|
||||
'sha256': _hash(content), 'verified': True}
|
||||
manifest['export'] = {'path': export.name, 'format': format,
|
||||
'sha256': _output_call(_hash_path, export), 'verified': True}
|
||||
source.seek(0)
|
||||
if _hash(source) != manifest['source']['sha256']:
|
||||
raise OSError('source archive changed during the operation')
|
||||
@@ -215,11 +220,18 @@ def _one(path, output, format, limits):
|
||||
raise
|
||||
except (OSError, ValueError, RuntimeError) as error:
|
||||
manifest['errors'].append(str(error))
|
||||
if container:
|
||||
_output_call(container.close)
|
||||
container = None
|
||||
if export and export.exists():
|
||||
export.unlink()
|
||||
_output_call(export.unlink)
|
||||
manifest['export'] = None
|
||||
for entry in manifest['entries']:
|
||||
entry['export_path'] = None
|
||||
if entry['kind'] == 'file':
|
||||
entry['status'] = 'error'
|
||||
entry['sha256'] = None
|
||||
entry['error'] = 'Archive-level failure; decoded result discarded'
|
||||
finally:
|
||||
if container:
|
||||
_output_call(container.close)
|
||||
@@ -228,13 +240,16 @@ def _one(path, output, format, limits):
|
||||
|
||||
|
||||
def _sources(inputs):
|
||||
def fail(error):
|
||||
raise error
|
||||
|
||||
result = []
|
||||
seen = set()
|
||||
for item in inputs:
|
||||
path = Path(item)
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
candidates = []
|
||||
for parent, directories, files in os.walk(path, followlinks=False):
|
||||
for parent, directories, files in os.walk(path, followlinks=False, onerror=fail):
|
||||
directories[:] = sorted(d for d in directories if not (Path(parent) / d).is_symlink())
|
||||
candidates.extend(Path(parent) / f for f in sorted(files)
|
||||
if f.lower().endswith('.uc2') and not (Path(parent) / f).is_symlink())
|
||||
|
||||
Reference in new issue
Block a user