215 lines
9.6 KiB
Python
215 lines
9.6 KiB
Python
from datetime import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import zipfile
|
|
|
|
import pytest
|
|
|
|
from uc2 import UC2File, UC2Writer
|
|
from uc2.preserve import preserve
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
@pytest.fixture(params=[4, 8])
|
|
def collection(tmp_path, request):
|
|
path = tmp_path / 'collection.UC2'
|
|
contents = [('folder/notes.txt', b'first version\n'),
|
|
('folder/notes.txt', b'second version\n' * 5000),
|
|
('folder/NOTES.TXT', b'case collision'),
|
|
('CON', b'reserved DOS name'),
|
|
('empty', b''), ('long café.txt', bytes(range(256)) * 200)]
|
|
with UC2Writer(path, level=request.param) as writer:
|
|
writer.mkdir('empty-folder', date_time=datetime(1996, 7, 4, 12, 14, 16))
|
|
for name, data in contents:
|
|
writer.writestr(name, data, date_time=datetime(1993, 2, 3, 4, 5, 6), attributes=0x23)
|
|
return path, contents
|
|
|
|
|
|
@pytest.mark.parametrize('format', ['inventory', 'zip', 'tar'])
|
|
def test_migration_retains_versions_metadata_and_bytes(collection, tmp_path, format):
|
|
source, contents = collection
|
|
before = source.read_bytes()
|
|
output = tmp_path / format
|
|
report = preserve([source], output, format=format)
|
|
assert report['status'] == 'ok'
|
|
manifest = json.loads((output / report['archives'][0]['manifest']).read_text())
|
|
assert manifest['source']['sha256'] == hashlib.sha256(before).hexdigest()
|
|
files = [entry for entry in manifest['entries'] if entry['kind'] == 'file']
|
|
assert [entry['path'] for entry in files] == [name for name, data in contents]
|
|
versions = [entry['version'] for entry in files if entry['path'] == 'folder/notes.txt']
|
|
assert versions == [1, 0]
|
|
for entry, (_, data) in zip(files, contents):
|
|
assert entry['status'] == 'verified'
|
|
assert entry['sha256'] == hashlib.sha256(data).hexdigest()
|
|
assert entry['dos_attributes'] == 0x23
|
|
assert entry['timestamp_local'] == '1993-02-03T04:05:06'
|
|
assert entry['dos_time'] == ((1993 - 1980) << 25 | 2 << 21 | 3 << 16 | 4 << 11 | 5 << 5 | 3)
|
|
if format != 'inventory':
|
|
paths = [entry['export_path'] for entry in files]
|
|
assert len({p.casefold() for p in paths}) == len(paths)
|
|
assert all('..' not in p.split('/') and not p.startswith('/') for p in paths)
|
|
export = (output / report['archives'][0]['manifest']).parent / manifest['export']['path']
|
|
if format == 'zip':
|
|
with zipfile.ZipFile(export) as archive:
|
|
for entry, (_, data) in zip(files, contents):
|
|
assert archive.read(entry['export_path']) == data
|
|
assert archive.getinfo(entry['export_path']).date_time == (1993, 2, 3, 4, 5, 6)
|
|
else:
|
|
with tarfile.open(export) as archive:
|
|
for entry, (_, data) in zip(files, contents):
|
|
member = archive.getmember(entry['export_path'])
|
|
assert archive.extractfile(member).read() == data
|
|
assert member.pax_headers['UC2.path'] == entry['path']
|
|
assert member.mtime == 0
|
|
assert manifest['export']['verified']
|
|
assert source.read_bytes() == before
|
|
with pytest.raises(FileExistsError):
|
|
preserve([source], output, format=format)
|
|
assert source.read_bytes() == before
|
|
|
|
|
|
@pytest.mark.parametrize('format', ['zip', 'tar'])
|
|
def test_original_encoder_corpus(tmp_path, format):
|
|
report = preserve([ROOT / 'tests/archives'], tmp_path / format, format=format)
|
|
assert report['status'] == 'ok'
|
|
assert len(report['archives']) == 5
|
|
for record in report['archives']:
|
|
manifest = json.loads((tmp_path / format / record['manifest']).read_text())
|
|
for entry in manifest['entries']:
|
|
if entry['kind'] == 'file':
|
|
expected = (ROOT / 'tests/corpus' / entry['path'].lower()).read_bytes()
|
|
assert entry['sha256'] == hashlib.sha256(expected).hexdigest()
|
|
|
|
|
|
def test_bad_source_does_not_stop_batch(tmp_path):
|
|
bad = tmp_path / 'bad.uc2'
|
|
bad.write_bytes(b'not UC2')
|
|
good = ROOT / 'tests/archives/single.uc2'
|
|
result = preserve([bad, good], tmp_path / 'out', format='zip')
|
|
assert result['status'] == 'partial'
|
|
assert [r['status'] for r in result['archives']] == ['error', 'ok']
|
|
broken = json.loads((tmp_path / 'out' / result['archives'][0]['manifest']).read_text())
|
|
assert broken['errors'] and broken['export'] is None
|
|
|
|
|
|
def test_corruption_is_recorded_without_partial_export(tmp_path):
|
|
path = tmp_path / 'bad-payload.uc2'
|
|
with UC2Writer(path) as writer:
|
|
writer.writestr('payload', bytes(range(256)) * 100)
|
|
data = bytearray(path.read_bytes())
|
|
directory = int.from_bytes(data[17:21], 'little')
|
|
data[directory - 16] ^= 255
|
|
path.write_bytes(data)
|
|
report = preserve([path], tmp_path / 'out', format='zip')
|
|
assert report['status'] == 'partial'
|
|
manifest_path = tmp_path / 'out' / report['archives'][0]['manifest']
|
|
manifest = json.loads(manifest_path.read_text())
|
|
assert manifest['entries'][0]['status'] == 'error'
|
|
assert manifest['entries'][0]['sha256'] is None
|
|
with zipfile.ZipFile(manifest_path.parent / 'content.zip') as archive:
|
|
assert archive.namelist() == []
|
|
|
|
|
|
@pytest.mark.parametrize('limit', ['max_archive_size', 'max_entries', 'max_decoder_memory', 'max_extract_size', 'max_total_size'])
|
|
def test_limits_record_failure(collection, tmp_path, limit):
|
|
source, _ = collection
|
|
report = preserve([source], tmp_path / limit, **{limit: 1})
|
|
assert report['status'] == 'partial'
|
|
manifest = json.loads((tmp_path / limit / report['archives'][0]['manifest']).read_text())
|
|
if limit in ('max_extract_size', 'max_total_size'):
|
|
assert manifest['status'] == 'partial'
|
|
assert any(e['status'] == 'error' for e in manifest['entries'])
|
|
assert next(e for e in manifest['entries'] if e['path'] == 'empty')['status'] == 'verified'
|
|
else:
|
|
assert manifest['status'] == 'error' and manifest['errors']
|
|
|
|
|
|
def test_cli_exit_status_and_recursive_discovery(tmp_path):
|
|
directory = tmp_path / 'input' / 'nested'
|
|
directory.mkdir(parents=True)
|
|
(directory / 'old.UC2').write_bytes((ROOT / 'tests/archives/single.uc2').read_bytes())
|
|
out = tmp_path / 'out'
|
|
command = [sys.executable, '-m', 'uc2.preserve', str(directory.parent), '-o', str(out), '--format', 'tar']
|
|
result = subprocess.run(command, capture_output=True, text=True)
|
|
assert result.returncode == 0, result.stderr
|
|
assert json.loads((out / 'inventory.json').read_text())['status'] == 'ok'
|
|
assert subprocess.run(command, capture_output=True).returncode == 2
|
|
bad = tmp_path / 'bad.uc2'
|
|
bad.write_bytes(b'invalid')
|
|
result = subprocess.run([sys.executable, '-m', 'uc2.preserve', str(bad), '-o', str(tmp_path / 'bad-output')], capture_output=True)
|
|
assert result.returncode == 1
|
|
|
|
|
|
def test_raw_time_api(collection):
|
|
source, _ = collection
|
|
with UC2File(source) as reader:
|
|
assert all(info.dos_time for info in reader.infolist(all_versions=True) if not info.is_dir())
|
|
|
|
|
|
def test_corpus_manifest_hashes():
|
|
manifest = json.loads((ROOT / 'tests/archives/manifest.json').read_text())
|
|
for group, directory in [('archives', 'archives'), ('payloads', 'corpus')]:
|
|
for item in manifest[group]:
|
|
data = (ROOT / 'tests' / directory / item['path']).read_bytes()
|
|
assert len(data) == item['size']
|
|
assert hashlib.sha256(data).hexdigest() == item['sha256']
|
|
|
|
|
|
def test_symlink_does_not_hide_real_input(tmp_path):
|
|
source = ROOT / 'tests/archives/single.uc2'
|
|
link = tmp_path / 'link.uc2'
|
|
try:
|
|
link.symlink_to(source)
|
|
except OSError:
|
|
pytest.skip('symlink creation is unavailable')
|
|
report = preserve([link, source], tmp_path / 'out')
|
|
assert [r['status'] for r in report['archives']] == ['error', 'ok']
|
|
|
|
|
|
@pytest.mark.skipif(not hasattr(os, 'mkfifo'), reason='POSIX named pipe')
|
|
def test_special_file_is_rejected_without_waiting(tmp_path):
|
|
fifo = tmp_path / 'fifo.uc2'
|
|
os.mkfifo(fifo)
|
|
result = subprocess.run([sys.executable, '-m', 'uc2.preserve', str(fifo), '-o', str(tmp_path / 'out')], timeout=10, capture_output=True)
|
|
assert result.returncode == 1
|
|
manifest = next((tmp_path / 'out').glob('*/manifest.json'))
|
|
assert 'regular file' in manifest.read_text()
|
|
|
|
|
|
@pytest.mark.parametrize('failure', ['spool', 'member', 'manifest'])
|
|
def test_output_failure_aborts_without_publishing(tmp_path, monkeypatch, failure):
|
|
import importlib
|
|
module = importlib.import_module('uc2.preserve')
|
|
def fail(*args, **kwargs):
|
|
raise OSError('simulated destination write failure')
|
|
if failure == 'spool':
|
|
monkeypatch.setattr(module._HashedWriter, 'write', lambda self, data: module._output_call(fail))
|
|
elif failure == 'member':
|
|
monkeypatch.setattr(module, '_write_member', fail)
|
|
else:
|
|
monkeypatch.setattr(Path, 'write_text', fail)
|
|
with pytest.raises(OSError, match='simulated destination'):
|
|
preserve([ROOT / 'tests/archives/single.uc2'], tmp_path / 'out', format='zip')
|
|
assert list(tmp_path.iterdir()) == []
|
|
|
|
|
|
def test_publish_does_not_replace_concurrently_created_directory(tmp_path, monkeypatch):
|
|
import importlib
|
|
module = importlib.import_module('uc2.preserve')
|
|
original = module._publish
|
|
def raced(stage, destination):
|
|
destination.mkdir()
|
|
original(stage, destination)
|
|
monkeypatch.setattr(module, '_publish', raced)
|
|
with pytest.raises(OSError):
|
|
preserve([ROOT / 'tests/archives/single.uc2'], tmp_path / 'out')
|
|
assert (tmp_path / 'out').is_dir()
|
|
assert list((tmp_path / 'out').iterdir()) == []
|
|
assert list(tmp_path.iterdir()) == [tmp_path / 'out']
|