Files
uc2/tests/scripts/check_historical.py

53 lines
2.3 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Fetch pinned historical archives and check migration against recorded hashes."""
import argparse
import hashlib
import json
from pathlib import Path
import tempfile
from urllib.request import urlopen
from uc2.preserve import preserve
ROOT = Path(__file__).resolve().parents[2]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cache', type=Path, help='optional directory for downloaded archives')
args = parser.parse_args()
corpus = json.loads((ROOT / 'tests/archives/historical.json').read_text())
with tempfile.TemporaryDirectory(prefix='uc2-historical-') as temporary:
scratch = Path(temporary)
cache = args.cache or scratch / 'sources'
cache.mkdir(parents=True, exist_ok=True)
inputs = []
for item in corpus['archives']:
source = cache / item['name']
data = source.read_bytes() if source.exists() else b''
if hashlib.sha256(data).hexdigest() != item['sha256']:
with urlopen(item['url'], timeout=30) as response:
data = response.read(item['size'] + 1)
if len(data) != item['size'] or hashlib.sha256(data).hexdigest() != item['sha256']:
raise ValueError(f"Source hash/size mismatch: {item['name']}")
source.write_bytes(data)
inputs.append(source)
for format in ('zip', 'tar'):
output = scratch / format
report = preserve(inputs, output, format=format)
if report['status'] != 'ok':
raise ValueError(f'{format} migration failed: {report}')
for item, record in zip(corpus['archives'], report['archives']):
manifest = json.loads((output / record['manifest']).read_text())
observed = [{key: entry[key] for key in item['expected_entries'][0]}
for entry in manifest['entries']]
if observed != item['expected_entries']:
raise ValueError(f"Decoded metadata/hash mismatch: {item['name']}")
print(f"{format}: {len(inputs)} historical archives, "
f"{sum(len(x['expected_entries']) for x in corpus['archives'])} entries verified")
if __name__ == '__main__':
main()