53 lines
2.3 KiB
Python
53 lines
2.3 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
"""Fetch pinned historical archives and check migration against recorded hashes."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
from urllib.request import urlopen
|
|
|
|
from uc2.preserve import preserve
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--cache', type=Path, help='optional directory for downloaded archives')
|
|
args = parser.parse_args()
|
|
corpus = json.loads((ROOT / 'tests/archives/historical.json').read_text())
|
|
with tempfile.TemporaryDirectory(prefix='uc2-historical-') as temporary:
|
|
scratch = Path(temporary)
|
|
cache = args.cache or scratch / 'sources'
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
inputs = []
|
|
for item in corpus['archives']:
|
|
source = cache / item['name']
|
|
data = source.read_bytes() if source.exists() else b''
|
|
if hashlib.sha256(data).hexdigest() != item['sha256']:
|
|
with urlopen(item['url'], timeout=30) as response:
|
|
data = response.read(item['size'] + 1)
|
|
if len(data) != item['size'] or hashlib.sha256(data).hexdigest() != item['sha256']:
|
|
raise ValueError(f"Source hash/size mismatch: {item['name']}")
|
|
source.write_bytes(data)
|
|
inputs.append(source)
|
|
for format in ('zip', 'tar'):
|
|
output = scratch / format
|
|
report = preserve(inputs, output, format=format)
|
|
if report['status'] != 'ok':
|
|
raise ValueError(f'{format} migration failed: {report}')
|
|
for item, record in zip(corpus['archives'], report['archives']):
|
|
manifest = json.loads((output / record['manifest']).read_text())
|
|
observed = [{key: entry[key] for key in item['expected_entries'][0]}
|
|
for entry in manifest['entries']]
|
|
if observed != item['expected_entries']:
|
|
raise ValueError(f"Decoded metadata/hash mismatch: {item['name']}")
|
|
print(f"{format}: {len(inputs)} historical archives, "
|
|
f"{sum(len(x['expected_entries']) for x in corpus['archives'])} entries verified")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|