Files
uc2/tests/scripts/benchmark_snapshot_scale.py
Eremey Valetov 8d57501c7b
Distributions / Wheels and CLI (windows-x86_64) (push) Canceled after 0s
Distributions / Wheels and CLI (macos-arm64) (push) Canceled after 0s
Distributions / Wheels and CLI (macos-x86_64) (push) Canceled after 0s
Distributions / Wheels and CLI (linux-x86_64) (push) Canceled after 0s
Distributions / Source distribution and metadata (push) Canceled after 0s
Build / Linux (push) Canceled after 0s
Build / Windows (MSVC) (push) Canceled after 0s
Build / macOS (push) Canceled after 0s
Build / libarchive plugin (push) Canceled after 0s
Build / Optional LGPL reader and libarchive proposal (push) Canceled after 0s
Build / Total Commander WCX (64-bit) (push) Canceled after 0s
Build / DOS (DJGPP) (push) Canceled after 0s
Build / Fuzz (libFuzzer + ASan) (push) Canceled after 0s
Integration releases / Native Total Commander (Win32) (push) Canceled after 0s
Integration releases / Native Total Commander (x64) (push) Canceled after 0s
Integration releases / packages (push) Canceled after 0s
Docs / build (push) Canceled after 0s
Docs / deploy (push) Canceled after 0s
Release alpha12 with indexed CLI metadata and scale recovery validation
2026-11-14 07:42:51 -05:00

201 lines
8.8 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later OR BSD-2-Clause
"""Opt-in large snapshot create/restore/maintenance measurement.
Default corpus: 1 GiB unique seeded random bytes, a 2.25 GiB file repeating
256 KiB of seeded random bytes, an empty directory and a small text file.
This is a 3.25 GiB logical collection, not 3.25 GiB of unique stored data.
The work directory and restored data are retained for inspection.
"""
import argparse
import gzip
import hashlib
import json
import os
from pathlib import Path
import random
import shutil
import sqlite3
import subprocess
import sys
import time
def digest(path):
sha = hashlib.sha256()
with path.open('rb') as source:
for data in iter(lambda: source.read(1 << 20), b''):
sha.update(data)
return sha.hexdigest()
def inventory(root):
result = {}
for path in [root, *sorted(root.rglob('*'))]:
info = path.stat()
row = {'type': 'directory' if path.is_dir() else 'file',
'mtime_ns': info.st_mtime_ns, 'mode': info.st_mode & 0o777}
if path.is_file():
row.update(sha256=digest(path), size=info.st_size)
result[path.relative_to(root).as_posix()] = row
return result
def stored(root):
size = allocated = files = 0
for path in root.rglob('*'):
if path.is_file():
info = path.stat()
size += info.st_size
allocated += getattr(info, 'st_blocks', 0) * 512
files += 1
return {'bytes': size, 'allocated_file_bytes': allocated, 'files': files}
def worker(request):
if sys.platform == 'linux':
import resource
resource.setrlimit(resource.RLIMIT_AS, (2 << 30, 2 << 30))
from uc2 import snapshot
operation = request.pop('operation')
started = time.monotonic()
function = getattr(snapshot, operation)
args = request.pop('args')
try:
result = function(*args, **request)
except (ValueError, OSError, sqlite3.Error) as error:
result = {'error': type(error).__name__, 'message': str(error)}
peak = None
if sys.platform == 'linux':
# ru_maxrss can retain the launcher's pre-exec high-water mark.
for line in Path('/proc/self/status').read_text().splitlines():
if line.startswith('VmHWM:'):
peak = int(line.split()[1]) * 1024
print(json.dumps({'operation': operation, 'seconds': time.monotonic() - started,
'peak_rss_bytes': peak, 'result': result}))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--work', type=Path)
parser.add_argument('--unique-mib', type=int, default=1024)
parser.add_argument('--repeat-mib', type=int, default=2304)
parser.add_argument('--worker')
args = parser.parse_args()
if args.worker:
worker(json.loads(args.worker))
return
if args.work is None or min(args.unique_mib, args.repeat_mib) <= 0:
parser.error('supply a new --work directory and positive corpus sizes')
root = args.work.resolve()
root.mkdir()
source = root / 'source'
source.mkdir()
(source / 'empty').mkdir()
(source / 'note.txt').write_bytes(b'first version\n')
rng = random.Random(20261114)
with (source / 'unique.bin').open('wb') as output:
for _ in range(args.unique_mib):
output.write(rng.randbytes(1 << 20))
block = random.Random(14112026).randbytes(256 << 10)
with (source / 'repeated.bin').open('wb') as output:
for _ in range(args.repeat_mib * 4):
output.write(block)
expected_first = inventory(source)
report = {'corpus': {'unique_mib': args.unique_mib,
'repeated_file_mib': args.repeat_mib,
'repeated_block_bytes': len(block), 'seed': 20261114},
'phases': [], 'restores': [], 'complete': False,
'notes': 'One Linux run. Each operation uses a fresh process with a 2 GiB address-space cap. VmHWM measures resident memory in that process. Timings include verification performed by the API. Data is synthetic; repeats are disclosed.'}
repo = root / 'repository'
def save():
(root / 'results.json').write_text(json.dumps(report, indent=2) + '\n')
def run(phase_name, operation, *positional, fail=False, **kwargs):
request = dict(operation=operation, args=list(map(str, positional)), **kwargs)
result = subprocess.run([sys.executable, str(Path(__file__).resolve()),
'--worker', json.dumps(request)],
capture_output=True, text=True, timeout=7200)
if result.returncode:
raise RuntimeError(f'{phase_name}: {result.returncode}: {result.stderr[-2000:]}')
row = json.loads(result.stdout)
row['label'] = phase_name
report['phases'].append(row)
save()
print(json.dumps(row), flush=True)
assert ('error' in row['result']) == fail, row
return row['result']
first = run('create loose v1', 'create', source, repo, label='large first')
report['first'] = first
report['loose_storage'] = stored(repo)
run('verify loose v1', 'verify', repo, first['snapshot'])
out = root / 'restore-v1'
run('restore loose v1', 'restore', repo, first['snapshot'], out)
assert inventory(out) == expected_first
assert (out / 'empty').is_dir()
report['restores'].append({'format': 1, 'bytes_paths_mtime_mode_equal': True})
run('compact v1 to v2', 'compact', repo)
report['packed_storage'] = stored(repo)
with (source / 'unique.bin').open('r+b') as output:
output.seek(min(123 << 20, (args.unique_mib << 20) // 2))
output.write(random.Random(99).randbytes(1 << 20))
with (source / 'repeated.bin').open('ab') as output:
output.write(b'changed after the large repeated prefix\n')
(source / 'note.txt').rename(source / 'renamed.txt')
(source / 'added.txt').write_bytes(b'second version\n')
expected_second = inventory(source)
second = run('create edited packed v2', 'create', source, repo, label='large second')
report['second'] = second
out = root / 'restore-v2'
run('restore edited packed v2', 'restore', repo, second['snapshot'], out)
assert inventory(out) == expected_second
assert (out / 'empty').is_dir()
report['restores'].append({'format': 2, 'bytes_paths_mtime_mode_equal': True})
replica = root / 'replica'
shutil.copytree(repo, replica)
with sqlite3.connect(repo / 'objects.sqlite3') as db:
def first_repeated_chunk(snapshot_id):
data = db.execute('SELECT data FROM manifests WHERE digest=?', (snapshot_id,)).fetchone()[0]
manifest = json.loads(gzip.decompress(data))
entry = next(e for e in manifest['entries'] if e['path'] == 'repeated.bin')
return entry['chunks'][0][0]
address = first_repeated_chunk(second['snapshot'])
assert first_repeated_chunk(first['snapshot']) == address
db.execute('UPDATE chunks SET data=? WHERE digest=?', (b'damaged', address))
# The deliberately truncated archive belongs to both committed versions.
for label, address in [('first', first['snapshot']), ('second', second['snapshot'])]:
request = dict(operation='verify', args=[str(repo), address])
result = subprocess.run([sys.executable, str(Path(__file__).resolve()),
'--worker', json.dumps(request)],
capture_output=True, text=True, timeout=7200, check=True)
row = json.loads(result.stdout)
row['label'] = 'corruption detection ' + label
report['phases'].append(row)
assert row['result'] == {'error': 'BadArchiveError', 'message': 'Truncated'}, row
report['corruption_detected'] = True
repaired = run('repair from packed replica', 'repair', repo, replica)
assert repaired['repaired_chunks'] == 1
run('verify repaired first', 'verify', repo, first['snapshot'])
run('verify repaired latest', 'verify', repo, second['snapshot'])
run('prune old history', 'prune', repo, keep_last=1, keep=[second['snapshot']], apply=True)
run('vacuum packed repository', 'compact', repo)
out = root / 'restore-after-maintenance'
run('restore after repair prune and vacuum', 'restore', repo, second['snapshot'], out)
assert inventory(out) == expected_second
report['restores'].append({'format': 2, 'after_maintenance': True,
'bytes_paths_mtime_mode_equal': True})
report['retained'] = run('list retained history', 'snapshots', repo)
assert [r['snapshot'] for r in report['retained']] == [second['snapshot']]
report['final_storage'] = stored(repo)
report['complete'] = True
save()
if __name__ == '__main__':
if not __debug__:
raise RuntimeError('validation requires Python without -O or PYTHONOPTIMIZE')
main()