Files

229 lines
10 KiB
Python

#!/usr/bin/env python3
"""CLI regressions for OTS suffix framing, wide offsets and benchmark input."""
import argparse
import hashlib
import os
from pathlib import Path
import shutil
import struct
import subprocess
import sys
import tempfile
MAGIC = b"UC2-OTS\0"
PROOF_LIMIT = 1 << 20
def trailer(prefix_len, proof):
return (MAGIC + struct.pack("<III", 1, prefix_len, len(proof)) + proof
+ struct.pack("<I", len(proof)) + MAGIC)
def proof_for(data, uri=b"https://example.com/digest"):
magic = b"\0OpenTimestamps\0\0Proof\0\xbf\x89\xe2\xe8\x84\xe8\x92\x94"
pending = b"\x83\xdf\xe3\x0d\x2e\xf9\x0c\x8e"
# Both lengths fit a one-byte canonical OTS varint.
payload = bytes([len(uri)]) + uri
return (magic + b"\x01\x08" + hashlib.sha256(data).digest()
+ b"\0" + pending + bytes([len(payload)]) + payload)
def sparse_file(path):
f = path.open("w+b")
if os.name == "nt":
# NTFS needs an explicit sparse-file flag before writing beyond EOF.
import ctypes
from ctypes import wintypes
import msvcrt
control = ctypes.WinDLL("kernel32", use_last_error=True).DeviceIoControl
control.argtypes = [wintypes.HANDLE, wintypes.DWORD, ctypes.c_void_p,
wintypes.DWORD, ctypes.c_void_p, wintypes.DWORD,
ctypes.POINTER(wintypes.DWORD), ctypes.c_void_p]
control.restype = wintypes.BOOL
returned = wintypes.DWORD()
if not control(msvcrt.get_osfhandle(f.fileno()), 0x900C4, None, 0,
None, 0, ctypes.byref(returned), None):
f.close()
raise OSError(ctypes.get_last_error(), "FSCTL_SET_SPARSE failed")
return f
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("uc2")
parser.add_argument("--no-memory-limit", action="store_true",
help="disable the Linux address-space cap for sanitizers")
args = parser.parse_args()
uc2 = str(Path(args.uc2).resolve())
if args.no_memory_limit or os.environ.get("ASAN_OPTIONS"):
print("Address-space cap disabled for sanitizers; sparse-offset checks remain enabled")
def run(*argv, ok=True, bounded=False, input=None, cwd=None):
preexec = None
if (bounded and sys.platform == "linux" and not args.no_memory_limit
and not os.environ.get("ASAN_OPTIONS")):
import resource
def preexec():
resource.setrlimit(resource.RLIMIT_AS, (192 << 20, 192 << 20))
result = subprocess.run([uc2, *map(str, argv)], input=input,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
timeout=30, preexec_fn=preexec, cwd=cwd)
assert (result.returncode == 0) == ok, (argv, result.returncode, result.stderr)
assert b": : " not in result.stderr, result.stderr
return result
with tempfile.TemporaryDirectory(prefix="uc2-cli-io-") as tmp:
work = Path(tmp)
source = work / "data.txt"
source.write_bytes(b"CLI I/O regression\n" * 8)
empty = work / "empty"
empty.write_bytes(b"")
empty_result = run("-B", empty)
assert b"0 bytes input" in empty_result.stdout
assert b"ERROR" not in empty_result.stdout
expected_size = f"{source.stat().st_size} bytes input".encode()
assert expected_size in run("-B", empty, source).stdout
run("-B", work, ok=False)
# Long-option spellings after -- are input filenames.
for index, name in enumerate(("--ots-info", "--ots-extract", "--ots-attach=x")):
literal = work / name
literal.write_bytes(name.encode())
literal_archive = work / f"literal-{index}.uc2"
run("-qw", "--", literal_archive, name, cwd=work)
assert run("--ingest-restore", literal_archive).stdout == literal.read_bytes()
archive = work / "small.uc2"
run("-w", "-q", archive, source)
original = archive.read_bytes()
assert run("--ingest-restore", archive).stdout == source.read_bytes()
run("-t", "-q", archive)
run("--ots-extract", archive, work / "absent.ots", ok=False)
long_proof = proof_for(original)
short_proof = proof_for(original, b"x")
proof_path = work / "proof.ots"
proof_path.write_bytes(long_proof)
run("--ots-attach", proof_path, archive)
assert archive.read_bytes() == original + trailer(len(original), long_proof)
run("--ots-attach", proof_path, archive, ok=False)
assert archive.read_bytes() == original + trailer(len(original), long_proof)
proof_path.write_bytes(short_proof)
run("--ots-attach", proof_path, "-f", archive)
assert archive.read_bytes() == original + trailer(len(original), short_proof)
run("-t", "-q", archive)
assert b"leaf matches archive: yes" in run("--ots-info", archive).stdout
extracted = work / "extracted.ots"
run("--ots-extract", archive, extracted)
assert extracted.read_bytes() == short_proof
pending_tag = b"\x83\xdf\xe3\x0d\x2e\xf9\x0c\x8e"
for payload in (b"\xff" * 9 + b"\x01", b"", b"\x80", b"\x02x"):
bad_proof = (short_proof[:65] + b"\0" + pending_tag
+ bytes([len(payload)]) + payload)
malformed_uri = work / "malformed-uri.uc2"
malformed_uri.write_bytes(original + trailer(len(original), bad_proof))
result = run("--ots-info", malformed_uri, ok=False)
assert b"walker error" in result.stderr, result.stderr
assert len(result.stdout) < 4096
if Path("/dev/full").exists():
for argv in (("-p", archive), ("--ingest-restore", archive),
("-l", archive), ("-B", source), ("--ots-info", archive)):
with open("/dev/full", "wb") as full:
result = subprocess.run([uc2, *map(str, argv)], stdout=full,
stderr=subprocess.PIPE, timeout=30)
assert result.returncode != 0, (argv, result.stderr)
run("--ots-extract", archive, "/dev/full", ok=False)
stdbuf = shutil.which("stdbuf")
if stdbuf and not os.environ.get("ASAN_OPTIONS"):
# Unbuffered printf fails before the final fflush, which can
# itself return success. The stream error must still win.
with open("/dev/full", "wb") as full:
result = subprocess.run([stdbuf, "-o0", uc2, "-l", str(archive)],
stdout=full, stderr=subprocess.PIPE, timeout=30)
assert result.returncode != 0, result
assert b"stdout" in result.stderr, result.stderr
else:
print("Skipping stdbuf check: unavailable or incompatible with ASan preload order")
proof_path.write_bytes(b"x" * (PROOF_LIMIT + 1))
before = archive.read_bytes()
run("--ots-attach", proof_path, "-f", archive, ok=False, bounded=True)
assert archive.read_bytes() == before
# These exercise the trailer API's framing error classes independently
# of the .ots envelope, which extraction must preserve verbatim.
framed = trailer(3, b"proof")
cases = [
("short", MAGIC, "no OTS trailer"),
("truncated", struct.pack("<I", 5) + MAGIC, "(-1)"),
("large", struct.pack("<I", PROOF_LIMIT + 1) + MAGIC, "(-8)"),
("front-magic", b"abc" + b"X" + framed[1:], "(-4)"),
("version", b"abc" + framed[:8] + struct.pack("<I", 2) + framed[12:], "(-5)"),
("length-copy", b"abc" + framed[:16] + struct.pack("<I", 4) + framed[20:], "(-2)"),
("prefix-length", b"abcd" + framed, "(-3)"),
]
for name, data, message in cases:
bad = work / (name + ".uc2")
bad.write_bytes(data)
result = run("--ots-extract", bad, extracted, ok=False)
assert message.encode() in result.stderr, (name, result.stderr)
for proof in (b"", b"opaque proof bytes", b"x" * PROOF_LIMIT):
framed_path = work / "framed.uc2"
framed_path.write_bytes(b"abc" + trailer(3, proof))
run("--ots-extract", framed_path, extracted, bounded=True)
assert extracted.read_bytes() == proof
# Relocating the central directory beyond signed-long range keeps the
# decoded payload tiny while exercising real archive callback seeks.
cdir = struct.unpack_from("<I", original, 17)[0]
high_offset = (1 << 31) + 4096
new_size = high_offset + len(original) - cdir
head = bytearray(original[:cdir])
component_len = new_size - 13
struct.pack_into("<II", head, 4, component_len,
(component_len + 0x01B2C3D4) & 0xFFFFFFFF)
struct.pack_into("<I", head, 17, high_offset)
large = work / "wide.uc2"
try:
f = sparse_file(large)
except OSError as error:
print(f"SKIP wide-offset checks: sparse files unavailable: {error}")
else:
with f:
f.write(head)
f.seek(high_offset)
f.write(original[cdir:])
run("-l", "-q", large, bounded=True)
run("-t", "-q", large, bounded=True)
assert run("--ingest-restore", large, bounded=True).stdout == source.read_bytes()
with large.open("ab") as f:
f.write(trailer(new_size, b"large-offset proof"))
run("--ots-extract", large, extracted, bounded=True)
assert extracted.read_bytes() == b"large-offset proof"
# Oversized benchmark input is rejected before allocating its contents.
huge = work / "oversized"
try:
f = sparse_file(huge)
except OSError:
pass # Same sparse-file limitation reported above.
else:
with f:
f.seek((1 << 32) - 1)
f.write(trailer((1 << 32) - 1, b"proof beyond 4 GiB"))
run("--ots-extract", huge, extracted, bounded=True)
assert extracted.read_bytes() == b"proof beyond 4 GiB"
result = run("-B", huge, ok=False, bounded=True)
assert b"benchmark input exceeds" in result.stderr, result.stderr
print("CLI I/O checks passed")
if __name__ == "__main__":
main()