diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3f789f7..8d84279 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -181,8 +181,8 @@ jobs: cmake --build build --target uc2 -j - name: Build fuzz harnesses run: | - for h in fuzz_extract fuzz_roundtrip fuzz_sea; do - clang -fsanitize=fuzzer,address -O1 -g \ + for h in fuzz_extract fuzz_roundtrip fuzz_sea fuzz_chunks; do + clang -DNDEBUG -fsanitize=fuzzer,address -O1 -g \ -Ilib/include -Ilib/src -Ibuild/lib \ tests/fuzz/$h.c lib/src/*.c build/lib/super_data.S \ -lm -o $h @@ -196,6 +196,12 @@ jobs: cp tests/fuzz/seeds/* tests/fuzz/regressions/* corpus_extract/ || true ./fuzz_extract -max_len=65536 -timeout=20 -rss_limit_mb=2048 \ -max_total_time=$FUZZ_SECONDS corpus_extract/ + - name: Fuzz shared chunk recipes and payloads + run: | + mkdir -p corpus_chunks + cp tests/fuzz/seeds/chunk-* corpus_chunks/ + ./fuzz_chunks -max_len=65536 -timeout=20 -rss_limit_mb=2048 \ + -max_total_time=$FUZZ_SECONDS corpus_chunks/ - name: Fuzz standalone SEA decoder run: | mkdir -p corpus_sea diff --git a/README.md b/README.md index a161d66..8f7b933 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Nico de Vries (1992-1996). UC2 used shared compression dictionaries ("master blocks") and file versioning. UC2 v3 is under development in portable C99. The current alpha is -[3.0.0a10](https://pypi.org/project/uc2/3.0.0a10/). +[3.0.0a11](https://pypi.org/project/uc2/3.0.0a11/). Compression levels 2-5 use the original Huffman format; levels 6-9 require the new rANS reader. Original-format compatibility is covered by DOSBox-X fixtures, including Turbo/method 80 reads. Multi-volume @@ -18,7 +18,8 @@ nested directories, stored file versions, shared master blocks, output to stdout, tab-separated listings, and a quiet mode for scripts. - LZ77 with Huffman coding at levels 2-5, or rANS at levels 6-9. -- Content-defined chunking and identical-chunk deduplication through `--ingest`. +- Optional shared payload chunks across named files and versions with `--chunked`; + see the [compatibility and format guide](docs/chunk-sharing.md). The resulting archive is self-contained; `--ingest-restore` reconstructs the input stream. A tar input remains a byte stream of `part-N` entries. - Compression comparisons through `-B`, including the library's LZ4 method. diff --git a/cli/src/main.c b/cli/src/main.c index 27b74a3..587ea84 100644 --- a/cli/src/main.c +++ b/cli/src/main.c @@ -92,6 +92,7 @@ struct options { bool help:1; bool quiet:1; bool benchmark:1; + bool chunked:1; int ots_mode; int ingest_mode; char sep; @@ -1098,6 +1099,7 @@ static int create_archive(int nargs, char **args) { struct uc2_writer_options options = UC2_WRITER_OPTIONS_INIT; options.level = opt.level; + options.chunked = opt.chunked; uc2_writer *writer = uc2_writer_create(&options); if (!writer) errx(EXIT_FAILURE, "cannot initialize archive writer"); uc2_say(stderr, "UC2 compression level: %s\n", level_name(opt.level)); @@ -1135,6 +1137,9 @@ static int create_archive(int nargs, char **args) uc2_writer_free(writer); if (stats.masters) uc2_say(stderr, " master[2]: custom master dictionaries: %u\n", stats.masters); + if (opt.chunked) + uc2_say(stderr," shared chunks: %u unique, %u references, %llu payload bytes reused\n", + stats.unique_chunks,stats.chunk_references,(unsigned long long)stats.shared_bytes); uc2_say(stderr, "Created %s (%u files, %u dirs, %u masters, %u bytes)\n", opt.archive, stats.files, stats.directories, stats.masters, stats.archive_size); uc2_say(stderr, "Everything went OK\n"); @@ -1269,6 +1274,12 @@ static int extract_ingest_long_opts(int *argcp, char **argv, char **out_path) *out_path = NULL; for (int i = 1; i < argc; ) { const char *a = argv[i]; + if (!strcmp(a,"--")) break; + if (!strcmp(a,"--chunked")) { + opt.chunked=true; + for (int j=i; j+1= 32) return 0; p = __archive_read_ahead(a, 4, NULL); if (p == NULL) return (-1); if (p[0] == 0x55 && p[1] == 0x43 && p[2] == 0x32 && p[3] == 0x1A) - return (64); + return (32); return (0); } diff --git a/contrib/libarchive/build_bsdtar.py b/contrib/libarchive/build_bsdtar.py index b218eb7..23efcd1 100755 --- a/contrib/libarchive/build_bsdtar.py +++ b/contrib/libarchive/build_bsdtar.py @@ -16,6 +16,8 @@ work = args.build.resolve() work.mkdir(parents=True, exist_ok=False) source = work / 'source' shutil.copytree(args.source, source) +p = source/'libarchive/archive_read_private.h' +p.write_text(p.read_text().replace('formats[16]', 'formats[17]')) p = source/'libarchive/archive_read_support_format_all.c' s = p.read_text() anchor = '\tarchive_read_support_format_ar(a);' diff --git a/contrib/libarchive/prepare_upstream.py b/contrib/libarchive/prepare_upstream.py index ef6b615..bda80cb 100644 --- a/contrib/libarchive/prepare_upstream.py +++ b/contrib/libarchive/prepare_upstream.py @@ -49,6 +49,7 @@ replace("libarchive/archive.h", "#define\tARCHIVE_FORMAT_RAR_V5\t\t\t0x100000", replace("libarchive/archive.h", "__LA_DECL int archive_read_support_format_all(struct archive *);", "__LA_DECL int archive_read_support_format_uc2(struct archive *);\n" "__LA_DECL int archive_read_support_format_all(struct archive *);") +replace("libarchive/archive_read_private.h", "formats[16]", "formats[17]") replace("libarchive/archive_read_support_format_all.c", "\tarchive_read_support_format_ar(a);", "\tarchive_read_support_format_uc2(a);\n\tarchive_read_support_format_ar(a);") replace("libarchive/archive_read_support_format_by_code.c", "\tcase ARCHIVE_FORMAT_RAW:", @@ -75,7 +76,7 @@ notice = (repo / "contrib/libarchive/LICENSE.BSD-2-Clause").read_text() adapter = ("/* SPDX-License-Identifier: BSD-2-Clause */\n/*\n * " + notice.rstrip().replace("\n", "\n * ") + "\n */\n" + "\n".join(adapter.splitlines()[2:]) + "\n") -write("libarchive/archive_read_support_format_uc2.c", adapter) +write("libarchive/archive_read_support_format_uc2.c", "\n".join(line.rstrip() for line in adapter.splitlines()) + "\n") replace("libarchive/CMakeLists.txt", " archive_read_support_format_7zip.c", " archive_read_support_format_uc2.c\n archive_read_support_format_7zip.c") replace("Makefile.am", "\tlibarchive/archive_read_support_format_7zip.c \\", @@ -90,7 +91,7 @@ IF(ENABLE_UC2READ) IF(NOT UC2READ_INCLUDE_DIR OR NOT UC2READ_LIBRARY) MESSAGE(FATAL_ERROR "ENABLE_UC2READ requires the external libuc2read shared reader") ENDIF() - IF(UC2READ_LIBRARY MATCHES "[.]a$") + IF(UC2READ_LIBRARY MATCHES "[.]a$" AND NOT UC2READ_LIBRARY MATCHES "[.]dll[.]a$") MESSAGE(FATAL_ERROR "UC2 requires a shared libuc2read, not a static archive") ENDIF() INCLUDE(CheckCSourceCompiles) @@ -118,6 +119,11 @@ replace("configure.ac", "AC_ARG_WITH([lz4],", '''AC_ARG_WITH([uc2read], [], [with_uc2read=no]) if test "x$with_uc2read" = "xyes"; then PKG_CHECK_MODULES(UC2READ, [libuc2read >= 3.0.0]) + dnl The published dependency is shared-only. Refuse explicit static overrides. + for uc2read_flag in $UC2READ_LIBS; do + AS_CASE([$uc2read_flag], [*.dll.a], [], [*.a|-static|-Wl,-Bstatic], + [AC_MSG_ERROR([UC2 requires the shared libuc2read dependency])]) + done CPPFLAGS="${CPPFLAGS} ${UC2READ_CFLAGS}" LIBS="${LIBS} ${UC2READ_LIBS}" AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], @@ -154,14 +160,19 @@ test = '''/* SPDX-License-Identifier: BSD-2-Clause */ DEFINE_TEST(test_read_format_uc2) { struct archive *a = archive_read_new(); - struct archive_entry *e; - char data[2000]; int result = archive_read_support_format_uc2(a); #ifndef HAVE_LIBUC2READ assertEqualInt(ARCHIVE_WARN, result); assertEqualInt(ARCHIVE_OK, archive_read_free(a)); #else + struct archive_entry *e; + char data[2000]; assertEqualInt(ARCHIVE_OK, result); + /* Keep room for explicit raw fallback after registering every format. */ + struct archive *all = archive_read_new(); + assertEqualInt(ARCHIVE_OK, archive_read_support_format_all(all)); + assertEqualIntA(all, ARCHIVE_OK, archive_read_support_format_raw(all)); + assertEqualInt(ARCHIVE_OK, archive_read_free(all)); extract_reference_file("test_read_format_uc2_4.uc2"); assertEqualIntA(a, ARCHIVE_OK, archive_read_open_filename(a, "test_read_format_uc2_4.uc2", 7)); assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &e)); diff --git a/contrib/libarchive/upstream/libarchive-3.8.11-uc2-review.patch b/contrib/libarchive/upstream/libarchive-3.8.11-uc2-review.patch index 25fa0ba..e578f4b 100644 --- a/contrib/libarchive/upstream/libarchive-3.8.11-uc2-review.patch +++ b/contrib/libarchive/upstream/libarchive-3.8.11-uc2-review.patch @@ -13,7 +13,7 @@ + IF(NOT UC2READ_INCLUDE_DIR OR NOT UC2READ_LIBRARY) + MESSAGE(FATAL_ERROR "ENABLE_UC2READ requires the external libuc2read shared reader") + ENDIF() -+ IF(UC2READ_LIBRARY MATCHES "[.]a$") ++ IF(UC2READ_LIBRARY MATCHES "[.]a$" AND NOT UC2READ_LIBRARY MATCHES "[.]dll[.]a$") + MESSAGE(FATAL_ERROR "UC2 requires a shared libuc2read, not a static archive") + ENDIF() + INCLUDE(CheckCSourceCompiles) @@ -75,7 +75,7 @@ +#cmakedefine HAVE_LIBUC2READ 1 --- a/configure.ac +++ b/configure.ac -@@ -491,6 +491,20 @@ +@@ -491,6 +491,25 @@ AC_CHECK_LIB(charset,locale_charset) fi fi @@ -86,6 +86,11 @@ + [], [with_uc2read=no]) +if test "x$with_uc2read" = "xyes"; then + PKG_CHECK_MODULES(UC2READ, [libuc2read >= 3.0.0]) ++ dnl The published dependency is shared-only. Refuse explicit static overrides. ++ for uc2read_flag in $UC2READ_LIBS; do ++ AS_CASE([$uc2read_flag], [*.dll.a], [], [*.a|-static|-Wl,-Bstatic], ++ [AC_MSG_ERROR([UC2 requires the shared libuc2read dependency])]) ++ done + CPPFLAGS="${CPPFLAGS} ${UC2READ_CFLAGS}" + LIBS="${LIBS} ${UC2READ_LIBS}" + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], @@ -98,7 +103,7 @@ AC_ARG_WITH([lz4], --- /dev/null +++ b/doc/uc2-reader.md -@@ -0,0 +1,79 @@ +@@ -0,0 +1,82 @@ +# Proposed UC2 reader for libarchive + +Prepared for review against libarchive 3.8.11. No contribution has been sent @@ -126,7 +131,10 @@ +they never silently truncate output. Listing and skipping do not decode files. +The fixed adapter state (about 65 KiB), stdio and libarchive's own allocations +are outside the shared memory budget. The input and current decoded entry use -+disk space; the decoded entry spool is released between files. ++disk space; the decoded entry spool is released between files. Spools use ++libarchive's anonymous temporary-file helper, including its Windows temp-directory ++handling. These libarchive build/runtime checks were performed on Linux; native ++Windows execution of the optional adapter has not been validated. + +Override limits before reading with libarchive format options, for example +`bsdtar --options uc2:memory-limit=134217728,uc2:entry-limit=2147483648 -xf data.uc2`. @@ -153,7 +161,7 @@ +cmake --build /tmp/uc2-reader +cmake --install /tmp/uc2-reader +python3 contrib/libarchive/prepare_upstream.py \ -+ --source /path/to/libarchive-3.8.11 --uc2 /path/to/uc2 \ ++ --source /path/to/libarchive-3.8.11 --uc2 /path/to/uc2/build/cli/uc2 \ + --output /tmp/uc2-review +cmake -S /tmp/uc2-review/libarchive -B /tmp/uc2-review/build \ + -DCMAKE_PREFIX_PATH=/tmp/uc2-reader-prefix -DENABLE_UC2READ=ON \ @@ -207,6 +215,17 @@ __LA_DECL int archive_read_support_format_all(struct archive *); __LA_DECL int archive_read_support_format_ar(struct archive *); __LA_DECL int archive_read_support_format_by_code(struct archive *, int); +--- a/libarchive/archive_read_private.h ++++ b/libarchive/archive_read_private.h +@@ -208,7 +208,7 @@ + int (*cleanup)(struct archive_read *); + int (*format_capabilties)(struct archive_read *); + int (*has_encrypted_entries)(struct archive_read *); +- } formats[16]; ++ } formats[17]; + struct archive_format_descriptor *format; /* Active format. */ + + /* --- a/libarchive/archive_read_set_format.c +++ b/libarchive/archive_read_set_format.c @@ -78,6 +78,9 @@ @@ -246,19 +265,19 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * BSD 2-Clause License -+ * ++ * + * Copyright (c) 2026, Eremey Valetov + * All rights reserved. -+ * ++ * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: -+ * ++ * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. -+ * ++ * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -578,14 +597,14 @@ +{ + const unsigned char *p; + -+ (void)best_bid; ++ if (best_bid >= 32) return 0; + + p = __archive_read_ahead(a, 4, NULL); + if (p == NULL) + return (-1); + + if (p[0] == 0x55 && p[1] == 0x43 && p[2] == 0x32 && p[3] == 0x1A) -+ return (64); ++ return (32); + return (0); +} + @@ -949,7 +968,7 @@ test_read_format_lha_bugfix_0.c --- /dev/null +++ b/libarchive/test/test_read_format_uc2.c -@@ -0,0 +1,67 @@ +@@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Copyright (c) 2026 Eremey Valetov. */ +#include "test.h" @@ -957,14 +976,19 @@ +DEFINE_TEST(test_read_format_uc2) +{ + struct archive *a = archive_read_new(); -+ struct archive_entry *e; -+ char data[2000]; + int result = archive_read_support_format_uc2(a); +#ifndef HAVE_LIBUC2READ + assertEqualInt(ARCHIVE_WARN, result); + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); +#else ++ struct archive_entry *e; ++ char data[2000]; + assertEqualInt(ARCHIVE_OK, result); ++ /* Keep room for explicit raw fallback after registering every format. */ ++ struct archive *all = archive_read_new(); ++ assertEqualInt(ARCHIVE_OK, archive_read_support_format_all(all)); ++ assertEqualIntA(all, ARCHIVE_OK, archive_read_support_format_raw(all)); ++ assertEqualInt(ARCHIVE_OK, archive_read_free(all)); + extract_reference_file("test_read_format_uc2_4.uc2"); + assertEqualIntA(a, ARCHIVE_OK, archive_read_open_filename(a, "test_read_format_uc2_4.uc2", 7)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &e)); diff --git a/contrib/libarchive/upstream/review.json b/contrib/libarchive/upstream/review.json index d3f370f..3b3b9a3 100644 --- a/contrib/libarchive/upstream/review.json +++ b/contrib/libarchive/upstream/review.json @@ -4,7 +4,7 @@ "format_id": "0x110000 (proposed, not allocated)", "decoder": "optional shared libuc2read; existing LGPL-3.0-only sources", "adapter_license": "BSD-2-Clause option", - "patch_sha256": "cdfe13c2fc58cbd0aa25d52c835eaaaedf1f5709e6733f8532e3165d3e8e2851", + "patch_sha256": "4b406969a27b96d94089bed70eac59e5e92bfda09ac65f03a964e8bf9b8e4f77", "files": [ "CMakeLists.txt", "Makefile.am", @@ -13,6 +13,7 @@ "doc/uc2-reader.md", "libarchive/CMakeLists.txt", "libarchive/archive.h", + "libarchive/archive_read_private.h", "libarchive/archive_read_set_format.c", "libarchive/archive_read_support_format_all.c", "libarchive/archive_read_support_format_by_code.c", diff --git a/docs/benchmarks/2026-11-07-chunk-sharing.md b/docs/benchmarks/2026-11-07-chunk-sharing.md new file mode 100644 index 0000000..507a362 --- /dev/null +++ b/docs/benchmarks/2026-11-07-chunk-sharing.md @@ -0,0 +1,59 @@ +# Named-file chunk sharing — 2026-11-07 + +Linux x86-64 development build of a11, level 4, three trials per case. All +restores matched the source SHA-256 manifests. Timings below are medians; +file sizes were identical across trials. These are small-collection measurements, +not capacity or universal compression claims. + +## Synthetic collections + +Each case has eight 512 KiB revisions, except generated source text (3.62 MB +total). The fixed seed and edit rules are in `scripts/benchmark_collections.py`. +Binary versions have eight local edits per revision; assets reorder shared +blocks; simulation checkpoints change one block; independent inputs share no +content. Tar modes preserve the complete stream; named modes expose files. + +| Collection | Default UC2 bytes | Shared-chunk UC2 bytes | Change | 7-Zip bytes | tar+Zstd bytes | +| --- | ---: | ---: | ---: | ---: | ---: | +| binary versions | 3,763,693 | 1,556,525 | -58.6% | 528,060 | 527,814 | +| game assets | 4,204,367 | 2,216,932 | -47.3% | 791,208 | 853,609 | +| simulation | 3,818,737 | 689,405 | -81.9% | 558,992 | 558,175 | +| source versions | 282,315 | 68,771 | -75.6% | 11,869 | 16,182 | +| independent | 4,201,233 | 4,213,383 | +0.3% | 4,194,795 | 4,198,532 | + +The chunk mode reduced creation time on all five cases (0.21–0.40 seconds +versus 0.60–2.27 seconds). Restores took 0.09–0.16 seconds versus 0.05–0.15 +seconds. Zstandard and 7-Zip still produced smaller archives in these comparisons; +Zstandard was also faster. Full times, process peak RSS, tool versions and input +parameters are in [the raw results](chunk-sharing-2026-11-07/synthetic.json). + +## Edited public source files + +The twelve largest `archive_read_support_format_*.c` files in libarchive commit +`eb57ffefe20e98eb7ba05829ee1c23b90e589a71` each have an original and a copy with +a short inserted comment at the midpoint: 24 files, 2,237,336 input bytes. + +| Mode | Archive bytes | Creation seconds | Restore seconds | +| --- | ---: | ---: | ---: | +| Default dictionary mode | 378,945 | 0.627 | 0.073 | +| Shared chunks | 413,429 | 0.319 | 0.169 | + +Here shared chunks made the archive **9.1% larger**. Independent small-chunk +compression loses some of the full-file dictionary benefit on this text corpus. +The writer therefore keeps its existing default. Choose `--chunked` when its +reuse benefits your collection; further codec/chunk-boundary tuning remains open. +[Raw source hashes and measurements](chunk-sharing-2026-11-07/public-source.json). + +## Reproduce + +```sh +python3 scripts/benchmark_collections.py --uc2 /path/to/uc2 \ + --output /tmp/uc2-comparison --size-kib 512 --repeat 3 +git clone https://github.com/libarchive/libarchive /tmp/libarchive-source +git -C /tmp/libarchive-source checkout eb57ffefe20e98eb7ba05829ee1c23b90e589a71 +python3 tests/scripts/benchmark_chunk_sharing.py --source /tmp/libarchive-source \ + --uc2 /path/to/uc2 --output /tmp/uc2-source-comparison --repeat 3 +``` + +The scripts require new output directories and `/usr/bin/time`; the comparison +also needs 7-Zip and Zstandard. No private collection data was used. diff --git a/docs/benchmarks/chunk-sharing-2026-11-07/public-source.json b/docs/benchmarks/chunk-sharing-2026-11-07/public-source.json new file mode 100644 index 0000000..2766298 --- /dev/null +++ b/docs/benchmarks/chunk-sharing-2026-11-07/public-source.json @@ -0,0 +1,83 @@ +{ + "platform": "Linux-6.12.0-211.73.1.el10_2.x86_64-x86_64-with-glibc2.39", + "cli_sha256": "e0e02429da449e41609abac781625935f7a023b06b683a74286e3193351c950f", + "source_revision": "eb57ffefe20e98eb7ba05829ee1c23b90e589a71", + "source_hashes": { + "archive_read_support_format_zip.c": "e165b2a6b56317aed9b06070813caa444ad6eeca370f4f6aaf51f85edac5886e", + "archive_read_support_format_7zip.c": "3456c0e2d79c3a7fb2b611ca28617b11a146e4ae57d40523f6767fc52412c2e3", + "archive_read_support_format_rar5.c": "0a21cc37ef7ae0f83bbffdac1b08414380787ddf1cddae7b8539985e4b28e3ab", + "archive_read_support_format_tar.c": "30bfbc0916511bdab1d2803bb7ed305a6d3721dd06b08ed8035032a7fd3af773", + "archive_read_support_format_rar.c": "c9afc9af8b2f537f1ecaee4ce521208562b3c37aa103f7d87e8e656e2150ab88", + "archive_read_support_format_iso9660.c": "c8402502d58f205c48e1da5a0a031491e932fd605548e10a710c1b2a7e3002d7", + "archive_read_support_format_xar.c": "fdfcd9280bf21dc9c1798b2b6b1b2f3e166d142198948b21c56e3e2a4d598e50", + "archive_read_support_format_cab.c": "aa3ca37bdb96011ac438afbd9d8248ff73fa4fd9f68053b5d6fb3d21b4c70d05", + "archive_read_support_format_lha.c": "9e0bc4c644cef5159ad4e07ccc69b973fc370b57fcb647a33cf78611531abfa3", + "archive_read_support_format_mtree.c": "8292142956d282b27d2d725ec9cf5201efb1858cfaf226e813c85bd550836b0f", + "archive_read_support_format_cpio.c": "7b2103a005b8d68fdd25a317751929617d19788248eeaba6b75df9bb7a566f78", + "archive_read_support_format_warc.c": "b7f64302ce6ade6b40eb902094d3920f5d99257c025f494047668cc28e0c5a60" + }, + "input_bytes": 2237336, + "files": 24, + "results": [ + { + "trial": 0, + "chunked": false, + "archive_bytes": 378945, + "create_seconds": 0.627, + "restore_seconds": 0.075, + "peak_kib": 2584, + "restore_peak_kib": 2448, + "verified": true + }, + { + "trial": 0, + "chunked": true, + "archive_bytes": 413429, + "create_seconds": 0.271, + "restore_seconds": 0.169, + "peak_kib": 2652, + "restore_peak_kib": 1592, + "verified": true + }, + { + "trial": 1, + "chunked": false, + "archive_bytes": 378945, + "create_seconds": 0.73, + "restore_seconds": 0.073, + "peak_kib": 2564, + "restore_peak_kib": 2116, + "verified": true + }, + { + "trial": 1, + "chunked": true, + "archive_bytes": 413429, + "create_seconds": 0.469, + "restore_seconds": 0.12, + "peak_kib": 2692, + "restore_peak_kib": 1796, + "verified": true + }, + { + "trial": 2, + "chunked": false, + "archive_bytes": 378945, + "create_seconds": 0.627, + "restore_seconds": 0.066, + "peak_kib": 2564, + "restore_peak_kib": 2248, + "verified": true + }, + { + "trial": 2, + "chunked": true, + "archive_bytes": 413429, + "create_seconds": 0.319, + "restore_seconds": 0.169, + "peak_kib": 2468, + "restore_peak_kib": 1796, + "verified": true + } + ] +} diff --git a/docs/benchmarks/chunk-sharing-2026-11-07/synthetic.json b/docs/benchmarks/chunk-sharing-2026-11-07/synthetic.json new file mode 100644 index 0000000..a290052 --- /dev/null +++ b/docs/benchmarks/chunk-sharing-2026-11-07/synthetic.json @@ -0,0 +1,934 @@ +{ + "metadata": { + "date_utc": "2026-11-07T17:34:27Z", + "platform": "Linux-6.12.0-211.73.1.el10_2.x86_64-x86_64-with-glibc2.39", + "cpu": "", + "uc2_sha256": "09ed0e51ab1230e1372a989267814cb90a1cffb60c54615697c512314197a8eb", + "seed": 20261105, + "size_kib": 512, + "repeat": 3, + "corpus": "synthetic, eight revisions per collection; not real user data", + "rss_scope": "per process peak RSS in KiB; tar materialization excluded equally", + "versions": { + "7zip": [ + "", + "7-Zip (a) 26.04 (x64) : Copyright (c) 1999-2026 Igor Pavlov : 2026-10-05", + " 64-bit locale=C.UTF-8 Threads:128 OPEN_MAX:524288", + "", + "Usage: 7za [...] [...] [@listfile]" + ], + "zstd": [ + "*** Zstandard CLI (64-bit) v1.5.5, by Yann Collet ***" + ], + "uc2": [ + "UC2 3.0.0 (UltraCompressor II)", + "\"Fast, reliable and superior compression.\"", + "", + "uc2 [-afpqDT] [-d destination] archive.uc2 [files]...", + "uc2 -l [-aqT] archive.uc2 [files]..." + ] + } + }, + "results": [ + { + "collection": "binary_versions", + "method": "uc2-files", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 3763693, + "compress_seconds": 1.638977, + "restore_seconds": 0.092364, + "compress_peak_kib": 2640, + "restore_peak_kib": 1764, + "verified": true + }, + { + "collection": "binary_versions", + "method": "uc2-chunks", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 1556525, + "compress_seconds": 0.31763, + "restore_seconds": 0.122534, + "compress_peak_kib": 2772, + "restore_peak_kib": 1792, + "verified": true + }, + { + "collection": "binary_versions", + "method": "tar-uc2", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 1674379, + "compress_seconds": 0.179262, + "restore_seconds": 0.091696, + "compress_peak_kib": 2516, + "restore_peak_kib": 1764, + "verified": true + }, + { + "collection": "binary_versions", + "method": "7z", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 528060, + "compress_seconds": 1.826092, + "restore_seconds": 0.030589, + "compress_peak_kib": 50088, + "restore_peak_kib": 9548, + "verified": true + }, + { + "collection": "binary_versions", + "method": "tar-zstd", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 527814, + "compress_seconds": 0.023188, + "restore_seconds": 0.022813, + "compress_peak_kib": 8672, + "restore_peak_kib": 5732, + "verified": true + }, + { + "collection": "binary_versions", + "method": "uc2-files", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 3763693, + "compress_seconds": 2.128321, + "restore_seconds": 0.075146, + "compress_peak_kib": 2648, + "restore_peak_kib": 1940, + "verified": true + }, + { + "collection": "binary_versions", + "method": "uc2-chunks", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 1556525, + "compress_seconds": 0.355798, + "restore_seconds": 0.112066, + "compress_peak_kib": 2804, + "restore_peak_kib": 1764, + "verified": true + }, + { + "collection": "binary_versions", + "method": "tar-uc2", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 1674379, + "compress_seconds": 0.175767, + "restore_seconds": 0.075982, + "compress_peak_kib": 2676, + "restore_peak_kib": 1764, + "verified": true + }, + { + "collection": "binary_versions", + "method": "7z", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 528060, + "compress_seconds": 1.50318, + "restore_seconds": 0.024666, + "compress_peak_kib": 50332, + "restore_peak_kib": 9628, + "verified": true + }, + { + "collection": "binary_versions", + "method": "tar-zstd", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 527814, + "compress_seconds": 0.018186, + "restore_seconds": 0.017152, + "compress_peak_kib": 10860, + "restore_peak_kib": 5428, + "verified": true + }, + { + "collection": "binary_versions", + "method": "uc2-files", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 3763693, + "compress_seconds": 1.268843, + "restore_seconds": 0.079027, + "compress_peak_kib": 2744, + "restore_peak_kib": 1768, + "verified": true + }, + { + "collection": "binary_versions", + "method": "uc2-chunks", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 1556525, + "compress_seconds": 0.267796, + "restore_seconds": 0.096384, + "compress_peak_kib": 2612, + "restore_peak_kib": 1764, + "verified": true + }, + { + "collection": "binary_versions", + "method": "tar-uc2", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 1674379, + "compress_seconds": 0.162703, + "restore_seconds": 0.129786, + "compress_peak_kib": 2424, + "restore_peak_kib": 1804, + "verified": true + }, + { + "collection": "binary_versions", + "method": "7z", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 528060, + "compress_seconds": 1.712246, + "restore_seconds": 0.030431, + "compress_peak_kib": 50520, + "restore_peak_kib": 9552, + "verified": true + }, + { + "collection": "binary_versions", + "method": "tar-zstd", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 527814, + "compress_seconds": 0.025576, + "restore_seconds": 0.022903, + "compress_peak_kib": 9392, + "restore_peak_kib": 6196, + "verified": true + }, + { + "collection": "game_assets", + "method": "uc2-files", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 4204367, + "compress_seconds": 1.536573, + "restore_seconds": 0.214796, + "compress_peak_kib": 2576, + "restore_peak_kib": 2008, + "verified": true + }, + { + "collection": "game_assets", + "method": "uc2-chunks", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 2216932, + "compress_seconds": 0.460623, + "restore_seconds": 0.114157, + "compress_peak_kib": 2812, + "restore_peak_kib": 1696, + "verified": true + }, + { + "collection": "game_assets", + "method": "tar-uc2", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 2246281, + "compress_seconds": 0.218358, + "restore_seconds": 0.086955, + "compress_peak_kib": 2468, + "restore_peak_kib": 1800, + "verified": true + }, + { + "collection": "game_assets", + "method": "7z", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 791208, + "compress_seconds": 1.750522, + "restore_seconds": 0.0585, + "compress_peak_kib": 50184, + "restore_peak_kib": 9872, + "verified": true + }, + { + "collection": "game_assets", + "method": "tar-zstd", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 853609, + "compress_seconds": 0.018772, + "restore_seconds": 0.014618, + "compress_peak_kib": 12904, + "restore_peak_kib": 5460, + "verified": true + }, + { + "collection": "game_assets", + "method": "uc2-files", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 4204367, + "compress_seconds": 1.261467, + "restore_seconds": 0.071452, + "compress_peak_kib": 2588, + "restore_peak_kib": 2052, + "verified": true + }, + { + "collection": "game_assets", + "method": "uc2-chunks", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 2216932, + "compress_seconds": 0.260462, + "restore_seconds": 0.081622, + "compress_peak_kib": 2660, + "restore_peak_kib": 1808, + "verified": true + }, + { + "collection": "game_assets", + "method": "tar-uc2", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 2246281, + "compress_seconds": 0.384236, + "restore_seconds": 0.25758, + "compress_peak_kib": 2440, + "restore_peak_kib": 1696, + "verified": true + }, + { + "collection": "game_assets", + "method": "7z", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 791208, + "compress_seconds": 1.420875, + "restore_seconds": 0.053211, + "compress_peak_kib": 50532, + "restore_peak_kib": 9888, + "verified": true + }, + { + "collection": "game_assets", + "method": "tar-zstd", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 853609, + "compress_seconds": 0.023485, + "restore_seconds": 0.017585, + "compress_peak_kib": 12484, + "restore_peak_kib": 5496, + "verified": true + }, + { + "collection": "game_assets", + "method": "uc2-files", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 4204367, + "compress_seconds": 1.924347, + "restore_seconds": 0.082141, + "compress_peak_kib": 2648, + "restore_peak_kib": 2064, + "verified": true + }, + { + "collection": "game_assets", + "method": "uc2-chunks", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 2216932, + "compress_seconds": 0.294265, + "restore_seconds": 0.120801, + "compress_peak_kib": 2812, + "restore_peak_kib": 1772, + "verified": true + }, + { + "collection": "game_assets", + "method": "tar-uc2", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 2246281, + "compress_seconds": 0.239292, + "restore_seconds": 0.106254, + "compress_peak_kib": 2428, + "restore_peak_kib": 1772, + "verified": true + }, + { + "collection": "game_assets", + "method": "7z", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 791208, + "compress_seconds": 2.492793, + "restore_seconds": 0.081404, + "compress_peak_kib": 50488, + "restore_peak_kib": 9872, + "verified": true + }, + { + "collection": "game_assets", + "method": "tar-zstd", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 853609, + "compress_seconds": 0.029237, + "restore_seconds": 0.025801, + "compress_peak_kib": 12592, + "restore_peak_kib": 5756, + "verified": true + }, + { + "collection": "source_versions", + "method": "uc2-files", + "trial": 0, + "input_bytes": 3618616, + "archive_bytes": 282315, + "compress_seconds": 0.587387, + "restore_seconds": 0.044849, + "compress_peak_kib": 2724, + "restore_peak_kib": 1892, + "verified": true + }, + { + "collection": "source_versions", + "method": "uc2-chunks", + "trial": 0, + "input_bytes": 3618616, + "archive_bytes": 68771, + "compress_seconds": 0.209431, + "restore_seconds": 0.220468, + "compress_peak_kib": 2492, + "restore_peak_kib": 1796, + "verified": true + }, + { + "collection": "source_versions", + "method": "tar-uc2", + "trial": 0, + "input_bytes": 3618616, + "archive_bytes": 66245, + "compress_seconds": 0.230022, + "restore_seconds": 0.104309, + "compress_peak_kib": 2388, + "restore_peak_kib": 1652, + "verified": true + }, + { + "collection": "source_versions", + "method": "7z", + "trial": 0, + "input_bytes": 3618616, + "archive_bytes": 11869, + "compress_seconds": 2.378033, + "restore_seconds": 0.109321, + "compress_peak_kib": 45300, + "restore_peak_kib": 8544, + "verified": true + }, + { + "collection": "source_versions", + "method": "tar-zstd", + "trial": 0, + "input_bytes": 3618616, + "archive_bytes": 16182, + "compress_seconds": 0.053411, + "restore_seconds": 0.028621, + "compress_peak_kib": 7572, + "restore_peak_kib": 4736, + "verified": true + }, + { + "collection": "source_versions", + "method": "uc2-files", + "trial": 1, + "input_bytes": 3618616, + "archive_bytes": 282315, + "compress_seconds": 0.731753, + "restore_seconds": 0.069715, + "compress_peak_kib": 2704, + "restore_peak_kib": 1928, + "verified": true + }, + { + "collection": "source_versions", + "method": "uc2-chunks", + "trial": 1, + "input_bytes": 3618616, + "archive_bytes": 68771, + "compress_seconds": 0.312596, + "restore_seconds": 0.15765, + "compress_peak_kib": 2372, + "restore_peak_kib": 1696, + "verified": true + }, + { + "collection": "source_versions", + "method": "tar-uc2", + "trial": 1, + "input_bytes": 3618616, + "archive_bytes": 66245, + "compress_seconds": 0.122976, + "restore_seconds": 0.066323, + "compress_peak_kib": 2608, + "restore_peak_kib": 1764, + "verified": true + }, + { + "collection": "source_versions", + "method": "7z", + "trial": 1, + "input_bytes": 3618616, + "archive_bytes": 11869, + "compress_seconds": 2.384093, + "restore_seconds": 0.027671, + "compress_peak_kib": 45232, + "restore_peak_kib": 8484, + "verified": true + }, + { + "collection": "source_versions", + "method": "tar-zstd", + "trial": 1, + "input_bytes": 3618616, + "archive_bytes": 16182, + "compress_seconds": 0.023151, + "restore_seconds": 0.022364, + "compress_peak_kib": 8032, + "restore_peak_kib": 4652, + "verified": true + }, + { + "collection": "source_versions", + "method": "uc2-files", + "trial": 2, + "input_bytes": 3618616, + "archive_bytes": 282315, + "compress_seconds": 0.597355, + "restore_seconds": 0.046043, + "compress_peak_kib": 2636, + "restore_peak_kib": 1856, + "verified": true + }, + { + "collection": "source_versions", + "method": "uc2-chunks", + "trial": 2, + "input_bytes": 3618616, + "archive_bytes": 68771, + "compress_seconds": 0.194641, + "restore_seconds": 0.160663, + "compress_peak_kib": 2476, + "restore_peak_kib": 1808, + "verified": true + }, + { + "collection": "source_versions", + "method": "tar-uc2", + "trial": 2, + "input_bytes": 3618616, + "archive_bytes": 66245, + "compress_seconds": 0.080883, + "restore_seconds": 0.042828, + "compress_peak_kib": 2616, + "restore_peak_kib": 1776, + "verified": true + }, + { + "collection": "source_versions", + "method": "7z", + "trial": 2, + "input_bytes": 3618616, + "archive_bytes": 11869, + "compress_seconds": 1.968079, + "restore_seconds": 0.026296, + "compress_peak_kib": 45160, + "restore_peak_kib": 8268, + "verified": true + }, + { + "collection": "source_versions", + "method": "tar-zstd", + "trial": 2, + "input_bytes": 3618616, + "archive_bytes": 16182, + "compress_seconds": 0.020668, + "restore_seconds": 0.018474, + "compress_peak_kib": 8228, + "restore_peak_kib": 4656, + "verified": true + }, + { + "collection": "simulation", + "method": "uc2-files", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 3818737, + "compress_seconds": 2.326753, + "restore_seconds": 0.174344, + "compress_peak_kib": 2604, + "restore_peak_kib": 1744, + "verified": true + }, + { + "collection": "simulation", + "method": "uc2-chunks", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 689405, + "compress_seconds": 0.4416, + "restore_seconds": 0.161133, + "compress_peak_kib": 2524, + "restore_peak_kib": 1744, + "verified": true + }, + { + "collection": "simulation", + "method": "tar-uc2", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 828765, + "compress_seconds": 0.149331, + "restore_seconds": 0.132954, + "compress_peak_kib": 2152, + "restore_peak_kib": 1792, + "verified": true + }, + { + "collection": "simulation", + "method": "7z", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 558992, + "compress_seconds": 2.197236, + "restore_seconds": 0.074768, + "compress_peak_kib": 50336, + "restore_peak_kib": 9636, + "verified": true + }, + { + "collection": "simulation", + "method": "tar-zstd", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 558175, + "compress_seconds": 0.075583, + "restore_seconds": 0.046413, + "compress_peak_kib": 9120, + "restore_peak_kib": 6120, + "verified": true + }, + { + "collection": "simulation", + "method": "uc2-files", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 3818737, + "compress_seconds": 2.272347, + "restore_seconds": 0.148374, + "compress_peak_kib": 2536, + "restore_peak_kib": 1788, + "verified": true + }, + { + "collection": "simulation", + "method": "uc2-chunks", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 689405, + "compress_seconds": 0.285805, + "restore_seconds": 0.150185, + "compress_peak_kib": 2552, + "restore_peak_kib": 1760, + "verified": true + }, + { + "collection": "simulation", + "method": "tar-uc2", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 828765, + "compress_seconds": 0.17788, + "restore_seconds": 0.151909, + "compress_peak_kib": 2424, + "restore_peak_kib": 1668, + "verified": true + }, + { + "collection": "simulation", + "method": "7z", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 558992, + "compress_seconds": 1.551102, + "restore_seconds": 0.031799, + "compress_peak_kib": 50504, + "restore_peak_kib": 9660, + "verified": true + }, + { + "collection": "simulation", + "method": "tar-zstd", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 558175, + "compress_seconds": 0.025413, + "restore_seconds": 0.025447, + "compress_peak_kib": 10184, + "restore_peak_kib": 5220, + "verified": true + }, + { + "collection": "simulation", + "method": "uc2-files", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 3818737, + "compress_seconds": 1.57079, + "restore_seconds": 0.1002, + "compress_peak_kib": 2740, + "restore_peak_kib": 1760, + "verified": true + }, + { + "collection": "simulation", + "method": "uc2-chunks", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 689405, + "compress_seconds": 0.192837, + "restore_seconds": 0.11998, + "compress_peak_kib": 2552, + "restore_peak_kib": 1776, + "verified": true + }, + { + "collection": "simulation", + "method": "tar-uc2", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 828765, + "compress_seconds": 0.118192, + "restore_seconds": 0.095067, + "compress_peak_kib": 2568, + "restore_peak_kib": 1648, + "verified": true + }, + { + "collection": "simulation", + "method": "7z", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 558992, + "compress_seconds": 1.211704, + "restore_seconds": 0.028826, + "compress_peak_kib": 50376, + "restore_peak_kib": 9576, + "verified": true + }, + { + "collection": "simulation", + "method": "tar-zstd", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 558175, + "compress_seconds": 0.025537, + "restore_seconds": 0.016905, + "compress_peak_kib": 10928, + "restore_peak_kib": 5604, + "verified": true + }, + { + "collection": "independent", + "method": "uc2-files", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 4201233, + "compress_seconds": 1.294164, + "restore_seconds": 0.076896, + "compress_peak_kib": 2468, + "restore_peak_kib": 1884, + "verified": true + }, + { + "collection": "independent", + "method": "uc2-chunks", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 4213383, + "compress_seconds": 0.398389, + "restore_seconds": 0.114191, + "compress_peak_kib": 2544, + "restore_peak_kib": 1768, + "verified": true + }, + { + "collection": "independent", + "method": "tar-uc2", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 4225083, + "compress_seconds": 0.603933, + "restore_seconds": 0.079515, + "compress_peak_kib": 2468, + "restore_peak_kib": 1808, + "verified": true + }, + { + "collection": "independent", + "method": "7z", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 4194795, + "compress_seconds": 2.027532, + "restore_seconds": 0.018995, + "compress_peak_kib": 50536, + "restore_peak_kib": 13184, + "verified": true + }, + { + "collection": "independent", + "method": "tar-zstd", + "trial": 0, + "input_bytes": 4194304, + "archive_bytes": 4198532, + "compress_seconds": 0.025947, + "restore_seconds": 0.013026, + "compress_peak_kib": 16544, + "restore_peak_kib": 5576, + "verified": true + }, + { + "collection": "independent", + "method": "uc2-files", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 4201233, + "compress_seconds": 1.188415, + "restore_seconds": 0.061086, + "compress_peak_kib": 2604, + "restore_peak_kib": 1932, + "verified": true + }, + { + "collection": "independent", + "method": "uc2-chunks", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 4213383, + "compress_seconds": 0.600238, + "restore_seconds": 0.091974, + "compress_peak_kib": 2456, + "restore_peak_kib": 1776, + "verified": true + }, + { + "collection": "independent", + "method": "tar-uc2", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 4225083, + "compress_seconds": 0.287276, + "restore_seconds": 0.068391, + "compress_peak_kib": 2504, + "restore_peak_kib": 1760, + "verified": true + }, + { + "collection": "independent", + "method": "7z", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 4194795, + "compress_seconds": 1.953398, + "restore_seconds": 0.017929, + "compress_peak_kib": 50516, + "restore_peak_kib": 13196, + "verified": true + }, + { + "collection": "independent", + "method": "tar-zstd", + "trial": 1, + "input_bytes": 4194304, + "archive_bytes": 4198532, + "compress_seconds": 0.024579, + "restore_seconds": 0.015071, + "compress_peak_kib": 16212, + "restore_peak_kib": 6180, + "verified": true + }, + { + "collection": "independent", + "method": "uc2-files", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 4201233, + "compress_seconds": 1.378445, + "restore_seconds": 0.074975, + "compress_peak_kib": 2576, + "restore_peak_kib": 1844, + "verified": true + }, + { + "collection": "independent", + "method": "uc2-chunks", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 4213383, + "compress_seconds": 0.351736, + "restore_seconds": 0.071957, + "compress_peak_kib": 2712, + "restore_peak_kib": 1752, + "verified": true + }, + { + "collection": "independent", + "method": "tar-uc2", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 4225083, + "compress_seconds": 0.345224, + "restore_seconds": 0.092703, + "compress_peak_kib": 2516, + "restore_peak_kib": 1772, + "verified": true + }, + { + "collection": "independent", + "method": "7z", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 4194795, + "compress_seconds": 1.846659, + "restore_seconds": 0.020069, + "compress_peak_kib": 50532, + "restore_peak_kib": 13200, + "verified": true + }, + { + "collection": "independent", + "method": "tar-zstd", + "trial": 2, + "input_bytes": 4194304, + "archive_bytes": 4198532, + "compress_seconds": 0.026454, + "restore_seconds": 0.017152, + "compress_peak_kib": 15500, + "restore_peak_kib": 5660, + "verified": true + } + ] +} diff --git a/docs/chunk-sharing.md b/docs/chunk-sharing.md new file mode 100644 index 0000000..761874b --- /dev/null +++ b/docs/chunk-sharing.md @@ -0,0 +1,98 @@ +# Shared chunks in named archives + +UC2 3.0.0a11 adds optional payload sharing between partly changed files and +stored versions. Each original file remains one named entry. The writer splits +it at content-defined boundaries and stores each distinct chunk once within +the archive, after comparing both SHA-256 and the original bytes. + +```sh +uc2 --chunked -w versions.uc2 old/ new/ +tar cf - project/ | uc2 --chunked --ingest-tar backup.uc2 +producer | uc2 --chunked --ingest stream.uc2 +uc2 --ingest-restore stream.uc2 > restored-stream +``` + +Opaque ingestion creates one entry named `data`; its bytes are the complete +original stream. The existing opaque v1/v2/v3-A restore paths remain available. +Tar ingestion preserves named entries, with the metadata limitations in the +[tar guide](tar-ingestion.md). Ordinary list, print and extract commands work +for both modes. The default writer continues to use shared dictionaries and +whole-file payload reuse in the existing format. + +```python +from uc2 import UC2Writer + +with UC2Writer("versions.uc2", chunked=True, level=4) as archive: + archive.write("old.bin", "data.bin") + archive.write("new.bin", "data.bin") +print(archive.stats) # includes unique_chunks, chunk_references, shared_bytes +``` + +Use the full a11 or newer reader, browser build or WCX plugin for these archives. +Original DOS UC2, earlier revival versions and the optional LGPL `libuc2read` +dependency reject them. The proposed upstream libarchive reader uses that +legacy dependency and therefore cannot read this extension. UC2's separate +full GPL libarchive adapter can. Existing archives remain readable. + +Chunks target 8 KiB, with a 2 KiB minimum except at EOF and a 32 KiB maximum. +Each unique chunk is independently compressed at the requested level; a chunk +that does not shrink is stored raw. Small or dissimilar files can grow because +each file needs an 88-byte recipe plus 48 bytes per chunk reference. There is no +cross-archive reuse. Whole-file duplicates also share the recipe. + +The writer caps the entire collection at 1,048,576 chunk references, counting +duplicates, in addition to existing input, output, entry and directory limits. Decoded central-directory +metadata is capped at 64 MiB, independent of its compression ratio. +Its retained chunk index uses at most 64 MiB plus 128 KiB of payload buffers; +array growth temporarily needs both old and new allocations. Existing file +metadata and compression workspaces are additional. Input and +output are spooled to temporary files. Reading checks one descriptor at a time +and decodes at most 32 KiB per chunk, with the codec workspace additional. +It does not retain a decoded-chunk cache. + +`shared_bytes` counts logical source bytes served by an earlier chunk or whole +file; it is not the compressed space saving. A method-11 entry's `compress_size` +is its recipe size, excluding shared payloads. Adding entry sizes cannot recover +the archive's disk size. Use the actual file size or writer `archive_size`. + +Each chunk, the ordered recipe and the complete decoded file have SHA-256 +checks. The historical UC2 XOR checksum is checked too. These detect corruption; +they do not authenticate the author. Python/CLI extraction stages output before +publication. C extraction callbacks can receive earlier verified chunks before +a later corruption or output error, so callers must stage output when needed. + +## Method 11 wire format + +All integers are little endian. Archive `versionMadeBy` and +`versionNeededToExtract` are 300. The central directory uses the existing codec +and has one ordinary entry per source file, with method 11, master ID 1 and an +offset to the recipe. The central-directory compressed size equals the recipe +length. Chunked archives produced by this writer have no master records. + +| Recipe header offset | Bytes | Meaning | +| --- | ---: | --- | +| 0 | 8 | `UC2CHN1` followed by NUL | +| 8 | 4 | Chunk count | +| 12 | 4 | Decoded file size, matching the central directory | +| 16 | 4 | Record table length, exactly count × 48 | +| 20 | 4 | Flags, currently zero | +| 24 | 32 | SHA-256 of the complete decoded file | +| 56 | 32 | SHA-256 of header bytes 0–55 followed by the ordered record table | + +| Record offset | Bytes | Meaning | +| --- | ---: | --- | +| 0 | 4 | Absolute chunk payload offset | +| 4 | 4 | Stored byte length | +| 8 | 4 | Decoded length, 1–32,768 | +| 12 | 2 | Method: 0 raw, 1–9 Huffman without a master/delta, 10 rANS | +| 14 | 2 | Historical XOR checksum of decoded chunk | +| 16 | 32 | SHA-256 of decoded chunk | + +All referenced payloads begin at offset 29 or later and end at or before their +recipe. Recipes must end before the central directory. References are direct, +never recursive. Raw stored and decoded sizes must match; compressed payloads +must have a positive even length of at most 65,536 bytes and consume exactly +their encoded word range. The sum of decoded lengths must equal the file size. +An empty file has zero records and SHA-256 of the empty byte string. Readers +validate the whole table before delivering any payload and check the full-file +hash after delivering the chunks. diff --git a/docs/index.rst b/docs/index.rst index a808581..333f2f4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,6 +20,10 @@ verified ZIP/tar exports. See the `preservation guide with shared chunks, verified restores, explicit retention and replica repair. See the `snapshot guide `_. +The optional ``--chunked`` writer shares payload chunks across named files and +stored versions; see the `chunk-sharing guide +`_ for compatibility. + The native CLI also supports tar ingestion with separate entries and data-only SAS/UC2SEA reads. See the `tar guide `_ and diff --git a/docs/library.rst b/docs/library.rst index 4ac710c..b2b6232 100644 --- a/docs/library.rst +++ b/docs/library.rst @@ -226,3 +226,10 @@ nesting at 64, and the CDC index at 1,048,576 chunks. Metadata grows with the collection while payload buffers remain fixed. Source and archive temporary files require disk space. The public header defines return values, metadata fields, limits, and ``uc2_writer_get_stats``. + +Set ``options.chunked = 1`` to share payload chunks across named files and +versions. Python exposes this as ``UC2Writer(..., chunked=True)``. This opt-in +format requires the full a11 or newer reader. See the `chunk-sharing guide +`_ for the wire +format, reference limits and compressed-size accounting. Rebuild C callers +against the matching header when upgrading writer option/stat structures. diff --git a/docs/license-audit.md b/docs/license-audit.md index a936f62..15012fa 100644 --- a/docs/license-audit.md +++ b/docs/license-audit.md @@ -133,7 +133,7 @@ been obtained. `contrib/libuc2read` builds `libuc2read` from the existing LGPL decoder and original SuperMaster data. `UC2_HUFFMAN_ONLY` removes the rANS include and -implementation from that target; none of the GPL compressor or rANS objects +implementation from that target; none of the GPL compressor, rANS or SHA-256 objects are linked. The target retains LGPL-3.0-only, uses a shared-library interface, and installs the LGPL and GPL license texts needed for its distribution. Its build recipe has the approved dual-license expression. Existing core file @@ -232,3 +232,9 @@ GPL-3.0-or-later OR BSD-2-Clause license for independently written Python glue. The Gear chunker and compressor it calls retain their GPL licenses. New test fixtures are generated by the repository test code; historical and external oracle archives remain optional downloads with pinned URLs/hashes. + +The a11 method-11 shared-chunk extension and its SHA-256 implementation are +GPL-3.0-or-later. `UC2_HUFFMAN_ONLY` excludes that code from `libuc2read`, which +rejects archives requiring version 300. `uc2_chunks.h` and the chunk-recipe +fuzz harness carry GPL notices; their small seed archives contain generated +pseudorandom bytes and repeated test strings, not third-party collection data. diff --git a/docs/releases/3.0.0a11-qa.md b/docs/releases/3.0.0a11-qa.md new file mode 100644 index 0000000..736cb7c --- /dev/null +++ b/docs/releases/3.0.0a11-qa.md @@ -0,0 +1,36 @@ +# Alpha11 named-file chunk sharing and libarchive submission + +Alpha11 adds optional shared payload chunks across named files and stored +versions, including tar-entry and opaque-stream ingestion. Use `--chunked` or +`UC2Writer(..., chunked=True)`. The default writer remains compatible with the +existing format. Method-11 archives require a11 or newer full readers; the +optional LGPL libuc2read dependency deliberately rejects them. + +Local checks cover 215 Python tests, 29 native CTests including DOSBox-X, +all compression levels, partial edits, versions, empty files, hostile references, +changed payloads, compressed word boundaries and staged extraction failure. +Native regressions cover invalid central-directory recipe lengths, rANS symbol +and end-marker validation, and the 64 MiB decoded-directory limit. Four thousand +versions with repeated long names now round-trip; a ratio heuristic previously +rejected these valid archives in both writer modes. + +The browser/WASM reader, WCX fixture driver and full libarchive adapter include +chunked-archive cases. Actual bsdtar reads named chunked files through paths and +pipes, and restores a separate 128 MiB legacy entry under a 96 MiB address-space +limit. Sanitizer runs exercise archive extraction, chunk recipes and codec +round-trips. The [benchmark](../benchmarks/2026-11-07-chunk-sharing.md) reports +both reuse gains and the 9.1% size regression on an edited public-source corpus. + +The upstream libarchive proposal is a BSD adapter using the separate optional +LGPL legacy reader, disabled by default. Current-master validation passes all +969 libarchive tests, plus focused enabled/disabled CMake and enabled Autotools +checks. A format-registration capacity bug was fixed after full-suite testing +exposed it. Upstream acceptance of the dependency and format ID remains external. +The [submission record](../../contrib/libarchive/upstream/REVIEW.md) tracks it. + +Independent scoped reviews use Kimi in place of Claude, Gemini, and opportunistic +Grok. Failed or empty reviewer calls do not count as clean reviews. The broader +project-wide audit remains open. TC and MC proposals remain unsent. + +Publication, exact-commit CI and installed/public artifact verification are +recorded here after release. diff --git a/docs/tar-ingestion.md b/docs/tar-ingestion.md index 0c30202..e93c684 100644 --- a/docs/tar-ingestion.md +++ b/docs/tar-ingestion.md @@ -32,5 +32,6 @@ file names create stored versions. Identical files share one compressed payload after SHA-256 and exact-byte comparison. The writer also uses content-defined chunks to form shared master -dictionaries. This is whole-file payload reuse plus dictionary sharing; named -files do not yet refer to a general pool of independently stored chunks. +dictionaries. Add `--chunked` to share independently stored payload chunks across partially +changed files. This opt-in format requires the full a11 or newer reader; see +[chunk sharing](chunk-sharing.md) for compatibility, limits and examples. diff --git a/lib/include/uc2/uc2_chunks.h b/lib/include/uc2/uc2_chunks.h new file mode 100644 index 0000000..e12a11b --- /dev/null +++ b/lib/include/uc2/uc2_chunks.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +#ifndef UC2_CHUNKS_H +#define UC2_CHUNKS_H + +/* Revival-only recipe format. See docs/chunk-sharing.md for the wire layout. */ +#define UC2_CHUNK_METHOD 11 +#define UC2_CHUNK_VERSION 300 +#define UC2_CHUNK_MAGIC "UC2CHN1\0" +#define UC2_CHUNK_HEADER 88u +#define UC2_CHUNK_RECORD 48u +#define UC2_CHUNK_MAX 32768u +#define UC2_CHUNK_LIMIT 1048576u + +#endif diff --git a/lib/include/uc2/uc2_writer.h b/lib/include/uc2/uc2_writer.h index 0461106..1cd3fda 100644 --- a/lib/include/uc2/uc2_writer.h +++ b/lib/include/uc2/uc2_writer.h @@ -22,9 +22,20 @@ struct uc2_writer_options { int level; /* 2-5 Huffman, 6-9 rANS */ unsigned max_entries; /* files plus directories; 1..1000000 */ uint64_t max_input_size; /* total spooled bytes; 1..UINT32_MAX */ + int chunked; /* 0: classic layout; 1: shared chunk recipes */ }; -#define UC2_WRITER_OPTIONS_INIT {4, 100000, UINT32_MAX} -struct uc2_writer_stats { unsigned files, directories, masters, archive_size; }; +#define UC2_WRITER_OPTIONS_INIT {4, 100000, UINT32_MAX, 0} +struct uc2_writer_stats { + unsigned files, directories, masters, archive_size; + unsigned unique_chunks, chunk_references; + uint64_t shared_bytes; +}; + +/* chunked=1 requires the full revival reader 3.0.0a11 or newer. It stores + * independent Gear chunks once across files/versions and a recipe per file. + * Maximum 1048576 logical chunk references across the input collection. + * Classic output remains the default; historical DOS readers cannot read + * chunked archives. No cross-archive references are created. */ /* NULL options selects defaults. NULL result: invalid options or allocation/ * temporary-file failure. Uses malloc/free and the platform's tmpfile(). */ diff --git a/lib/src/decompress.c b/lib/src/decompress.c index 878fef5..41d07be 100644 --- a/lib/src/decompress.c +++ b/lib/src/decompress.c @@ -22,6 +22,8 @@ #include "uc2/libuc2.h" #ifndef UC2_HUFFMAN_ONLY #include "uc2/uc2_rans.h" +#include "uc2/uc2_sha256.h" +#include "uc2/uc2_chunks.h" #endif #if !defined NDEBUG && !defined NDIAG @@ -195,6 +197,7 @@ struct uc2_context { u8 *cdir_buf; struct range cdir_range; + unsigned data_end; enum { Start, @@ -205,6 +208,7 @@ struct uc2_context { u8 scanned:1; u8 pcp:1; + u8 chunked:1; }; /* callback */ @@ -684,28 +688,20 @@ static int use_master(struct uc2_context *uc2, u8 buffer[65535], u32 id) static int cdir_damaged(struct uc2_context *uc2); -/* Writer for the central-directory decode that also enforces a - compression-ratio ceiling. A tiny crafted cdir stream can expand via - long matches into tens of megabytes (a decompression bomb), turning a - few-hundred-byte archive into a multi-second decode. Abort once the - output far outgrows the compressed bytes consumed. */ +/* Repeated names and stored versions can legitimately exceed a 64:1 ratio. + * Bound decoded metadata directly, matching the archive writer's 64 MiB cap. */ +enum { CdirLimit = 64u * 1024u * 1024u }; struct cdir_writer { - struct range out; - struct archive_ctx *src; /* reader context, for bytes consumed */ - unsigned base; /* src->offset at decode start */ - unsigned long produced; + struct range out; + unsigned produced; }; static int cdir_write(void *context, const void *ptr, unsigned size) { - struct cdir_writer *w = context; - w->produced += size; - unsigned consumed = w->src->offset - w->base; - /* Real cdir metadata compresses well under ~20:1; 64:1 with a - 64 KiB floor leaves ample headroom while stopping bombs. */ - if (w->produced > 65536 + 64ul * consumed) - return UC2_Damaged; - return buf_write(&w->out, ptr, size); + struct cdir_writer *w = context; + if (size > CdirLimit - w->produced) return UC2_Damaged; + w->produced += size; + return buf_write(&w->out, ptr, size); } static int decompress_cdir(struct uc2_context *uc2, u32 offset, u16 csum) @@ -736,12 +732,11 @@ static int decompress_cdir(struct uc2_context *uc2, u32 offset, u16 csum) struct archive_ctx ar = {.offset = offset, .uc2 = uc2}; struct reader rd = {.read = archive_read, .context = &ar}; struct cdir_writer wctx = { - .out = {uc2->cdir_buf, uc2->cdir_buf + size}, - .src = &ar, .base = offset + .out = {uc2->cdir_buf, uc2->cdir_buf + size} }; struct writer wr = {.write = cdir_write, .context = &wctx}; u16 cs; - ret = decompressor(uc2, get16(c.method), &rd, &wr, NoMaster, 100000000, &cs); + ret = decompressor(uc2, get16(c.method), &rd, &wr, NoMaster, CdirLimit + 1, &cs); if (ret < 0) goto fail; @@ -777,6 +772,11 @@ fail: static int start_read(struct uc2_context *uc2); static int read_entry(struct uc2_context *uc2, struct uc2_entry *e, u8 type); static void copy_dos_name(u8 *dos_name, u8 *s); +#ifndef UC2_HUFFMAN_ONLY +static int chunk_header(struct uc2_context *, const struct uc2_xinfo *, unsigned, const unsigned *, u8 *); +static int extract_chunks(struct uc2_context *, const struct uc2_xinfo *, unsigned, + int (*)(void *, const void *, unsigned), void *); +#endif int uc2_read_cdir(struct uc2_context *uc2, struct uc2_entry *e) { @@ -896,6 +896,16 @@ static int read_entry(struct uc2_context *uc2, struct uc2_entry *e, u8 type) e->xi.master = get32(rc->u.f.c.masterPrefix); e->xi.csum = get16(rc->u.f.m.fletch); e->xi.method = get16(rc->u.f.c.method); + if (e->xi.method==11) { +#ifdef UC2_HUFFMAN_ONLY + uc2->message="Chunk sharing requires the full UC2 reader"; + return UC2_Unimplemented; +#else + u8 header[UC2_CHUNK_HEADER]; + int result=chunk_header(uc2,&e->xi,e->size,&e->csize,header); + if (result<0) return result; +#endif + } e->is_dir = 0; } else { e->id = get32(rc->u.d.m.index); @@ -1063,6 +1073,14 @@ int uc2_extract( if (!uc2->scanned) return UC2_BadState; + if (xi->method==11) { +#ifdef UC2_HUFFMAN_ONLY + uc2->message="Chunk sharing requires the full UC2 reader"; + return UC2_Unimplemented; +#else + return extract_chunks(uc2,xi,size,write,context); +#endif + } ret = resolve_master(uc2, xi->master); if (ret < 0) return ret; @@ -1085,7 +1103,7 @@ int uc2_extract( static int decompressor_ultra(struct uc2_context *uc2, unsigned master, unsigned delta, struct reader *rd, struct writer *wr, unsigned limit, u16 *csum, uint64_t *used); static int decompressor_turbo(struct uc2_context *uc2, unsigned master, struct reader *rd, struct writer *wr, unsigned limit, u16 *csum); #ifndef UC2_HUFFMAN_ONLY -static int decompressor_rans(struct uc2_context *uc2, unsigned master_id, struct reader *rd, struct writer *wr, unsigned limit, u16 *csum); +static int decompressor_rans(struct uc2_context *uc2, unsigned master_id, struct reader *rd, struct writer *wr, unsigned limit, u16 *csum, uint64_t *used); #endif static int decompressor(struct uc2_context *uc2, int method, struct reader *rd, struct writer *wr, unsigned master, unsigned len, u16 *csum) @@ -1113,7 +1131,7 @@ ultra: uc2->message = "rANS requires the full UC2 reader"; ret = UC2_Unimplemented; #else - ret = decompressor_rans(uc2, master, rd, wr, len, csum); + ret = decompressor_rans(uc2, master, rd, wr, len, csum, NULL); #endif } else if (method == 80) { ret = decompressor_turbo(uc2, master, rd, wr, len, csum); @@ -1202,8 +1220,9 @@ static int rans_flush(struct writer *wr, struct csum *cs, const u8 *buf, unsigned lin = 0x10000u - *wpos; if (n > lin) n = lin; csum_update(cs, buf + *wpos, n); - if (wr->write(wr->context, buf + *wpos, n) < 0) - return UC2_UserFault; + int result = wr->write(wr->context, buf + *wpos, n); + if (result < 0) + return result; *wpos = (u16)(*wpos + n); } return 0; @@ -1212,7 +1231,7 @@ static int rans_flush(struct writer *wr, struct csum *cs, const u8 *buf, /* rANS decompressor (method 10) */ static int decompressor_rans(struct uc2_context *uc2, unsigned master_id, struct reader *rd, struct writer *wr, - unsigned limit, u16 *csum) + unsigned limit, u16 *csum, uint64_t *used) { const unsigned EOB = 64001; int ret; @@ -1263,9 +1282,12 @@ static int decompressor_rans(struct uc2_context *uc2, unsigned master_id, /* Decode all nsyms symbols, including the trailing EOB pair and its extra bits: stopping at remaining == 0 would leave the bit cursor mid-block and desynchronize the next block-present bit. */ + int ended = 0; for (unsigned s = 0; s < nsyms; s++) { int sym = uc2_rans_decode(&dec); + if (used && (sym<0 || sym>=316)) { bi.err=UC2_Damaged; break; } if (sym < 256) { + if (used && !remaining) { bi.err=UC2_Damaged; break; } if (remaining) { buf[tail++] = (u8)sym; remaining--; @@ -1282,10 +1304,20 @@ static int decompressor_rans(struct uc2_context *uc2, unsigned master_id, (ds < 45) ? (ds-30+1)*256 + (bits_get(&bi, 8) & 0xff) : (ds-45+1)*4096 + (bits_get(&bi, 12) & 0xfff); if (bi.err) break; - if (dist == EOB) { s++; if (s < nsyms) uc2_rans_decode(&dec); break; } + if (dist == EOB) { + ended = 1; + s++; + if (used && s>=nsyms) { bi.err=UC2_Damaged; break; } + if (s=344 || s+1!=nsyms)) bi.err=UC2_Damaged; + } + break; + } s++; - if (s >= nsyms) break; + if (s >= nsyms) { if (used) bi.err=UC2_Damaged; break; } int ls = uc2_rans_decode(&dec) - 316; + if (used && (ls<0 || ls>=28)) { bi.err=UC2_Damaged; break; } if (ls < 0) ls = 0; unsigned length = (ls < 8) ? ls + 3 : (ls < 16) ? (ls-8)*2+11+(bits_get(&bi,1) & 0x1) : @@ -1295,6 +1327,7 @@ static int decompressor_rans(struct uc2_context *uc2, unsigned master_id, (ls == 26) ? 667+(bits_get(&bi,11) & 0x7ff) : 2715+(bits_get(&bi,15) & 0x7fff); if (bi.err) break; + if (used && length>remaining) { bi.err=UC2_Damaged; break; } if (dist > produced) { bi.err = UC2_Damaged; break; } for (unsigned j = 0; j < length && remaining > 0; j++) { buf[tail] = buf[(u16)(tail - dist)]; @@ -1308,6 +1341,7 @@ static int decompressor_rans(struct uc2_context *uc2, unsigned master_id, if (bi.err) break; } } + if (used && !ended) bi.err=UC2_Damaged; u_free(uc2, rdata); if (bi.err) break; } @@ -1324,6 +1358,7 @@ static int decompressor_rans(struct uc2_context *uc2, unsigned master_id, bits_destroy(&bi); u_free(uc2, buf); if (csum) *csum = csum_get(&cs); + if (used) *used=bi.consumed; return limit - remaining; } @@ -1791,7 +1826,15 @@ not_uc2: get32(h.xhead.cdir.offset), get16(h.xhead.versionMadeBy), ver); - if (ver >= 203) { + uc2->data_end=get32(h.xhead.cdir.offset); + if (ver==300) { +#ifdef UC2_HUFFMAN_ONLY + uc2->message="Chunk sharing requires the full UC2 reader"; + return UC2_Unimplemented; +#else + uc2->chunked=1; +#endif + } else if (ver >= 203) { if (ver > 203) goto not_uc2; uc2->pcp = 1; @@ -1809,6 +1852,7 @@ static int raw_read(void *context, void *buffer, unsigned size) if (r->remaining) { if (size > r->remaining) size = r->remaining; int n = u_read(r->uc2, r->offset, buffer, size); + if (n>0 && (unsigned)n>size) return UC2_UserFault; if (n > 0) { r->offset += n; r->remaining -= n; } return n; } @@ -1839,6 +1883,94 @@ int uc2_decompress_raw(struct uc2_io *io, void *context, unsigned offset, return result; } +#ifndef UC2_HUFFMAN_ONLY +static unsigned chunk_u32(const u8 *p) +{ return p[0]|(unsigned)p[1]<<8|(unsigned)p[2]<<16|(unsigned)p[3]<<24; } +static unsigned chunk_u16(const u8 *p) { return p[0]|(unsigned)p[1]<<8; } +static int chunk_header(struct uc2_context *uc2, const struct uc2_xinfo *xi, + unsigned size, const unsigned *csize, u8 *header) +{ + if (!uc2->chunked || xi->master!=NoMaster || xi->offset<29 || + xi->offset>uc2->data_end || UC2_CHUNK_HEADER>uc2->data_end-xi->offset) + return UC2_Damaged; + int ret=u_read_all(uc2,xi->offset,header,UC2_CHUNK_HEADER); + if (ret<0) return ret; + unsigned count=chunk_u32(header+8), table=chunk_u32(header+16); + if (memcmp(header,UC2_CHUNK_MAGIC,8) || chunk_u32(header+12)!=size || + chunk_u32(header+20) || count>UC2_CHUNK_LIMIT || count>size || + (!count)!=(!size) || table!=count*UC2_CHUNK_RECORD || + table>uc2->data_end-xi->offset-UC2_CHUNK_HEADER || + (csize && *csize!=UC2_CHUNK_HEADER+table)) return UC2_Damaged; + return 0; +} +static int chunk_record_valid(const u8 *record, unsigned before) +{ + unsigned offset=chunk_u32(record), csize=chunk_u32(record+4); + unsigned size=chunk_u32(record+8), method=chunk_u16(record+12); + return size && size<=UC2_CHUNK_MAX && csize && csize<=2*UC2_CHUNK_MAX && + offset>=29 && offset<=before && csize<=before-offset && + (method==0 ? csize==size : method<=10 && !(csize&1)); +} +static int extract_chunks(struct uc2_context *uc2, const struct uc2_xinfo *xi, + unsigned size, int (*write)(void *, const void *, unsigned), void *output) +{ + u8 header[UC2_CHUNK_HEADER], record[UC2_CHUNK_RECORD], digest[32]; + int ret=chunk_header(uc2,xi,size,NULL,header); + if (ret<0) return ret; + unsigned count=chunk_u32(header+8), total=0; + struct uc2_sha256 table_hash; + uc2_sha256_init(&table_hash); uc2_sha256_update(&table_hash,header,56); + for (unsigned i=0; ioffset+UC2_CHUNK_HEADER+i*UC2_CHUNK_RECORD,record,sizeof record); + if (ret<0) return ret; + unsigned n=chunk_u32(record+8); + if (!chunk_record_valid(record,xi->offset) || n>size-total) return UC2_Damaged; + total+=n; uc2_sha256_update(&table_hash,record,sizeof record); + } + uc2_sha256_final(&table_hash,digest); + if (total!=size || memcmp(digest,header+56,32)) return UC2_Damaged; + u8 *data=u_alloc(uc2,UC2_CHUNK_MAX); + if (!data) return UC2_UserFault; + struct uc2_sha256 file_hash; struct csum file_sum; + uc2_sha256_init(&file_hash); csum_init(&file_sum); total=0; + for (unsigned i=0; ioffset+UC2_CHUNK_HEADER+i*UC2_CHUNK_RECORD,record,sizeof record); + if (ret<0) goto done; + unsigned n=chunk_u32(record+8), csize=chunk_u32(record+4), method=chunk_u16(record+12); + if (!chunk_record_valid(record,xi->offset) || n>size-total) { ret=UC2_Damaged; goto done; } + u16 sum=0; + if (!method) { + ret=u_read_all(uc2,chunk_u32(record),data,n); + if (ret<0) goto done; + struct csum cs; csum_init(&cs); csum_update(&cs,data,n); sum=csum_get(&cs); + } else { + struct raw_reader source={uc2,chunk_u32(record),csize,0}; + struct reader rd={.read=raw_read,.context=&source}; + struct range buffer={data,data+n}; + struct user_write_ctx sink={.write=buf_write,.context=&buffer,.remaining=n}; + struct writer wr={.write=user_write,.context=&sink}; + uint64_t used=0; + ret=method==10 ? decompressor_rans(uc2,NoMaster,&rd,&wr,n,&sum,&used) : + decompressor_ultra(uc2,NoMaster,0,&rd,&wr,n,&sum,&used); + if (ret<0) goto done; + if (sink.remaining || used>(uint64_t)csize*8 || (used+15)/16*2!=csize) { + ret=UC2_Damaged; goto done; + } + } + struct uc2_sha256 chunk_hash; + uc2_sha256_init(&chunk_hash); uc2_sha256_update(&chunk_hash,data,n); uc2_sha256_final(&chunk_hash,digest); + if (sum!=chunk_u16(record+14) || memcmp(digest,record+16,32)) { ret=UC2_Damaged; goto done; } + uc2_sha256_update(&file_hash,data,n); csum_update(&file_sum,data,n); total+=n; + if (write(output,data,n)<0) { ret=UC2_UserFault; goto done; } + } + uc2_sha256_final(&file_hash,digest); + ret=total==size && csum_get(&file_sum)==xi->csum && !memcmp(digest,header+24,32) ? 0 : UC2_Damaged; +done: + u_free(uc2,data); + return ret; +} +#endif + /* public */ struct uc2_context *uc2_open(struct uc2_io *io, void *io_ctx) @@ -1853,6 +1985,8 @@ struct uc2_context *uc2_open(struct uc2_io *io, void *io_ctx) uc2->cdir_state = Start; uc2->scanned = 0; uc2->pcp = 0; + uc2->chunked = 0; + uc2->data_end = 0; list_init(&uc2->masters); } return uc2; diff --git a/lib/src/uc2_writer.c b/lib/src/uc2_writer.c index c7f5146..a1313aa 100644 --- a/lib/src/uc2_writer.c +++ b/lib/src/uc2_writer.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,7 @@ struct entry { char name[256]; unsigned parent, id, depth, time, attr; - unsigned size, csize, offset, master, spool, duplicate; + unsigned size, csize, offset, master, spool, duplicate, chunk_count; unsigned char digest[32]; unsigned short checksum; }; @@ -138,7 +139,8 @@ uc2_writer *uc2_writer_create(const struct uc2_writer_options *options) if (!options) options=&defaults; if (options->level<2 || options->level>9 || !options->max_entries || options->max_entries>1000000 || !options->max_input_size || - options->max_input_size>UINT32_MAX) return NULL; + options->max_input_size>UINT32_MAX || + (options->chunked!=0 && options->chunked!=1)) return NULL; uc2_writer *w=calloc(1,sizeof *w); if (!w) return NULL; w->options=*options; @@ -211,6 +213,7 @@ int uc2_writer_add_file(uc2_writer *w, unsigned parent, const char *name, e->spool=(unsigned)w->input_size; unsigned char buffer[65536]; uint64_t consumed=0; + unsigned sum=0xa55a; struct uc2_sha256 sha; uc2_sha256_init(&sha); w->state=1; for (;;) { @@ -223,11 +226,14 @@ int uc2_writer_add_file(uc2_writer *w, unsigned parent, const char *name, if ((unsigned)n>limit) return fail(w,UC2_WRITER_LIMIT,"input exceeds declared size or total input limit"); if (fwrite(buffer,1,(size_t)n,w->input)!=(size_t)n) return fail(w,UC2_UserFault,"temporary input write failed"); uc2_sha256_update(&sha,buffer,(unsigned)n); + if (w->options.chunked) + for (unsigned i=0; i<(unsigned)n; ++i) + sum^=(unsigned)buffer[i]<<(((consumed+i)&1)*8); consumed+=(unsigned)n; w->input_size+=(unsigned)n; } if (size!=UC2_WRITER_UNKNOWN_SIZE && consumed!=size) return fail(w,UC2_Truncated,"input ended before its declared size"); uc2_sha256_final(&sha,e->digest); - e->size=(unsigned)consumed; w->state=0; + e->size=(unsigned)consumed; e->checksum=(unsigned short)sum; w->state=0; return 0; } static int load_at(uc2_writer *w, unsigned pos, void *buf, unsigned n) @@ -378,11 +384,155 @@ static int duplicates(uc2_writer *w) } free(index); return 0; } + +struct shared_chunk { + unsigned spool, size, offset, csize; + unsigned short checksum, method; + unsigned char digest[32]; +}; +struct chunk_index { + struct shared_chunk *chunks; + unsigned *slots, *refs; + unsigned count, capacity, slot_count, ref_capacity; + unsigned char *raw, *compressed, *compare; +}; +static unsigned chunk_slot(const unsigned char digest[32], unsigned count) +{ + uint64_t h=0; + for (unsigned i=0; i<8; ++i) h=(h<<8)|digest[i]; + return (unsigned)(h^(h>>32))&(count-1); +} +static int chunk_index_grow(uc2_writer *w, struct chunk_index *index) +{ + if ((index->count+1)*2<=index->slot_count) return 0; + unsigned count=index->slot_count ? index->slot_count*2 : 256; + unsigned *slots=calloc(count,sizeof *slots); + if (!slots) return fail(w,UC2_UserFault,"cannot allocate shared chunk index"); + for (unsigned i=0; icount; ++i) { + unsigned slot=chunk_slot(index->chunks[i].digest,count); + while (slots[slot]) slot=(slot+1)&(count-1); + slots[slot]=i+1; + } + free(index->slots); index->slots=slots; index->slot_count=count; + return 0; +} +struct compressed_chunk { unsigned char *data; unsigned size; }; +static int chunk_output(void *ctx, const void *data, unsigned n) +{ + struct compressed_chunk *b=ctx; + if (n>2*UC2_CHUNK_MAX-b->size) return -1; + memcpy(b->data+b->size,data,n); b->size+=n; return 0; +} +static void chunk_record(unsigned char record[UC2_CHUNK_RECORD], const struct shared_chunk *c) +{ + w32(record,c->offset); w32(record+4,c->csize); w32(record+8,c->size); + w16(record+12,c->method); w16(record+14,c->checksum); + memcpy(record+16,c->digest,32); +} +static int shared_chunk(uc2_writer *w, struct chunk_index *index, + unsigned spool, unsigned size, unsigned *id) +{ + struct uc2_sha256 sha; unsigned char digest[32]; + uc2_sha256_init(&sha); uc2_sha256_update(&sha,index->raw,size); uc2_sha256_final(&sha,digest); + if (chunk_index_grow(w,index)<0) return w->error; + unsigned slot=chunk_slot(digest,index->slot_count); + while (index->slots[slot]) { + unsigned i=index->slots[slot]-1; + struct shared_chunk *c=&index->chunks[i]; + if (c->size==size && !memcmp(c->digest,digest,32)) { + if (load_at(w,c->spool,index->compare,size)<0) return w->error; + if (!memcmp(index->raw,index->compare,size)) { + *id=i; w->stats.shared_bytes+=size; return 0; + } + } + slot=(slot+1)&(index->slot_count-1); + } + if (index->count==index->capacity) { + unsigned cap=index->capacity ? index->capacity*2 : 256; + void *p=realloc(index->chunks,(size_t)cap*sizeof *index->chunks); + if (!p) return fail(w,UC2_UserFault,"cannot allocate shared chunk metadata"); + index->chunks=p; index->capacity=cap; + } + struct shared_chunk c={0}; + c.spool=spool; c.size=size; c.offset=(unsigned)w->output_size; + memcpy(c.digest,digest,32); + struct memory in={index->raw,0,size}; + struct compressed_chunk out={index->compressed,0}; + int r=uc2_compress(w->options.level,memory_cb,&in,chunk_output,&out,size,&c.checksum,&c.csize); + if (r<0) return fail(w,r,"shared chunk compression failed"); + c.method=w->options.level>=6 ? 10 : (unsigned short)w->options.level; + const unsigned char *data=index->compressed; + if (out.size>=size) { c.method=0; c.csize=size; data=index->raw; } + if (output_cb(w,data,c.csize)<0) return w->error; + *id=index->count; index->chunks[index->count++]=c; index->slots[slot]=index->count; + return 0; +} +static int write_chunked(uc2_writer *w) +{ + struct chunk_index index={0}; + index.raw=malloc(4*UC2_CHUNK_MAX); + if (!index.raw) return fail(w,UC2_UserFault,"cannot allocate chunk buffers"); + index.compressed=index.raw+UC2_CHUNK_MAX; + index.compare=index.raw+3*UC2_CHUNK_MAX; + for (unsigned i=0; icount; ++i) { + struct entry *e=&w->entries[i]; + if (e->id) continue; + e->master=1; + if (e->duplicate) { + const struct entry *first=&w->entries[e->duplicate-1]; + e->offset=first->offset; e->csize=first->csize; e->chunk_count=first->chunk_count; + if (e->chunk_count>UC2_CHUNK_LIMIT-w->stats.chunk_references) { + fail(w,UC2_WRITER_LIMIT,"shared chunk reference limit exceeded"); goto done; + } + w->stats.chunk_references+=e->chunk_count; w->stats.shared_bytes+=e->size; + continue; + } + for (unsigned pos=0; possize;) { + if (w->stats.chunk_references==UC2_CHUNK_LIMIT) { + fail(w,UC2_WRITER_LIMIT,"shared chunk reference limit exceeded"); goto done; + } + unsigned n=e->size-pos; + if (n>UC2_CHUNK_MAX) n=UC2_CHUNK_MAX; + if (load_at(w,e->spool+pos,index.raw,n)<0) goto done; + struct uc2_chunker c; size_t off=0,len=0; + uc2_chunker_init(&c,13,0,UC2_CHUNK_MAX); + uc2_chunker_next(&c,index.raw,n,&off,&len); + if (!len) { fail(w,UC2_InternalError,"chunker made no progress"); goto done; } + if (e->chunk_count==index.ref_capacity) { + unsigned cap=index.ref_capacity ? index.ref_capacity*2 : 256; + void *p=realloc(index.refs,(size_t)cap*sizeof *index.refs); + if (!p) { fail(w,UC2_UserFault,"cannot allocate chunk recipe"); goto done; } + index.refs=p; index.ref_capacity=cap; + } + if (shared_chunk(w,&index,e->spool+pos,(unsigned)len,&index.refs[e->chunk_count])<0) goto done; + ++e->chunk_count; ++w->stats.chunk_references; pos+=(unsigned)len; + } + unsigned char header[UC2_CHUNK_HEADER]={0}, record[UC2_CHUNK_RECORD]; + memcpy(header,UC2_CHUNK_MAGIC,8); w32(header+8,e->chunk_count); w32(header+12,e->size); + w32(header+16,e->chunk_count*UC2_CHUNK_RECORD); memcpy(header+24,e->digest,32); + struct uc2_sha256 sha; uc2_sha256_init(&sha); uc2_sha256_update(&sha,header,56); + for (unsigned j=0; jchunk_count; ++j) { + chunk_record(record,&index.chunks[index.refs[j]]); uc2_sha256_update(&sha,record,sizeof record); + } + uc2_sha256_final(&sha,header+56); + e->offset=(unsigned)w->output_size; + e->csize=UC2_CHUNK_HEADER+e->chunk_count*UC2_CHUNK_RECORD; + if (output_cb(w,header,sizeof header)<0) goto done; + for (unsigned j=0; jchunk_count; ++j) { + chunk_record(record,&index.chunks[index.refs[j]]); + if (output_cb(w,record,sizeof record)<0) goto done; + } + } +done: + w->stats.unique_chunks=index.count; + free(index.raw); free(index.chunks); free(index.slots); free(index.refs); + return w->error; +} static int assemble(uc2_writer *w) { if (fflush(w->input)!=0) return fail(w,UC2_UserFault,"temporary input flush failed"); - if (duplicates(w)<0 || group_files(w)<0) return w->error; - for (unsigned i=0; icount; ++i) if (w->entries[i].duplicate) { + if (duplicates(w)<0 || (!w->options.chunked && group_files(w)<0)) return w->error; + for (unsigned i=0; !w->options.chunked && icount; ++i) if (w->entries[i].duplicate) { struct entry *e=&w->entries[i]; e->master=w->entries[e->duplicate-1].master; struct master *m=&w->masters[e->master-2]; @@ -391,8 +541,11 @@ static int assemble(uc2_writer *w) w->output=tmpfile(); if (!w->output) return fail(w,UC2_UserFault,"cannot create temporary archive"); unsigned char header[29]={0}, super[49152], master_data[MASTER_MAX]; - if (uc2_get_supermaster(super,sizeof super)<0) return fail(w,UC2_InternalError,"SuperMaster decoding failed"); if (output_cb(w,header,sizeof header)<0) return w->error; + if (w->options.chunked) { + if (write_chunked(w)<0) return w->error; + } else { + if (uc2_get_supermaster(super,sizeof super)<0) return fail(w,UC2_InternalError,"SuperMaster decoding failed"); for (unsigned i=0; imaster_count; ++i) { struct master *m=&w->masters[i]; if (load_master(w,m,master_data)<0) return w->error; @@ -415,6 +568,7 @@ static int assemble(uc2_writer *w) int r=uc2_compress_ex(w->options.level,master_data,m->size,input_cb,w->input,output_cb,w,e->size,&e->checksum,&e->csize); if (r<0) return fail(w,r,"file compression failed"); } + } uint64_t cap=22+(uint64_t)39*w->master_count; for (unsigned i=0; icount; ++i) cap+=(w->entries[i].id ? 27 : 47)+22+strlen(w->entries[i].name); if (cap>CDIR_MAX) return fail(w,UC2_WRITER_LIMIT,"central directory exceeds 64 MiB"); @@ -437,7 +591,7 @@ static int assemble(uc2_writer *w) if (e->id) { w32(p,e->id); p+=4; } else { w32(p,e->size); p+=4; w16(p,e->checksum); p+=2; - w32(p,e->csize); p+=4; w16(p,method); p+=2; w32(p,e->master); p+=4; + w32(p,e->csize); p+=4; w16(p,w->options.chunked ? UC2_CHUNK_METHOD : method); p+=2; w32(p,e->master); p+=4; w32(p,1); p+=4; w32(p,e->offset); p+=4; } unsigned n=(unsigned)strlen(e->name)+1; @@ -454,7 +608,9 @@ static int assemble(uc2_writer *w) free(raw); if (r<0) return fail(w,r,"central directory compression failed"); unsigned total=(unsigned)w->output_size, length=total-13; w32(header,0x1a324355); w32(header+4,length); w32(header+8,length+0x01b2c3d4); - w32(header+13,1); w32(header+17,cdir_offset); w16(header+21,csum); w16(header+24,203); w16(header+26,200); + w32(header+13,1); w32(header+17,cdir_offset); w16(header+21,csum); + w16(header+24,w->options.chunked ? UC2_CHUNK_VERSION : 203); + w16(header+26,w->options.chunked ? UC2_CHUNK_VERSION : 200); if (seek_file(w->output,0)!=0 || fwrite(header,1,sizeof header,w->output)!=sizeof header || fflush(w->output)!=0) return fail(w,UC2_UserFault,"archive header write or flush failed"); w->stats.archive_size=total; diff --git a/pyproject.toml b/pyproject.toml index c14a734..3d3adb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build" [project] name = "uc2" -version = "3.0.0a10" +version = "3.0.0a11" description = "Read and create UltraCompressor II archives in Python, with a bundled native CLI" readme = "python/README.md" requires-python = ">=3.10" @@ -50,7 +50,7 @@ UC2_BUILD_PYTHON = "ON" UC2_BUILD_TESTS = "OFF" UC2_BUILD_LIBARCHIVE_PLUGIN = "OFF" UC2_BUILD_WCX = "OFF" -UC2_VERSION_LABEL = "3.0.0a10" +UC2_VERSION_LABEL = "3.0.0a11" [tool.cibuildwheel] build = "cp3{10,11,12,13,14}-*" diff --git a/python/README.md b/python/README.md index b18b076..52685e8 100644 --- a/python/README.md +++ b/python/README.md @@ -136,7 +136,7 @@ through EOF without seeking and leaves the input open. A supplied size must match exactly; detecting excess input can consume one extra byte. Inputs spool to anonymous temporary files as they are added. Compression runs -at close, using CDC-selected custom master dictionaries. Payload buffers have +at close, using CDC-selected custom master dictionaries by default. Payload buffers have fixed sizes; metadata grows with entries and chunks. Temporary disk space covers the uncompressed inputs and completed archive, plus the destination's sibling temporary file during the final copy. Duplicate file paths store @@ -212,3 +212,8 @@ Filesystem paths now resolve SAS and DOS UC2SEA sets without executing input. Keep all volumes together. Snapshot maintenance adds explicit `prune`, `repair` and `compact`; packed repositories require alpha10 or later. See the [repository guides](https://github.com/evvaletov/uc2/tree/main/docs). + +Optional `UC2Writer(..., chunked=True)` shares payload chunks across partly +changed files and stored versions. Use the full 3.0.0a11 or newer reader. +For method 11, `UC2Info.compress_size` counts the recipe only; shared payloads +are excluded. See the [chunk-sharing guide](../docs/chunk-sharing.md). diff --git a/python/tests/test_chunk_sharing.py b/python/tests/test_chunk_sharing.py new file mode 100644 index 0000000..b59a202 --- /dev/null +++ b/python/tests/test_chunk_sharing.py @@ -0,0 +1,207 @@ +"""Shared payloads preserve named files; malformed recipes fail explicitly.""" +import hashlib +import io +import random +import struct +import subprocess +import tarfile + +import pytest +from uc2 import UC2File, UC2Writer, BadArchiveError +from uc2._cli import executable + +MAGIC = b"UC2CHN1\0" +HEADER, RECORD = 88, 48 + + +@pytest.mark.parametrize("level", range(2, 10)) +def test_named_chunks_versions_and_codecs(tmp_path, level): + base = random.Random(7).randbytes(180000) + b"ABCD" * 22000 + changed = b"prefix change" + base[:160000] + b"X" + base[160000:] + path = tmp_path / "shared.uc2" + with UC2Writer(path, level=level, chunked=True) as writer: + writer.writestr("folder/café.bin", base) + writer.writestr("folder/café.bin", changed) + writer.writestr("empty", b"") + writer.writestr("copy", base) + with UC2File(io.BytesIO(path.read_bytes())) as archive: + files = [i for i in archive.infolist(all_versions=True) if not i.is_dir()] + assert len(files) == 4 + assert all(i.method == 11 for i in files) + assert archive.read("folder/café.bin", version=1) == base + assert archive.read("folder/café.bin") == changed + assert archive.read("copy") == base + assert archive.read("empty") == b"" + assert writer.stats["chunk_references"] > writer.stats["unique_chunks"] + assert writer.stats["shared_bytes"] > len(base) + assert writer.stats["masters"] == 0 + + +def test_partial_edits_share_beyond_dictionary_window(tmp_path): + base = random.Random(341).randbytes(512 * 1024) + changed = base[:200000] + b"an inserted line\n" + base[200000:] + sizes = {} + for chunked in (False, True): + path = tmp_path / f"{chunked}.uc2" + with UC2Writer(path, chunked=chunked) as writer: + writer.writestr("old.bin", base) + writer.writestr("new.bin", changed) + sizes[chunked] = path.stat().st_size + with UC2File(path) as archive: + assert archive.namelist() == ["old.bin", "new.bin"] + assert archive.read("old.bin") == base + assert archive.read("new.bin") == changed + if chunked: + assert writer.stats["shared_bytes"] > 400000 + else: + assert writer.stats["chunk_references"] == 0 + assert sizes[True] < sizes[False] * 0.75 + + +def recipe_digest(data, start): + count = struct.unpack_from(" bool: diff --git a/python/writer.c b/python/writer.c index 79b9abf..ecb8833 100644 --- a/python/writer.c +++ b/python/writer.c @@ -30,14 +30,14 @@ static void set_error(Writer *self, int result) } static int writer_init(Writer *self, PyObject *args, PyObject *kwargs) { - int level=4; unsigned maximum=100000; long long total=UINT32_MAX; - static char *names[]={"level","max_entries","max_total_size",NULL}; + int level=4,chunked=0; unsigned maximum=100000; long long total=UINT32_MAX; + static char *names[]={"level","max_entries","max_total_size","chunked",NULL}; if (self->initialized) { PyErr_SetString(PyExc_RuntimeError,"writer cannot be initialized twice"); return -1; } - if (!PyArg_ParseTupleAndKeywords(args,kwargs,"|iIL",names,&level,&maximum,&total)) return -1; + if (!PyArg_ParseTupleAndKeywords(args,kwargs,"|iILp",names,&level,&maximum,&total,&chunked)) return -1; if (level<2 || level>9 || maximum<1 || maximum>1000000 || total<1 || total>UINT32_MAX) { PyErr_SetString(PyExc_ValueError,"invalid writer level or limits"); return -1; } - struct uc2_writer_options options={level,maximum,(uint64_t)total}; + struct uc2_writer_options options={level,maximum,(uint64_t)total,chunked}; self->initialized=1; self->writer=uc2_writer_create(&options); if (!self->writer) { PyErr_SetString(PyExc_OSError,"cannot allocate writer or create temporary input file"); return -1; } return 0; @@ -108,8 +108,10 @@ static PyObject *writer_finish(Writer *self, PyObject *destination) if (r<0 || PyErr_Occurred()) { set_error(self,r); return NULL; } struct uc2_writer_stats stats; uc2_writer_get_stats(self->writer,&stats); uc2_writer_free(self->writer); self->writer=NULL; - return Py_BuildValue("{sI,sI,sI,sI}","files",stats.files,"directories",stats.directories, - "masters",stats.masters,"archive_size",stats.archive_size); + return Py_BuildValue("{sI,sI,sI,sI,sI,sI,sK}","files",stats.files,"directories",stats.directories, + "masters",stats.masters,"archive_size",stats.archive_size, + "unique_chunks",stats.unique_chunks,"chunk_references",stats.chunk_references, + "shared_bytes",(unsigned long long)stats.shared_bytes); } static PyMethodDef methods[]={ {"abort",(PyCFunction)writer_abort,METH_NOARGS,NULL}, diff --git a/scripts/benchmark_collections.py b/scripts/benchmark_collections.py index 9ec3a43..c21923e 100755 --- a/scripts/benchmark_collections.py +++ b/scripts/benchmark_collections.py @@ -125,15 +125,16 @@ def main(): tar.addfile(info, f) (root / f'{kind}.manifest.json').write_text(json.dumps(expected, indent=2)+'\n') for trial in range(args.repeat): - for method in ['uc2-files', 'tar-uc2', '7z', 'tar-zstd']: + for method in ['uc2-files', 'uc2-chunks', 'tar-uc2', '7z', 'tar-zstd']: work = root / f'{kind}-{method}-{trial}' work.mkdir() archive = work / ('archive.7z' if method == '7z' else 'archive') restored = work / 'restored' restored.mkdir() with tar_path.open('rb') as stream: - if method == 'uc2-files': - cmd = [uc2, '-q', '-w', '-L', '4', archive, *sorted(expected)] + if method in ('uc2-files', 'uc2-chunks'): + cmd = [uc2, *(['--chunked'] if method == 'uc2-chunks' else []), + '-q', '-w', '-L', '4', archive, *sorted(expected)] inp = None elif method == 'tar-uc2': cmd, inp = [uc2, '--ingest', archive], stream @@ -142,7 +143,7 @@ def main(): else: cmd, inp = ['zstd', '-q', '-3', '-T1', '-o', archive], stream enc, enc_rss = run(cmd, folder, work, 'create', stdin=inp) - if method == 'uc2-files': + if method in ('uc2-files', 'uc2-chunks'): dec, dec_rss = run([uc2, '-q', '-d', restored, archive], root, work, 'restore') elif method == '7z': dec, dec_rss = run([seven, 'x', '-y', '-o'+str(restored), archive], root, work, 'restore') diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d8dd781..b505a2f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -232,6 +232,11 @@ target_link_libraries(test_writer PRIVATE uc2) target_compile_features(test_writer PRIVATE c_std_99) add_test(NAME writer COMMAND test_writer) +add_executable(test_chunk_corruption src/test_chunk_corruption.c) +target_link_libraries(test_chunk_corruption PRIVATE uc2) +target_compile_features(test_chunk_corruption PRIVATE c_std_99) +add_test(NAME chunk_corruption COMMAND test_chunk_corruption) + add_executable(test_sea src/test_sea.c) target_link_libraries(test_sea PRIVATE uc2) target_compile_features(test_sea PRIVATE c_std_99) diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 5e9e032..bbeb5c7 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -59,3 +59,10 @@ Two layers run automatically: `regressions/turbo-empty-checksum.uc2` is a generated method80 archive with a zero-length file and deliberately wrong file checksum. Libarchive must verify the stream even when the declared file size is zero. + +`fuzz_chunks.c` replays method-11 archives and recomputes the bounded recipe +digest after mutations, reaching descriptor and compressed-payload checks. +`chunk-raw.uc2`, `chunk-huffman.uc2` and `chunk-rans.uc2` are generated a11 +fixtures containing pseudorandom data or repeated test strings. The regular +read-path replay also opens them. CI builds harnesses with `NDEBUG`, matching +the production library and suppressing diagnostic output. diff --git a/tests/fuzz/fuzz_chunks.c b/tests/fuzz/fuzz_chunks.c new file mode 100644 index 0000000..8d66cb6 --- /dev/null +++ b/tests/fuzz/fuzz_chunks.c @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later + * Mutate real method-11 archives, repairing their recipe digest so mutations + * reach descriptor and codec validation instead of stopping at the table hash. + * The central directory and all payload hashes remain independently checked. */ +#define LLVMFuzzerTestOneInput extract_archive +#include "fuzz_extract.c" +#undef LLVMFuzzerTestOneInput +#include +#include + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + if (size > (1u << 20)) return 0; + uint8_t *copy = malloc(size ? size : 1); + if (!copy) return 0; + memcpy(copy, data, size); + for (size_t i = 29; i + UC2_CHUNK_HEADER <= size; ++i) { + if (memcmp(copy + i, UC2_CHUNK_MAGIC, 8)) continue; + const uint8_t *p = copy + i + 8; + uint32_t count = p[0] | (uint32_t)p[1] << 8 | + (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; + if (count > (size - i - UC2_CHUNK_HEADER) / UC2_CHUNK_RECORD) continue; + struct uc2_sha256 hash; + uc2_sha256_init(&hash); + uc2_sha256_update(&hash, copy + i, 56); + uc2_sha256_update(&hash, copy + i + UC2_CHUNK_HEADER, count * UC2_CHUNK_RECORD); + uc2_sha256_final(&hash, copy + i + 56); + } + extract_archive(copy, size); + free(copy); + return 0; +} diff --git a/tests/fuzz/seeds/chunk-huffman.uc2 b/tests/fuzz/seeds/chunk-huffman.uc2 new file mode 100644 index 0000000..71e8a93 Binary files /dev/null and b/tests/fuzz/seeds/chunk-huffman.uc2 differ diff --git a/tests/fuzz/seeds/chunk-rans.uc2 b/tests/fuzz/seeds/chunk-rans.uc2 new file mode 100644 index 0000000..99c4025 Binary files /dev/null and b/tests/fuzz/seeds/chunk-rans.uc2 differ diff --git a/tests/fuzz/seeds/chunk-raw.uc2 b/tests/fuzz/seeds/chunk-raw.uc2 new file mode 100644 index 0000000..1dfa98d Binary files /dev/null and b/tests/fuzz/seeds/chunk-raw.uc2 differ diff --git a/tests/scripts/benchmark_chunk_sharing.py b/tests/scripts/benchmark_chunk_sharing.py new file mode 100644 index 0000000..7755692 --- /dev/null +++ b/tests/scripts/benchmark_chunk_sharing.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Compare named-file sharing on edited public C sources; verify every restore.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import subprocess +import time + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def measure(command, cwd, report): + started = time.perf_counter() + with report.with_suffix('.log').open('wb') as log: + subprocess.run(['/usr/bin/time', '-f', '%M', '-o', str(report), + *map(str, command)], cwd=cwd, stdout=log, stderr=log, + timeout=600, check=True) + return round(time.perf_counter() - started, 3), int(report.read_text()) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--source', type=Path, required=True, help='libarchive source checkout') + ap.add_argument('--uc2', type=Path, required=True) + ap.add_argument('--output', type=Path, required=True, help='new output directory') + ap.add_argument('--repeat', type=int, default=3) + args = ap.parse_args() + root, source, cli = args.output.resolve(), args.source.resolve(), args.uc2.resolve() + root.mkdir(parents=True, exist_ok=False) + inputs = root / 'input' + inputs.mkdir() + paths = sorted((source / 'libarchive').glob('archive_read_support_format_*.c')) + paths = sorted(paths, key=lambda p: (-p.stat().st_size, p.name))[:12] + if len(paths) != 12 or args.repeat < 1: + raise SystemExit('Need 12 libarchive format sources and repeat >= 1') + provenance = {} + for path in paths: + data = path.read_bytes() + provenance[path.name] = digest(path) + for version in (0, 1): + item = inputs / f'{version}-{path.name}' + position = len(data) // 2 + changed = data[:position] + b'\n/* benchmark inserted edit */\n' + data[position:] + item.write_bytes(changed if version else data) + os.utime(item, (946684800, 946684800)) + expected = {p.name: digest(p) for p in inputs.iterdir()} + results = [] + for trial in range(args.repeat): + for chunked in (False, True): + name = f'{trial}-' + ('chunks' if chunked else 'classic') + archive, restored = root / (name + '.uc2'), root / (name + '-restored') + restored.mkdir() + flags = ['--chunked'] if chunked else [] + enc, rss = measure([cli, *flags, '-w', '-L', '4', archive, *sorted(expected)], + inputs, root / (name + '-create.time')) + dec, dec_rss = measure([cli, '-q', '-d', restored, archive], root, + root / (name + '-restore.time')) + actual = {p.name: digest(p) for p in restored.iterdir()} + if actual != expected: + raise AssertionError('Restored bytes differ') + row = dict(trial=trial, chunked=chunked, archive_bytes=archive.stat().st_size, + create_seconds=enc, restore_seconds=dec, peak_kib=rss, + restore_peak_kib=dec_rss, verified=True) + results.append(row) + print(json.dumps(row), flush=True) + report = dict(platform=platform.platform(), cli_sha256=digest(cli), + source_revision=subprocess.check_output( + ['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip(), + source_hashes=provenance, input_bytes=sum(p.stat().st_size for p in inputs.iterdir()), + files=len(expected), results=results) + (root / 'results.json').write_text(json.dumps(report, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tests/scripts/check_bsdtar.py b/tests/scripts/check_bsdtar.py index 50656ee..572cbd1 100755 --- a/tests/scripts/check_bsdtar.py +++ b/tests/scripts/check_bsdtar.py @@ -20,14 +20,15 @@ with tempfile.TemporaryDirectory(prefix='uc2-bsdtar-') as temp: (src/'folder'/'long filename.txt').write_bytes(b'libarchive integration\n' * 10000) (src/'empty').write_bytes(b'') (src/'root.bin').write_bytes(bytes(range(256))*400) - for level in ((4,) if '--legacy-only' in sys.argv[3:] else (4, 6)): - archive=root/f'level{level}.uc2' - subprocess.run([uc2,'-qw','-L',str(level),str(archive),'folder','empty','root.bin'],cwd=src,check=True) + for level, chunked in (((4, False),) if '--legacy-only' in sys.argv[3:] else + ((4, False), (6, False), (4, True), (6, True))): + archive=root/f'level{level}-{chunked}.uc2' + subprocess.run([uc2,*(['--chunked'] if chunked else []),'-qw','-L',str(level),str(archive),'folder','empty','root.bin'],cwd=src,check=True) listed=subprocess.check_output([tar,'-tf',str(archive)]).decode().splitlines() for name in ('folder/long filename.txt','empty','root.bin'): if name not in listed: raise AssertionError('missing '+name) for pipe in (False,True): - out=root/f'out-{level}-{pipe}';out.mkdir() + out=root/f'out-{level}-{chunked}-{pipe}';out.mkdir() with archive.open('rb') as inp: subprocess.run([tar,'-xf','-' if pipe else str(archive),'-C',str(out)], stdin=inp if pipe else None,check=True) @@ -35,7 +36,7 @@ with tempfile.TemporaryDirectory(prefix='uc2-bsdtar-') as temp: if path.is_file() and sha(path)!=sha(out/path.relative_to(src)): raise AssertionError('restore mismatch') # Skip earlier entries and extract just one member through libarchive. - single=root/f'single-{level}';single.mkdir() + single=root/f'single-{level}-{chunked}';single.mkdir() subprocess.run([tar,'-xf',str(archive),'-C',str(single),'folder/long filename.txt'],check=True) if sha(src/'folder/long filename.txt') != sha(single/'folder/long filename.txt'): raise AssertionError('selective extraction mismatch') diff --git a/tests/src/test_chunk_corruption.c b/tests/src/test_chunk_corruption.c new file mode 100644 index 0000000..daa1dc1 --- /dev/null +++ b/tests/src/test_chunk_corruption.c @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(x) do { int ok=!!(x); assert(ok); (void)ok; } while (0) +struct bytes { unsigned char *p; unsigned n, pos; }; +static int append(void *ctx,const void *p,unsigned n) +{ + struct bytes *b=ctx; + void *next=realloc(b->p,b->n+n); CHECK(next || !n); + b->p=next; memcpy(b->p+b->n,p,n); b->n+=n; return 0; +} +static int read_at(void *ctx,unsigned pos,void *p,unsigned n) +{ + struct bytes *b=ctx; + if (pos>=b->n) return 0; + if (n>b->n-pos) n=b->n-pos; + memcpy(p,b->p+pos,n); return (int)n; +} +static int read_next(void *ctx,void *p,unsigned n) +{ struct bytes *b=ctx; int got=read_at(b,b->pos,p,n); b->pos+=(unsigned)got; return got; } +static void *allocate(void *ctx,unsigned n) { (void)ctx; return malloc(n); } +static void release(void *ctx,void *p) { (void)ctx; free(p); } +static void w16(unsigned char *p,unsigned n) { p[0]=(unsigned char)n; p[1]=(unsigned char)(n>>8); } +static void w32(unsigned char *p,unsigned n) { w16(p,n); w16(p+2,n>>16); } +static unsigned sum(const unsigned char *p,unsigned n) +{ unsigned s=0xa55a; for (unsigned i=0;iword=(b->word<<1)|((n>>--count)&1); ++b->count; + if (b->count==16) { + unsigned char word[2]; w16(word,b->word); append(&b->out,word,2); + b->word=b->count=0; + } + } +} +static struct bytes rans_payload(int fault) +{ + /* fault 1 omits EOB; fault 2 puts a length symbol in primary position. */ + int symbols[]={65,316,315,316}, count=4; + if (!fault) { symbols[1]=315; symbols[2]=316; count=3; } + if (fault==1) count=1; + uint32_t frequencies[344]={0}; frequencies[66]=1; + for (int i=0;i=0;--i) uc2_rans_encode(&enc,symbols[i]); + unsigned char *data; size_t size=uc2_rans_enc_finish(&enc,&data); + uc2_rans_enc_free(&enc); CHECK(size<=65535); + struct bits b={0}; put(&b,1,1); put(&b,(unsigned)count,16); put(&b,(unsigned)size,16); + for (unsigned i=0;i<344;++i) put(&b,table.freq[i],12); + for (unsigned i=0;i*left) n=*left; + memset(p,'A',n); *left-=n; return (int)n; +} +static void oversized_cdir(void) +{ + /* Highly compressible metadata must stop at the absolute byte ceiling, + * reporting damaged input rather than falsely blaming an I/O callback. */ + unsigned left=64u*1024u*1024u+1, encoded; + unsigned char header[29]={0}, crec[10]={0}; + struct bytes compressed={0}, archive={0}; unsigned short checksum; + CHECK(uc2_compress(6,repeated,&left,append,&compressed,left,&checksum,&encoded)==0); + unsigned total=39+compressed.n; + w32(header,0x1a324355); w32(header+4,total-13); w32(header+8,total-13+0x01b2c3d4); + w32(header+13,1); w32(header+17,29); w16(header+21,checksum); + w16(header+24,203); w16(header+26,200); w16(crec+4,10); w32(crec+6,1); + append(&archive,header,sizeof header); append(&archive,crec,sizeof crec); + append(&archive,compressed.p,compressed.n); + struct uc2_io io={read_at,allocate,release,NULL}; uc2_handle h=uc2_open(&io,&archive); CHECK(h); + struct uc2_entry entry; CHECK(uc2_read_cdir(h,&entry)==UC2_Damaged); + uc2_close(h); free(archive.p); free(compressed.p); +} +int main(void) +{ + struct bytes raw={(unsigned char *)"A",1,0}; + check_archive(raw,0,136,0); check_archive(raw,0,UINT_MAX,1); check_archive(raw,0,137,1); + for (int fault=0;fault<3;++fault) { + struct bytes payload=rans_payload(fault); check_archive(payload,10,136,fault!=0); free(payload.p); + } + oversized_cdir(); + puts("chunk corruption: cdir sizes, rANS primary symbols and EOB validated"); return 0; +} diff --git a/tests/src/test_writer.c b/tests/src/test_writer.c index f059d7a..1c3e2c5 100644 --- a/tests/src/test_writer.c +++ b/tests/src/test_writer.c @@ -30,18 +30,20 @@ static int read_at(void *ctx, unsigned pos, void *data, unsigned n) if (n>b->size-pos) n=b->size-pos; memcpy(data,b->data+pos,n); return (int)n; } -static void *allocate(void *ctx, unsigned n) { return malloc(n); } +static int refuse_allocation; +static void *allocate(void *ctx, unsigned n) { return refuse_allocation ? NULL : malloc(n); } static void release(void *ctx, void *p) { free(p); } static struct uc2_io io={read_at,allocate,release,NULL}; static int fail_read(void *ctx,void *p,unsigned n) { return -1; } static int bad_read(void *ctx,void *p,unsigned n) { return (int)n+1; } static int fail_write(void *ctx,const void *p,unsigned n) { return -1; } -static void roundtrip(int level) +static void roundtrip(int level, int chunked) { unsigned char data[70001]; for (unsigned i=0; istats.unique_chunks && stats.shared_bytes>=sizeof data); uc2_writer_free(w); uc2_handle h=uc2_open(&io,&encoded); CHECK(h); struct uc2_entry entries[4]; unsigned count=0; @@ -66,6 +69,7 @@ static void roundtrip(int level) } CHECK(count==4 && uc2_finish_cdir(h,NULL)==0); for (unsigned i=0; i
IncludePathVersionSizeCheck

-

Keep the original

Stored versions remain separate. Verification checks UC2 checksums; it does not establish who created an archive.

Working with a collection?

Use uc2-preserve for batch inventory and verified ZIP/tar migration, including original paths and metadata.

Compatibility

Legacy Huffman, Turbo/method 80 and UC2 v3 rANS are supported. Individual files are limited to 64 MiB here.

+

Keep the original

Stored versions remain separate. Verification checks UC2 checksums; it does not establish who created an archive.

Working with a collection?

Use uc2-preserve for batch inventory and verified ZIP/tar migration, including original paths and metadata.

Compatibility

Legacy Huffman, Turbo/method 80 and UC2 v3 rANS and a11 shared chunks are supported. Individual files are limited to 64 MiB here.

diff --git a/web/test_browser.py b/web/test_browser.py index 3f86ef2..8459a2a 100644 --- a/web/test_browser.py +++ b/web/test_browser.py @@ -8,6 +8,7 @@ from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer import json from pathlib import Path import re +import random import tempfile from threading import Thread from urllib.parse import urlsplit @@ -126,6 +127,24 @@ def main(): downloaded.value.save_as(target) assert target.read_bytes() == reference.read(info), info.filename + # Exercise shared raw/compressed chunks through the WASM decoder and ZIP exporter. + base = random.Random(11).randbytes(80000) + b'chunk-sharing' * 8000 + for level in (4, 8): + shared = scratch / f'shared-{level}.uc2' + with UC2Writer(shared, level=level, chunked=True) as writer: + writer.writestr('old.bin', base) + writer.writestr('new.bin', base[:40000] + b'inserted' + base[40000:]) + assert writer.stats['shared_bytes'] > 100000 + page.locator('#archive').set_input_files(shared) + expect(page.locator('#archive-name')).to_have_text(shared.name, timeout=30000) + page.get_by_role('button', name='Verify all versions').click() + expect(page.locator('#status')).to_contain_text('All 2 stored files passed', timeout=30000) + page.get_by_role('button', name='Select matching files').click() + with page.expect_download() as downloaded: + page.get_by_role('button', name='Save selected ZIP').click() + downloaded.value.save_as(scratch / f'shared-{level}.zip') + check_zip(scratch / f'shared-{level}.zip', shared) + modern = scratch / 'versions.uc2' with UC2Writer(modern, level=8) as writer: writer.writestr('folder/café.txt', b'old version') @@ -373,7 +392,7 @@ def main(): if method != 'GET' or not target.startswith((url, 'blob:' + url))] assert not unexpected, unexpected browser.close() - print(f'{args.engine} {browser.version} checks passed: original fixtures, rANS, versions, verified ZIP/CRC/hash/paths, selection, limits, corruption, cancellation, timeout recovery, offline use/repair/update/integrity, mobile layout, and no uploads.') + print(f'{args.engine} {browser.version} checks passed: original fixtures, rANS, shared chunks, versions, verified ZIP/CRC/hash/paths, selection, limits, corruption, cancellation, timeout recovery, offline use/repair/update/integrity, mobile layout, and no uploads.') finally: server.shutdown() server.server_close()