diff --git a/docs/usage/export-tar.rst.inc b/docs/usage/export-tar.rst.inc index 7bb8cb19f5..8bc2de07ae 100644 --- a/docs/usage/export-tar.rst.inc +++ b/docs/usage/export-tar.rst.inc @@ -12,39 +12,41 @@ borg export-tar .. class:: borg-options-table - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | **positional arguments** | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``NAME`` | specify the archive name | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``FILE`` | output tar file. "-" to write to stdout instead. | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``PATH`` | paths to extract; patterns are supported | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | **options** | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``--tar-filter`` | filter program to pipe data through | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``--list`` | output verbose list of items (files, dirs, ...) | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``--tar-format FMT`` | select tar format: BORG, PAX or GNU | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | .. class:: borg-common-opt-ref | - | | - | :ref:`common_options` | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | **Include/Exclude options** | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``-e PATTERN``, ``--exclude PATTERN`` | exclude paths matching PATTERN | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``--exclude-from EXCLUDEFILE`` | read exclude patterns from EXCLUDEFILE, one per line | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``--pattern PATTERN`` | include/exclude paths matching PATTERN | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``--patterns-from PATTERNFILE`` | read include/exclude patterns from PATTERNFILE, one per line | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ - | | ``--strip-components NUMBER`` | Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. | - +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | **positional arguments** | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``NAME`` | specify the archive name | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``FILE`` | output tar file. "-" to write to stdout instead. | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``PATH`` | paths to extract; patterns are supported | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | **options** | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``--tar-filter`` | filter program to pipe data through | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``--list`` | output verbose list of items (files, dirs, ...) | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``--tar-format FMT`` | select tar format: BORG, PAX or GNU | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``--sparse`` | write sparse tar members (GNU sparse format 1.0) for files containing all-zero chunks (BORG and PAX formats only) | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | **Include/Exclude options** | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``-e PATTERN``, ``--exclude PATTERN`` | exclude paths matching PATTERN | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``--exclude-from EXCLUDEFILE`` | read exclude patterns from EXCLUDEFILE, one per line | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``--pattern PATTERN`` | include/exclude paths matching PATTERN | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``--patterns-from PATTERNFILE`` | read include/exclude patterns from PATTERNFILE, one per line | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ + | | ``--strip-components NUMBER`` | Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. | + +-------------------------------------------------------+---------------------------------------+-------------------------------------------------------------------------------------------------------------------+ .. raw:: html @@ -68,6 +70,7 @@ borg export-tar --tar-filter filter program to pipe data through --list output verbose list of items (files, dirs, ...) --tar-format FMT select tar format: BORG, PAX or GNU + --sparse write sparse tar members (GNU sparse format 1.0) for files containing all-zero chunks (BORG and PAX formats only) :ref:`common_options` @@ -120,7 +123,20 @@ Depending on the ``--tar-format`` option, these formats are created: | | | no ACLs/xattrs/bsdflags | +--------------+---------------------------+----------------------------+ -A ``--sparse`` option (as found in borg extract) is not supported. +With ``--sparse``, files whose content contains runs of all-zero chunks are written +as sparse tar members (GNU sparse format 1.0, as GNU tar creates it in POSIX mode), +storing only a hole map and the non-zero data. This requires ``--tar-format BORG`` +or ``PAX``. Such tarballs can be much smaller for sparse files (e.g. disk images) +and extract to sparse files again with GNU tar's or bsdtar's sparse support +(as well as with ``borg import-tar`` / ``borg extract --sparse``). +Notes: hole detection works at the granularity of borg's content chunks (it does not +depend on the original file having been a sparse file - but some short or unaligned +zero runs may be stored literally); sparse-unaware tar implementations will extract +a member as ``GNUSparseFile.0/`` containing the raw hole map and data (the +same caveat applies to tarballs created by GNU tar); for members needing >= 8 GiB +of stored (non-hole) data, the stored size is base-256 encoded in the tar header +(the GNU/star encoding of big numbers, understood by GNU tar, libarchive/bsdtar +and python) - logical file sizes are unlimited anyway. By default the entire archive is extracted but a subset of files and directories can be selected by passing a list of ``PATHs`` as arguments. diff --git a/docs/usage/import-tar.rst.inc b/docs/usage/import-tar.rst.inc index 12a949d678..b8e0bf9c72 100644 --- a/docs/usage/import-tar.rst.inc +++ b/docs/usage/import-tar.rst.inc @@ -110,7 +110,9 @@ stdout. Most documentation of borg create applies. Note that this command does not support excluding files. -A ``--sparse`` option (as found in borg create) is not supported. +A ``--sparse`` option (as found in borg create) is not needed: sparse members in +input tarballs (old GNU and PAX sparse formats) are read correctly and their +holes are stored as deduplicated all-zero chunks. About tar formats and metadata conservation or loss, please see ``borg export-tar``. diff --git a/docs/usage/tar.rst b/docs/usage/tar.rst index f20ea8c673..d5d64faf93 100644 --- a/docs/usage/tar.rst +++ b/docs/usage/tar.rst @@ -27,6 +27,9 @@ Examples # Remote extraction via 'tarpipe' $ borg export-tar Monday - | ssh somewhere "cd extracted; tar x" + # Export sparse files (e.g. disk images) as sparse tar members (GNU sparse format 1.0) + $ borg export-tar --sparse disk-images disk-images.tar + Archives transfer script ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -54,4 +57,5 @@ Please note: - the pipe is dumb, there is no data or transfer time reduction there due to deduplication - maybe add compression - pipe over ssh for remote transfer -- no special sparse file support +- maybe add ``--sparse`` to the export-tar command, so runs of all-zero chunks + travel as a compact sparse map instead of literal zeros diff --git a/src/borg/archive.py b/src/borg/archive.py index 23dc784323..b907cdccb7 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -371,23 +371,8 @@ def fetch_many(self, chunks, ro_type=None, replacement_chunk=True): sizes = [None] * len(ids) else: raise TypeError(f"unsupported or mixed element types: {chunks}") - # All-zero chunks can be served directly from the zeros constant, without repository - # access, by comparing against the (memoized) id of an all-zero chunk of same size. - # Only compute that id for ids occurring repeatedly within this stream: a repeated id - # means repeating plaintext, which usually is a run of zeros (e.g. the "holes" of a - # sparse file, see issue #1678) - and the repetition also keeps the memoization - # effective, as it bounds the computations to a few chunk sizes. - id_hash = self.repo_objs.key.id_hash - counts = Counter(ids) - zero_flags = [] - for id, size in zip(ids, sizes): - if size is None or not 0 < size <= len(zeros): - zero_flags.append(False) - elif counts[id] > 1: - zero_flags.append(id == zero_chunk_id(id_hash, size)) - else: - # unique id: only compare against already memoized zero chunk ids (cheap). - zero_flags.append(id == zero_chunk_ids.get((id_hash, size))) + # All-zero chunks can be served directly from the zeros constant, without repository access. + zero_flags = zero_chunk_flags(ids, sizes, self.repo_objs.key.id_hash) fetch_ids = [id for id, zero in zip(ids, zero_flags) if not zero] fetched = self.repository.get_many(fetch_ids, raise_missing=False) for id, size, zero in zip(ids, sizes, zero_flags): @@ -1275,6 +1260,30 @@ def zero_chunk_id(id_hash, size): return chunk_id +def zero_chunk_flags(ids, sizes, id_hash): + """ + Return a list of bools telling whether the chunk with ids[i] / sizes[i] is an all-zero chunk, + detected by comparing against the (memoized) id of an all-zero chunk of same size. + + Only compute that id for ids occurring repeatedly within this stream: a repeated id + means repeating plaintext, which usually is a run of zeros (e.g. the "holes" of a + sparse file, see issue #1678) - and the repetition also keeps the memoization + effective, as it bounds the computations to a few chunk sizes. For unique ids, only + already memoized zero chunk ids are compared against (cheap), so the detection may + have false negatives. + """ + counts = Counter(ids) + zero_flags = [] + for id, size in zip(ids, sizes): + if size is None or not 0 < size <= len(zeros): + zero_flags.append(False) + elif counts[id] > 1: + zero_flags.append(id == zero_chunk_id(id_hash, size)) + else: + zero_flags.append(id == zero_chunk_ids.get((id_hash, size))) + return zero_flags + + def cached_hash(chunk, id_hash): allocation = chunk.meta["allocation"] if allocation == CH_DATA: diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index f4a49ed627..18fd8f3feb 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -11,7 +11,7 @@ else: from backports import zstd -from ..archive import Archive, TarfileObjectProcessors, ChunksProcessor +from ..archive import Archive, TarfileObjectProcessors, ChunksProcessor, zero_chunk_flags, zero_chunk_id from ..compress import get_zstd_mt_workers from ..constants import * # NOQA from ..helpers import Error @@ -156,6 +156,140 @@ def item_to_paxheaders(format, item): return ph +def chunks_to_sparse_info(chunks, zero_flags): + """Compute a GNU sparse map from an item's chunk list. + + *zero_flags* tells for each chunk in *chunks* whether it is an all-zero chunk; + runs of all-zero chunks become the holes of the sparse file. Holes are shrunk + to whole 512-byte tar blocks (a trailing hole may end unaligned at the file's + end): GNU tar's sparse reader processes the data segments block-wise, so it + desyncs on unaligned segments - and this also matches what GNU tar itself + produces, as its own hole detection is block-granular. The zero bytes shaved + off the hole edges become part of the neighboring data segments. + + Return a tuple (map_entries, stream_plan, realsize): *map_entries* is a list of + (offset, length) data segments - with a terminating (realsize, 0) entry if the + file ends in a hole, like GNU tar creates it -, *stream_plan* tells how to + produce the data segments' bytes: a list of ChunkListEntry (a chunk to fetch and + emit completely) and int (a count of zero bytes to emit literally) elements, + *realsize* is the logical file size. + Return None if there is no hole (the file shall be a normal dense tar member). + """ + if not chunks or all(not zero for zero in zero_flags): + return None + realsize = sum(chunk.size for chunk in chunks) + # runs of all-zero chunks, shrunk to 512-byte block alignment, become the holes. + holes = [] + + def add_hole(start, end): + start = -(-start // tarfile.BLOCKSIZE) * tarfile.BLOCKSIZE + if end != realsize: # a trailing hole may end unaligned, gtar just truncates the file + end = end // tarfile.BLOCKSIZE * tarfile.BLOCKSIZE + if end > start: + holes.append((start, end)) + + offset = 0 + run_start = None # start offset of the current run of all-zero chunks + for chunk, zero in zip(chunks, zero_flags): + if zero and run_start is None: + run_start = offset + elif not zero and run_start is not None: + add_hole(run_start, offset) + run_start = None + offset += chunk.size + if run_start is not None: + add_hole(run_start, realsize) + if not holes: + return None # all zero runs were too short to make a block-aligned hole + # the data segments are the complement of the holes. + map_entries = [] + pos = 0 + for hole_start, hole_end in holes: + if hole_start > pos: + map_entries.append((pos, hole_start - pos)) + pos = hole_end + if pos < realsize: + map_entries.append((pos, realsize - pos)) + else: + # the file ends in a hole: terminate the map like GNU tar does. + map_entries.append((realsize, 0)) + # plan the emission of the data segments' bytes, chunk by chunk: data chunks are + # emitted completely, all-zero chunks only with their parts sticking out of the + # hole (shaved-off hole edges and the chunks of too-short zero runs) - those are + # emitted as literal zero bytes, so all-zero chunks never need to be fetched. + stream_plan = [] + offset = 0 + hole_idx = 0 + for chunk, zero in zip(chunks, zero_flags): + chunk_end = offset + chunk.size + if not zero: + stream_plan.append(chunk) + else: + pos = offset + while pos < chunk_end: + while hole_idx < len(holes) and holes[hole_idx][1] <= pos: + hole_idx += 1 + if hole_idx < len(holes) and holes[hole_idx][0] <= pos: + pos = min(holes[hole_idx][1], chunk_end) # covered by a hole: emit nothing + else: + emit_end = min(holes[hole_idx][0], chunk_end) if hole_idx < len(holes) else chunk_end + stream_plan.append(emit_end - pos) + pos = emit_end + offset = chunk_end + return map_entries, stream_plan, realsize + + +class SparseTarInfo(tarfile.TarInfo): + """TarInfo for GNU sparse format 1.0 members. + + A sparse member must never get a pax "size" record: readers recalculate the offset of + the next header from it *after* the sparse processing already consumed the sparse map, + so they desync on it - the pax-standard way of representing big sizes is broken for + sparse members. Thus, the member's stored size (sparse map + data segments) always + lives in the ustar size field: as standard octal while it fits (< 8 GiB, so those + members stay as standard-conforming as sparse members can be), else base-256 encoded + (the GNU/star big-number encoding; not POSIX, but GNU tar does the same for big + numbers in pax mode, and GNU tar, libarchive/bsdtar and python's tarfile all read it + in any tar format). The logical file size (GNU.sparse.realsize) is unlimited anyway. + + The base-256 field is patched in manually: python's tarfile writes base-256 only for + GNU_FORMAT - for PAX_FORMAT it conforms to POSIX and emits a pax "size" record, the + very thing that must be avoided here. + """ + + __slots__ = () # keep the TarInfo object layout, so plain TarInfos can be converted + + octal_size_limit = 8**11 # what fits into the 12-digit octal ustar size field + + def create_pax_header(self, info, encoding): + stored_size = info["size"] + if stored_size < self.octal_size_limit: + return super().create_pax_header(info, encoding) + info["size"] = 0 # suppresses both the automatic pax "size" record and the octal overflow + buf = super().create_pax_header(info, encoding) + # patch the base-256 encoded stored size into the ustar block's size field + # and recompute the block's checksum. + ustar = bytearray(buf[-tarfile.BLOCKSIZE :]) + ustar[124:136] = tarfile.itn(stored_size, 12, tarfile.GNU_FORMAT) + chksum = tarfile.calc_chksums(bytes(ustar))[0] + ustar[148:156] = bytes("%06o\0 " % chksum, "ascii") + return buf[: -tarfile.BLOCKSIZE] + bytes(ustar) + + +def gnu_sparse_10_map(map_entries): + """Serialize a sparse map as a GNU sparse format 1.0 map block. + + That is a series of decimal numbers delimited by newlines: the number of map + entries, then offset and length of each entry - NUL-padded to a multiple of + the 512 byte tar block size. It precedes the data segments in the tar member. + """ + numbers = [len(map_entries)] + for offset, length in map_entries: + numbers += [offset, length] + text = "".join(f"{number}\n" for number in numbers).encode() + return text + b"\0" * (-len(text) % tarfile.BLOCKSIZE) + + # Sentinel returned by get_tar_filter for zstd suffixes: (de)compress in-process # via the zstd module instead of piping through an external filter program. IN_PROCESS_ZSTD = "zstd (in-process)" @@ -214,6 +348,9 @@ def do_export_tar(self, args, repository, manifest, archive): """Export archive contents as a tarball""" self.output_list = args.output_list + if args.sparse and args.tar_format not in ("BORG", "PAX"): + raise Error("--sparse requires --tar-format BORG or PAX (GNU sparse format 1.0 members are PAX-based).") + # A quick note about the general design of tar_filter and tarfile; # The tarfile module of Python can provide some compression mechanisms # by itself, using the built-in gzip, bz2, and lzma modules (and "tar modes" @@ -276,11 +413,24 @@ def _export_tar(self, args, archive, tarstream): else: pi = None - def item_content_stream(item): + def sparse_chunk_iterator(stream_plan, map_bytes): + """Generate a sparse member's payload: the sparse map, then the data segments' + bytes as told by *stream_plan* (chunks to fetch, counts of literal zero bytes).""" + yield map_bytes + data_chunks = [entry for entry in stream_plan if not isinstance(entry, int)] + fetched = archive.pipeline.fetch_many(data_chunks, ro_type=ROBJ_FILE_STREAM) + for entry in stream_plan: + yield zeros[:entry] if isinstance(entry, int) else next(fetched) + + def item_content_stream(item, stream_plan=None, map_bytes=None): """ - Return a file-like object that reads from the chunks of *item*. + Return a file-like object that reads from the chunks of *item* + (or produces a sparse member's payload from *stream_plan* / *map_bytes*). """ - chunk_iterator = archive.pipeline.fetch_many(item.chunks, ro_type=ROBJ_FILE_STREAM) + if stream_plan is not None: + chunk_iterator = sparse_chunk_iterator(stream_plan, map_bytes) + else: + chunk_iterator = archive.pipeline.fetch_many(item.chunks, ro_type=ROBJ_FILE_STREAM) if pi: info = [remove_surrogates(item.path)] return ChunkIteratorFileWrapper( @@ -289,6 +439,57 @@ def item_content_stream(item): else: return ChunkIteratorFileWrapper(chunk_iterator) + def sparsify_tarinfo(item, tarinfo): + """Try to turn *tarinfo* into a GNU sparse format 1.0 member. + + If the item's content has detectable holes (runs of all-zero chunks) and storing + it sparsely is possible and worthwhile, modify *tarinfo* accordingly (mangled + name, stored size, GNU.sparse.* pax headers) and return (data_chunks, map_bytes); + else return None and leave *tarinfo* alone (dense member). + """ + if not item.chunks: + return None + ids = [chunk.id for chunk in item.chunks] + sizes = [chunk.size for chunk in item.chunks] + # Warm up the zero chunk id memo for likely hole chunk sizes, so such zero chunks + # get detected even when their id does not repeat within this item: all-zero + # chunks usually are max-chunk-sized (a power of two, as the chunkers do not cut + # within runs of zeros) or a file's last chunk (a trailing hole of any size). + candidate_sizes = {size for size in sizes if size & (size - 1) == 0} | {sizes[-1]} + for size in candidate_sizes: + if 0 < size <= len(zeros): + zero_chunk_id(archive.key.id_hash, size) + sparse_info = chunks_to_sparse_info(item.chunks, zero_chunk_flags(ids, sizes, archive.key.id_hash)) + if sparse_info is None: + return None + map_entries, stream_plan, realsize = sparse_info + if realsize != item.get_size(): + return None # do not write self-contradicting sparse headers for an inconsistent item + map_bytes = gnu_sparse_10_map(map_entries) + stored_size = len(map_bytes) + sum(length for _, length in map_entries) + if stored_size >= realsize: + return None # not worthwhile, the map costs more than the holes save + # SparseTarInfo suppresses the pax "size" record (sparse readers desync on it) + # and base-256 encodes a stored size beyond the octal ustar field limit. + tarinfo.__class__ = SparseTarInfo + # Do like GNU tar: store the member under a mangled name, so that a sparse-unaware + # tar does not extract the raw map + data segments under the original name - the + # real name is in the GNU.sparse.name pax header. GNU tar uses its pid where we + # always use 0, for reproducible output. + mangled_name = "GNUSparseFile.0/" + tarinfo.name + # pax record order matters for readers applying them in-order: "path" (mangled) + # first, the GNU.sparse.* records (real name / size) last, so the latter win. + ph = {"path": mangled_name} + ph.update(tarinfo.pax_headers) + ph["GNU.sparse.major"] = "1" + ph["GNU.sparse.minor"] = "0" + ph["GNU.sparse.name"] = tarinfo.name + ph["GNU.sparse.realsize"] = str(realsize) + tarinfo.pax_headers = ph + tarinfo.name = mangled_name + tarinfo.size = stored_size # sparse map + data segments, excluding the holes + return stream_plan, map_bytes + for item in archive.iter_items(filter): orig_path = item.path if strip_components: @@ -299,8 +500,17 @@ def item_content_stream(item): tarinfo.pax_headers = item_to_paxheaders(args.tar_format, item) if output_list: logging.getLogger("borg.output.list").info(remove_surrogates(orig_path)) - stream = item_content_stream(item) if needs_content else None + sparse_content = sparsify_tarinfo(item, tarinfo) if args.sparse and needs_content else None + if sparse_content is not None: + stream_plan, map_bytes = sparse_content + stream = item_content_stream(item, stream_plan=stream_plan, map_bytes=map_bytes) + else: + stream = item_content_stream(item) if needs_content else None tar.addfile(tarinfo, stream) + if pi and sparse_content is not None: + # the stream callback counted only the stored bytes (sparse map + data + # segments), but the progress total is based on the logical file sizes. + pi.show(increase=max(0, item.get_size() - tarinfo.size), info=[remove_surrogates(item.path)]) if pi: pi.finish() @@ -434,7 +644,20 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser): | | | no ACLs/xattrs/bsdflags | +--------------+---------------------------+----------------------------+ - A ``--sparse`` option (as found in borg extract) is not supported. + With ``--sparse``, files whose content contains runs of all-zero chunks are written + as sparse tar members (GNU sparse format 1.0, as GNU tar creates it in POSIX mode), + storing only a hole map and the non-zero data. This requires ``--tar-format BORG`` + or ``PAX``. Such tarballs can be much smaller for sparse files (e.g. disk images) + and extract to sparse files again with GNU tar's or bsdtar's sparse support + (as well as with ``borg import-tar`` / ``borg extract --sparse``). + Notes: hole detection works at the granularity of borg's content chunks (it does not + depend on the original file having been a sparse file - but some short or unaligned + zero runs may be stored literally); sparse-unaware tar implementations will extract + a member as ``GNUSparseFile.0/`` containing the raw hole map and data (the + same caveat applies to tarballs created by GNU tar); for members needing >= 8 GiB + of stored (non-hole) data, the stored size is base-256 encoded in the tar header + (the GNU/star encoding of big numbers, understood by GNU tar, libarchive/bsdtar + and python) - logical file sizes are unlimited anyway. By default the entire archive is extracted but a subset of files and directories can be selected by passing a list of ``PATHs`` as arguments. @@ -469,6 +692,13 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser): action=Highlander, help="select tar format: BORG, PAX or GNU", ) + subparser.add_argument( + "--sparse", + dest="sparse", + action="store_true", + help="write sparse tar members (GNU sparse format 1.0) for files containing all-zero " + "chunks (BORG and PAX formats only)", + ) subparser.add_argument("name", metavar="NAME", type=archivename_validator, help="specify the archive name") subparser.add_argument( "tarfile", metavar="FILE", type=FilesystemPathSpec, help='output tar file. "-" to write to stdout instead.' @@ -504,7 +734,9 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser): Most documentation of borg create applies. Note that this command does not support excluding files. - A ``--sparse`` option (as found in borg create) is not supported. + A ``--sparse`` option (as found in borg create) is not needed: sparse members in + input tarballs (old GNU and PAX sparse formats) are read correctly and their + holes are stored as deduplicated all-zero chunks. About tar formats and metadata conservation or loss, please see ``borg export-tar``. diff --git a/src/borg/testsuite/archive_test.py b/src/borg/testsuite/archive_test.py index cb2aee520e..e41e6bdfdb 100644 --- a/src/borg/testsuite/archive_test.py +++ b/src/borg/testsuite/archive_test.py @@ -13,6 +13,7 @@ from ..crypto.key import ChecksumKey from ..archive import Archive, CacheChunkBuffer, DownloadPipeline, RobustUnpacker, valid_msgpacked_dict from ..archive import ITEM_KEYS, Statistics +from ..archive import zero_chunk_flags, zero_chunk_id, zero_chunk_ids from ..archive import BackupOSError, backup_io, backup_io_iter, get_item_uid_gid from ..helpers import msgpack from ..repoobj import RepoObj @@ -340,6 +341,31 @@ def test_download_pipeline_zero_chunks_served_locally(): assert repository.requested_ids == [other_id] +def test_zero_chunk_flags(): + # cheap all-zero chunk detection from the chunk ids/sizes alone: the zero chunk id + # is computed for ids occurring repeatedly, while unique ids are only compared + # against already memoized zero chunk ids. + key = ChecksumKey(None) + id_hash = key.id_hash + data = b"foobar" * 100 + data_id = id_hash(data) + repeated_size, unique_size = 1234, 4321 + repeated_id = id_hash(zeros[:repeated_size]) + unique_id = id_hash(zeros[:unique_size]) + # make sure nothing is memoized for these sizes yet + for size in (repeated_size, unique_size): + zero_chunk_ids.pop((id_hash, size), None) + ids = [repeated_id, data_id, repeated_id, unique_id] + sizes = [repeated_size, len(data), repeated_size, unique_size] + # the repeated zero id gets detected, the unique one is missed (nothing memoized yet) + assert zero_chunk_flags(ids, sizes, id_hash) == [True, False, True, False] + # once its size's zero chunk id is memoized, the unique one gets detected, too + zero_chunk_id(id_hash, unique_size) + assert zero_chunk_flags(ids, sizes, id_hash) == [True, False, True, True] + # unknown (None) or out-of-range sizes disable the detection + assert zero_chunk_flags([repeated_id, repeated_id], [None, None], id_hash) == [False, False] + + def make_chunks(items): return b"".join(msgpack.packb({"path": item}) for item in items) diff --git a/src/borg/testsuite/archiver/tar_cmds_test.py b/src/borg/testsuite/archiver/tar_cmds_test.py index 3121e9a549..5a1ea8f4fb 100644 --- a/src/borg/testsuite/archiver/tar_cmds_test.py +++ b/src/borg/testsuite/archiver/tar_cmds_test.py @@ -1,11 +1,17 @@ import os +import random import shutil import subprocess +import sys +import tarfile import pytest from ... import xattr +from ...archiver.tar_cmds import chunks_to_sparse_info, gnu_sparse_10_map, SparseTarInfo from ...constants import * # NOQA +from ...helpers import Error +from ...item import ChunkListEntry from .. import changedir from . import assert_dirs_equal, _extract_hardlinks_setup, cmd, requires_hardlinks, RK_ENCRYPTION from . import create_test_files, create_regular_file @@ -342,6 +348,260 @@ def test_roundtrip_pax_xattrs(archivers, request): assert xa_value_extracted == xa_value +def _sparse_entries(sizes): + return [ChunkListEntry(id=bytes([i]) * 32, size=size) for i, size in enumerate(sizes)] + + +def test_chunks_to_sparse_info(): + # no chunks / no holes: not sparse + assert chunks_to_sparse_info([], []) is None + assert chunks_to_sparse_info(_sparse_entries([512, 1024]), [False, False]) is None + # a zero run shorter than a tar block cannot make a (block-aligned) hole + assert chunks_to_sparse_info(_sparse_entries([512, 511, 512]), [False, True, False]) is None + # leading hole; adjacent zero chunks coalesce into one hole + chunks = _sparse_entries([1024, 512, 1536]) + map_entries, stream_plan, realsize = chunks_to_sparse_info(chunks, [True, True, False]) + assert map_entries == [(1536, 1536)] + assert stream_plan == [chunks[2]] + assert realsize == 3072 + # middle hole + chunks = _sparse_entries([512, 1024, 1024, 512]) + map_entries, stream_plan, realsize = chunks_to_sparse_info(chunks, [False, True, True, False]) + assert map_entries == [(0, 512), (2560, 512)] + assert stream_plan == [chunks[0], chunks[3]] + assert realsize == 3072 + # trailing hole: terminating (realsize, 0) entry, like GNU tar creates it + chunks = _sparse_entries([512, 1024]) + map_entries, stream_plan, realsize = chunks_to_sparse_info(chunks, [False, True]) + assert map_entries == [(0, 512), (1536, 0)] + assert stream_plan == [chunks[0]] + assert realsize == 1536 + # all-hole file (its trailing hole may end unaligned) + chunks = _sparse_entries([512, 100]) + map_entries, stream_plan, realsize = chunks_to_sparse_info(chunks, [True, True]) + assert map_entries == [(612, 0)] + assert stream_plan == [] + assert realsize == 612 + # holes shrink to whole 512-byte tar blocks (GNU tar's sparse reader needs block-aligned + # data segments); the shaved-off zero bytes at the hole edges are emitted literally. + chunks = _sparse_entries([10000, 40000, 10000]) + map_entries, stream_plan, realsize = chunks_to_sparse_info(chunks, [False, True, False]) + assert map_entries == [(0, 10240), (49664, 10336)] + assert stream_plan == [chunks[0], 240, 336, chunks[2]] + assert realsize == 60000 + # the stream plan produces exactly the data segments' bytes + emitted = sum(entry if isinstance(entry, int) else entry.size for entry in stream_plan) + assert emitted == sum(length for _, length in map_entries) + + +def test_gnu_sparse_10_map(): + # exactly the numbers/layout GNU tar writes, NUL-padded to a multiple of 512. + map_bytes = gnu_sparse_10_map([(0, 1048576), (5242880, 1048576), (10485760, 0)]) + assert map_bytes == b"3\n0\n1048576\n5242880\n1048576\n10485760\n0\n" + b"\0" * (512 - 39) + assert len(gnu_sparse_10_map([(1000, 0)])) == 512 + # a large map spills into multiple 512 byte blocks + map_bytes = gnu_sparse_10_map([(i * 10000, 5000) for i in range(100)]) + assert len(map_bytes) % 512 == 0 and len(map_bytes) > 512 + numbers = [int(n) for n in map_bytes.rstrip(b"\0").split()] + assert numbers[0] == 100 and numbers[1:3] == [0, 5000] + + +# chunkers used by the sparse tests: a fixed chunker (all-zero chunks exactly aligned with +# the zero runs) and a small-target fastcdc chunker (content-defined chunk boundaries, so the +# zero runs yield multiple repeated pure all-zero chunks of max. chunk size (64 KiB), possibly +# surrounded by mixed data/zeros chunks at the edges - like real sparse files chunked by the +# default chunker, just scaled down to small test files). +SPARSE_CHUNKER_FIXED = "--chunker-params=fixed,65536" +SPARSE_CHUNKER_CDC = "--chunker-params=fastcdc,12,16,14,2" # 4 KiB min, 16 KiB target, 64 KiB max + + +def _create_sparse_test_input(input_path): + """Create files containing runs of all-zero chunks (the files need not be sparse on disk).""" + B = 65536 # the max. chunk size of the chunkers used by the sparse tests + rnd = random.Random(42) # pseudorandom data, so the cdc chunker cuts realistic chunks + contents = { + "sparse_img": rnd.randbytes(B) + b"\0" * (4 * B) + rnd.randbytes(B) + b"\0" * (4 * B), + "allzero": b"\0" * (3 * B), + "unaligned": b"\0" * (2 * B) + b"tail", # leading hole, small unaligned trailing data chunk + "dense": rnd.randbytes(B + 42), + "empty": b"", + } + for name, data in contents.items(): + create_regular_file(input_path, name, contents=data) + return contents + + +@pytest.mark.parametrize("chunker_params", [SPARSE_CHUNKER_FIXED, SPARSE_CHUNKER_CDC]) +def test_export_tar_sparse(archivers, request, chunker_params): + archiver = request.getfixturevalue(archivers) + contents = _create_sparse_test_input(archiver.input_path) + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "create", chunker_params, "test", "input") + cmd(archiver, "export-tar", "test", "dense.tar") + cmd(archiver, "export-tar", "--sparse", "test", "sparse.tar", "--progress") + # storing the holes as sparse members must save space (~832 KiB of holes here) + assert os.path.getsize("sparse.tar") < os.path.getsize("dense.tar") - 500000 + expected_sparse = {"input/sparse_img", "input/allzero", "input/unaligned"} + seen = set() + with tarfile.open("sparse.tar") as tar: + for tarinfo in tar.getmembers(): + if not tarinfo.isreg(): + continue + seen.add(tarinfo.name) + # the tarinfo has the real (unmangled) name and the logical size, + # and the member expands to the original content. + name = tarinfo.name.rsplit("/", 1)[-1] + assert tarinfo.size == len(contents[name]) + assert tar.extractfile(tarinfo).read() == contents[name] + assert tarinfo.issparse() == (tarinfo.name in expected_sparse) + assert seen == {"input/" + name for name in contents} + + +@pytest.mark.parametrize("chunker_params", [SPARSE_CHUNKER_FIXED, SPARSE_CHUNKER_CDC]) +@pytest.mark.parametrize("tar_format", ["PAX", "BORG"]) +def test_export_tar_sparse_roundtrip(archivers, request, tar_format, chunker_params): + archiver = request.getfixturevalue(archivers) + _create_sparse_test_input(archiver.input_path) + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "create", chunker_params, "src", "input") + cmd(archiver, "export-tar", "--sparse", f"--tar-format={tar_format}", "src", "sparse.tar") + cmd(archiver, "import-tar", "dst", "sparse.tar") + with changedir(archiver.output_path): + cmd(archiver, "extract", "dst") + assert_dirs_equal("input", "output/input", ignore_ns=True, ignore_xattrs=True) + + +def test_export_tar_sparse_not_worthwhile(archivers, request): + # when the sparse map would cost more space than the holes save, store a dense member. + archiver = request.getfixturevalue(archivers) + contents = b"X" * 448 + b"\0" * 64 + create_regular_file(archiver.input_path, "tinyhole", contents=contents) + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "create", "--chunker-params=fixed,64", "test", "input") + cmd(archiver, "export-tar", "--sparse", "test", "sparse.tar") + with tarfile.open("sparse.tar") as tar: + tarinfo = tar.getmember("input/tinyhole") + assert not tarinfo.issparse() + assert tar.extractfile(tarinfo).read() == contents + + +@requires_gnutar +def test_export_tar_sparse_gnutar(archivers, request): + archiver = request.getfixturevalue(archivers) + _create_sparse_test_input(archiver.input_path) + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "create", SPARSE_CHUNKER_CDC, "test", "input") + cmd(archiver, "export-tar", "--sparse", "test", "sparse.tar") + with changedir("output"): + subprocess.check_call(["tar", "xpf", "../sparse.tar", "--warning=no-timestamp"]) + assert_dirs_equal("input", "output/input", ignore_flags=True, ignore_xattrs=True, ignore_ns=True) + if sys.platform == "linux": + # GNU tar recreates the holes when extracting sparse members. + st = os.stat("output/input/sparse_img") + assert st.st_blocks * 512 < st.st_size + + +@requires_hardlinks +def test_export_tar_sparse_hardlinks(archivers, request): + archiver = request.getfixturevalue(archivers) + contents = b"\0" * 2 * 65536 + b"data" + create_regular_file(archiver.input_path, "sparse1", contents=contents) + os.link(os.path.join(archiver.input_path, "sparse1"), os.path.join(archiver.input_path, "sparse2")) + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "create", SPARSE_CHUNKER_CDC, "src", "input") + cmd(archiver, "export-tar", "--sparse", "src", "sparse.tar") + with tarfile.open("sparse.tar") as tar: + members = [ti for ti in tar.getmembers() if ti.name.startswith("input/sparse")] + regs = [ti for ti in members if ti.isreg()] + lnks = [ti for ti in members if ti.islnk()] + assert len(regs) == 1 and len(lnks) == 1 + # the first occurrence carries the sparse content, the second one is a tar hard link + # referencing the first one's real (unmangled) name. + assert regs[0].issparse() + assert tar.extractfile(regs[0]).read() == contents + assert lnks[0].linkname == regs[0].name + # roundtrip: as usual for import-tar, tar hard links become separate files (sharing chunks). + cmd(archiver, "import-tar", "dst", "sparse.tar") + with changedir(archiver.output_path): + cmd(archiver, "extract", "dst") + for name in "input/sparse1", "input/sparse2": + with open(name, "rb") as f: + assert f.read() == contents + + +def test_export_tar_sparse_strip_components(archivers, request): + # the mangled member name and GNU.sparse.name must be based on the stripped path. + archiver = request.getfixturevalue(archivers) + contents = b"\0" * 2 * 65536 + b"end" + create_regular_file(archiver.input_path, "dir/sparsefile", contents=contents) + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "create", SPARSE_CHUNKER_CDC, "test", "input") + cmd(archiver, "export-tar", "--sparse", "--strip-components=1", "test", "sparse.tar") + with tarfile.open("sparse.tar") as tar: + tarinfo = tar.getmember("dir/sparsefile") + assert tarinfo.issparse() + assert tar.extractfile(tarinfo).read() == contents + + +def test_sparse_tarinfo_base256_size(): + # a stored size beyond the 12-digit octal ustar field limit is base-256 encoded in the + # ustar size field (with a correct checksum), and no pax "size" record is emitted + # (sparse readers desync on such a record). + tarinfo = SparseTarInfo(name="GNUSparseFile.0/big") + tarinfo.size = 3 * 8**11 # 24 GiB of stored data, does not fit the octal field + tarinfo.pax_headers = {"GNU.sparse.realsize": str(100 * 8**11)} + buf = tarinfo.tobuf(tarfile.PAX_FORMAT, tarfile.ENCODING, "surrogateescape") + assert b" size=" not in buf # no pax "size" record (" realsize=" does not match) + # frombuf validates the checksum and decodes the base-256 size field: + parsed = tarfile.TarInfo.frombuf(buf[-tarfile.BLOCKSIZE :], tarfile.ENCODING, "surrogateescape") + assert parsed.size == 3 * 8**11 + # small stored sizes keep the plain octal encoding: + tarinfo.size = 4711 + buf = tarinfo.tobuf(tarfile.PAX_FORMAT, tarfile.ENCODING, "surrogateescape") + assert buf[-tarfile.BLOCKSIZE :][124:136] == b"00000011147\x00" + + +def test_export_tar_sparse_base256_size(archivers, request, monkeypatch): + # end-to-end wire-format test of the base-256 stored size: the encoding does not depend + # on the value, so force it for small members instead of storing 8 GiB of data. + archiver = request.getfixturevalue(archivers) + if archiver.EXE: + pytest.skip("monkeypatching does not reach a borg binary") + monkeypatch.setattr(SparseTarInfo, "octal_size_limit", 1) + contents = _create_sparse_test_input(archiver.input_path) + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "create", SPARSE_CHUNKER_CDC, "src", "input") + cmd(archiver, "export-tar", "--sparse", "src", "sparse.tar") + with tarfile.open("sparse.tar") as tar: + for tarinfo in tar.getmembers(): + if tarinfo.isreg(): + name = tarinfo.name.rsplit("/", 1)[-1] + assert tar.extractfile(tarinfo).read() == contents[name] + if have_gnutar(): + with changedir("output"): + subprocess.check_call(["tar", "xpf", "../sparse.tar", "--warning=no-timestamp"]) + assert_dirs_equal("input", "output/input", ignore_flags=True, ignore_xattrs=True, ignore_ns=True) + shutil.rmtree("output/input") + cmd(archiver, "import-tar", "dst", "sparse.tar") + with changedir(archiver.output_path): + cmd(archiver, "extract", "dst") + assert_dirs_equal("input", "output/input", ignore_ns=True, ignore_xattrs=True) + + +def test_export_tar_sparse_gnu_format_error(archivers, request): + # --sparse requires a PAX-based tar format, the GNU format cannot store the sparse headers. + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file", contents=b"x") + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "create", "test", "input") + if archiver.FORK_DEFAULT: + output = cmd(archiver, "export-tar", "--sparse", "--tar-format=GNU", "test", "out.tar", exit_code=2) + assert "--sparse requires --tar-format" in output + else: + with pytest.raises(Error, match="--sparse requires --tar-format"): + cmd(archiver, "export-tar", "--sparse", "--tar-format=GNU", "test", "out.tar") + + @skipif_not_linux @skipif_acls_not_working def test_acl_roundtrip(archivers, request):