From b9da5fdfd740b7aa0952cd91b6bd85dd4b499fb2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 14 Aug 2026 13:50:53 +0200 Subject: [PATCH] compression: use zstd,-4 as the new default, fixes #10085 lz4 -> zstd,-4 for the -C default of create, recreate, import-tar, transfer, repo-compress and debug put-obj. Benchmarked on real data (see PR): end-to-end create is as fast as or faster than lz4 on small-file-heavy and binary corpora - libzstd's multithreading (chunks >= 768 KiB) more than makes up for lz4's higher single-thread speed on small chunks - at the same or slightly smaller repo size. lz4 stays available, nothing changes for existing repos. Also: zstd,-4 added to the borg benchmark cpu codec list, docs updated, usage/man files regenerated for the affected commands. Co-Authored-By: Claude Fable 5 --- docs/faq.rst | 6 +-- docs/man/borg-compression.1 | 17 ++++-- docs/man/borg-create.1 | 12 ++++- docs/quickstart.rst | 4 +- docs/usage/benchmark_cpu.rst.inc | 53 +++++++++++++++---- docs/usage/create.rst | 5 +- docs/usage/help.rst.inc | 15 ++++-- src/borg/archiver/benchmark_cmd.py | 1 + src/borg/archiver/create_cmd.py | 2 +- src/borg/archiver/debug_cmd.py | 2 +- src/borg/archiver/help_cmd.py | 6 +-- src/borg/archiver/recreate_cmd.py | 2 +- src/borg/archiver/repo_compress_cmd.py | 2 +- src/borg/archiver/tar_cmds.py | 2 +- src/borg/archiver/transfer_cmd.py | 2 +- .../archiver/restricted_permissions_test.py | 2 +- 16 files changed, 95 insertions(+), 38 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 2c731ebc8e..6b9056077a 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -842,7 +842,7 @@ and disk space on subsequent runs. Here what Borg does when you run ``borg creat - Then it checks whether this chunk is already in the repo (local hashtable lookup, fast). If so, the processing of the chunk is completed here. Otherwise it needs to process the chunk: -- Compresses (the default lz4 is super fast) +- Compresses (the default zstd,-4 is very fast) - Encrypts and authenticates (AES-OCB, usually fast if your CPU has AES acceleration as usual since about 10y, or chacha20-poly1305, fast pure-software crypto) - Transmits to repo. If the repo is remote, this usually involves an SSH connection @@ -879,8 +879,8 @@ If you feel your Borg backup is too slow somehow, here is what you can do: files you have) - Use one of the blake2 modes for --encryption except if you positively know your CPU (and openssl) accelerates sha256 (then stay with hmac-sha256). -- Don't use any expensive compression. The default is lz4 and super fast. - Uncompressed is often slower than lz4. +- Don't use any expensive compression. The default is zstd,-4 and very fast. + Uncompressed is often slower than fast compression. - Just wait. You can also interrupt it and start it again as often as you like, it will converge against a valid "completed" state. It is starting from the beginning each time, but it is still faster then as it does not store diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index d3eb201c67..e4ad63ff4b 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "borg-compression" "1" "2026-07-21" "" "borg backup tool" +.TH "borg-compression" "1" "2026-08-14" "" "borg backup tool" .SH Name borg-compression \- Details regarding compression .SH DESCRIPTION @@ -42,7 +42,7 @@ another backup that also uses this chunk will not change the stored chunk. So if you use different compression specs for the backups, whichever stores a chunk first determines its compression. See also \fBborg recreate\fP\&. .sp -Compression is lz4 by default. If you want something else, you have to specify what you want. +Compression is zstd,\-4 by default. If you want something else, you have to specify what you want. .sp Valid compression specifiers are: .INDENT 0.0 @@ -51,12 +51,19 @@ Valid compression specifiers are: Do not compress. .TP .B lz4 -Use lz4 compression. Very high speed, very low compression. (default) +Use lz4 compression. Very high speed, very low compression. .TP .B zstd[,L] -Use zstd (\(dqzstandard\(dq) compression, a modern wide\-range algorithm. -If you do not explicitly give the compression level L (ranging from 1 +Use zstd (\(dqzstandard\(dq) compression, a modern wide\-range algorithm. (default: zstd,\-4) +If you do not explicitly give the compression level L (ranging from \-128 to 22), it will use level 3. +Negative levels are zstd\(aqs \(dqfast\(dq levels (level \-N is what the zstd command +line tool calls \-\-fast=N): they give up compression ratio for speed. +\-1 to \-10 is the useful range for general data; going lower only pays off for +data with long repeats (disk/VM images, database files), where it stays fast +while still finding the big matches. +Level 0 selects zstd\(aqs default level (3) \- for zstd it does not mean +\(dqno compression\(dq (use \(dqnone\(dq for that), unlike for zlib below. .TP .B zlib[,L] Use zlib (\(dqgz\(dq) compression. Medium speed, medium compression. diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index a5fc9ece74..1b0441f41e 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "borg-create" "1" "2026-08-13" "" "borg backup tool" +.TH "borg-create" "1" "2026-08-14" "" "borg backup tool" .SH Name borg-create \- Creates a new archive. .SH SYNOPSIS @@ -134,6 +134,11 @@ The \fB\-\-progress\fP option shows (from left to right) Original and (uncompres deduplicated size (O and U respectively), then the Number of files (N) processed so far, followed by the currently processed path. .sp +Sizes of GB and above are shown with enough decimal places that even MB\-sized progress +stays visible. On a terminal, this needs a width of at least 110 columns \- on narrower +terminals, the compact format is used, so that the path stays readable. If the output +does not go to a terminal (e.g. into a logfile), the precise format is always used. +.sp When using \fB\-\-stats\fP, you will get some statistics about how much data was added \- the \(dqThis Archive\(dq deduplicated size there is most interesting as that is how much your repository will grow. Please note that the \(dqAll archives\(dq stats refer to @@ -372,9 +377,12 @@ $ borg create \-\-sparse \-\-chunker\-params fixed,4194304 my\-disk my\-disk.raw # No compression (none) $ borg create \-\-compression none arch ~ -# Super fast, low compression (lz4, default) +# Fast, moderate compression (zstd,\-4, default) $ borg create arch ~ +# Super fast, low compression (lz4) +$ borg create \-\-compression lz4 arch ~ + # Less fast, higher compression (zlib, N = 0..9) $ borg create \-\-compression zlib,N arch ~ diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 9ceede63e0..8b6a4cb154 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -292,12 +292,12 @@ E.g. if your crypto device supports GPG and borg calls ``gpg`` via Backup compression ------------------ -The default is lz4 (very fast, but low compression ratio), but other methods are +The default is zstd,-4 (very fast, moderate compression ratio), but other methods are supported for different situations. Compression not only helps you save disk space, but will especially speed up remote backups since less data needs to be transferred. zstd is a modern compression algorithm which can be parametrized to anything between -N=1 for highest speed (and relatively low compression) to N=22 for highest compression +N=-128 for highest speed (and low compression) to N=22 for highest compression (and lower speed):: $ borg create --compression zstd,N arch ~ diff --git a/docs/usage/benchmark_cpu.rst.inc b/docs/usage/benchmark_cpu.rst.inc index 3cb75edbf4..3a7a8bc148 100644 --- a/docs/usage/benchmark_cpu.rst.inc +++ b/docs/usage/benchmark_cpu.rst.inc @@ -12,15 +12,25 @@ borg benchmark cpu .. class:: borg-options-table - +-------------------------------------------------------+------------+-----------------------+ - | **options** | - +-------------------------------------------------------+------------+-----------------------+ - | | ``--json`` | format output as JSON | - +-------------------------------------------------------+------------+-----------------------+ - | .. class:: borg-common-opt-ref | - | | - | :ref:`common_options` | - +-------------------------------------------------------+------------+-----------------------+ + +-------------------------------------------------------+-------------------+--------------------------------+ + | **options** | + +-------------------------------------------------------+-------------------+--------------------------------+ + | | ``--json`` | format output as JSON | + +-------------------------------------------------------+-------------------+--------------------------------+ + | | ``--chunking`` | benchmark the chunkers | + +-------------------------------------------------------+-------------------+--------------------------------+ + | | ``--hashing`` | benchmark the hashes / MACs | + +-------------------------------------------------------+-------------------+--------------------------------+ + | | ``--encrypting`` | benchmark the encryption modes | + +-------------------------------------------------------+-------------------+--------------------------------+ + | | ``--compressing`` | benchmark the compressors | + +-------------------------------------------------------+-------------------+--------------------------------+ + | | ``--msgpacking`` | benchmark msgpack item packing | + +-------------------------------------------------------+-------------------+--------------------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-------------------------------------------------------+-------------------+--------------------------------+ .. raw:: html @@ -35,7 +45,12 @@ borg benchmark cpu options - --json format output as JSON + --json format output as JSON + --chunking benchmark the chunkers + --hashing benchmark the hashes / MACs + --encrypting benchmark the encryption modes + --compressing benchmark the compressors + --msgpacking benchmark msgpack item packing :ref:`common_options` @@ -50,4 +65,20 @@ It creates input data in memory, runs the operation and then displays throughput To reduce outside influence on the timings, please make sure to run this with: - an otherwise as idle as possible machine -- enough free memory so there will be no slow down due to paging activity \ No newline at end of file +- enough free memory so there will be no slow down due to paging activity + +By default all benchmarks run. Give one or more of --chunking, --hashing, +--encrypting, --compressing, --msgpacking to run only those. + +Some algorithms use multiple threads only above a size threshold, so the +hashes and the compressors are measured at more than one buffer size: the +hashes at 64MiB (a borg pack) and 2MiB (a typical borg chunk), both above +blake3's threshold, the compressors at 2MiB and 128kiB, which is below +zstd's. Within a section every row processes the same total number of +bytes - 1 GiB, or 10 MiB for the compressors - so the throughput column is +comparable between rows. + +The compressors work on synthetic text-like data that compresses about 4x +at zstd,3. Random data would be the worst possible input: no codec can +compress it, so all of them would take their incompressible fast path and +the levels would barely differ. \ No newline at end of file diff --git a/docs/usage/create.rst b/docs/usage/create.rst index 3a96c71798..510113845d 100644 --- a/docs/usage/create.rst +++ b/docs/usage/create.rst @@ -64,9 +64,12 @@ Examples # No compression (none) $ borg create --compression none arch ~ - # Super fast, low compression (lz4, default) + # Fast, moderate compression (zstd,-4, default) $ borg create arch ~ + # Super fast, low compression (lz4) + $ borg create --compression lz4 arch ~ + # Less fast, higher compression (zlib, N = 0..9) $ borg create --compression zlib,N arch ~ diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index d7877b84e4..743863fef3 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -476,7 +476,7 @@ another backup that also uses this chunk will not change the stored chunk. So if you use different compression specs for the backups, whichever stores a chunk first determines its compression. See also ``borg recreate``. -Compression is lz4 by default. If you want something else, you have to specify what you want. +Compression is zstd,-4 by default. If you want something else, you have to specify what you want. Valid compression specifiers are: @@ -484,12 +484,19 @@ none Do not compress. lz4 - Use lz4 compression. Very high speed, very low compression. (default) + Use lz4 compression. Very high speed, very low compression. zstd[,L] - Use zstd ("zstandard") compression, a modern wide-range algorithm. - If you do not explicitly give the compression level L (ranging from 1 + Use zstd ("zstandard") compression, a modern wide-range algorithm. (default: zstd,-4) + If you do not explicitly give the compression level L (ranging from -128 to 22), it will use level 3. + Negative levels are zstd's "fast" levels (level -N is what the zstd command + line tool calls --fast=N): they give up compression ratio for speed. + -1 to -10 is the useful range for general data; going lower only pays off for + data with long repeats (disk/VM images, database files), where it stays fast + while still finding the big matches. + Level 0 selects zstd's default level (3) - for zstd it does not mean + "no compression" (use "none" for that), unlike for zlib below. zlib[,L] Use zlib ("gz") compression. Medium speed, medium compression. diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py index 614640adf0..3f52164580 100644 --- a/src/borg/archiver/benchmark_cmd.py +++ b/src/borg/archiver/benchmark_cmd.py @@ -399,6 +399,7 @@ def blake3_hash(d): number = max(3, comp_total // nbytes) # a few reps even for the fast codecs for spec in [ "lz4", + "zstd,-4", "zstd,1", "zstd,3", "zstd,5", diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 748dbe3be9..9e2a9c9783 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -1062,7 +1062,7 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=CompressionSpec("zstd,-4"), action=Highlander, help="select compression algorithm, see the output of the " '"borg help compression" command for details.', ) diff --git a/src/borg/archiver/debug_cmd.py b/src/borg/archiver/debug_cmd.py index 40eb210176..4a4ba335c9 100644 --- a/src/borg/archiver/debug_cmd.py +++ b/src/borg/archiver/debug_cmd.py @@ -483,7 +483,7 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser): metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=CompressionSpec("zstd,-4"), action=Highlander, help="select compression algorithm, see the output of the " '"borg help compression" command for details.', ) diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py index 28980f7db6..3997835bf7 100644 --- a/src/borg/archiver/help_cmd.py +++ b/src/borg/archiver/help_cmd.py @@ -468,7 +468,7 @@ class HelpMixIn: So if you use different compression specs for the backups, whichever stores a chunk first determines its compression. See also ``borg recreate``. - Compression is lz4 by default. If you want something else, you have to specify what you want. + Compression is zstd,-4 by default. If you want something else, you have to specify what you want. Valid compression specifiers are: @@ -476,10 +476,10 @@ class HelpMixIn: Do not compress. lz4 - Use lz4 compression. Very high speed, very low compression. (default) + Use lz4 compression. Very high speed, very low compression. zstd[,L] - Use zstd ("zstandard") compression, a modern wide-range algorithm. + Use zstd ("zstandard") compression, a modern wide-range algorithm. (default: zstd,-4) If you do not explicitly give the compression level L (ranging from -128 to 22), it will use level 3. Negative levels are zstd's "fast" levels (level -N is what the zstd command diff --git a/src/borg/archiver/recreate_cmd.py b/src/borg/archiver/recreate_cmd.py index 85057b87f5..70ecaa2002 100644 --- a/src/borg/archiver/recreate_cmd.py +++ b/src/borg/archiver/recreate_cmd.py @@ -154,7 +154,7 @@ def build_parser_recreate(self, subparsers, common_parser, mid_common_parser): metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=CompressionSpec("zstd,-4"), action=Highlander, help="select compression algorithm, see the output of the " '"borg help compression" command for details.', ) diff --git a/src/borg/archiver/repo_compress_cmd.py b/src/borg/archiver/repo_compress_cmd.py index 273abc787e..2a9e5a9dc1 100644 --- a/src/borg/archiver/repo_compress_cmd.py +++ b/src/borg/archiver/repo_compress_cmd.py @@ -237,7 +237,7 @@ def build_parser_repo_compress(self, subparsers, common_parser, mid_common_parse metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=CompressionSpec("zstd,-4"), action=Highlander, help='select compression algorithm, see the output of the "borg help compression" command for details.', ) diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index f4a49ed627..291911dce3 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -599,7 +599,7 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser): metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=CompressionSpec("zstd,-4"), action=Highlander, help="select compression algorithm, see the output of the " '"borg help compression" command for details.', ) diff --git a/src/borg/archiver/transfer_cmd.py b/src/borg/archiver/transfer_cmd.py index abe7f25a8c..e351dd5617 100644 --- a/src/borg/archiver/transfer_cmd.py +++ b/src/borg/archiver/transfer_cmd.py @@ -373,7 +373,7 @@ def build_parser_transfer(self, subparsers, common_parser, mid_common_parser): metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=CompressionSpec("zstd,-4"), action=Highlander, help="select compression algorithm, see the output of the " '"borg help compression" command for details.', ) diff --git a/src/borg/testsuite/archiver/restricted_permissions_test.py b/src/borg/testsuite/archiver/restricted_permissions_test.py index 38bf1655f3..2f5949c528 100644 --- a/src/borg/testsuite/archiver/restricted_permissions_test.py +++ b/src/borg/testsuite/archiver/restricted_permissions_test.py @@ -83,7 +83,7 @@ def test_repository_permissions_no_delete(archivers, request, monkeypatch): # A dry run only reads and reports, so it is allowed even without write/delete access. cmd(archiver, "compact", "--dry-run") - # Try to repo-compress (and change compression from lz4 to zstd), which should fail up front: + # Try to repo-compress (and change compression to zstd,3), which should fail up front: # it rewrites (and deletes) pack files, which is disallowed by no-delete. with pytest.raises(Repository.PermissionDenied): cmd(archiver, "repo-compress", "-C", "zstd")