Add fastcdc chunker (keyed Gear hash)

.gitignore

@@ -9,6 +9,7 @@ src/borg/crypto/low_level.c
 src/borg/item.c
 src/borg/chunkers/buzhash.c
 src/borg/chunkers/buzhash64.c
+src/borg/chunkers/fastcdc.c
 src/borg/chunkers/reader.c
 src/borg/checksums.c
 src/borg/platform/darwin.c

docs/changes.rst

@@ -176,6 +176,12 @@ New features:
   ``chunker-params`` for buzhash64 gains a required 6th field ``nc_level``
   (``buzhash64,chunk_min,chunk_max,chunk_mask,window_size,nc_level``).
   buzhash (32bit) is unchanged and stays bit-compatible with borg 1.x.
+- new ``fastcdc`` chunker: a FastCDC content-defined chunker using a window-less, keyed Gear
+  rolling hash (the gear table is derived from the repo's id key, like buzhash64, so cut points
+  stay unpredictable without the key). It supports the same normalized chunking as buzhash64 and
+  produces the same chunk-size distribution and deduplication, but chunks roughly 1.3-1.5x faster.
+  Select it via ``--chunker-params fastcdc,chunk_min,chunk_max,chunk_mask,nc_level`` (no window
+  field; e.g. ``fastcdc,19,23,21,2``). ``borg benchmark cpu`` now reports its throughput too.
 - repo-create: split ``--encryption`` into orthogonal options. ``--encryption`` now
   selects only the cipher / AE algorithm (``none``, ``authenticated``, ``aes256-ocb``
   or ``chacha20-poly1305``), the new ``--id-hash`` selects the id hash function

docs/global.rst.inc

@@ -19,6 +19,7 @@
 .. _OpenSSL: https://www.openssl.org/
 .. _`Python 3`: https://www.python.org/
 .. _Buzhash: https://en.wikipedia.org/wiki/Buzhash
+.. _FastCDC: https://www.usenix.org/conference/atc16/technical-sessions/presentation/xia
 .. _msgpack: https://msgpack.org/
 .. _`msgpack-python`: https://pypi.org/project/msgpack-python/
 .. _llfuse: https://pypi.org/project/llfuse/

docs/internals/data-structures.rst

@@ -403,6 +403,8 @@ Borg has these chunkers:
 - "buzhash": variable, content-defined blocksize, uses a rolling hash
   computed by the Buzhash_ algorithm.
 - "buzhash64": similar to "buzhash", but improved 64bit implementation
+- "fastcdc": variable, content-defined blocksize, uses the window-less, keyed
+  Gear rolling hash (FastCDC_); faster than buzhash, same deduplication.
 
 For some more general usage hints see also ``--chunker-params``.
 
@@ -483,6 +485,22 @@ The buzhash table is cryptographically derived from secret key material.
 These changes should improve resistance against attacks and also solve
 some of the issues of the original (32bit / XORed table) implementation.
 
+"fastcdc" chunker
++++++++++++++++++
+
+FastCDC_ content-defined chunker using the Gear rolling hash. Unlike buzhash it
+is window-less (each byte's influence simply decays out of the hash), so its
+update is cheaper and it chunks noticeably faster, while producing the same
+deduplication and (with normalized chunking) the same chunk-size distribution.
+
+Like "buzhash64", the Gear table is cryptographically derived from secret key
+material, so chunk cut points are unpredictable without the key.
+
+``borg create --chunker-params fastcdc,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,NC_LEVEL``
+
+There is no window size (Gear is window-less). NC_LEVEL is the normalized
+chunking level (0 disables it); 2 is a good default. E.g.: ``fastcdc,19,23,21,2``.
+
 .. _cache:
 
 The cache

scripts/chunker_bench.py

@@ -129,7 +129,10 @@ def chunk_stats(algo, data, min_exp, max_exp, mask_bits, win, nc_level=0, normal
     params = [min_exp, max_exp, mask_bits, win]
     kw = dict(key=None, sparse=False)
     if algo == "buzhash64":
-        params.append(nc_level)  # nc_level is a positional buzhash64 param
+        params.append(nc_level)  # nc_level is a positional param
+        kw["normal_size"] = normal_size
+    elif algo == "fastcdc":
+        params = [min_exp, max_exp, mask_bits, nc_level]  # fastcdc is window-less
         kw["normal_size"] = normal_size
     chunker = get_chunker(algo, *params, **kw)
     sizes = []
@@ -285,11 +288,11 @@ def main():
     print(f"shift test: {args.shift_edits} edits   repeats: {args.repeat}")
     print("-" * 118)
 
-    # build (algo, nc_level) variants; for buzhash64 also run the requested NC level
+    # build (algo, nc_level) variants; for buzhash64/fastcdc also run the requested NC level
     variants = []
     for algo in args.algo:
         variants.append((algo, 0))
-        if algo == "buzhash64" and args.nc_level > 0:
+        if algo in ("buzhash64", "fastcdc") and args.nc_level > 0:
             variants.append((algo, args.nc_level))
 
     for algo, nc in variants:

setup.py

@@ -56,6 +56,7 @@
 crypto_legacy_ll_source = "src/borg/legacy/crypto/low_level.pyx"
 buzhash_source = "src/borg/chunkers/buzhash.pyx"
 buzhash64_source = "src/borg/chunkers/buzhash64.pyx"
+fastcdc_source = "src/borg/chunkers/fastcdc.pyx"
 reader_source = "src/borg/chunkers/reader.pyx"
 hashindex_source = "src/borg/hashindex.pyx"
 item_source = "src/borg/item.pyx"
@@ -73,6 +74,7 @@
     crypto_legacy_ll_source,
     buzhash_source,
     buzhash64_source,
+    fastcdc_source,
     reader_source,
     hashindex_source,
     item_source,
@@ -189,6 +191,7 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
         Extension("borg.item", [item_source], extra_compile_args=cflags),
         Extension("borg.chunkers.buzhash", [buzhash_source], extra_compile_args=cflags),
         Extension("borg.chunkers.buzhash64", [buzhash64_source], extra_compile_args=cflags),
+        Extension("borg.chunkers.fastcdc", [fastcdc_source], extra_compile_args=cflags),
         Extension("borg.chunkers.reader", [reader_source], extra_compile_args=cflags),
     ]
 

src/borg/archiver/benchmark_cmd.py

@@ -204,6 +204,8 @@ def chunkit(ch):
                 "chunkit(ch)",
                 locals(),
             ),
+            # fastcdc (window-less keyed gear hash); gear table creation is slow, keep it in setup
+            ("fastcdc,19,23,21,2", "ch = get_chunker('fastcdc', 19, 23, 21, 2, sparse=False)", "chunkit(ch)", locals()),
             ("fixed,1048576", "ch = get_chunker('fixed', 1048576, sparse=False)", "chunkit(ch)", locals()),
         ]:
             dt = timeit(func, setup, number=number_default, globals=vars)

src/borg/archiver/completion_cmd.py

@@ -708,7 +708,13 @@ def do_completion(self, args):
         comp_spec_choices_str = " ".join(comp_spec_choices)
 
         # Chunker params choices (static list)
-        chunker_params_choices = ["default", "fixed,4194304", "buzhash,19,23,21,4095", "buzhash64,19,23,21,4095,2"]
+        chunker_params_choices = [
+            "default",
+            "fixed,4194304",
+            "buzhash,19,23,21,4095",
+            "buzhash64,19,23,21,4095,2",
+            "fastcdc,19,23,21,2",
+        ]
         chunker_params_choices_str = " ".join(chunker_params_choices)
 
         # Relative time marker choices (static list)

src/borg/chunkers/__init__.py

@@ -1,5 +1,6 @@
 from .buzhash import Chunker
 from .buzhash64 import ChunkerBuzHash64
+from .fastcdc import ChunkerFastCDC
 from .failing import ChunkerFailing
 from .fixed import ChunkerFixed
 from .reader import *  # noqa
@@ -10,16 +11,23 @@ def get_chunker(algo, *params, **kw):
     sparse = kw.get("sparse", False)
     # key.chunk_seed only has 32 bits
     seed = key.chunk_seed if key is not None else 0
-    # for buzhash64, we want a much longer key, so we derive it from the id key
-    bh64_key = (
-        key.derive_key(salt=b"", domain=b"buzhash64", size=32, from_id_key=True) if key is not None else b"\0" * 32
-    )
     if algo == "buzhash":
         return Chunker(seed, *params, sparse=sparse)
     if algo == "buzhash64":
+        # for buzhash64, we want a much longer key, so we derive it from the id key.
         # params is (chunk_min_exp, chunk_max_exp, hash_mask_bits, hash_window_size, nc_level);
         # nc_level is passed positionally. normal_size is an optional tuning knob (0 = auto).
+        bh64_key = (
+            key.derive_key(salt=b"", domain=b"buzhash64", size=32, from_id_key=True) if key is not None else b"\0" * 32
+        )
         return ChunkerBuzHash64(bh64_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse)
+    if algo == "fastcdc":
+        # keyed gear table, derived from the id key (own domain). params is
+        # (chunk_min_exp, chunk_max_exp, hash_mask_bits, nc_level) - no window (Gear is window-less).
+        fc_key = (
+            key.derive_key(salt=b"", domain=b"fastcdc", size=32, from_id_key=True) if key is not None else b"\0" * 32
+        )
+        return ChunkerFastCDC(fc_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse)
     if algo == "fixed":
         return ChunkerFixed(*params, sparse=sparse)
     if algo == "fail":