create: support --stats with --dry-run

docs/internals/frontends.rst

@@ -356,6 +356,10 @@ Archive formats
 array under the *archives* key, while :ref:`borg_create` returns a single archive object under the
 *archive* key.
 
+:ref:`borg_create` with ``--dry-run`` does not create an archive, so there is no *archive* key.
+Instead, it returns *dry_run* (true) and a reduced *stats* object with *nfiles* and *original_size*,
+both computed from file system metadata without reading the file contents.
+
 Both formats contain a *name* key with the archive name, the *id* key with the hexadecimal archive ID,
 and the *start* key with the start timestamp.
 

docs/usage/create.rst.inc

@@ -274,8 +274,14 @@ does not go to a terminal (e.g. into a logfile), the precise format is always us
 When using ``--stats``, you will get some statistics about how much data was
 added - the "This Archive" deduplicated size there is most interesting as that is
 how much your repository will grow. Please note that the "All archives" stats refer to
-the state after creation. Also, the ``--stats`` and ``--dry-run`` options are mutually
-exclusive because the data is not actually compressed and deduplicated during a dry run.
+the state after creation.
+
+When ``--stats`` is used together with ``--dry-run``, only the number of files and the
+original size are reported. They are computed from file system metadata, without reading
+the file contents, so a dry run stays fast. As data is not actually read, chunked, and
+deduplicated during a dry run, the deduplicated size is unknown. The sizes of data read
+from standard input, from a command's output, or from special files (``--read-special``)
+are also unknown in a dry run and counted as zero.
 
 The ``--stats`` output also reports the store statistics (lines prefixed with
 "Store"), taken from the storage layer after this run. These cover the backend and

src/borg/archiver/__init__.py

@@ -473,7 +473,8 @@ def run(self, args):
         self._setup_implied_logging(vars(args))
         self._setup_topic_debugging(args)
         # extract --dry-run reads, decrypts and decompresses every object, so its store stats are accurate.
-        stats_supported_with_dry_run = func_name == "do_extract"
+        # create --dry-run counts the files and sums up their sizes from file system metadata.
+        stats_supported_with_dry_run = func_name in ("do_extract", "do_create")
         if getattr(args, "stats", False) and getattr(args, "dry_run", False) and not stats_supported_with_dry_run:
             logger.warning("Ignoring --stats. It is not supported when using --dry-run.")
             args.stats = False

src/borg/archiver/create_cmd.py

@@ -10,7 +10,7 @@
 
 from ._common import with_repository, Highlander
 from .. import helpers
-from ..archive import Archive, is_special, SF_DATALESS
+from ..archive import Archive, Statistics, is_special, SF_DATALESS
 from ..archive import BackupError, BackupOSError, BackupItemExcluded, backup_io, OsOpen, stat_update_check
 from ..archive import FilesystemObjectProcessors, MetadataCollector, ChunksProcessor
 from ..cache import Cache
@@ -22,7 +22,7 @@
 from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify
 from ..helpers import dir_is_tagged
 from ..helpers import log_multi
-from ..helpers import basic_json_data, json_print
+from ..helpers import basic_json_data, json_print, FileSize
 from ..helpers import flags_dir, flags_special_follow, flags_special
 from ..helpers import prepare_subprocess_env
 from ..helpers import sig_int, ignore_sigint
@@ -97,6 +97,7 @@ def create_inner(archive, cache, fso):
                         raise Error(f"{path!r}: {e}")
                 else:
                     status = "+"  # included
+                    self.dry_run_stats.nfiles += 1  # size unknown without running the command
                 self.print_file_status(status, path)
             elif args.paths_from_command or args.paths_from_shell_command or args.paths_from_stdin:
                 paths_sep = eval_escapes(args.paths_delimiter) if args.paths_delimiter is not None else "\n"
@@ -174,6 +175,7 @@ def create_inner(archive, cache, fso):
                                 status = "E"
                         else:
                             status = "+"  # included
+                            self.dry_run_stats.nfiles += 1  # size unknown without reading stdin
                         self.print_file_status(status, path)
                         if not dry_run and status is not None:
                             fso.stats.files_stats[status] += 1
@@ -226,6 +228,7 @@ def create_inner(archive, cache, fso):
         self.noxattrs = args.noxattrs
         self.exclude_dataless = args.exclude_dataless
         dry_run = args.dry_run
+        self.dry_run_stats = Statistics() if dry_run else None
         self.start_backup = time.time_ns()
         t0 = archive_ts_now()
         logger.info('Creating archive "%s" in repository %s' % (args.name, args.location.processed))
@@ -288,13 +291,47 @@ def create_inner(archive, cache, fso):
                     log_multi(str(archive), str(archive.stats), logger=logging.getLogger("borg.output.stats"))
         else:
             create_inner(None, None, None)
+            args.stats |= args.json
+            if args.stats:
+                stats = self.dry_run_stats
+                if args.json:
+                    json_data = basic_json_data(
+                        manifest,
+                        extra={
+                            "dry_run": True,
+                            "stats": {"nfiles": stats.nfiles, "original_size": FileSize(stats.osize)},
+                        },
+                    )
+                    json_print(json_data)
+                else:
+                    log_multi(
+                        f"Number of files: {stats.nfiles}",
+                        f"Original size: {stats.osize_fmt}",
+                        logger=logging.getLogger("borg.output.stats"),
+                    )
 
     def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, dry_run, strip_prefix):
         """
         Call the right method on the given FilesystemObjectProcessor.
         """
 
         if dry_run:
+            stats = self.dry_run_stats
+            if stat.S_ISREG(st.st_mode):
+                stats.nfiles += 1
+                stats.osize += st.st_size
+            elif read_special:
+                if stat.S_ISLNK(st.st_mode):
+                    try:
+                        st_target = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=True)
+                    except OSError:
+                        special = False
+                    else:
+                        special = is_special(st_target.st_mode)
+                else:
+                    special = is_special(st.st_mode)
+                if special:
+                    stats.nfiles += 1  # size unknown without reading the special file
             return "+"  # included
         MAX_RETRIES = 10  # count includes the initial try (initial try == "retry 0")
         for retry in range(MAX_RETRIES):
@@ -673,8 +710,14 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
         When using ``--stats``, you will get some statistics about how much data was
         added - the "This Archive" deduplicated size there is most interesting as that is
         how much your repository will grow. Please note that the "All archives" stats refer to
-        the state after creation. Also, the ``--stats`` and ``--dry-run`` options are mutually
-        exclusive because the data is not actually compressed and deduplicated during a dry run.
+        the state after creation.
+
+        When ``--stats`` is used together with ``--dry-run``, only the number of files and the
+        original size are reported. They are computed from file system metadata, without reading
+        the file contents, so a dry run stays fast. As data is not actually read, chunked, and
+        deduplicated during a dry run, the deduplicated size is unknown. The sizes of data read
+        from standard input, from a command's output, or from special files (``--read-special``)
+        are also unknown in a dry run and counted as zero.
 
         The ``--stats`` output also reports the store statistics (lines prefixed with
         "Store"), taken from the storage layer after this run. These cover the backend and
@@ -834,8 +877,6 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
         subparser = ArgumentParser(parents=[common_parser], description=self.do_create.__doc__, epilog=create_epilog)
         subparsers.add_subcommand("create", subparser, help="create a backup")
 
-        # note: --dry-run and --stats are mutually exclusive, but we do not want to abort when
-        #  parsing, but rather proceed with the dry-run, but without stats (see run() method).
         subparser.add_argument(
             "-n", "--dry-run", dest="dry_run", action="store_true", help="do not create a backup archive"
         )

src/borg/testsuite/archiver/create_cmd_test.py

@@ -687,6 +687,39 @@ def test_create_dry_run(archivers, request):
         assert manifest.archives.count() == 0
 
 
+def test_create_dry_run_stats(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file1", size=1024 * 80)
+    create_regular_file(archiver.input_path, "file2", size=1024 * 20)
+    expected_nfiles = 2
+    if are_hardlinks_supported():
+        os.link(os.path.join(archiver.input_path, "file1"), os.path.join(archiver.input_path, "hardlink1"))
+        expected_nfiles = 3  # as in a real create, each hardlink counts as a file
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    output = cmd(archiver, "create", "--dry-run", "--stats", "test", "input")
+    assert f"Number of files: {expected_nfiles}" in output
+    assert "Original size:" in output
+    assert "Deduplicated size:" not in output
+    # Make sure no archive has been created
+    with Repository(archiver.repository_path) as repository:
+        manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
+        assert manifest.archives.count() == 0
+
+
+def test_create_dry_run_json(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file1", size=1024 * 80)
+    create_regular_file(archiver.input_path, "file2", size=1024 * 20)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    output = cmd(archiver, "create", "--dry-run", "--json", "test", "input")
+    result = json.loads(output)
+    assert result["dry_run"] is True
+    assert result["stats"]["nfiles"] == 2
+    assert result["stats"]["original_size"] == 1024 * 100
+    assert "archive" not in result
+    assert "repository" in result
+
+
 def test_progress_on(archivers, request):
     archiver = request.getfixturevalue(archivers)
     create_regular_file(archiver.input_path, "file1", size=1024 * 80)