diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index 6af8114b2d..5cec57a0f6 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -356,6 +356,10 @@ Archive formats array under the *archives* key, while :ref:`borg_create` returns a single archive object under the *archive* key. +:ref:`borg_create` with ``--dry-run`` does not create an archive, so there is no *archive* key. +Instead, it returns *dry_run* (true) and a reduced *stats* object with *nfiles* and *original_size*, +both computed from file system metadata without reading the file contents. + Both formats contain a *name* key with the archive name, the *id* key with the hexadecimal archive ID, and the *start* key with the start timestamp. diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index b914e9f204..f3a5f7b4a5 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -274,8 +274,14 @@ does not go to a terminal (e.g. into a logfile), the precise format is always us When using ``--stats``, you will get some statistics about how much data was added - the "This Archive" deduplicated size there is most interesting as that is how much your repository will grow. Please note that the "All archives" stats refer to -the state after creation. Also, the ``--stats`` and ``--dry-run`` options are mutually -exclusive because the data is not actually compressed and deduplicated during a dry run. +the state after creation. + +When ``--stats`` is used together with ``--dry-run``, only the number of files and the +original size are reported. They are computed from file system metadata, without reading +the file contents, so a dry run stays fast. As data is not actually read, chunked, and +deduplicated during a dry run, the deduplicated size is unknown. The sizes of data read +from standard input, from a command's output, or from special files (``--read-special``) +are also unknown in a dry run and counted as zero. The ``--stats`` output also reports the store statistics (lines prefixed with "Store"), taken from the storage layer after this run. These cover the backend and diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py index e3a35c99be..92d96d7fbd 100644 --- a/src/borg/archiver/__init__.py +++ b/src/borg/archiver/__init__.py @@ -473,7 +473,8 @@ def run(self, args): self._setup_implied_logging(vars(args)) self._setup_topic_debugging(args) # extract --dry-run reads, decrypts and decompresses every object, so its store stats are accurate. - stats_supported_with_dry_run = func_name == "do_extract" + # create --dry-run counts the files and sums up their sizes from file system metadata. + stats_supported_with_dry_run = func_name in ("do_extract", "do_create") if getattr(args, "stats", False) and getattr(args, "dry_run", False) and not stats_supported_with_dry_run: logger.warning("Ignoring --stats. It is not supported when using --dry-run.") args.stats = False diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 1727ca14d2..c53c6e149d 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -10,7 +10,7 @@ from ._common import with_repository, Highlander from .. import helpers -from ..archive import Archive, is_special, SF_DATALESS +from ..archive import Archive, Statistics, is_special, SF_DATALESS from ..archive import BackupError, BackupOSError, BackupItemExcluded, backup_io, OsOpen, stat_update_check from ..archive import FilesystemObjectProcessors, MetadataCollector, ChunksProcessor from ..cache import Cache @@ -22,7 +22,7 @@ from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify from ..helpers import dir_is_tagged from ..helpers import log_multi -from ..helpers import basic_json_data, json_print +from ..helpers import basic_json_data, json_print, FileSize from ..helpers import flags_dir, flags_special_follow, flags_special from ..helpers import prepare_subprocess_env from ..helpers import sig_int, ignore_sigint @@ -97,6 +97,7 @@ def create_inner(archive, cache, fso): raise Error(f"{path!r}: {e}") else: status = "+" # included + self.dry_run_stats.nfiles += 1 # size unknown without running the command self.print_file_status(status, path) elif args.paths_from_command or args.paths_from_shell_command or args.paths_from_stdin: paths_sep = eval_escapes(args.paths_delimiter) if args.paths_delimiter is not None else "\n" @@ -174,6 +175,7 @@ def create_inner(archive, cache, fso): status = "E" else: status = "+" # included + self.dry_run_stats.nfiles += 1 # size unknown without reading stdin self.print_file_status(status, path) if not dry_run and status is not None: fso.stats.files_stats[status] += 1 @@ -226,6 +228,7 @@ def create_inner(archive, cache, fso): self.noxattrs = args.noxattrs self.exclude_dataless = args.exclude_dataless dry_run = args.dry_run + self.dry_run_stats = Statistics() if dry_run else None self.start_backup = time.time_ns() t0 = archive_ts_now() logger.info('Creating archive "%s" in repository %s' % (args.name, args.location.processed)) @@ -288,6 +291,24 @@ def create_inner(archive, cache, fso): log_multi(str(archive), str(archive.stats), logger=logging.getLogger("borg.output.stats")) else: create_inner(None, None, None) + args.stats |= args.json + if args.stats: + stats = self.dry_run_stats + if args.json: + json_data = basic_json_data( + manifest, + extra={ + "dry_run": True, + "stats": {"nfiles": stats.nfiles, "original_size": FileSize(stats.osize)}, + }, + ) + json_print(json_data) + else: + log_multi( + f"Number of files: {stats.nfiles}", + f"Original size: {stats.osize_fmt}", + logger=logging.getLogger("borg.output.stats"), + ) def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, dry_run, strip_prefix): """ @@ -295,6 +316,22 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d """ if dry_run: + stats = self.dry_run_stats + if stat.S_ISREG(st.st_mode): + stats.nfiles += 1 + stats.osize += st.st_size + elif read_special: + if stat.S_ISLNK(st.st_mode): + try: + st_target = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=True) + except OSError: + special = False + else: + special = is_special(st_target.st_mode) + else: + special = is_special(st.st_mode) + if special: + stats.nfiles += 1 # size unknown without reading the special file return "+" # included MAX_RETRIES = 10 # count includes the initial try (initial try == "retry 0") for retry in range(MAX_RETRIES): @@ -673,8 +710,14 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): When using ``--stats``, you will get some statistics about how much data was added - the "This Archive" deduplicated size there is most interesting as that is how much your repository will grow. Please note that the "All archives" stats refer to - the state after creation. Also, the ``--stats`` and ``--dry-run`` options are mutually - exclusive because the data is not actually compressed and deduplicated during a dry run. + the state after creation. + + When ``--stats`` is used together with ``--dry-run``, only the number of files and the + original size are reported. They are computed from file system metadata, without reading + the file contents, so a dry run stays fast. As data is not actually read, chunked, and + deduplicated during a dry run, the deduplicated size is unknown. The sizes of data read + from standard input, from a command's output, or from special files (``--read-special``) + are also unknown in a dry run and counted as zero. The ``--stats`` output also reports the store statistics (lines prefixed with "Store"), taken from the storage layer after this run. These cover the backend and @@ -834,8 +877,6 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): subparser = ArgumentParser(parents=[common_parser], description=self.do_create.__doc__, epilog=create_epilog) subparsers.add_subcommand("create", subparser, help="create a backup") - # note: --dry-run and --stats are mutually exclusive, but we do not want to abort when - # parsing, but rather proceed with the dry-run, but without stats (see run() method). subparser.add_argument( "-n", "--dry-run", dest="dry_run", action="store_true", help="do not create a backup archive" ) diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index 15c2f064e8..b3a51ec785 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -687,6 +687,39 @@ def test_create_dry_run(archivers, request): assert manifest.archives.count() == 0 +def test_create_dry_run_stats(archivers, request): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024 * 80) + create_regular_file(archiver.input_path, "file2", size=1024 * 20) + expected_nfiles = 2 + if are_hardlinks_supported(): + os.link(os.path.join(archiver.input_path, "file1"), os.path.join(archiver.input_path, "hardlink1")) + expected_nfiles = 3 # as in a real create, each hardlink counts as a file + cmd(archiver, "repo-create", RK_ENCRYPTION) + output = cmd(archiver, "create", "--dry-run", "--stats", "test", "input") + assert f"Number of files: {expected_nfiles}" in output + assert "Original size:" in output + assert "Deduplicated size:" not in output + # Make sure no archive has been created + with Repository(archiver.repository_path) as repository: + manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + assert manifest.archives.count() == 0 + + +def test_create_dry_run_json(archivers, request): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024 * 80) + create_regular_file(archiver.input_path, "file2", size=1024 * 20) + cmd(archiver, "repo-create", RK_ENCRYPTION) + output = cmd(archiver, "create", "--dry-run", "--json", "test", "input") + result = json.loads(output) + assert result["dry_run"] is True + assert result["stats"]["nfiles"] == 2 + assert result["stats"]["original_size"] == 1024 * 100 + assert "archive" not in result + assert "repository" in result + + def test_progress_on(archivers, request): archiver = request.getfixturevalue(archivers) create_regular_file(archiver.input_path, "file1", size=1024 * 80)