diff --git a/src/cfengine_cli/cfengine_wrapper/arg_parse.py b/src/cfengine_cli/cfengine_wrapper/arg_parse.py index 70362d3..a7b6871 100644 --- a/src/cfengine_cli/cfengine_wrapper/arg_parse.py +++ b/src/cfengine_cli/cfengine_wrapper/arg_parse.py @@ -2,9 +2,41 @@ def parse_wrapper_args(subp: argparse._SubParsersAction): + + sp = subp.add_parser( + "save", help="Save host(s) with a group name to use in other commands" + ) + sp.add_argument( + "--role", + help="Role of the hosts", + choices=["hub", "hubs", "client", "clients"], + required=True, + ) + sp.add_argument( + "--name", + help="Name of the group of hosts (can be used in other commands)", + required=True, + ) + sp.add_argument( + "--hosts", + "-H", + help="SSH usernames and IPs for SSH and CFEngine in the form of user@ip", + required=True, + ) + + sp = subp.add_parser( + "setup-code", help="Fetches a new setup-code for mission-portal login" + ) + sp.add_argument( + "--hub", + "-H", + help="Hub from which to fetch new setup-code", + type=str, + default=None, + ) + subp.add_parser("build", help="Build a policy set from a CFEngine Build project\n\ A wrapper arount the cfbs `build`-function.") - sp = subp.add_parser("deploy", help="Deploy policy-set (masterfiles) to hub\n\ A wrapper around the cf-remote `deploy`-function with some added niceties.") sp.add_argument("--hub", help="Hub(s) to deploy to", type=str) @@ -101,14 +133,19 @@ def parse_wrapper_args(subp: argparse._SubParsersAction): report_parser = subp.add_parser( "report", - help="Run the agent and hub commands necessary to get new reporting data", + help="Refresh reporting data", ) report_parser.add_argument( - "--host", + "--run-agent", + action="store_true", + help="Runs the agent on the chosen host(s) before collecting report data.", + ) + report_parser.add_argument( + "--hub", + "-H", type=str, default=None, - help="Select which installation to use by name/IP (e.g. 'local' or '192.168.56.90'). " - "If omitted and multiple installations of cf-agent+cf-hub are found, you'll be prompted.", + help="Only refresh one hub specified by name/IP (e.g. 'local' or '192.168.56.90') and accompanying clients", ) run_parser = subp.add_parser( diff --git a/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py b/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py index 968c2a3..ba4bba4 100644 --- a/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py +++ b/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py @@ -4,19 +4,20 @@ from cfbs.commands import build_command from cf_remote.commands import deploy as deploy_command from cf_remote.commands import destroy as destroy_command +from cf_remote.commands import save as save_command from cf_remote.remote import run_command, transfer_file from cfengine_cli.utils import UserError from cfengine_cli.cfengine_wrapper.cfengine_objects import ( Executable, - _ensure_default_agent_flags, + ensure_default_agent_flags, ) from cfengine_cli.cfengine_wrapper.cfengine_utils import ( extract_agent_file, prompt_two_options, prompt_yes_no, require_executable, - require_installation, + select_report_targets, ) _DEFAULT_CFENGINE_INPUTS_DIR = "/var/cfengine/inputs" @@ -113,7 +114,7 @@ def _resolve_command_for_agent(agent: Executable, command: str) -> str: if agent.name != "cf-agent": return command - command = _ensure_default_agent_flags(command) + command = ensure_default_agent_flags(command) file_arg = extract_agent_file(command) if not file_arg: return command @@ -131,14 +132,74 @@ def _resolve_command_for_agent(agent: Executable, command: str) -> str: # --------------------------------------------------------------------------- -def report(target: str | None = None) -> int: # TODO? ENT-14122 - installation = require_installation(target) - rc = installation.agent.run("-KIf update.cf", "-KI") - if rc != 0: - return rc - return installation.hub.run( - "--query rebase -H 127.0.0.1", "--query delta -H 127.0.0.1" - ) +def save(hosts: str, role: str, name: str) -> int: # TODO: Add to existing group + return save_command(hosts=hosts, role=role, name=name) + + +def _refresh_agent(agent: Executable) -> int: + try: + return agent.run("-KIf update.cf", "-KI") + except (Exception, SystemExit) as e: + logging.error(f"Skipping {agent.label}: {e}") + return 1 + + +def _query_hub_delta(hub: Executable, client_ips: list[str]) -> int: + """ + Ask a hub to recompute delta report data for itself and + for every client bootstrapped to it. + """ + try: + queries = ["--query delta -H 127.0.0.1"] + [ + f"--query delta -H {ip}" for ip in client_ips + ] + return hub.run(*queries) + except (Exception, SystemExit) as e: + logging.error(f"Skipping hub {hub.label}: {e}") + return 1 + + +def report( + target: str | None = None, + run_agent: bool = False, +) -> int: + errors = 0 + hubs, clients = select_report_targets(target) + + hub_agent_failed = {} + if run_agent: + for hub in hubs: + rc = _refresh_agent(hub.agent) + hub_agent_failed[hub.location] = rc != 0 + if rc != 0: + logging.error(f"Agent run failed on {hub.agent.label}") + errors += 1 + + for agent in clients: + rc = _refresh_agent(agent) + if rc != 0: + logging.error(f"Refresh failed on {agent.label})") + errors += 1 + + for hub in hubs: + if run_agent and hub_agent_failed[hub.location]: + logging.warning( + f"Agent run failed for {hub.location}, some data may be stale." + ) + client_ips = [client.location.split("@", 1)[1] for client in clients] + rc = _query_hub_delta(hub.hub, client_ips) + if rc != 0: + logging.error(f"Hub refresh failed on {hub.label})") + errors += 1 + + if errors > 0: + logging.error(f"Encountered {errors}.") + return errors + + +def setup_code(target: str | None = None) -> int: + hub = require_executable("cf-hub", target) + return hub.run("--new-setup-code") def run(*args, target: str | None = None) -> int: diff --git a/src/cfengine_cli/cfengine_wrapper/cfengine_objects.py b/src/cfengine_cli/cfengine_wrapper/cfengine_objects.py index 854674c..5ac5b9e 100644 --- a/src/cfengine_cli/cfengine_wrapper/cfengine_objects.py +++ b/src/cfengine_cli/cfengine_wrapper/cfengine_objects.py @@ -5,7 +5,7 @@ import os -def _ensure_default_agent_flags(command: str) -> str: +def ensure_default_agent_flags(command: str) -> str: """ cf-agent needs -K (no-lock), -I (inform), and -f (specify file) to actually run against a policy file the way people expect. @@ -52,17 +52,14 @@ def label(self) -> str: return self.location def run(self, *commands) -> int: - rc = 0 + errors = 0 for command in commands: rc = self._run_one(command) if rc != 0: - return rc - return rc + errors += 1 + return errors def _run_one(self, command: str) -> int: - if self.name == "cf-agent": - command = _ensure_default_agent_flags(command) - if self.is_local: args = [self.path] + command.split() # cf-agent picks its workdir based on privilege: as root it uses diff --git a/src/cfengine_cli/cfengine_wrapper/cfengine_utils.py b/src/cfengine_cli/cfengine_wrapper/cfengine_utils.py index 5d37d41..510e6aa 100644 --- a/src/cfengine_cli/cfengine_wrapper/cfengine_utils.py +++ b/src/cfengine_cli/cfengine_wrapper/cfengine_utils.py @@ -1,20 +1,22 @@ import os import shutil import logging -import sys +import random from collections.abc import Iterator +from functools import lru_cache + +from cf_remote.remote import get_info from cfengine_cli.paths import bin from cfengine_cli.utils import UserError from cfengine_cli.cfengine_wrapper.cfengine_objects import Executable, Installation -from cf_remote.remote import get_info from cf_remote.paths import CLOUD_STATE_FPATH from cf_remote.utils import read_json +DEFAULT_MAX_REPORT_HOSTS = 25 + def prompt_yes_no(prompt: str, default: bool = True) -> bool: - if not sys.stdin.isatty(): - raise UserError(f"{prompt} -- no terminal to confirm.") suffix = "[Y/n]" if default else "[y/N]" answer = input(f"{prompt} {suffix} ").strip().lower() if not answer: @@ -85,6 +87,22 @@ def _known_hosts(role_filter=None) -> Iterator[tuple[str, list[str]]]: yield host_id, aliases +@lru_cache(maxsize=None) +def _host_info(host: str): + try: + return get_info(host) or None + except (Exception, SystemExit) as e: + logging.warning(f"Skipping {host}: {e}") + return None + + +def _hosts_with_info(role_filter=None): + for host, aliases in _known_hosts(role_filter): + data = _host_info(host) + if data: + yield host, aliases, data + + def _find_all(binary_name: str) -> list[Executable]: """Every location -- local, plus every matching remote host -- with `binary_name` installed.""" executables = [] @@ -93,75 +111,26 @@ def _find_all(binary_name: str) -> list[Executable]: if local_path: executables.append(Executable(binary_name, "local", local_path)) - role_filter = "hub" if binary_name == "cf-hub" else None - key = "agent" if binary_name == "cf-agent" else "hub" - for host, aliases in _known_hosts(role_filter=role_filter): - try: - data = get_info(host) - except (Exception, SystemExit) as e: - """Need to catch SystemExit as cf-remote's get_info() will SystemExit if - any ssh-connections does not work, for our case we still want to fetch - the ones that are up in case the user wants to use a different host""" - logging.warning(f"Skipping {host}: {e}") - continue - if not data: - continue - binary_path = data.get(key) - if binary_path: - executables.append( - Executable(binary_name, host, binary_path, aliases=aliases) - ) - + is_agent = binary_name == "cf-agent" + for host, aliases, data in _hosts_with_info(None if is_agent else "hub"): + # band-aid: hostinfo has no path for cf-hub, so assume it's on PATH + path = data.get("agent") if is_agent else "cf-hub" + if path: + executables.append(Executable(binary_name, host, path, aliases)) return executables def _find_all_paired() -> list[Installation]: """Every location -- local or remote -- that has BOTH cf-agent and cf-hub.""" - installations = [] - - local_agent_path = _find_local_path("cf-agent") - local_hub_path = _find_local_path("cf-hub") - if local_agent_path and local_hub_path: - installations.append( - Installation( - location="local", - agent=Executable("cf-agent", "local", local_agent_path), - hub=Executable("cf-hub", "local", local_hub_path), - ) - ) - - for host, aliases in _known_hosts(role_filter="hub"): - try: - data = get_info(host) - except (Exception, SystemExit) as e: - # Same reasoning as _find_all() - logging.warning(f"Skipping {host}: {e}") - continue - if not data: - continue - agent_path = data.get("agent") - # If role is hub, assume hub exists and path resolves correctly - is_hub = data.get("role") == "hub" - hub_path = "cf-hub" if is_hub else None - if agent_path and hub_path: - installations.append( - Installation( - location=host, - agent=Executable("cf-agent", host, agent_path, aliases=aliases), - hub=Executable("cf-hub", host, hub_path, aliases=aliases), - ) - ) - - return installations + hubs = {e.location: e for e in _find_all("cf-hub")} + return [ + Installation(agent.location, agent, hubs[agent.location]) + for agent in _find_all("cf-agent") + if agent.location in hubs + ] def _prompt_choice(candidates, description): - if not sys.stdin.isatty(): - labels = ", ".join(c.label for c in candidates) - raise UserError( - f"Multiple installations of {description} found ({labels}) " - f"and no terminal to prompt on. Specify one with --host." - ) print(f"Multiple installations of {description} found:") for i, c in enumerate(candidates, 1): print(f" {i}) {c.label}") @@ -198,6 +167,14 @@ def _select(candidates, description, target: str | None = None): f"Could not find {description} locally or on any configured remote host." ) + if isinstance(target, list): + if len(target) > 1: + raise UserError( + f"Expected a single {description}, but got {len(target)}: " + f"{', '.join(target)}." + ) + target = target[0] if target else None + if target: matches = [c for c in candidates if _exact_match(c, target)] if not matches: @@ -225,9 +202,69 @@ def require_executable(name: str, target: str | None = None) -> Executable: return chosen -def require_installation(target: str | None = None) -> Installation: - chosen = _select(_find_all_paired(), "cf-agent + cf-hub", target) +def select_report_targets( + target: str | None = None, +) -> tuple[list[Installation], list[Executable]]: + """ + Decide which hosts `report()` should refresh new reporting data on. + + - `target` given -> just that one hub + - `target` omitted -> every known hub, and a random + sample of the remaining (non-hub) clients, capped so the + total (hubs + sampled hosts) doesn't exceed 25. + """ + installations = _find_all_paired() + if not installations: + raise UserError( + "Could not find any installation of cf-agent + cf-hub locally " + "or on any configured remote host." + ) + + if target: + hub_client_map = clients_by_hub_ip() + chosen_hub = _select(installations, "cf-agent + cf-hub", target) + hub_ip = None if chosen_hub.is_local else chosen_hub.location.split("@", 1)[1] + installations = [chosen_hub] + other_agents = [ + agent + for agent in _find_all("cf-agent") + if hub_ip + and any( + client_ip in agent.location + for client_ip in hub_client_map.get(hub_ip, []) + ) + ] + + else: + hub_locations = {installation.location for installation in installations} + other_agents = [ + agent + for agent in _find_all("cf-agent") + if agent.location not in hub_locations + ] + + cap = DEFAULT_MAX_REPORT_HOSTS + budget = max(0, cap - len(installations)) + if len(other_agents) <= budget: + return installations, other_agents + + sampled_agents = random.sample(other_agents, budget) logging.warning( - f"Using {'local' if chosen.is_local else 'remote'} installation of cf-agent and cf-hub ({chosen.label})" + f"{len(other_agents)} additional host(s) found; refreshing a random " + f"{budget} of them (plus {len(installations)} hub(s)) to keep this fast. " ) - return chosen + return installations, sampled_agents + + +def clients_by_hub_ip() -> dict[str, list[str]]: + """ + Maps each hub's IP (as every host reports it via its 'policy_server' + field) to the list of client IPs bootstrapped to it. + """ + mapping: dict[str, list[str]] = {} + for host, _, data in _hosts_with_info(): + policy_server = data.get("policy_server") + client_ip = data.get("ssh_host") or host.split("@", 1)[1] + if policy_server and policy_server != client_ip: + mapping.setdefault(policy_server, []).append(client_ip) + return mapping diff --git a/src/cfengine_cli/main.py b/src/cfengine_cli/main.py index 6bdcc13..00823b9 100644 --- a/src/cfengine_cli/main.py +++ b/src/cfengine_cli/main.py @@ -179,6 +179,8 @@ def run_command_with_args(args) -> int: if args.command == "version": return commands.version() # The real commands: + if args.command == "save": + return cfengine_commands.save(hosts=args.hosts, role=args.role, name=args.name) if args.command == "build": return cfengine_commands.build() if args.command == "deploy": @@ -192,10 +194,13 @@ def run_command_with_args(args) -> int: args.syntax_description, ) if args.command == "report": - return cfengine_commands.report(target=args.host) - + return cfengine_commands.report( + target=args.hub, + run_agent=args.run_agent, + ) + if args.command == "setup-code": + return cfengine_commands.setup_code(target=args.hub) if args.command == "install": - print(args) if args.trust_keys: trust_keys = args.trust_keys.split(",") else: