diff --git a/Jenkinsfile_config_dir b/Jenkinsfile_config_dir index 1720fde..f269316 100644 --- a/Jenkinsfile_config_dir +++ b/Jenkinsfile_config_dir @@ -19,7 +19,6 @@ pipeline { skipDefaultCheckout(true) } environment { - SSH_CREDENTIALS = credentials('SSH') TEST_INSTRUMENT_LIST = "${TEST_INSTRUMENT_LIST}" USE_TEST_INSTRUMENT_LIST = "${USE_TEST_INSTRUMENT_LIST}" DEBUG_MODE = "${DEBUG_MODE}" @@ -42,11 +41,18 @@ pipeline { stage('Check Instrument has any Hotfixes and then any uncommitteed changes') { steps { echo 'Check Instrument has any commits or any uncommitteed changes' - timeout(time: 1, unit: 'HOURS') { - bat ''' - call get_python.bat - python -u hotfix_checker.py - ''' + withCredentials([sshUserPrivateKey(credentialsId: '9f81ea0c-9740-4e2e-b58a-46d426645acb', + usernameVariable: 'SSH_CREDENTIALS_USER', + passphraseVariable: 'SSH_CREDENTIALS_PASSPHRASE', + keyFileVariable: 'SSH_CREDENTIALS_KEY_FILE')]) { + timeout(time: 1, unit: 'HOURS') { + bat ''' + @echo off + setlocal + call get_python.bat + python -u hotfix_checker.py + ''' + } } } } diff --git a/Jenkinsfile_epics_dir b/Jenkinsfile_epics_dir index e004f74..db50cba 100644 --- a/Jenkinsfile_epics_dir +++ b/Jenkinsfile_epics_dir @@ -20,7 +20,6 @@ pipeline { } environment { - SSH_CREDENTIALS = credentials('SSH') TEST_INSTRUMENT_LIST = "${TEST_INSTRUMENT_LIST}" USE_TEST_INSTRUMENT_LIST = "${USE_TEST_INSTRUMENT_LIST}" DEBUG_MODE = "${DEBUG_MODE}" @@ -43,11 +42,18 @@ pipeline { stage('Check Instrument has any Hotfixes and then any uncommitteed changes') { steps { echo 'Check Instrument has any Hotfixes and then any uncommitteed changes' - timeout(time: 1, unit: 'HOURS') { - bat ''' - call get_python.bat - python -u hotfix_checker.py - ''' + withCredentials([sshUserPrivateKey(credentialsId: '9f81ea0c-9740-4e2e-b58a-46d426645acb', + usernameVariable: 'SSH_CREDENTIALS_USER', + passphraseVariable: 'SSH_CREDENTIALS_PASSPHRASE', + keyFileVariable: 'SSH_CREDENTIALS_KEY_FILE')]) { + timeout(time: 1, unit: 'HOURS') { + bat ''' + @echo off + setlocal + call get_python.bat + python -u hotfix_checker.py + ''' + } } } } diff --git a/hotfix_checker.py b/hotfix_checker.py index 753c1e4..146ebb3 100644 --- a/hotfix_checker.py +++ b/hotfix_checker.py @@ -1,4 +1,8 @@ -"""Creates a RepoChecker object and calls the check_instruments method to check for changes in the instruments repository.""" +"""Checks a repository. + +Creates a RepoChecker object and calls the check_instruments method +to check for changes in the instruments repository. +""" import os @@ -7,10 +11,13 @@ # Load environment variables from .env file load_dotenv(find_dotenv()) -# importing here so it doesn't set variables that are populated from env vars before actually having the env vars loaded +# importing here so it doesn't set variables that are populated from env vars +# before actually having the env vars loaded # needed for when running locally to get the contents of a .env fil # Jenkins will have the env vars set in the pipeline -from utils.hotfix_utils.RepoChecker import RepoChecker +from utils.hotfix_utils.repo_checker import ( + RepoChecker, # see above comments +) if __name__ == "__main__": if os.environ["DEBUG_MODE"] == "true": @@ -18,9 +25,10 @@ print(f"INFO: REPO_DIR: {os.environ['REPO_DIR']}") print(f"INFO: UPSTREAM_BRANCH: {os.environ['UPSTREAM_BRANCH_CONFIG']}") print(f"INFO: ARTEFACT_DIR: {os.environ['WORKSPACE']}") - print(f"INFO: USE_TEST_INSTRUMENT_LIST: {os.environ['USE_TEST_INSTRUMENT_LIST']}") + print( + f"INFO: USE_TEST_INSTRUMENT_LIST: {os.environ['USE_TEST_INSTRUMENT_LIST']}" + ) print(f"INFO: TEST_INSTRUMENT_LIST: {os.environ['TEST_INSTRUMENT_LIST']}") print(f"INFO: DEBUG_MODE: {os.environ['DEBUG_MODE']}") - repo_checker = RepoChecker() repo_checker.check_instruments() diff --git a/utils/communication_utils/ssh_access.py b/utils/communication_utils/ssh_access.py index 78d7ac6..7e66982 100644 --- a/utils/communication_utils/ssh_access.py +++ b/utils/communication_utils/ssh_access.py @@ -1,28 +1,28 @@ -"""This module provides utilities for SSH access.""" - -from typing import Dict +"""Module provides utilities for SSH access.""" import paramiko SSH_PORT = 22 -class SSHAccessUtils(object): +class SSHAccessUtils: """Class containing utility methods for SSH access.""" @staticmethod def run_ssh_command( host: str, username: str, - password: str, + key_file: str, + passphrase: str, command: str, - ) -> Dict[str, bool | str]: + ) -> dict[str, bool | str]: """Run a command on a remote host using SSH. Args: host (str): The hostname to connect to. username (str): The username to use to connect. - password (str): The password to use to connect. + key_file (str): The ssh key file to use to connect. + passphrase (str): The ssh key passphrase to use. command (str): The command to run on the remote host. Returns: @@ -36,15 +36,16 @@ def run_ssh_command( host, port=SSH_PORT, username=username, - password=password, + key_filename=key_file, + passphrase=passphrase, ) ( - stdin, + _stdin, stdout, stderr, ) = client.exec_command(command) - output = stdout.read().decode("utf-8") - error = stderr.read().decode("utf-8") + output = stdout.read().decode("utf-8", errors="backslashreplace") + error = stderr.read().decode("utf-8", errors="backslashreplace") client.close() if error: return { @@ -56,7 +57,7 @@ def run_ssh_command( "success": True, "output": output, } - except Exception as e: + except Exception as e: # noqa: BLE001 print(str(e)) return { "success": False, diff --git a/utils/hotfix_utils/InstrumentChecker.py b/utils/hotfix_utils/instrument_checker.py similarity index 68% rename from utils/hotfix_utils/InstrumentChecker.py rename to utils/hotfix_utils/instrument_checker.py index b7eca4d..4472af6 100644 --- a/utils/hotfix_utils/InstrumentChecker.py +++ b/utils/hotfix_utils/instrument_checker.py @@ -1,7 +1,7 @@ """A module for checking the status of an instrument in relation to it's repo.""" import os -from typing import List, Tuple, Union +from typing import Any from ..communication_utils.ssh_access import ( SSHAccessUtils, @@ -15,14 +15,22 @@ class InstrumentChecker: repo_dir = os.environ["REPO_DIR"] - def __init__(self, hostname: str) -> None: + def __init__( + self, hostname: str, ssh_username: str, ssh_key_file: str, ssh_passphrase: str + ) -> None: """Initialize the Instrument object. Args: hostname (str): The hostname of the instrument. + ssh_username (str): The ssh username. + ssh_key_file (str): The ssh key file. + ssh_passphrase (str): The ssh passphrase. """ self._hostname = hostname + self._ssh_username = ssh_username + self._ssh_key_file = ssh_key_file + self._ssh_passphrase = ssh_passphrase self._commits_local_not_on_upstream_enum = None self._commits_local_not_on_upstream_messages = None @@ -43,37 +51,80 @@ def hostname(self) -> str: """ return self._hostname - def check_for_uncommitted_changes(self) -> Tuple[CHECK, List[any]]: - """Check if there are any uncommitted changes on the instrument via SSH. + @property + def ssh_username(self) -> str: + """Get the ssh username for the instrument. - Args: - hostname (str): The hostname to connect to. + Returns: + str: The ssh username for the instrument. + + """ + return self._ssh_username + + @property + def ssh_key_file(self) -> str: + """Get the ssh key file for the instrument. Returns: - CHECK: The result of the check. + str: The ssh key file for the instrument. """ - command = f"cd /d {self.repo_dir} && git status --porcelain" - ssh_process = SSHAccessUtils.run_ssh_command( + return self._ssh_key_file + + @property + def ssh_passphrase(self) -> str: + """Get the ssh passphrase for the key. + + Returns: + str: The ssh passphrase for the key. + + """ + return self._ssh_passphrase + + def run_ssh_command(self, command: str) -> dict[str, bool | str]: + """Run ssh command. + + Returns: + dict: command result and output. + + """ + ssh_out = SSHAccessUtils.run_ssh_command( self.hostname, - os.environ["SSH_CREDENTIALS_USR"], - os.environ["SSH_CREDENTIALS_PSW"], + self.ssh_username, + self.ssh_key_file, + self.ssh_passphrase, command, ) - if os.environ["DEBUG_MODE"] == "true": print(f"DEBUG: Running command {command}") + return ssh_out - command = f"cd /d {self.repo_dir} && git --no-pager diff --ignore-cr-at-eol" - ssh_process_diff = SSHAccessUtils.run_ssh_command( - self.hostname, - os.environ["SSH_CREDENTIALS_USR"], - os.environ["SSH_CREDENTIALS_PSW"], - command, + def check_for_uncommitted_changes(self) -> tuple[CHECK, list[Any]]: + """Check if there are any uncommitted changes on the instrument via SSH. + + Args: + hostname (str): The hostname to connect to. + + Returns: + CHECK: The result of the check. + + """ + xout = self.run_ssh_command( + f"cd /d {self.repo_dir} && " + r'"c:\Program Files\Git\bin\bash.exe" -c ' + '"git ls-files | grep ^tools/master/ | ' + 'xargs echo"' ) + # 'xargs git update-index --assume-unchanged"') + print(xout["output"]) - if os.environ["DEBUG_MODE"] == "true": - print(f"DEBUG: Running command {command}") + ssh_process = self.run_ssh_command( + f"cd /d {self.repo_dir} && git status --porcelain" + ) + + ssh_process_diff = self.run_ssh_command( + f"cd /d {self.repo_dir} && git --no-pager diff --ignore-cr-at-eol" + ) if ssh_process["success"]: status = ssh_process["output"] @@ -81,10 +132,15 @@ def check_for_uncommitted_changes(self) -> Tuple[CHECK, List[any]]: status_save = status + "\n\n" + ssh_process_diff["output"] else: status_save = status - JenkinsUtils.save_git_status(self.hostname, status_save, os.environ["WORKSPACE"]) + JenkinsUtils.save_git_status( + self.hostname, str(status_save), os.environ["WORKSPACE"] + ) status_stripped = status.strip() - if status_stripped != "" and os.environ["SHOW_UNCOMMITTED_CHANGES_MESSAGES"] == "true": + if ( + status_stripped != "" + and os.environ["SHOW_UNCOMMITTED_CHANGES_MESSAGES"] == "true" + ): return CHECK.TRUE, status_stripped.split("\n") elif status_stripped != "": return CHECK.TRUE, [] @@ -93,10 +149,15 @@ def check_for_uncommitted_changes(self) -> Tuple[CHECK, List[any]]: else: return CHECK.UNDETERMINABLE, [] + # ssh_set_unchanged = self.run_ssh_command(f'cd /d {self.repo_dir} && ' + # '"c:\Program Files\Git\bin\bash.exe" -c ' + # '"git ls-files | grep ^tools/master/ | ' + # 'xargs git update-index --no-assume-unchanged"') + def get_parent_epics_branch( self, hostname: str, - ) -> Union[str | bool]: + ) -> str | bool: """Get the parent branch of the instrument branch. Args: @@ -109,8 +170,9 @@ def get_parent_epics_branch( command = f"cd /d {self.repo_dir} && git log" ssh_process = SSHAccessUtils.run_ssh_command( hostname, - os.environ["SSH_CREDENTIALS_USR"], - os.environ["SSH_CREDENTIALS_PSW"], + self.ssh_username, + self.ssh_key_file, + self.ssh_passphrase, command, ) if ssh_process["success"]: @@ -125,8 +187,8 @@ def git_branch_comparer( self, hostname: str, changes_on: str, - subtracted_against: str = None, - prefix: str = None, + subtracted_against: str | None = None, + prefix: str | None = None, ) -> CHECK: """Get the commit messages between two branches on the instrument. @@ -151,8 +213,9 @@ def git_branch_comparer( fetch_command = f"cd /d {self.repo_dir} && git fetch origin" ssh_process_fetch = SSHAccessUtils.run_ssh_command( hostname, - os.environ["SSH_CREDENTIALS_USR"], - os.environ["SSH_CREDENTIALS_PSW"], + self.ssh_username, + self.ssh_key_file, + self.ssh_passphrase, fetch_command, ) @@ -172,8 +235,9 @@ def git_branch_comparer( ssh_process = SSHAccessUtils.run_ssh_command( hostname, - os.environ["SSH_CREDENTIALS_USR"], - os.environ["SSH_CREDENTIALS_PSW"], + self.ssh_username, + self.ssh_key_file, + self.ssh_passphrase, command, ) @@ -203,9 +267,11 @@ def split_git_log(self, git_log: str, prefix: str) -> dict: Args: git_log (str): The git log to split. + prefix (str): message prefix to match Returns: - dict: A dictionary with the commit hashes as keys and the commit messages as values. + dict: A dictionary with the commit hashes as keys and the + commit messages as values. """ commit_dict = {} @@ -229,7 +295,8 @@ def check_instrument(self) -> dict: dict: A dictionary with the result of the checks. """ - # Examples of how to use the git_branch_comparer function decided to not be used in this iteration of the check + # Examples of how to use the git_branch_comparer function decided to not + # be used in this iteration of the check # Check if any hotfixes run on the instrument with the prefix "Hotfix:" # hotfix_commits_enum, hotfix_commits_messages = git_branch_comparer( # hostname, local_branch, upstream_branch, prefix="Hotfix:") @@ -245,7 +312,9 @@ def check_instrument(self) -> dict: elif os.environ["UPSTREAM_BRANCH_CONFIG"] == "master": upstream_branch = "origin/master" else: - # if the UPSTREAM_BRANCH_CONFIG is not set to any of the above, set it to the value of the environment variable assuming user wants custom branch + # if the UPSTREAM_BRANCH_CONFIG is not set to any of the above, + # set it to the value of the environment variable assuming user + # wants custom branch upstream_branch = os.environ["UPSTREAM_BRANCH_CONFIG"] # Check if any commits on upstream that are not on the local branch @@ -282,4 +351,11 @@ def as_string(self) -> str: str: The Instrument object as a string. """ - return f"Hostname: {self.hostname} - Uncommitted changes: {self.uncommitted_changes_enum} - Commits on local not on upstream: {self.commits_local_not_on_upstream_enum} - Commits on upstream not on local: {self.commits_upstream_not_on_local_enum}" + return ( + f"Hostname: {self.hostname} - " + f"Uncommitted changes: {self.uncommitted_changes_enum} " + "- Commits on local not on upstream: " + f"{self.commits_local_not_on_upstream_enum} " + "- Commits on upstream not on local: " + f"{self.commits_upstream_not_on_local_enum}" + ) diff --git a/utils/hotfix_utils/RepoChecker.py b/utils/hotfix_utils/repo_checker.py similarity index 73% rename from utils/hotfix_utils/RepoChecker.py rename to utils/hotfix_utils/repo_checker.py index 1fe7d61..5dc7a76 100644 --- a/utils/hotfix_utils/RepoChecker.py +++ b/utils/hotfix_utils/repo_checker.py @@ -1,4 +1,7 @@ -"""Contains the RepoChecker class which is used to check the status of a specified repo on an instrument.""" +"""Contains the RepoChecker class. + +This is used to check the status of a specified repo on an instrument. +""" import os import sys @@ -6,7 +9,7 @@ import requests from packaging.version import InvalidVersion, Version -from utils.hotfix_utils.InstrumentChecker import InstrumentChecker +from utils.hotfix_utils.instrument_checker import InstrumentChecker from ..communication_utils.channel_access import ( ChannelAccessUtils, @@ -27,8 +30,12 @@ def __init__(self) -> None: self.use_test_inst_list = os.environ["USE_TEST_INSTRUMENT_LIST"] == "true" self.test_inst_list = os.environ["TEST_INSTRUMENT_LIST"] self.debug_mode = os.environ["DEBUG_MODE"] == "true" + self.ssh_username = os.environ["SSH_CREDENTIALS_USER"] + self.ssh_key_file = os.environ["SSH_CREDENTIALS_KEY_FILE"] + self.ssh_passphrase = os.environ["SSH_CREDENTIALS_PASSPHRASE"] - # You can get the versions of insts a variety of ways, inst config, CS:VERSION:SVN:REV pv etc + # You can get the versions of insts a variety of ways, inst config, + # CS:VERSION:SVN:REV pv etc def get_insts_on_latest_ibex_via_inst_config(self) -> list: """Get a list of instruments that are on the latest version of IBEX. @@ -41,15 +48,18 @@ def get_insts_on_latest_ibex_via_inst_config(self) -> list: for instrument in instrument_list: if not instrument["seci"]: version_string = requests.get( - "https://control-svcs.isis.cclrc.ac.uk/git/?p=instconfigs/inst.git;a=blob_plain;f=configurations/config_version.txt;hb=refs/heads/" - + instrument["hostName"], verify=False + "https://control-svcs.isis.cclrc.ac.uk/git/?p=instconfigs/inst.git;" + "a=blob_plain;f=configurations/config_version.txt;hb=refs/heads/" + + instrument["hostName"], + verify=False, ).text try: version = Version(version_string) if self.debug_mode: print( - f"DEBUG: Found instrument {instrument['name']} on IBEX version {version}" + f"DEBUG: Found instrument {instrument['name']} " + f"on IBEX version {version}" ) result_list.append( { @@ -59,23 +69,28 @@ def get_insts_on_latest_ibex_via_inst_config(self) -> list: ) except InvalidVersion as e: print( - f"Could not parse {instrument['name']}'s Version({version_string}): {str(e)}" + f"Could not parse {instrument['name']}'s " + f"Version({version_string}): {e!s}" ) # Get the latest versions of IBEX - versions = sorted(set([inst["version"] for inst in result_list])) + versions = sorted({inst["version"] for inst in result_list}) latest_major_version = versions[-1].major second_latest_major_version = latest_major_version - 1 print( - f"INFO: checking versions {latest_major_version}.x.x and {second_latest_major_version}.x.x" + f"INFO: checking versions {latest_major_version}.x.x " + f"and {second_latest_major_version}.x.x" ) # filter out the instruments that are not on the latest version insts_on_latest_ibex = [ inst["hostname"] for inst in result_list - if (inst["version"].major in [latest_major_version, second_latest_major_version, 15, 14]) + if ( + inst["version"].major + in [latest_major_version, second_latest_major_version, 15, 14] + ) ] return insts_on_latest_ibex @@ -103,14 +118,22 @@ def check_instruments(self) -> None: self._commits_on_upstream_not_local_key: [], } - def update_instrument_status_lists(instrument, status_list_key, messages=None): + def update_instrument_status_lists( + instrument: InstrumentChecker, + status_list_key: str, + messages: list[str] | None = None, + ) -> None: if messages: - instrument_status_lists[status_list_key].append({instrument.hostname: messages}) + instrument_status_lists[status_list_key].append( + {instrument.hostname: messages} + ) else: instrument_status_lists[status_list_key].append(instrument.hostname) for hostname in instrument_list: - instrument = InstrumentChecker(hostname) + instrument = InstrumentChecker( + hostname, self.ssh_username, self.ssh_key_file, self.ssh_passphrase + ) try: print(f"INFO: Checking {instrument.hostname}") instrument.check_instrument() @@ -139,22 +162,29 @@ def update_instrument_status_lists(instrument, status_list_key, messages=None): ) if ( - instrument.commits_local_not_on_upstream_enum == CHECK.UNDETERMINABLE + instrument.commits_local_not_on_upstream_enum + == CHECK.UNDETERMINABLE or instrument.uncommitted_changes_enum == CHECK.UNDETERMINABLE - or instrument.commits_upstream_not_on_local_enum == CHECK.UNDETERMINABLE + or instrument.commits_upstream_not_on_local_enum + == CHECK.UNDETERMINABLE ): update_instrument_status_lists( instrument, self._undeterminable_at_some_point_key ) - except Exception as e: - print(f"ERROR: Could not connect to {instrument.hostname} ({str(e)})") - update_instrument_status_lists(instrument, self._undeterminable_at_some_point_key) + except Exception as e: # noqa: BLE001 + print(f"ERROR: Could not connect to {instrument.hostname} ({e!s})") + update_instrument_status_lists( + instrument, self._undeterminable_at_some_point_key + ) keys_and_prefixes = [ (self._uncommitted_changes_key, "Uncommitted changes"), (self._commits_on_local_not_upstream_key, "Commits on local not upstream"), - (self._commits_on_upstream_not_local_key, "Commits on upstream not on local"), + ( + self._commits_on_upstream_not_local_key, + "Commits on upstream not on local", + ), (self._undeterminable_at_some_point_key, "Undeterminable at some point"), ] @@ -166,10 +196,11 @@ def update_instrument_status_lists(instrument, status_list_key, messages=None): else: print(f"{prefix}: {status_list}".replace("'", '"')) - for key in instrument_status_lists: - if len(instrument_status_lists[key]) > 0: + for key, value in instrument_status_lists.items(): + if len(value) > 0: sys.exit(1) - # If no instruments have uncommitted changes, local branch matches upstream branch, and no undeterminable results then - # exit with ok status + # If no instruments have uncommitted changes, + # local branch matches upstream branch, + # and no undeterminable results then exit with ok status sys.exit(0) diff --git a/utils/jenkins_utils/jenkins_utils.py b/utils/jenkins_utils/jenkins_utils.py index a00d773..2f565f0 100644 --- a/utils/jenkins_utils/jenkins_utils.py +++ b/utils/jenkins_utils/jenkins_utils.py @@ -30,6 +30,6 @@ def save_git_status( with open( os.path.join(artefact_dir, "git_status", f"{hostname}.txt"), "w", - encoding="utf-8" + encoding="utf-8", ) as file: file.write(status)