Merge pull request #56 from Albireo-B/Refactor/Vip_optional_inputs

VIP input maps handling

Author: axlbonnet

src/vip_client/classes/VipGirder.py

@@ -119,7 +119,7 @@ def output_location(self, value: str) -> None:
     ################ Constructor ##################
                     ############# 
     def __init__(
-        self, output_location='girder', output_dir=None, pipeline_id: str=None, input_settings: dict=None,
+        self, output_location='girder', output_dir=None, pipeline_id: str=None, input_settings: dict | list[dict]=None,
         session_name: str=None, verbose: bool=None, custom_wf_metadata: dict=None
     ) -> None:
         """
@@ -260,7 +260,7 @@ def upload_inputs(self, input_dir=None, update_files=True) -> VipSession:
 
     # Launch the pipeline on VIP
     def launch_pipeline(
-            self, pipeline_id: str=None, input_settings: dict=None, nb_runs=1
+            self, pipeline_id: str=None, input_settings: dict | list[dict]=None, nb_runs=1
         ) -> VipGirder:
         """
         Launches pipeline executions on VIP.
@@ -738,7 +738,7 @@ def parse_value(input):
     # ------------------------------------------------
 
     # Get the input settings after files are parsed as PathLib objects
-    def _get_input_settings(self, location="girder") -> dict:
+    def _get_input_settings(self, location="girder") -> list[dict]:
         """
         Returns the input settings with filenames adapted to `location`.
         - if `location` = "girder", returns Girder paths string format.
@@ -747,7 +747,7 @@ def _get_input_settings(self, location="girder") -> dict:
         Returns a string version of any other parameter.
         """
         # Function to get the VIP-Girder standard from 1 input path
-        def get_input(value, location) -> str:
+        def get_input(value, location) -> str | list[str]:
             """
             If `value` is a path, returns the corresponding string.
             Value can be a single input or a list of inputs.
@@ -768,10 +768,13 @@ def get_input(value, location) -> str:
         if location not in ("girder", "vip-girder"):
             return super()._get_input_settings(location)
         # Browse input settings
-        return {
-            key: get_input(value, location)
-            for key, value in self._input_settings.items()
-        }
+        return [
+            {
+                key: get_input(value, location)
+                for key, value in input_dict.items()
+            }
+            for input_dict in self._input_settings
+        ]
     # ------------------------------------------------
 
 ######################################################

src/vip_client/classes/VipLauncher.py

@@ -130,34 +130,40 @@ def pipeline_id(self) -> None:
 
     # Input settings
     @property
-    def input_settings(self) -> dict:
+    def input_settings(self) -> list[dict] | None:
         """All parameters needed to run the pipeline 
         Run show_pipeline() for more information"""
         # Return None if the private attribute is unset
         return self._get_input_settings() if self._is_defined("_input_settings") else None
 
     @input_settings.setter
-    def input_settings(self, input_settings: dict):
+    def input_settings(self, input_settings: dict | list[dict]):
         # Call deleter if agument is None
         if input_settings is None: 
             del self.input_settings
             return
         # Display
         self._print("Input Settings --> ", end="", flush=True)
         # Check type
-        if not isinstance(input_settings, dict):
-            raise TypeError("`input_settings` should be a dictionary")
-        
+        is_dict = isinstance(input_settings, dict)
+        is_nonempty_dict_list = isinstance(input_settings, list) and input_settings and all(isinstance(item, dict) for item in input_settings)
+        if not (is_dict or is_nonempty_dict_list):
+            raise TypeError("`input_settings` should be a dictionary or a non-empty list of dictionaries")
+
         # Check if each input can be converted to a string with valid characters and no empty strings
-        self._check_invalid_input(input_settings)
-        
-        # Parse the input settings
-        new_settings = self._parse_input_settings(input_settings)
-        self._print("parsed")
-        # Check conflicts with private attribute
-        self._check_value("_input_settings", new_settings)
-        # Update
-        self._input_settings = new_settings
+        inputs = input_settings if is_nonempty_dict_list else [input_settings]
+        new_input_settings = []
+        for input_dict in inputs:
+            self._check_invalid_input(input_dict)
+            # Parse the input settings
+            new_settings = self._parse_input_settings(input_dict)
+            self._print("parsed")
+            # Check conflicts with private attribute
+            self._check_value("_input_settings", new_settings)
+            # Update
+            new_input_settings.append(new_settings)
+
+        self._input_settings = new_input_settings
 
     @input_settings.deleter
     def input_settings(self) -> None:
@@ -308,7 +314,7 @@ def _pipeline_def(self) -> dict:
 
     def __init__(
             self, output_dir=None, pipeline_id: str=None,  
-            input_settings: dict=None, session_name: str=None, verbose: bool=None
+            input_settings: dict | list[dict]=None, session_name: str=None, verbose: bool=None
         ) -> None:
         """
         Create a VipLauncher instance and sets properties from keyword arguments. 
@@ -439,7 +445,7 @@ def init(cls, api_key="VIP_API_KEY", verbose=True, vip_portal_url=None,
 
     # Launch executions on VIP 
     def launch_pipeline(
-            self, pipeline_id: str=None, input_settings: dict=None, output_dir=None, nb_runs=1,
+            self, pipeline_id: str=None, input_settings: dict | list[dict]=None, output_dir=None, nb_runs=1,
         ) -> VipLauncher:
         """
         Launches pipeline executions on VIP.
@@ -556,7 +562,7 @@ def monitor_workflows(self, refresh_time=30) -> VipLauncher:
             self._print("Run launch_pipeline() to launch workflows on VIP.")
             return self
         # Update existing workflows
-        self._print("Updating worflow inventory ... ", end="", flush=True)
+        self._print("Updating workflow inventory ... ", end="", flush=True)
         self._update_workflows()
         self._print("Done.")
         # Check if workflows are still running
@@ -565,7 +571,7 @@ def monitor_workflows(self, refresh_time=30) -> VipLauncher:
             self._execution_report()
             # Display standby
             self._print("\n-------------------------------------------------------------")
-            self._print("The current proccess will wait until all executions are over.")
+            self._print("The current process will wait until all executions are over.")
             self._print("Their progress can be monitored on VIP portal:")
             self._print(f"\t{self._VIP_PORTAL}")
             self._print("-------------------------------------------------------------")
@@ -1538,21 +1544,21 @@ def parse_value(input):
         }
 
     # Get the input settings after files are parsed as PathLib objects
-    def _get_input_settings(self, location="vip") -> dict:
+    def _get_input_settings(self, location="vip") -> list[dict]:
         """
         Returns the input settings with their orignal values in string format.
         `location` is destined to subclasses.
         """
         if location != "vip":
             raise NotImplementedError(f"Unknown location: {location}")
-        return {
-            key: [str(v) for v in value] if isinstance(value, list) else str(value)
-            for key, value in self._input_settings.items()
-        }
+        return [
+            {key: [str(v) for v in value] if isinstance(value, list) else str(value) for key, value in input_dict.items()}
+            for input_dict in self._input_settings
+        ]
     # ------------------------------------------------
 
     # Check the input settings based on the pipeline descriptor
-    def _check_input_settings(self, input_settings: dict=None, location: str=None) -> None:
+    def _check_input_settings(self, input_settings: list[dict]=None, location: str=None) -> None:
         """
         Checks `input_settings` with respect to pipeline descriptor. If not provided, checks the instance property.
         Prerequisite: input_settings contains only strings or lists of strings.
@@ -1580,10 +1586,12 @@ def _check_input_settings(self, input_settings: dict=None, location: str=None) -
         # Check the pipeline identifier
         if not self._is_defined("_pipeline_id"): 
             raise AttributeError("Input settings could not be checked without a pipeline identifier.")
-        # Parameter names
-        self._check_input_keys(input_settings)
-        # Parameter values
-        self._check_input_values(input_settings, location=location)
+
+        for input_dict in input_settings:
+            # Parameter names
+            self._check_input_keys(input_dict)
+            # Parameter values
+            self._check_input_values(input_dict, location=location)
         # Return True when all checks are complete
         return True
     # ------------------------------------------------      

src/vip_client/classes/VipSession.py

@@ -218,7 +218,7 @@ def _vip_dir(self) -> str:
                     #############
     def __init__(
             self, session_name: str=None,  input_dir=None, pipeline_id: str=None,  
-            input_settings: dict=None, output_dir=None, verbose: bool=None
+            input_settings: dict | list[dict]=None, output_dir=None, verbose: bool=None
         ) -> None:
         """
         Create a VipSession instance and sets its properties from keyword arguments.
@@ -396,7 +396,7 @@ def upload_inputs(self, input_dir=None, update_files=True) -> VipSession:
 
     # Launch executions on VIP 
     def launch_pipeline(
-            self, pipeline_id: str=None, input_settings: dict=None, nb_runs=1
+            self, pipeline_id: str=None, input_settings: dict | list[dict]=None, nb_runs=1
         ) -> VipSession:
         """
         Launches pipeline executions on VIP.
@@ -1140,7 +1140,7 @@ def parse_value(input):
     # ------------------------------------------------
 
     # Get the input settings after they are parsed
-    def _get_input_settings(self, location="vip") -> dict:
+    def _get_input_settings(self, location="vip") -> list[dict]:
         """
         Fits `self._input_settings` to `location`, i.e. write the input paths relatively to `location`.
         Returns the modified settings.
@@ -1173,10 +1173,10 @@ def get_input(value, location) -> str:
         if location not in ("vip", "local"):
             raise NotImplementedError(f"Unknown location: {location}")
         # Browse input settings
-        return {
-            key: get_input(value, location)
-            for key, value in self._input_settings.items()
-        }
+        return [
+            {key: get_input(value, location) for key, value in input_dict.items()}
+            for input_dict in self._input_settings
+        ]
     # ------------------------------------------------
 
     def _update_input_settings(self) -> None:
@@ -1185,7 +1185,7 @@ def _update_input_settings(self) -> None:
         This method does nothing if `input_settings` is unset.
         """
         if self._is_defined('_input_settings'):
-            self._input_settings = self._parse_input_settings(self._input_settings)
+            self._input_settings = [self._parse_input_settings(input_dict) for input_dict in self._input_settings]
     # ------------------------------------------------
 
     # Function to convert a VIP path to local output directory

src/vip_client/utils/vip.py

@@ -350,7 +350,7 @@ def count_executions()->int:
     return int(rq.text)
 
 # -----------------------------------------------------------------------------
-def init_exec(pipeline, name="default", inputValues={}, resultsLocation="/vip/Home") -> str:
+def init_exec(pipeline, name="default", inputValues=[], resultsLocation="/vip/Home") -> str:
     url = __PREFIX + 'executions'
     headers = {
                 'apikey': __apikey,
@@ -359,15 +359,15 @@ def init_exec(pipeline, name="default", inputValues={}, resultsLocation="/vip/Ho
     data_ = {
             "name": name, 
             'pipelineIdentifier': pipeline,
-            "inputValues": inputValues,
+            "inputValues": [inputValues] if isinstance(inputValues, dict) else inputValues,
             "resultsLocation": resultsLocation
            }
     rq = SESSION.post(url, headers=headers, json=data_)
     manage_errors(rq)
     return rq.json()["identifier"]
 # -----------------------------------------------------------------------------
 
-def init_exec_without_resultsLocation(pipeline, name="default", inputValues={}) -> str:
+def init_exec_without_resultsLocation(pipeline, name="default", inputValues=[]) -> str:
     """Initiate executions with "results-directory" in the `inputValues`"""
     url = __PREFIX + 'executions'
     headers = {
@@ -377,7 +377,7 @@ def init_exec_without_resultsLocation(pipeline, name="default", inputValues={})
     data_ = {
             "name": name, 
             'pipelineIdentifier': pipeline,
-            "inputValues": inputValues
+            "inputValues": [inputValues] if isinstance(inputValues, dict) else inputValues,
            }
     rq = requests.post(url, headers=headers, json=data_)
     manage_errors(rq)

tests/test_VipGirder.py

@@ -163,11 +163,14 @@ def test_properties_interface(mocker):
     # Backup the inputs
     backup = s.input_settings
     # Run a subtest for each property
-    for prop in s.input_settings:
-        setattr(s, prop, None) # Calls deleter
-        assert getattr(s, prop) is None # Public attribute must be None
-        assert not s._is_defined("_" + prop) # Private attribute must be unset
-        setattr(s, prop, backup[prop]) # Reset
+    for i, map in enumerate(s.input_settings):
+        for key, value in map.items():
+            setattr(s, key, None) # Calls deleter
+            assert getattr(s, key) is None # Public attribute must be None
+            assert not s._is_defined("_" + key) # Private attribute must be unset
+            setattr(s, key, backup[i][key]) # Reset
+
     # Test correct reset
-    for key, value in s.input_settings.items():
-        assert getattr(s, key) == value
+    for map in s.input_settings:
+        for key, value in map.items():
+            assert getattr(s, key) == value

tests/test_VipLauncher.py

@@ -202,11 +202,14 @@ def test_properties_interface(mocker):
     # Backup the inputs
     backup = s.input_settings
     # Run a subtest for each property
-    for prop in s.input_settings:
-        setattr(s, prop, None) # Calls deleter
-        assert getattr(s, prop) is None # Public attribute must be None
-        assert not s._is_defined("_" + prop) # Private attribute must be unset
-        setattr(s, prop, backup[prop]) # Reset
+    for i, map in enumerate(s.input_settings):
+        for key, value in map.items():
+            setattr(s, key, None) # Calls deleter
+            assert getattr(s, key) is None # Public attribute must be None
+            assert not s._is_defined("_" + key) # Private attribute must be unset
+            setattr(s, key, backup[i][key]) # Reset
+
     # Test correct reset
-    for key, value in s.input_settings.items():
-        assert getattr(s, key) == value
+    for map in s.input_settings:
+        for key, value in map.items():
+            assert getattr(s, key) == value

tests/test_VipSession.py

@@ -189,11 +189,14 @@ def test_properties_interface(mocker):
     # Backup the inputs
     backup = s.input_settings
     # Run a subtest for each property
-    for prop in s.input_settings:
-        setattr(s, prop, None) # Calls deleter
-        assert getattr(s, prop) is None # Public attribute must be None
-        assert not s._is_defined("_" + prop) # Private attribute must be unset
-        setattr(s, prop, backup[prop]) # Reset
+    for i, map in enumerate(s.input_settings):
+        for key, value in map.items():
+            setattr(s, key, None) # Calls deleter
+            assert getattr(s, key) is None # Public attribute must be None
+            assert not s._is_defined("_" + key) # Private attribute must be unset
+            setattr(s, key, backup[i][key]) # Reset
+
     # Test correct reset
-    for key, value in s.input_settings.items():
-        assert getattr(s, key) == value
+    for map in s.input_settings:
+        for key, value in map.items():
+            assert getattr(s, key) == value