checkpointing

Author: mitzimorris

cmdstanpy/cmdstan_args.py

@@ -315,6 +315,7 @@ def __init__(
         algorithm: Optional[str] = None,
         init_alpha: Optional[float] = None,
         iter: Optional[int] = None,
+        save_iterations: bool = False,
         tol_obj: Optional[float] = None,
         tol_rel_obj: Optional[float] = None,
         tol_grad: Optional[float] = None,
@@ -326,6 +327,7 @@ def __init__(
         self.algorithm = algorithm
         self.init_alpha = init_alpha
         self.iter = iter
+        self.save_iterations = save_iterations
         self.tol_obj = tol_obj
         self.tol_rel_obj = tol_rel_obj
         self.tol_grad = tol_grad
@@ -457,6 +459,8 @@ def compose(self, idx: int, cmd: List[str]) -> List[str]:
             cmd.append('history_size={}'.format(self.history_size))
         if self.iter is not None:
             cmd.append('iter={}'.format(self.iter))
+        if self.save_iterations:
+            cmd.append('save_iterations=1')
 
         return cmd
 

cmdstanpy/model.py

@@ -397,6 +397,7 @@ def optimize(
         tol_param: Optional[float] = None,
         history_size: Optional[int] = None,
         iter: Optional[int] = None,
+        save_iterations: bool = False,
         refresh: Optional[int] = None,
     ) -> CmdStanMLE:
         """
@@ -481,6 +482,9 @@ def optimize(
 
         :param iter: Total number of iterations
 
+        :param save_iterations: When ``True``, save intermediate approximations
+            to the output CSV file.  Default is ``False``.
+
         :param refresh: Specify the number of iterations cmdstan will take
             between progress messages. Default value is 100.
 
@@ -496,6 +500,7 @@ def optimize(
             tol_param=tol_param,
             history_size=history_size,
             iter=iter,
+            save_iterations=save_iterations,
         )
 
         with MaybeDictToFilePath(data, inits) as (_data, _inits):
@@ -518,11 +523,8 @@ def optimize(
             self._run_cmdstan(runset, dummy_chain_id)
 
         if not runset._check_retcodes():
-            msg = 'Error during optimization:\n{}'.format(runset.get_err_msgs())
-            msg = '{}Command and output files:\n{}'.format(
-                msg, runset.__repr__()
-            )
-            raise RuntimeError(msg)
+            msg = 'Error during optimization: {}'.format(runset.get_err_msgs())
+            get_logger().warn(msg)  # https://github.com/stan-dev/cmdstanr/issues/314
         mle = CmdStanMLE(runset)
         return mle
 

cmdstanpy/stanfit.py

@@ -303,23 +303,29 @@ def get_err_msgs(self) -> str:
                             self._chain_ids[i], fd.read()
                         )
                     )
-            # pre 2.27, all msgs sent to stdout, including errors
-            if (
-                not cmdstan_version_at(2, 27)
-                and os.path.exists(self._stdout_files[i])
-                and os.stat(self._stdout_files[i]).st_size > 0
-            ):
-                with open(self._stdout_files[i], 'r') as fd:
-                    contents = fd.read()
-                    # pattern matches initial "Exception" or "Error" msg
-                    pat = re.compile(r'^E[rx].*$', re.M)
-                errors = re.findall(pat, contents)
-                if len(errors) > 0:
-                    msgs.append(
-                        'chain_id {}:\n\t{}\n'.format(
-                            self._chain_ids[i], '\n\t'.join(errors)
-                        )
-                    )
+            # pre 2.27, all sampler msgs go to stdout, including errors
+            if self._args.method == Method.SAMPLE:
+                if (
+                    not cmdstan_version_at(2, 27)
+                    and os.path.exists(self._stdout_files[i])
+                    and os.stat(self._stdout_files[i]).st_size > 0
+                ):
+                    with open(self._stdout_files[i], 'r') as fd:
+                        contents = fd.read()
+                        # pattern matches initial "Exception" or "Error" msg
+                        pat = re.compile(r'^E[rx].*$', re.M)
+                        errors = re.findall(pat, contents)
+                        if len(errors) > 0:
+                            msgs.append(
+                                'chain_id {}:\n\t{}\n'.format(
+                                    self._chain_ids[i], '\n\t'.join(errors)
+                                    )
+                                )
+            elif self._args.method == Method.OPTIMIZE:
+                msgs.append('console log output:\n')
+                with open(self._stdout_files[0], 'r') as fd:
+                    msgs.append(fd.read())
+
         return '\n'.join(msgs)
 
     def save_csvfiles(self, dir: Optional[str] = None) -> None:
@@ -1240,7 +1246,9 @@ def __repr__(self) -> str:
             '\n\t'.join(self.runset.csv_files),
             '\n\t'.join(self.runset.stdout_files),
         )
-        # TODO - profiling files
+        if not self.runset._check_retcodes():
+            repr = '{}\n Warning: invalid estimate, '.format(repr)
+            repr = '{} optimization failed to converge.'.format(repr)
         return repr
 
     def _set_mle_attrs(self, sample_csv_0: str) -> None:
@@ -1269,20 +1277,33 @@ def metadata(self) -> InferenceMetadata:
     @property
     def optimized_params_np(self) -> np.ndarray:
         """Returns optimized params as numpy array."""
+        if not self.runset._check_retcodes():
+            get_logger().warning(
+                'invalid estimate, optimization failed to converge'
+            )
         return np.asarray(self._mle)
 
     @property
     def optimized_params_pd(self) -> pd.DataFrame:
         """Returns optimized params as pandas DataFrame."""
+        if not self.runset._check_retcodes():
+            get_logger().warning(
+                'invalid estimate, optimization failed to converge'
+            )
         return pd.DataFrame([self._mle], columns=self.column_names)
 
     @property
     def optimized_params_dict(self) -> Dict[str, float]:
         """Returns optimized params as Dict."""
+        if not self.runset._check_retcodes():
+            get_logger().warning(
+                'invalid estimate, optimization failed to converge'
+            )
         return OrderedDict(zip(self.column_names, self._mle))
 
     def stan_variable(
-        self, var: Optional[str] = None, *, name: Optional[str] = None
+        self, var: Optional[str] = None,
+        check_convergence:bool = True, *, name: Optional[str] = None
     ) -> np.ndarray:
         """
         Return a numpy.ndarray which contains the estimates for the
@@ -1291,6 +1312,11 @@ def stan_variable(
 
         :param var: variable name
 
+        :param check_convergence: Checks for failure to converge and
+            prints warning.failed to converge.  ``False`` will supress
+            check and warning, default is ``True``.
+
+
         See Also
         --------
         CmdStanMLE.stan_variables
@@ -1311,6 +1337,11 @@ def stan_variable(
             raise ValueError('no variable name specified.')
         if var not in self._metadata.stan_vars_dims:
             raise ValueError('unknown variable name: {}'.format(var))
+        if check_convergence and not self.runset._check_retcodes():
+            get_logger().warning(
+                'invalid estimate, optimization failed to converge'
+            )
+
         col_idxs = list(self._metadata.stan_vars_cols[var])
         vals = list(self._mle)
         xs = [vals[x] for x in col_idxs]
@@ -1319,21 +1350,29 @@ def stan_variable(
             shape = self._metadata.stan_vars_dims[var]
         return np.array(xs).reshape(shape)
 
-    def stan_variables(self) -> Dict[str, np.ndarray]:
+    def stan_variables(self, check_convergence:bool = True) -> Dict[str, np.ndarray]:
         """
         Return a dictionary mapping Stan program variables names
         to the corresponding numpy.ndarray containing the inferred values.
 
+        :param check_convergence: Checks for failure to converge and
+            prints warning.failed to converge.  ``False`` will supress
+            check and warning, default is ``True``.
+
         See Also
         --------
         CmdStanMLE.stan_variable
         CmdStanMCMC.stan_variables
         CmdStanVB.stan_variables
         CmdStanGQ.stan_variables
         """
+        if check_convergence and not self.runset._check_retcodes():
+            get_logger().warning(
+                'invalid estimate, optimization failed to converge'
+            )
         result = {}
         for name in self._metadata.stan_vars_dims.keys():
-            result[name] = self.stan_variable(name)
+            result[name] = self.stan_variable(name, False)  # don't warn twice
         return result
 
     def save_csvfiles(self, dir: Optional[str] = None) -> None:

cmdstanpy/utils.py

@@ -578,14 +578,24 @@ def scan_sampler_csv(path: str, is_fixed_param: bool = False) -> Dict[str, Any]:
     return dict
 
 
-def scan_optimize_csv(path: str) -> Dict[str, Any]:
+def scan_optimize_csv(path: str, save_iters: bool = False) -> Dict[str, Any]:
     """Process optimizer stan_csv output file line by line."""
     dict: Dict[str, Any] = {}
     lineno = 0
+    # scan to find config, header, num saved iters
     with open(path, 'r') as fd:
         lineno = scan_config(fd, dict, lineno)
         lineno = scan_column_names(fd, dict, lineno)
-        line = fd.readline().lstrip(' #\t').rstrip()
+        iters = 0
+        for line in fd:
+            iters += 1
+    # rescan to capture estimates
+    with open(path, 'r') as fd:
+        for i in range(lineno):
+            fd.readline()
+        line = fd.readline()
+        # allocate numpy ndarray for 
+        print(line)
         xs = line.split(',')
         dict['mle'] = [float(x) for x in xs]
     return dict