Merge branch 'develop' of https://github.com/stan-dev/cmdstanpy into develop

Author: mitzimorris

.github/workflows/main.yml

@@ -39,8 +39,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9, "3.10"]
-      fail-fast: false
+        python-version: [3.7, 3.8, 3.9, "3.10"]
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     steps:
@@ -60,6 +59,7 @@ jobs:
           pip install codecov
 
       - name: Run flake8, pylint, mypy
+        if: matrix.python-version == '3.10'
         run: |
           flake8 cmdstanpy test
           pylint -v cmdstanpy test

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
       - id: mypy
         # Copied from setup.cfg
         exclude: ^test/
-        additional_dependencies: [ numpy >= 1.21 ]
+        additional_dependencies: [ numpy >= 1.22, types-ujson ]
   # local uses the user-installed pylint, this allows dependency checking
   - repo: local
     hooks:

cmdstanpy/cmdstan_args.py

@@ -202,10 +202,7 @@ def validate(self, chains: Optional[int]) -> None:
                 if all(isinstance(elem, dict) for elem in self.metric):
                     metric_files: List[str] = []
                     for i, metric in enumerate(self.metric):
-                        assert isinstance(
-                            metric, dict
-                        )  # make the typechecker happy
-                        metric_dict: Dict[str, Any] = metric
+                        metric_dict: Dict[str, Any] = metric  # type: ignore
                         if 'inv_metric' not in metric_dict:
                             raise ValueError(
                                 'Entry "inv_metric" not found in metric dict '

cmdstanpy/compiler_opts.py

@@ -3,21 +3,27 @@
 """
 
 import os
+from copy import copy
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
 from cmdstanpy.utils import get_logger
 
 STANC_OPTS = [
     'O',
-    'allow_undefined',
+    'allow-undefined',
     'use-opencl',
     'warn-uninitialized',
-    'include_paths',
+    'include-paths',
     'name',
     'warn-pedantic',
 ]
 
+STANC_DEPRECATED_OPTS = {
+    'allow_undefined': 'allow-undefined',
+    'include_paths': 'include-paths',
+}
+
 STANC_IGNORE_OPTS = [
     'debug-lex',
     'debug-parse',
@@ -121,19 +127,37 @@ def validate_stanc_opts(self) -> None:
             return
         ignore = []
         paths = None
+        for deprecated, replacement in STANC_DEPRECATED_OPTS.items():
+            if deprecated in self._stanc_options:
+                if replacement:
+                    get_logger().warning(
+                        'compiler option "%s" is deprecated, use "%s" instead',
+                        deprecated,
+                        replacement,
+                    )
+                    self._stanc_options[replacement] = copy(
+                        self._stanc_options[deprecated]
+                    )
+                    del self._stanc_options[deprecated]
+                else:
+                    get_logger().warning(
+                        'compiler option "%s" is deprecated and '
+                        'should not be used',
+                        deprecated,
+                    )
         for key, val in self._stanc_options.items():
             if key in STANC_IGNORE_OPTS:
                 get_logger().info('ignoring compiler option: %s', key)
                 ignore.append(key)
             elif key not in STANC_OPTS:
                 raise ValueError(f'unknown stanc compiler option: {key}')
-            elif key == 'include_paths':
+            elif key == 'include-paths':
                 paths = val
                 if isinstance(val, str):
                     paths = val.split(',')
                 elif not isinstance(val, list):
                     raise ValueError(
-                        'Invalid include_paths, expecting list or '
+                        'Invalid include-paths, expecting list or '
                         f'string, found type: {type(val)}.'
                     )
             elif key == 'use-opencl':
@@ -145,17 +169,16 @@ def validate_stanc_opts(self) -> None:
         for opt in ignore:
             del self._stanc_options[opt]
         if paths is not None:
-            self._stanc_options['include_paths'] = paths
-            bad_paths = [
-                dir
-                for dir in self._stanc_options['include_paths']
-                if not os.path.exists(dir)
-            ]
+            bad_paths = [dir for dir in paths if not os.path.exists(dir)]
             if any(bad_paths):
                 raise ValueError(
                     'invalid include paths: {}'.format(', '.join(bad_paths))
                 )
 
+            self._stanc_options['include-paths'] = [
+                os.path.abspath(os.path.expanduser(path)) for path in paths
+            ]
+
     def validate_cpp_opts(self) -> None:
         """
         Check cpp compiler args.
@@ -190,8 +213,8 @@ def validate_user_header(self) -> None:
                 raise ValueError(
                     f"Header file must end in .hpp, got {self._user_header}"
                 )
-            if "allow_undefined" not in self._stanc_options:
-                self._stanc_options["allow_undefined"] = True
+            if "allow-undefined" not in self._stanc_options:
+                self._stanc_options["allow-undefined"] = True
             # set full path
             self._user_header = os.path.abspath(self._user_header)
 
@@ -218,7 +241,7 @@ def add(self, new_opts: "CompilerOptions") -> None:  # noqa: disable=Q000
                 self._stanc_options = new_opts.stanc_options
             else:
                 for key, val in new_opts.stanc_options.items():
-                    if key == 'include_paths':
+                    if key == 'include-paths':
                         self.add_include_path(str(val))
                     else:
                         self._stanc_options[key] = val
@@ -230,30 +253,35 @@ def add(self, new_opts: "CompilerOptions") -> None:  # noqa: disable=Q000
 
     def add_include_path(self, path: str) -> None:
         """Adds include path to existing set of compiler options."""
-        if 'include_paths' not in self._stanc_options:
-            self._stanc_options['include_paths'] = [path]
-        elif path not in self._stanc_options['include_paths']:
-            self._stanc_options['include_paths'].append(path)
+        path = os.path.abspath(os.path.expanduser(path))
+        if 'include-paths' not in self._stanc_options:
+            self._stanc_options['include-paths'] = [path]
+        elif path not in self._stanc_options['include-paths']:
+            self._stanc_options['include-paths'].append(path)
 
-    def compose(self) -> List[str]:
-        """Format makefile options as list of strings."""
+    def compose_stanc(self) -> List[str]:
         opts = []
         if self._stanc_options is not None and len(self._stanc_options) > 0:
             for key, val in self._stanc_options.items():
-                if key == 'include_paths':
+                if key == 'include-paths':
                     opts.append(
-                        'STANCFLAGS+=--include_paths='
+                        '--include-paths='
                         + ','.join(
                             (
                                 Path(p).as_posix()
-                                for p in self._stanc_options['include_paths']
+                                for p in self._stanc_options['include-paths']
                             )
                         )
                     )
                 elif key == 'name':
-                    opts.append(f'STANCFLAGS+=--name={val}')
+                    opts.append(f'--name={val}')
                 else:
-                    opts.append(f'STANCFLAGS+=--{key}')
+                    opts.append(f'--{key}')
+        return opts
+
+    def compose(self) -> List[str]:
+        """Format makefile options as list of strings."""
+        opts = ['STANCFLAGS+=' + flag for flag in self.compose_stanc()]
         if self._cpp_options is not None and len(self._cpp_options) > 0:
             for key, val in self._cpp_options.items():
                 opts.append(f'{key}={val}')

cmdstanpy/install_cmdstan.py

@@ -460,7 +460,9 @@ def main(args: Dict[str, Any]) -> None:
         print('Installing CmdStan version: {}'.format(version))
     else:
         raise ValueError(
-            'Invalid version requested: {}, cannot install.'.format(version)
+            f'Version {version} cannot be downloaded. '
+            'Connection to GitHub failed. '
+            'Check firewall settings or ensure this version exists.'
         )
 
     cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTAN))

cmdstanpy/model.py

@@ -128,6 +128,8 @@ def __init__(
                 )
             self._name = model_name.strip()
 
+        self._compiler_options.validate()
+
         if stan_file is None:
             if exe_file is None:
                 raise ValueError(
@@ -146,19 +148,14 @@ def __init__(
             if not self._name:
                 self._name, _ = os.path.splitext(filename)
 
-            # TODO: When minimum version is 2.27, use --info instead
             # if program has include directives, record path
             with open(self._stan_file, 'r') as fd:
                 program = fd.read()
             if '#include' in program:
                 path, _ = os.path.split(self._stan_file)
-                if self._compiler_options is None:
-                    self._compiler_options = CompilerOptions(
-                        stanc_options={'include_paths': [path]}
-                    )
-                elif self._compiler_options._stanc_options is None:
+                if self._compiler_options._stanc_options is None:
                     self._compiler_options._stanc_options = {
-                        'include_paths': [path]
+                        'include-paths': [path]
                     }
                 else:
                     self._compiler_options.add_include_path(path)
@@ -186,8 +183,6 @@ def __init__(
                         ' found: {}.'.format(self._name, exename)
                     )
 
-        self._compiler_options.validate()
-
         if platform.system() == 'Windows':
             try:
                 do_command(['where.exe', 'tbb.dll'], fd_out=None)
@@ -279,12 +274,15 @@ def src_info(self) -> Dict[str, Any]:
         if self.stan_file is None:
             return result
         try:
-
-            cmd = [
-                os.path.join('.', 'bin', 'stanc' + EXTENSION),
-                '--info',
-                self.stan_file,
-            ]
+            cmd = (
+                [os.path.join('.', 'bin', 'stanc' + EXTENSION)]
+                # handle include-paths, allow-undefined etc
+                + self._compiler_options.compose_stanc()
+                + [
+                    '--info',
+                    self.stan_file,
+                ]
+            )
             sout = io.StringIO()
             do_command(cmd=cmd, cwd=cmdstan_path(), fd_out=sout)
             result = json.loads(sout.getvalue())

cmdstanpy/stanfit/mcmc.py

@@ -95,9 +95,9 @@ def __init__(
         self._save_warmup = sampler_args.save_warmup
         self._sig_figs = runset._args.sig_figs
         # info from CSV values, instantiated lazily
-        self._metric = np.array(())
-        self._step_size = np.array(())
-        self._draws = np.array(())
+        self._metric: np.ndarray = np.array(())
+        self._step_size: np.ndarray = np.array(())
+        self._draws: np.ndarray = np.array(())
         # info from CSV initial comments and header
         config = self._validate_csv_files()
         self._metadata: InferenceMetadata = InferenceMetadata(config)
@@ -246,7 +246,7 @@ def draws(
 
         if concat_chains:
             return flatten_chains(self._draws[start_idx:, :, :])
-        return self._draws[start_idx:, :, :]  # type: ignore
+        return self._draws[start_idx:, :, :]
 
     def _validate_csv_files(self) -> Dict[str, Any]:
         """
@@ -675,9 +675,7 @@ def stan_variable(
         if len(col_idxs) > 0:
             dims.extend(self._metadata.stan_vars_dims[var])
         # pylint: disable=redundant-keyword-arg
-        return self._draws[draw1:, :, col_idxs].reshape(  # type: ignore
-            dims, order='F'
-        )
+        return self._draws[draw1:, :, col_idxs].reshape(dims, order='F')
 
     def stan_variables(self) -> Dict[str, np.ndarray]:
         """
@@ -748,7 +746,7 @@ def __init__(
             )
         self.runset = runset
         self.mcmc_sample = mcmc_sample
-        self._draws = np.array(())
+        self._draws: np.ndarray = np.array(())
         config = self._validate_csv_files()
         self._metadata = InferenceMetadata(config)
 
@@ -765,7 +763,7 @@ def __repr__(self) -> str:
         )
         return repr
 
-    def _validate_csv_files(self) -> dict:
+    def _validate_csv_files(self) -> Dict[str, Any]:
         """
         Checks that Stan CSV output files for all chains are consistent
         and returns dict containing config and column names.
@@ -910,13 +908,13 @@ def draws(
         if concat_chains:
             return flatten_chains(self._draws[start_idx:, :, :])
         if inc_sample:
-            return np.dstack(  # type: ignore
+            return np.dstack(
                 (
                     np.delete(self.mcmc_sample.draws(), drop_cols, axis=1),
                     self._draws,
                 )
             )[start_idx:, :, :]
-        return self._draws[start_idx:, :, :]  # type: ignore
+        return self._draws[start_idx:, :, :]
 
     def draws_pd(
         self,
@@ -1195,9 +1193,7 @@ def stan_variable(
             if len(col_idxs) > 0:
                 dims.extend(self._metadata.stan_vars_dims[var])
             # pylint: disable=redundant-keyword-arg
-            return self._draws[draw1:, :, col_idxs].reshape(  # type: ignore
-                dims, order='F'
-            )
+            return self._draws[draw1:, :, col_idxs].reshape(dims, order='F')
 
     def stan_variables(self, inc_warmup: bool = False) -> Dict[str, np.ndarray]:
         """
@@ -1229,7 +1225,7 @@ def _assemble_generated_quantities(self) -> None:
         # use numpy loadtxt
         warmup = self.mcmc_sample.metadata.cmdstan_config['save_warmup']
         num_draws = self.mcmc_sample.draws(inc_warmup=warmup).shape[0]
-        gq_sample = np.empty(
+        gq_sample: np.ndarray = np.empty(
             (num_draws, self.chains, len(self.column_names)),
             dtype=float,
             order='F',