Merge pull request #518 from stan-dev/src-info-includes

Add include paths to src_info, update stanc3 argument handling

Author: WardBrian

cmdstanpy/compiler_opts.py

@@ -3,21 +3,27 @@
 """
 
 import os
+from copy import copy
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
 from cmdstanpy.utils import get_logger
 
 STANC_OPTS = [
     'O',
-    'allow_undefined',
+    'allow-undefined',
     'use-opencl',
     'warn-uninitialized',
-    'include_paths',
+    'include-paths',
     'name',
     'warn-pedantic',
 ]
 
+STANC_DEPRECATED_OPTS = {
+    'allow_undefined': 'allow-undefined',
+    'include_paths': 'include-paths',
+}
+
 STANC_IGNORE_OPTS = [
     'debug-lex',
     'debug-parse',
@@ -121,19 +127,37 @@ def validate_stanc_opts(self) -> None:
             return
         ignore = []
         paths = None
+        for deprecated, replacement in STANC_DEPRECATED_OPTS.items():
+            if deprecated in self._stanc_options:
+                if replacement:
+                    get_logger().warning(
+                        'compiler option "%s" is deprecated, use "%s" instead',
+                        deprecated,
+                        replacement,
+                    )
+                    self._stanc_options[replacement] = copy(
+                        self._stanc_options[deprecated]
+                    )
+                    del self._stanc_options[deprecated]
+                else:
+                    get_logger().warning(
+                        'compiler option "%s" is deprecated and '
+                        'should not be used',
+                        deprecated,
+                    )
         for key, val in self._stanc_options.items():
             if key in STANC_IGNORE_OPTS:
                 get_logger().info('ignoring compiler option: %s', key)
                 ignore.append(key)
             elif key not in STANC_OPTS:
                 raise ValueError(f'unknown stanc compiler option: {key}')
-            elif key == 'include_paths':
+            elif key == 'include-paths':
                 paths = val
                 if isinstance(val, str):
                     paths = val.split(',')
                 elif not isinstance(val, list):
                     raise ValueError(
-                        'Invalid include_paths, expecting list or '
+                        'Invalid include-paths, expecting list or '
                         f'string, found type: {type(val)}.'
                     )
             elif key == 'use-opencl':
@@ -145,10 +169,10 @@ def validate_stanc_opts(self) -> None:
         for opt in ignore:
             del self._stanc_options[opt]
         if paths is not None:
-            self._stanc_options['include_paths'] = paths
+            self._stanc_options['include-paths'] = paths
             bad_paths = [
                 dir
-                for dir in self._stanc_options['include_paths']
+                for dir in self._stanc_options['include-paths']
                 if not os.path.exists(dir)
             ]
             if any(bad_paths):
@@ -190,8 +214,8 @@ def validate_user_header(self) -> None:
                 raise ValueError(
                     f"Header file must end in .hpp, got {self._user_header}"
                 )
-            if "allow_undefined" not in self._stanc_options:
-                self._stanc_options["allow_undefined"] = True
+            if "allow-undefined" not in self._stanc_options:
+                self._stanc_options["allow-undefined"] = True
             # set full path
             self._user_header = os.path.abspath(self._user_header)
 
@@ -218,7 +242,7 @@ def add(self, new_opts: "CompilerOptions") -> None:  # noqa: disable=Q000
                 self._stanc_options = new_opts.stanc_options
             else:
                 for key, val in new_opts.stanc_options.items():
-                    if key == 'include_paths':
+                    if key == 'include-paths':
                         self.add_include_path(str(val))
                     else:
                         self._stanc_options[key] = val
@@ -230,23 +254,23 @@ def add(self, new_opts: "CompilerOptions") -> None:  # noqa: disable=Q000
 
     def add_include_path(self, path: str) -> None:
         """Adds include path to existing set of compiler options."""
-        if 'include_paths' not in self._stanc_options:
-            self._stanc_options['include_paths'] = [path]
-        elif path not in self._stanc_options['include_paths']:
-            self._stanc_options['include_paths'].append(path)
+        if 'include-paths' not in self._stanc_options:
+            self._stanc_options['include-paths'] = [path]
+        elif path not in self._stanc_options['include-paths']:
+            self._stanc_options['include-paths'].append(path)
 
     def compose(self) -> List[str]:
         """Format makefile options as list of strings."""
         opts = []
         if self._stanc_options is not None and len(self._stanc_options) > 0:
             for key, val in self._stanc_options.items():
-                if key == 'include_paths':
+                if key == 'include-paths':
                     opts.append(
-                        'STANCFLAGS+=--include_paths='
+                        'STANCFLAGS+=--include-paths='
                         + ','.join(
                             (
                                 Path(p).as_posix()
-                                for p in self._stanc_options['include_paths']
+                                for p in self._stanc_options['include-paths']
                             )
                         )
                     )

cmdstanpy/model.py

@@ -128,6 +128,8 @@ def __init__(
                 )
             self._name = model_name.strip()
 
+        self._compiler_options.validate()
+
         if stan_file is None:
             if exe_file is None:
                 raise ValueError(
@@ -152,13 +154,9 @@ def __init__(
                 program = fd.read()
             if '#include' in program:
                 path, _ = os.path.split(self._stan_file)
-                if self._compiler_options is None:
-                    self._compiler_options = CompilerOptions(
-                        stanc_options={'include_paths': [path]}
-                    )
-                elif self._compiler_options._stanc_options is None:
+                if self._compiler_options._stanc_options is None:
                     self._compiler_options._stanc_options = {
-                        'include_paths': [path]
+                        'include-paths': [path]
                     }
                 else:
                     self._compiler_options.add_include_path(path)
@@ -186,8 +184,6 @@ def __init__(
                         ' found: {}.'.format(self._name, exename)
                     )
 
-        self._compiler_options.validate()
-
         if platform.system() == 'Windows':
             try:
                 do_command(['where.exe', 'tbb.dll'], fd_out=None)
@@ -279,9 +275,19 @@ def src_info(self) -> Dict[str, Any]:
         if self.stan_file is None:
             return result
         try:
-
+            includes = ''
+            if (
+                self.stanc_options is not None
+                and 'include-paths' in self.stanc_options
+            ):
+                print(self.stanc_options)
+                includes = '--include-paths=' + ','.join(
+                    Path(p).as_posix()
+                    for p in self.stanc_options['include-paths']  # type: ignore
+                )
             cmd = [
                 os.path.join('.', 'bin', 'stanc' + EXTENSION),
+                includes,
                 '--info',
                 self.stan_file,
             ]

docsrc/examples/Using External C++.ipynb

@@ -60,7 +60,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Even enabling the `--allow_undefined` flag to stanc3 will not allow this model to be compiled quite yet."
+    "Even enabling the `--allow-undefined` flag to stanc3 will not allow this model to be compiled quite yet."
    ]
   },
   {
@@ -69,7 +69,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "model_external.compile(stanc_options={'allow_undefined':True})"
+    "model_external.compile(stanc_options={'allow-undefined':True})"
    ]
   },
   {
@@ -80,7 +80,7 @@
     "\n",
     "We can provide a definition in a C++ header file by using the `user_header` argument to either the CmdStanModel constructor or the `compile` method. \n",
     "\n",
-    "This will enables the `allow_undefined` flag automatically."
+    "This will enables the `allow-undefined` flag automatically."
    ]
   },
   {

test/test_compiler_opts.py

@@ -3,6 +3,8 @@
 import os
 import unittest
 
+from testfixtures import LogCapture
+
 from cmdstanpy.compiler_opts import CompilerOptions
 
 HERE = os.path.dirname(os.path.abspath(__file__))
@@ -70,6 +72,28 @@ def test_opts_stanc(self):
             ['STANCFLAGS+=--warn-uninitialized', 'STANCFLAGS+=--name=foo'],
         )
 
+    def test_opts_stanc_deprecated(self):
+        stanc_opts = {}
+        stanc_opts['allow_undefined'] = True
+        opts = CompilerOptions(stanc_options=stanc_opts)
+        with LogCapture() as log:
+            opts.validate()
+        log.check_present(
+            (
+                'cmdstanpy',
+                'WARNING',
+                'compiler option "allow_undefined" is deprecated,'
+                ' use "allow-undefined" instead',
+            )
+        )
+        self.assertEqual(opts.compose(), ['STANCFLAGS+=--allow-undefined'])
+
+        stanc_opts['include_paths'] = DATAFILES_PATH
+        opts = CompilerOptions(stanc_options=stanc_opts)
+        opts.validate()
+        self.assertIn('include-paths', opts.stanc_options)
+        self.assertNotIn('include_paths', opts.stanc_options)
+
     def test_opts_stanc_opencl(self):
         stanc_opts = {}
         stanc_opts['use-opencl'] = 'foo'
@@ -89,22 +113,22 @@ def test_opts_stanc_ignore(self):
     def test_opts_stanc_includes(self):
         path2 = os.path.join(HERE, 'data', 'optimize')
         paths_str = ','.join([DATAFILES_PATH, path2]).replace('\\', '/')
-        expect = 'STANCFLAGS+=--include_paths=' + paths_str
+        expect = 'STANCFLAGS+=--include-paths=' + paths_str
 
-        stanc_opts = {'include_paths': paths_str}
+        stanc_opts = {'include-paths': paths_str}
         opts = CompilerOptions(stanc_options=stanc_opts)
         opts.validate()
         opts_list = opts.compose()
         self.assertTrue(expect in opts_list)
 
-        stanc_opts = {'include_paths': [DATAFILES_PATH, path2]}
+        stanc_opts = {'include-paths': [DATAFILES_PATH, path2]}
         opts = CompilerOptions(stanc_options=stanc_opts)
         opts.validate()
         opts_list = opts.compose()
         self.assertTrue(expect in opts_list)
 
     def test_opts_add_include_paths(self):
-        expect = 'STANCFLAGS+=--include_paths=' + DATAFILES_PATH.replace(
+        expect = 'STANCFLAGS+=--include-paths=' + DATAFILES_PATH.replace(
             '\\', '/'
         )
         stanc_opts = {'warn-uninitialized': True}
@@ -120,7 +144,7 @@ def test_opts_add_include_paths(self):
 
         path2 = os.path.join(HERE, 'data', 'optimize')
         paths_str = ','.join([DATAFILES_PATH, path2]).replace('\\', '/')
-        expect = 'STANCFLAGS+=--include_paths=' + paths_str
+        expect = 'STANCFLAGS+=--include-paths=' + paths_str
         opts.add_include_path(path2)
         opts.validate()
         opts_list = opts.compose()
@@ -169,7 +193,7 @@ def test_user_header(self):
         header_file = os.path.join(DATAFILES_PATH, 'return_one.hpp')
         opts = CompilerOptions(user_header=header_file)
         opts.validate()
-        self.assertTrue(opts.stanc_options['allow_undefined'])
+        self.assertTrue(opts.stanc_options['allow-undefined'])
 
         bad = os.path.join(DATAFILES_PATH, 'nonexistant.hpp')
         opts = CompilerOptions(user_header=bad)
@@ -209,20 +233,20 @@ def test_opts_add(self):
         self.assertTrue('STAN_OPENCL=FALSE' in opts_list)
         self.assertTrue('OPENCL_DEVICE_ID=2' in opts_list)
 
-        expect = 'STANCFLAGS+=--include_paths=' + DATAFILES_PATH.replace(
+        expect = 'STANCFLAGS+=--include-paths=' + DATAFILES_PATH.replace(
             '\\', '/'
         )
-        stanc_opts2 = {'include_paths': DATAFILES_PATH}
+        stanc_opts2 = {'include-paths': DATAFILES_PATH}
         new_opts2 = CompilerOptions(stanc_options=stanc_opts2)
         opts.add(new_opts2)
         opts_list = opts.compose()
         self.assertTrue(expect in opts_list)
 
         path2 = os.path.join(HERE, 'data', 'optimize')
-        expect = 'STANCFLAGS+=--include_paths=' + ','.join(
+        expect = 'STANCFLAGS+=--include-paths=' + ','.join(
             [DATAFILES_PATH, path2]
         ).replace('\\', '/')
-        stanc_opts3 = {'include_paths': path2}
+        stanc_opts3 = {'include-paths': path2}
         new_opts3 = CompilerOptions(stanc_options=stanc_opts3)
         opts.add(new_opts3)
         opts_list = opts.compose()

test/test_model.py

@@ -129,7 +129,7 @@ def test_model_bad(self):
     def test_stanc_options(self):
         opts = {
             'O': True,
-            'allow_undefined': True,
+            'allow-undefined': True,
             'use-opencl': True,
             'name': 'foo',
         }
@@ -138,7 +138,7 @@ def test_stanc_options(self):
         )
         stanc_opts = model.stanc_options
         self.assertTrue(stanc_opts['O'])
-        self.assertTrue(stanc_opts['allow_undefined'])
+        self.assertTrue(stanc_opts['allow-undefined'])
         self.assertTrue(stanc_opts['use-opencl'])
         self.assertTrue(stanc_opts['name'] == 'foo')
 
@@ -151,12 +151,12 @@ def test_stanc_options(self):
                 stan_file=BERN_STAN, compile=False, stanc_options=bad_opts
             )
         with self.assertRaises(ValueError):
-            bad_opts = {'include_paths': True}
+            bad_opts = {'include-paths': True}
             model = CmdStanModel(
                 stan_file=BERN_STAN, compile=False, stanc_options=bad_opts
             )
         with self.assertRaises(ValueError):
-            bad_opts = {'include_paths': 'lkjdf'}
+            bad_opts = {'include-paths': 'lkjdf'}
             model = CmdStanModel(
                 stan_file=BERN_STAN, compile=False, stanc_options=bad_opts
             )
@@ -190,6 +190,15 @@ def test_model_info(self):
         self.assertNotEqual(model_info, {})
         self.assertIn('theta', model_info['parameters'])
 
+        model_include = CmdStanModel(
+            stan_file=os.path.join(DATAFILES_PATH, "bernoulli_include.stan"),
+            compile=False,
+        )
+        model_info_include = model_include.src_info()
+        self.assertNotEqual(model_info_include, {})
+        self.assertIn('theta', model_info_include['parameters'])
+        self.assertIn('included_files', model_info_include)
+
     def test_compile_force(self):
         if os.path.exists(BERN_EXE):
             os.remove(BERN_EXE)
@@ -349,7 +358,7 @@ def test_model_includes_explicit(self):
         if os.path.exists(BERN_EXE):
             os.remove(BERN_EXE)
         model = CmdStanModel(
-            stan_file=BERN_STAN, stanc_options={'include_paths': DATAFILES_PATH}
+            stan_file=BERN_STAN, stanc_options={'include-paths': DATAFILES_PATH}
         )
         self.assertEqual(BERN_STAN, model.stan_file)
         self.assertPathsEqual(model.exe_file, BERN_EXE)