Merge pull request #464 from stan-dev/external-fns

Allow user to supply a c++ header file

Author: mitzimorris

cmdstanpy/compiler_opts.py

@@ -48,18 +48,21 @@ class CompilerOptions:
     Attributes:
         stanc_options - stanc compiler flags, options
         cpp_options - makefile options (NAME=value)
+        user_header - path to a user .hpp file to include during compilation
     """
 
     def __init__(
         self,
         *,
         stanc_options: Optional[Dict[str, Any]] = None,
         cpp_options: Optional[Dict[str, Any]] = None,
+        user_header: Optional[str] = None,
         logger: Optional[logging.Logger] = None,
     ) -> None:
         """Initialize object."""
         self._stanc_options = stanc_options if stanc_options is not None else {}
         self._cpp_options = cpp_options if cpp_options is not None else {}
+        self._user_header = user_header if user_header is not None else ''
         if logger is not None:
             get_logger().warning(
                 "Parameter 'logger' is deprecated."
@@ -88,6 +91,7 @@ def validate(self) -> None:
         """
         self.validate_stanc_opts()
         self.validate_cpp_opts()
+        self.validate_user_header()
 
     def validate_stanc_opts(self) -> None:
         """
@@ -104,17 +108,15 @@ def validate_stanc_opts(self) -> None:
                 get_logger().info('ignoring compiler option: %s', key)
                 ignore.append(key)
             elif key not in STANC_OPTS:
-                raise ValueError(
-                    'unknown stanc compiler option: {}'.format(key)
-                )
+                raise ValueError(f'unknown stanc compiler option: {key}')
             elif key == 'include_paths':
                 paths = val
                 if isinstance(val, str):
                     paths = val.split(',')
                 elif not isinstance(val, list):
                     raise ValueError(
                         'Invalid include_paths, expecting list or '
-                        'string, found type: {}.'.format(type(val))
+                        f'string, found type: {type(val)}.'
                     )
             elif key == 'use-opencl':
                 if self._cpp_options is None:
@@ -149,10 +151,48 @@ def validate_cpp_opts(self) -> None:
                 val = self._cpp_options[key]
                 if not isinstance(val, int) or val < 0:
                     raise ValueError(
-                        '{} must be a non-negative integer value,'
-                        ' found {}.'.format(key, val)
+                        f'{key} must be a non-negative integer value,'
+                        f' found {val}.'
                     )
 
+    def validate_user_header(self) -> None:
+        """
+        User header exists.
+        Raise ValueError if bad config is found.
+        """
+        if self._user_header != "":
+            if not (
+                os.path.exists(self._user_header)
+                and os.path.isfile(self._user_header)
+            ):
+                raise ValueError(
+                    f"User header file {self._user_header} cannot be found"
+                )
+            if self._user_header[-4:] != '.hpp':
+                raise ValueError(
+                    f"Header file must end in .hpp, got {self._user_header}"
+                )
+            if "allow_undefined" not in self._stanc_options:
+                self._stanc_options["allow_undefined"] = True
+            # set full path
+            self._user_header = os.path.abspath(self._user_header)
+
+            if ' ' in self._user_header:
+                raise ValueError(
+                    "User header must be in a location with no spaces in path!"
+                )
+
+            if (
+                'USER_HEADER' in self._cpp_options
+                and self._user_header != self._cpp_options['USER_HEADER']
+            ):
+                raise ValueError(
+                    "Disagreement in user_header C++ options found!\n"
+                    f"{self._user_header}, {self._cpp_options['USER_HEADER']}"
+                )
+
+            self._cpp_options['USER_HEADER'] = self._user_header
+
     def add(self, new_opts: "CompilerOptions") -> None:  # noqa: disable=Q000
         """Adds options to existing set of compiler options."""
         if new_opts.stanc_options is not None:
@@ -167,6 +207,8 @@ def add(self, new_opts: "CompilerOptions") -> None:  # noqa: disable=Q000
         if new_opts.cpp_options is not None:
             for key, val in new_opts.cpp_options.items():
                 self._cpp_options[key] = val
+        if new_opts._user_header != '' and self._user_header == '':
+            self._user_header = new_opts._user_header
 
     def add_include_path(self, path: str) -> None:
         """Adds include path to existing set of compiler options."""
@@ -191,10 +233,10 @@ def compose(self) -> List[str]:
                         )
                     )
                 elif key == 'name':
-                    opts.append('STANCFLAGS+=--{}={}'.format(key, val))
+                    opts.append(f'STANCFLAGS+=--name={val}')
                 else:
-                    opts.append('STANCFLAGS+=--{}'.format(key))
+                    opts.append(f'STANCFLAGS+=--{key}')
         if self._cpp_options is not None and len(self._cpp_options) > 0:
             for key, val in self._cpp_options.items():
-                opts.append('{}={}'.format(key, val))
+                opts.append(f'{key}={val}')
         return opts

cmdstanpy/model.py

@@ -76,6 +76,10 @@ class CmdStanModel:
     :param cpp_options: Options for C++ compiler, specified as a Python
         dictionary containing C++ compiler option name, value pairs.
         Optional.
+
+    :param user_header: A path to a header file to include during C++
+        compilation.
+        Optional.
     """
 
     def __init__(
@@ -86,6 +90,7 @@ def __init__(
         compile: bool = True,
         stanc_options: Optional[Dict[str, Any]] = None,
         cpp_options: Optional[Dict[str, Any]] = None,
+        user_header: Optional[str] = None,
         logger: Optional[logging.Logger] = None,
     ) -> None:
         """
@@ -97,12 +102,16 @@ def __init__(
         :param compile: Whether or not to compile the model.
         :param stanc_options: Options for stanc compiler.
         :param cpp_options: Options for C++ compiler.
+        :param user_header: A path to a header file to include during C++
+            compilation.
         """
         self._name = ''
         self._stan_file = None
         self._exe_file = None
         self._compiler_options = CompilerOptions(
-            stanc_options=stanc_options, cpp_options=cpp_options
+            stanc_options=stanc_options,
+            cpp_options=cpp_options,
+            user_header=user_header,
         )
         if logger is not None:
             get_logger().warning(
@@ -235,6 +244,11 @@ def cpp_options(self) -> Dict[str, Union[bool, int]]:
         """Options to C++ compilers."""
         return self._compiler_options._cpp_options
 
+    @property
+    def user_header(self) -> str:
+        """The user header file if it exists, otherwise empty"""
+        return self._compiler_options._user_header
+
     def code(self) -> Optional[str]:
         """Return Stan program as a string."""
         if not self._stan_file:
@@ -255,6 +269,7 @@ def compile(
         force: bool = False,
         stanc_options: Optional[Dict[str, Any]] = None,
         cpp_options: Optional[Dict[str, Any]] = None,
+        user_header: Optional[str] = None,
         override_options: bool = False,
     ) -> None:
         """
@@ -272,6 +287,8 @@ def compile(
 
         :param stanc_options: Options for stanc compiler.
         :param cpp_options: Options for C++ compiler.
+        :param user_header: A path to a header file to include during C++
+            compilation.
 
         :param override_options: When ``True``, override existing option.
             When ``False``, add/replace existing options.  Default is ``False``.
@@ -280,9 +297,15 @@ def compile(
             raise RuntimeError('Please specify source file')
 
         compiler_options = None
-        if not (stanc_options is None and cpp_options is None):
+        if not (
+            stanc_options is None
+            and cpp_options is None
+            and user_header is None
+        ):
             compiler_options = CompilerOptions(
-                stanc_options=stanc_options, cpp_options=cpp_options
+                stanc_options=stanc_options,
+                cpp_options=cpp_options,
+                user_header=user_header,
             )
             compiler_options.validate()
             if self._compiler_options is None:

docsrc/env.yml

@@ -6,6 +6,7 @@ dependencies:
   - python=3.7
   - ipykernel
   - ipython
+  - ipywidgets
   - numpy>=1.15
   - pandas
   - xarray

docsrc/examples.rst

@@ -8,3 +8,4 @@ __________________
     examples/Maximum Likelihood Estimation.ipynb
     examples/Variational Inference.ipynb
     examples/Run Generated Quantities.ipynb
+    examples/Using External C++.ipynb

docsrc/examples/.gitignore

@@ -7,4 +7,5 @@
 *.hpp
 *.exe
 *.csv
-.ipynb_checkpoints
+.ipynb_checkpoints
+!make_odds.hpp

docsrc/examples/Using External C++.ipynb

@@ -0,0 +1,157 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Advanced Topic: Using External C++ Functions\n",
+    "\n",
+    "This is based on the relevant portion of the CmdStan documentation [here](https://mc-stan.org/docs/cmdstan-guide/using-external-cpp-code.html)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Consider the following Stan model, based on the bernoulli example."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {"nbsphinx": "hidden"},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "try:\n",
+    "    os.remove('bernoulli_external')\n",
+    "except:\n",
+    "    pass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from cmdstanpy import CmdStanModel\n",
+    "model_external = CmdStanModel(stan_file='bernoulli_external.stan', compile=False)\n",
+    "print(model_external.code())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "As you can see, it features a function declaration for `make_odds`, but no definition. If we try to compile this, we will get an error. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_external.compile()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Even enabling the `--allow_undefined` flag to stanc3 will not allow this model to be compiled quite yet."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_external.compile(stanc_options={'allow_undefined':True})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To resolve this, we need to both tell the Stan compiler an undefined function is okay **and** let C++ know what it should be. \n",
+    "\n",
+    "We can provide a definition in a C++ header file by using the `user_header` argument to either the CmdStanModel constructor or the `compile` method. \n",
+    "\n",
+    "This will enables the `allow_undefined` flag automatically."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_external.compile(user_header='make_odds.hpp')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can then run this model and inspect the output"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fit = model_external.sample(data={'N':10, 'y':[0,1,0,0,0,0,0,0,0,1]})\n",
+    "fit.stan_variable('odds')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The contents of this header file are a bit complicated unless you are familiar with the C++ internals of Stan, so they are presented without comment:\n",
+    "\n",
+    "```c++\n",
+    "#include <boost/math/tools/promotion.hpp>\n",
+    "#include <ostream>\n",
+    "\n",
+    "namespace bernoulli_model_namespace {\n",
+    "    template <typename T0__>  inline  typename\n",
+    "          boost::math::tools::promote_args<T0__>::type  \n",
+    "          make_odds(const T0__& theta, std::ostream* pstream__) {\n",
+    "            return theta / (1 - theta);  \n",
+    "       }\n",
+    "}\n",
+    "```"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "8765ce46b013071999fc1966b52035a7309a0da7551e066cc0f0fa23e83d4f60"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.9.5 64-bit ('stan': conda)",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.5"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}