Merge remote-tracking branch 'origin/master' into feature/martin_docs1

Author: Sjogren

Docker/Dockerfile-ORE-Dependencies

@@ -40,6 +40,6 @@ RUN python3 -m venv /venv
 ENV PATH="/venv/bin:$PATH"
 
 RUN \
-pip3 install cibuildwheel==3.2.1 && \
+pip3 install twine cibuildwheel==3.2.1 && \
 rm -rf ~/.cache/pip
 

Docker/Dockerfile-Wheels-CIBW

@@ -3,6 +3,7 @@ FROM ${DOCKER_REPO}ore-build-dependencies:${ORE_BUILD_VERSION}
 
 ARG WHEEL_ARCH
 ARG WHEEL_IMAGE
+ARG WHEELS_VERSION
 
 ENV PATH="/venv/bin:$PATH"
 
@@ -12,7 +13,9 @@ ENV CIBW_MANYLINUX_AARCH64_IMAGE=${WHEEL_IMAGE}
 ENV CIBW_BEFORE_BUILD="chmod +x ORE-SWIG/Wheels-gitlab/before_all_linux.sh && ORE-SWIG/Wheels-gitlab/before_all_linux.sh"
 
 WORKDIR /builds/qs/oreplus/ore
-RUN cibuildwheel --output-dir wheelhouse ORE-SWIG
+RUN \
+python ../ore/Docker/update_version_number.py -f setup.py -v ${WHEELS_VERSION} && \
+cibuildwheel --output-dir wheelhouse ORE-SWIG
 
 COPY wheelhouse /wheelhouse
 

Docker/Dockerfile-Wheels-ORE

@@ -4,6 +4,7 @@ FROM ${WHEEL_IMG}
 
 ARG CMAKE_BUILD_TYPE=Release
 ARG NUM_CORES=16
+ARG WHEELS_VERSION
 
 COPY CMakeLists.txt /ore/CMakeLists.txt
 COPY QuantLib /ore/QuantLib
@@ -12,6 +13,7 @@ COPY OREData /ore/OREData
 COPY OREAnalytics /ore/OREAnalytics
 COPY ThirdPartyLibs /ore/ThirdPartyLibs
 COPY cmake /ore/cmake
+COPY Docker/update_version_number.py /ore
 
 RUN \
 dnf -y install clang && \
@@ -24,6 +26,7 @@ ENV CCACHE_MAXSIZE="10G"
 
 WORKDIR /ore
 RUN \
+python3 update_version_number.py -f version.hpp -v ${WHEELS_VERSION} && \
 mkdir build && \
 cd build && \
 cmake .. -GNinja -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_CXX_FLAGS="-D BOOST_ENABLE_ASSERT_HANDLER $([ "$(uname -m)" = "x86_64" ] && echo "-mavx2")" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DQL_BUILD_EXAMPLES=OFF -DQL_BUILD_TEST_SUITE=OFF -DQL_ENABLE_PARALLEL_UNIT_TEST_RUNNER=ON -DQL_ENABLE_SESSIONS=ON -DORE_BUILD_APP=OFF -DORE_BUILD_DOC=OFF -DORE_BUILD_EXAMPLES=OFF -DORE_BUILD_SWIG=OFF -DORE_BUILD_TESTS=OFF -DORE_ENABLE_OPENCL=OFF -DORE_ENABLE_PARALLEL_UNIT_TEST_RUNNER=ON -DORE_PYTHON_INTEGRATION=OFF

Docker/update_version_number.py

@@ -0,0 +1,81 @@
+
+# Invoke this file from the ore directory to
+# overwrite the version number in one of the files below:
+#
+# ORE-SWIG/setup.py
+# QuantExt/qle/version.hpp
+#
+# e.g:
+# cd /path/to/ore
+# python /path/to/update_version_number.py -v 1.23.45 -f setup.py
+# python /path/to/update_version_number.py -v 1.23.45 -f version.hpp
+
+import optparse
+import shutil
+import re
+import os.path
+
+SETUP_PY = "ORE-SWIG/setup.py"
+VERSION_HPP = "QuantExt/qle/version.hpp"
+
+# Convert a version string into a string
+# containing a corresponding numerical value:
+# 1.8.14.0     ->  1081400
+# 1.234.5      -> 12340500
+# 1.234.5.dev1 -> 12340501
+def version_string_to_number(v):
+    ret = ""
+    # match x.x.x or x.x.x.x
+    # where x = one or more alphanumeric characters
+    p = r"(\w*)\.(\w*)\.(\w*)(?:\.(\w*))?"
+    m = re.match(p, v)
+    if not m:
+        raise Exception("version string '{v}' has invalid format, expected x.x.x or x.x.x.x where x in [a-zA-Z0-9_]")
+    for x in m.groups():
+        x = re.sub("[^0-9]", "", x or "")
+        if x:
+            ret += x.zfill(2)
+        else:
+            ret += '00'
+    return ret.lstrip('0')
+
+# Parse the command line arguments
+parser = optparse.OptionParser()
+parser.add_option('-f', '--file')
+parser.add_option('-v', '--version')
+opts, args = parser.parse_args()
+file = opts.file
+version = opts.version
+
+if file is None:
+    raise Exception("missing input parameter --file")
+
+if version is None:
+    raise Exception("missing input parameter --version")
+
+def find_and_replace(file, repl):
+
+    if not os.path.isfile(file):
+        raise Exception(f"invalid path: {file}")
+
+    shutil.copy(file, file + ".bak")
+
+    with open(file) as f:
+        s = f.read()
+
+    for (p, r) in repl:
+        s = re.sub(p, r, s)
+
+    with open(file, 'w') as f:
+        f.write(s)
+
+if file == "setup.py":
+    find_and_replace(SETUP_PY, [(r'(version\s*=\s*").*(")', fr'\g<1>{version}\g<2>')])
+elif file == "version.hpp":
+    version_num = version_string_to_number(version)
+    find_and_replace(VERSION_HPP, [
+        (r'(#define OPEN_SOURCE_RISK_VERSION ").*(")', fr'\g<1>{version}\g<2>'),
+        (r'(#define OPEN_SOURCE_RISK_VERSION_NUM ).*', fr'\g<1>{version_num}')])
+else:
+    raise Exception(f"unrecognized file: {file}")
+

Docs/ScriptedTrade/docs/language.tex

@@ -41,6 +41,8 @@
     \verb+max+ & \\
     \verb+min+ & \\
     \verb+pow+ & \\
+    \verb+frac+ & \\
+    \verb+round+ & \\
     \verb+black+ & \\
     \verb+dcf+ & \\
     \verb+days+ & \\ \hline
@@ -398,21 +400,21 @@
 \item {\tt REQUIRE Strike >= 0;}
 \end{itemize}
 
-% ====================================================
-\stsubsection{Functions {\tt min}, {\tt max}, {\tt pow}}
-% ====================================================
+% =========================================================================
+\stsubsection{Functions {\tt min}, {\tt max}, {\tt pow}}, {\tt round(x,y)}
+% =========================================================================
 
-Binary functions {\tt min(x,y)}, {\tt max(x,y)}, {\tt pow(x,y)}, applicable to numbers only.
+Binary functions {\tt min(x,y)}, {\tt max(x,y)}, {\tt pow(x,y)}, {\tt round(x,y)}, applicable to numbers only.
 
-% ====================================================
-\stsubsection{Functions {\tt -}, abs, exp, ln, sqrt}
-% ====================================================
+% ========================================================
+\stsubsection{Functions {\tt -}, abs, exp, ln, sqrt, frac}
+% ========================================================
 
-Unary functions {\tt -x}, {\tt abs(x)}, {\tt exp(x)}, {\tt ln(x)}, {\tt sqrt(x)}, applicable to numbers only.
+Unary functions {\tt -x}, {\tt abs(x)}, {\tt exp(x)}, {\tt ln(x)}, {\tt sqrt(x)}, {\tt frac(x)} which take the fractional part of a float, applicable to numbers only.
 
-% ====================================================
+% =======================================================
 \stsubsection{Functions {\tt normalPdf}, {\tt normalCdf}}
-% ====================================================
+% =======================================================
 
 Returns the standard normal pdf $\phi(x)$ resp. cdf $\Phi(x)$, applicable to numbers only.
 

Docs/UserGuide/curve_configurations/yieldcurves.tex

@@ -56,11 +56,19 @@ \subsubsection{Yield Curves}
 \item Extrapolation [Optional]: Set to \emph{True} or \emph{False} to enable or disable extrapolation respectively. If
 the element is omitted or left blank, then it defaults to \emph{True}.
 
-\item ExcludeT0FromInterpolation [Optional]: Set to \emph{True} to exclude the synthetic time-zero (t0) point from
-yield curve interpolation. When enabled, the curve interpolates only on actual market pillar dates, while still
-ensuring proper behavior at the reference date (e.g., $DF(t_0) = 1.0$ for discount curves). This is useful when the
-first market pillar date does not coincide with the as-of date and you want to avoid including a synthetic point in
-the interpolation.
+\item ExtrapolationMethod [Optional]: Only applies to bootstrapped curves and interpolated zero / discount curves.
+Controls how the curve is extrapolated:
+\begin{itemize}
+\item \emph{ContinuousForward}: The instantaneous forward rate at the last curve pillar date $T$ is kept constant
+\item \emph{DiscreteForward}: The discrete forward rate between $T-1$ and $T$ is kept constant, where $T$ is the last
+curve pillar date and $T-1$ lies one calendar day before $T$.
+\end{itemize}
+
+\item ExcludeT0FromInterpolation [Optional]: Only applies to bootstrapped curves and interpolated zero / discount curves.
+Set to \emph{True} to exclude the  synthetic time-zero (t0) point from yield curve interpolation. When enabled, the curve
+interpolates only on actual market pillar dates, while still ensuring proper behavior at the reference date
+(e.g., $DF(t_0) = 1.0$ for discount curves). This is useful when the first market pillar date does not coincide with the
+as-of date and you want to avoid including a synthetic point in the interpolation.
 
 This feature is supported for all three interpolation variables:
 \begin{itemize}
@@ -194,7 +202,10 @@ \subsubsection*{Simple Segment}
 floating rates on the instruments underlying the quotes listed in the \lstinline!Quote! nodes during the bootstrap
 procedure. This is an optional node. If it is left blank or omitted, then the projection curve is assumed to equal the
 curve being bootstrapped i.e.\ the current CurveId. The \lstinline!PillarChoice! node determines the bootstrap pillars
-that are used (MaturityDate, LastRelevantDate, if not given 'LastRelevantDate' is the default value).
+that are used (MaturityDate, LastRelevantDate, NoPillar, StartDate, StartDateAndMaturityDate,
+StartDateAndLastRelevantDate; if not given 'LastRelevantDate' is the default
+value). The \lstinline!DuplicatePillarPolcicy! node determined how curve instruments within the same segment with
+identical pillar dates are handled (Keep Last, KeepFirst, KeepAll, ThrowError; if not given 'KeepLast' is used).
 
 The \lstinline!Priority! node determines the priority of the segment, this has to be a non-negative integer. A lower
 number means a higher priority (more ``important'') segment. If two adjacent segments overlap w.r.t. the pillar dates of
@@ -223,6 +234,7 @@ \subsubsection*{Simple Segment}
   </Quotes>
   <Conventions> </Conventions>
   <PillarChoice> </PillarChoice>
+  <DuplicatePillarPolicy> </DuplicatePillarPolicy>
   <Priority> </Priority>
   <MinDistance> </MinDistance>
   <ProjectionCurve> </ProjectionCurve>
@@ -261,11 +273,10 @@ \subsubsection*{Average OIS Segment}
 quote for an Average OIS instrument (a typical example in a USD Overnight Index Swap) consists of two quotes, a vanilla
 IRS quote and an OIS-LIBOR basis swap spread quote.  The IDs of these two quotes are stored in the
 \lstinline!CompositeQuote! node. The \lstinline!RateQuote! node holds the ID of the vanilla IRS quote and the
-\lstinline!SpreadQuote! node holds the ID of the OIS-LIBOR basis swap spread quote. The \lstinline!PillarChoice! node
-determines the bootstrap pillars that are used (MaturityDate, LastRelevantDate, if not given `LastRelevantDate' is the
-default value).
+\lstinline!SpreadQuote! node holds the ID of the OIS-LIBOR basis swap spread quote.
 
-For the \lstinline!Priority! and \lstinline!MinDistance! nodes see the explanation under ``Simple Segment''.
+For the \lstinline!PillarChoice!, \lstinline!DuplicatePillarPolicy!, \lstinline!Priority! and \lstinline!MinDistance!
+nodes see the explanation under ``Simple Segment''.
 
 \begin{listing}[H]
 %\hrule\medskip
@@ -279,6 +290,7 @@ \subsubsection*{Average OIS Segment}
   </Quotes>
   <Conventions> </Conventions>
   <PillarChoice> </PillarChoice>
+  <DuplicatePillarPolicy> </DuplicatePillarPolicy>
   <Priority> </Priority>
   <MinDistance> </MinDistance>
   <ProjectionCurve> </ProjectionCurve>
@@ -313,13 +325,13 @@ \subsubsection*{Tenor Basis Segment}
 node has the value \emph{Tenor Basis Two Swaps}. Again, the structure is similar to the simple segment in Listing
 \ref{lst:simple_segment} except that there are two projection curve nodes. There is a \lstinline!ProjectionCurveReceive!
 node for the index with the shorter tenor. This node holds the CurveId of a curve for projecting the floating rates on
-the receiving side. Similarly, there is a \lstinline!ProjectionCurvePay! node for the index of the pay side. The deprecated
-values are short for receive, and long for pay. These are optional nodes. If they are left blank or omitted, then the projection
-curve is assumed to equal the curve being bootstrapped i.e.\ the current CurveId. However, at least one of the nodes
-needs to be populated to allow the bootstrap to proceed. The \lstinline!PillarChoice! node determines the bootstrap pillars
-that are used (MaturityDate, LastRelevantDate, if not given `LastRelevantDate' is the default value).
+the receiving side. Similarly, there is a \lstinline!ProjectionCurvePay! node for the index of the pay side. The
+deprecated values are short for receive, and long for pay. These are optional nodes. If they are left blank or omitted,
+then the projection curve is assumed to equal the curve being bootstrapped i.e.\ the current CurveId. However, at least
+one of the nodes needs to be populated to allow the bootstrap to proceed.
 
-For the \lstinline!Priority! and \lstinline!MinDistance! nodes see the explanation under ``Simple Segment''.
+For the \lstinline!PillarChoice!, \lstinline!DuplicatePillarPolicy!, \lstinline!Priority! and \lstinline!MinDistance!
+nodes see the explanation under ``Simple Segment''.
 
 \begin{listing}[H]
 %\hrule\medskip
@@ -333,6 +345,7 @@ \subsubsection*{Tenor Basis Segment}
   </Quotes>
   <Conventions> </Conventions>
   <PillarChoice> </PillarChoice>
+  <DuplicatePillarPolicy> </DuplicatePillarPolicy>
   <Priority> </Priority>
   <MinDistance> </MinDistance>
   <ProjectionCurvePay> </ProjectionCurvePay>
@@ -350,8 +363,7 @@ \subsubsection*{Cross Currency Segment}
 for FX forward instruments. The \lstinline!DiscountCurve! node holds the CurveId of a curve used to discount cash flows
 in the other currency i.e.\ the currency in the currency pair that is not equal to the currency in Listing
 \ref{lst:top_level_yc}. The \lstinline!SpotRate! node holds the ID of a spot FX quote for the currency pair that is
-looked up in the {\tt market.txt} file. The \lstinline!PillarChoice! node determines the bootstrap pillars that are used
-(MaturityDate, LastRelevantDate, if not given `LastRelevantDate' is the default value).
+looked up in the {\tt market.txt} file.
 
 \begin{listing}[H]
 %\hrule\medskip
@@ -365,6 +377,7 @@ \subsubsection*{Cross Currency Segment}
   </Quotes>
   <Conventions> </Conventions>
   <PillarChoice> </PillarChoice>
+  <DuplicatePillarPolicy> </DuplicatePillarPolicy>
   <Priority> </Priority>
   <MinDistance> </MinDistance>
   <DiscountCurve> </DiscountCurve>
@@ -388,7 +401,8 @@ \subsubsection*{Cross Currency Segment}
 the other currency. If it is left blank or omitted, then it is assumed to equal the CurveId provided in the
 \lstinline!DiscountCurve! node in this segment.
 
-For the \lstinline!Priority! and \lstinline!MinDistance! nodes see the explanation under ``Simple Segment''.
+For the \lstinline!PillarChoice!, \lstinline!DuplicatePillarPolicy!, \lstinline!Priority! and \lstinline!MinDistance!
+nodes see the explanation under ``Simple Segment''.
 
 \begin{listing}[H]
 %\hrule\medskip
@@ -402,6 +416,7 @@ \subsubsection*{Cross Currency Segment}
   </Quotes>
   <Conventions> </Conventions>
   <PillarChoice> </PillarChoice>
+  <DuplicatePillarPolicy> </DuplicatePillarPolicy>
   <Priority> </Priority>
   <MinDistance> </MinDistance>
   <DiscountCurve> </DiscountCurve>

Docs/UserGuide/examples/examples.tex

@@ -615,8 +615,9 @@ \subsubsection{Historical Simulation VaR}
 The analytic is specified as usual in {\tt ore.xml} with the following parameters:
 \begin{itemize}
 \item outputFile: csv file name of the resulting VaR report
-%\item breakdown: boolean, if true the VaR report will contain a breakdown by risk class and risk type, otherwise the report shows the portfolio-lvel VaR only.
-\item tradePnl: boolean, if true the VaR report will contain a breakdown by tradeID, risk class and risk type, otherwise the report shows the portfolio-lvel VaR only.
+%\item breakdown: boolean, if true the VaR report will contain a breakdown by risk class and risk type, otherwise the report shows the portfolio-level VaR only.
+\item tradePnl: boolean, if true the VaR report will contain a breakdown by tradeID, risk class and risk type, otherwise the report shows the portfolio-level VaR only.
+\item riskFactorBreakdown: boolean, if true the VaR report will contain a breakdown by risk factor.
 \item quantiles: comma separated list of quantiles to be reported
 \item portfolioFilter (optional): Only trades with {\tt portfolioId} equal to the provided filter name are processed, see {\tt portfolio.xml}; the entire portfolio is processed, if omitted
 \item historicalPeriod: comma-separated date list, an even number of ordered dates is required (d1, d2, d3, d4, ...), where each pair (d1-d2, d3-d4, ...) defines the start and end of historical observation periods used

Docs/UserGuide/parameterisation/curveconfig.tex

@@ -1263,11 +1263,21 @@ \subsubsection{Bootstrap Configuration}
 \item \lstinline!DontThrowSteps! [Optional]:
 This node is used only if \lstinline!DontThrow! is \lstinline!true!. The meaning of this node is given in the description of the \lstinline!DontThrow! node. This node should hold a positive integer. If omitted, the default value is 10.
 
-\item \lstinline!Global! [Optional]: This nodes specifies whether the curve should use a global bootstrap over all
-  instruments of the curve (true) or iterative bootstrap (false, default value). If global bootstrap is used, only the
-  Accuracy field of the BootstrapConfig is relevant and specifies the accuracy of the global optimizer. If a cycle is
-  present in the dependency graph of the curves, all members of the cycle must use global bootstrap, in this case a
-  global optimization is run over all instrument of all members of the cycle simultaneously.
+\item \lstinline!Global! [Optional]: Defaults to false. This nodes specifies whether the curve should use a global
+  bootstrap over all instruments of the curve (true) or iterative bootstrap (false, default value). If global bootstrap
+  is used, only the fields Accuracy and SmoothnessLambda of the BootstrapConfig are relevant. Accurcy specifies the
+  accuracy of the global optimizer in this case.  Global bootstrap is required to build curves that form a cycle in the
+  dependency graph, in this case all members of the cycle must have Global set to true in their bootstrap configs. In
+  this case a global optimization is run over all instrument of all members of the cycle simultaneously. Global
+  bootstrap is also required if according to the PillarChoice configuration of each segment there are curve instrument
+  without or with more than one associated curve pillar date. Finally, global bootstrap can be preferable over iterative
+  bootstrap if the latter requires an outer convergence loop, i.e. if a pillar choice is not set to LastRelevatDate or
+  if the interpolation is non-local.
+
+\item \lstinline!SmoothnessLambda! [Optional]: Defaults to zero. Only applies if GLobal is set to true. If positive,
+  penalty components are included in the global optimization used to build a curve. For each adjacent pair of discrete
+  forward rates $F_i$, $F_{i+1}$ between curve interpolation pillars, the term $\lambda\cdot(F_{i+1} - F_i)$ is added to
+  the target function vector of which the MSE is optimized.
 
 \end{itemize}