diff --git a/Dataproducts-Summary-table.tex b/Dataproducts-Summary-table.tex new file mode 100644 index 0000000..1870500 --- /dev/null +++ b/Dataproducts-Summary-table.tex @@ -0,0 +1,33 @@ +% Mireille Louys- 2026-June 24 +%proposal for taking into account VEP for analysis products and response function +%\newpage +\subsubsection{Summary Table} + +The proposed vocabulary entries are listed in Table~\ref{tab:dp_vocabulary} with their labels and parents identified. Two vocabularies are proposed for response functions and analysis products following the semantics procedure for VEPs (see \url{https://github.com/ivoa-std/VEPs}). + +\begin{landscape} +\begin{longtable}{p{0.1\linewidth}p{0.175\linewidth}p{0.475\linewidth}p{0.175\linewidth}} +\sptablerule +\textbf{Term} & \textbf{Label} & \textbf{Description} &\textbf{Parent}\cr +\sptablerule +\multicolumn{4}{|r|}{\bf Added to \emph{product-type} Vocabulary \url{https://www.ivoa.net/rdf/product-type/}} \\ \hline +{\bf hea-event-bundle} & Event Bundle & A compounded dataset containing containing an {\bf hea-event-list} and multiple files or other substructures that are products necessary to analyze the {\bf hea-event-list} & none \cr +{\bf hea-event-list} & Event list & A dataset that records a collection of observed particle-detection events, such as incoming high-energy particles, where an event is typically characterized by a spatial position, a time, and a spectral value ({\em e.g.\/}, an energy, a channel, a pulse height) & none \cr + \hline +\multicolumn{4}{|r|}{\bf Added to \emph{response-type} vocabulary \url{https://www.ivoa.net/rdf/response-type/} } \\ \hline +{\bf response-function} & Response Function & A dataset that maps a physical quantity to an observable quantitiy. This term is mainly intended for retrieval. To annotate datasets, use a narrower term Narrower terms are preferred to indicate more precisely the type of {\bf response-function} & \cr +{\bf rmf} &\raggedright Redistribution Matrix File & A dataset that records the probability density function mapping from energy space into detector pulse height (or position) space & \#response-function \cr +{\bf aeff} & Effective Area & A dataset that records the ``effective area'' of a telescope and/or instrument. The effective area is the geometric area of the telescope and/or instrument reduced by efficiency factors such as reflectivity and vignetting, among other effects & \#response-function \cr +{\bf arf} &\raggedright Ancillary Response File & A dataset that records the combined telescope/instrument effective area and detector quantum efficiency as a function of energy & \#response-function \cr +{\bf bkgrate} & Background Rate & A dataset that models the rate of residual events that are not from the expected source type ({\em e.g.\/}, for gamma-ray instrument {\bf bkgrate} measures residual non-gamma-ray events coming from charged cosmic rays) & \#response-function \cr +{\bf edisp} & Energy Dispersion & A dataset that records the probability density of detecting an event with an energy estimator (proxy) given the true energy of the event & \#response-function \cr +{\bf psf} &\raggedright Point Spread Function & A dataset that records the probability density function of spatial/angular spreading of incident particles from a point source caused by the instrument (detector and/or mirror and/or analysis) & \#response-function \cr \hline +\multicolumn{4}{|r|}{\bf Added to \emph{analysis-product} vocabulary \url{https://github.com/ivoa-std/VEPs}} \\ \hline +{\bf draws} & Draws & A dataset that records statistical draws computed from a probability distribution, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable & none \cr +{\bf pdf} &\raggedright Probability Density Function & A dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable & none \cr +{\bf region} & Region & A dataset that encodes (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary & none \cr +%\sptablerule +\caption{IVOA Vocabulary Extension for High energy data products. } +\label{tab:dp_vocabulary} +\end{longtable} +\end{landscape} diff --git a/HighEnergyObsCoreExt.bib b/HighEnergyObsCoreExt.bib index 673d980..565f2ff 100644 --- a/HighEnergyObsCoreExt.bib +++ b/HighEnergyObsCoreExt.bib @@ -7,6 +7,15 @@ @MISC{2024ivoa.note.heig url = {https://www.ivoa.net/documents/Notes/VOHE/20241112/index.html}, } +@MISC{2024ivoa.note.datalink, + author = {{Bonnarel}, F. and {Demleitner}, M. and {Dowler}, P. and {Michel}, L. and {Taylor}, M. }, + title = "{IVOA DataLink Implementation note}", + howpublished = {IVOA Note 12 November 2024}, + year = 2024, + month = jan, + url = {https://www.ivoa.net/documents/Notes/DataLinkImp/20240719/index.html}, +} + @INPROCEEDINGS{2006SPIE.6270E..1VF, author = {{Fruscione}, Antonella and {McDowell}, Jonathan C. and {Allen}, Glenn E. and {Brickhouse}, Nancy S. and {Burke}, Douglas J. and {Davis}, John E. and {Durham}, Nick and {Elvis}, Martin and {Galle}, Elizabeth C. and {Harris}, Daniel E. and {Huenemoerder}, David P. and {Houck}, John C. and {Ishibashi}, Bish and {Karovska}, Margarita and {Nicastro}, Fabrizio and {Noble}, Michael S. and {Nowak}, Michael A. and {Primini}, Frank A. and {Siemiginowska}, Aneta and {Smith}, Randall K. and {Wise}, Michael}, title = "{CIAO: Chandra's data analysis system}", @@ -71,6 +80,14 @@ @misc{ogip_psf_2011 year = 2011, url = {https://heasarc.gsfc.nasa.gov/docs/heasarc/caldb/docs/memos/cal_gen_92_027/cal_gen_92_027.pdf}, } +@misc{ogip_Fits_event, + title = {GUIDELINES FOR DEFINING FITS FORMATS FOR EVENT LISTS}, + author = {{George}, I.M. and {Rots}, A. and {Mukai}, K.}, + howpublished = {Technical Report OGIP/94-003, NASA/GSFC, Dec 1994}, + month = dec, + year = 1994, + url = {https://heasarc.gsfc.nasa.gov/docs/heasarc/ofwg/docs/events/ogip_94_003/ogip_94_003.html}, +} @misc{ogip_spectrum_1998, title = {The Calibration Requirements for Spectral Analysis}, @@ -109,4 +126,4 @@ @INCOLLECTION{2019adds.book..283C doi = {10.1007/978-94-024-1631-2_7}, adsurl = {https://ui.adsabs.harvard.edu/abs/2019adds.book..283C}, adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} \ No newline at end of file +} diff --git a/HighEnergyObsCoreExt.tex b/HighEnergyObsCoreExt.tex index d4ab57e..8d2406c 100644 --- a/HighEnergyObsCoreExt.tex +++ b/HighEnergyObsCoreExt.tex @@ -48,7 +48,7 @@ % mireille : in order to flag changes to fill \newcommand{\TODO}[1]{% \noindent% - \colorbox{todocolor}{% + \colorbox{green}{% \parbox{0.85\linewidth}{\sffamily \textbf{TODO:}\\ #1} }% @@ -144,7 +144,7 @@ \section{High Energy Astrophysics Data} Though {\bf hea-event-list}s {\em may\/} include estimators for calibrated physical values, they typically still have to be corrected for the photometric, spectral, spatial, and/or temporal responses of the telescope and detector combination to yield scientifically interpretable information. The mappings between physical measurements of the source properties and the observables are called Instrument Response Functions (\glspl{IRF}\footnote{We try to avoid using the term \gls{IRF} in a normative sense since historical usage across the broad \gls{HEA} community (and from facility to facility) varies. In some cases, \gls{IRF} has been used to mean specifically the product of the \gls{ARF} and \gls{RMF}, whereas in other cases \gls{IRF} has been used more generally to mean any instrumental response function regardless of type.}). Some \glspl{IRF} are probabilistic in nature\footnote{For example, the energy matrix is a probability density function.}, and in addition may depend on the set of events selected for analysis by the end user. They are usually not invertible, so methods such as forward-folding fitting (using source models with any combination of spectral, spatial, temporal, and/or polarization components that are estimated) are needed to estimate physical properties, such as the true flux of particles from a source arriving at the instrument, given the measured observable quantities. The \glspl{IRF} generally evolve over time with the instrument and observation characteristics, and are usually defined for a specific time interval and may be decomposed into a standard set of independent components (see \S~3.1.5 of \citealt{2024ivoa.note.heig}), such as the spatial point-spread function or the energy-migration matrix or different messenger particle types, where each component may be stored or computed separately. Since both \glspl{IRF} and {\bf hea-event-list}s are required to analyze \gls{HEA} data, some \gls{IVOA} standards must be modified to expose both of them via the \gls{VO}. -In the following, the current ObsCore standard will be discussed in \S~\ref{sec:obscore}, focusing on attributes that need to be modified. Then, we propose the creation of a \gls{HEA} extension of ObsCore in \S~\ref{sec:obscoreext}, as some attributes are very specific to our domain. In these two sections, the discussion focuses on the attribute definitions rather on the attribute values. In \S~\ref{sec:voc}, enhancement of vocabulary is proposed for some ObsCore attributes, DataLink semantics, UCDs, and MIME-types. +In the following, the current ObsCore standard will be discussed in \S~\ref{sec:obscore}, focusing on attributes that need to be modified. Then, we propose the creation of a \gls{HEA} extension of ObsCore in \S~\ref{sec:obscoreext}, as some attributes are very specific to our domain. In these two sections, the discussion focuses on the attribute definitions rather than on the attribute values. In \S~\ref{sec:voc}, enhancement of vocabulary is proposed for some ObsCore attributes, DataLink semantics, UCDs, and MIME-types. \section{ObsCore Attribute Definitions for High Energy Astrophysics Data} \label{sec:obscore} @@ -166,14 +166,14 @@ \subsection{{\em dataproduct\_type}} {\bf event}: an event-counting ({\em e.g.\/}, X-ray or other high energy) dataset of some sort. Typically this is instrumental data, {\em i.e.\/}, ``event data''. An event dataset is often a complex object containing multiple files or other substructures. An event dataset may contain data with spatial, spectral, and time information for each measured event, although the spectral resolution (energy) is sometimes limited. Event data may be used to produce higher level data products such as images or spectra. \end{quote} -We propose to add the following {\em dataproduct\_type\/} terms to ObsCore to better define a \gls{HEA} {\bf he-event-list} and an {\bf he-event-bundle} that includes the {\bf hea-event-list} and associated data: +We propose to add the following {\em dataproduct\_type\/} terms to ObsCore to better define a \gls{HEA} {\bf hea-event-list} and an {\bf hea-event-bundle} that includes the {\bf hea-event-list} and associated data: \begin{quote} {\bf hea-event-list}: a dataset that records a collection of observed particle-detection events, such as incoming high-energy particles, where an event is typically characterized by a spatial position, a time, and a spectral value ({\em e.g.\/}, an energy, a channel, a pulse height). {\bf hea-event-bundle}: a compounded dataset containing an {\bf hea-event-list} and multiple files or other substructures that are products necessary to analyze the hea-event-list. Data in an {\bf hea-event-bundle} may thus be used to produce higher level data products calibrated in physical units when containing \glspl{IRF} or other data products that can be used to construct \glspl{IRF}. \end{quote} -We note that the term {\bf event} has caused confusion in the past, since ``event'' may also used to describe notifications ({\em e.g.\/}, as in ``VOEvent'') of astrophysical events such as supernova explosions. Such ``events'' are quite different from the particle detection events being described herein. Using {\bf hea-event-list} will help to resolve this ambiguity. +We note that the term {\bf event} has caused confusion in the past, since ``event'' may also be used to describe notifications ({\em e.g.\/}, as in ``VOEvent'') of astrophysical events such as supernova explosions. Such ``events'' are quite different from the particle detection events being described herein. Using {\bf hea-event-list} will help to resolve this ambiguity. In addition to {\em dataproduct\_type\/} terms that focus on event data, we note that existing ObsCore definitions do not adequately span the breadth of ``advanced data products'' (typically with {\em calib\_level\/} $\ge$ 3) that may be generated from astronomical observations by users or observatories. The computational complexity of analyzing \gls{HEA} data robustly in the extreme Poisson regime ({\em e.g.\/}, Bayesian X-ray aperture photometry applied simultaneously to multiple overlapping detections and observations, or Frequentist adjustment of models of electron populations for multi-wavelength data spanning from X-rays to PeV gamma rays) means that data providers may choose to provide such analysis products directly to the end user. For example, the Chandra Source Catalog includes 38 types of advanced data products (for a total of $\sim\!90$ million files) and $\sim\!50$\% of these data product types are not well represented by a {\em dataproduct\_type\/} value that allows for meaningful data discovery. Users will certainly want to discover these data products independently from the associated progenitor observation data (and many of these data products combine data from multiple observations). We therefore propose the following additional {\em dataproduct\_type\/} (or {\em dataproduct\_subtype\/}) terms for these advanced data products, and note that these terms will certainly be useful independent of waveband ({\em i.e.\/}, they can be equally applicable to UV/optical, IR, and radio datasets): @@ -187,11 +187,16 @@ \subsection{{\em dataproduct\_type}} {\bf response-function}: a dataset that records a mapping from a physical quantity to an observable quantity. For \gls{HEA}, this may be the components of the composite \gls{IRF} such as an Auxiliary Response File ({\bf arf}), Redistribution Matrix File ({\bf rmf}), Effective Area ({\bf aeff}), Energy Dispersion ({\bf edisp}), the Background Rate ({\bf bkgrate}). The Point Spread Function ({\bf psf}) is a response function that is generally applicable across multiple wavebands. While these datasets may generally be represented as an N-dimensional data cube, designating them as {\bf response-function}s enhances data discovery for very common types of \gls{HEA} dataset (see the use cases in Appendix~\ref{sec:uc}). \end{quote} +%\TODO{ mireille: Rewrite this paragraph to avoid measurements: +%no agreement exist for this term among ObsCore Implementors in practice. } + The {\bf measurements} {\em dataproduct\_type\/} is quite useful for many different types of advanced data products (which may be derived from multiple observations). But users of those products often may not be interested in the progenitor datasets, especially as multiple advanced data products may be extracted from the same single progenitor or a few progenitors ({\em e.g.\/}, measurements associated with multiple sources detected in a single observation field). We propose to delete the caveat associated with {\em dataproduct\_type\/} = ``measurements'' in the ObsCore IVOA Recommendation (\S~4.1.1) that requires the derived data products be exposed ``{\bf together} with the progenitor observation dataset''. The recovery of progenitor observation datasets may be achieved using provenance information, if desired. \subsection{{\em dataproduct\_subtype}} -The optional attribute {\em dataproduct\_subtype} may be used by the data provider to specify more precisely the scientific nature of a data product. Although no vocabulary is defined for {\em dataproduct\_subtype\/}, we recommend that data providers formulate and use a standardized vocabulary for this attribute for data products that are commonly used in \gls{HEA}\null. We have proposed several terms in \S~5 for commonly used \gls{HEA} {\bf response-function} types ({\em e.g.\/}, {\bf aeff}, {\bf edisp}, {\bf psf}), but additional terms could be standardized for other common data products. For example, standardizing using {\bf exposuremap} for an exposure map would enable queries such as ({\em dataproduct\_type\/} = {\bf image}) AND ({\em dataproduct\_subtype\/} = {\bf exposuremap}) to work across multiple facilities. Other possible terms could include (but are not limited to) {\bf significancemap} for a significance map, {\bf probabilitymap} for aprobability map, and {\bf exclusionmap} for an exclusion map ({\em e.g.\/}, as used to adjust TeV background models). +The optional attribute {\em dataproduct\_subtype} may be used by the data provider to specify more precisely the scientific nature of a data product. Although no vocabulary is defined for {\em dataproduct\_subtype\/}, we recommend that data providers formulate and use a standardized vocabulary for this attribute for data products that are commonly used in \gls{HEA}\null. We have proposed several terms in \S~5 for commonly used \gls{HEA} {\bf response-function} types ({\em e.g.\/}, {\bf aeff}, {\bf edisp}, {\bf psf}), but additional terms could be standardized for other common data products. For example, standardizing using {\bf exposure-map} for an exposure map would enable queries such as ({\em dataproduct\_type\/} = {\bf image}) AND ({\em dataproduct\_subtype\/} = {\bf exposure-map}) to work across multiple facilities. Other possible terms could include (but are not limited to) {\bf significance-map} for a significance map, {\bf probability-map} for a probability map, and {\bf exclusion-map} for an exclusion map ({\em e.g.\/}, as used to adjust TeV background models). + +\TODO{ Propose a small table of definitions for all these maps : could be added in the analysis-product vocabulary} \subsection{{\em calib\_level}} @@ -257,7 +262,7 @@ \subsection{{\em o\_ucd}} In the example {\em o\_ucd\/} above, the UCD {\em instr.pulse;arith.sum\/} is used to represent the detector Pulse Height Amplitude (PHA)\null. There is currently no UCD defined for a raw measure like PHA, but we propose the addition of {\em instr.pulse\/} to the UCDs list vocabulary, together with other UCDs that are relevant for \gls{HEA} data, in \S~\ref{sec:UCDs}. Several additional UCDs, including electromagnetic spectrum, physical quantities, and statistical parameters UCDs, are also proposed in \S~\ref{sec:UCDs} that are relevant for \gls{HEA} data products but could also be of use for other domains such as cosmology. -Advanced data products may similarly record multiple observables that can only be differentiated through their UCDs. For example, a Chandra Source Catalog {\bf pdf} dataset for a detection may include multiple marginalized probability density functions computed using a Bayesian X-ray aperture photometry algorithm in units of net counts, net count rates, photon fluxes, and energy fluxes in multiple apertures. The observables recorded in the different MPDFs may be distinguished by their UCDs which then become relevant for data discovery when a user is searching for specific aperture photometry datasets. +Advanced data products may similarly record multiple observables that can only be differentiated through their UCDs. For example, a Chandra Source Catalog {\bf pdf} dataset for a detection may include multiple marginalized probability density functions computed using a Bayesian X-ray aperture photometry algorithm in units of net counts, net count rates, photon fluxes, and energy fluxes in multiple apertures. The observables recorded in the different marginal probability density functions (MPDF) may be distinguished by their UCDs which then become relevant for data discovery when a user is searching for specific aperture photometry datasets. \subsection{{\em proposal\_id}} @@ -336,11 +341,10 @@ \subsection{{\em pointing\_mode}} \subsection{{\em analysis\_mode}} -Most \gls{HEA} instruments employ significant software processing to transform raw data into the {\bf hea-event-bundle} data exposed to users, including algorithms for calibration and event property reconstruction. The way in which this processing is configured therefore has a potentially large impact the content of the reduced datasets; indeed the same observation processed with two different configurations may result in different scientific performance. In some cases, multiple processing configurations within the same observation collection are used to provide users with a wider range of scientific coverage. +Most \gls{HEA} instruments employ significant software processing to transform raw data into the {\bf hea-event-bundle} data exposed to users, including algorithms for calibration and event property reconstruction. The way in which this processing is configured therefore has a potentially large impact on the content of the reduced datasets; indeed the same observation processed with two different configurations may result in different scientific performance. In some cases, multiple processing configurations within the same observation collection are used to provide users with a wider range of scientific coverage. We propose to add an optional attribute {\bf analysis\_mode} that allows the data provider to specify the data reduction/analysis mode for an observation, in case more than one is applied. Constraints on analysis mode can provide a simple way to discover data sets for a specific facility/instrument combination. We note that permissible {\bf analysis\_mode} values may vary from facility to facility and from instrument to instrument. - \subsection{{\em event\_type}} \label{sec:evttype} @@ -377,7 +381,10 @@ \subsection{Evolution of the Data Product Type Vocabulary} The \gls{IVOA} Data Product Type Vocabulary\footnote{See \url{http://www.ivoa.net/rdf/product-type}.} provides terms, labels, and descriptions for many types of astronomical data products. However, there are some additions and changes that are appropriate to better support \gls{HEA} datasets. -We propose to add vocabulary entries for the new data product types outlined in \S~\ref{sec:dataproduct_type} and also propose to slightly modify the existing definition of {\bf hea-event-list} so that it aligns more accurately with the definition in that section. Additionally, we propose to add several more specific entries to the data product type vocabulary that specialize these types (especially {\bf response-function}). +We propose to add vocabulary entries for the new data product types outlined in \S~\ref{sec:dataproduct_type} and also propose to slightly modify the existing definition of {\bf hea-event-list} so that it aligns more accurately with the definition in that section. + +%explain here the 2 vocabularies + Additionally, we propose to add several more specific entries to the data product type vocabulary that specialize these types (especially {\bf response-function}). \subsubsection{Event List} @@ -388,6 +395,17 @@ \subsubsection{Event List} {\bf hea-event-list}: A dataset that records a collection of observed particle-detection events, such as incoming high-energy particles, where an event is typically characterized by a spatial position, a time, and a spectral value ({\em e.g.\/}, an energy, a channel, a pulse height). \end{quote} +\subsubsection{Event Bundle} + +Some use cases require access to a bundle of datasets that includes the {\bf hea-event-list} and associated data products. We define an {\bf hea-event-bundle}: + +\begin{quote} +{\bf hea-event-bundle}: A compounded dataset containing an {\bf hea-event-list} and multiple files or other substructures that are products necessary to analyze the hea-event-list. Data in an {\bf hea-event-bundle} may thus be used to produce higher level data products calibrated in physical units when containing \glspl{IRF} or other data products that can be used to construct \glspl{IRF}. +\end{quote} + +An {\bf hea-event-bundle} might for example consist of an {\bf hea-event-list} and the associated {\bf response-function}s used to calibrate the dataset, and may also contain provenance information, data quality time-series, and preview images or plots. + + \subsubsection{Response Functions} \label{sec:responsefct} @@ -413,17 +431,7 @@ \subsubsection{Response Functions} {\bf rmf}: A dataset that records the probability density function mapping from energy space into detector pulse height (or position) space \citep{ogip_spectrum_1998}. \end{quote} -\subsubsection{Event Bundle} - -Some use cases require access to a bundle of datasets that includes the {\bf hea-event-list} and associated data products. We define an {\bf hea-event-bundle}: - -\begin{quote} -{\bf hea-event-bundle}: A compounded dataset containing an {\bf hea-event-list} and multiple files or other substructures that are products necessary to analyze the hea-event-list. Data in an {\bf hea-event-bundle} may thus be used to produce higher level data products calibrated in physical units when containing \glspl{IRF} or other data products that can be used to construct \glspl{IRF}. -\end{quote} - -An {\bf hea-event-bundle} might for example consist of an {\bf hea-event-list} and the associated {\bf response-function}s used to calibrate the dataset, and may also contain provenance information, data quality time-series, and preview images or plots. - - +These terms have been declared in coordination with the IVOA Semantics working group in a specialized IVOA data product vocabulary dedicated to response functions and available at \url{https://www.ivoa.net/rdf/response-type/}. \subsubsection{Advanced Data Products} @@ -439,35 +447,10 @@ \subsubsection{Advanced Data Products} {\bf region}: A dataset that includes an encoding of (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary. \end{quote} +%mireille proposal for vocabularies +\input{Dataproducts-Summary-table} -%\newpage -\subsubsection{Summary Table} - -The proposed vocabulary entries are listed in Table~\ref{tab:dp_vocabulary} with their labels and parents identified. - -\begin{landscape} -\begin{longtable}{p{0.125\linewidth}p{0.175\linewidth}p{0.475\linewidth}p{0.175\linewidth}} -\sptablerule -\textbf{Term} & \textbf{Label} & \textbf{Description} &\textbf{Parent}\cr -\sptablerule -{\bf aeff} & Effective Area & A dataset that records the ``effective area'' of a telescope and/or instrument. The effective area is the geometric area of the telescope and/or instrument reduced by efficiency factors such as reflectivity and vignetting, among other effects & \#response-function \cr -{\bf arf} &\raggedright Ancillary Response File & A dataset that records the combined telescope/instrument effective area and detector quantum efficiency as a function of energy & \#response-function \cr -{\bf bkgrate} & Background Rate & A dataset that models the rate of residual events that are not from the expected source type ({\em e.g.\/}, for gamma-ray instrument {\bf bkgrate} measures residual non-gamma-ray events coming from charged cosmic rays) & \#response-function \cr -{\bf draws} & Draws & A dataset that records statistical draws computed from a probability distribution, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable & \#measurements \cr -{\bf edisp} & Energy Dispersion & A dataset that records the probability density of detecting an event with an energy estimator (proxy) given the true energy of the event & \#response-function, \#pdf \cr -{\bf hea-event-bundle} & Event Bundle & A compounded dataset containing containing an {\bf hea-event-list} and multiple files or other substructures that are products necessary to analyze the {\bf hea-event-list} & \cr -{\bf hea-event-list} & Event list & A dataset that records a collection of observed particle-detection events, such as incoming high-energy particles, where an event is typically characterized by a spatial position, a time, and a spectral value ({\em e.g.\/}, an energy, a channel, a pulse height) & \#temporally-resolved-dataset \cr -{\bf pdf} &\raggedright Probability Density Function & A dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable & \#measurements \cr -{\bf psf} &\raggedright Point Spread Function & A dataset that records the probability density function of spatial/angular spreading of incident particles from a point source caused by the instrument (detector and/or mirror and/or analysis) & \#response-function, \#pdf \cr -{\bf region} & Region & A dataset that encodes (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary & \#measurements \cr -{\bf response-function} & Response Function & A dataset that maps a physical quantity to an observable quantitiy. This term is mainly intended for retrieval. To annotate datasets, use a narrower term Narrower terms are preferred to indicate more precisely the type of {\bf response-function} & \cr -{\bf rmf} &\raggedright Redistribution Matrix File & A dataset that records the probability density function mapping from energy space into detector pulse height (or position) space & \#response-function, \#pdf \cr -\sptablerule -\caption{IVOA Data Product Type Vocabulary Extension} -\label{tab:dp_vocabulary} -\end{longtable} -\end{landscape} - +% \subsubsection{Clarification of ``Flux'' in Data Product Type Vocabulary Definitions} \label{sec:flux} @@ -524,7 +507,7 @@ \subsubsection{Instrument-related Quantities} \paragraph{Event Type} -For \gls{VHE} (and GeV) gamma-ray data, there is the notion of event type (see \S~\ref{sec:evttype}) that can be mandatory for some data releases. We propose to use a combined UCD {\em meta.code.qual;instr.detection;phys.particle\/} to tag the proposed {\bf event\_type} parameter. +For \gls{VHE} (and GeV) gamma-ray data, there is the notion of event type (see \S~\ref{sec:evttype}) that can be mandatory for some data releases. We propose to use a combined UCD \\{\em meta.code.qual;instr.detection;phys.particle\/} \\ to tag the proposed {\bf event\_type} parameter. \subsubsection{Physical Quantities} @@ -551,12 +534,12 @@ \subsubsection{Statistical Parameters} % mireille We should use here the terms that have been recently discussed and approved by the semantics group . I would like the semantics discussions that happened to appear in this note. -The shape of any statistical distribution is an essential quantity for interpreting the meaning of any statistical properties. Too often a Gaussian distribution or a distribution that can be characterized by a simple set of moments ({\em e.g.\/}, mean, variance, skewness, kurtosis) are assumed, but in the extreme Poisson regime common in \gls{HEA} these assumptions are often invalid. We propose adding a UCD {\em stat.distribution\/} to identify a quantity that defines the distribution of a statistical variable such as a likelihood profile. +The shape of any statistical distribution is an essential quantity for interpreting the meaning of any statistical properties. Too often a Gaussian distribution or a distribution that can be characterized by a simple set of moments ({\em e.g.\/}, mean, variance, skewness, kurtosis) are assumed, but in the extreme Poisson regime common in \gls{HEA} these assumptions are often invalid. We propose adding a UCD {\em stat.distribution \/} to identify a quantity that defines the distribution of a statistical variable such as a likelihood profile. % mireille This assumes that a column will contain a full set of data values. Is this ? %In fact it would make the UCD work as a term compatible to a data product type , so ambiguous in terms of role . % in the case we want to mention the kind of distribution this parameter describes , then the UCD for that would be : meta.name;stat.distribution and a new UCD ' stat.distribution' is needed to cover the idea of a statistical distribution. %This would be -%. S& {\em stat.distribution\/} & Related to a statistical distribution \cr as inserted below +%. S& {\em stat.distribution \/} & Related to a statistical distribution \cr as inserted below \subsubsection{Evolution of UCD list} @@ -571,7 +554,7 @@ \subsubsection{Evolution of UCD list} S & {\em em.gamma.uhe\/} & Ultra-High-Energy gamma ray ($>100$ TeV) \cr Q & {\em instr.detection\/} & Particle event detection \cr %Q & {\em instr.event.grade\/} & Particle event grade \cr -Q & {\em instr.pulse\/} & Pulse height amplitude measure \cr +Q & {\em instr.pulse\/} & Pulse height amplitude measure \cr %Q & {\em instr.event.type\/} & Particle event type \cr E & {\em phot.count.density\/} & Count flux density (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}]$) \cr E & {\em phot.count.density.sb\/} & Count flux density surface brightness (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}\,\hbox{sr}^{-1}]$) \cr @@ -591,11 +574,12 @@ \subsubsection{Evolution of UCD list} S & {\em phys.particle.electron\/} & Related to electron \cr S & {\em phys.particle.photon\/} & Related to photon \cr S & {\em phys.particle.positron\/} & Related to positron \cr -% Mireille we cannot have these terms in the UCD tree ; that would imply importing all possible encoding of any kind -S & {\em phys.particle.pdgid\/} & Particle Data Group Identifier \cr +%S & {\em phys.particle.pdgid\/} & Particle Data Group Identifier \cr +P& meta.ref.pdgid &Particle data group identifier encoding a type of particle \cr +% we stated with semantics to use meta.ref.pdgdid;phys.particle for messenger %S & {\em phys.particle.pdgid$\pm$XX\/} & Related to a particle with PDG ID $\pm$XX \cr %mireille -S& {\em stat.distribution\/} & Related to a statistical distribution \cr +S& {\em stat.distribution \/} & Related to a statistical distribution \cr %mireille update to latest discussed term P & {\em stat.error.minus\/} & Negative statistical error \cr P & {\em stat.error.plus\/} & Positive statistical error \cr @@ -633,9 +617,13 @@ \subsection{MIME-type Enhancements}\label{sec:mimetypes} Data files used in the \gls{HEA} domain should have appropriate MIME-types, so that they can be included in ObsCore tables or elsewhere. -Many \gls{HEA} FITS format data products will comply with existing MIME-types discussed in the ObsCore Recommendation, such as {\bf application/fits} or {\bf application/x-fits-bintable}. However, the gamma-ray community has developed two additional data format standards and we propose to add the following MIME-types: +Many \gls{HEA} FITS format data products will comply with existing MIME-types discussed in the ObsCore Recommendation, such as {\bf application/fits} or {\bf application/x-fits-bintable}, but specialized FITS formats have been established for high energy data. + +Many data collections distributed for instance at HEASARC adopted the OGIP format in FITS. +The gamma-ray community has developed two additional data format standards, so we propose to add the following MIME-types: \begin{itemize} +\item {\bf x-fits-OGIP}: OGIP specialized format based on FITS and used in legacy high energy surveys at HEASARC for instance \citep{ogip_Fits_event}. \item {\bf x-fits-gadf}: for FITS files following the Gamma-Astro-Data-Format (GADF) specification \citep{deil_2022_7304668}, \item {\bf x-fits-vodf}: for FITS files following the Very-high-energy Open Data Format (VODF) specification \citep{2023arXiv230813385K}. \end{itemize} @@ -682,6 +670,9 @@ \section{Proposed ivoa.obscore\_hea Table Attributes}\label{sec:ibscoreext} \end{center} \end{landscape} +\input{extendedAccessTonewtypesOfproducts.tex} + + \pagebreak %\printglossaries diff --git a/Makefile b/Makefile index dcae619..a4deef9 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DOCNAME = HighEnergyObsCoreExt DOCVERSION = 1.0 # Publication date, ISO format; update manually for "releases" -DOCDATE = 2026-05-20 +DOCDATE = 2026-06-24 # What is it you're writing: NOTE, WD, PR, REC, PEN, or EN DOCTYPE = PEN diff --git a/Obscore+datalink.png b/Obscore+datalink.png new file mode 100644 index 0000000..6c434eb Binary files /dev/null and b/Obscore+datalink.png differ diff --git a/extendedAccessTonewtypesOfproducts.tex b/extendedAccessTonewtypesOfproducts.tex new file mode 100644 index 0000000..48f9593 --- /dev/null +++ b/extendedAccessTonewtypesOfproducts.tex @@ -0,0 +1,300 @@ + +%\newpage +\newcommand{\blinks}{\{links\}} +% François Version 2: +\section{Extending access to new types of data products in ObsTAP services} +%\section{Access methods for data related to sky datasets discovered with ObsTAP} +%(Various implementation strategies for data access) + +\subsection{Difference between hea-event-list, hea-event-bundle, response function and analysis data products} +As discussed in the previous section, high energy data distribution requires a special focus on event lists and associated response functions as well as analysis data products. +Event lists without corresponding response functions do not allow data interpretation. That’s the reason why hea-event-lists are often gathered in the same package with the response function in an “hea-event-bundle” with a specific format (OGIP, GADF or VODF). In addition, the High Energy astronomical community has developed a lot of probabilistic analysis methods to analyse and extract information from the event lists. They cannot all be described by ObsCore parameters due to the richness and complexity of their parameters. They are listed in section~\ref{sec:voc_product_type}. Access to this category of data can be managed similarly to response functions in some cases, as proposed in section \ref{sec:analysis-dp}. + +\subsection{Direct access to an hea-event-bundle} + The simplest method to access hea-event-list and associated response function is to provide a link to an hea-event-bundle package; this can be done by providing an URL to the package in the {\em access\_url \/} FIELD of the ObsCore (or ObsCore + extension) table. The {\em access\_format \/} FIELD will contain the appropriate MIME type for the bundle package (for example x-fits-ogip, x-fits-gadf, or x-fits-vodf). Table~\ref{tab:bundle} shows an excerpt of an ObsCore result and illustrates this method. This method can be combined with methods described below in~\ref{sec:datalink} for a common access to the bundle and other resources such as previews or some specific analysis data products. + + +%\begin{landscape} +\begin{longtable}{|p{4.1cm}|p{4.4cm}|p{4.4cm}|} +\sptablerule +\textbf{\em Column name\/}&\textbf{\em Row 1 value\/}&\textbf{\em Row 2 value\/}\\ +\sptablerule +\sptablerule + \textbf{dataproduct\_type}& hea-event-list&hea-event-list\\ +\sptablerule + \textbf{dataproduct\_subtype}& GADF DL3&GADF DL3\\ +\sptablerule +\textbf{calib\_level}& 2&2\\ +\sptablerule + \textbf{obs\_collection}& HESS-DL3-DR1&HESS-DL3-DR1\\ +\sptablerule + \textbf{obs\_id}&20136&20317\\ +\sptablerule + \textbf{obs\_title}& & \\ +\sptablerule + \textbf{obs\_publisher\_did}&ivo:\/\/padc.obspm\/hess\#20136&ivo:\/\/padc.obspm\/hess\#20137\\ +\sptablerule + \textbf{obs\_creator\_did}&&\\ +\sptablerule +\textbf{access\_url}& \footnotesize{https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz}&\footnotesize{https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020137.fits.gz}\\ +\sptablerule + \textbf{access\_format}&x-fits-gadf&x-fits-gadf\\ +\sptablerule +\textbf{access\_estsize}&414720&216000\\ +\sptablerule + \textbf{target\_name}&MSH15-52&MSH15-52\\ +\sptablerule + \textbf{target\_class} &&\\ +\sptablerule + \textbf{s\_ra}&228.6125&228.6125\\ +\sptablerule + \textbf{s\_dec}&-58.771667&-59.77771677\\ +\sptablerule +\textbf{s\_fov}&5.0&5.0\\ +%\sptablerule \newpage +\sptablerule + \textbf{s\_region}&&\\ +\sptablerule + \textbf{s\_resolution}&0.10000000149011612&0.10000000149011612\\ +\sptablerule + \textbf{t\_min}&53090.123451203704&53090.144735925925\\ +\sptablerule + \textbf{t\_max}&53090.14291879629&53090.155175740736\\ + \sptablerule + \textbf{t\_exptime}&1521.02685546875&819.2053833007812\\ +\sptablerule +\textbf{t\_resolution}&9.999999974752427E-7&9.999999974752427E-7\\ +\sptablerule +\textbf {em\_min}&1.1889202653543914E-20&1.208503847135941E-20\\ +\sptablerule + \textbf{em\_max}&5.5549210638422004E-18&5.2809212082467665E-18\\ +\sptablerule +\textbf {em\_res\_power}&&\\ +\sptablerule + \textbf{o\_ucd}&time;pos;phys.energy&time;pos;phys.energy\\ +\sptablerule + \textbf{pol\_states}& &\\ +\sptablerule + \textbf{facility\_name}&H.E.S.S.&H.E.S.S\\ +\sptablerule + \textbf{instrument\_name}& 1,2,3,4&1,2,3,4\\ +\sptablerule + \textbf{s\_xel1}&-1&-1\\ +\sptablerule + \textbf{s\_xel2}& -1&-1\\ +\sptablerule + \textbf{t\_xel}&-1&-1\\ +\sptablerule + \textbf{em\_xel}& -1&-1\\ +\sptablerule +\textbf{pol\_xel}& -1&-1\\ +\sptablerule +\caption{ObsCore response for hea-event-bundles (inspired from the H.E.S.S. ObsTAP prototype at Paris Observatory)} +\label{tab:bundle} +\end{longtable} +%\end{landscape} + +\subsection{Access via DataLink} +\label{sec:datalink} + +DataLink \citep{2023ivoa.spec.1215B} allows to relate a main record provided by a service (ObsTAP, SSA/SIA or a catalogue service) to a various number of links which complement the information contained in the record. An implementation note \citep{2024ivoa.note.datalink} gives some hints on how to use this standard. +There are at least three ways to implement this standard for hea-event-lists depending how many steps are used in the access method to the associated datasets.\\ + \begin{itemize} + \item Subsection~\ref{sec:datalinkinobs} is a full 2-step approach, + \item Subsection~\ref{sec:datalinksd} allows both direct access to hea-event-list and 2-step access to the additional material, + \item Subsection~\ref{sec:datalinktap} is not described in the DataLink standard but is fully consistant with TAP and the DataLink \blinks\ endpoint response specification. It allows direct access to both hea-event-list and additional material, like response files, or analysis data products. + \end{itemize} + + \subsubsection{From an hea-event-list in ObsCore: DataLink to access the event list itself and response functions via {\em access\_url \/} } +\label{sec:datalinkinobs} + The { \em data\_product type \/} exposed in ObsCore is “hea-event-list” and the DataLink \blinks\ endpoint response points to the event list itself and to the various response functions individually. This solution allows retrieval of data with explicit access to the various components of a “bundle” which doesn't need to be packaged in that case. In addition to response functions, analysis data products can also be linked to the event list this way. Figure~\ref{fig:obscoretodl} shows an ObsCore response where the selected record points to a \blinks\ endpoint response illustrating this method. Table~\ref{tab:datalink} shows this DataLink response content. +\begin{figure} + +\includegraphics[width=1.5\textwidth, angle=90]{Obscore+datalink.png} +\caption{Selection of H.E.S.S. data from Aladin Desktop: In the galactic center area, an ObsTAP query has been performed and H.E.S.S. datasets are discovered. One of the ObsCore record has been selected and points to a DataLink response, with 8 links provided in the white box. The content of this DataLink table is described in Table~\ref{tab:datalink}. } +\label{fig:obscoretodl} +\end{figure} + +\begin{landscape} +\begin{center} +\begin{longtable}{|p{0.08\linewidth}|p{0.3\linewidth}|p{0.08\linewidth}|p{0.22\linewidth}|p{0.1\linewidth}|p{0.14\linewidth}|p{0.13\linewidth}|} +\hline%\sptablerule +\textbf{ID} &\textbf{\footnotesize access\_url} &\textbf{\footnotesize semantics}&\textbf{\footnotesize description} &\textbf{\footnotesize content\_type} &\textbf{\footnotesize content\_qualifier}\cr +\hline%\sptablerule +{\footnotesize 20136} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz\#EVENTS} & {\footnotesize \#this} & {\footnotesize Event list dataset (HDU=EVENTS) } & {\footnotesize x-fits-gadf} & {\footnotesize hea-event-list} \cr +\hline%\sptablerule +{\footnotesize 20136} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136\_image.png} & {\footnotesize \#preview-image} & {\footnotesize Preview image of the observation} & {\footnotesize image/png} & {\footnotesize preview} \cr +\hline%\sptablerule +{\footnotesize 20136} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136\_plot.png} & {\footnotesize \#preview-plot} & {\footnotesize Preview significance distribution of the observation } & {\footnotesize image/png} & {\footnotesize preview} \cr +\hline%\sptablerule +{\footnotesize 20136} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz\#AEFF } & {\footnotesize \#calibration} & {\footnotesize Effective Area (HDU=AEFF)} & {\footnotesize x-fits-gadf} & {\footnotesize aeff} \cr +\hline%\sptablerule +{\footnotesize 20316} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz\#EDISP} & {\footnotesize \#calibration} & {\footnotesize Energy Dispersion (HDU=EDISP)} & {\footnotesize x-fits-gadf} & {\footnotesize edisp} \cr +\hline%\sptablerule +{\footnotesize 20316} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz\#PSF} & {\footnotesize \#calibration} & {\footnotesize Point Spread Function (HDU=PSF) } & {\footnotesize x-fits-gadf} & {\footnotesize psf} \cr +\hline%\sptablerule +{\footnotesize 20316} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz\#BKG} & {\footnotesize \#calibration} & {\footnotesize Background Rate (HDU=BKG) } & {\footnotesize x-fits-gadf} & {\footnotesize bkgrate} \cr +\hline%\sptablerule +{\footnotesize 20316} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz} & {\footnotesize \#package} & {\footnotesize Event bundle (including HDU=EVENTS,GTI,AEFF,\newline EDISP,PSF,BKG) } & {\footnotesize x-fits-gadf} & {\footnotesize hea-event-bundle } \cr +\hline%\sptablerule + +\caption{DataLink response table attached to an hea-event-list record in ObsCore. Inspired from Paris Observatory H.E.S.S. ObsTAP prototype} +\label{tab:datalink} +\end{longtable} +\end{center} +\end{landscape} + + \subsubsection{Datalink access using a service descriptor} +\label{sec:datalinksd} + To avoid preventing a direct access to the hea-event-list while keeping the explicit access to the various response functions, it is possible to add a “DataLink” service descriptor in the ObsTAP result as can be seen in the example below. + +\begin{verbatim} + +Datalink service + + + + + + +\end{verbatim} + + In this case the service descriptor provides the root URL to the DataLink \blinks\ endpoint. The full \blinks\ endpoint URL for a specific row (or group of rows) is then built by adding the content of the referred ID FIELD this way: \\ http://provider.org/dataservice/datalink?ID={\it referred\_ID\_FIELD\_content}. + It is important to notice that it is not mandatory that the {\it referred\_ID\_FIELD} be {\em obs\_publisher\_did/}. It can be {\em obs\_id}, or any free identifier FIELD allowing to group several rows together. For example, referring to {\em obs\_id} will allow all the datasets belonging to the same observation to be bound to the same \blinks\ endpoint response. + + This method is implemented in many archive services such as ESO, CADC, GAVO, HEASARC and even VizieR. + + \subsubsection{Datalink table in a TAP service / join with ObsCore table} + \label{sec:datalinktap} + +The \blinks\ endpoint table can also be implemented as a TAP table of its own. This table will contain columns describing all the links (hea-event-list, response functions, analysis data products) desirable for any dataset record in the \texttt{ivoa.obscore} table of ObsTAP service. +A direct access to the ObsCore record and the links provided by the \blinks\ endpoint response table is then possible. An identifier column would be common in the ObsCore table and the DataLink table and used as a keyreference. Suppose we have {\em dataproduct\_type} = ‘hea-event-list’ in the \texttt{ivoa.obscore} table, and we name this DataLink table “datalink.response”, we can then query the TAP service this way in order to access directly to the hea-event-list product: + +\begin{verbatim} +SELECT * FROM ivoa.obscore + NATURAL JOIN ivoa.obscore_hea + JOIN datalink.response ON + ivoa.obscore.obs_publisher_did = datalink.response.ID + WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 6) + AND (dataproduct_type = ’hea-event-list’ ) + AND ( semantics = '#this') +\end{verbatim} +While to access directly the psf we can select this way : +\begin{verbatim} +SELECT * FROM ivoa.obscore + NATURAL JOIN ivoa.obscore_hea + JOIN datalink.response ON + ivoa.obscore.obs_publisher_did = datalink.response.ID + WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 6) + AND (dataproduct_type = ’hea-event-list’ ) + AND ( semantics = '#calibration') + AND (content_qualifier = 'psf') +\end{verbatim} +and to access directly a background image : +\begin{verbatim} +SELECT * FROM ivoa.obscore + NATURAL JOIN ivoa.obscore_hea + JOIN datalink.response ON + ivoa.obscore.obs_publisher_did = datalink.response.ID + WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 6) + AND (dataproduct_type = ’hea-event-list’ ) + AND ( semantics = '#auxiliary') + AND (content_qualifier = 'bkgimage') +\end{verbatim} +One caveat: if response functions and other auxiliary or analysis data are common to a full observation (instead of dataset) or even worse to several observations, then the DataLink-like table will repeat the information several times. To avoid that it's possible to create an additional index column (for example {\em dl\_association\_id \/} or whatever appropriate term) and modify the first query this way (for others it would be similar of course) : +\begin{verbatim} +SELECT * FROM ivoa.obscore + NATURAL JOIN ivoa.obscore_hea + JOIN datalink.response ON ivoa.obscore.response_id = datalink.response.ID + WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 6) + AND (dataproduct_type = ’hea-event-list’ ) + AND ( semantics = '#this') +\end{verbatim} + +\subsection{Accessing Response datasets via a specific table joined with the \texttt{ivoa.obscore} table} + +For the discovery of response data products, the use cases recorded show that selection criteria are based on 1) the type of response 2) the observed data set properties, and on the observation, like the observing position and field of view, the observing date and time, the energy and time bounds. However, the cardinality of the relationship between a response dataset and an hea-event-list dataset varies according to projects and may be 1 to 1, like in CTAO project, or 1 to N, N to 1 or N to N. + +Compared to the \texttt{ivoa.obscore} table, it can happen that less attributes are needed to select a response file. The response details could be into the response files, in the metadata encoded according to the file format (OGIP, GADF, VODF, etc). In this case, response data products can be described by a specific response table, with a short list of attributes as proposed in Tab. \ref{tab:response_table}. + +In order to handle the various cardinality between response and observation datasets, the foreign keys are defined in the response table and will allow JOIN operations between the \texttt{ivoa.obscore} and \texttt{ivoa.response} table. + +\section*{Response table proposal} +%\TODO{ check fields for this new table } +%\begin{table}[htbp] +%\begin{center} +\begin{longtable}{|p{0.25\linewidth}|p{0.12\linewidth}|p{0.1\linewidth}|p{0.39\linewidth}|} +\hline +Column Name & Unit & Type & Description\\\hline +%obs\_collection & unitless & String & Name of the data collection \\\hline +% to bind a response with an observation data set +obs\_id & unitless & String & Related Observation ID (foreign key) \\\hline +obs\_publisher\_did & unitless & String & Related dataset ID (foreign key) \\\hline +% copied from related observation if needed +target\_name & unitless & String & Astronomical object observed, if any\\\hline +facility\_name & unitless & String & Facility Name used for observed data products \\\hline +instrument\_name & unitless & String & Instrument Name used for observed data products \\\hline +% response specific fields +resp\_product\_type & unitless & String & Product type as defined in response-type vocabulary\\\hline +resp\_publisher\_did & unitless & String & Dataset identifier given by the publisher\\\hline +resp\_access\_url & unitless & String & URL used to access (download) response dataset\\\hline +resp\_access\_format & unitless & String & File content format (see in ObsCore MIME Types)\\\hline +% response spatial coverage +resp\_s\_ra & deg & double & ICRS Central right ascension, for the region where the response-product applies \\\hline +resp\_s\_dec & deg & double & ICRS Central declination, for the region where the response-product applies\\\hline +resp\_s\_region & unitless & String & Region of interest for response use (expressed in ICRS frame)\\\hline +resp\_t\_intervals & unitless& String (TMOC) & time coverage for response use \\\hline +%response energy coverage +resp\_energy\_min & m & double & Energy band minimal value for response use \\\hline +resp\_energy\_max & m & double & Energy band maximal value for response use \\\hline + +\caption{Possible mandatory fields of the Response table proposed in this HEIG ObsCore extension serialisation} +\label{tab:response_table} +\end{longtable} +%\end{center} +%\end{table} + +Implementing this extra response table will allow to write queries like: \\ + +\noindent Find all datasets satisfying: +\begin{enumerate}[(i)] + \item Position inside 3 arcmin from (83.84358, $-5.43639$) + \item Response product\_type = ``psf'' + \item obs\_id = ``1527'' + \item obs\_collection = ``HESS'' +\end{enumerate} + +When using the response table defined in Table \ref{tab:response_table}, +we can join the \texttt{ivoa.obscore} table with the \texttt{ivoa.response} table on +the two keys {\em obs\_id \/} and {\em obs\_collection \/} for instance, as in: +\begin{verbatim} +SELECT obs_publisher_did, s_ra, s_dec, s_fov, t_min, t_max, +energy_min, energy_max, access_url, access_format, +resp_publisher_did, resp_access_url, resp_access_format, +resp_energy_max, resp_energy_min +FROM ivoa.obscore +NATURAL JOIN ivoa.response +WHERE +(target_name = ’Crab’ OR target_name = ’M1’ OR +CONTAINS(POINT(s_ra, s_dec), CIRCLE, 83.6324, +22.0174, 0.083333) = 1) +AND (resp_product_type = 'psf') +AND (obs_id = '1527') +AND (obs_collection = 'HESS') +\end{verbatim} + +If a time constraint is needed, then we can use {\em resp\_t\_min \/}, and {\em resp\_t\_max \/} and constrain the interval like: +\begin{verbatim} +AND (resp_t_min > 56000.0 and resp_t_max < 56001.5) +\end{verbatim} + +\section*{Querying for Analysis Data products} +\label{sec:analysis-dp} + +The same strategy can apply for searching analysis data products for one observation or for a hea-event-list data product, using a dedicated \texttt{ivoa.obscore.adp} table . +The benefice is to describe more specific properties of these data products in the optional columns, without overloading the main \texttt{ivoa.obscore} table. + +