feat(proxy): add the service to the proxy metrics

pkg/metrics/metrics.go

@@ -1,6 +1,7 @@
 package metrics
 
 import (
+	"errors"
 	"fmt"
 	"reflect"
 	"strings"
@@ -48,6 +49,21 @@ func describe(metric prometheus.Collector, initialize func() error) (string, err
 	return fams[0].GetName(), nil
 }
 
+func postProcess(c prometheus.Collector) error {
+	// special post-treatment for the BuildInfo metric, as we have that one pretty much
+	// everywhere: set its value with the current version so we don't need to do that every time
+	switch buildInfo := c.(type) {
+	case BuildInfoMetric:
+		if name, err := describe(buildInfo, func() error { buildInfo.WithLabelValues("0").Set(0.0); return nil }); err != nil {
+			return err
+		} else if strings.HasSuffix(name, "_build_info") {
+			buildInfo.Reset()
+			buildInfo.WithLabelValues(version.GetString()).Set(1)
+		}
+	}
+	return nil
+}
+
 // Take a struct that contains metrics as attributes and register all of them
 // with the specified Registerer.
 func RegisterAll(registerer prometheus.Registerer, m any, logger *log.Logger) error {
@@ -85,17 +101,8 @@ func RegisterAll(registerer prometheus.Registerer, m any, logger *log.Logger) er
 				}
 			} else {
 				succeeded = append(succeeded, n)
-
-				// special post-treatment for the BuildInfo metric, as we have that one pretty much
-				// everywhere: set its value with the current version so we don't need to do that every time
-				switch buildInfo := c.(type) {
-				case BuildInfoMetric:
-					if name, err := describe(buildInfo, func() error { buildInfo.WithLabelValues("0").Set(0.0); return nil }); err != nil {
-						failed[n] = err
-					} else if strings.HasSuffix(name, "_build_info") {
-						buildInfo.Reset()
-						buildInfo.WithLabelValues(version.GetString()).Set(1)
-					}
+				if err := postProcess(c); err != nil {
+					failed[n] = err
 				}
 			}
 		case *prometheus.Desc,
@@ -135,9 +142,18 @@ func Register[M any](reg prometheus.Registerer, m M, logger *log.Logger) (M, err
 	return m, err
 }
 
-// Register a single metric.
-func RegisterMetric[M prometheus.Collector](reg prometheus.Registerer, m M, logger *log.Logger) error {
-	return NewLoggingPrometheusRegisterer(reg, logger).Register(m)
+// Register individual metrics.
+func RegisterMetrics(reg prometheus.Registerer, logger *log.Logger, metrics ...prometheus.Collector) error {
+	lreg := NewLoggingPrometheusRegisterer(reg, logger)
+	errs := []error{}
+	for _, c := range metrics {
+		if err := lreg.Register(c); err != nil {
+			errs = append(errs, err)
+		} else {
+			errs = append(errs, postProcess(c))
+		}
+	}
+	return errors.Join(errs...)
 }
 
 // Prometheus Registerer wrapper that logs every error that occurs when registering

services/graph/pkg/metrics/metrics.go

@@ -118,7 +118,9 @@ func New(registerer prometheus.Registerer, logger *log.Logger, httpPathSplitter
 
 	m, err := ocmetrics.Register(registerer, m, logger)
 	// must additionally register unexported metrics:
-	err = errors.Join(err, ocmetrics.RegisterMetric(registerer, m.httpRequestDuration, logger))
+	err = errors.Join(err, ocmetrics.RegisterMetrics(registerer, logger,
+		m.httpRequestDuration,
+	))
 	return m, err
 }
 

services/proxy/README.md

@@ -300,12 +300,17 @@ In this mode, the proxy service only exposes its own metrics. The metrics of the
 ### Available Metrics
 The following metrics are exposed by the proxy service:
 
-| Metric Name                      | Description                                                                                                                                                                                                                   | Labels                                |
-|----------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------|
-| `opencloud_proxy_requests_total`      | [Counter](https://prometheus.io/docs/tutorials/understanding_metric_types/#counter) metric which reports the total number of HTTP requests.                                                                                   | `method`: HTTP method of the request  |
-| `opencloud_proxy_errors_total`        | [Counter](https://prometheus.io/docs/tutorials/understanding_metric_types/#counter) metric which reports the total number of HTTP requests which have failed. That counts all response codes >= 500                           | `method`: HTTP method of the request  |
-| `opencloud_proxy_duration_seconds`    | [Histogram](https://prometheus.io/docs/tutorials/understanding_metric_types/#histogram) of the time (in seconds) each request took. A histogram metric uses buckets to count the number of events that fall into each bucket. | `method`: HTTP method of the request  |
-| `opencloud_proxy_build_info{version}` | A metric with a constant `1` value labeled by version, exposing the version of the OpenCloud proxy service.                                                                                                                        | `version`: Build version of the proxy |
+| Name | Labels | Description |
+| ---- | ------ | ----------- |
+| `opencloud_proxy_requests_total` | • `method`: HTTP method of the request | [Counter](https://prometheus.io/docs/tutorials/understanding_metric_types/#counter) metric which reports the total number of HTTP requests |
+| `opencloud_proxy_errors_total` | • `method`: HTTP method of the request | [Counter](https://prometheus.io/docs/tutorials/understanding_metric_types/#counter) metric which reports the total number of HTTP requests which have failed. That counts all response codes >= 500. |
+| `opencloud_proxy_duration_seconds` | • `method`: HTTP method of the request | [Histogram](https://prometheus.io/docs/tutorials/understanding_metric_types/#histogram) of the time (in seconds) each request took. A histogram metric uses buckets to count the number of events that fall into each bucket |
+| `opencloud_proxy_concurrent_service_requests` | • `service`: identifier of the service the request is proxied to | Counts the number of in-flight requests that are being processed at a given time |
+| `opencloud_proxy_routing_failure_count` | | Counts the number of inbound requests that cannot be proxied due to a failure of determining how to route it |
+| `opencloud_proxy_duration_seconds` | • `service`: identifier of the service the request is proxied to | Classic histogram that measures the duration of proxied HTTP requests, per service |
+| `opencloud_proxy_request_total` | • `method`: the HTTP method<br>• `result`: one of `success` (<=299), `client-error` (<= 499), `server-error` (>= 500), depending on the status code in the response of the proxied HTTP request<br>• `services`: identifier of the service the request is proxied to | Counts the number of proxied requests |
+| `opencloud_proxy_request_duration_seconds_bucket` | • `method`: the HTTP method<br>• `result`: one of `success`, `client-error`, `server-error`, depending on the status code in the response of the proxied HTTP request<br>• `services`: identifier of the service the request is proxied to | Native histogram that measures the duration of proxied HTTP requests, per service |
+| `opencloud_proxy_build_info` | • `version`: build version of the proxy | A gauge with a constant value of `1` |
 
 ### Prometheus Configuration
 The following is an example prometheus configuration for the single process mode. It assumes that the proxy debug address is configured to bind on all interfaces `PROXY_DEBUG_ADDR=0.0.0.0:9205` and that the proxy is available via the `opencloud` service name (typically in docker-compose). The prometheus service detects the `/metrics` endpoint automatically and scrapes it every 15 seconds.
@@ -318,3 +323,17 @@ scrape_configs:
     static_configs:
     - targets: ["opencloud:9205"]
 ```
+
+In order to process native histograms, use this configuration instead:
+
+```yaml
+global:
+  scrape_interval: 15s
+  scrape_native_histograms: true
+  scrape_protocols: ['PrometheusProto', 'OpenMetricsText1.0.0']
+scrape_configs:
+  - job_name: opencloud_proxy
+    static_configs:
+    - targets: ["opencloud:9205"]
+```
+

services/proxy/pkg/command/server.go

@@ -26,7 +26,6 @@ import (
 	"github.com/opencloud-eu/opencloud/pkg/runner"
 	"github.com/opencloud-eu/opencloud/pkg/service/grpc"
 	"github.com/opencloud-eu/opencloud/pkg/tracing"
-	"github.com/opencloud-eu/opencloud/pkg/version"
 	policiessvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/policies/v0"
 	settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
 	"github.com/opencloud-eu/opencloud/services/proxy/pkg/config"
@@ -123,8 +122,20 @@ func Server(cfg *config.Config) *cobra.Command {
 				defer cancel()
 			}
 
-			m := metrics.New()
-			m.BuildInfo.WithLabelValues(version.GetString()).Set(1)
+			m, err := metrics.New(func(yield func(config.Route) bool) {
+				// provide the metrics with an iterator that gives a list of the routes,
+				// to allow the metrics to initialize empty collectors accordingly
+				for _, pol := range cfg.Policies {
+					for _, r := range pol.Routes {
+						if !yield(r) {
+							return
+						}
+					}
+				}
+			}, &logger)
+			if err != nil {
+				return fmt.Errorf("failed to initialize metrics in reverse proxy: %w", err)
+			}
 
 			rp, err := proxy.NewMultiHostReverseProxy(
 				proxy.Logger(logger),
@@ -202,7 +213,7 @@ func Server(cfg *config.Config) *cobra.Command {
 					proxyHTTP.Logger(logger),
 					proxyHTTP.Context(cfg.Context),
 					proxyHTTP.Config(cfg),
-					proxyHTTP.Metrics(metrics.New()),
+					proxyHTTP.Metrics(m),
 					proxyHTTP.Middlewares(middlewares),
 				)
 				if err != nil {
@@ -355,7 +366,6 @@ func loadMiddlewares(logger log.Logger, cfg *config.Config,
 		),
 		middleware.Tracer(traceProvider),
 		pkgmiddleware.TraceContext,
-		middleware.Instrumenter(metrics),
 		middleware.AccessLog(logger),
 		middleware.ContextLogger(logger),
 		middleware.HTTPSRedirect, // redirect to https if enabled
@@ -367,7 +377,8 @@ func loadMiddlewares(logger log.Logger, cfg *config.Config,
 		middleware.Security(cspConfig),
 
 		// 3. Routing & Authentication
-		router.Middleware(serviceSelector, cfg.PolicySelector, cfg.Policies, logger),
+		router.Middleware(serviceSelector, cfg.PolicySelector, cfg.Policies, metrics.RoutingFailed, logger),
+		middleware.Instrumenter(metrics), // must come after the router middleware as it needs to know the routeInfo for detailed metrics
 		middleware.Authentication(
 			authenticators,
 			middleware.CredentialsByUserAgent(cfg.AuthMiddleware.CredentialsByUserAgent),