feat(server): add a storage-aware GET /readiness endpoint

hugegraph-server/README.md

@@ -33,6 +33,40 @@ Memory remains a test-only backend throughout. Historical backend users must ope
 maintain a compatible release; these implementations are not restored to the current source
 tree or distribution packages.
 
+## Readiness endpoint
+
+`GET /readiness` answers `200` while this Server can serve graph traffic and `503` otherwise. It is
+unauthenticated and needs no graphspace prefix, so a Kubernetes `httpGet` probe can call it as is. The
+JSON body carries `ready`, `storage` (`embedded`, `hstore`, `hbase`, or a comma-separated list), a
+`reason`, whether the answer was `cached`, and one `probes` entry per probed backend configuration
+(backend, `ready`, `reason`); it carries no addresses and no graph names.
+
+What ready means:
+
+- `embedded`: no graph on a remote storage; ready once the REST layer is up.
+- `hstore`: this Server knows a Store list and at least one Store answers a direct status ping
+  (`HgStoreState.getScanState`). A Store whose status answers while its raft or partition path is
+  broken is not detected.
+- `hbase`: every table of the graph (schema store, graph store, system store) exists, is enabled and
+  is available.
+- A configured graph that failed to load at startup makes the Server not ready whatever its backend
+  (`reason` says how many, `failed_graphs` carries the count, `failed_backends` the backends of the
+  failed configs, and `storage` includes them). The gate holds until the Server restarts or a graph of
+  that name is created through the graphs API (which replaces the failed load).
+
+Graphs that share a backend configuration (the same `pd.peers`; for hbase every graph is its own
+scope, since the table namespace is derived from the graph name) share one probe; independent
+configurations are probed side by side within one time budget and the Server is ready only when all
+of them are.
+
+Options in `rest-server.properties`:
+
+| option | default | meaning |
+|---|---|---|
+| `readiness.timeout` | `1000` | time budget of one probe in ms; a backend that does not answer within it reads as not ready |
+| `readiness.cache_ttl` | `2000` | a probe result is reused for this many ms |
+| `readiness.max_waiters` | `16` | callers that may wait for the probe in flight; beyond it a caller gets `503` at once |
+
 ## Docker
 
 ### Standalone Mode

hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AuthenticationFilter.java

@@ -81,6 +81,7 @@ public class AuthenticationFilter implements ContainerRequestFilter, ContainerRe
     // https://github.com/apache/hugegraph/issues/3212
     private static final Set<String> FIXED_WHITE_API_SET = ImmutableSet.of(
             "versions",
+            "readiness",
             "openapi.json"
     );
     /** Remove auth/login API from whitelist */

hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/LoadDetectFilter.java

@@ -53,7 +53,8 @@ public class LoadDetectFilter implements ContainerRequestFilter {
             "",
             "apis",
             "metrics",
-            "versions"
+            "versions",
+            "readiness"
     );
 
     // Call gc every 30+ seconds if memory is low and request frequently

hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java

@@ -57,6 +57,7 @@ public class PathFilter implements ContainerRequestFilter {
             "apis",
             "metrics",
             "versions",
+            "readiness",
             "health",
             "gremlin",
             "graphs/auth",

hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/profile/ReadinessAPI.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.api.profile;
+
+import java.util.Map;
+
+import org.apache.hugegraph.api.API;
+import org.apache.hugegraph.config.HugeConfig;
+import org.apache.hugegraph.config.ServerOptions;
+import org.apache.hugegraph.core.GraphManager;
+import org.apache.hugegraph.util.JsonUtil;
+
+import com.codahale.metrics.annotation.Timed;
+
+import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.annotation.security.PermitAll;
+import jakarta.inject.Singleton;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.Response;
+
+/**
+ * Storage-aware readiness for Kubernetes and load balancers: 503 while a
+ * configured graph failed to load; on hstore 200 while at least one known
+ * Store answers this server, 503 while none does (or, before any Store list
+ * is known, while PD does not answer); on hbase 200 while every table of
+ * every store of the graph is enabled and available within the budget.
+ * Unauthenticated, like /versions, so that an httpGet probe needs no
+ * credential; the body carries no addresses, graph names or raw exception
+ * text. The contract and the options are described in the server README.
+ */
+@Path("readiness")
+@Singleton
+@Tag(name = "ReadinessAPI")
+public class ReadinessAPI extends API {
+
+    @GET
+    @Timed
+    @Produces(APPLICATION_JSON_WITH_CHARSET)
+    @PermitAll
+    public Response get(@Context GraphManager manager, @Context HugeConfig conf) {
+        Map<String, Object> body = StorageReadiness.check(
+                manager, conf.get(ServerOptions.READINESS_TIMEOUT),
+                conf.get(ServerOptions.READINESS_CACHE_TTL),
+                conf.get(ServerOptions.READINESS_MAX_WAITERS));
+        Response.Status status = StorageReadiness.isReady(body) ?
+                                 Response.Status.OK :
+                                 Response.Status.SERVICE_UNAVAILABLE;
+        return Response.status(status)
+                       .type(APPLICATION_JSON_WITH_CHARSET)
+                       .entity(JsonUtil.toJson(body))
+                       .build();
+    }
+}