From 85f4de9778a9d139f3c79f86a0d855da55e0d3a1 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Thu, 15 Oct 2026 05:03:38 +0530 Subject: [PATCH 1/4] fix(server): bind locally loaded graphs in Gremlin Server Gremlin Server opens the graphs in conf/graphs on its own, once, while it is being prepared, and drops any graph whose open fails there. With PD briefly unreachable at that moment, the REST GraphManager still opened the graph a few seconds later, after its PD readiness wait, but never told Gremlin about it: loadGraph() stored the graph without firing GRAPH_CREATE, unlike the create paths. The Server then answered REST and passed readiness while every Gremlin call on that graph failed with "Could not rebind" until the process restarted. loadGraph() now offers each locally loaded graph to Gremlin through the same GRAPH_CREATE event the create paths use. When Gremlin's own open succeeded, both opens return the same cached instance, so the rebind is harmless. reload() also gets its Gremlin binding back this way, since dropping a graph removed it. A graph whose spaceGraphName differs from DEFAULT- (a different store name or a non-DEFAULT graphspace) is skipped with a warning so it is not bound under a second name, and a failed bind is logged without unloading the graph from REST. Fixes #3228 --- helm/hugegraph/docs/operations.md | 2 +- .../apache/hugegraph/core/GraphManager.java | 38 +++- .../apache/hugegraph/unit/UnitTestSuite.java | 2 + .../core/GraphManagerLocalGraphBindTest.java | 184 ++++++++++++++++++ 4 files changed, 218 insertions(+), 8 deletions(-) create mode 100644 hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java diff --git a/helm/hugegraph/docs/operations.md b/helm/hugegraph/docs/operations.md index cb7f2e8048..6ddc90abec 100644 --- a/helm/hugegraph/docs/operations.md +++ b/helm/hugegraph/docs/operations.md @@ -293,7 +293,7 @@ Updating chart values alone does not change an existing image's behavior. | `HG_PD_AUTH_SECRET_KEY` missing | PD Pod Secret reference and key; chart injects the shared PD REST password automatically | | Server storage wait timeout | Ready PD majority, Store registration, network policies and `server.storageWaitTimeoutSeconds` | | NodePort/Ingress unreachable | Assigned port, reachable node/controller address, firewall and NetworkPolicy rules | -| REST works but Gremlin reports `Could not rebind` | Run `helm test`; after PD stabilizes, replace the failing Server Pod and retry | +| REST works but Gremlin reports `Could not rebind` | Run `helm test`. Images that include apache/hugegraph#3228 rebind Gremlin when REST loads the graph; if it persists, check the Server log for `could not be bound in Gremlin Server` or `can't be rebound in Gremlin Server`, then replace the failing Server Pod after PD stabilizes and retry | ```bash kubectl -n hugegraph get pods,pvc,svc diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java index c7d821bf38..9ccb04be35 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java @@ -1811,13 +1811,6 @@ private String defaultSpaceGraphName(String graphName) { } private void loadGraph(String name, String graphConfPath) { - // TODO: offer a loaded graph to Gremlin Server (notify GRAPH_CREATE, as the create paths - // do) or fail startup when its static Gremlin instantiation failed. Today a Server whose - // first open failed against a PD member mid-restart serves REST and passes readiness - // while every Gremlin call on it fails for the life of the process. The Helm chart - // (helm/hugegraph) detects this with a per-Pod Gremlin query in `helm test` and documents - // the manual Pod deletion; retire both once this is fixed. - // https://github.com/apache/hugegraph/issues/3228 HugeConfig config = new HugeConfig(graphConfPath); // Transfer `raft.group_peers` from server config to graph config @@ -1836,6 +1829,8 @@ private void loadGraph(String name, String graphConfPath) { LOG.info("Graph '{}' was successfully configured via '{}'", name, graphConfPath); + this.bindLocalGraph(name, graph); + if (this.requireAuthentication() && !(graph instanceof HugeGraphAuthProxy)) { LOG.warn("You may need to support access control for '{}' with {}", @@ -1843,6 +1838,35 @@ private void loadGraph(String name, String graphConfPath) { } } + /* + * Gremlin Server opens the graphs in conf/graphs once, while it is being + * prepared, and drops any graph whose open fails there (for example when + * PD is briefly unreachable). Offer each locally loaded graph to it again, + * as the create paths do, so such a graph still gets its Gremlin binding. + * Rebinding the graph Gremlin already holds is harmless: both opens return + * the same cached instance. + */ + private void bindLocalGraph(String name, Graph graph) { + if (!(graph instanceof HugeGraph)) { + return; + } + HugeGraph hugegraph = (HugeGraph) graph; + String graphName = defaultSpaceGraphName(name); + if (!graphName.equals(hugegraph.spaceGraphName())) { + LOG.warn("Graph '{}' is named '{}' by its config, so it can't be " + + "rebound in Gremlin Server if Gremlin failed to open it", + graphName, hugegraph.spaceGraphName()); + return; + } + try { + this.notifyEvent(Events.GRAPH_CREATE, hugegraph); + } catch (Throwable e) { + LOG.error("Graph '{}' is served by REST but could not be bound " + + "in Gremlin Server, restart the Server to retry", + graphName, e); + } + } + private void transferPdPeersConfig(HugeConfig config) { if (config.containsKey(CoreOptions.PD_PEERS.name())) { return; diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index 2ab8d4e8fa..6b20b9fec5 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -77,6 +77,7 @@ import org.apache.hugegraph.unit.core.ExceptionTest; import org.apache.hugegraph.unit.core.GraphManagerAdminInitTest; import org.apache.hugegraph.unit.core.GraphManagerConfigTest; +import org.apache.hugegraph.unit.core.GraphManagerLocalGraphBindTest; import org.apache.hugegraph.unit.core.GroovyScriptEngineCompatibilityTest; import org.apache.hugegraph.unit.core.HugeFeaturesTest; import org.apache.hugegraph.unit.core.HstoreSessionsTest; @@ -220,6 +221,7 @@ ExceptionTest.class, GraphManagerAdminInitTest.class, GraphManagerConfigTest.class, + GraphManagerLocalGraphBindTest.class, HstoreSessionsTest.class, HstoreTableRoutingTest.class, BackendStoreInfoTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java new file mode 100644 index 0000000000..cd5b4ea73b --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.core; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.configuration2.PropertiesConfiguration; +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.HugeFactory; +import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.auth.ContextGremlinServer; +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.core.GraphManager; +import org.apache.hugegraph.event.EventHub; +import org.apache.hugegraph.event.EventListener; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.util.Events; +import org.apache.tinkerpop.gremlin.server.Settings; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableMap; + +/** + * Gremlin Server opens conf/graphs on its own, once, and drops a graph whose + * open fails there. The REST load of the same graph has to offer it to + * Gremlin again, or that Server answers REST while every Gremlin call on the + * graph fails until it restarts (#3228). + */ +public class GraphManagerLocalGraphBindTest { + + private File graphsDir; + private EventHub hub; + private GraphManager manager; + private ContextGremlinServer gremlin; + private List created; + private EventListener listener; + + @Before + public void setup() throws IOException { + this.graphsDir = Files.createTempDirectory("local-graph-bind").toFile(); + this.hub = new EventHub("local-graph-bind-test"); + this.manager = new GraphManager( + new HugeConfig(new PropertiesConfiguration()), this.hub); + this.created = new CopyOnWriteArrayList<>(); + this.listener = event -> { + event.checkArgs(HugeGraph.class); + this.created.add(((HugeGraph) event.args()[0]).spaceGraphName()); + return null; + }; + this.hub.listen(Events.GRAPH_CREATE, this.listener); + } + + @After + public void teardown() throws Exception { + List graphs = new ArrayList<>(); + for (String name : this.manager.graphs()) { + graphs.add(this.manager.graph(name)); + } + try { + if (this.gremlin != null) { + this.gremlin.stop().get(30, TimeUnit.SECONDS); + } + this.hub.unlisten(Events.GRAPH_CREATE, this.listener); + this.manager.close(); + } finally { + graphs.forEach(HugeFactory::remove); + FileUtils.deleteQuietly(this.graphsDir); + } + } + + @Test + public void testLocalLoadOffersGraphToGremlin() throws IOException { + String path = this.writeGraphConf("bind_local", "bind_local"); + + this.manager.loadGraphsFromLocal(ImmutableMap.of("bind_local", path)); + + Assert.assertEquals(1, this.created.size()); + Assert.assertEquals("DEFAULT-bind_local", this.created.get(0)); + Assert.assertNotNull(this.manager.graph("DEFAULT", "bind_local")); + } + + @Test + public void testLoadingTwiceRebindsTheSameGraph() throws IOException { + String path = this.writeGraphConf("bind_twice", "bind_twice"); + Map confs = ImmutableMap.of("bind_twice", path); + + this.manager.loadGraphsFromLocal(confs); + HugeGraph first = this.manager.graph("DEFAULT", "bind_twice"); + this.manager.loadGraphsFromLocal(confs); + + Assert.assertEquals(2, this.created.size()); + Assert.assertSame(first, this.manager.graph("DEFAULT", "bind_twice")); + } + + @Test + public void testGraphNamedDifferentlyByConfigIsNotRebound() throws IOException { + String path = this.writeGraphConf("bind_file", "bind_store"); + + this.manager.loadGraphsFromLocal(ImmutableMap.of("bind_file", path)); + + Assert.assertTrue(this.created.isEmpty()); + Assert.assertNotNull(this.manager.graph("DEFAULT", "bind_file")); + } + + @Test + public void testGraphMissingFromGremlinGetsBound() throws IOException { + String path = this.writeGraphConf("bind_missing", "bind_missing"); + // No settings.graphs entry stands in for a failed static open + this.gremlin = new ContextGremlinServer(newSettings(), this.hub); + org.apache.tinkerpop.gremlin.server.GraphManager bindings = + this.gremlin.getServerGremlinExecutor().getGraphManager(); + Assert.assertNull(bindings.getGraph("DEFAULT-bind_missing")); + + this.manager.loadGraphsFromLocal(ImmutableMap.of("bind_missing", path)); + + Assert.assertSame(this.manager.graph("DEFAULT", "bind_missing"), + bindings.getGraph("DEFAULT-bind_missing")); + Assert.assertNotNull(bindings.getTraversalSource("__g_DEFAULT-bind_missing")); + } + + @Test + public void testGraphOpenedByGremlinKeepsItsInstance() throws IOException { + String path = this.writeGraphConf("bind_static", "bind_static"); + Settings settings = newSettings(); + settings.graphs.put("DEFAULT-bind_static", path); + this.gremlin = new ContextGremlinServer(settings, this.hub); + org.apache.tinkerpop.gremlin.server.GraphManager bindings = + this.gremlin.getServerGremlinExecutor().getGraphManager(); + Graph opened = bindings.getGraph("DEFAULT-bind_static"); + Assert.assertNotNull(opened); + + this.manager.loadGraphsFromLocal(ImmutableMap.of("bind_static", path)); + + Assert.assertSame(opened, bindings.getGraph("DEFAULT-bind_static")); + Assert.assertSame(opened, this.manager.graph("DEFAULT", "bind_static")); + } + + private String writeGraphConf(String fileName, String store) + throws IOException { + File conf = new File(this.graphsDir, fileName + ".properties"); + String content = "gremlin.graph=org.apache.hugegraph.HugeFactory\n" + + "backend=memory\n" + + "serializer=text\n" + + "store=" + store + "\n"; + FileUtils.writeStringToFile(conf, content, StandardCharsets.UTF_8); + return conf.getPath(); + } + + private static Settings newSettings() { + Settings settings = new Settings(); + settings.host = "127.0.0.1"; + settings.port = 0; + settings.gremlinPool = 1; + settings.threadPoolBoss = 1; + settings.threadPoolWorker = 1; + return settings; + } +} From 3fb3e39188b7a7efd95aa93fde15785b374236eb Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Thu, 15 Oct 2026 10:21:11 +0530 Subject: [PATCH 2/4] test(server): cover a failed Gremlin bind of a local graph Add a case where a GRAPH_CREATE listener fails: the error is logged and the graph stays loaded in REST. Also drop the instanceof HugeGraph guard in bindLocalGraph(); loadGraph() already casts the graph's configuration to HugeConfig, so only HugeGraph instances reach it. --- .../apache/hugegraph/core/GraphManager.java | 14 +++++--------- .../core/GraphManagerLocalGraphBindTest.java | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java index 9ccb04be35..4ffed13da6 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java @@ -1829,7 +1829,7 @@ private void loadGraph(String name, String graphConfPath) { LOG.info("Graph '{}' was successfully configured via '{}'", name, graphConfPath); - this.bindLocalGraph(name, graph); + this.bindLocalGraph(name, (HugeGraph) graph); if (this.requireAuthentication() && !(graph instanceof HugeGraphAuthProxy)) { @@ -1846,20 +1846,16 @@ private void loadGraph(String name, String graphConfPath) { * Rebinding the graph Gremlin already holds is harmless: both opens return * the same cached instance. */ - private void bindLocalGraph(String name, Graph graph) { - if (!(graph instanceof HugeGraph)) { - return; - } - HugeGraph hugegraph = (HugeGraph) graph; + private void bindLocalGraph(String name, HugeGraph graph) { String graphName = defaultSpaceGraphName(name); - if (!graphName.equals(hugegraph.spaceGraphName())) { + if (!graphName.equals(graph.spaceGraphName())) { LOG.warn("Graph '{}' is named '{}' by its config, so it can't be " + "rebound in Gremlin Server if Gremlin failed to open it", - graphName, hugegraph.spaceGraphName()); + graphName, graph.spaceGraphName()); return; } try { - this.notifyEvent(Events.GRAPH_CREATE, hugegraph); + this.notifyEvent(Events.GRAPH_CREATE, graph); } catch (Throwable e) { LOG.error("Graph '{}' is served by REST but could not be bound " + "in Gremlin Server, restart the Server to retry", diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java index cd5b4ea73b..1311d38801 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java @@ -128,6 +128,24 @@ public void testGraphNamedDifferentlyByConfigIsNotRebound() throws IOException { Assert.assertNotNull(this.manager.graph("DEFAULT", "bind_file")); } + @Test + public void testFailedBindKeepsGraphLoaded() throws IOException { + String path = this.writeGraphConf("bind_failed", "bind_failed"); + EventListener failing = event -> { + throw new IllegalStateException("gremlin binding failed"); + }; + this.hub.listen(Events.GRAPH_CREATE, failing); + + try { + this.manager.loadGraphsFromLocal(ImmutableMap.of("bind_failed", path)); + } finally { + this.hub.unlisten(Events.GRAPH_CREATE, failing); + } + + Assert.assertEquals(1, this.created.size()); + Assert.assertNotNull(this.manager.graph("DEFAULT", "bind_failed")); + } + @Test public void testGraphMissingFromGremlinGetsBound() throws IOException { String path = this.writeGraphConf("bind_missing", "bind_missing"); From 664eb9a8a181dc35573df118908f1d976af6bccf Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Thu, 15 Oct 2026 12:43:55 +0530 Subject: [PATCH 3/4] fix(server): let loadGraph() accept a graph Gremlin opened first When Gremlin Server opens a conf/graphs file first, HugeFactory caches the graph with a HugeConfig built from a Configuration, which has no file. The REST load of the same path gets that cached instance back, and the assert comparing its config file with the loaded path threw a NullPointerException under -ea before bindLocalGraph() ran. loadGraphsFromLocal() logged and swallowed it, so the unit test for this normal case passed without the rebind ever happening. The assert now skips the comparison when the config has no file, and the test checks that GRAPH_CREATE fired and that the __g_ traversal source still belongs to the instance Gremlin opened. --- .../main/java/org/apache/hugegraph/core/GraphManager.java | 4 +++- .../hugegraph/unit/core/GraphManagerLocalGraphBindTest.java | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java index 4ffed13da6..a60b507be6 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java @@ -1824,7 +1824,9 @@ private void loadGraph(String name, String graphConfPath) { this.graphs.put(defaultSpaceGraphName(name), graph); HugeConfig graphConfig = (HugeConfig) graph.configuration(); - assert graphConfPath.equals(Objects.requireNonNull(graphConfig.file()).getPath()); + // A graph Gremlin Server opened first from this path is cached without a file + assert graphConfig.file() == null || + graphConfPath.equals(graphConfig.file().getPath()); LOG.info("Graph '{}' was successfully configured via '{}'", name, graphConfPath); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java index 1311d38801..91a5ea2bed 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerLocalGraphBindTest.java @@ -44,6 +44,7 @@ import org.junit.Before; import org.junit.Test; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; /** @@ -175,8 +176,12 @@ public void testGraphOpenedByGremlinKeepsItsInstance() throws IOException { this.manager.loadGraphsFromLocal(ImmutableMap.of("bind_static", path)); + // The rebind ran and handed Gremlin the instance it already held + Assert.assertEquals(ImmutableList.of("DEFAULT-bind_static"), this.created); Assert.assertSame(opened, bindings.getGraph("DEFAULT-bind_static")); Assert.assertSame(opened, this.manager.graph("DEFAULT", "bind_static")); + Assert.assertSame(opened, bindings.getTraversalSource("__g_DEFAULT-bind_static") + .getGraph()); } private String writeGraphConf(String fileName, String store) From 5419eaa056cf6dbbf82d29203706ac83857d843e Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Fri, 16 Oct 2026 10:00:36 +0530 Subject: [PATCH 4/4] docs(helm): point the Could not rebind row at the fix, not the issue "Images that include apache/hugegraph#3228" named the bug report, which reads as images that have the bug. Name the fixing PR instead. --- helm/hugegraph/docs/operations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/hugegraph/docs/operations.md b/helm/hugegraph/docs/operations.md index 6ddc90abec..3e9ecad7ad 100644 --- a/helm/hugegraph/docs/operations.md +++ b/helm/hugegraph/docs/operations.md @@ -293,7 +293,7 @@ Updating chart values alone does not change an existing image's behavior. | `HG_PD_AUTH_SECRET_KEY` missing | PD Pod Secret reference and key; chart injects the shared PD REST password automatically | | Server storage wait timeout | Ready PD majority, Store registration, network policies and `server.storageWaitTimeoutSeconds` | | NodePort/Ingress unreachable | Assigned port, reachable node/controller address, firewall and NetworkPolicy rules | -| REST works but Gremlin reports `Could not rebind` | Run `helm test`. Images that include apache/hugegraph#3228 rebind Gremlin when REST loads the graph; if it persists, check the Server log for `could not be bound in Gremlin Server` or `can't be rebound in Gremlin Server`, then replace the failing Server Pod after PD stabilizes and retry | +| REST works but Gremlin reports `Could not rebind` | Run `helm test`. Server images built from source that includes apache/hugegraph#3351 (the fix for #3228) rebind Gremlin when REST loads the graph; if it persists, check the Server log for `could not be bound in Gremlin Server` or `can't be rebound in Gremlin Server`, then replace the failing Server Pod after PD stabilizes and retry | ```bash kubectl -n hugegraph get pods,pvc,svc