refactor(core): drain tasks before graph deletion

docs/server-runtime.md

@@ -463,3 +463,23 @@ Structure or Process suites must execute tests; an all-skipped report is not
 compatibility evidence. Run the read-only application examples against representative
 existing data and validate the serializers, client versions and authentication
 combinations actually used by the application.
+
+
+## Deleting a graph with local tasks
+
+Stop application writes before deletion and keep them stopped during retries. The local
+scheduler closes task admission and waits for actual executions, cancellation callbacks
+and their transaction cleanup before data or metadata is removed. If draining times out,
+the graph stays registered with its data and configuration, but task admission remains
+closed: new asynchronous tasks are rejected even after the existing owners exit. Keep
+application writes stopped and retry deletion after those owners exit. If no destructive
+cleanup has begun, restart the Server to resume task admission instead of completing
+the deletion. Closing the scheduler alone leaves unstarted persistent work in storage.
+
+This is a local shutdown guarantee. For a graph shared by several Servers, stop writes
+and tasks on every Server before deletion; notification delivery is not a cluster-wide
+barrier. Do not recreate the same graph name until deletion finishes. Use a restart for
+configuration changes rather than hot reload, whose recovery is handled separately.
+Deletion is not transactional: a failure after destructive cleanup needs operator
+inspection, and retry does not restore removed data. An owner-cleanup failure must not
+be treated as a successful drain; inspect it instead of force-removing the scheduler.

hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java

@@ -36,6 +36,7 @@
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
@@ -114,6 +115,7 @@
 import org.apache.hugegraph.space.register.dto.ServiceDTO;
 import org.apache.hugegraph.space.register.registerImpl.PdRegister;
 import org.apache.hugegraph.task.TaskManager;
+import org.apache.hugegraph.task.TaskScheduler;
 import org.apache.hugegraph.testutil.Whitebox;
 import org.apache.hugegraph.traversal.optimize.HugeScriptTraversal;
 import org.apache.hugegraph.type.define.CollectionType;
@@ -1107,23 +1109,21 @@ public void getSpaceStorage(String graphSpace) {
     }
 
     public void clearGraphSpace(String name) {
-        // Clear all roles
-        this.metaManager.clearGraphAuth(name);
-
-        // Clear all schemaTemplate
-        this.metaManager.clearSchemaTemplate(name);
-
         // Clear all graphs
         for (String key : this.graphs.keySet()) {
-            if (key.startsWith(name)) {
+            if (key.startsWith(name + DELIMITER)) {
                 String[] parts = key.split(DELIMITER);
                 this.dropGraph(parts[0], parts[1], true);
             }
         }
 
+        // Shared metadata remains available if a graph refuses deletion.
+        this.metaManager.clearGraphAuth(name);
+        this.metaManager.clearSchemaTemplate(name);
+
         // Clear all services
         for (String key : this.services.keySet()) {
-            if (key.startsWith(name)) {
+            if (key.startsWith(name + DELIMITER)) {
                 String[] parts = key.split(DELIMITER);
                 this.dropService(parts[0], parts[1]);
             }
@@ -1457,12 +1457,13 @@ private HugeGraph createGraphLocal(HugeConfig config, String name) {
             LOG.error("Failed to create graph '{}' due to: {}",
                       name, e.getMessage(), e);
             if (graph != null) {
-                this.graphs.remove(graph.spaceGraphName(), graph);
                 try {
                     this.dropGraphLocal(graph);
-                } finally {
-                    // The create event may have partially registered the graph
+                    this.graphs.remove(graph.spaceGraphName(), graph);
                     this.notifyEventLenient(Events.GRAPH_DROP, graph);
+                } catch (Throwable cleanup) {
+                    e.addSuppressed(cleanup);
+                    this.graphs.putIfAbsent(graph.spaceGraphName(), graph);
                 }
             }
             throw e;
@@ -1472,16 +1473,8 @@ private HugeGraph createGraphLocal(HugeConfig config, String name) {
     }
 
     private void dropGraphLocal(HugeGraph graph) {
-        try {
-            // Clear data and config files
-            graph.drop();
-        } finally {
-            /*
-             * Will fill graph instance into HugeFactory.graphs after
-             * GraphFactory.open() succeed, remove it when the graph drops
-             */
-            HugeFactory.remove(graph);
-        }
+        graph.drop();
+        HugeFactory.remove(graph);
     }
 
     public HugeGraph createGraph(String graphSpace, String name, String creator,
@@ -1614,16 +1607,14 @@ public HugeGraph createGraph(String graphSpace, String name, String creator,
         try {
             this.notifyEvent(Events.GRAPH_CREATE, graph);
         } catch (Throwable e) {
-            this.notifyEventLenient(Events.GRAPH_DROP, graph);
-            this.graphs.remove(graphName, graph);
             try {
                 graph.close();
+                this.notifyEventLenient(Events.GRAPH_DROP, graph);
+                this.graphs.remove(graphName, graph);
+                HugeFactory.remove(graph);
             } catch (Exception e1) {
-                if (graph instanceof StandardHugeGraph) {
-                    ((StandardHugeGraph) graph).clearSchedulerAndLock();
-                }
+                e.addSuppressed(e1);
             }
-            HugeFactory.remove(graph);
             throw e;
         }
 
@@ -2086,12 +2077,11 @@ private HugeGraph createGraph(String graphSpace, HugeConfig config,
             } catch (BackendException e) {
                 try {
                     graph.close();
+                    HugeFactory.remove(graph);
                 } catch (Exception e1) {
-                    if (graph instanceof StandardHugeGraph) {
-                        ((StandardHugeGraph) graph).clearSchedulerAndLock();
-                    }
+                    e.addSuppressed(e1);
+                    this.graphs.putIfAbsent(graph.spaceGraphName(), graph);
                 }
-                HugeFactory.remove(graph);
                 throw e;
             }
         }
@@ -2269,6 +2259,21 @@ public void dropGraph(String graphSpace, String name, boolean clear) {
 
         String graphName = spaceGraphName(graphSpace, name);
         if (clear) {
+            // The auth proxy checks ADMIN before the local deletion fence.
+            TaskScheduler scheduler = g.taskScheduler();
+            if (!scheduler.close()) {
+                try {
+                    scheduler.waitUntilAllTasksCompleted(g.configuration().get(CoreOptions.TASK_WAIT_TIMEOUT));
+                } catch (TimeoutException e) {
+                    throw new HugeException("Tasks are still active in graph '%s'; task admission remains closed. " +
+                                            "Retry deletion after owners exit, or restart to resume task admission",
+                                            e, graphName);
+                }
+                E.checkState(scheduler.close(),
+                             "Tasks are still active in graph '%s'; task admission remains closed. " +
+                             "Retry deletion after owners exit, or restart to resume task admission",
+                             graphName);
+            }
             this.removingGraphs.add(graphName);
             try {
                 this.metaManager.removeGraphConfig(graphSpace, name);
@@ -2278,24 +2283,13 @@ public void dropGraph(String graphSpace, String name, boolean clear) {
                         "Failed to remove graph config of '%s'", name, e);
             }
 
-            /**
-             * close task scheduler before clear data,
-             * because taskinfo stored in backend in
-             * {@link org.apache.hugegraph.task.DistributedTaskScheduler}
-             */
-            try {
-                g.taskScheduler().close();
-            } catch (Throwable t) {
-                LOG.warn(String.format(
-                                 "Error when close TaskScheduler of %s",
-                                 graphName),
-                         t);
-            }
-
             g.clearBackend();
             try {
                 g.close();
             } catch (Exception e) {
+                if (!g.closed()) {
+                    throw new HugeException("Failed to close graph '%s', please retry later", e, graphName);
+                }
                 LOG.warn("Failed to close graph", e);
             }
         }
@@ -2309,13 +2303,17 @@ public void dropGraph(String graphSpace, String name, boolean clear) {
         }
         // Let gremlin server and rest server context remove graph
         LOG.info("Notify remove graph {} by GRAPH_DROP event", name);
-        Graph graph = this.graphs.remove(graphName);
+        Graph graph = this.graphs.get(graphName);
         if (graph != null) {
             try {
                 graph.close();
             } catch (Exception e) {
+                if (!((HugeGraph) graph).closed()) {
+                    throw new HugeException("Failed to close graph '%s', please retry later", e, graphName);
+                }
                 LOG.warn("Failed to close graph", e);
             }
+            this.graphs.remove(graphName, graph);
             try {
                 // Delete alias graph in HugeFactory
                 HugeFactory.remove((HugeGraph) graph);

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java

@@ -537,6 +537,7 @@ public void initBackend() {
 
     @Override
     public void clearBackend() {
+        this.checkGraphNotClosed();
         this.waitUntilAllTasksCompleted();
 
         this.loadSchemaStore().open(this.configuration);
@@ -1212,12 +1213,24 @@ public void create(String configPath, GlobalMasterInfo nodeInfo) {
 
     @Override
     public void drop() {
+        this.checkGraphNotClosed();
+        TaskScheduler scheduler = this.taskManager.getScheduler(this.params);
+        if (scheduler != null && !scheduler.close()) {
+            // Fence admission before waiting for actual task owners to exit.
+            try {
+                scheduler.waitUntilAllTasksCompleted(this.configuration().get(CoreOptions.TASK_WAIT_TIMEOUT));
+            } catch (TimeoutException e) {
+                throw new HugeException("Tasks are still active in graph '%s'; task admission remains closed. " +
+                                        "Retry deletion after owners exit, or restart to resume task admission",
+                                        e, this.spaceGraphName());
+            }
+            E.checkState(scheduler.close(),
+                         "Tasks are still active in graph '%s'; task admission remains closed. " +
+                         "Retry deletion after owners exit, or restart to resume task admission",
+                         this.spaceGraphName());
+        }
         this.clearBackend();
 
-        HugeConfig config = this.configuration();
-        this.storeProvider.onDeleteConfig(config);
-        ConfigUtil.deleteFile(config.file());
-
         try {
             /*
              * It's hard to ensure all threads close the tx.
@@ -1228,8 +1241,15 @@ public void drop() {
              */
             this.close();
         } catch (Throwable e) {
+            if (!this.closed()) {
+                throw new HugeException("Failed to close graph '%s', please retry later",
+                                        e, this.spaceGraphName());
+            }
             LOG.warn("Failed to close graph {} {}", this, e);
         }
+        HugeConfig config = this.configuration();
+        this.storeProvider.onDeleteConfig(config);
+        ConfigUtil.deleteFile(config.file());
     }
 
     @Override