From 8b75c81da8b711ce3b2a295aa84f4a7f89a6dfa2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:19:04 +0800 Subject: [PATCH 1/3] Support asynchronous timeseries and table column deletion --- .../itbase/runtime/ClusterTestStatement.java | 4 +- .../iotdb/db/it/IoTDBExecuteBatchIT.java | 9 + .../apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 6 +- .../IoTDBQueryWithRecreatedTimeseriesIT.java | 3 +- .../IoTDBRecreatedTimeseriesCompactionIT.java | 3 +- .../apache/iotdb/db/it/IoTDBRestartIT.java | 4 +- .../iotdb/db/it/IoTDBRestartRatisIT.java | 4 +- .../IoTDBAggregationWithDeletion2IT.java | 3 +- .../IoTDBAggregationWithDeletionIT.java | 3 +- ...IoTDBAlignedSeriesQueryWithMisMatchIT.java | 3 +- ...ByQueryWithValueFilterWithDeletion2IT.java | 3 +- ...pByQueryWithValueFilterWithDeletionIT.java | 3 +- ...ueryWithoutValueFilterWithDeletion2IT.java | 3 +- ...QueryWithoutValueFilterWithDeletionIT.java | 3 +- .../IoTDBLastQueryWithDeletion2IT.java | 3 +- .../aligned/IoTDBLastQueryWithDeletionIT.java | 3 +- ...tQueryWithoutLastCacheWithDeletion2IT.java | 3 +- ...stQueryWithoutLastCacheWithDeletionIT.java | 3 +- ...ueryWithoutValueFilterWithDeletion2IT.java | 3 +- ...QueryWithoutValueFilterWithDeletionIT.java | 3 +- .../db/it/audit/IoTDBAuditLogBasicIT.java | 47 +- .../IoTDBIoTConsensusV23C3DBasicITBase.java | 3 +- .../it/schema/IoTDBAlterTimeSeriesTypeIT.java | 39 +- .../IoTDBCreateAlignedTimeseriesIT.java | 2 +- .../IoTDBDeleteAlignedTimeseriesIT.java | 15 +- .../db/it/schema/IoTDBDeleteTimeSeriesIT.java | 68 ++- .../db/it/schema/IoTDBMetadataFetchIT.java | 3 +- .../db/it/schema/IoTDBSchemaTemplateIT.java | 5 +- ...oTDBShowTimeseriesOrderByTimeseriesIT.java | 5 +- .../apache/iotdb/db/it/schema/IoTDBTagIT.java | 7 +- .../schema/quota/IoTDBClusterMixQuotaIT.java | 3 +- .../it/schema/quota/IoTDBClusterQuotaIT.java | 3 +- .../iotdb/db/it/utils/IoTDBTestUtilsIT.java | 46 ++ .../apache/iotdb/db/it/utils/TestUtils.java | 166 +++++- .../it/db/it/IoTDBDeletionTableIT.java | 4 +- .../it/db/it/IoTDBFlushQueryTableIT.java | 3 +- .../it/db/it/IoTDBRestartTableIT.java | 4 +- .../it/schema/IoTDBAlterColumnTypeIT.java | 8 +- .../schema/IoTDBNullableAttributeCacheIT.java | 41 +- .../it/schema/IoTDBTableDDLHAIT.java | 3 + .../relational/it/schema/IoTDBTableIT.java | 50 +- .../session/it/IoTDBSessionSimpleIT.java | 9 + .../apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 5 + .../confignode/i18n/ManagerMessages.java | 10 + .../confignode/i18n/ManagerMessages.java | 10 + .../confignode/conf/ConfigNodeConfig.java | 42 ++ .../confignode/conf/ConfigNodeDescriptor.java | 16 + .../confignode/manager/ConfigManager.java | 148 ++---- .../iotdb/confignode/manager/IManager.java | 3 + .../confignode/manager/ProcedureManager.java | 319 ++++++++++-- .../procedure/ProcedureExecutor.java | 126 ++++- .../schema/DeleteTimeSeriesProcedure.java | 204 +++++++- .../DeleteTimeSeriesProcedureScheduler.java | 226 +++++++++ .../state/schema/DeleteTimeSeriesState.java | 5 +- .../procedure/store/ConfigProcedureStore.java | 50 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 6 + .../manager/AsyncTimeSeriesDeletionTest.java | 419 ++++++++++++++++ .../ConfigManagerDatabaseDeletionTest.java | 60 +++ .../procedure/ConfigProcedureStoreTest.java | 151 ++++++ .../schema/DeleteTimeSeriesProcedureTest.java | 472 ++++++++++++++++++ .../table/DropTableColumnProcedureTest.java | 104 ++++ ...eleteTimeSeriesProcedureSchedulerTest.java | 141 ++++++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 4 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 4 + .../db/protocol/client/ConfigNodeClient.java | 8 + .../common/header/DatasetHeaderFactory.java | 4 + .../config/TreeConfigTaskVisitor.java | 8 + .../executor/ClusterConfigTaskExecutor.java | 59 ++- .../config/executor/IConfigTaskExecutor.java | 2 + .../sys/ShowDeleteTimeSeriesProgressTask.java | 89 ++++ .../queryengine/plan/parser/ASTVisitor.java | 7 + .../security/TreeAccessCheckVisitor.java | 8 + .../plan/statement/StatementType.java | 1 + .../plan/statement/StatementVisitor.java | 6 + ...ShowDeleteTimeSeriesProgressStatement.java | 52 ++ .../plan/parser/StatementGeneratorTest.java | 11 + .../conf/iotdb-system.properties.template | 16 + .../schema/column/ColumnHeaderConstant.java | 9 + .../src/main/thrift/confignode.thrift | 16 + 79 files changed, 3142 insertions(+), 285 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/DeleteTimeSeriesProcedureScheduler.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/AsyncTimeSeriesDeletionTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ConfigProcedureStoreTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/scheduler/DeleteTimeSeriesProcedureSchedulerTest.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ShowDeleteTimeSeriesProgressTask.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowDeleteTimeSeriesProgressStatement.java diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java index eccea1d143709..9f6644cc38cfa 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java @@ -280,8 +280,8 @@ public int[] executeBatch() throws SQLException { } @Override - public Connection getConnection() { - throw new UnsupportedOperationException(); + public Connection getConnection() throws SQLException { + return writeStatement.getConnection(); } @Override diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java index 9ba4e4e7ce980..4088a9873cf6a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -71,6 +72,10 @@ public void testJDBCExecuteBatch() { statement.addBatch( "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465600001,2.3)"); statement.addBatch("delete timeseries root.ln.wf01.wt01.**"); + statement.executeBatch(); + statement.clearBatch(); + TestUtils.waitForTimeSeriesDeletion( + EnvFactory.getEnv(), "delete timeseries root.ln.wf01.wt01.**"); statement.addBatch( "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465600002,3.4)"); statement.executeBatch(); @@ -104,6 +109,10 @@ public void testJDBCExecuteBatchForCreateMultiTimeSeriesPlan() { statement.addBatch( "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465600001,2.3)"); statement.addBatch("delete timeseries root.ln.wf01.wt01.**"); + statement.executeBatch(); + statement.clearBatch(); + TestUtils.waitForTimeSeriesDeletion( + EnvFactory.getEnv(), "delete timeseries root.ln.wf01.wt01.**"); statement.addBatch( "create timeseries root.turbine.d1.s1(s1) with datatype=boolean, encoding=plain , compression=snappy " + "tags('tag1'='v1', 'tag2'='v2') attributes('attr1'='v3', 'attr2'='v4')"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index 6640277eddbab..9e2f4213c31b0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -261,7 +261,8 @@ public void testLoad() throws Exception { // Try to delete after loading. Expect no deadlock try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute( + TestUtils.deleteTimeSeries( + statement, String.format( "delete timeseries %s.%s", SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_00.getMeasurementName())); @@ -363,7 +364,8 @@ public void testLoadWithAlignmentMismatch() throws Exception { // Try to delete after loading. Expect no deadlock try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute( + TestUtils.deleteTimeSeries( + statement, String.format( "delete timeseries %s.%s", SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_00.getMeasurementName())); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBQueryWithRecreatedTimeseriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBQueryWithRecreatedTimeseriesIT.java index 5472fabec5e10..a6abb41c67a56 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBQueryWithRecreatedTimeseriesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBQueryWithRecreatedTimeseriesIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -61,7 +62,7 @@ public void testQueryDiffTypeTimeseries() { Thread.sleep(100); statement.execute("FLUSH"); Thread.sleep(100); - statement.execute("DELETE TIMESERIES root.sg.d1.s1"); + TestUtils.deleteTimeSeries(statement, "DELETE TIMESERIES root.sg.d1.s1"); statement.execute("CREATE TIMESERIES root.sg.d1.s1 with datatype=INT32,encoding=PLAIN"); try (ResultSet resultSet = statement.executeQuery("SELECT s1 FROM root.sg.d1 WHERE s1 > 10")) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java index 8d3609ee87289..a8f4b77cd9bd9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.it; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -77,7 +78,7 @@ public void testDeletionAfterRecreatingTwoLevelTimeseriesSurvivesCompaction() th statement.execute("INSERT INTO root.repro(time, s1) VALUES (1, 1)"); statement.execute("FLUSH"); - statement.execute("DELETE TIMESERIES root.repro.s1"); + TestUtils.deleteTimeSeries(statement, "DELETE TIMESERIES root.repro.s1"); statement.execute("CREATE TIMESERIES root.repro.s1 WITH DATATYPE=INT64"); statement.execute("INSERT INTO root.repro(time, s1) VALUES (2, 2)"); statement.execute("FLUSH"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java index 302d059959a5a..f4fa93e125679 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java @@ -382,7 +382,7 @@ public void testRecoverWALMismatchDataType() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("insert into root.turbine1.d1(timestamp,s1,s2) values(1,1.1,2.2)"); - statement.execute("delete timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.turbine1.d1.s1"); statement.execute( "create timeseries root.turbine1.d1.s1 with datatype=INT32, encoding=RLE, compression=SNAPPY"); } @@ -412,7 +412,7 @@ public void testRecoverWALDeleteSchema() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("insert into root.turbine1.d1(timestamp,s1,s2) values(1,1.1,2.2)"); - statement.execute("delete timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.turbine1.d1.s1"); } TestUtils.restartDataNodes(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartRatisIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartRatisIT.java index 22f5234ee9239..7c34294a673cd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartRatisIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartRatisIT.java @@ -251,7 +251,7 @@ public void testRecoverWALMismatchDataType() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("insert into root.turbine1.d1(timestamp,s1,s2) values(1,1.1,2.2)"); - statement.execute("delete timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.turbine1.d1.s1"); statement.execute( "create timeseries root.turbine1.d1.s1 with datatype=INT32, encoding=RLE, compression=SNAPPY"); } @@ -281,7 +281,7 @@ public void testRecoverWALDeleteSchema() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("insert into root.turbine1.d1(timestamp,s1,s2) values(1,1.1,2.2)"); - statement.execute("delete timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.turbine1.d1.s1"); } TestUtils.restartDataNodes(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletion2IT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletion2IT.java index 8d69a09e89725..c401ea6844fc1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletion2IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletion2IT.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -55,7 +56,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s1 where time <= 21"); } catch (Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletionIT.java index d16587ac45354..82e003424d006 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletionIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -62,7 +63,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s1 where time <= 21"); } catch (Exception e) { fail(e.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryWithMisMatchIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryWithMisMatchIT.java index a8fcf713bcda6..fefe4bb84004c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryWithMisMatchIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryWithMisMatchIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -61,7 +62,7 @@ public static void setUp() throws Exception { statement.execute("insert into root.sg1.d1(time, s1, s2) aligned values (1, 1, true)"); statement.execute("flush"); - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("insert into root.sg1.d1(time, s1, s2) aligned values (10, 10, 100)"); statement.execute("flush"); } catch (Exception e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithValueFilterWithDeletion2IT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithValueFilterWithDeletion2IT.java index de074bfe88c73..c4877303954a8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithValueFilterWithDeletion2IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithValueFilterWithDeletion2IT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -54,7 +55,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("delete from root.sg1.d1.s1 where time <= 15"); - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); } catch (Exception e) { e.printStackTrace(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithValueFilterWithDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithValueFilterWithDeletionIT.java index 7b228b700b816..c1a43444dd4de 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithValueFilterWithDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithValueFilterWithDeletionIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -65,7 +66,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("delete from root.sg1.d1.s1 where time <= 15"); - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); } catch (Exception e) { e.printStackTrace(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithoutValueFilterWithDeletion2IT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithoutValueFilterWithDeletion2IT.java index 46b964a78adb2..7bc5818d8ede8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithoutValueFilterWithDeletion2IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithoutValueFilterWithDeletion2IT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -53,7 +54,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("delete from root.sg1.d1.s1 where time <= 15"); - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s3 where time > 25"); } catch (Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithoutValueFilterWithDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithoutValueFilterWithDeletionIT.java index 3461bbb32b2f1..0b6f8055e7fa2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithoutValueFilterWithDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByQueryWithoutValueFilterWithDeletionIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -65,7 +66,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("delete from root.sg1.d1.s1 where time <= 15"); - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s3 where time > 25"); } catch (Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletion2IT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletion2IT.java index 0810a0abe5953..e5849cbae5d49 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletion2IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletion2IT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -52,7 +53,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s1 where time <= 27"); } catch (Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java index cd2ce061c0031..a845eb2720a0b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -64,7 +65,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s1 where time <= 27"); } catch (Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheWithDeletion2IT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheWithDeletion2IT.java index efc2d65dd9185..c972424f1f9a6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheWithDeletion2IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheWithDeletion2IT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -55,7 +56,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s1 where time <= 27"); } catch (Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheWithDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheWithDeletionIT.java index eaa134759c389..97225d6cbd496 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheWithDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheWithDeletionIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -53,7 +54,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s1 where time <= 27"); } catch (Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletion2IT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletion2IT.java index 01e9466fcbab6..81135b1994c2e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletion2IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletion2IT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -54,7 +55,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s2 where time <= 40"); statement.execute("delete from root.sg1.d1.s1 where time <= 21"); statement.execute("delete from root.sg1.d1.s5 where time <= 31 and time > 20"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java index 67787a8ddd233..ee676c8a8a5c6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -58,7 +59,7 @@ public static void setUp() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("delete timeseries root.sg1.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s2 where time <= 40"); statement.execute("delete from root.sg1.d1.s1 where time <= 21"); statement.execute("delete from root.sg1.d1.s5 where time <= 31 and time > 20"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/audit/IoTDBAuditLogBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/audit/IoTDBAuditLogBasicIT.java index 83aa06dc61dce..d4e16dda53e93 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/audit/IoTDBAuditLogBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/audit/IoTDBAuditLogBasicIT.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.audit.AbstractAuditLogger; import org.apache.iotdb.commons.audit.AuditLogOperation; import org.apache.iotdb.commons.audit.PrivilegeLevel; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -49,6 +50,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.awaitility.Awaitility.await; + /** * This test class ensures the audit log behave exactly the same as we expected, including the * number, sequence and content of the audit logs. @@ -2389,7 +2392,7 @@ public void basicAuditLogTestForTableModel() throws SQLException, InterruptedExc Stream.of(7).collect(Collectors.toSet()); @Test - public void basicAuditLogTestForTreeModel() throws SQLException, InterruptedException { + public void basicAuditLogTestForTreeModel() throws Exception { Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); Statement statement = connection.createStatement(); for (String sql : TREE_MODEL_AUDIT_SQLS_USER_ROOT) { @@ -2417,7 +2420,47 @@ public void basicAuditLogTestForTreeModel() throws SQLException, InterruptedExce connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); statement = connection.createStatement(); for (String sql : TREE_MODEL_AUDIT_SQLS_USER_ROOT_FINAL) { - statement.execute(sql); + if (sql.startsWith("delete timeseries ") || sql.startsWith("drop timeseries ")) { + final String paths = "[" + sql.substring(sql.lastIndexOf(' ') + 1) + "]"; + // Poll the ConfigNode directly so diagnostics do not add SQL audit records to this test. + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + final long previousProcedureId = + client.showDeleteTimeSeriesProcedures().getProcedureInfoList().stream() + .filter(p -> paths.equals(p.getPathPatterns())) + .mapToLong(p -> p.getProcedureId()) + .max() + .orElse(-1L); + statement.execute(sql); + await() + .atMost(60, TimeUnit.SECONDS) + .pollInSameThread() + .pollInterval(20, TimeUnit.MILLISECONDS) + .until( + () -> { + final java.util.Optional< + org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesProcedureInfo> + procedure = + client.showDeleteTimeSeriesProcedures().getProcedureInfoList().stream() + .filter(p -> paths.equals(p.getPathPatterns())) + .filter(p -> p.getProcedureId() > previousProcedureId) + .max(java.util.Comparator.comparingLong(p -> p.getProcedureId())); + if (!procedure.isPresent()) { + return false; + } + if ("ROLLEDBACK".equals(procedure.get().getState())) { + throw new AssertionError( + "Timeseries deletion procedure " + + procedure.get().getProcedureId() + + " rolled back: " + + procedure.get().getErrorMessage()); + } + return "SUCCESS".equals(procedure.get().getState()); + }); + } + } else { + statement.execute(sql); + } } ResultSet resultSet = statement.executeQuery("SELECT * FROM root.__audit.log.** ORDER BY TIME ALIGN BY DEVICE"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/iotconsensusv2/IoTDBIoTConsensusV23C3DBasicITBase.java b/integration-test/src/test/java/org/apache/iotdb/db/it/iotconsensusv2/IoTDBIoTConsensusV23C3DBasicITBase.java index 094b5aebb94e7..08cc15cafafd6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/iotconsensusv2/IoTDBIoTConsensusV23C3DBasicITBase.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/iotconsensusv2/IoTDBIoTConsensusV23C3DBasicITBase.java @@ -21,6 +21,7 @@ import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; @@ -255,7 +256,7 @@ public void testDeleteTimeSeriesReplicaConsistency() throws Exception { // Step 3: Delete one timeseries LOGGER.info("Step 3: Deleting timeseries root.sg.d1.speed..."); - statement.execute(DELETE_TIMESERIES_SPEED); + TestUtils.deleteTimeSeries(statement, DELETE_TIMESERIES_SPEED); // Step 4: Flush again to persist the deletion LOGGER.info("Step 4: Flushing to persist deletion..."); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java index 8737df331d993..89f2a3509519e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.utils.MetadataUtils; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionDataSet; @@ -348,8 +349,8 @@ private void doWriteAndAlter(TSDataType from, TSDataType to) } } finally { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - session.executeNonQueryStatement( - "DELETE TIMESERIES " + database + ".write_and_alter_column_type.s1"); + TestUtils.deleteTimeSeries( + session, "DELETE TIMESERIES " + database + ".write_and_alter_column_type.s1"); } } } @@ -428,8 +429,8 @@ private void doWriteAndAlterWriteForAccumulatorQuery(TSDataType from, TSDataType standardAccumulatorQueryByTimeInvolveAllDataTypeTest(session, from, to); } finally { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - session.executeNonQueryStatement( - "DELETE TIMESERIES " + database + ".write_and_alter_column_type_write.s1"); + TestUtils.deleteTimeSeries( + session, "DELETE TIMESERIES " + database + ".write_and_alter_column_type_write.s1"); } } } @@ -512,8 +513,8 @@ private void doAlterWithoutWrite(TSDataType from, TSDataType to, boolean flush) assertFalse(dataSet.hasNext()); - session.executeNonQueryStatement( - "DELETE TIMESERIES " + database + ".just_alter_column_type.**"); + TestUtils.deleteTimeSeries( + session, "DELETE TIMESERIES " + database + ".just_alter_column_type.**"); } } @@ -1155,7 +1156,7 @@ public void testLoadAndAccumulator() resourceFile.delete(); }); filesToLoad.clear(); - session.executeNonQueryStatement("DELETE TIMESERIES root.sg1.d1.s1"); + TestUtils.deleteTimeSeries(session, "DELETE TIMESERIES root.sg1.d1.s1"); } } @@ -1285,8 +1286,8 @@ public void testNonAlignDeviceSequenceDataQuery(TSDataType from, TSDataType to) "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) .hasNext(); if (isExist) { - session.executeNonQueryStatement( - "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + TestUtils.deleteTimeSeries( + session, "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); } } } @@ -1425,8 +1426,8 @@ public void testAlignDeviceSequenceDataQuery(TSDataType from, TSDataType to) thr "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) .hasNext(); if (isExist) { - session.executeNonQueryStatement( - "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + TestUtils.deleteTimeSeries( + session, "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); } } } @@ -1555,8 +1556,8 @@ public void testNonAlignDeviceUnSequenceDataQuery(TSDataType from, TSDataType to "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) .hasNext(); if (isExist) { - session.executeNonQueryStatement( - "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + TestUtils.deleteTimeSeries( + session, "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); } } } @@ -1688,8 +1689,8 @@ public void testAlignDeviceUnSequenceDataQuery(TSDataType from, TSDataType to) t "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) .hasNext(); if (isExist) { - session.executeNonQueryStatement( - "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + TestUtils.deleteTimeSeries( + session, "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); } } } @@ -1818,8 +1819,8 @@ public void testNonAlignDeviceUnSequenceOverlappedDataQuery(TSDataType from, TSD "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) .hasNext(); if (isExist) { - session.executeNonQueryStatement( - "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + TestUtils.deleteTimeSeries( + session, "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); } } } @@ -1949,8 +1950,8 @@ public void testAlignDeviceUnSequenceOverlappedDataQuery(TSDataType from, TSData "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) .hasNext(); if (isExist) { - session.executeNonQueryStatement( - "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + TestUtils.deleteTimeSeries( + session, "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateAlignedTimeseriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateAlignedTimeseriesIT.java index bd2730c39d5bc..4ea29833522bf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateAlignedTimeseriesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateAlignedTimeseriesIT.java @@ -106,7 +106,7 @@ public void testCreateAlignedTimeseriesWithDeletion() throws Exception { try { statement.execute( "CREATE ALIGNED TIMESERIES root.sg1.d1.vector1(s1 FLOAT encoding=PLAIN compressor=UNCOMPRESSED,s2 INT64 encoding=RLE)"); - statement.execute("DELETE TIMESERIES root.sg1.d1.vector1.s1"); + TestUtils.deleteTimeSeries(statement, "DELETE TIMESERIES root.sg1.d1.vector1.s1"); statement.execute( "CREATE ALIGNED TIMESERIES root.sg1.d1.vector1(s1 DOUBLE encoding=PLAIN compressor=SNAPPY)"); } catch (SQLException e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java index 2ade000fbb664..72125fb68db20 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.it.schema; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.util.AbstractSchemaIT; @@ -84,7 +85,7 @@ public void deleteTimeseriesAndCreateDifferentTypeTest() throws Exception { } } - statement.execute("DELETE timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "DELETE timeseries root.turbine1.d1.s1"); statement.execute("INSERT INTO root.turbine1.d1(timestamp,s1) ALIGNED VALUES(2,1.1)"); statement.execute("FLUSH"); @@ -130,7 +131,7 @@ public void deleteTimeseriesAndCreateSameTypeTest() throws Exception { cnt++; } } - statement.execute("DELETE timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "DELETE timeseries root.turbine1.d1.s1"); statement.execute("INSERT INTO root.turbine1.d1(timestamp,s1) ALIGNED VALUES(2,5)"); statement.execute("FLUSH"); @@ -167,8 +168,8 @@ public void deleteTimeseriesAndChangeDeviceAlignmentTest() throws Exception { } } cnt = 0; - statement.execute("DELETE timeseries root.sg3.d1.s1"); - statement.execute("DELETE timeseries root.sg3.d1.s2"); + TestUtils.deleteTimeSeries(statement, "DELETE timeseries root.sg3.d1.s1"); + TestUtils.deleteTimeSeries(statement, "DELETE timeseries root.sg3.d1.s2"); statement.execute("INSERT INTO root.sg3.d1(timestamp,s1,s2) VALUES(1,1,2)"); try (ResultSet resultSet = statement.executeQuery("SHOW DEVICES root.sg3.**")) { while (resultSet.next()) { @@ -188,8 +189,8 @@ public void deleteTimeseriesAndChangeDeviceAlignmentTest() throws Exception { } } cnt = 0; - statement.execute("DELETE timeseries root.sg3.d1.s1"); - statement.execute("DELETE timeseries root.sg3.d1.s2"); + TestUtils.deleteTimeSeries(statement, "DELETE timeseries root.sg3.d1.s1"); + TestUtils.deleteTimeSeries(statement, "DELETE timeseries root.sg3.d1.s2"); statement.execute("INSERT INTO root.sg3.d1(timestamp,s1,s2) ALIGNED VALUES(1,1,2)"); try (ResultSet resultSet = statement.executeQuery("SHOW DEVICES root.sg3.**")) { while (resultSet.next()) { @@ -266,7 +267,7 @@ public void deleteTimeseriesAndCreateSameTypeTest2() throws Exception { } } // delete series in the middle - statement.execute("DELETE timeseries root.turbine1.d1.s4"); + TestUtils.deleteTimeSeries(statement, "DELETE timeseries root.turbine1.d1.s4"); statement.execute( "INSERT INTO root.turbine1.d1(timestamp,s3,s4) ALIGNED VALUES(2,false,8.0)"); statement.execute("FLUSH"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteTimeSeriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteTimeSeriesIT.java index 0866b37836e23..50212cd6b54e1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteTimeSeriesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteTimeSeriesIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.schema; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -64,6 +65,47 @@ public void tearDown() throws Exception { clearSchema(); } + @Test + public void testAsynchronousDeletionProgress() throws Exception { + final String path = "root.async_progress.d.s"; + final String sql = "DELETE TIMESERIES " + path; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = TestUtils.createSingleNodeStatement(connection)) { + statement.execute("INSERT INTO root.async_progress.d(time,s) VALUES(1,1)"); + statement.execute(sql); + boolean found = false; + try (ResultSet result = statement.executeQuery("SHOW DELETE TIMESERIES PROGRESS")) { + Assert.assertEquals(6, result.getMetaData().getColumnCount()); + while (result.next()) { + if (("[" + path + "]").equals(result.getString("Paths"))) { + found = true; + Assert.assertTrue(result.getLong("ProcedureId") >= 0); + Assert.assertNotNull(result.getString("QueryId")); + Assert.assertNotNull(result.getString("Progress")); + Assert.assertTrue( + java.util.Arrays.asList("QUEUED", "RUNNING", "SUCCESS") + .contains(result.getString("State"))); + Assert.assertNull(result.getString("ErrorMessage")); + } + } + } + Assert.assertTrue("Accepted deletion must be visible immediately", found); + TestUtils.waitForTimeSeriesDeletion(statement, sql); + try (ResultSet result = statement.executeQuery("SHOW TIMESERIES " + path)) { + Assert.assertFalse(result.next()); + } + // A nonexistent path is accepted too; the error belongs to the asynchronous task. + statement.execute(sql); + try { + TestUtils.waitForTimeSeriesDeletion(statement, sql); + Assert.fail("Expected the task to fail for a nonexistent path"); + } catch (SQLException e) { + Assert.assertTrue( + e.getMessage().contains(Integer.toString(TSStatusCode.PATH_NOT_EXIST.getStatusCode()))); + } + } + } + @Test public void deleteTimeSeriesAndCreateDifferentTypeTest() throws Exception { String[] retArray = new String[] {"1,1,", "2,1.1,"}; @@ -87,7 +129,7 @@ public void deleteTimeSeriesAndCreateDifferentTypeTest() throws Exception { cnt++; } } - statement.execute("DELETE timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "DELETE timeseries root.turbine1.d1.s1"); statement.execute( "create timeseries root.turbine1.d1.s1 with datatype=DOUBLE, encoding=PLAIN, compression=SNAPPY"); statement.execute("INSERT INTO root.turbine1.d1(timestamp,s1) VALUES(2,1.1)"); @@ -139,7 +181,7 @@ public void deleteTimeSeriesAndCreateSameTypeTest() throws Exception { cnt++; } } - statement.execute("DELETE timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "DELETE timeseries root.turbine1.d1.s1"); statement.execute( "create timeseries root.turbine1.d1.s1 with datatype=INT64, encoding=PLAIN, compression=SNAPPY"); statement.execute("INSERT INTO root.turbine1.d1(timestamp,s1) VALUES(2,5)"); @@ -225,7 +267,7 @@ public void deleteTimeSeriesAndAutoDeleteDeviceTest() throws Exception { Assert.assertEquals(retArray1.length, cnt); } - statement.execute("delete timeseries root.sg.d1.*"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg.d1.*"); try (ResultSet resultSet = statement.executeQuery("select * from root.sg.d1")) { Assert.assertFalse(resultSet.next()); } @@ -245,7 +287,7 @@ public void deleteTimeSeriesAndInvalidationTest() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { statement.execute("insert into root.sg.d1 (c1, c2) values (1, 1)"); - statement.execute("delete timeSeries root.sg.d1.**"); + TestUtils.deleteTimeSeries(statement, "delete timeSeries root.sg.d1.**"); try (final ResultSet resultSet = statement.executeQuery("select c1, c2 from root.sg.d1")) { Assert.assertEquals(1, resultSet.getMetaData().getColumnCount()); Assert.assertFalse(resultSet.next()); @@ -279,7 +321,7 @@ public void deleteTimeSeriesCrossSchemaRegionTest() throws Exception { Assert.assertEquals(retArray1.length, cnt); } - statement.execute("delete timeseries root.sg.*.s1"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg.*.s1"); try (ResultSet resultSet = statement.executeQuery("select s1 from root.sg.*")) { Assert.assertFalse(resultSet.next()); } @@ -329,7 +371,7 @@ public void deleteTimeSeriesCrossDatabaseTest() throws Exception { Assert.assertEquals(retArray1.length, cnt); } - statement.execute("delete timeseries root.*.d1.s1"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.*.d1.s1"); try (ResultSet resultSet = statement.executeQuery("select s1 from root.*.*")) { Assert.assertFalse(resultSet.next()); } @@ -351,7 +393,7 @@ public void deleteTimeSeriesCrossDatabaseTest() throws Exception { Assert.assertEquals(retArray1.length, cnt); } - statement.execute("delete timeseries root.sg1.d1.s2, root.sg2.**"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d1.s2, root.sg2.**"); try (ResultSet resultSet = statement.executeQuery("select s2 from root.sg1.*")) { Assert.assertFalse(resultSet.next()); } @@ -402,7 +444,7 @@ public void deleteTimeSeriesWithMultiPatternTest() throws Exception { Assert.assertEquals(retArray1.length, cnt); } - statement.execute("delete timeseries root.*.d1.s1, root.*.d1.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.*.d1.s1, root.*.d1.s2"); try (ResultSet resultSet = statement.executeQuery("select * from root.*.*")) { Assert.assertFalse(resultSet.next()); } @@ -422,7 +464,7 @@ public void deleteTemplateTimeSeriesTest() throws Exception { statement.execute("SET DEVICE TEMPLATE t1 to root.db"); statement.execute("CREATE TIMESERIES USING DEVICE TEMPLATE ON root.db.d1"); try { - statement.execute("DELETE TIMESERIES root.db.**"); + TestUtils.deleteTimeSeries(statement, "DELETE TIMESERIES root.db.**"); Assert.fail(); } catch (SQLException e) { Assert.assertTrue( @@ -432,7 +474,7 @@ public void deleteTemplateTimeSeriesTest() throws Exception { + ": Timeseries [root.db.**] does not exist or is represented by device template")); } try { - statement.execute("DELETE TIMESERIES root.db.**"); + TestUtils.deleteTimeSeries(statement, "DELETE TIMESERIES root.db.**"); Assert.fail(); } catch (SQLException e) { Assert.assertTrue( @@ -449,7 +491,7 @@ public void deleteTimeSeriesAndReturnPathNotExistsTest() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { try { - statement.execute("delete timeseries root.db.**"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.db.**"); Assert.fail(); } catch (SQLException e) { Assert.assertTrue( @@ -483,7 +525,7 @@ public void deleteTimeSeriesAndReturnPathNotExistsTest() throws Exception { } try { - statement.execute("delete timeseries root.*.d1.s3"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.*.d1.s3"); Assert.fail(); } catch (SQLException e) { Assert.assertTrue( @@ -518,7 +560,7 @@ public void dropTimeSeriesTest() throws Exception { cnt++; } } - statement.execute("DROP timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "DROP timeseries root.turbine1.d1.s1"); statement.execute("FLUSH"); try (ResultSet resultSet = statement.executeQuery("show timeseries root.turbine1.d1")) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java index 82f08cb81fc14..059610046b14d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.it.schema; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -882,7 +883,7 @@ public void showDeadbandInfo() throws SQLException { } catch (SQLException e) { fail(e.getMessage()); } finally { - statement.execute("delete timeseries root.sg1.d0.*"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d0.*"); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaTemplateIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaTemplateIT.java index f7683355073ad..7b9547e5ccd34 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaTemplateIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaTemplateIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.schema; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -462,7 +463,7 @@ public void testDeleteTimeSeriesWhenUsingTemplate() throws SQLException { } Assert.assertTrue(expectedResult.isEmpty()); - statement.execute("DELETE TIMESERIES root.**.s1"); + TestUtils.deleteTimeSeries(statement, "DELETE TIMESERIES root.**.s1"); expectedResult = new HashSet<>( @@ -634,7 +635,7 @@ public void testTemplateSetAndTimeSeriesExistenceCheck() throws SQLException { e.getMessage()); } - statement.execute("DELETE TIMESERIES root.sg1.d1.s"); + TestUtils.deleteTimeSeries(statement, "DELETE TIMESERIES root.sg1.d1.s"); statement.execute("SET DEVICE TEMPLATE t1 TO root.sg1.d1"); expectedResult = new String[] {"root.sg1.d1"}; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBShowTimeseriesOrderByTimeseriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBShowTimeseriesOrderByTimeseriesIT.java index 8218d9ef511a2..fc35fb9afb8ab 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBShowTimeseriesOrderByTimeseriesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBShowTimeseriesOrderByTimeseriesIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.schema; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -147,7 +148,7 @@ public void testDeleteSubtreeThenQueryOrder() throws Exception { prepareComplexSchema(); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("delete timeseries root.db2.devC.**"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.db2.devC.**"); } List expected = new ArrayList<>(BASE_TIMESERIES_DB2); @@ -164,7 +165,7 @@ public void testOffsetLimitAfterDeletesAndAdds() throws Exception { prepareComplexSchema(); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("delete timeseries root.db1.devB.x"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.db1.devB.x"); statement.execute( "create timeseries root.db1.devC.m0 with datatype=INT32, encoding=RLE, compression=SNAPPY"); statement.execute( diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagIT.java index a403758cc364f..3e85aeff3095c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagIT.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.it.schema; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -452,7 +453,7 @@ public void deleteTest() { } assertEquals(ret1.size(), count); - statement.execute("delete timeseries root.turbine.d7.s2"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.turbine.d7.s2"); count = 0; try (ResultSet resultSet = statement.executeQuery("show timeseries root.turbine.**")) { while (resultSet.next()) { @@ -535,7 +536,7 @@ public void deleteWithAliasTest() { } assertEquals(ret1.size(), count); - statement.execute("delete timeseries root.turbine.d7.status"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.turbine.d7.status"); count = 0; try (ResultSet resultSet = statement.executeQuery("show timeseries root.turbine.**")) { while (resultSet.next()) { @@ -855,7 +856,7 @@ public void queryWithWhereAndDeleteTest() { statement.execute(sql); } - statement.execute("delete timeseries root.turbine.d2.s0"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.turbine.d2.s0"); // with *; int count = 0; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/quota/IoTDBClusterMixQuotaIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/quota/IoTDBClusterMixQuotaIT.java index 8518ffccf3534..dda47be78a6ef 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/quota/IoTDBClusterMixQuotaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/quota/IoTDBClusterMixQuotaIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.schema.quota; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.util.AbstractSchemaIT; @@ -72,7 +73,7 @@ public void testClusterSchemaQuota2() { } // delete some timeseries and database statement.execute("delete database root.sg2;"); - statement.execute("delete timeseries root.sg1.d0.s0;"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d0.s0;"); waitForSchemaQuotaUpdate(); // now we can create 3 new timeseries and 1 new device statement.execute("SET DEVICE TEMPLATE t1 TO root.sg1.d4"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/quota/IoTDBClusterQuotaIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/quota/IoTDBClusterQuotaIT.java index 51f85a3712da0..40c79bbc6f2a9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/quota/IoTDBClusterQuotaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/quota/IoTDBClusterQuotaIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.it.schema.quota; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; @@ -123,7 +124,7 @@ public void testClusterSchemaQuota() { } // delete some timeseries and database statement.execute("delete database root.sg2;"); - statement.execute("delete timeseries root.sg1.d0.s0;"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.sg1.d0.s0;"); waitForSchemaQuotaUpdate(); // now we can create 3 new timeseries or 1 new device statement.execute("SET DEVICE TEMPLATE t1 TO root.sg1.d4"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/IoTDBTestUtilsIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/IoTDBTestUtilsIT.java index 234f59cbe0b57..50ef422f77bbe 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/IoTDBTestUtilsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/IoTDBTestUtilsIT.java @@ -24,6 +24,7 @@ import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -34,6 +35,8 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Proxy; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.util.Collections; import java.util.concurrent.TimeUnit; @@ -60,6 +63,49 @@ public static void tearDown() { EnvFactory.getEnv().cleanClusterEnvironment(); } + @Test + public void testColumnDeletionWaitsForFinalization() { + final AtomicInteger probes = new AtomicInteger(); + final ResultSet absentColumn = + (ResultSet) + Proxy.newProxyInstance( + ResultSet.class.getClassLoader(), + new Class[] {ResultSet.class}, + (proxy, method, args) -> { + if ("next".equals(method.getName())) { + return false; + } + if ("close".equals(method.getName())) { + return null; + } + throw new UnsupportedOperationException(method.getName()); + }); + final Statement statement = + (Statement) + Proxy.newProxyInstance( + Statement.class.getClassLoader(), + new Class[] {Statement.class}, + (proxy, method, args) -> { + if ("executeQuery".equals(method.getName())) { + return absentColumn; + } + if ("execute".equals(method.getName())) { + assertEquals( + "ALTER TABLE test.t DROP COLUMN IF EXISTS \"removed\"\"column\"", args[0]); + if (probes.incrementAndGet() == 1) { + throw new SQLException( + "deletion still finalizing", + null, + TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()); + } + return false; + } + throw new UnsupportedOperationException(method.getName()); + }); + TestUtils.waitForColumnDeletion(statement, "test.t", "removed\"column"); + assertEquals(2, probes.get()); + } + @Test public void testAssertAlwaysFail() { assertObservationPeriod( diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java index b9e9512970568..e1192746515c9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java @@ -21,7 +21,14 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.cluster.RegionStatus; +import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropColumn; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.DeleteTimeSeriesStatement; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; @@ -63,6 +70,7 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.stream.Collectors; import static org.apache.iotdb.itbase.constant.TestConstant.DELTA; import static org.apache.iotdb.itbase.constant.TestConstant.NULL; @@ -147,6 +155,156 @@ public static String defaultFormatDataTime(long time) { RpcUtils.DEFAULT_TIME_FORMAT, TIME_PRECISION_IN_MS, time, DEFAULT_ZONE_ID); } + /** Submits deletion using the caller's credentials, then waits for its durable task. */ + public static void deleteTimeSeries(final Statement statement, final String sql) + throws SQLException { + statement.execute(sql); + waitForTimeSeriesDeletion(EnvFactory.getEnv(), sql); + } + + public static void deleteTimeSeries(final ISession session, final String sql) + throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement(sql); + try { + waitForTimeSeriesDeletion(EnvFactory.getEnv(), sql); + } catch (SQLException e) { + throw new StatementExecutionException(e); + } + } + + public static void deleteTimeSeries(final ITableSession session, final String sql) + throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement(sql); + try { + waitForTimeSeriesDeletion(EnvFactory.getEnv(), sql); + } catch (SQLException e) { + throw new StatementExecutionException(e); + } + } + + public static void waitForColumnDeletion( + final BaseEnv env, final String database, final String table, final String column) + throws SQLException { + try (final Connection connection = env.getConnection(TABLE_SQL_DIALECT); + final Statement statement = createSingleNodeStatement(connection)) { + waitForColumnDeletion(statement, database + "." + table, column); + } + } + + public static void waitForTimeSeriesDeletion(final BaseEnv env, final String sql) + throws SQLException { + // Use the administrator only for diagnostics; submission keeps the original user's authority. + try (final Connection connection = env.getConnection(); + final Statement statement = createSingleNodeStatement(connection)) { + waitForTimeSeriesDeletion(statement, sql); + } + } + + public static void waitForTimeSeriesDeletion(final Statement statement, final String sql) + throws SQLException { + final DeleteTimeSeriesStatement deletion = + (DeleteTimeSeriesStatement) StatementGenerator.createStatement(sql, DEFAULT_ZONE_ID); + final PathPatternTree tree = new PathPatternTree(); + deletion.getPathPatternList().forEach(tree::appendPathPattern); + tree.constructTree(); + final String paths = tree.getAllPathPatterns().toString(); + await() + .atMost(60, TimeUnit.SECONDS) + .pollInSameThread() + .pollInterval(20, TimeUnit.MILLISECONDS) + .until( + () -> { + long latestId = -1; + String state = null; + String error = null; + try (final ResultSet result = + statement.executeQuery("SHOW DELETE TIMESERIES PROGRESS")) { + while (result.next()) { + if (paths.equals(result.getString("Paths")) + && result.getLong("ProcedureId") > latestId) { + latestId = result.getLong("ProcedureId"); + state = result.getString("State"); + error = result.getString("ErrorMessage"); + } + } + } + if ("ROLLEDBACK".equals(state)) { + throw new SQLException(error); + } + return "SUCCESS".equals(state); + }); + } + + /** Waits for column removal and for the durable task to release conflicting table changes. */ + public static void waitForColumnDeletion( + final Statement statement, final String table, final String column) { + await() + .atMost(60, TimeUnit.SECONDS) + .pollInSameThread() + .pollInterval(20, TimeUnit.MILLISECONDS) + .until( + () -> { + try (final ResultSet result = + statement.executeQuery("DESCRIBE " + table + " DETAILS")) { + while (result.next()) { + if (column.equals(result.getString(1))) { + return false; + } + } + } + // Schema removal precedes the terminal procedure store update. No read-only schema + // query exposes root cleanup, so probe the absent column with an idempotent no-op. + try { + statement.execute( + "ALTER TABLE " + + table + + " DROP COLUMN IF EXISTS \"" + + column.replace("\"", "\"\"") + + "\""); + return true; + } catch (SQLException e) { + if (e.getErrorCode() == TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) { + return false; + } + throw e; + } + }); + } + + public static void dropColumn( + final Statement statement, final String sql, final String table, final String column) + throws SQLException { + statement.execute(sql); + try (final Statement progress = createSingleNodeStatement(statement.getConnection())) { + waitForColumnDeletion(progress, table, column); + } + } + + public static void dropColumn(final Statement statement, final String sql) throws SQLException { + final DropColumn drop = + (DropColumn) new SqlParser().createStatement(sql, DEFAULT_ZONE_ID, null); + final String table = + drop.getTable().getParts().stream() + .map(part -> "\"" + part.replace("\"", "\"\"") + "\"") + .collect(Collectors.joining(".")); + dropColumn(statement, sql, table, drop.getField().getValue()); + } + + private static void executeNonQueryAndWait( + final BaseEnv env, final Statement statement, final String sql, final String dialect) + throws SQLException { + if (TABLE_SQL_DIALECT.equals(dialect) + && sql.matches("(?is)^\\s*ALTER\\s+TABLE\\s+.*\\s+DROP\\s+COLUMN\\s+.*")) { + dropColumn(statement, sql); + } else { + statement.execute(sql); + if (!TABLE_SQL_DIALECT.equals(dialect) + && sql.matches("(?is)^\\s*(DELETE|DROP)\\s+TIMESERIES\\s+(?!OF\\b)\\S.*")) { + waitForTimeSeriesDeletion(env, sql); + } + } + } + public static void prepareData(String[] sqls) { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -1086,7 +1244,7 @@ public static void executeNonQuery(String sql) { public static void executeNonQuery(String sql, String userName, String password) { try (Connection connection = EnvFactory.getEnv().getConnection(userName, password); Statement statement = connection.createStatement()) { - statement.execute(sql); + executeNonQueryAndWait(EnvFactory.getEnv(), statement, sql, TREE_SQL_DIALECT); } catch (SQLException e) { e.printStackTrace(); fail(e.getMessage()); @@ -1174,7 +1332,7 @@ public static void executeNonQueriesWithRetry( } for (int i = lastIndex; i < sqlList.size(); ++i) { lastIndex = i; - statement.execute(sqlList.get(i)); + executeNonQueryAndWait(env, statement, sqlList.get(i), sqlDialect); } return; } catch (final SQLException e) { @@ -1330,7 +1488,7 @@ public static void executeNonQueries( statement.execute("use " + dataBase); } for (int i = lastIndex; i < sqlList.size(); ++i) { - statement.execute(sqlList.get(i)); + executeNonQueryAndWait(env, statement, sqlList.get(i), sqlDialect); } } catch (SQLException e) { // the default connection should be closed by the upper level @@ -1409,7 +1567,7 @@ public static boolean tryExecuteNonQueriesWithRetry( } for (int i = lastIndex; i < sqlList.size(); ++i) { lastIndex = i; - statement.execute(sqlList.get(i)); + executeNonQueryAndWait(env, statement, sqlList.get(i), sqlDialect); } return true; } catch (SQLException e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 2af7b3720749a..0af08a558402c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -3508,7 +3508,7 @@ public void testDeleteDataByTag() throws IoTDBConnectionException, StatementExec } @Test - public void testDropAndAlter() throws IoTDBConnectionException, StatementExecutionException { + public void testDropAndAlter() throws Exception { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS drop_and_alter (s1 int32)"); @@ -3538,6 +3538,7 @@ public void testDropAndAlter() throws IoTDBConnectionException, StatementExecuti tablet.reset(); session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); + TestUtils.waitForColumnDeletion(EnvFactory.getEnv(), "test", "drop_and_alter", "s1"); // time=3 and time=4 are STRING tablet = @@ -3565,6 +3566,7 @@ public void testDropAndAlter() throws IoTDBConnectionException, StatementExecuti tablet.reset(); session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); + TestUtils.waitForColumnDeletion(EnvFactory.getEnv(), "test", "drop_and_alter", "s1"); session.executeNonQueryStatement("ALTER TABLE drop_and_alter ADD COLUMN s1 TEXT"); // time=5 and time=6 are TEXT diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBFlushQueryTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBFlushQueryTableIT.java index 85af3ad0ff7c0..0756b924c3a85 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBFlushQueryTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBFlushQueryTableIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.relational.it.db.it; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; @@ -202,7 +203,7 @@ public void testFlushTableAfterDropColumn() { statement.execute("use db1"); statement.execute("create table t2(s1 text field, s2 text field)"); statement.execute("insert into t2(time,s2) values(2,'t1')"); - statement.execute("alter table t2 drop column s2"); + TestUtils.dropColumn(statement, "alter table t2 drop column s2"); statement.execute("flush"); statement.execute("insert into t2(time,s1) values(2,'t1')"); } catch (Exception e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java index d24c4737c25e5..206454a19c002 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java @@ -363,7 +363,7 @@ public void testRecoverWALMismatchDataType() throws Exception { Statement statement = connection.createStatement()) { statement.execute("use \"test\""); statement.execute("insert into root.turbine1.d1(time,s1,s2) values(1,1.1,2.2)"); - statement.execute("delete timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.turbine1.d1.s1"); statement.execute( "create timeseries root.turbine1.d1.s1 with datatype=INT32, encoding=RLE, compression=SNAPPY"); } @@ -396,7 +396,7 @@ public void testRecoverWALDeleteSchema() throws Exception { Statement statement = connection.createStatement()) { statement.execute("use \"test\""); statement.execute("insert into turbine1(time,tag1,s1,s2) values(1,\'d1\',1.1,2.2)"); - statement.execute("delete timeseries root.turbine1.d1.s1"); + TestUtils.deleteTimeSeries(statement, "delete timeseries root.turbine1.d1.s1"); } TestUtils.restartDataNodes(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java index 00c3df4f39078..0fc6c7e6996ab 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java @@ -20,6 +20,7 @@ package org.apache.iotdb.relational.it.schema; import org.apache.iotdb.commons.utils.MetadataUtils; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.ITableSession; @@ -517,7 +518,7 @@ public void testAlterWrongType() throws IoTDBConnectionException, StatementExecu } @Test - public void testDropAndAlter() throws IoTDBConnectionException, StatementExecutionException { + public void testDropAndAlter() throws Exception { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { session.executeNonQueryStatement("SET CONFIGURATION enable_unseq_space_compaction='false'"); session.executeNonQueryStatement("SET CONFIGURATION enable_seq_space_compaction='false'"); @@ -549,6 +550,7 @@ public void testDropAndAlter() throws IoTDBConnectionException, StatementExecuti tablet.reset(); session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); + TestUtils.waitForColumnDeletion(EnvFactory.getEnv(), "test", "drop_and_alter", "s1"); // time=3 and time=4 are STRING tablet = @@ -1147,7 +1149,7 @@ public void testAlterViewType() throws IoTDBConnectionException, StatementExecut } try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - session.executeNonQueryStatement("DELETE TIMESERIES root.db.battery.**"); + TestUtils.deleteTimeSeries(session, "DELETE TIMESERIES root.db.battery.**"); } } @@ -1332,7 +1334,7 @@ public void testLoadAndAccumulator() resourceFile.delete(); }); filesToLoad.clear(); - session.executeNonQueryStatement("DELETE TIMESERIES root.sg1.d1.s1"); + TestUtils.deleteTimeSeries(session, "DELETE TIMESERIES root.sg1.d1.s1"); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBNullableAttributeCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBNullableAttributeCacheIT.java index f431800bc6984..b945dad7fc1ea 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBNullableAttributeCacheIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBNullableAttributeCacheIT.java @@ -25,6 +25,7 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.relational.it.schema.fault.MetadataFaultController.Node; +import org.apache.iotdb.rpc.TSStatusCode; import org.awaitility.Awaitility; import org.junit.Test; @@ -44,6 +45,7 @@ import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @RunWith(IoTDBTestRunner.class) @@ -112,7 +114,7 @@ private void invalidationFailure(boolean loseAck) throws Exception { () -> { assertLease(target, lease); if (ddl.isDone()) { - assertInvalidationFailure(ddl.get()); + assertNull("The asynchronous drop must be accepted", ddl.get()); } for (Node node : NODES.values()) { assertEquals( @@ -133,12 +135,37 @@ private void invalidationFailure(boolean loseAck) throws Exception { assertTrue(target.count(INVALIDATE) > 0); } target.release(hook); - SQLException failure = ddl.get(30, TimeUnit.SECONDS); - if (failure != null) { - assertInvalidationFailure(failure); - // DropTableColumnProcedure does not roll back its pre-delete marker. Reissuing the same - // drop is supported: preDeleteColumn adds that marker idempotently and completes the drop. - update("ALTER TABLE sensor DROP COLUMN changing"); + assertNull("The asynchronous drop must be accepted", ddl.get(30, TimeUnit.SECONDS)); + try (Connection connection = connection(remote); + Statement statement = connection.createStatement()) { + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(60, TimeUnit.SECONDS) + .until( + () -> { + boolean present = false; + try (ResultSet result = + statement.executeQuery("DESCRIBE " + database + ".sensor DETAILS")) { + while (result.next()) { + present |= "changing".equals(result.getString(1)); + } + } + if (!present) { + return true; + } + // A failed task retains its pre-delete marker. Retrying the drop after restoring + // the RPC completes it; a still-running task rejects the overlapping request. + try { + update("ALTER TABLE sensor DROP COLUMN changing"); + } catch (SQLException e) { + assertTrue( + e.getMessage(), + e.getErrorCode() == TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() + || e.getErrorCode() == TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode()); + } + return false; + }); + TestUtils.waitForColumnDeletion(statement, database + ".sensor", "changing"); } TestUtils.tableResultSetEqualTestUnordered( "SELECT device, kept, reading FROM sensor", diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java index 39bcd6d46f540..42cf12b7b38a7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java @@ -276,6 +276,9 @@ private void assertStatementEffect( final String message) throws Exception { statement.execute(sql); + if (sql.startsWith("DELETE TIMESERIES ")) { + TestUtils.waitForTimeSeriesDeletion(EnvFactory.getEnv(), sql); + } assertTrue(message, effect.call()); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index 078a6a71a71cd..27e6d988d0168 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -542,7 +542,7 @@ public void testManageTable() { "insert into table2(region_id, plant_id, color, temperature, speed) values(1, 1, 1, 1, 1)"); // Test drop column - statement.execute("alter table table2 drop column color"); + TestUtils.dropColumn(statement, "alter table table2 drop column color"); // Test comment // Before @@ -599,7 +599,7 @@ public void testManageTable() { assertEquals(columnNames.length, cnt); } - statement.execute("alter table table2 drop column speed"); + TestUtils.dropColumn(statement, "alter table table2 drop column speed"); try { statement.executeQuery("select color from table2"); @@ -616,13 +616,13 @@ public void testManageTable() { } try { - statement.execute("alter table table2 drop column speed"); + TestUtils.dropColumn(statement, "alter table table2 drop column speed"); } catch (final SQLException e) { assertEquals("616: Column speed in table 'test2.table2' does not exist.", e.getMessage()); } try { - statement.execute("alter table table2 drop column t1"); + TestUtils.dropColumn(statement, "alter table table2 drop column t1"); } catch (final SQLException e) { assertEquals("701: Dropping tag or time column is not supported.", e.getMessage()); } @@ -685,7 +685,7 @@ public void testManageTable() { } try { - statement.execute("alter table test1.test drop column a"); + TestUtils.dropColumn(statement, "alter table test1.test drop column a"); fail(); } catch (final SQLException e) { assertEquals("500: Unknown database test1", e.getMessage()); @@ -959,7 +959,8 @@ public void testConcurrentAutoCreateAndDropColumn() throws Exception { () -> { for (int i = 0; i < 100; ++i) { try { - adminStmt.execute(String.format("alter table db1.table8 drop column m%s", i)); + TestUtils.dropColumn( + adminStmt, String.format("alter table db1.table8 drop column m%s", i)); } catch (final SQLException e) { throw new RuntimeException(e); } @@ -1111,7 +1112,7 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("drop timeSeries root.`重庆`.`1`.b.`1`.s1"); + TestUtils.deleteTimeSeries(statement, "drop timeSeries root.`重庆`.`1`.b.`1`.s1"); statement.execute("create device template t1 (S1 boolean, s9 int32)"); statement.execute("set schema template t1 to root.`重庆`.`1`.b.`1`"); statement.execute("create timeSeries root.`重庆`.`1`.b.`2`.f.g.h.S1 int32"); @@ -1387,6 +1388,41 @@ public void testTreeViewTable() throws Exception { } } + @Test + public void testAsyncDropColumn() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement(); + Statement progress = TestUtils.createSingleNodeStatement(connection)) { + statement.execute("CREATE DATABASE async_drop_column_db"); + try { + statement.execute("USE async_drop_column_db"); + statement.execute( + "CREATE TABLE t (device STRING TAG, attr STRING ATTRIBUTE, removed INT32, kept INT32)"); + statement.execute("INSERT INTO t(time,device,attr,removed,kept) VALUES(1,'d','a',11,21)"); + statement.execute("FLUSH"); + statement.execute("INSERT INTO t(time,device,attr,removed,kept) VALUES(2,'d','a',12,22)"); + statement.execute("ALTER TABLE t DROP COLUMN removed"); + TestUtils.waitForColumnDeletion(progress, "async_drop_column_db.t", "removed"); + statement.execute("ALTER TABLE t ADD COLUMN removed STRING"); + try (ResultSet result = + statement.executeQuery("SELECT removed,kept,attr FROM t ORDER BY time")) { + TestUtils.assertResultSetEqual( + result, "removed,kept,attr,", new String[] {"null,21,a,", "null,22,a,"}); + } + // Attribute deletion has a different region/cache path from field deletion. + statement.execute("ALTER TABLE t DROP COLUMN attr"); + TestUtils.waitForColumnDeletion(progress, "async_drop_column_db.t", "attr"); + statement.execute("ALTER TABLE t ADD COLUMN attr STRING ATTRIBUTE"); + try (ResultSet result = statement.executeQuery("SELECT attr,kept FROM t ORDER BY time")) { + TestUtils.assertResultSetEqual( + result, "attr,kept,", new String[] {"null,21,", "null,22,"}); + } + } finally { + statement.execute("DROP DATABASE async_drop_column_db"); + } + } + } + @Test public void testAsyncDropTableProgress() throws Exception { try (final Connection connection = diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java index 101f1be9b731e..0aba39c638e56 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java @@ -65,6 +65,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.SecureRandom; +import java.sql.SQLException; import java.sql.Timestamp; import java.time.LocalDate; import java.util.ArrayList; @@ -820,6 +821,14 @@ public void deleteNonExistTimeSeriesTest() { try { session.deleteTimeseries( Arrays.asList("root.sg1.d1.t6", "root.sg1.d1.t2", "root.sg1.d1.t3")); + try { + TestUtils.waitForTimeSeriesDeletion( + EnvFactory.getEnv(), + "DELETE TIMESERIES root.sg1.d1.t6, root.sg1.d1.t2, root.sg1.d1.t3"); + fail("Expected asynchronous deletion to fail for a nonexistent timeseries"); + } catch (SQLException e) { + assertTrue(e.getMessage().contains("Path [root.sg1.d1.t6] does not exist;")); + } } catch (BatchExecutionException e) { assertTrue(e.getMessage().contains("Path [root.sg1.d1.t6] does not exist;")); } diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 4424d31843a8a..65a64cac4051a 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -97,6 +97,7 @@ utilityStatement | revokeWatermarkEmbedding | loadConfiguration | loadTimeseries | loadFile | removeFile | unloadFile | setSqlDialect | showCurrentSqlDialect | showCurrentUser | repairDataPartitionTable | showRepairDataPartitionTableProgress + | showDeleteTimeSeriesProgress ; /** @@ -1297,6 +1298,10 @@ showRepairDataPartitionTableProgress : SHOW REPAIR DATA PARTITION TABLE PROGRESS ; +showDeleteTimeSeriesProgress + : SHOW DELETE TIMESERIES PROGRESS + ; + // Explain explain : EXPLAIN (ANALYZE VERBOSE?)? selectStatement? diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index a6df250eed70d..a375c013774b4 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -1056,4 +1056,14 @@ private ManagerMessages() {} EXCEPTION_REJECT_REGISTER_PLEASE_ENSURE_THAT_THE_PARAMETER_ARG_IS_CONSISTENT_WITH_THE_SEED_CONFIGNODE_C072CE7C = "Reject register, please ensure that the parameter %s is consistent with the" + " Seed-ConfigNode."; + public static final String MESSAGE_TIMESERIES_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_USE_SHOW_DELETE_TIMESERIES_PROGRESS_TO_CHECK_ITS_PROGRESS_75958076 = + "Timeseries deletion task %s has been submitted for asynchronous execution. Use SHOW DELETE TIMESERIES PROGRESS to check its progress."; + public static final String MESSAGE_COLUMN_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_USE_DESCRIBE_DETAILS_TO_CHECK_ITS_PROGRESS_A0578678 = + "Column deletion task %s has been submitted for asynchronous execution. Use DESCRIBE ... DETAILS to check its progress."; + public static final String MESSAGE_TABLE_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_QUERY_INFORMATION_SCHEMA_DROP_TABLE_PROCEDURES_TO_CHECK_ITS_PROGRESS_D7AB21D3 = + "Table deletion task %s has been submitted for asynchronous execution. Query information_schema.drop_table_procedures to check its progress."; + public static final String MESSAGE_THE_ASYNCHRONOUS_TIMESERIES_DELETION_QUEUE_IS_FULL_ARG_WAITING_TASKS_RETRY_LATER_93F6F407 = + "The asynchronous timeseries deletion queue is full (%s waiting tasks). Retry later."; + public static final String MESSAGE_SOME_OTHER_ASYNCHRONOUS_TASK_IS_DELETING_TARGET_TIMESERIES_USE_SHOW_DELETE_TIMESERIES_PROGRESS_TO_CHECK_ITS_PROGRESS_AND_RETRY_LATER_A799B41E = + "Some other asynchronous task is deleting target timeseries. Use SHOW DELETE TIMESERIES PROGRESS to check its progress and retry later."; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 34aa75da47957..f99384704ae12 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -1004,4 +1004,14 @@ private ManagerMessages() {} public static final String EXCEPTION_REJECT_REGISTER_PLEASE_ENSURE_THAT_THE_PARAMETER_ARG_IS_CONSISTENT_WITH_THE_SEED_CONFIGNODE_C072CE7C = "拒绝注册，请确保参数 %s 与 Seed-ConfigNode 一致。"; + public static final String MESSAGE_TIMESERIES_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_USE_SHOW_DELETE_TIMESERIES_PROGRESS_TO_CHECK_ITS_PROGRESS_75958076 = + "时间序列删除任务 %s 已提交，将异步执行。请使用 SHOW DELETE TIMESERIES PROGRESS 查看进度。"; + public static final String MESSAGE_COLUMN_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_USE_DESCRIBE_DETAILS_TO_CHECK_ITS_PROGRESS_A0578678 = + "列删除任务 %s 已提交，将异步执行。请使用 DESCRIBE ... DETAILS 查看进度。"; + public static final String MESSAGE_TABLE_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_QUERY_INFORMATION_SCHEMA_DROP_TABLE_PROCEDURES_TO_CHECK_ITS_PROGRESS_D7AB21D3 = + "表删除任务 %s 已提交，将异步执行。请查询 information_schema.drop_table_procedures 查看进度。"; + public static final String MESSAGE_THE_ASYNCHRONOUS_TIMESERIES_DELETION_QUEUE_IS_FULL_ARG_WAITING_TASKS_RETRY_LATER_93F6F407 = + "异步时间序列删除队列已满（%s 个等待任务），请稍后重试。"; + public static final String MESSAGE_SOME_OTHER_ASYNCHRONOUS_TASK_IS_DELETING_TARGET_TIMESERIES_USE_SHOW_DELETE_TIMESERIES_PROGRESS_TO_CHECK_ITS_PROGRESS_AND_RETRY_LATER_A799B41E = + "其他异步任务正在删除目标时间序列。请使用 SHOW DELETE TIMESERIES PROGRESS 查看进度，并稍后重试。"; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index c5facfb371ece..517c8dd16e90e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -250,6 +250,15 @@ public class ConfigNodeConfig { private int procedureCoreWorkerThreadsCount = Math.max(Runtime.getRuntime().availableProcessors() / 4, 16); + /** Maximum number of tree-model timeseries deletion tasks waiting to execute. */ + private int timeseriesDeletionQueueCapacity = 1024; + + /** Minimum interval between starting tree-model timeseries deletion tasks, in milliseconds. */ + private long timeseriesDeletionExecutionIntervalMs = 1000; + + /** Maximum number of tree-model timeseries deletion tasks executing concurrently. */ + private int timeseriesDeletionMaxConcurrentTasks = 1; + /** Thread pool size for publishing cluster load statistics changes. */ private int loadStatisticsPublisherThreadCount = 5; @@ -793,6 +802,39 @@ public int getLoadStatisticsPublisherThreadCount() { return loadStatisticsPublisherThreadCount; } + public int getTimeseriesDeletionQueueCapacity() { + return timeseriesDeletionQueueCapacity; + } + + public void setTimeseriesDeletionQueueCapacity(int capacity) { + if (capacity <= 0) { + throw new IllegalArgumentException(CommonMessages.SIZE_MUST_BE_POSITIVE); + } + timeseriesDeletionQueueCapacity = capacity; + } + + public long getTimeseriesDeletionExecutionIntervalMs() { + return timeseriesDeletionExecutionIntervalMs; + } + + public void setTimeseriesDeletionExecutionIntervalMs(long intervalMs) { + if (intervalMs <= 0) { + throw new IllegalArgumentException(CommonMessages.SIZE_MUST_BE_POSITIVE); + } + timeseriesDeletionExecutionIntervalMs = intervalMs; + } + + public int getTimeseriesDeletionMaxConcurrentTasks() { + return timeseriesDeletionMaxConcurrentTasks; + } + + public void setTimeseriesDeletionMaxConcurrentTasks(int maxConcurrentTasks) { + if (maxConcurrentTasks <= 0) { + throw new IllegalArgumentException(CommonMessages.SIZE_MUST_BE_POSITIVE); + } + timeseriesDeletionMaxConcurrentTasks = maxConcurrentTasks; + } + public void setLoadStatisticsPublisherThreadCount(int loadStatisticsPublisherThreadCount) { if (loadStatisticsPublisherThreadCount <= 0) { throw new IllegalArgumentException(CommonMessages.SIZE_MUST_BE_POSITIVE); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index e9ab7657c50a6..e01d634b76fd5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -402,6 +402,22 @@ private void loadProperties(TrimProperties properties) throws BadNodeUrlExceptio "procedure_core_worker_thread_count", String.valueOf(conf.getProcedureCoreWorkerThreadsCount())))); + conf.setTimeseriesDeletionQueueCapacity( + Integer.parseInt( + properties.getProperty( + "timeseries_deletion_queue_capacity", + String.valueOf(conf.getTimeseriesDeletionQueueCapacity())))); + conf.setTimeseriesDeletionExecutionIntervalMs( + Long.parseLong( + properties.getProperty( + "timeseries_deletion_execution_interval_ms", + String.valueOf(conf.getTimeseriesDeletionExecutionIntervalMs())))); + conf.setTimeseriesDeletionMaxConcurrentTasks( + Integer.parseInt( + properties.getProperty( + "timeseries_deletion_max_concurrent_tasks", + String.valueOf(conf.getTimeseriesDeletionMaxConcurrentTasks())))); + conf.setLoadStatisticsPublisherThreadCount( Integer.parseInt( properties.getProperty( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 8b0b2d4751fe0..f862d388eaf5c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -50,13 +50,11 @@ import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.enums.RepairDataPartitionTableProgressState; import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.i18n.PipeMessages; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapPseudoTPipeTransferRequest; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; @@ -79,7 +77,6 @@ import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.conf.SystemPropertiesUtils; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.read.ainode.GetAINodeConfigurationPlan; import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; @@ -147,7 +144,6 @@ import org.apache.iotdb.confignode.persistence.quota.QuotaInfo; import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo; import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo; -import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.rpc.thrift.TAINodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartReq; import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartResp; @@ -246,6 +242,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodes4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowDeleteTimeSeriesProceduresResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDropTableProceduresResp; import org.apache.iotdb.confignode.rpc.thrift.TShowPipePluginReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; @@ -732,7 +729,13 @@ public TSStatus setSchemaReplicationFactor( SetSchemaReplicationFactorPlan setSchemaReplicationFactorPlan) { TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return clusterSchemaManager.setSchemaReplicationFactor(setSchemaReplicationFactorPlan); + synchronized (procedureManager) { + if (procedureManager.hasPendingFullDatabaseTimeSeriesDeletion( + setSchemaReplicationFactorPlan.getDatabase())) { + return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } + return clusterSchemaManager.setSchemaReplicationFactor(setSchemaReplicationFactorPlan); + } } else { return status; } @@ -743,7 +746,13 @@ public TSStatus setDataReplicationFactor( SetDataReplicationFactorPlan setDataReplicationFactorPlan) { TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return clusterSchemaManager.setDataReplicationFactor(setDataReplicationFactorPlan); + synchronized (procedureManager) { + if (procedureManager.hasPendingFullDatabaseTimeSeriesDeletion( + setDataReplicationFactorPlan.getDatabase())) { + return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } + return clusterSchemaManager.setDataReplicationFactor(setDataReplicationFactorPlan); + } } else { return status; } @@ -754,7 +763,13 @@ public TSStatus setTimePartitionInterval( SetTimePartitionIntervalPlan setTimePartitionIntervalPlan) { TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return clusterSchemaManager.setTimePartitionInterval(setTimePartitionIntervalPlan); + synchronized (procedureManager) { + if (procedureManager.hasPendingFullDatabaseTimeSeriesDeletion( + setTimePartitionIntervalPlan.getDatabase())) { + return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } + return clusterSchemaManager.setTimePartitionInterval(setTimePartitionIntervalPlan); + } } else { return status; } @@ -800,7 +815,13 @@ public DataSet showTTL(ShowTTLPlan showTTLPlan) { public TSStatus setDatabase(final DatabaseSchemaPlan databaseSchemaPlan) { final TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return clusterSchemaManager.setDatabase(databaseSchemaPlan, false); + synchronized (procedureManager) { + if (procedureManager.hasPendingFullDatabaseTimeSeriesDeletion( + databaseSchemaPlan.getSchema().getName())) { + return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } + return clusterSchemaManager.setDatabase(databaseSchemaPlan, false); + } } else { return status; } @@ -810,7 +831,13 @@ public TSStatus setDatabase(final DatabaseSchemaPlan databaseSchemaPlan) { public TSStatus alterDatabase(final DatabaseSchemaPlan databaseSchemaPlan) { final TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return clusterSchemaManager.alterDatabase(databaseSchemaPlan, false); + synchronized (procedureManager) { + if (procedureManager.hasPendingFullDatabaseTimeSeriesDeletion( + databaseSchemaPlan.getSchema().getName())) { + return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } + return clusterSchemaManager.alterDatabase(databaseSchemaPlan, false); + } } else { return status; } @@ -2511,92 +2538,14 @@ public TSStatus alterEncodingCompressor(final TAlterEncodingCompressorReq req) { @Override public TSStatus deleteTimeSeries(TDeleteTimeSeriesReq req) { - TSStatus status = confirmLeader(); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - String queryId = req.getQueryId(); - PathPatternTree rawPatternTree = - PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree())); - boolean isGeneratedByPipe = req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe(); - /** - * If delete pattern is prefix path (such as root.db.**), it may be optimized to delete - * database plus create database. We need to determine two conditions: whether the pattern - * ends in **, and that the device is a full path and is matched in the ConfigMTree. - */ - boolean canOptimize = false; - HashSet deleteDatabaseSchemas = new HashSet<>(); - List deleteTimeSeriesPatternPaths = new ArrayList<>(); - List deleteDatabasePatternPaths = new ArrayList<>(); - for (PartialPath path : rawPatternTree.getAllPathPatterns()) { - if (PathPatternUtil.isMultiLevelMatchWildcard(path.getMeasurement()) - && !path.getDevicePath().hasWildcard()) { - Map databaseSchemaMap = - getClusterSchemaManager().getMatchedDatabaseSchemasByPrefix(path.getDevicePath()); - // root.__audit can never be deleted - databaseSchemaMap.remove(TREE_MODEL_AUDIT_DATABASE); - if (!databaseSchemaMap.isEmpty()) { - deleteDatabaseSchemas.addAll(databaseSchemaMap.values()); - deleteDatabasePatternPaths.add(path); - canOptimize = true; - continue; - } - } - deleteTimeSeriesPatternPaths.add(path); - } - if (!canOptimize) { - return procedureManager.deleteTimeSeries( - queryId, - rawPatternTree, - isGeneratedByPipe, - req.isSetMayDeleteAudit() && req.isMayDeleteAudit()); - } - // check if the database is using template - try { - SchemaUtils.checkSchemaRegionUsingTemplate(this, deleteDatabasePatternPaths); - } catch (MetadataException e) { - return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); - } - if (!deleteTimeSeriesPatternPaths.isEmpty()) { - // 1. delete time series that can not be optimized - PathPatternTree deleteTimeSeriesPatternTree = new PathPatternTree(); - for (PartialPath path : deleteTimeSeriesPatternPaths) { - deleteTimeSeriesPatternTree.appendPathPattern(path); - } - deleteTimeSeriesPatternTree.constructTree(); - status = - procedureManager.deleteTimeSeries( - queryId, - deleteTimeSeriesPatternTree, - isGeneratedByPipe, - req.isSetMayDeleteAudit() && req.isMayDeleteAudit()); - } - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - // 2. delete database - List failedStatus = new ArrayList<>(); - status = - procedureManager.deleteDatabases( - new ArrayList<>(deleteDatabaseSchemas), isGeneratedByPipe); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - failedStatus.add(status); - } - // 3. create database whatever the delete database operation is successful or not - for (TDatabaseSchema databaseSchema : deleteDatabaseSchemas) { - status = - clusterSchemaManager.setDatabase( - new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, databaseSchema), - isGeneratedByPipe); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - failedStatus.add(status); - } - } - // 4. squash or generate status - if (!failedStatus.isEmpty()) { - status = RpcUtils.squashResponseStatusList(failedStatus); - } else { - status = StatusUtils.OK; - } - } - } - return status; + final TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? procedureManager.deleteTimeSeries( + req.getQueryId(), + PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree())), + req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe(), + req.isSetMayDeleteAudit() && req.isMayDeleteAudit()) + : status; } @Override @@ -3498,6 +3447,15 @@ public TShowTable4InformationSchemaResp showTables4InformationSchema() { : new TShowTable4InformationSchemaResp(status); } + @Override + public TShowDeleteTimeSeriesProceduresResp showDeleteTimeSeriesProcedures() { + final TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? new TShowDeleteTimeSeriesProceduresResp(status) + .setProcedureInfoList(procedureManager.getDeleteTimeSeriesProcedures()) + : new TShowDeleteTimeSeriesProceduresResp(status); + } + @Override public TShowDropTableProceduresResp showDropTableProcedures() { final TSStatus status = confirmLeader(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 9d2ea4c725d03..a059f0d73293a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -153,6 +153,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodes4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowDeleteTimeSeriesProceduresResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDropTableProceduresResp; import org.apache.iotdb.confignode.rpc.thrift.TShowPipePluginReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; @@ -929,6 +930,8 @@ TPermissionInfoResp login( TShowTable4InformationSchemaResp showTables4InformationSchema(); + TShowDeleteTimeSeriesProceduresResp showDeleteTimeSeriesProcedures(); + TShowDropTableProceduresResp showDropTableProcedures(); TDescTableResp describeTable( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index f01681b6b9d64..e593c72fa87bd 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -32,6 +32,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.exception.table.ColumnNotExistsException; import org.apache.iotdb.commons.exception.table.TableNotExistsException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; @@ -43,6 +44,8 @@ import org.apache.iotdb.commons.schema.table.TableNodeStatus; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; @@ -60,6 +63,7 @@ import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; +import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.manager.partition.PartitionManager; @@ -165,6 +169,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp; +import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesProcedureInfo; import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTableProcedureInfo; import org.apache.iotdb.confignode.rpc.thrift.TExtendRegionReq; @@ -197,6 +202,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -324,17 +330,41 @@ public TSStatus testSubProcedure() { public TSStatus deleteDatabases( final List deleteSgSchemaList, final boolean isGeneratedByPipe) { + return deleteDatabases(deleteSgSchemaList, isGeneratedByPipe, PROCEDURE_WAIT_TIME_OUT); + } + + public TSStatus deleteDatabases( + final List deleteSgSchemaList, + final boolean isGeneratedByPipe, + final long waitTimeout) { + return deleteDatabases(deleteSgSchemaList, isGeneratedByPipe, waitTimeout, null); + } + + public TSStatus deleteDatabases( + final List deleteSgSchemaList, + final boolean isGeneratedByPipe, + final long waitTimeout, + final @Nullable String deletingTimeSeriesQueryId) { final List procedures = new ArrayList<>(); final long startCheckTimeForProcedures = System.currentTimeMillis(); for (final TDatabaseSchema databaseSchema : deleteSgSchemaList) { final String database = databaseSchema.getName(); boolean hasOverlappedTask = false; synchronized (this) { + if (deletingTimeSeriesQueryId == null + && hasPendingFullDatabaseTimeSeriesDeletion(database)) { + return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } while (executor.isRunning() - && System.currentTimeMillis() - startCheckTimeForProcedures < PROCEDURE_WAIT_TIME_OUT) { + && System.currentTimeMillis() - startCheckTimeForProcedures < waitTimeout) { final Pair procedureIdDuplicatePair = checkDuplicateTableTask( - database, null, null, null, null, ProcedureType.DELETE_DATABASE_PROCEDURE); + database, + null, + null, + null, + deletingTimeSeriesQueryId, + ProcedureType.DELETE_DATABASE_PROCEDURE); hasOverlappedTask = procedureIdDuplicatePair.getRight(); if (Boolean.FALSE.equals(procedureIdDuplicatePair.getRight())) { @@ -361,7 +391,7 @@ public TSStatus deleteDatabases( } } List results = new ArrayList<>(procedures.size()); - procedures.forEach(procedure -> results.add(waitingProcedureFinished(procedure))); + procedures.forEach(procedure -> results.add(waitingProcedureFinished(procedure, waitTimeout))); // Clear the previously deleted regions final PartitionManager partitionManager = getConfigManager().getPartitionManager(); partitionManager.getRegionMaintainer().submit(partitionManager::maintainRegionReplicas); @@ -418,40 +448,102 @@ public TSStatus deleteTimeSeries( PathPatternTree patternTree, boolean isGeneratedByPipe, boolean mayDeleteAudit) { - DeleteTimeSeriesProcedure procedure = null; + final DeleteTimeSeriesProcedure procedure; synchronized (this) { - boolean hasOverlappedTask = false; - ProcedureType type; - DeleteTimeSeriesProcedure deleteTimeSeriesProcedure; - for (Procedure runningProcedure : executor.getProcedures().values()) { - type = ProcedureFactory.getProcedureType(runningProcedure); - if (type == null || !type.equals(ProcedureType.DELETE_TIMESERIES_PROCEDURE)) { - continue; - } - deleteTimeSeriesProcedure = ((DeleteTimeSeriesProcedure) runningProcedure); - if (queryId.equals(deleteTimeSeriesProcedure.getQueryId())) { - procedure = deleteTimeSeriesProcedure; - break; - } - if (patternTree.isOverlapWith(deleteTimeSeriesProcedure.getPatternTree())) { - hasOverlappedTask = true; - break; - } + if (!executor.isRunning()) { + return RpcUtils.getStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP); } - - if (procedure == null) { - if (hasOverlappedTask) { + final Optional duplicate = + snapshotProcedures().stream() + .filter(p -> p instanceof DeleteTimeSeriesProcedure) + .map(p -> (DeleteTimeSeriesProcedure) p) + .filter(p -> queryId.equals(p.getQueryId())) + .findFirst(); + if (duplicate.isPresent()) { + procedure = duplicate.get(); + } else { + if (executor.getProcedures().values().stream() + .filter(p -> p instanceof DeleteTimeSeriesProcedure) + .map(p -> (DeleteTimeSeriesProcedure) p) + .anyMatch(p -> patternTree.isOverlapWith(p.getPatternTree()))) { return RpcUtils.getStatus( TSStatusCode.OVERLAP_WITH_EXISTING_TASK, ManagerMessages - .EXCEPTION_SOME_OTHER_TASK_IS_DELETING_SOME_TARGET_TIMESERIES_2D824DD9); + .MESSAGE_SOME_OTHER_ASYNCHRONOUS_TASK_IS_DELETING_TARGET_TIMESERIES_USE_SHOW_DELETE_TIMESERIES_PROGRESS_TO_CHECK_ITS_PROGRESS_AND_RETRY_LATER_A799B41E); + } + final int waiting = executor.getDeleteTimeSeriesScheduler().getWaitingTaskCount(); + if (waiting >= CONFIG_NODE_CONFIG.getTimeseriesDeletionQueueCapacity()) { + return RpcUtils.getStatus( + TSStatusCode.WRITE_PROCESS_REJECT, + String.format( + ManagerMessages + .MESSAGE_THE_ASYNCHRONOUS_TIMESERIES_DELETION_QUEUE_IS_FULL_ARG_WAITING_TASKS_RETRY_LATER_93F6F407, + waiting)); } procedure = new DeleteTimeSeriesProcedure(queryId, patternTree, isGeneratedByPipe, mayDeleteAudit); - this.executor.submitProcedure(procedure); + try { + executor.submitProcedureAndPersist(procedure); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR, e.getMessage()); + } } } - return waitingProcedureFinished(procedure, PROCEDURE_WAIT_TIME_OUT << 1); + // Pipe reception waits for the operation's result before acknowledging its event. + return isGeneratedByPipe + ? waitingProcedureFinished(procedure, PROCEDURE_WAIT_TIME_OUT << 1) + : RpcUtils.getStatus( + TSStatusCode.SUCCESS_STATUS, + String.format( + ManagerMessages + .MESSAGE_TIMESERIES_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_USE_SHOW_DELETE_TIMESERIES_PROGRESS_TO_CHECK_ITS_PROGRESS_75958076, + procedure.getProcId())); + } + + public List getDeleteTimeSeriesProcedures() { + final Map procedures = new HashMap<>(); + snapshotProcedures().stream() + .filter(p -> p instanceof DeleteTimeSeriesProcedure) + .map(p -> (DeleteTimeSeriesProcedure) p) + .forEach(p -> procedures.put(p.getProcId(), p)); + return procedures.values().stream() + .sorted(Comparator.comparingLong(Procedure::getProcId)) + .map( + p -> { + // A terminal in-memory state precedes its final store update and root cleanup. + final boolean finished = executor.isCompleted(p.getProcId()); + final TDeleteTimeSeriesProcedureInfo info = + new TDeleteTimeSeriesProcedureInfo( + p.getProcId(), + p.getQueryId(), + p.getPatternTree().getAllPathPatterns().toString(), + finished + ? p.getState().name() + : executor.getDeleteTimeSeriesScheduler().isQueued(p.getProcId()) + ? "QUEUED" + : "RUNNING", + !finished && p.isFinished() ? "FINALIZING" : p.getDeleteProgress()); + if (p.hasException()) { + if (p.getException().getCause() instanceof IoTDBException) { + final IoTDBException failure = (IoTDBException) p.getException().getCause(); + info.setErrorMessage(failure.getErrorCode() + ": " + failure.getMessage()); + } else { + info.setErrorMessage( + TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + + ": " + + p.getException().getMessage()); + } + } + return info; + }) + .collect(Collectors.toList()); + } + + public boolean hasPendingFullDatabaseTimeSeriesDeletion(final String database) { + return executor.getProcedures().values().stream() + .filter(DeleteTimeSeriesProcedure.class::isInstance) + .map(DeleteTimeSeriesProcedure.class::cast) + .anyMatch(procedure -> procedure.deletesDatabase(database)); } public TSStatus deleteLogicalView(TDeleteLogicalViewReq req) { @@ -2217,8 +2309,12 @@ protected TSStatus waitingProcedureFinished( } // Conflict checks can fail before a procedure is assigned a persistent id. In that case the // procedure remains at NO_PROC_ID but already carries the user-facing failure. - if (!procedure.isFinished() - && !(procedure.getProcId() == Procedure.NO_PROC_ID && procedure.isFailed())) { + // Pipe acknowledgements must wait for durable root cleanup even if shutdown interrupts the + // wait while the deletion already has a terminal in-memory state. + if ((procedure instanceof DeleteTimeSeriesProcedure + && !executor.isFinished(procedure.getProcId())) + || !procedure.isFinished() + && !(procedure.getProcId() == Procedure.NO_PROC_ID && procedure.isFailed())) { // The procedure is still executing status = RpcUtils.getStatus(TSStatusCode.INTERNAL_REQUEST_TIME_OUT, PROCEDURE_TIMEOUT_MESSAGE); @@ -2565,29 +2661,115 @@ public TSStatus alterTableRenameColumn(final TAlterOrDropTableReq req) { false)); } + private List> snapshotProcedures() { + // Root cleanup inserts into completed before removing from the active map. Fully snapshot the + // active map first, so a completion between these reads is visible in at least one snapshot. + final List> procedures = new ArrayList<>(executor.getProcedures().values()); + procedures.addAll(executor.getCompletedProcedures()); + return procedures; + } + + private long tableProcedureId( + String queryId, Class type) { + return snapshotProcedures().stream() + .filter(p -> p.getClass() == type) + .map(p -> (AbstractAlterOrDropTableProcedure) p) + .filter(p -> queryId.equals(p.getQueryId())) + .mapToLong(Procedure::getProcId) + .findFirst() + .orElse(-1L); + } + public TSStatus alterTableDropColumn(final TAlterOrDropTableReq req) { - final boolean isView = req.isSetIsView() && req.isIsView(); - return executeWithoutDuplicate( - req.database, - null, - req.tableName, - req.queryId, - isView - ? ProcedureType.DROP_VIEW_COLUMN_PROCEDURE - : ProcedureType.DROP_TABLE_COLUMN_PROCEDURE, - isView - ? new DropViewColumnProcedure( - req.database, - req.tableName, - req.queryId, - ReadWriteIOUtils.readString(req.updateInfo), - false) - : new DropTableColumnProcedure( - req.database, - req.tableName, - req.queryId, - ReadWriteIOUtils.readString(req.updateInfo), - false)); + final String columnName = ReadWriteIOUtils.readString(req.updateInfo); + if (req.isSetIsView() && req.isIsView()) { + return executeWithoutDuplicate( + req.database, + null, + req.tableName, + req.queryId, + ProcedureType.DROP_VIEW_COLUMN_PROCEDURE, + new DropViewColumnProcedure(req.database, req.tableName, req.queryId, columnName, false)); + } + synchronized (this) { + if (!executor.isRunning()) { + return RpcUtils.getStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP); + } + final long completedId = tableProcedureId(req.queryId, DropTableColumnProcedure.class); + if (completedId != -1) { + return RpcUtils.getStatus( + TSStatusCode.SUCCESS_STATUS, + String.format( + ManagerMessages + .MESSAGE_COLUMN_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_USE_DESCRIBE_DETAILS_TO_CHECK_ITS_PROGRESS_A0578678, + completedId)); + } + final Pair duplicate = + checkDuplicateTableTask( + req.database, + null, + req.tableName, + null, + req.queryId, + ProcedureType.DROP_TABLE_COLUMN_PROCEDURE); + if (duplicate.getLeft() != -1) { + return RpcUtils.getStatus( + TSStatusCode.SUCCESS_STATUS, + String.format( + ManagerMessages + .MESSAGE_COLUMN_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_USE_DESCRIBE_DETAILS_TO_CHECK_ITS_PROGRESS_A0578678, + duplicate.getLeft())); + } + if (Boolean.TRUE.equals(duplicate.getRight())) { + return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } + try { + final Optional> table = + configManager + .getClusterSchemaManager() + .getTableAndStatusIfExists(req.database, req.tableName); + if (table.isEmpty()) { + final TableNotExistsException e = + new TableNotExistsException(req.database, req.tableName); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } + if (table.get().getRight() != TableNodeStatus.USING) { + return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } + final Optional> viewCheck = + ClusterSchemaManager.checkTable4View(req.database, table.get().getLeft(), false); + if (viewCheck.isPresent()) { + return viewCheck.get().getLeft(); + } + final TsTableColumnSchema column = table.get().getLeft().getColumnSchema(columnName); + if (column == null) { + final ColumnNotExistsException e = + new ColumnNotExistsException(req.database, req.tableName, columnName); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } + if (column.getColumnCategory() == TsTableColumnCategory.TAG + || column.getColumnCategory() == TsTableColumnCategory.TIME) { + return RpcUtils.getStatus( + TSStatusCode.SEMANTIC_ERROR, + ConfigNodeMessages.DROPPING_TAG_OR_TIME_COLUMN_IS_NOT_SUPPORTED); + } + } catch (MetadataException e) { + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } + final DropTableColumnProcedure procedure = + new DropTableColumnProcedure(req.database, req.tableName, req.queryId, columnName, false); + try { + executor.submitProcedureAndPersist(procedure); + return RpcUtils.getStatus( + TSStatusCode.SUCCESS_STATUS, + String.format( + ManagerMessages + .MESSAGE_COLUMN_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_USE_DESCRIBE_DETAILS_TO_CHECK_ITS_PROGRESS_A0578678, + procedure.getProcId())); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } } public TSStatus alterTableColumnDataType(TAlterOrDropTableReq req) { @@ -2621,6 +2803,15 @@ public TSStatus dropTable(final TAlterOrDropTableReq req) { if (!executor.isRunning()) { return RpcUtils.getStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP); } + final long completedId = tableProcedureId(req.queryId, DropTableProcedure.class); + if (completedId != -1) { + return RpcUtils.getStatus( + TSStatusCode.SUCCESS_STATUS, + String.format( + ManagerMessages + .MESSAGE_TABLE_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_QUERY_INFORMATION_SCHEMA_DROP_TABLE_PROCEDURES_TO_CHECK_ITS_PROGRESS_D7AB21D3, + completedId)); + } final Pair duplicate = checkDuplicateTableTask( req.database, @@ -2630,7 +2821,12 @@ public TSStatus dropTable(final TAlterOrDropTableReq req) { req.queryId, ProcedureType.DROP_TABLE_PROCEDURE); if (duplicate.getLeft() != -1) { - return StatusUtils.OK; + return RpcUtils.getStatus( + TSStatusCode.SUCCESS_STATUS, + String.format( + ManagerMessages + .MESSAGE_TABLE_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_QUERY_INFORMATION_SCHEMA_DROP_TABLE_PROCEDURES_TO_CHECK_ITS_PROGRESS_D7AB21D3, + duplicate.getLeft())); } if (Boolean.TRUE.equals(duplicate.getRight())) { return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); @@ -2660,7 +2856,12 @@ public TSStatus dropTable(final TAlterOrDropTableReq req) { new DropTableProcedure(req.database, req.tableName, req.queryId, false); try { executor.submitProcedureAndPersist(procedure); - return StatusUtils.OK; + return RpcUtils.getStatus( + TSStatusCode.SUCCESS_STATUS, + String.format( + ManagerMessages + .MESSAGE_TABLE_DELETION_TASK_ARG_HAS_BEEN_SUBMITTED_FOR_ASYNCHRONOUS_EXECUTION_QUERY_INFORMATION_SCHEMA_DROP_TABLE_PROCEDURES_TO_CHECK_ITS_PROGRESS_D7AB21D3, + procedure.getProcId())); } catch (final Exception e) { return RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR, e.getMessage()); } @@ -2886,9 +3087,19 @@ public Pair checkDuplicateTableTask( ProcedureType type; for (final Procedure procedure : executor.getProcedures().values()) { type = ProcedureFactory.getProcedureType(procedure); - if (type == null || procedure.isFinished()) { + if (type == null + || (procedure.isFinished() + && !(procedure instanceof DropTableColumnProcedure + && !(procedure instanceof DropViewColumnProcedure)) + && !(procedure instanceof DeleteTimeSeriesProcedure))) { continue; } + if (procedure instanceof DeleteTimeSeriesProcedure + && (queryId == null + || !queryId.equals(((DeleteTimeSeriesProcedure) procedure).getQueryId())) + && ((DeleteTimeSeriesProcedure) procedure).deletesDatabase(database)) { + return new Pair<>(-1L, true); + } // A table shall not be concurrently operated or else the dataNode cache // may record fake values switch (type) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java index 32797ba0c7a27..5114f26b36291 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java @@ -22,10 +22,16 @@ import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewProcedure; +import org.apache.iotdb.confignode.procedure.scheduler.DeleteTimeSeriesProcedureScheduler; import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler; import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler; import org.apache.iotdb.confignode.procedure.state.ProcedureLockState; @@ -77,6 +83,7 @@ public class ProcedureExecutor { // leader transition. private volatile CopyOnWriteArrayList workerThreads; private volatile CopyOnWriteArrayList dropTableWorkerThreads; + private volatile CopyOnWriteArrayList deleteTimeSeriesWorkerThreads; private TimeoutExecutorThread timeoutExecutor; @@ -98,6 +105,7 @@ public class ProcedureExecutor { // Drop table can spend a long time deleting data on DataNodes. Keep its worker separate so // unrelated procedures can still make progress. private final ProcedureScheduler dropTableScheduler = new SimpleProcedureScheduler(); + private DeleteTimeSeriesProcedureScheduler deleteTimeSeriesScheduler; private final AtomicLong workId = new AtomicLong(0); private final AtomicInteger activeExecutorCount = new AtomicInteger(0); @@ -146,12 +154,26 @@ public synchronized void init(int numThreads) { workerThreads.add(new WorkerThread(threadGroup, scheduler)); } dropTableWorkerThreads = new CopyOnWriteArrayList<>(); + deleteTimeSeriesWorkerThreads = new CopyOnWriteArrayList<>(); + deleteTimeSeriesScheduler = + new DeleteTimeSeriesProcedureScheduler( + ConfigNodeDescriptor.getInstance().getConf().getTimeseriesDeletionMaxConcurrentTasks(), + ConfigNodeDescriptor.getInstance() + .getConf() + .getTimeseriesDeletionExecutionIntervalMs()); // Add worker monitor workerMonitorExecutor.add(new WorkerMonitor()); + // This executor is reused after leader changes. An interrupted deletion can still be active + // here even when another leader has since persisted its completion. Recover only store state. + procedures.clear(); + completed.clear(); + rollbackStack.clear(); + scheduler.clear(); dropTableScheduler.clear(); scheduler.start(); dropTableScheduler.start(); + deleteTimeSeriesScheduler.start(); recover(); } @@ -241,6 +263,7 @@ private void recover() { runnableList.forEach(procedure -> getScheduler(procedure).addBack(procedure)); scheduler.signalAll(); dropTableScheduler.signalAll(); + deleteTimeSeriesScheduler.signalAll(); } private List> getProcedureListFromDifferentVersion() { @@ -322,6 +345,9 @@ public synchronized void startWorkers() { for (WorkerThread workerThread : dropTableWorkerThreads) { workerThread.start(); } + for (WorkerThread workerThread : deleteTimeSeriesWorkerThreads) { + workerThread.start(); + } LOG.info(ProcedureMessages.PROCEDURE_WORKERS_ARE_STARTED, workerThreads.size()); } @@ -484,7 +510,8 @@ private void executeProcedure(Procedure proc) throws InterruptedException { * @param rootProcStack procedure's root proc stack * @param proc procedure */ - private void executeProcedure(RootProcedureStack rootProcStack, Procedure proc) { + private void executeProcedure(RootProcedureStack rootProcStack, Procedure proc) + throws InterruptedException { if (proc.getState() != ProcedureState.RUNNABLE) { LOG.error( ProcedureMessages @@ -601,7 +628,14 @@ private void submitChildrenProcedures(Procedure[] subprocs) { } private void updateStoreOnExecution( - RootProcedureStack rootProcStack, Procedure proc, Procedure[] subprocs) { + RootProcedureStack rootProcStack, Procedure proc, Procedure[] subprocs) + throws InterruptedException { + if (requiresDurableDeletionState(proc)) { + // In particular, persist RECREATE_DATABASES before recreating a database, and persist the + // terminal state before admitting a conflicting task or exposing successful completion. + persistDeletionState(proc); + return; + } if (subprocs != null && !proc.isFailed()) { if (LOG.isDebugEnabled()) { LOG.debug(ProcedureMessages.STORED_CHILDREN, proc, Arrays.toString(subprocs)); @@ -832,9 +866,13 @@ private void finishWithoutRollback(final Procedure procedure) throws Interr // The stack indexes and the states just dropped are persisted here, so that the rollback // recovered by a new leader leaves this procedure out. Endless retry since this step is not // idempotent. - RetryUtils.executeWithEndlessBackoffRetry( - () -> store.update(procedure), - ProcedureMessages.LOG_FINISH_PROCEDURE_WITHOUT_ROLLBACK_41F7A354); + if (requiresDurableDeletionState(procedure)) { + persistDeletionState(procedure); + } else { + RetryUtils.executeWithEndlessBackoffRetry( + () -> store.update(procedure), + ProcedureMessages.LOG_FINISH_PROCEDURE_WITHOUT_ROLLBACK_41F7A354); + } if (Thread.currentThread().isInterrupted()) { throw new InterruptedException(); } @@ -875,7 +913,7 @@ private ProcedureLockState acquireLock(Procedure proc) { * @param procedure procedure * @return procedure lock state */ - private ProcedureLockState executeRollback(Procedure procedure) { + private ProcedureLockState executeRollback(Procedure procedure) throws InterruptedException { boolean rollbackFailed = false; try { procedure.doRollback(this.environment); @@ -904,7 +942,15 @@ private ProcedureLockState executeRollback(Procedure procedure) { return ProcedureLockState.LOCK_ACQUIRED; } - private void cleanupAfterRollback(Procedure procedure) { + private void cleanupAfterRollback(Procedure procedure) throws InterruptedException { + if (requiresDurableDeletionState(procedure)) { + if (procedure.removeStackIndex()) { + procedure.setState(ProcedureState.ROLLEDBACK); + procedure.updateMetricsOnFinish(getEnvironment(), procedure.elapsedTime(), false); + } + persistDeletionState(procedure); + return; + } if (procedure.removeStackIndex()) { if (!procedure.isSuccess()) { procedure.setState(ProcedureState.ROLLEDBACK); @@ -954,12 +1000,41 @@ private void executeCompletionCleanup(Procedure proc) { } } + private boolean requiresDurableDeletionState(Procedure procedure) { + return !procedure.hasParent() + && (procedure instanceof DeleteTimeSeriesProcedure + || procedure instanceof DeleteDatabaseProcedure + || (procedure instanceof DropTableColumnProcedure + && !(procedure instanceof DropViewColumnProcedure))); + } + + private void persistDeletionState(Procedure procedure) throws InterruptedException { + long backoffMs = 100; + while (isRunning() && store.isRunning()) { + try { + store.update(procedure); + return; + } catch (InterruptedException e) { + throw e; + } catch (Exception e) { + LOG.warn(ProcedureMessages.FAILED_TO_UPDATE_PROCEDURE_ON_EXECUTION, e); + Thread.sleep(backoffMs); + backoffMs = Math.min(backoffMs * 2, 1000); + } + } + // Leave the last durable state for the next leader; do not clean up this root or its slot. + throw new InterruptedException(); + } + private void rootProcedureCleanup(Procedure proc) { executeCompletionCleanup(proc); CompletedProcedureContainer retainer = new CompletedProcedureContainer<>(proc); completed.put(proc.getProcId(), retainer); rollbackStack.remove(proc.getProcId()); procedures.remove(proc.getProcId()); + if (proc instanceof DeleteTimeSeriesProcedure) { + deleteTimeSeriesScheduler.completionCleanup(proc); + } } /** @@ -1085,6 +1160,8 @@ public void run() { } if (workerScheduler == dropTableScheduler) { dropTableWorkerThreads.remove(this); + } else if (workerScheduler == deleteTimeSeriesScheduler) { + deleteTimeSeriesWorkerThreads.remove(this); } else { workerThreads.remove(this); } @@ -1216,6 +1293,7 @@ public synchronized void stop() { LOG.info(ProcedureMessages.STOPPING); scheduler.stop(); dropTableScheduler.stop(); + deleteTimeSeriesScheduler.stop(); timeoutExecutor.sendStopSignal(); } @@ -1228,6 +1306,9 @@ public void join() { for (WorkerThread workerThread : dropTableWorkerThreads) { workerThread.awaitTermination(); } + for (WorkerThread workerThread : deleteTimeSeriesWorkerThreads) { + workerThread.awaitTermination(); + } } public boolean isStarted(long procId) { @@ -1242,6 +1323,10 @@ public boolean isFinished(final long procId) { return !procedures.containsKey(procId); } + public boolean isCompleted(final long procId) { + return completed.containsKey(procId); + } + public ConcurrentHashMap> getProcedures() { return procedures; } @@ -1294,11 +1379,36 @@ public ProcedureScheduler getScheduler() { } public ProcedureScheduler getScheduler(final Procedure procedure) { - return procedure instanceof DropTableProcedure && !(procedure instanceof DropViewProcedure) + if (procedure instanceof DeleteTimeSeriesProcedure) { + return getDeleteTimeSeriesScheduler(); + } + return (procedure instanceof DropTableProcedure && !(procedure instanceof DropViewProcedure)) + || (procedure instanceof DropTableColumnProcedure + && !(procedure instanceof DropViewColumnProcedure)) ? getDropTableScheduler() : scheduler; } + public synchronized DeleteTimeSeriesProcedureScheduler getDeleteTimeSeriesScheduler() { + if (deleteTimeSeriesWorkerThreads.isEmpty()) { + for (int i = 0; + i + < ConfigNodeDescriptor.getInstance() + .getConf() + .getTimeseriesDeletionMaxConcurrentTasks(); + i++) { + final WorkerThread worker = + new WorkerThread( + threadGroup, deleteTimeSeriesScheduler, "DeleteTimeSeriesProcedureWorker-"); + deleteTimeSeriesWorkerThreads.add(worker); + if (isRunning()) { + worker.start(); + } + } + } + return deleteTimeSeriesScheduler; + } + private synchronized ProcedureScheduler getDropTableScheduler() { // Recovery can enqueue drops before startWorkers(), while new submissions arrive after it. // Share the lifecycle lock so the first drop creates and starts exactly one worker. diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java index 4bc3f9df83b79..018b2e07730da 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java @@ -23,18 +23,28 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.commons.utils.ThriftConfigNodeSerDeUtils; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteTimeSeriesPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan; +import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; import org.apache.iotdb.confignode.i18n.ProcedureMessages; +import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; import org.apache.iotdb.confignode.procedure.state.schema.DeleteTimeSeriesState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.mpp.rpc.thrift.TConstructSchemaBlackListReq; @@ -42,6 +52,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TDeleteTimeSeriesReq; import org.apache.iotdb.mpp.rpc.thrift.TRollbackSchemaBlackListReq; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -53,7 +64,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -62,6 +75,8 @@ import java.util.function.Consumer; import java.util.stream.Collectors; +import static org.apache.iotdb.commons.schema.table.Audit.TREE_MODEL_AUDIT_DATABASE; + public class DeleteTimeSeriesProcedure extends StateMachineProcedure { @@ -70,6 +85,9 @@ public class DeleteTimeSeriesProcedure private String queryId; private PathPatternTree patternTree; + private PathPatternTree originalPatternTree; + private final List deleteDatabaseSchemas = new ArrayList<>(); + private String databaseDeletionFailure; private transient ByteBuffer patternTreeBytes; private boolean mayDeleteAudit; @@ -104,6 +122,9 @@ protected Flow executeFromState( final long startTime = System.currentTimeMillis(); try { switch (state) { + case PREPARE_DELETE: + prepareDelete(env); + break; case CONSTRUCT_BLACK_LIST: LOGGER.info( ProcedureMessages.CONSTRUCT_SCHEMAENGINE_BLACK_LIST_OF_TIMESERIES, requestMessage); @@ -133,6 +154,28 @@ protected Flow executeFromState( LOGGER.info(ProcedureMessages.DELETE_TIMESERIES_SCHEMAENGINE_OF, requestMessage); deleteTimeSeriesSchema(env); collectPayload4Pipe(env); + if (!isFailed() && !deleteDatabaseSchemas.isEmpty()) { + setNextState(DeleteTimeSeriesState.DELETE_DATABASES); + break; + } + return Flow.NO_MORE_STATE; + case DELETE_DATABASES: + // This is a background task: an RPC wait timeout must not allow recreation while a + // database-deletion procedure is still running. + final TSStatus deleteStatus = + env.getConfigManager() + .getProcedureManager() + .deleteDatabases( + deleteDatabaseSchemas, isGeneratedByPipe, Long.MAX_VALUE, queryId); + if (deleteStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + databaseDeletionFailure = deleteStatus.toString(); + } + // Persist deletion's result before recreating any database. Recovery must never delete + // databases again once their schemas have been recreated and accept new writes. + setNextState(DeleteTimeSeriesState.RECREATE_DATABASES); + break; + case RECREATE_DATABASES: + recreateDatabases(env); return Flow.NO_MORE_STATE; default: setFailure(new ProcedureException(ProcedureMessages.UNRECOGNIZED_STATE + state)); @@ -147,6 +190,86 @@ protected Flow executeFromState( } } + /** Keep the full-database optimization inside the durable asynchronous deletion task. */ + private void prepareDelete(final ConfigNodeProcedureEnv env) { + final ConfigManager manager = env.getConfigManager(); + final Set databases = new HashSet<>(); + final List databasePatterns = new ArrayList<>(); + final PathPatternTree remaining = new PathPatternTree(); + for (PartialPath path : patternTree.getAllPathPatterns()) { + if (PathPatternUtil.isMultiLevelMatchWildcard(path.getMeasurement()) + && path.getNodeLength() > 1) { + // Pattern-tree paths are PartialPaths, whose getDevicePath() includes the final **. + final PartialPath prefix = + new PartialPath(Arrays.copyOf(path.getNodes(), path.getNodeLength() - 1)); + if (prefix.hasWildcard()) { + remaining.appendPathPattern(path); + continue; + } + final Map matched = + manager.getClusterSchemaManager().getMatchedDatabaseSchemasByPrefix(prefix); + matched.remove(TREE_MODEL_AUDIT_DATABASE); + if (!matched.isEmpty()) { + databases.addAll(matched.values()); + databasePatterns.add(path); + continue; + } + } + remaining.appendPathPattern(path); + } + if (!databases.isEmpty()) { + try { + SchemaUtils.checkSchemaRegionUsingTemplate(manager, databasePatterns); + } catch (MetadataException e) { + setFailure(new ProcedureException(e)); + return; + } + remaining.constructTree(); + setPatternTree(remaining); + deleteDatabaseSchemas.addAll(databases); + } + setNextState( + patternTree.isEmpty() && !deleteDatabaseSchemas.isEmpty() + ? DeleteTimeSeriesState.DELETE_DATABASES + : DeleteTimeSeriesState.CONSTRUCT_BLACK_LIST); + } + + private void recreateDatabases(final ConfigNodeProcedureEnv env) { + final ConfigManager manager = env.getConfigManager(); + final List failures = new ArrayList<>(); + if (databaseDeletionFailure != null) { + failures.add( + RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, databaseDeletionFailure)); + } + // Recreate the original database schemas even when deletion reports a failure, as before. + for (TDatabaseSchema database : deleteDatabaseSchemas) { + final TSStatus status = + manager + .getClusterSchemaManager() + .setDatabase( + new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, database), + isGeneratedByPipe); + if (status.getCode() == TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode()) { + try { + if (database.equals( + manager.getClusterSchemaManager().getDatabaseSchemaByName(database.getName()))) { + // A previous execution may have recreated this database before the state was stored. + continue; + } + } catch (DatabaseNotExistsException e) { + failures.add(RpcUtils.getStatus(TSStatusCode.DATABASE_NOT_EXIST, e.getMessage())); + } + } + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + failures.add(status); + } + } + if (!failures.isEmpty()) { + setFailure( + new ProcedureException(new IoTDBException(RpcUtils.squashResponseStatusList(failures)))); + } + } + // Return the total num of timeSeries in schemaEngine black list private long constructBlackList(final ConfigNodeProcedureEnv env) { final Map targetSchemaRegionGroup = @@ -317,7 +440,7 @@ protected int getStateId(final DeleteTimeSeriesState deleteTimeSeriesState) { @Override protected DeleteTimeSeriesState getInitialState() { - return DeleteTimeSeriesState.CONSTRUCT_BLACK_LIST; + return DeleteTimeSeriesState.PREPARE_DELETE; } public String getQueryId() { @@ -325,10 +448,47 @@ public String getQueryId() { } public PathPatternTree getPatternTree() { - return patternTree; + return originalPatternTree == null ? patternTree : originalPatternTree; + } + + public boolean deletesDatabase(final String database) { + // Table and tree databases have separate namespaces, even with the same qualified path. + if (PathUtils.isTableModelDatabase(database)) { + return false; + } + final PartialPath databasePath; + try { + databasePath = PartialPath.getQualifiedDatabasePartialPath(database); + } catch (IllegalPathException e) { + return false; + } + final String[] databaseNodes = databasePath.getNodes(); + for (PartialPath path : getPatternTree().getAllPathPatterns()) { + if (!PathPatternUtil.isMultiLevelMatchWildcard(path.getMeasurement()) + || path.getNodeLength() <= 1) { + continue; + } + final PartialPath prefix = + new PartialPath(Arrays.copyOf(path.getNodes(), path.getNodeLength() - 1)); + if (!prefix.hasWildcard() + && prefix.getNodeLength() <= databaseNodes.length + && Arrays.equals( + prefix.getNodes(), Arrays.copyOf(databaseNodes, prefix.getNodeLength()))) { + return true; + } + } + return false; + } + + public String getDeleteProgress() { + final DeleteTimeSeriesState state = getCurrentState(); + return isFinished() || state == null ? getState().name() : state.name(); } public void setPatternTree(final PathPatternTree patternTree) { + if (originalPatternTree == null) { + originalPatternTree = patternTree; + } this.patternTree = patternTree; requestMessage = patternTree.getAllPathPatterns().toString(); patternTreeBytes = preparePatternTreeBytesData(patternTree); @@ -355,12 +515,24 @@ public void serialize(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(queryId, stream); patternTree.serialize(stream); ReadWriteIOUtils.write(mayDeleteAudit, stream); + ReadWriteIOUtils.write(originalPatternTree != patternTree, stream); + if (originalPatternTree != patternTree) { + originalPatternTree.serialize(stream); + } + ReadWriteIOUtils.write(deleteDatabaseSchemas.size(), stream); + for (TDatabaseSchema database : deleteDatabaseSchemas) { + ThriftConfigNodeSerDeUtils.serializeTDatabaseSchema(database, stream); + } + ReadWriteIOUtils.write(databaseDeletionFailure, stream); } @Override public void deserialize(final ByteBuffer byteBuffer) { super.deserialize(byteBuffer); queryId = ReadWriteIOUtils.readString(byteBuffer); + originalPatternTree = null; + deleteDatabaseSchemas.clear(); + databaseDeletionFailure = null; setPatternTree(PathPatternTree.deserialize(byteBuffer)); if (getCurrentState() == DeleteTimeSeriesState.CLEAN_DATANODE_SCHEMA_CACHE || getCurrentState() == DeleteTimeSeriesState.DELETE_DATA) { @@ -369,6 +541,20 @@ public void deserialize(final ByteBuffer byteBuffer) { if (byteBuffer.hasRemaining()) { mayDeleteAudit = ReadWriteIOUtils.readBoolean(byteBuffer); } + if (byteBuffer.hasRemaining()) { + originalPatternTree = + ReadWriteIOUtils.readBoolean(byteBuffer) + ? PathPatternTree.deserialize(byteBuffer) + : patternTree; + final int size = ReadWriteIOUtils.readInt(byteBuffer); + for (int i = 0; i < size; i++) { + deleteDatabaseSchemas.add( + ThriftConfigNodeSerDeUtils.deserializeTDatabaseSchema(byteBuffer)); + } + if (byteBuffer.hasRemaining()) { + databaseDeletionFailure = ReadWriteIOUtils.readString(byteBuffer); + } + } } @Override @@ -384,13 +570,23 @@ public boolean equals(final Object o) { && Objects.equals(this.getCurrentState(), that.getCurrentState()) && this.getCycles() == getCycles() && this.isGeneratedByPipe == that.isGeneratedByPipe - && this.patternTree.equals(that.patternTree); + && this.patternTree.equals(that.patternTree) + && Objects.equals(originalPatternTree, that.originalPatternTree) + && deleteDatabaseSchemas.equals(that.deleteDatabaseSchemas) + && Objects.equals(databaseDeletionFailure, that.databaseDeletionFailure); } @Override public int hashCode() { return Objects.hash( - getProcId(), getCurrentState(), getCycles(), isGeneratedByPipe, patternTree); + getProcId(), + getCurrentState(), + getCycles(), + isGeneratedByPipe, + patternTree, + originalPatternTree, + deleteDatabaseSchemas, + databaseDeletionFailure); } private class DeleteTimeSeriesRegionTaskExecutor extends DataNodeTSStatusTaskExecutor { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/DeleteTimeSeriesProcedureScheduler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/DeleteTimeSeriesProcedureScheduler.java new file mode 100644 index 0000000000000..08806ea2b0e91 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/DeleteTimeSeriesProcedureScheduler.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.scheduler; + +import org.apache.iotdb.confignode.procedure.Procedure; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +/** Limits whole deletion tasks, retaining their slots across state transitions and rollback. */ +public class DeleteTimeSeriesProcedureScheduler implements ProcedureScheduler { + private final ReentrantLock lock = new ReentrantLock(); + private final Condition changed = lock.newCondition(); + private final Deque waiting = new ArrayDeque<>(); + private final Deque runnable = new ArrayDeque<>(); + private final Set active = new HashSet<>(); + private final int maxConcurrentTasks; + private final long executionIntervalNanos; + private boolean running; + private boolean taskStarted; + private long lastStartNanos; + + public DeleteTimeSeriesProcedureScheduler(int maxConcurrentTasks, long executionIntervalMs) { + this.maxConcurrentTasks = maxConcurrentTasks; + this.executionIntervalNanos = TimeUnit.MILLISECONDS.toNanos(executionIntervalMs); + } + + @Override + public void start() { + lock.lock(); + try { + running = true; + changed.signalAll(); + } finally { + lock.unlock(); + } + } + + @Override + public void stop() { + lock.lock(); + try { + running = false; + changed.signalAll(); + } finally { + lock.unlock(); + } + } + + @Override + public void signalAll() { + lock.lock(); + try { + changed.signalAll(); + } finally { + lock.unlock(); + } + } + + @Override + public void addFront(Procedure procedure) { + addFront(procedure, true); + } + + @Override + public void addFront(Procedure procedure, boolean notify) { + enqueue(procedure, true, notify); + } + + @Override + public void addBack(Procedure procedure) { + addBack(procedure, true); + } + + @Override + public void addBack(Procedure procedure, boolean notify) { + enqueue(procedure, false, notify); + } + + private void enqueue(Procedure procedure, boolean front, boolean notify) { + if (procedure.isSuccess()) { + return; + } + lock.lock(); + try { + // Enqueue without rate limiting; only starting a new task is throttled in poll(). + // Subsequent states retain their concurrency slot. + Deque queue = active.contains(procedure.getProcId()) ? runnable : waiting; + if (front) { + queue.addFirst(procedure); + } else { + queue.addLast(procedure); + } + if (notify) { + changed.signalAll(); + } + } finally { + lock.unlock(); + } + } + + @Override + public void yield(Procedure procedure) { + addBack(procedure); + } + + @Override + public Procedure poll() { + return poll(-1, TimeUnit.NANOSECONDS); + } + + @Override + public Procedure poll(long timeout, TimeUnit unit) { + final long deadline = timeout < 0 ? 0 : System.nanoTime() + unit.toNanos(timeout); + lock.lock(); + try { + while (running) { + if (!runnable.isEmpty()) { + return runnable.removeFirst(); + } + long delay = Long.MAX_VALUE; + if (!waiting.isEmpty() && active.size() < maxConcurrentTasks) { + delay = taskStarted ? executionIntervalNanos - (System.nanoTime() - lastStartNanos) : 0; + if (delay <= 0) { + final Procedure procedure = waiting.removeFirst(); + active.add(procedure.getProcId()); + lastStartNanos = System.nanoTime(); + taskStarted = true; + return procedure; + } + } + if (timeout >= 0) { + delay = Math.min(delay, deadline - System.nanoTime()); + if (delay <= 0) { + return null; + } + } + changed.awaitNanos(delay); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + lock.unlock(); + } + return null; + } + + /** Called only after the entire root procedure, including rollback, has completed. */ + public void completionCleanup(Procedure procedure) { + lock.lock(); + try { + active.remove(procedure.getProcId()); + changed.signalAll(); + } finally { + lock.unlock(); + } + } + + public boolean isQueued(long procedureId) { + lock.lock(); + try { + return waiting.stream().anyMatch(procedure -> procedure.getProcId() == procedureId); + } finally { + lock.unlock(); + } + } + + public int getWaitingTaskCount() { + lock.lock(); + try { + return waiting.size(); + } finally { + lock.unlock(); + } + } + + @Override + public boolean hasRunnables() { + return size() > 0; + } + + @Override + public int size() { + lock.lock(); + try { + return waiting.size() + runnable.size(); + } finally { + lock.unlock(); + } + } + + @Override + public void clear() { + lock.lock(); + try { + waiting.clear(); + runnable.clear(); + active.clear(); + taskStarted = false; + } finally { + lock.unlock(); + } + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DeleteTimeSeriesState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DeleteTimeSeriesState.java index 9fcf045e3a60b..879a709eab2b3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DeleteTimeSeriesState.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DeleteTimeSeriesState.java @@ -23,5 +23,8 @@ public enum DeleteTimeSeriesState { CONSTRUCT_BLACK_LIST, CLEAN_DATANODE_SCHEMA_CACHE, DELETE_DATA, - DELETE_TIMESERIES_SCHEMA + DELETE_TIMESERIES_SCHEMA, + PREPARE_DELETE, + DELETE_DATABASES, + RECREATE_DATABASES } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ConfigProcedureStore.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ConfigProcedureStore.java index d9d3bf88cf87a..99b85c32699df 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ConfigProcedureStore.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ConfigProcedureStore.java @@ -19,9 +19,12 @@ package org.apache.iotdb.confignode.procedure.store; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.write.procedure.DeleteProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; @@ -29,13 +32,19 @@ import org.apache.iotdb.confignode.persistence.ProcedureInfo; import org.apache.iotdb.confignode.procedure.Procedure; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewColumnProcedure; import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -71,12 +80,41 @@ public void setRunning(boolean running) { @Override public List> load() { - return procedureInfo.oldLoad(); + return copyDeletionProcedures(procedureInfo.oldLoad()); } @Override public List> getProcedures() { - return procedureInfo.getProcedures(); + return copyDeletionProcedures(procedureInfo.getProcedures()); + } + + private List> copyDeletionProcedures( + List> procedures) { + final List> copies = new ArrayList<>(procedures.size()); + for (Procedure procedure : procedures) { + try { + copies.add(copyDeletionProcedure(procedure)); + } catch (IOException e) { + throw new IllegalStateException(ConfigNodeMessages.LOAD_PROCEDURE_WAL_FAILED, e); + } + } + return copies; + } + + private Procedure copyDeletionProcedure( + Procedure procedure) throws IOException { + if (procedure instanceof DeleteTimeSeriesProcedure + || procedure instanceof DeleteDatabaseProcedure + || (procedure instanceof DropTableColumnProcedure + && !(procedure instanceof DropViewColumnProcedure))) { + // Consensus may apply the original plan on the leader. Keep both newly submitted and + // recovered executions separate from the last confirmed record in ProcedureInfo. + return ((UpdateProcedurePlan) + ConfigPhysicalPlan.Factory.create( + new UpdateProcedurePlan(procedure).serializeToByteBuffer())) + .getProcedure(); + } + return procedure; } @Override @@ -94,9 +132,13 @@ public void update(Procedure procedure) throws Exception Objects.requireNonNull( ProcedureFactory.getProcedureType(procedure), ConfigNodeMessages.EXCEPTION_PROCEDURE_TYPE_IS_NULL_93147BD3); - final UpdateProcedurePlan updateProcedurePlan = new UpdateProcedurePlan(procedure); + final UpdateProcedurePlan updateProcedurePlan = + new UpdateProcedurePlan(copyDeletionProcedure(procedure)); try { - configManager.getConsensusManager().write(updateProcedurePlan); + final TSStatus status = configManager.getConsensusManager().write(updateProcedurePlan); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IoTDBException(status); + } } catch (ConsensusException e) { LOG.warn( ConfigNodeMessages.LOG_PID_ARG_FAILED_WRITE_UPDATE_API_EXECUTING_CONSENSUS_LAYER_824FB30E, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index d9310622e0a53..0cce5ac3264f0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -212,6 +212,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodes4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowDeleteTimeSeriesProceduresResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDropTableProceduresResp; import org.apache.iotdb.confignode.rpc.thrift.TShowPipePluginReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; @@ -1506,6 +1507,11 @@ public TShowTable4InformationSchemaResp showTables4InformationSchema() { return configManager.showTables4InformationSchema(); } + @Override + public TShowDeleteTimeSeriesProceduresResp showDeleteTimeSeriesProcedures() { + return configManager.showDeleteTimeSeriesProcedures(); + } + @Override public TShowDropTableProceduresResp showDropTableProcedures() { return configManager.showDropTableProcedures(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/AsyncTimeSeriesDeletionTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/AsyncTimeSeriesDeletionTest.java new file mode 100644 index 0000000000000..22ca6d7e61eb8 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/AsyncTimeSeriesDeletionTest.java @@ -0,0 +1,419 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.table.TableNodeStatus; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; +import org.apache.iotdb.confignode.conf.ConfigNodeConfig; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; +import org.apache.iotdb.confignode.persistence.ProcedureInfo; +import org.apache.iotdb.confignode.procedure.Procedure; +import org.apache.iotdb.confignode.procedure.ProcedureExecutor; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; +import org.apache.iotdb.confignode.procedure.scheduler.DeleteTimeSeriesProcedureScheduler; +import org.apache.iotdb.confignode.procedure.state.ProcedureState; +import org.apache.iotdb.confignode.procedure.store.ConfigProcedureStore; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class AsyncTimeSeriesDeletionTest { + private ProcedureManager manager; + private ProcedureExecutor executor; + private ConcurrentHashMap> running; + private List> completed; + private ClusterSchemaManager schemaManager; + + @Before + public void setUp() throws Exception { + executor = mock(ProcedureExecutor.class); + running = new ConcurrentHashMap<>(); + completed = new ArrayList<>(); + final DeleteTimeSeriesProcedureScheduler scheduler = + new DeleteTimeSeriesProcedureScheduler(1, 1000); + when(executor.isRunning()).thenReturn(true); + when(executor.getProcedures()).thenReturn(running); + when(executor.getCompletedProcedures()).thenReturn(completed); + when(executor.getDeleteTimeSeriesScheduler()).thenReturn(scheduler); + final AtomicLong ids = new AtomicLong(); + doAnswer( + invocation -> { + final Procedure procedure = invocation.getArgument(0); + procedure.setProcId(ids.incrementAndGet()); + procedure.setProcRunnable(); + running.put(procedure.getProcId(), procedure); + scheduler.addBack(procedure); + return procedure.getProcId(); + }) + .when(executor) + .submitProcedureAndPersist(any()); + final ConfigManager configManager = mock(ConfigManager.class); + schemaManager = mock(ClusterSchemaManager.class); + when(configManager.getClusterSchemaManager()).thenReturn(schemaManager); + manager = + new ProcedureManager(configManager, mock(ProcedureInfo.class)) { + @Override + protected TSStatus waitingProcedureFinished(Procedure procedure, long timeout) { + throw new AssertionError("SQL submission must not wait for deletion"); + } + }; + manager.setExecutor(executor); + } + + private PathPatternTree pattern(String path) throws Exception { + final PathPatternTree tree = new PathPatternTree(); + tree.appendPathPattern(new PartialPath(path)); + tree.constructTree(); + return tree; + } + + private TAlterOrDropTableReq dropColumnRequest(String queryId, String column) throws IOException { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ReadWriteIOUtils.write(column, new DataOutputStream(bytes)); + return new TAlterOrDropTableReq() + .setDatabase("test") + .setTableName("t") + .setQueryId(queryId) + .setUpdateInfo(ByteBuffer.wrap(bytes.toByteArray())); + } + + @Test + public void testConsensusRejectionDoesNotEnqueueDeletion() throws Exception { + final ConfigManager configManager = mock(ConfigManager.class); + final ConsensusManager consensus = mock(ConsensusManager.class); + when(configManager.getConsensusManager()).thenReturn(consensus); + when(consensus.write(any())) + .thenReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())); + final ConfigProcedureStore store = + new ConfigProcedureStore(configManager, mock(ProcedureInfo.class)); + final ProcedureExecutor realExecutor = + new ProcedureExecutor<>(null, store); + try { + realExecutor.submitProcedureAndPersist( + new DeleteTimeSeriesProcedure("query", pattern("root.async.d.s"), false, false)); + fail("A rejected consensus write must not acknowledge or enqueue deletion"); + } catch (IoTDBException e) { + assertEquals(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(), e.getErrorCode()); + } + assertTrue(realExecutor.getProcedures().isEmpty()); + } + + @Test + public void testFinalizingTreeDeletionStillRejectsOverlap() throws Exception { + final DeleteTimeSeriesProcedure procedure = mock(DeleteTimeSeriesProcedure.class); + when(procedure.getPatternTree()).thenReturn(pattern("root.async.**")); + when(procedure.getQueryId()).thenReturn("first"); + when(procedure.isFinished()).thenReturn(true); + running.put(1L, procedure); + assertEquals( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode(), + manager.deleteTimeSeries("another", pattern("root.async.d.s"), false, false).getCode()); + verify(executor, times(0)).submitProcedureAndPersist(any()); + } + + @Test + public void testFullDatabaseDeletionReservesTableAndDatabaseChanges() throws Exception { + manager.deleteTimeSeries("query", pattern("root.async.**"), false, false); + assertTrue(manager.hasPendingFullDatabaseTimeSeriesDeletion("root.async")); + assertTrue( + manager + .checkDuplicateTableTask( + "root.async", null, "t", null, "table-query", ProcedureType.CREATE_TABLE_PROCEDURE) + .getRight()); + assertTrue( + manager + .checkDuplicateTableTask( + "root.async", null, null, null, null, ProcedureType.DELETE_DATABASE_PROCEDURE) + .getRight()); + assertTrue( + !manager + .checkDuplicateTableTask( + "root.other", null, "t", null, "other-query", ProcedureType.CREATE_TABLE_PROCEDURE) + .getRight()); + assertTrue( + !manager + .checkDuplicateTableTask( + "root.async", null, null, null, "query", ProcedureType.DELETE_DATABASE_PROCEDURE) + .getRight()); + assertEquals( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode(), + manager + .deleteDatabases( + Collections.singletonList(new TDatabaseSchema("root.async")), false, 100) + .getCode()); + + completed.add(running.remove(1L)); + assertTrue(!manager.hasPendingFullDatabaseTimeSeriesDeletion("root.async")); + } + + @Test + public void testTreeDeletionDoesNotReserveTableModelDatabase() throws Exception { + manager.deleteTimeSeries("query", pattern("root.async.**"), false, false); + assertTrue(!manager.hasPendingFullDatabaseTimeSeriesDeletion("async")); + assertTrue( + !manager + .checkDuplicateTableTask( + "async", null, "t", null, "table-query", ProcedureType.CREATE_TABLE_PROCEDURE) + .getRight()); + assertTrue(manager.hasPendingFullDatabaseTimeSeriesDeletion("root.async")); + + running.clear(); + manager.deleteTimeSeries("all", pattern("root.**"), false, false); + assertTrue(!manager.hasPendingFullDatabaseTimeSeriesDeletion("other_table_database")); + assertTrue(manager.hasPendingFullDatabaseTimeSeriesDeletion("root.other_tree_database")); + } + + @Test + public void testFinalizingColumnDeletionStillRejectsTableChanges() { + final DropTableColumnProcedure procedure = mock(DropTableColumnProcedure.class); + when(procedure.getDatabase()).thenReturn("test"); + when(procedure.getTableName()).thenReturn("t"); + when(procedure.isFinished()).thenReturn(true); + running.put(1L, procedure); + assertTrue( + manager + .checkDuplicateTableTask( + "test", null, "t", null, "another", ProcedureType.ADD_TABLE_COLUMN_PROCEDURE) + .getRight()); + } + + @Test + public void testColumnDeletionIsDurableAsynchronousAndIdempotent() throws Exception { + final TsTable table = new TsTable("t"); + table.addColumnSchema(new FieldColumnSchema("s", TSDataType.INT32)); + when(schemaManager.getTableAndStatusIfExists("test", "t")) + .thenReturn(Optional.of(new Pair<>(table, TableNodeStatus.USING))); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + manager.alterTableDropColumn(dropColumnRequest("query", "s")).getCode()); + assertTrue( + manager + .alterTableDropColumn(dropColumnRequest("query", "s")) + .getMessage() + .contains("asynchronous")); + assertEquals( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode(), + manager.alterTableDropColumn(dropColumnRequest("another", "s")).getCode()); + completed.add(running.remove(1L)); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + manager.alterTableDropColumn(dropColumnRequest("query", "s")).getCode()); + verify(executor, times(1)).submitProcedureAndPersist(any()); + } + + @Test + public void testInvalidColumnDeletionIsRejectedBeforeSubmission() throws Exception { + final TsTable table = new TsTable("t"); + table.addColumnSchema(new TimeColumnSchema("time", TSDataType.TIMESTAMP)); + table.addColumnSchema(new TagColumnSchema("tag", TSDataType.STRING)); + when(schemaManager.getTableAndStatusIfExists("test", "t")) + .thenReturn(Optional.of(new Pair<>(table, TableNodeStatus.USING))); + assertEquals( + TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode(), + manager.alterTableDropColumn(dropColumnRequest("missing", "s")).getCode()); + assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + manager.alterTableDropColumn(dropColumnRequest("time", "time")).getCode()); + assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + manager.alterTableDropColumn(dropColumnRequest("tag", "tag")).getCode()); + verify(executor, times(0)).submitProcedureAndPersist(any()); + } + + @Test + public void testCompletionDuringSnapshotDoesNotLoseTaskOrResubmit() throws Exception { + final PathPatternTree tree = pattern("root.async.d.s"); + manager.deleteTimeSeries("query", tree, false, false); + completeDuringSnapshot(); + assertEquals(1, manager.getDeleteTimeSeriesProcedures().size()); + // Repeat the race for submission-time deduplication. + running.put(1L, completed.remove(0)); + completeDuringSnapshot(); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + manager.deleteTimeSeries("query", tree, false, false).getCode()); + verify(executor, times(1)).submitProcedureAndPersist(any()); + } + + @Test + public void testProgressWaitsForDurableCompletion() throws Exception { + final DeleteTimeSeriesProcedure procedure = mock(DeleteTimeSeriesProcedure.class); + when(procedure.getProcId()).thenReturn(1L); + when(procedure.getQueryId()).thenReturn("query"); + when(procedure.getPatternTree()).thenReturn(pattern("root.async.d.s")); + when(procedure.isFinished()).thenReturn(true); + when(procedure.getState()).thenReturn(ProcedureState.SUCCESS); + when(procedure.getDeleteProgress()).thenReturn("SUCCESS"); + when(executor.isCompleted(1L)).thenAnswer(invocation -> completed.contains(procedure)); + running.put(1L, procedure); + assertEquals("RUNNING", manager.getDeleteTimeSeriesProcedures().get(0).getState()); + assertEquals("FINALIZING", manager.getDeleteTimeSeriesProcedures().get(0).getProgress()); + completed.add(running.remove(1L)); + assertEquals("SUCCESS", manager.getDeleteTimeSeriesProcedures().get(0).getState()); + assertEquals("SUCCESS", manager.getDeleteTimeSeriesProcedures().get(0).getProgress()); + } + + @Test + public void testColumnCompletionDuringSnapshotDoesNotResubmit() throws Exception { + final TsTable table = new TsTable("t"); + table.addColumnSchema(new FieldColumnSchema("s", TSDataType.INT32)); + when(schemaManager.getTableAndStatusIfExists("test", "t")) + .thenReturn(Optional.of(new Pair<>(table, TableNodeStatus.USING))); + manager.alterTableDropColumn(dropColumnRequest("query", "s")); + completeDuringSnapshot(); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + manager.alterTableDropColumn(dropColumnRequest("query", "s")).getCode()); + verify(executor, times(1)).submitProcedureAndPersist(any()); + } + + private void completeDuringSnapshot() { + when(executor.getCompletedProcedures()) + .thenAnswer( + invocation -> { + final List> snapshot = new ArrayList<>(completed); + completed.add(running.remove(1L)); + return snapshot; + }) + .thenReturn(completed); + } + + @Test + public void testPipeDoesNotAcknowledgeUnpersistedCompletionDuringShutdown() throws Exception { + final PathPatternTree tree = pattern("root.async.d.s"); + final DeleteTimeSeriesProcedure procedure = + new SuccessfulDeleteTimeSeriesProcedure("pipe-query", tree); + procedure.setProcId(1L); + running.put(1L, procedure); + when(executor.isRunning()).thenReturn(true, false); + when(executor.isFinished(1L)).thenAnswer(invocation -> !running.containsKey(1L)); + final ProcedureManager pipeManager = + new ProcedureManager(mock(ConfigManager.class), mock(ProcedureInfo.class)); + pipeManager.setExecutor(executor); + assertEquals( + TSStatusCode.INTERNAL_REQUEST_TIME_OUT.getStatusCode(), + pipeManager.deleteTimeSeries("pipe-query", tree, true, false).getCode()); + completed.add(running.remove(1L)); + when(executor.isRunning()).thenReturn(true); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + pipeManager.deleteTimeSeries("pipe-query", tree, true, false).getCode()); + verify(executor, times(0)).submitProcedureAndPersist(any()); + } + + private static class SuccessfulDeleteTimeSeriesProcedure extends DeleteTimeSeriesProcedure { + private SuccessfulDeleteTimeSeriesProcedure(String queryId, PathPatternTree tree) { + super(queryId, tree, true, false); + setState(ProcedureState.SUCCESS); + } + } + + @Test + public void testAcknowledgementRequiresPersistenceAndRetriesAreIdempotent() throws Exception { + final PathPatternTree tree = pattern("root.async.d.s"); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + manager.deleteTimeSeries("query", tree, false, false).getCode()); + assertTrue( + manager + .deleteTimeSeries("query", tree, false, false) + .getMessage() + .contains("asynchronous")); + completed.add(running.remove(1L)); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + manager.deleteTimeSeries("query", tree, false, false).getCode()); + verify(executor, times(1)).submitProcedureAndPersist(any()); + assertEquals(1, manager.getDeleteTimeSeriesProcedures().size()); + } + + @Test + public void testPersistenceFailureDoesNotAcknowledgeOrConsumeQueueCapacity() throws Exception { + doThrow(new IOException("store unavailable")).when(executor).submitProcedureAndPersist(any()); + assertEquals( + TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode(), + manager.deleteTimeSeries("query", pattern("root.async.d.s"), false, false).getCode()); + assertTrue(running.isEmpty()); + assertEquals(0, executor.getDeleteTimeSeriesScheduler().getWaitingTaskCount()); + } + + @Test + public void testQueueCapacityAndOverlapAreCheckedBeforePersistence() throws Exception { + final ConfigNodeConfig config = ConfigNodeDescriptor.getInstance().getConf(); + final int capacity = config.getTimeseriesDeletionQueueCapacity(); + config.setTimeseriesDeletionQueueCapacity(1); + try { + manager.deleteTimeSeries("first", pattern("root.async.d.s1"), false, false); + assertEquals( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode(), + manager.deleteTimeSeries("overlap", pattern("root.async.**"), false, false).getCode()); + assertEquals( + TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode(), + manager.deleteTimeSeries("full", pattern("root.async.d.s2"), false, false).getCode()); + // Repeating an accepted request succeeds even when its queue is full. + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + manager.deleteTimeSeries("first", pattern("root.async.d.s1"), false, false).getCode()); + verify(executor, times(1)).submitProcedureAndPersist(any()); + } finally { + config.setTimeseriesDeletionQueueCapacity(capacity); + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerDatabaseDeletionTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerDatabaseDeletionTest.java index 10770df247bf9..23410bcb2cfcb 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerDatabaseDeletionTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerDatabaseDeletionTest.java @@ -19,6 +19,11 @@ package org.apache.iotdb.confignode.manager; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; +import org.apache.iotdb.confignode.consensus.request.write.database.SetDataReplicationFactorPlan; +import org.apache.iotdb.confignode.consensus.request.write.database.SetSchemaReplicationFactorPlan; +import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.confignode.rpc.thrift.TDeleteDatabasesReq; @@ -26,6 +31,7 @@ import org.junit.Test; +import java.lang.reflect.Field; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -34,10 +40,64 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class ConfigManagerDatabaseDeletionTest { + @Test + public void testFullDatabaseDeletionRejectsDatabaseConfigurationChanges() throws Exception { + final ConfigManager manager = mock(ConfigManager.class); + final ClusterSchemaManager schemaManager = mock(ClusterSchemaManager.class); + final ProcedureManager procedureManager = mock(ProcedureManager.class); + for (String name : new String[] {"clusterSchemaManager", "procedureManager"}) { + final Field field = ConfigManager.class.getDeclaredField(name); + field.setAccessible(true); + field.set(manager, "clusterSchemaManager".equals(name) ? schemaManager : procedureManager); + } + when(manager.confirmLeader()) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + final String database = "root.async"; + when(procedureManager.hasPendingFullDatabaseTimeSeriesDeletion(database)).thenReturn(true); + final DatabaseSchemaPlan alter = + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.AlterDatabase, + new TDatabaseSchema(database).setMaxSchemaRegionGroupNum(2)); + final SetSchemaReplicationFactorPlan schemaFactor = + new SetSchemaReplicationFactorPlan(database, 2); + final SetDataReplicationFactorPlan dataFactor = new SetDataReplicationFactorPlan(database, 2); + final SetTimePartitionIntervalPlan interval = new SetTimePartitionIntervalPlan(database, 1000); + final TSStatus success = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + when(schemaManager.alterDatabase(alter, false)).thenReturn(success); + when(schemaManager.setSchemaReplicationFactor(schemaFactor)).thenReturn(success); + when(schemaManager.setDataReplicationFactor(dataFactor)).thenReturn(success); + when(schemaManager.setTimePartitionInterval(interval)).thenReturn(success); + when(manager.alterDatabase(alter)).thenCallRealMethod(); + when(manager.setSchemaReplicationFactor(schemaFactor)).thenCallRealMethod(); + when(manager.setDataReplicationFactor(dataFactor)).thenCallRealMethod(); + when(manager.setTimePartitionInterval(interval)).thenCallRealMethod(); + for (TSStatus result : + new TSStatus[] { + manager.alterDatabase(alter), + manager.setSchemaReplicationFactor(schemaFactor), + manager.setDataReplicationFactor(dataFactor), + manager.setTimePartitionInterval(interval) + }) { + assertEquals(TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode(), result.getCode()); + } + verify(schemaManager, never()).alterDatabase(alter, false); + verify(schemaManager, never()).setSchemaReplicationFactor(schemaFactor); + verify(schemaManager, never()).setDataReplicationFactor(dataFactor); + verify(schemaManager, never()).setTimePartitionInterval(interval); + + when(procedureManager.hasPendingFullDatabaseTimeSeriesDeletion(database)).thenReturn(false); + assertEquals(success, manager.alterDatabase(alter)); + assertEquals(success, manager.setSchemaReplicationFactor(schemaFactor)); + assertEquals(success, manager.setDataReplicationFactor(dataFactor)); + assertEquals(success, manager.setTimePartitionInterval(interval)); + } + @Test public void testTableAuditDatabaseIsExcludedFromDeletion() { final ConfigManager manager = mock(ConfigManager.class); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ConfigProcedureStoreTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ConfigProcedureStoreTest.java new file mode 100644 index 0000000000000..32bdaa3ed2c52 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ConfigProcedureStoreTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.persistence.ProcedureInfo; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; +import org.apache.iotdb.confignode.procedure.state.ProcedureState; +import org.apache.iotdb.confignode.procedure.store.ConfigProcedureStore; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class ConfigProcedureStoreTest { + + @Test + public void testRejectedTreeDeletionUpdatePreservesRecoveryState() throws Exception { + assertRejectedUpdatePreservesRecovery(newTreeDeletion(), false); + } + + @Test + public void testRejectedColumnDeletionUpdatePreservesRecoveryState() throws Exception { + assertRejectedUpdatePreservesRecovery( + new DropTableColumnProcedure("test", "t", "query", "s", false), false); + } + + @Test + public void testRejectedDatabaseDeletionUpdatePreservesRecoveryState() throws Exception { + assertRejectedUpdatePreservesRecovery( + new DeleteDatabaseProcedure(new TDatabaseSchema("root.async"), false), false); + } + + @Test + public void testRejectedLegacyDatabaseDeletionUpdatePreservesRecoveryState() throws Exception { + assertRejectedUpdatePreservesRecovery( + new DeleteDatabaseProcedure(new TDatabaseSchema("root.async"), false), true); + } + + @Test + public void testRejectedLegacyTreeDeletionUpdatePreservesRecoveryState() throws Exception { + assertRejectedUpdatePreservesRecovery(newTreeDeletion(), true); + } + + @Test + public void testRejectedLegacyColumnDeletionUpdatePreservesRecoveryState() throws Exception { + assertRejectedUpdatePreservesRecovery( + new DropTableColumnProcedure("test", "t", "query", "s", false), true); + } + + private DeleteTimeSeriesProcedure newTreeDeletion() throws Exception { + final PathPatternTree tree = new PathPatternTree(); + tree.appendPathPattern(new PartialPath("root.async.d.s")); + tree.constructTree(); + return new DeleteTimeSeriesProcedure("query", tree, false, false); + } + + private void assertRejectedUpdatePreservesRecovery( + Procedure procedure, boolean legacyStore) throws Exception { + final ConfigManager configManager = mock(ConfigManager.class); + final ConsensusManager consensus = mock(ConsensusManager.class); + final ProcedureInfo procedureInfo = spy(new ProcedureInfo(configManager)); + if (legacyStore) { + // oldLoad also inserts the objects it returns into ProcedureInfo's in-memory map. + // Reproduce that boundary without relying on or changing the shared legacy WAL directory. + doAnswer(invocation -> procedureInfo.getProcedures()).when(procedureInfo).oldLoad(); + } + when(configManager.getConsensusManager()).thenReturn(consensus); + final AtomicBoolean rejectUpdates = new AtomicBoolean(); + when(consensus.write(any(UpdateProcedurePlan.class))) + .thenAnswer( + invocation -> + rejectUpdates.get() + ? new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) + : procedureInfo.updateProcedure(invocation.getArgument(0))); + final ConfigProcedureStore store = new ConfigProcedureStore(configManager, procedureInfo); + procedure.setProcId(1L); + procedure.setProcRunnable(); + store.update(procedure); + + // A failed terminal write must not change the last successful consensus record, including + // when the leader applies the original UpdateProcedurePlan without deserializing it. + procedure.setState(ProcedureState.SUCCESS); + rejectUpdates.set(true); + assertRejectedUpdate(store, procedure); + final Procedure recovered = + (legacyStore ? store.load() : store.getProcedures()).get(0); + assertEquals(ProcedureState.RUNNABLE, recovered.getState()); + + // Execution of a recovered object must be isolated as well: another failed terminal write + // must leave the same resumable state available for the following recovery. + recovered.setState(ProcedureState.SUCCESS); + assertRejectedUpdate(store, recovered); + assertEquals( + ProcedureState.RUNNABLE, + (legacyStore ? store.load() : store.getProcedures()).get(0).getState()); + assertEquals(ProcedureState.RUNNABLE, procedureInfo.getProcedures().get(0).getState()); + + rejectUpdates.set(false); + store.update(recovered); + assertEquals( + ProcedureState.SUCCESS, + (legacyStore ? store.load() : store.getProcedures()).get(0).getState()); + } + + private void assertRejectedUpdate( + ConfigProcedureStore store, Procedure procedure) throws Exception { + try { + store.update(procedure); + fail("A rejected consensus write must fail persistence"); + } catch (IoTDBException e) { + assertEquals(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(), e.getErrorCode()); + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedureTest.java index 8223cac998e66..b31e6594f0d79 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedureTest.java @@ -19,10 +19,27 @@ package org.apache.iotdb.confignode.procedure.impl.schema; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.ProcedureManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.partition.PartitionManager; +import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; +import org.apache.iotdb.confignode.persistence.ProcedureInfo; +import org.apache.iotdb.confignode.procedure.NoopProcedureStore; +import org.apache.iotdb.confignode.procedure.Procedure; +import org.apache.iotdb.confignode.procedure.ProcedureExecutor; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.state.ProcedureState; +import org.apache.iotdb.confignode.procedure.state.schema.DeleteTimeSeriesState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.confignode.procedure.util.ProcedureTestUtil; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; import org.junit.Test; @@ -31,10 +48,465 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class DeleteTimeSeriesProcedureTest { + @Test + public void testLeaderReelectionDiscardsUnpersistedDeletionState() throws Exception { + final AtomicBoolean allowCompletion = new AtomicBoolean(); + final CountDownLatch completionRetried = new CountDownLatch(1); + final AtomicReference> recovered = + new AtomicReference<>(Collections.emptyList()); + final NoopProcedureStore store = + new NoopProcedureStore() { + @Override + public List load() { + return recovered.get(); + } + + @Override + public void update(Procedure procedure) { + if (procedure.isSuccess() && !allowCompletion.get()) { + completionRetried.countDown(); + throw new IllegalStateException("completion store unavailable"); + } + } + }; + final PathPatternTree tree = new PathPatternTree(); + tree.appendPathPattern(new PartialPath("root.reelection.d.s")); + tree.constructTree(); + final DeleteTimeSeriesProcedure procedure = + new DeleteTimeSeriesProcedure("reelection", tree, false, false) { + @Override + protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteTimeSeriesState state) { + return Flow.NO_MORE_STATE; + } + }; + final ProcedureExecutor executor = new ProcedureExecutor<>(null, store); + store.start(); + executor.init(1); + executor.startWorkers(); + try { + executor.submitProcedureAndPersist(procedure); + Assert.assertTrue(completionRetried.await(10, TimeUnit.SECONDS)); + executor.stop(); + executor.join(); + Assert.assertTrue(executor.getProcedures().containsKey(procedure.getProcId())); + Assert.assertFalse(executor.isCompleted(procedure.getProcId())); + + // A different leader completes the task before this same executor becomes leader again. + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + procedure.serialize(new DataOutputStream(bytes)); + final ByteBuffer buffer = ByteBuffer.wrap(bytes.toByteArray()); + buffer.getShort(); + final DeleteTimeSeriesProcedure finished = new DeleteTimeSeriesProcedure(false); + finished.deserialize(buffer); + Assert.assertTrue(finished.isSuccess()); + recovered.set(Collections.singletonList(finished)); + allowCompletion.set(true); + executor.init(1); + executor.startWorkers(); + Assert.assertTrue(executor.isFinished(procedure.getProcId())); + Assert.assertTrue(executor.getProcedures().isEmpty()); + Assert.assertEquals(Collections.singletonList(finished), executor.getCompletedProcedures()); + Assert.assertEquals(0, executor.getDeleteTimeSeriesScheduler().getWaitingTaskCount()); + Assert.assertNull(executor.getRollbackStack(procedure.getProcId())); + } finally { + allowCompletion.set(true); + executor.stop(); + executor.join(); + store.stop(); + } + } + + @Test + public void testStatePersistenceFailureBlocksNextStepAndCompletion() throws Exception { + final AtomicBoolean allowTransition = new AtomicBoolean(); + final AtomicBoolean allowCompletion = new AtomicBoolean(); + final AtomicInteger transitionAttempts = new AtomicInteger(); + final AtomicInteger completionAttempts = new AtomicInteger(); + final CountDownLatch transitionRetried = new CountDownLatch(1); + final CountDownLatch completionRetried = new CountDownLatch(1); + final CountDownLatch recreated = new CountDownLatch(1); + final NoopProcedureStore store = + new NoopProcedureStore() { + @Override + public void update(Procedure procedure) { + final DeleteTimeSeriesProcedure deletion = (DeleteTimeSeriesProcedure) procedure; + if ("RECREATE_DATABASES".equals(deletion.getDeleteProgress()) + && !allowTransition.get()) { + if (transitionAttempts.incrementAndGet() >= 2) { + transitionRetried.countDown(); + } + throw new IllegalStateException("transition store unavailable"); + } + if (deletion.isSuccess() && !allowCompletion.get()) { + if (completionAttempts.incrementAndGet() >= 2) { + completionRetried.countDown(); + } + throw new IllegalStateException("completion store unavailable"); + } + } + }; + final DeleteTimeSeriesProcedure procedure = + new DeleteTimeSeriesProcedure(false) { + @Override + protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteTimeSeriesState state) { + if (state == DeleteTimeSeriesState.PREPARE_DELETE) { + setNextState(DeleteTimeSeriesState.RECREATE_DATABASES); + return Flow.HAS_MORE_STATE; + } + recreated.countDown(); + return Flow.NO_MORE_STATE; + } + }; + final ProcedureExecutor executor = new ProcedureExecutor<>(null, store); + store.start(); + executor.init(1); + executor.startWorkers(); + try { + executor.submitProcedureAndPersist(procedure); + Assert.assertTrue(transitionRetried.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, recreated.getCount()); + Assert.assertFalse(executor.isCompleted(procedure.getProcId())); + allowTransition.set(true); + Assert.assertTrue(completionRetried.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(0, recreated.getCount()); + Assert.assertFalse(executor.isCompleted(procedure.getProcId())); + Assert.assertTrue(executor.getProcedures().containsKey(procedure.getProcId())); + allowCompletion.set(true); + ProcedureTestUtil.waitForProcedure(executor, procedure.getProcId()); + Assert.assertTrue(executor.isCompleted(procedure.getProcId())); + } finally { + executor.stop(); + executor.join(); + store.stop(); + } + } + + @Test + public void testRollbackPersistenceFailureDoesNotCompleteDuringShutdown() throws Exception { + final AtomicInteger attempts = new AtomicInteger(); + final CountDownLatch rollbackRetried = new CountDownLatch(1); + final NoopProcedureStore store = + new NoopProcedureStore() { + @Override + public void update(Procedure procedure) { + if (procedure.getState() == ProcedureState.ROLLEDBACK) { + if (attempts.incrementAndGet() >= 2) { + rollbackRetried.countDown(); + } + throw new IllegalStateException("rollback store unavailable"); + } + } + }; + final ProcedureExecutor executor = new ProcedureExecutor<>(null, store); + final DeleteTimeSeriesProcedure procedure = + new DeleteTimeSeriesProcedure(false) { + @Override + protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteTimeSeriesState state) { + setFailure(new ProcedureException("deletion failed")); + return Flow.NO_MORE_STATE; + } + }; + store.start(); + executor.init(1); + executor.startWorkers(); + try { + executor.submitProcedureAndPersist(procedure); + Assert.assertTrue(rollbackRetried.await(10, TimeUnit.SECONDS)); + Assert.assertFalse(executor.isCompleted(procedure.getProcId())); + executor.stop(); + executor.join(); + Assert.assertFalse(executor.isCompleted(procedure.getProcId())); + Assert.assertTrue(executor.getProcedures().containsKey(procedure.getProcId())); + } finally { + executor.stop(); + executor.join(); + store.stop(); + } + } + + @Test + public void testDeletionDoesNotOccupyRegularWorkerAndRetainsConcurrencySlot() throws Exception { + final NoopProcedureStore store = new NoopProcedureStore(); + final ProcedureExecutor executor = new ProcedureExecutor<>(null, store); + final CountDownLatch firstStarted = new CountDownLatch(1); + final CountDownLatch releaseFirst = new CountDownLatch(1); + final CountDownLatch secondStarted = new CountDownLatch(1); + final CountDownLatch regularCompleted = new CountDownLatch(1); + store.start(); + executor.init(1); + executor.startWorkers(); + try { + executor.submitProcedureAndPersist( + new DeleteTimeSeriesProcedure(false) { + @Override + protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteTimeSeriesState state) + throws InterruptedException { + firstStarted.countDown(); + releaseFirst.await(); + return Flow.NO_MORE_STATE; + } + }); + Assert.assertTrue(firstStarted.await(10, TimeUnit.SECONDS)); + executor.submitProcedureAndPersist( + new DeleteTimeSeriesProcedure(false) { + @Override + protected Flow executeFromState( + ConfigNodeProcedureEnv env, DeleteTimeSeriesState state) { + secondStarted.countDown(); + return Flow.NO_MORE_STATE; + } + }); + executor.submitProcedure( + new Procedure() { + @Override + protected Procedure[] execute(ConfigNodeProcedureEnv env) { + regularCompleted.countDown(); + return null; + } + + @Override + protected void rollback(ConfigNodeProcedureEnv env) {} + }); + Assert.assertTrue(regularCompleted.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, secondStarted.getCount()); + Assert.assertEquals(1, executor.getDeleteTimeSeriesScheduler().getWaitingTaskCount()); + releaseFirst.countDown(); + Assert.assertTrue(secondStarted.await(10, TimeUnit.SECONDS)); + } finally { + releaseFirst.countDown(); + executor.stop(); + executor.join(); + store.stop(); + } + } + + @Test + public void testDatabaseRecreationRecoveryDoesNotDeleteAgain() throws Exception { + assertDatabaseRecreationRecovery(false); + } + + @Test + public void testDatabaseDeletionFailureSurvivesRecoveryAndStillRecreates() throws Exception { + assertDatabaseRecreationRecovery(true); + } + + @Test + public void testDatabaseRecreationWaitsForDurableDatabaseDeletion() throws Exception { + final AtomicBoolean allowCompletion = new AtomicBoolean(); + final AtomicInteger terminalAttempts = new AtomicInteger(); + final CountDownLatch completionRetried = new CountDownLatch(1); + final AtomicReference databaseDeletion = new AtomicReference<>(); + final AtomicReference databaseWorker = new AtomicReference<>(); + final NoopProcedureStore store = + new NoopProcedureStore() { + @Override + public void update(Procedure procedure) { + if (procedure instanceof DeleteDatabaseProcedure + && procedure.isSuccess() + && !allowCompletion.get()) { + databaseDeletion.set((DeleteDatabaseProcedure) procedure); + databaseWorker.set(Thread.currentThread()); + if (terminalAttempts.incrementAndGet() >= 2) { + completionRetried.countDown(); + } + throw new IllegalStateException("database completion store unavailable"); + } + } + }; + final ConfigNodeProcedureEnv env = mock(ConfigNodeProcedureEnv.class); + final ConfigManager manager = mock(ConfigManager.class); + final ClusterSchemaManager schemaManager = mock(ClusterSchemaManager.class); + final PartitionManager partitionManager = mock(PartitionManager.class); + final TDatabaseSchema database = new TDatabaseSchema("root.durable_database_deletion"); + when(env.getConfigManager()).thenReturn(manager); + when(env.invalidateCache(database.getName())).thenReturn(true); + when(env.getAllReplicaSets(database.getName())).thenReturn(Collections.emptyList()); + when(env.deleteDatabaseConfig(database.getName(), false)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + when(manager.getClusterSchemaManager()).thenReturn(schemaManager); + when(manager.getLoadManager()).thenReturn(mock(LoadManager.class)); + when(manager.getPartitionManager()).thenReturn(partitionManager); + when(partitionManager.getRegionMaintainer()).thenReturn(mock(ScheduledExecutorService.class)); + when(schemaManager.getMatchedDatabaseSchemasByPrefix(new PartialPath(database.getName()))) + .thenReturn(new HashMap<>(Collections.singletonMap(database.getName(), database))); + when(manager.getRelatedSchemaRegionGroup(any())).thenReturn(Collections.emptyMap()); + when(schemaManager.setDatabase(any(), anyBoolean())) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + final ProcedureExecutor executor = new ProcedureExecutor<>(env, store); + final ProcedureManager procedureManager = + new ProcedureManager(manager, mock(ProcedureInfo.class)); + procedureManager.setExecutor(executor); + when(manager.getProcedureManager()).thenReturn(procedureManager); + final PathPatternTree tree = new PathPatternTree(); + tree.appendPathPattern(new PartialPath(database.getName() + ".**")); + tree.constructTree(); + final DeleteTimeSeriesProcedure procedure = + new DeleteTimeSeriesProcedure("durable-database-deletion", tree, false, false); + store.start(); + executor.init(1); + executor.startWorkers(); + try { + executor.submitProcedureAndPersist(procedure); + Assert.assertTrue(completionRetried.await(10, TimeUnit.SECONDS)); + Assert.assertTrue(terminalAttempts.get() >= 2); + Assert.assertEquals("DELETE_DATABASES", procedure.getDeleteProgress()); + verify(schemaManager, never()).setDatabase(any(), anyBoolean()); + // An interrupted unlock retry must not turn an unpersisted child into a completed root. + final Thread worker = databaseWorker.get(); + worker.interrupt(); + worker.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse(worker.isAlive()); + Assert.assertFalse(executor.isCompleted(databaseDeletion.get().getProcId())); + Assert.assertTrue(executor.getProcedures().containsKey(databaseDeletion.get().getProcId())); + executor.stop(); + executor.join(); + verify(schemaManager, never()).setDatabase(any(), anyBoolean()); + } finally { + allowCompletion.set(true); + executor.stop(); + executor.join(); + store.stop(); + } + } + + private void assertDatabaseRecreationRecovery(boolean deleteFails) throws Exception { + final TDatabaseSchema database = new TDatabaseSchema("root.async_full"); + final PathPatternTree tree = new PathPatternTree(); + tree.appendPathPattern(new PartialPath(database.getName() + ".**")); + tree.constructTree(); + final SteppableDeletion procedure = new SteppableDeletion("recovery", tree); + final ConfigNodeProcedureEnv env = mock(ConfigNodeProcedureEnv.class); + final ConfigManager manager = mock(ConfigManager.class); + final ClusterSchemaManager schemaManager = mock(ClusterSchemaManager.class); + final ProcedureManager procedureManager = mock(ProcedureManager.class); + when(env.getConfigManager()).thenReturn(manager); + when(manager.getClusterSchemaManager()).thenReturn(schemaManager); + when(manager.getProcedureManager()).thenReturn(procedureManager); + when(manager.getLoadManager()).thenReturn(mock(LoadManager.class)); + when(schemaManager.getMatchedDatabaseSchemasByPrefix(new PartialPath(database.getName()))) + .thenReturn(new HashMap<>(Collections.singletonMap(database.getName(), database))); + when(manager.getRelatedSchemaRegionGroup(any())).thenReturn(Collections.emptyMap()); + when(procedureManager.deleteDatabases( + Collections.singletonList(database), false, Long.MAX_VALUE, "recovery")) + .thenReturn( + new TSStatus( + deleteFails + ? TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + : TSStatusCode.SUCCESS_STATUS.getStatusCode()) + .setMessage(deleteFails ? "database deletion failed" : null)); + procedure.step(env); + procedure.step(env); + Assert.assertEquals("RECREATE_DATABASES", procedure.getDeleteProgress()); + verify(schemaManager, never()).setDatabase(any(), anyBoolean()); + + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + procedure.serialize(new DataOutputStream(bytes)); + final ByteBuffer buffer = ByteBuffer.wrap(bytes.toByteArray()); + buffer.getShort(); + final SteppableDeletion restored = new SteppableDeletion(); + restored.deserialize(buffer); + Assert.assertEquals("RECREATE_DATABASES", restored.getDeleteProgress()); + // Successful recreation can also have happened just before a restart. + when(schemaManager.setDatabase(any(), anyBoolean())) + .thenReturn( + new TSStatus( + deleteFails + ? TSStatusCode.SUCCESS_STATUS.getStatusCode() + : TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode())); + when(schemaManager.getDatabaseSchemaByName(database.getName())).thenReturn(database); + restored.step(env); + Assert.assertEquals(deleteFails, restored.hasException()); + if (deleteFails) { + Assert.assertTrue(restored.getException().getMessage().contains("database deletion failed")); + } + verify(procedureManager, times(1)) + .deleteDatabases(Collections.singletonList(database), false, Long.MAX_VALUE, "recovery"); + verify(schemaManager, times(1)).setDatabase(any(), anyBoolean()); + } + + private static class SteppableDeletion extends DeleteTimeSeriesProcedure { + SteppableDeletion() { + super(false); + } + + SteppableDeletion(String queryId, PathPatternTree tree) { + super(queryId, tree, false, false); + } + + void step(ConfigNodeProcedureEnv env) throws InterruptedException { + super.execute(env); + } + } + + @Test + public void testPreparedDatabaseDeletionSurvivesSerialization() throws Exception { + final PathPatternTree tree = new PathPatternTree(); + tree.appendPathPattern(new PartialPath("root.async_full.**")); + tree.appendPathPattern(new PartialPath("root.async_other.d.s")); + tree.constructTree(); + final DeleteTimeSeriesProcedure procedure = + new DeleteTimeSeriesProcedure("prepared", tree, false, false); + final ConfigNodeProcedureEnv env = mock(ConfigNodeProcedureEnv.class); + final ConfigManager manager = mock(ConfigManager.class); + final ClusterSchemaManager schemaManager = mock(ClusterSchemaManager.class); + when(env.getConfigManager()).thenReturn(manager); + when(manager.getClusterSchemaManager()).thenReturn(schemaManager); + when(manager.getLoadManager()).thenReturn(mock(LoadManager.class)); + when(schemaManager.getMatchedDatabaseSchemasByPrefix(new PartialPath("root.async_full"))) + .thenReturn( + new HashMap<>( + Collections.singletonMap( + "root.async_full", new TDatabaseSchema("root.async_full")))); + when(manager.getRelatedSchemaRegionGroup(any())).thenReturn(Collections.emptyMap()); + procedure.executeFromState(env, DeleteTimeSeriesState.PREPARE_DELETE); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + procedure.serialize(new DataOutputStream(bytes)); + final ByteBuffer buffer = ByteBuffer.wrap(bytes.toByteArray()); + buffer.getShort(); + final DeleteTimeSeriesProcedure restored = new DeleteTimeSeriesProcedure(false); + restored.deserialize(buffer); + Assert.assertEquals(procedure, restored); + Assert.assertEquals(tree, restored.getPatternTree()); + } + + @Test + public void testLegacyProcedureWithoutAsyncSuffixCanBeRecovered() throws Exception { + final PathPatternTree tree = new PathPatternTree(); + tree.appendPathPattern(new PartialPath("root.legacy.d.s")); + tree.constructTree(); + final DeleteTimeSeriesProcedure procedure = + new DeleteTimeSeriesProcedure("legacy", tree, false, false); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + procedure.serialize(new DataOutputStream(bytes)); + // Remove the original-tree flag, empty database-schema count and null deletion-failure string. + final ByteBuffer buffer = ByteBuffer.wrap(bytes.toByteArray(), 0, bytes.size() - 9); + buffer.getShort(); + final DeleteTimeSeriesProcedure restored = new DeleteTimeSeriesProcedure(false); + restored.deserialize(buffer); + Assert.assertEquals(tree, restored.getPatternTree()); + Assert.assertEquals("legacy", restored.getQueryId()); + } + @Test public void serializeDeserializeTest() throws IllegalPathException, IOException { String queryId = "1"; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedureTest.java index 6271201b04dd0..517632d24e8dd 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedureTest.java @@ -20,7 +20,15 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.confignode.procedure.NoopProcedureStore; +import org.apache.iotdb.confignode.procedure.Procedure; +import org.apache.iotdb.confignode.procedure.ProcedureExecutor; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.state.ProcedureState; +import org.apache.iotdb.confignode.procedure.state.schema.DropTableColumnState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.confignode.procedure.util.ProcedureTestUtil; import org.junit.Assert; import org.junit.Test; @@ -29,8 +37,104 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; public class DropTableColumnProcedureTest { + @Test + public void testColumnFailurePersistenceDoesNotBlockShutdown() throws Exception { + final AtomicBoolean allowCompletion = new AtomicBoolean(); + final CountDownLatch completionRetried = new CountDownLatch(1); + final AtomicInteger attempts = new AtomicInteger(); + final NoopProcedureStore store = + new NoopProcedureStore() { + @Override + public void update(Procedure procedure) { + if (procedure.getState() == ProcedureState.ROLLEDBACK && !allowCompletion.get()) { + if (attempts.incrementAndGet() >= 2) { + completionRetried.countDown(); + } + throw new IllegalStateException("column failure store unavailable"); + } + } + }; + final DropTableColumnProcedure procedure = + new DropTableColumnProcedure(false) { + @Override + protected Flow executeFromState(ConfigNodeProcedureEnv env, DropTableColumnState state) { + setFailure(new ProcedureException("column deletion failed")); + return Flow.NO_MORE_STATE; + } + }; + final ProcedureExecutor executor = new ProcedureExecutor<>(null, store); + final ExecutorService joiner = Executors.newSingleThreadExecutor(); + store.start(); + executor.init(1); + executor.startWorkers(); + try { + executor.submitProcedureAndPersist(procedure); + Assert.assertTrue(completionRetried.await(10, TimeUnit.SECONDS)); + executor.stop(); + final Future stopped = joiner.submit(executor::join); + stopped.get(5, TimeUnit.SECONDS); + Assert.assertFalse(executor.isCompleted(procedure.getProcId())); + Assert.assertTrue(executor.getProcedures().containsKey(procedure.getProcId())); + } finally { + allowCompletion.set(true); + executor.stop(); + executor.join(); + joiner.shutdownNow(); + store.stop(); + } + } + + @Test + public void testColumnCompletionWaitsForPersistence() throws Exception { + final AtomicBoolean allowCompletion = new AtomicBoolean(); + final AtomicInteger attempts = new AtomicInteger(); + final CountDownLatch completionRetried = new CountDownLatch(1); + final NoopProcedureStore store = + new NoopProcedureStore() { + @Override + public void update(Procedure procedure) { + if (procedure.isSuccess() && !allowCompletion.get()) { + if (attempts.incrementAndGet() >= 2) { + completionRetried.countDown(); + } + throw new IllegalStateException("column completion store unavailable"); + } + } + }; + final DropTableColumnProcedure procedure = + new DropTableColumnProcedure(false) { + @Override + protected Flow executeFromState(ConfigNodeProcedureEnv env, DropTableColumnState state) { + return Flow.NO_MORE_STATE; + } + }; + final ProcedureExecutor executor = new ProcedureExecutor<>(null, store); + store.start(); + executor.init(1); + executor.startWorkers(); + try { + executor.submitProcedureAndPersist(procedure); + Assert.assertTrue(completionRetried.await(10, TimeUnit.SECONDS)); + Assert.assertFalse(executor.isCompleted(procedure.getProcId())); + allowCompletion.set(true); + ProcedureTestUtil.waitForProcedure(executor, procedure.getProcId()); + Assert.assertTrue(executor.isCompleted(procedure.getProcId())); + } finally { + executor.stop(); + executor.join(); + store.stop(); + } + } + @Test public void serializeDeserializeTest() throws IllegalPathException, IOException { final DropTableColumnProcedure dropTableColumnProcedure = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/scheduler/DeleteTimeSeriesProcedureSchedulerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/scheduler/DeleteTimeSeriesProcedureSchedulerTest.java new file mode 100644 index 0000000000000..ebdd6410b933c --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/scheduler/DeleteTimeSeriesProcedureSchedulerTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.scheduler; + +import org.apache.iotdb.confignode.procedure.Procedure; + +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DeleteTimeSeriesProcedureSchedulerTest { + private Procedure procedure(long id) { + final Procedure procedure = mock(Procedure.class); + when(procedure.getProcId()).thenReturn(id); + return procedure; + } + + @Test + public void testConcurrencySlotSurvivesStateTransitionsAndRollback() { + final DeleteTimeSeriesProcedureScheduler scheduler = + new DeleteTimeSeriesProcedureScheduler(1, 1); + final Procedure first = procedure(1); + final Procedure second = procedure(2); + scheduler.start(); + scheduler.addBack(first); + scheduler.addBack(second); + assertSame(first, scheduler.poll()); + assertTrue(scheduler.isQueued(2)); + assertEquals(1, scheduler.getWaitingTaskCount()); + scheduler.addFront(first); + assertSame(first, scheduler.poll(0, TimeUnit.MILLISECONDS)); + // A root remains admitted even while none of its states is runnable. + assertNull(scheduler.poll(0, TimeUnit.MILLISECONDS)); + scheduler.completionCleanup(first); + assertSame(second, scheduler.poll(1, TimeUnit.SECONDS)); + scheduler.stop(); + } + + @Test + public void testStartIntervalAlsoAppliesWhenMultipleSlotsAreAvailable() { + final DeleteTimeSeriesProcedureScheduler scheduler = + new DeleteTimeSeriesProcedureScheduler(2, 100); + final Procedure first = procedure(1); + final Procedure second = procedure(2); + final Procedure third = procedure(3); + // All tasks may be queued before any worker starts. Their execution starts still need spacing. + scheduler.addBack(first); + scheduler.addBack(second); + scheduler.addBack(third); + scheduler.start(); + final long started = System.nanoTime(); + assertSame(first, scheduler.poll()); + assertSame(second, scheduler.poll(2, TimeUnit.SECONDS)); + assertTrue(System.nanoTime() - started >= TimeUnit.MILLISECONDS.toNanos(100)); + assertNull(scheduler.poll(0, TimeUnit.MILLISECONDS)); + scheduler.completionCleanup(first); + assertSame(third, scheduler.poll(2, TimeUnit.SECONDS)); + scheduler.stop(); + } + + @Test + public void testEnqueueDoesNotWaitForExecutionInterval() throws Exception { + final DeleteTimeSeriesProcedureScheduler scheduler = + new DeleteTimeSeriesProcedureScheduler(1, TimeUnit.HOURS.toMillis(1)); + final ExecutorService submitter = Executors.newSingleThreadExecutor(); + try { + final Future submitted = + submitter.submit( + () -> { + scheduler.addBack(procedure(1)); + scheduler.addBack(procedure(2)); + scheduler.addBack(procedure(3)); + }); + submitted.get(5, TimeUnit.SECONDS); + assertEquals(3, scheduler.getWaitingTaskCount()); + } finally { + scheduler.stop(); + submitter.shutdownNow(); + } + } + + @Test + public void testStopUnblocksAdmissionAndClearAllowsRecovery() throws Exception { + final DeleteTimeSeriesProcedureScheduler scheduler = + new DeleteTimeSeriesProcedureScheduler(1, 1); + final Procedure first = procedure(1); + final Procedure recovered = procedure(2); + scheduler.start(); + scheduler.addBack(first); + assertSame(first, scheduler.poll()); + scheduler.addBack(recovered); + final ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + final CountDownLatch started = new CountDownLatch(1); + final Future waiting = + executor.submit( + () -> { + started.countDown(); + return scheduler.poll(); + }); + assertTrue(started.await(1, TimeUnit.SECONDS)); + scheduler.stop(); + assertNull(waiting.get(1, TimeUnit.SECONDS)); + scheduler.clear(); + scheduler.start(); + scheduler.addBack(recovered); + assertSame(recovered, scheduler.poll(0, TimeUnit.MILLISECONDS)); + } finally { + scheduler.stop(); + executor.shutdownNow(); + } + } +} diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index c0b6c32d3fce5..7372323c0deda 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4925,4 +4925,8 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_ARG_ARG_MS_ARG_INVOCATIONS_ARG_APPLICATIONS_F056BD84 = "%s: %s ms, %s invocations, %s applications"; + public static final String MESSAGE_COLUMN_DELETION_IS_ASYNCHRONOUS_THE_SPECIFIED_TABLE_OR_COLUMN_DOES_NOT_EXIST_NO_TASK_WAS_SUBMITTED_08F7DB64 = + "Column deletion is asynchronous. The specified table or column does not exist; no task was submitted."; + public static final String MESSAGE_TABLE_DELETION_IS_ASYNCHRONOUS_THE_SPECIFIED_TABLE_DOES_NOT_EXIST_NO_TASK_WAS_SUBMITTED_74E6521A = + "Table deletion is asynchronous. The specified table does not exist; no task was submitted."; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 522bf93d99ad3..4096c0c152e63 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -5625,4 +5625,8 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_ARG_ARG_MS_ARG_INVOCATIONS_ARG_APPLICATIONS_F056BD84 = "%s：%s ms，调用 %s 次，应用 %s 次"; + public static final String MESSAGE_COLUMN_DELETION_IS_ASYNCHRONOUS_THE_SPECIFIED_TABLE_OR_COLUMN_DOES_NOT_EXIST_NO_TASK_WAS_SUBMITTED_08F7DB64 = + "列删除采用异步执行。指定的表或列不存在，未提交任务。"; + public static final String MESSAGE_TABLE_DELETION_IS_ASYNCHRONOUS_THE_SPECIFIED_TABLE_DOES_NOT_EXIST_NO_TASK_WAS_SUBMITTED_74E6521A = + "表删除采用异步执行。指定的表不存在，未提交任务。"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 02d6c51ef2dcf..17cc7bbbd14bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -166,6 +166,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodes4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowDeleteTimeSeriesProceduresResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDropTableProceduresResp; import org.apache.iotdb.confignode.rpc.thrift.TShowPipePluginReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; @@ -1277,6 +1278,13 @@ public TShowTable4InformationSchemaResp showTables4InformationSchema() throws TE () -> client.showTables4InformationSchema(), resp -> !updateConfigNodeLeader(resp.status)); } + @Override + public TShowDeleteTimeSeriesProceduresResp showDeleteTimeSeriesProcedures() throws TException { + return executeRemoteCallWithRetry( + () -> client.showDeleteTimeSeriesProcedures(), + resp -> !updateConfigNodeLeader(resp.status)); + } + @Override public TShowDropTableProceduresResp showDropTableProcedures() throws TException { return executeRemoteCallWithRetry( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java index 671c817d8f4ee..64fe4fdbe8fec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java @@ -169,6 +169,10 @@ public static DatasetHeader getShowPipeHeader() { return new DatasetHeader(ColumnHeaderConstant.showPipeColumnHeaders, true); } + public static DatasetHeader getShowDeleteTimeSeriesProgressHeader() { + return new DatasetHeader(ColumnHeaderConstant.showDeleteTimeSeriesProgressColumnHeaders, true); + } + public static DatasetHeader getShowRepairDataPartitionTableProgressHeader() { return new DatasetHeader( ColumnHeaderConstant.showRepairDataPartitionTableProgressColumnHeaders, true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index 269db0cb85f50..102a79f371dcc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -113,6 +113,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetSystemStatusTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.ShowConfigurationTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.ShowDeleteTimeSeriesProgressTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.ShowRepairDataPartitionTableProgressTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StartRepairDataTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StopRepairDataTask; @@ -228,6 +229,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentSqlDialectStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDeleteTimeSeriesProgressStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowRepairDataPartitionTableProgressStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; @@ -436,6 +438,12 @@ public IConfigTask visitRepairDataPartitionTable( return new RepairDataPartitionTableTask(); } + @Override + public IConfigTask visitShowDeleteTimeSeriesProgress( + ShowDeleteTimeSeriesProgressStatement statement, MPPQueryContext context) { + return new ShowDeleteTimeSeriesProgressTask(); + } + @Override public IConfigTask visitShowRepairDataPartitionTableProgress( ShowRepairDataPartitionTableProgressStatement showRepairDataPartitionTableProgressStatement, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 0575c6938c7be..c9eaff05a35b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -167,6 +167,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowConfigNodesResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowDeleteTimeSeriesProceduresResp; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipePluginReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; @@ -262,6 +263,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentUserTask; import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowVersionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.ShowConfigurationTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.ShowDeleteTimeSeriesProgressTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.ShowRepairDataPartitionTableProgressTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.TestConnectionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.ShowPipeTask; @@ -1548,6 +1550,23 @@ public SettableFuture repairDataPartitionTable() { return future; } + @Override + public SettableFuture showDeleteTimeSeriesProgress() { + final SettableFuture future = SettableFuture.create(); + try (ConfigNodeClient client = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + final TShowDeleteTimeSeriesProceduresResp response = client.showDeleteTimeSeriesProcedures(); + if (response.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + ShowDeleteTimeSeriesProgressTask.buildTsBlock(response.getProcedureInfoList(), future); + } else { + future.setException(new IoTDBException(response.getStatus())); + } + } catch (ClientManagerException | TException e) { + future.setException(e); + } + return future; + } + @Override public SettableFuture showRepairDataPartitionTableProgress() { SettableFuture future = SettableFuture.create(); @@ -3462,21 +3481,7 @@ public SettableFuture deleteTimeSeries( req.setMayDeleteAudit(deleteTimeSeriesStatement.isMayDeleteAudit()); try (ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TSStatus tsStatus; - do { - try { - tsStatus = client.deleteTimeSeries(req); - } catch (TTransportException e) { - if (e.getType() == TTransportException.TIMED_OUT - || e.getCause() instanceof SocketTimeoutException) { - // time out mainly caused by slow execution, wait until - tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); - } else { - throw e; - } - } - // keep waiting until task ends - } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + final TSStatus tsStatus = client.deleteTimeSeries(req); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { if (tsStatus.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { @@ -3486,7 +3491,7 @@ public SettableFuture deleteTimeSeries( future.setException(new IoTDBException(tsStatus)); } } else { - future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + future.set(new ConfigTaskResult(tsStatus)); } } catch (final ClientManagerException | TException e) { future.setException(e); @@ -5162,7 +5167,16 @@ public SettableFuture alterTableDropColumn( || (TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() == tsStatus.getCode() && tableIfExists) || (TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode() == tsStatus.getCode() && columnIfExists)) { - future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + future.set( + new ConfigTaskResult( + tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? tsStatus + : isView + ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) + : RpcUtils.getStatus( + TSStatusCode.SUCCESS_STATUS, + DataNodeQueryMessages + .MESSAGE_COLUMN_DELETION_IS_ASYNCHRONOUS_THE_SPECIFIED_TABLE_OR_COLUMN_DOES_NOT_EXIST_NO_TASK_WAS_SUBMITTED_08F7DB64))); } else { future.setException( new IoTDBException(getTableErrorMessage(tsStatus, database), tsStatus.getCode())); @@ -5373,7 +5387,16 @@ public SettableFuture dropTable( if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() || (TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() == tsStatus.getCode() && ifExists)) { - future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + future.set( + new ConfigTaskResult( + tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? tsStatus + : isView + ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) + : RpcUtils.getStatus( + TSStatusCode.SUCCESS_STATUS, + DataNodeQueryMessages + .MESSAGE_TABLE_DELETION_IS_ASYNCHRONOUS_THE_SPECIFIED_TABLE_DOES_NOT_EXIST_NO_TASK_WAS_SUBMITTED_74E6521A))); } else { future.setException( new IoTDBException(getTableErrorMessage(tsStatus, database), tsStatus.getCode())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 9fd3f7e9d5df0..c552f784d4e08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -161,6 +161,8 @@ SettableFuture showPipePlugins( SettableFuture repairDataPartitionTable(); + SettableFuture showDeleteTimeSeriesProgress(); + SettableFuture showRepairDataPartitionTableProgress(); SettableFuture flush(TFlushReq tFlushReq, boolean onCluster); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ShowDeleteTimeSeriesProgressTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ShowDeleteTimeSeriesProgressTask.java new file mode 100644 index 0000000000000..efc6bf9ae2488 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ShowDeleteTimeSeriesProgressTask.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.sys; + +import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesProcedureInfo; +import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.rpc.TSStatusCode; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.Binary; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class ShowDeleteTimeSeriesProgressTask implements IConfigTask { + @Override + public ListenableFuture execute(IConfigTaskExecutor executor) + throws InterruptedException { + return executor.showDeleteTimeSeriesProgress(); + } + + public static void buildTsBlock( + List procedures, SettableFuture future) { + final TsBlockBuilder builder = + new TsBlockBuilder( + Arrays.asList( + TSDataType.INT64, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT)); + for (TDeleteTimeSeriesProcedureInfo info : + procedures == null ? Collections.emptyList() : procedures) { + builder.getTimeColumnBuilder().writeLong(0); + builder.getColumnBuilder(0).writeLong(info.getProcedureId()); + builder + .getColumnBuilder(1) + .writeBinary(new Binary(info.getQueryId(), TSFileConfig.STRING_CHARSET)); + builder + .getColumnBuilder(2) + .writeBinary(new Binary(info.getPathPatterns(), TSFileConfig.STRING_CHARSET)); + builder + .getColumnBuilder(3) + .writeBinary(new Binary(info.getState(), TSFileConfig.STRING_CHARSET)); + builder + .getColumnBuilder(4) + .writeBinary(new Binary(info.getProgress(), TSFileConfig.STRING_CHARSET)); + if (info.isSetErrorMessage()) { + builder + .getColumnBuilder(5) + .writeBinary(new Binary(info.getErrorMessage(), TSFileConfig.STRING_CHARSET)); + } else { + builder.getColumnBuilder(5).appendNull(); + } + builder.declarePosition(); + } + future.set( + new ConfigTaskResult( + TSStatusCode.SUCCESS_STATUS, + builder.build(), + DatasetHeaderFactory.getShowDeleteTimeSeriesProgressHeader())); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index f12b2bf159dd4..c0be20613b978 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -250,6 +250,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentSqlDialectStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDeleteTimeSeriesProgressStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; @@ -3876,6 +3877,12 @@ public Statement visitRepairDataPartitionTable( return new RepairDataPartitionTable(); } + @Override + public Statement visitShowDeleteTimeSeriesProgress( + IoTDBSqlParser.ShowDeleteTimeSeriesProgressContext ctx) { + return new ShowDeleteTimeSeriesProgressStatement(); + } + @Override public Statement visitShowRepairDataPartitionTableProgress( IoTDBSqlParser.ShowRepairDataPartitionTableProgressContext ctx) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java index aa1cab64b0f05..48d1cfbd8e2b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java @@ -161,6 +161,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentSqlDialectStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDeleteTimeSeriesProgressStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; @@ -2025,6 +2026,13 @@ public TSStatus visitRepairDataPartitionTable( AuditEventType.INTEGRITY_CHECK); } + @Override + public TSStatus visitShowDeleteTimeSeriesProgress( + ShowDeleteTimeSeriesProgressStatement statement, TreeAccessCheckContext context) { + return checkGlobalAuth( + context.setAuditLogOperation(AuditLogOperation.QUERY), PrivilegeType.MAINTAIN, () -> ""); + } + @Override public TSStatus visitShowRepairDataPartitionTableProgress( ShowRepairDataPartitionTableProgressStatement showRepairDataPartitionTableProgressStatement, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java index 2384a2e34ac20..cbdb5d6b9e891 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java @@ -205,4 +205,5 @@ public enum StatementType { SHOW_DISK_USAGE, SHOW_RECEIVERS, + SHOW_DELETE_TIMESERIES_PROGRESS, } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index 467e2dc3bc879..8df35f609940e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -147,6 +147,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentSqlDialectStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDeleteTimeSeriesProgressStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; @@ -530,6 +531,11 @@ public R visitRepairDataPartitionTable( return visitStatement(repairDataPartitionTable, context); } + public R visitShowDeleteTimeSeriesProgress( + ShowDeleteTimeSeriesProgressStatement statement, C context) { + return visitStatement(statement, context); + } + public R visitShowRepairDataPartitionTableProgress( ShowRepairDataPartitionTableProgressStatement showRepairDataPartitionTableProgressStatement, C context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowDeleteTimeSeriesProgressStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowDeleteTimeSeriesProgressStatement.java new file mode 100644 index 0000000000000..931fd92f5a5d7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowDeleteTimeSeriesProgressStatement.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.sys; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; + +public class ShowDeleteTimeSeriesProgressStatement extends Statement implements IConfigStatement { + + public ShowDeleteTimeSeriesProgressStatement() { + this.statementType = StatementType.SHOW_DELETE_TIMESERIES_PROGRESS; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public QueryType getQueryType() { + return QueryType.READ; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitShowDeleteTimeSeriesProgress(this, context); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java index 0acb6816bfb8a..952e825bc272b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java @@ -104,6 +104,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -125,6 +126,16 @@ public class StatementGeneratorTest { + @Test + public void testShowDeleteTimeSeriesProgress() { + for (String sql : + new String[] {"SHOW DELETE TIMESERIES PROGRESS", "show delete timeseries progress"}) { + assertEquals( + StatementType.SHOW_DELETE_TIMESERIES_PROGRESS, + StatementGenerator.createStatement(sql, ZoneId.systemDefault()).getType()); + } + } + @Test public void testAlterTopicStatement() { final Statement statement = diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index a3fe0970318c8..068e8a6561ce3 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -2381,6 +2381,22 @@ procedure_completed_clean_interval=30 # Datatype: int procedure_completed_evict_ttl=60 +# Maximum number of tree-model timeseries deletion tasks waiting to execute on the ConfigNode. +# effectiveMode: restart +# Datatype: int, greater than 0 +timeseries_deletion_queue_capacity=1024 + +# Minimum interval between starting tree-model timeseries deletion tasks, in milliseconds. +# Measured when tasks leave the queue to start execution; submissions are not rate-limited. +# effectiveMode: restart +# Datatype: long, greater than 0 +timeseries_deletion_execution_interval_ms=1000 + +# Maximum number of tree-model timeseries deletion tasks executing concurrently. +# effectiveMode: restart +# Datatype: int, greater than 0 +timeseries_deletion_max_concurrent_tasks=1 + #################### ### MQTT Broker Configuration #################### diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index d439b35609460..ce25c907840f4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -688,6 +688,15 @@ private ColumnHeaderConstant() { new ColumnHeader(IS_DEGRADED, TSDataType.BOOLEAN), new ColumnHeader(RECENT_FAILURES, TSDataType.TEXT)); + public static final List showDeleteTimeSeriesProgressColumnHeaders = + ImmutableList.of( + new ColumnHeader("ProcedureId", TSDataType.INT64), + new ColumnHeader(QUERY_ID, TSDataType.TEXT), + new ColumnHeader(PATHS, TSDataType.TEXT), + new ColumnHeader(STATE, TSDataType.TEXT), + new ColumnHeader("Progress", TSDataType.TEXT), + new ColumnHeader("ErrorMessage", TSDataType.TEXT)); + public static final List showRepairDataPartitionTableProgressColumnHeaders = ImmutableList.of( new ColumnHeader(REPAIR_DATA_PARTITION_TABLE_STATUS, TSDataType.TEXT), diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index b9a4eb4a49aed..44bd751354b23 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -988,6 +988,20 @@ struct TDeleteTimeSeriesReq { 4: optional bool mayDeleteAudit } +struct TDeleteTimeSeriesProcedureInfo { + 1: required i64 procedureId + 2: required string queryId + 3: required string pathPatterns + 4: required string state + 5: required string progress + 6: optional string errorMessage +} + +struct TShowDeleteTimeSeriesProceduresResp { + 1: required common.TSStatus status + 2: optional list procedureInfoList +} + struct TAlterTimeSeriesReq { 1: required string queryId 2: required binary measurementPath @@ -1987,6 +2001,8 @@ service IConfigNodeRPCService { */ common.TSStatus deleteTimeSeries(TDeleteTimeSeriesReq req) + TShowDeleteTimeSeriesProceduresResp showDeleteTimeSeriesProcedures() + /** * Alter timeseries measurement **/ From 3e66ecaf42e763ae42fc653dd27cf548fcbb9352 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:27:33 +0800 Subject: [PATCH 2/3] Fix Session IT expectations for asynchronous timeseries deletion --- .../session/it/IoTDBSessionSimpleIT.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java index 0aba39c638e56..da78132d16190 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java @@ -29,7 +29,6 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; -import org.apache.iotdb.rpc.BatchExecutionException; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.rpc.TSStatusCode; @@ -814,30 +813,30 @@ public void createWrongTimeSeriesTest() { @Test @Category({ClusterIT.class}) - public void deleteNonExistTimeSeriesTest() { + public void deleteNonExistTimeSeriesTest() throws Exception { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { session.insertRecord( "root.sg1.d1", 0, Arrays.asList("t1", "t2", "t3"), Arrays.asList("123", "333", "444")); - try { - session.deleteTimeseries( - Arrays.asList("root.sg1.d1.t6", "root.sg1.d1.t2", "root.sg1.d1.t3")); - try { - TestUtils.waitForTimeSeriesDeletion( - EnvFactory.getEnv(), - "DELETE TIMESERIES root.sg1.d1.t6, root.sg1.d1.t2, root.sg1.d1.t3"); - fail("Expected asynchronous deletion to fail for a nonexistent timeseries"); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("Path [root.sg1.d1.t6] does not exist;")); - } - } catch (BatchExecutionException e) { - assertTrue(e.getMessage().contains("Path [root.sg1.d1.t6] does not exist;")); - } + // A mixed request succeeds when at least one timeseries matches. + session.deleteTimeseries(Arrays.asList("root.sg1.d1.t6", "root.sg1.d1.t2", "root.sg1.d1.t3")); + TestUtils.waitForTimeSeriesDeletion( + EnvFactory.getEnv(), "DELETE TIMESERIES root.sg1.d1.t6, root.sg1.d1.t2, root.sg1.d1.t3"); assertTrue(session.checkTimeseriesExists("root.sg1.d1.t1")); assertFalse(session.checkTimeseriesExists("root.sg1.d1.t2")); assertFalse(session.checkTimeseriesExists("root.sg1.d1.t3")); - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); + + // When no timeseries matches, admission succeeds and the asynchronous task fails. + session.deleteTimeseries("root.sg1.d1.t6"); + SQLException exception = + assertThrows( + SQLException.class, + () -> + TestUtils.waitForTimeSeriesDeletion( + EnvFactory.getEnv(), "DELETE TIMESERIES root.sg1.d1.t6")); + assertTrue( + exception.getMessage().contains(TSStatusCode.PATH_NOT_EXIST.getStatusCode() + ":")); + assertTrue(exception.getMessage().contains("root.sg1.d1.t6")); + assertTrue(session.checkTimeseriesExists("root.sg1.d1.t1")); } } From 971feafe81b9dec11a115f36cedfb8b715aa3aa2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:01:14 +0800 Subject: [PATCH 3/3] Fix HA test progress polling for stopped DataNode --- .../apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java index 42cf12b7b38a7..4f60bdcb76c60 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java @@ -277,7 +277,10 @@ private void assertStatementEffect( throws Exception { statement.execute(sql); if (sql.startsWith("DELETE TIMESERIES ")) { - TestUtils.waitForTimeSeriesDeletion(EnvFactory.getEnv(), sql); + try (final Statement progressStatement = + TestUtils.createSingleNodeStatement(statement.getConnection())) { + TestUtils.waitForTimeSeriesDeletion(progressStatement, sql); + } } assertTrue(message, effect.call()); }