From 9cd8ee1d58f9ae5b25bae405b533c4b0ad1508e7 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Fri, 6 Nov 2026 16:25:18 +0800 Subject: [PATCH 1/2] Fix NPE of approx_most_frequent and approx_count_distinct on empty input --- .../query/recent/IoTDBTableAggregationIT.java | 57 +++++++++++++++---- ...AbstractApproxMostFrequentAccumulator.java | 19 ++++++- .../ApproxCountDistinctAccumulator.java | 12 +++- .../BlobApproxMostFrequentAccumulator.java | 4 ++ .../approximate/HyperLogLogStateFactory.java | 2 +- .../aggregation/approximate/SpaceSaving.java | 4 ++ ...tGroupedApproxMostFrequentAccumulator.java | 18 ++++-- ...GroupedApproxCountDistinctAccumulator.java | 12 ++-- 8 files changed, 104 insertions(+), 24 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index 99188903b99a7..36581d5540bfb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -57,6 +57,7 @@ public class IoTDBTableAggregationIT { "CREATE DATABASE " + DATABASE_NAME, "USE " + DATABASE_NAME, "CREATE TABLE table1(province STRING TAG, city STRING TAG, region STRING TAG, device_id STRING TAG, color STRING ATTRIBUTE, type STRING ATTRIBUTE, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", + "CREATE TABLE empty_table(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s6 TEXT FIELD, s8 BLOB FIELD)", "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',30,30.0,'shanghai_huangpu_red_A_d01_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',35000,35.0,35.0,'shanghai_huangpu_red_A_d01_35','shanghai_huangpu_red_A_d01_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',40,40.0,true,'shanghai_huangpu_red_A_d01_40',2024-09-24T06:15:40.000+00:00)", @@ -4377,26 +4378,26 @@ public void approxMostFrequentTest() { expectedHeader = new String[] {"time", "province", "_col2"}; retArray = new String[] { - "2024-09-24T06:15:30.000Z,beijing,{},", + "2024-09-24T06:15:30.000Z,beijing,null,", "2024-09-24T06:15:31.000Z,beijing,{\"31000\":2},", "2024-09-24T06:15:35.000Z,beijing,{\"35000\":2},", - "2024-09-24T06:15:36.000Z,beijing,{},", + "2024-09-24T06:15:36.000Z,beijing,null,", "2024-09-24T06:15:40.000Z,beijing,{\"40000\":2},", - "2024-09-24T06:15:41.000Z,beijing,{},", + "2024-09-24T06:15:41.000Z,beijing,null,", "2024-09-24T06:15:46.000Z,beijing,{\"46000\":2},", "2024-09-24T06:15:50.000Z,beijing,{\"50000\":4},", - "2024-09-24T06:15:51.000Z,beijing,{},", - "2024-09-24T06:15:55.000Z,beijing,{},", - "2024-09-24T06:15:30.000Z,shanghai,{},", + "2024-09-24T06:15:51.000Z,beijing,null,", + "2024-09-24T06:15:55.000Z,beijing,null,", + "2024-09-24T06:15:30.000Z,shanghai,null,", "2024-09-24T06:15:31.000Z,shanghai,{\"31000\":2},", "2024-09-24T06:15:35.000Z,shanghai,{\"35000\":2},", - "2024-09-24T06:15:36.000Z,shanghai,{},", + "2024-09-24T06:15:36.000Z,shanghai,null,", "2024-09-24T06:15:40.000Z,shanghai,{\"40000\":2},", - "2024-09-24T06:15:41.000Z,shanghai,{},", + "2024-09-24T06:15:41.000Z,shanghai,null,", "2024-09-24T06:15:46.000Z,shanghai,{\"46000\":2},", "2024-09-24T06:15:50.000Z,shanghai,{\"50000\":4},", - "2024-09-24T06:15:51.000Z,shanghai,{},", - "2024-09-24T06:15:55.000Z,shanghai,{},", + "2024-09-24T06:15:51.000Z,shanghai,null,", + "2024-09-24T06:15:55.000Z,shanghai,null,", }; tableResultSetEqualTest( @@ -4406,6 +4407,42 @@ public void approxMostFrequentTest() { DATABASE_NAME); } + @Test + public void approxAggregationEmptyInputTest() { + // No block reaches the aggregation operator, so the accumulator state is never created + tableResultSetEqualTest( + "select approx_most_frequent(s1, 2, 10), approx_most_frequent(s2, 2, 10), approx_most_frequent(s6, 2, 10), approx_most_frequent(s8, 2, 10) from empty_table", + buildHeaders(4), + new String[] {"null,null,null,null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "select approx_count_distinct(s1), approx_count_distinct(s6) from empty_table", + buildHeaders(2), + new String[] {"0,0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "select device_id, approx_most_frequent(s1, 2, 10) from empty_table group by device_id", + new String[] {"device_id", "_col1"}, + new String[] {}, + DATABASE_NAME); + + // All rows are filtered out by the predicate + tableResultSetEqualTest( + "select approx_most_frequent(s1, 2, 10), approx_count_distinct(s1) from table1 where s1 < 0", + buildHeaders(2), + new String[] {"null,0,"}, + DATABASE_NAME); + + // Rows exist but every value is null, so there is nothing to count + tableResultSetEqualTest( + "select approx_most_frequent(s2, 2, 10), approx_count_distinct(s2) from table1 where s2 is null", + buildHeaders(2), + new String[] {"null,0,"}, + DATABASE_NAME); + } + @Test public void approxPercentileTest() { tableResultSetEqualTest( diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java index 0b2d9ab5f0ad9..b929b6c27a268 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java @@ -19,6 +19,7 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.SpaceSaving; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.SpaceSavingStateFactory; import org.apache.iotdb.calc.i18n.CalcMessages; @@ -36,13 +37,23 @@ public abstract class AbstractApproxMostFrequentAccumulator implements TableA @Override public void evaluateIntermediate(ColumnBuilder columnBuilder) { - columnBuilder.writeBinary(new Binary(state.getSpaceSaving().serialize())); + SpaceSaving spaceSaving = state.getSpaceSaving(); + if (spaceSaving == null || spaceSaving.isEmpty()) { + columnBuilder.appendNull(); + return; + } + columnBuilder.writeBinary(new Binary(spaceSaving.serialize())); } @Override public void evaluateFinal(ColumnBuilder columnBuilder) { + SpaceSaving spaceSaving = state.getSpaceSaving(); + if (spaceSaving == null || spaceSaving.isEmpty()) { + columnBuilder.appendNull(); + return; + } columnBuilder.writeBinary( - new Binary(new Gson().toJson(state.getSpaceSaving().getBuckets()), StandardCharsets.UTF_8)); + new Binary(new Gson().toJson(spaceSaving.getBuckets()), StandardCharsets.UTF_8)); } @Override @@ -58,6 +69,8 @@ public void addStatistics(Statistics[] statistics) { @Override public void reset() { - state.getSpaceSaving().reset(); + if (state.getSpaceSaving() != null) { + state.getSpaceSaving().reset(); + } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java index 65824142a079d..97cf60e8d8b17 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java @@ -106,11 +106,19 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { columnBuilder instanceof BinaryColumnBuilder, CalcMessages .EXCEPTION_INTERMEDIATE_INPUT_AND_OUTPUT_OF_APPROX_COUNT_DISTINCT_SHOULD_BE_BINARYCOLUMN_F444D3BE); + if (state.getHyperLogLog() == null) { + columnBuilder.appendNull(); + return; + } columnBuilder.writeBinary(new Binary(state.getHyperLogLog().serialize())); } @Override public void evaluateFinal(ColumnBuilder columnBuilder) { + if (state.getHyperLogLog() == null) { + columnBuilder.writeLong(0); + return; + } columnBuilder.writeLong(state.getHyperLogLog().cardinality()); } @@ -127,7 +135,9 @@ public void addStatistics(Statistics[] statistics) { @Override public void reset() { - state.getHyperLogLog().reset(); + if (state.getHyperLogLog() != null) { + state.getHyperLogLog().reset(); + } } public void addBooleanInput(Column valueColumn, AggregationMask mask, HyperLogLog hll) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/BlobApproxMostFrequentAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/BlobApproxMostFrequentAccumulator.java index 9f2df5d94552f..c70e0855c831a 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/BlobApproxMostFrequentAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/BlobApproxMostFrequentAccumulator.java @@ -31,6 +31,10 @@ public class BlobApproxMostFrequentAccumulator extends BinaryApproxMostFrequentAccumulator { @Override public void evaluateFinal(ColumnBuilder columnBuilder) { + if (state.getSpaceSaving() == null || state.getSpaceSaving().isEmpty()) { + columnBuilder.appendNull(); + return; + } Map buckets = state.getSpaceSaving().getBuckets(); Map formatedBuckets = buckets.entrySet().stream() diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/HyperLogLogStateFactory.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/HyperLogLogStateFactory.java index cc0780efa1ae1..5746019d32456 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/HyperLogLogStateFactory.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/HyperLogLogStateFactory.java @@ -50,7 +50,7 @@ public void setHyperLogLog(HyperLogLog value) { public long getEstimatedSize() { // not used - return INSTANCE_SIZE + hll.getEstimatedSize(); + return hll == null ? INSTANCE_SIZE : INSTANCE_SIZE + hll.getEstimatedSize(); } public void merge(HyperLogLog other) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/SpaceSaving.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/SpaceSaving.java index 0ac134cfa0a14..610620a92e394 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/SpaceSaving.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/SpaceSaving.java @@ -124,6 +124,10 @@ public Map getBuckets() { return buckets.buildOrThrow(); } + public boolean isEmpty() { + return streamSummary.size() == 0; + } + public byte[] serialize() { List> counters = streamSummary.topK(capacity); // Calculate the size of the keys diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxMostFrequentAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxMostFrequentAccumulator.java index 96826bf0dde07..925c74ce26179 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxMostFrequentAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxMostFrequentAccumulator.java @@ -19,6 +19,7 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.SpaceSaving; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.SpaceSavingStateFactory; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.SpaceSavingBigArray; @@ -42,15 +43,22 @@ public void setGroupCount(long groupCount) { @Override public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { - columnBuilder.writeBinary(new Binary(state.getSpaceSavings().get(groupId).serialize())); + SpaceSaving spaceSaving = state.getSpaceSavings().get(groupId); + if (spaceSaving == null || spaceSaving.isEmpty()) { + columnBuilder.appendNull(); + return; + } + columnBuilder.writeBinary(new Binary(spaceSaving.serialize())); } @Override public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { - Binary result = - new Binary( - new Gson().toJson(state.getSpaceSavings().get(groupId).getBuckets()), - StandardCharsets.UTF_8); + SpaceSaving spaceSaving = state.getSpaceSavings().get(groupId); + if (spaceSaving == null || spaceSaving.isEmpty()) { + columnBuilder.appendNull(); + return; + } + Binary result = new Binary(new Gson().toJson(spaceSaving.getBuckets()), StandardCharsets.UTF_8); columnBuilder.writeBinary(result); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java index ae6e2ec828bc0..40811e24f69b5 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java @@ -101,14 +101,18 @@ public void addIntermediate(int[] groupIds, Column argument) { @Override public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { - HyperLogLogBigArray hlls = state.getHyperLogLogs(); - columnBuilder.writeBinary(new Binary(hlls.get(groupId).serialize())); + HyperLogLog hll = state.getHyperLogLogs().get(groupId); + if (hll == null) { + columnBuilder.appendNull(); + return; + } + columnBuilder.writeBinary(new Binary(hll.serialize())); } @Override public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { - HyperLogLogBigArray hlls = state.getHyperLogLogs(); - columnBuilder.writeLong(hlls.get(groupId).cardinality()); + HyperLogLog hll = state.getHyperLogLogs().get(groupId); + columnBuilder.writeLong(hll == null ? 0 : hll.cardinality()); } @Override From 38adbe7d2c201aa758e4eb60a30292983335b3f1 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Fri, 6 Nov 2026 17:08:30 +0800 Subject: [PATCH 2/2] fix BlobGroupedApproxMostFrequentAccumulator --- .../query/recent/IoTDBTableAggregationIT.java | 33 +++++++++++++++++++ ...bGroupedApproxMostFrequentAccumulator.java | 9 ++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index 36581d5540bfb..6bf49726a0d0a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -4407,6 +4407,39 @@ public void approxMostFrequentTest() { DATABASE_NAME); } + @Test + public void blobGroupedApproxMostFrequentTest() { + // Groups whose BLOB column is entirely NULL must yield NULL instead of an empty map. + // Under a PARTIAL -> FINAL aggregation the partial side emits NULL for these groups and + // the final side never initializes their state, so evaluateFinal must handle that too. + tableResultSetEqualTest( + "select time,province,approx_most_frequent(s8, 1, 10) from table1 group by 1,2 order by 2,1", + new String[] {"time", "province", "_col2"}, + new String[] { + "2024-09-24T06:15:30.000Z,beijing,{\"0xcafebabe30\":2},", + "2024-09-24T06:15:31.000Z,beijing,{\"0xcafebabe31\":2},", + "2024-09-24T06:15:35.000Z,beijing,null,", + "2024-09-24T06:15:36.000Z,beijing,null,", + "2024-09-24T06:15:40.000Z,beijing,null,", + "2024-09-24T06:15:41.000Z,beijing,{\"0xcafebabe41\":2},", + "2024-09-24T06:15:46.000Z,beijing,null,", + "2024-09-24T06:15:50.000Z,beijing,{\"0xcafebabe50\":2},", + "2024-09-24T06:15:51.000Z,beijing,null,", + "2024-09-24T06:15:55.000Z,beijing,{\"0xcafebabe55\":4},", + "2024-09-24T06:15:30.000Z,shanghai,{\"0xcafebabe30\":2},", + "2024-09-24T06:15:31.000Z,shanghai,{\"0xcafebabe31\":2},", + "2024-09-24T06:15:35.000Z,shanghai,null,", + "2024-09-24T06:15:36.000Z,shanghai,null,", + "2024-09-24T06:15:40.000Z,shanghai,null,", + "2024-09-24T06:15:41.000Z,shanghai,{\"0xcafebabe41\":2},", + "2024-09-24T06:15:46.000Z,shanghai,null,", + "2024-09-24T06:15:50.000Z,shanghai,{\"0xcafebabe50\":2},", + "2024-09-24T06:15:51.000Z,shanghai,null,", + "2024-09-24T06:15:55.000Z,shanghai,{\"0xcafebabe55\":4},", + }, + DATABASE_NAME); + } + @Test public void approxAggregationEmptyInputTest() { // No block reaches the aggregation operator, so the accumulator state is never created diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/BlobGroupedApproxMostFrequentAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/BlobGroupedApproxMostFrequentAccumulator.java index b83583ee4e8d9..36f7278224f69 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/BlobGroupedApproxMostFrequentAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/BlobGroupedApproxMostFrequentAccumulator.java @@ -19,6 +19,8 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.SpaceSaving; + import com.google.gson.Gson; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.utils.Binary; @@ -32,7 +34,12 @@ public class BlobGroupedApproxMostFrequentAccumulator extends BinaryGroupedApproxMostFrequentAccumulator { @Override public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { - Map buckets = state.getSpaceSavings().get(groupId).getBuckets(); + SpaceSaving spaceSaving = state.getSpaceSavings().get(groupId); + if (spaceSaving == null || spaceSaving.isEmpty()) { + columnBuilder.appendNull(); + return; + } + Map buckets = spaceSaving.getBuckets(); Map formatedBuckets = buckets.entrySet().stream() .collect(