diff --git a/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java b/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java index 2e79a47474775..62bba86ae2b8f 100644 --- a/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java +++ b/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java @@ -123,6 +123,12 @@ public final class LibraryUdfMessages { // MADSketch public static final String NO_VALUES_IN_TIME_SERIES = "No values in the time series"; + // UDAFMad + public static final String + EXCEPTION_MAD_SKETCH_CANNOT_PRODUCE_A_RESULT_WITH_THE_GIVEN_ERROR_PLEASE_RETRY_WITH_A_SMALLER_ERROR_OR_USE_ERROR_0_FOR_THE_EXACT_RESULT_E168B5FE = + "MAD sketch cannot produce a result with the given error. Please retry with a smaller" + + " 'error', or use 'error'=0 for the exact result."; + // SlidingCollector public static final String COMBINING_NOT_POSSIBLE = "Combining not possible"; diff --git a/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java b/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java index 42e145ffd65eb..b3a3d552b22c7 100644 --- a/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java +++ b/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java @@ -111,6 +111,11 @@ public final class LibraryUdfMessages { // MADSketch public static final String NO_VALUES_IN_TIME_SERIES = "时间序列中没有值"; + // UDAFMad + public static final String + EXCEPTION_MAD_SKETCH_CANNOT_PRODUCE_A_RESULT_WITH_THE_GIVEN_ERROR_PLEASE_RETRY_WITH_A_SMALLER_ERROR_OR_USE_ERROR_0_FOR_THE_EXACT_RESULT_E168B5FE = + "在给定的 error 下，MAD 近似计算无法得出结果。请减小 error 后重试，或使用 error=0 进行精确计算。"; + // SlidingCollector public static final String COMBINING_NOT_POSSIBLE = "无法合并"; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java index 8a5f99abdb690..58987cc5866f6 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java @@ -21,6 +21,7 @@ import org.apache.iotdb.library.dprofile.util.ExactOrderStatistics; import org.apache.iotdb.library.dprofile.util.MADSketch; +import org.apache.iotdb.library.dprofile.util.Mad; import org.apache.iotdb.library.i18n.LibraryUdfMessages; import org.apache.iotdb.library.util.Util; import org.apache.iotdb.udf.api.UDTF; @@ -30,6 +31,7 @@ import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy; +import org.apache.iotdb.udf.api.exception.UDFException; import org.apache.iotdb.udf.api.type.Type; /** calculate the exact or approximate median absolute deviation (mad). */ @@ -78,7 +80,13 @@ public void terminate(PointCollector collector) throws Exception { if (exact) { collector.putDouble(0, statistics.getMad()); } else { - collector.putDouble(0, sketch.getMad().result); + Mad mad = sketch.getMad(); + if (mad.errorBound == Double.MAX_VALUE) { + throw new UDFException( + LibraryUdfMessages + .EXCEPTION_MAD_SKETCH_CANNOT_PRODUCE_A_RESULT_WITH_THE_GIVEN_ERROR_PLEASE_RETRY_WITH_A_SMALLER_ERROR_OR_USE_ERROR_0_FOR_THE_EXACT_RESULT_E168B5FE); + } + collector.putDouble(0, mad.result); } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java index c4c49f0fdb9fb..c225a40ed6a41 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java @@ -127,9 +127,9 @@ public static double getMad(FloatArrayList nums) { throw new NoSuchElementException(); } else { double median = getMedian(nums); - DoubleArrayList dal = new DoubleArrayList(); + DoubleArrayList dal = new DoubleArrayList(nums.size()); for (int i = 0; i < nums.size(); ++i) { - dal.set(i, Math.abs(nums.get(i) - median)); + dal.add(Math.abs(nums.get(i) - median)); } return getMedian(dal); } @@ -144,9 +144,9 @@ public static double getMad(DoubleArrayList nums) { throw new NoSuchElementException(); } else { double median = getMedian(nums); - DoubleArrayList dal = new DoubleArrayList(); + DoubleArrayList dal = new DoubleArrayList(nums.size()); for (int i = 0; i < nums.size(); ++i) { - dal.set(i, Math.abs(nums.get(i) - median)); + dal.add(Math.abs(nums.get(i) - median)); } return getMedian(dal); } @@ -157,9 +157,9 @@ public static double getMad(IntArrayList nums) { throw new NoSuchElementException(); } else { double median = getMedian(nums); - DoubleArrayList dal = new DoubleArrayList(); + DoubleArrayList dal = new DoubleArrayList(nums.size()); for (int i = 0; i < nums.size(); ++i) { - dal.set(i, Math.abs(nums.get(i) - median)); + dal.add(Math.abs(nums.get(i) - median)); } return getMedian(dal); } @@ -170,9 +170,9 @@ public static double getMad(LongArrayList nums) { throw new NoSuchElementException(); } else { double median = getMedian(nums); - DoubleArrayList dal = new DoubleArrayList(); + DoubleArrayList dal = new DoubleArrayList(nums.size()); for (int i = 0; i < nums.size(); ++i) { - dal.set(i, Math.abs(nums.get(i) - median)); + dal.add(Math.abs(nums.get(i) - median)); } return getMedian(dal); } diff --git a/library-udf/src/test/java/org/apache/iotdb/library/dprofile/UDAFMadTest.java b/library-udf/src/test/java/org/apache/iotdb/library/dprofile/UDAFMadTest.java new file mode 100644 index 0000000000000..c4d64cf3075aa --- /dev/null +++ b/library-udf/src/test/java/org/apache/iotdb/library/dprofile/UDAFMadTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.library.dprofile; + +import org.apache.iotdb.udf.api.access.Row; +import org.apache.iotdb.udf.api.collector.PointCollector; +import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.type.Type; + +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class UDAFMadTest { + + private static final double[] VALUES = { + 2403.71, 2404.36, 2402.88, 2403.95, 2404.12, 2401.63, 2403.52, 2404.77, 2402.34, 2403.18, + 2405.01, 2397.77, 2404.63, 2402.07, 2403.85, 2404.44, 2402.61, 2403.29, 2404.90, 2402.15, + 2403.63, 2404.05, 2402.72, 2403.97 + }; + private static final double EXPECTED_MAD = 0.78; + + @Test + public void testExactMad() throws Exception { + UDAFMad mad = new UDAFMad(); + mad.beforeStart(parameters(0), new UDTFConfigurations(ZoneId.systemDefault())); + RecordingCollector collector = new RecordingCollector(); + for (double value : VALUES) { + mad.transform(doubleRow(value), collector.collector); + } + mad.terminate(collector.collector); + Assert.assertEquals(EXPECTED_MAD, collector.value, 1e-9); + } + + @Test + public void testSketchRejectsDegenerateResult() throws Exception { + UDAFMad mad = new UDAFMad(); + mad.beforeStart(parameters(0.01), new UDTFConfigurations(ZoneId.systemDefault())); + RecordingCollector collector = new RecordingCollector(); + for (double value : VALUES) { + mad.transform(doubleRow(value), collector.collector); + } + Assert.assertThrows(UDFException.class, () -> mad.terminate(collector.collector)); + } + + private static UDFParameters parameters(double error) { + Map attributes = new HashMap<>(); + if (error != 0) { + attributes.put("error", Double.toString(error)); + } + return new UDFParameters(Arrays.asList("s0"), Arrays.asList(Type.DOUBLE), attributes); + } + + private static Row doubleRow(double value) { + return (Row) + Proxy.newProxyInstance( + Row.class.getClassLoader(), + new Class[] {Row.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "getTime": + return 0L; + case "getDouble": + return value; + case "getDataType": + return Type.DOUBLE; + case "size": + return 1; + default: + return null; + } + }); + } + + private static class RecordingCollector { + + private double value = Double.NaN; + private final PointCollector collector = + (PointCollector) + Proxy.newProxyInstance( + PointCollector.class.getClassLoader(), + new Class[] {PointCollector.class}, + (proxy, method, args) -> { + if ("putDouble".equals(method.getName())) { + value = (Double) args[1]; + } + return null; + }); + } +} diff --git a/library-udf/src/test/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatisticsTest.java b/library-udf/src/test/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatisticsTest.java new file mode 100644 index 0000000000000..7d11e594e3cb8 --- /dev/null +++ b/library-udf/src/test/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatisticsTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.library.dprofile.util; + +import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList; +import org.eclipse.collections.impl.list.mutable.primitive.FloatArrayList; +import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; +import org.eclipse.collections.impl.list.mutable.primitive.LongArrayList; +import org.junit.Assert; +import org.junit.Test; + +public class ExactOrderStatisticsTest { + + private static final double[] VALUES = { + 2403.71, 2404.36, 2402.88, 2403.95, 2404.12, 2401.63, 2403.52, 2404.77, 2402.34, 2403.18, + 2405.01, 2397.77, 2404.63, 2402.07, 2403.85, 2404.44, 2402.61, 2403.29, 2404.90, 2402.15, + 2403.63, 2404.05, 2402.72, 2403.97 + }; + private static final double EXPECTED_MAD = 0.78; + + @Test + public void testGetMadDouble() { + DoubleArrayList nums = new DoubleArrayList(VALUES.length); + for (double value : VALUES) { + nums.add(value); + } + Assert.assertEquals(EXPECTED_MAD, ExactOrderStatistics.getMad(nums), 1e-9); + } + + @Test + public void testGetMadFloat() { + FloatArrayList nums = new FloatArrayList(VALUES.length); + for (double value : VALUES) { + nums.add((float) value); + } + Assert.assertEquals(EXPECTED_MAD, ExactOrderStatistics.getMad(nums), 1e-3); + } + + @Test + public void testGetMadInt() { + IntArrayList nums = new IntArrayList(VALUES.length); + for (int i = 1; i <= VALUES.length; i++) { + nums.add(i); + } + Assert.assertEquals(6.0, ExactOrderStatistics.getMad(nums), 0); + } + + @Test + public void testGetMadLong() { + LongArrayList nums = new LongArrayList(VALUES.length); + for (long i = 1; i <= VALUES.length; i++) { + nums.add(i); + } + Assert.assertEquals(6.0, ExactOrderStatistics.getMad(nums), 0); + } +}