Fix MAD UDF exact calculation and silent approximate result

library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java

@@ -123,6 +123,12 @@ public final class LibraryUdfMessages {
   // MADSketch
   public static final String NO_VALUES_IN_TIME_SERIES = "No values in the time series";
 
+  // UDAFMad
+  public static final String
+      EXCEPTION_MAD_SKETCH_CANNOT_PRODUCE_A_RESULT_WITH_THE_GIVEN_ERROR_PLEASE_RETRY_WITH_A_SMALLER_ERROR_OR_USE_ERROR_0_FOR_THE_EXACT_RESULT_E168B5FE =
+          "MAD sketch cannot produce a result with the given error. Please retry with a smaller"
+              + " 'error', or use 'error'=0 for the exact result.";
+
   // SlidingCollector
   public static final String COMBINING_NOT_POSSIBLE = "Combining not possible";
 

library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java

@@ -111,6 +111,11 @@ public final class LibraryUdfMessages {
   // MADSketch
   public static final String NO_VALUES_IN_TIME_SERIES = "时间序列中没有值";
 
+  // UDAFMad
+  public static final String
+      EXCEPTION_MAD_SKETCH_CANNOT_PRODUCE_A_RESULT_WITH_THE_GIVEN_ERROR_PLEASE_RETRY_WITH_A_SMALLER_ERROR_OR_USE_ERROR_0_FOR_THE_EXACT_RESULT_E168B5FE =
+          "在给定的 error 下，MAD 近似计算无法得出结果。请减小 error 后重试，或使用 error=0 进行精确计算。";
+
   // SlidingCollector
   public static final String COMBINING_NOT_POSSIBLE = "无法合并";
 

library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java

@@ -21,6 +21,7 @@
 
 import org.apache.iotdb.library.dprofile.util.ExactOrderStatistics;
 import org.apache.iotdb.library.dprofile.util.MADSketch;
+import org.apache.iotdb.library.dprofile.util.Mad;
 import org.apache.iotdb.library.i18n.LibraryUdfMessages;
 import org.apache.iotdb.library.util.Util;
 import org.apache.iotdb.udf.api.UDTF;
@@ -30,6 +31,7 @@
 import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator;
 import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters;
 import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy;
+import org.apache.iotdb.udf.api.exception.UDFException;
 import org.apache.iotdb.udf.api.type.Type;
 
 /** calculate the exact or approximate median absolute deviation (mad). */
@@ -78,7 +80,13 @@ public void terminate(PointCollector collector) throws Exception {
     if (exact) {
       collector.putDouble(0, statistics.getMad());
     } else {
-      collector.putDouble(0, sketch.getMad().result);
+      Mad mad = sketch.getMad();
+      if (mad.errorBound == Double.MAX_VALUE) {
+        throw new UDFException(
+            LibraryUdfMessages
+                .EXCEPTION_MAD_SKETCH_CANNOT_PRODUCE_A_RESULT_WITH_THE_GIVEN_ERROR_PLEASE_RETRY_WITH_A_SMALLER_ERROR_OR_USE_ERROR_0_FOR_THE_EXACT_RESULT_E168B5FE);
+      }
+      collector.putDouble(0, mad.result);
     }
   }
 }

library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java

@@ -127,9 +127,9 @@ public static double getMad(FloatArrayList nums) {
       throw new NoSuchElementException();
     } else {
       double median = getMedian(nums);
-      DoubleArrayList dal = new DoubleArrayList();
+      DoubleArrayList dal = new DoubleArrayList(nums.size());
       for (int i = 0; i < nums.size(); ++i) {
-        dal.set(i, Math.abs(nums.get(i) - median));
+        dal.add(Math.abs(nums.get(i) - median));
       }
       return getMedian(dal);
     }
@@ -144,9 +144,9 @@ public static double getMad(DoubleArrayList nums) {
       throw new NoSuchElementException();
     } else {
       double median = getMedian(nums);
-      DoubleArrayList dal = new DoubleArrayList();
+      DoubleArrayList dal = new DoubleArrayList(nums.size());
       for (int i = 0; i < nums.size(); ++i) {
-        dal.set(i, Math.abs(nums.get(i) - median));
+        dal.add(Math.abs(nums.get(i) - median));
       }
       return getMedian(dal);
     }
@@ -157,9 +157,9 @@ public static double getMad(IntArrayList nums) {
       throw new NoSuchElementException();
     } else {
       double median = getMedian(nums);
-      DoubleArrayList dal = new DoubleArrayList();
+      DoubleArrayList dal = new DoubleArrayList(nums.size());
       for (int i = 0; i < nums.size(); ++i) {
-        dal.set(i, Math.abs(nums.get(i) - median));
+        dal.add(Math.abs(nums.get(i) - median));
       }
       return getMedian(dal);
     }
@@ -170,9 +170,9 @@ public static double getMad(LongArrayList nums) {
       throw new NoSuchElementException();
     } else {
       double median = getMedian(nums);
-      DoubleArrayList dal = new DoubleArrayList();
+      DoubleArrayList dal = new DoubleArrayList(nums.size());
       for (int i = 0; i < nums.size(); ++i) {
-        dal.set(i, Math.abs(nums.get(i) - median));
+        dal.add(Math.abs(nums.get(i) - median));
       }
       return getMedian(dal);
     }

library-udf/src/test/java/org/apache/iotdb/library/dprofile/UDAFMadTest.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.library.dprofile;
+
+import org.apache.iotdb.udf.api.access.Row;
+import org.apache.iotdb.udf.api.collector.PointCollector;
+import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations;
+import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters;
+import org.apache.iotdb.udf.api.exception.UDFException;
+import org.apache.iotdb.udf.api.type.Type;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Proxy;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class UDAFMadTest {
+
+  private static final double[] VALUES = {
+    2403.71, 2404.36, 2402.88, 2403.95, 2404.12, 2401.63, 2403.52, 2404.77, 2402.34, 2403.18,
+    2405.01, 2397.77, 2404.63, 2402.07, 2403.85, 2404.44, 2402.61, 2403.29, 2404.90, 2402.15,
+    2403.63, 2404.05, 2402.72, 2403.97
+  };
+  private static final double EXPECTED_MAD = 0.78;
+
+  @Test
+  public void testExactMad() throws Exception {
+    UDAFMad mad = new UDAFMad();
+    mad.beforeStart(parameters(0), new UDTFConfigurations(ZoneId.systemDefault()));
+    RecordingCollector collector = new RecordingCollector();
+    for (double value : VALUES) {
+      mad.transform(doubleRow(value), collector.collector);
+    }
+    mad.terminate(collector.collector);
+    Assert.assertEquals(EXPECTED_MAD, collector.value, 1e-9);
+  }
+
+  @Test
+  public void testSketchRejectsDegenerateResult() throws Exception {
+    UDAFMad mad = new UDAFMad();
+    mad.beforeStart(parameters(0.01), new UDTFConfigurations(ZoneId.systemDefault()));
+    RecordingCollector collector = new RecordingCollector();
+    for (double value : VALUES) {
+      mad.transform(doubleRow(value), collector.collector);
+    }
+    Assert.assertThrows(UDFException.class, () -> mad.terminate(collector.collector));
+  }
+
+  private static UDFParameters parameters(double error) {
+    Map<String, String> attributes = new HashMap<>();
+    if (error != 0) {
+      attributes.put("error", Double.toString(error));
+    }
+    return new UDFParameters(Arrays.asList("s0"), Arrays.asList(Type.DOUBLE), attributes);
+  }
+
+  private static Row doubleRow(double value) {
+    return (Row)
+        Proxy.newProxyInstance(
+            Row.class.getClassLoader(),
+            new Class<?>[] {Row.class},
+            (proxy, method, args) -> {
+              switch (method.getName()) {
+                case "getTime":
+                  return 0L;
+                case "getDouble":
+                  return value;
+                case "getDataType":
+                  return Type.DOUBLE;
+                case "size":
+                  return 1;
+                default:
+                  return null;
+              }
+            });
+  }
+
+  private static class RecordingCollector {
+
+    private double value = Double.NaN;
+    private final PointCollector collector =
+        (PointCollector)
+            Proxy.newProxyInstance(
+                PointCollector.class.getClassLoader(),
+                new Class<?>[] {PointCollector.class},
+                (proxy, method, args) -> {
+                  if ("putDouble".equals(method.getName())) {
+                    value = (Double) args[1];
+                  }
+                  return null;
+                });
+  }
+}

library-udf/src/test/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatisticsTest.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.library.dprofile.util;
+
+import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList;
+import org.eclipse.collections.impl.list.mutable.primitive.FloatArrayList;
+import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList;
+import org.eclipse.collections.impl.list.mutable.primitive.LongArrayList;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ExactOrderStatisticsTest {
+
+  private static final double[] VALUES = {
+    2403.71, 2404.36, 2402.88, 2403.95, 2404.12, 2401.63, 2403.52, 2404.77, 2402.34, 2403.18,
+    2405.01, 2397.77, 2404.63, 2402.07, 2403.85, 2404.44, 2402.61, 2403.29, 2404.90, 2402.15,
+    2403.63, 2404.05, 2402.72, 2403.97
+  };
+  private static final double EXPECTED_MAD = 0.78;
+
+  @Test
+  public void testGetMadDouble() {
+    DoubleArrayList nums = new DoubleArrayList(VALUES.length);
+    for (double value : VALUES) {
+      nums.add(value);
+    }
+    Assert.assertEquals(EXPECTED_MAD, ExactOrderStatistics.getMad(nums), 1e-9);
+  }
+
+  @Test
+  public void testGetMadFloat() {
+    FloatArrayList nums = new FloatArrayList(VALUES.length);
+    for (double value : VALUES) {
+      nums.add((float) value);
+    }
+    Assert.assertEquals(EXPECTED_MAD, ExactOrderStatistics.getMad(nums), 1e-3);
+  }
+
+  @Test
+  public void testGetMadInt() {
+    IntArrayList nums = new IntArrayList(VALUES.length);
+    for (int i = 1; i <= VALUES.length; i++) {
+      nums.add(i);
+    }
+    Assert.assertEquals(6.0, ExactOrderStatistics.getMad(nums), 0);
+  }
+
+  @Test
+  public void testGetMadLong() {
+    LongArrayList nums = new LongArrayList(VALUES.length);
+    for (long i = 1; i <= VALUES.length; i++) {
+      nums.add(i);
+    }
+    Assert.assertEquals(6.0, ExactOrderStatistics.getMad(nums), 0);
+  }
+}