From b157c0c700283f6fbea4d48f2a524632139cdcce Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Wed, 16 Sep 2026 19:20:34 +0800 Subject: [PATCH 01/11] [feature](function) Support Hive-compatible encode and decode ### What problem does this PR solve? Issue Number: #48203 Related PR: None Problem Summary: Doris lacks Hive-compatible encode(string, charset) and decode(binary, charset) scalar functions. Add FE signatures and constant folding, BE vectorized ICU conversion with strict malformed and unmappable input handling, case-insensitive support for the six Hive-documented character sets, Java-compatible UTF-16 BOM behavior, null propagation, and focused tests. ### Release note Add Hive-compatible encode and decode scalar functions for US-ASCII, ISO-8859-1, UTF-8, UTF-16BE, UTF-16LE, and UTF-16. ### Check List (For Author) - Test: Unit and regression tests - BE function_character_encoding_test.*: 3 tests passed. - FE StringArithmeticTest: 10 tests passed; Maven reactor succeeded. - Native Linux FE/BE build succeeded. - test_encode_decode regression suite passed in generated-output and comparison modes. - Behavior changed: Yes. Add encode and decode with Hive-compatible types, supported character sets, UTF-16 BOM semantics, null propagation, and strict conversion errors. - Does this need documentation: Yes. A follow-up doris-website PR is required. --- .../function/function_character_encoding.cpp | 359 ++++++++++++++++++ .../exprs/function/simple_function_factory.h | 2 + .../function_character_encoding_test.cpp | 116 ++++++ .../doris/catalog/BuiltinScalarFunctions.java | 4 + .../executable/StringArithmetic.java | 66 ++++ .../expressions/functions/scalar/Decode.java | 74 ++++ .../expressions/functions/scalar/Encode.java | 74 ++++ .../visitor/ScalarFunctionVisitor.java | 10 + .../executable/StringArithmeticTest.java | 54 +++ .../test_encode_decode.groovy | 33 ++ 10 files changed, 792 insertions(+) create mode 100644 be/src/exprs/function/function_character_encoding.cpp create mode 100644 be/test/exprs/function/function_character_encoding_test.cpp create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java create mode 100644 regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy diff --git a/be/src/exprs/function/function_character_encoding.cpp b/be/src/exprs/function/function_character_encoding.cpp new file mode 100644 index 00000000000000..edbaa72843a58f --- /dev/null +++ b/be/src/exprs/function/function_character_encoding.cpp @@ -0,0 +1,359 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/string_ref.h" +#include "exprs/function/function.h" +#include "exprs/function/simple_function_factory.h" + +namespace doris { +namespace { + +enum class CharacterSet : uint8_t { + US_ASCII, + ISO_8859_1, + UTF_8, + UTF_16BE, + UTF_16LE, + UTF_16, + SIZE, +}; + +constexpr std::array(CharacterSet::SIZE)> + SUPPORTED_CHARACTER_SETS = {"US-ASCII", "ISO-8859-1", "UTF-8", + "UTF-16BE", "UTF-16LE", "UTF-16"}; + +bool equals_ignore_case(StringRef value, std::string_view expected) { + if (value.size != expected.size()) { + return false; + } + for (size_t i = 0; i < value.size; ++i) { + const char current = value.data[i] >= 'a' && value.data[i] <= 'z' + ? value.data[i] - ('a' - 'A') + : value.data[i]; + if (current != expected[i]) { + return false; + } + } + return true; +} + +Status parse_character_set(StringRef value, CharacterSet& character_set) { + for (size_t i = 0; i < SUPPORTED_CHARACTER_SETS.size(); ++i) { + if (equals_ignore_case(value, SUPPORTED_CHARACTER_SETS[i])) { + character_set = static_cast(i); + return Status::OK(); + } + } + return Status::InvalidArgument( + "Unsupported character set '{}'. Supported character sets are US-ASCII, " + "ISO-8859-1, UTF-8, UTF-16BE, UTF-16LE, and UTF-16", + std::string(value.data, value.size)); +} + +using ConverterPtr = std::unique_ptr; + +class ConverterPair { +public: + ConverterPair() : _source(nullptr, ucnv_close), _target(nullptr, ucnv_close) {} + + Status open(std::string_view source_name, std::string_view target_name) { + UErrorCode error = U_ZERO_ERROR; + _source.reset(ucnv_open(source_name.data(), &error)); + if (U_FAILURE(error)) { + return Status::InternalError("Failed to open ICU converter '{}': {}", source_name, + u_errorName(error)); + } + + error = U_ZERO_ERROR; + ucnv_setToUCallBack(_source.get(), UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr, nullptr, + &error); + if (U_FAILURE(error)) { + return Status::InternalError("Failed to configure ICU converter '{}': {}", source_name, + u_errorName(error)); + } + + error = U_ZERO_ERROR; + _target.reset(ucnv_open(target_name.data(), &error)); + if (U_FAILURE(error)) { + return Status::InternalError("Failed to open ICU converter '{}': {}", target_name, + u_errorName(error)); + } + + error = U_ZERO_ERROR; + ucnv_setFromUCallBack(_target.get(), UCNV_FROM_U_CALLBACK_STOP, nullptr, nullptr, nullptr, + &error); + if (U_FAILURE(error)) { + return Status::InternalError("Failed to configure ICU converter '{}': {}", target_name, + u_errorName(error)); + } + return Status::OK(); + } + + Status convert(StringRef input, std::string_view character_set_name, std::string& output) { + output.clear(); + if (input.size == 0) { + return Status::OK(); + } + if (input.size > static_cast(std::numeric_limits::max())) { + return Status::InvalidArgument("Input is too large for character conversion using '{}'", + character_set_name); + } + + UErrorCode error = U_ZERO_ERROR; + int32_t utf16_size = ucnv_toUChars(_source.get(), nullptr, 0, input.data, + static_cast(input.size), &error); + if (error != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(error)) { + return conversion_error(character_set_name, error); + } + + _utf16.resize(static_cast(utf16_size)); + error = U_ZERO_ERROR; + ucnv_toUChars(_source.get(), _utf16.data(), utf16_size, input.data, + static_cast(input.size), &error); + if (U_FAILURE(error)) { + return conversion_error(character_set_name, error); + } + + error = U_ZERO_ERROR; + int32_t output_size = + ucnv_fromUChars(_target.get(), nullptr, 0, _utf16.data(), utf16_size, &error); + if (error != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(error)) { + return conversion_error(character_set_name, error); + } + + output.resize(static_cast(output_size)); + error = U_ZERO_ERROR; + ucnv_fromUChars(_target.get(), output.data(), output_size, _utf16.data(), utf16_size, + &error); + if (U_FAILURE(error)) { + return conversion_error(character_set_name, error); + } + return Status::OK(); + } + +private: + static Status conversion_error(std::string_view character_set_name, UErrorCode error) { + return Status::InvalidArgument("Character conversion using '{}' failed: {}", + character_set_name, u_errorName(error)); + } + + ConverterPtr _source; + ConverterPtr _target; + std::vector _utf16; +}; + +template +class FunctionCharacterEncoding : public IFunction { +public: + static constexpr auto name = Encode ? "encode" : "decode"; + + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { return 2; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + DataTypePtr result_type; + if constexpr (Encode) { + result_type = std::make_shared(); + } else { + result_type = std::make_shared(); + } + return have_nullable(arguments) ? make_nullable(result_type) : result_type; + } + + bool use_default_implementation_for_nulls() const override { return false; } + + Status execute_impl(FunctionContext* /*context*/, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + auto [input_column, input_is_const] = + unpack_if_const(block.get_by_position(arguments[0]).column); + auto [character_set_column, character_set_is_const] = + unpack_if_const(block.get_by_position(arguments[1]).column); + const auto* input_nullable = check_and_get_column(input_column.get()); + const auto* character_set_nullable = + check_and_get_column(character_set_column.get()); + const IColumn* input_nested = + input_nullable ? &input_nullable->get_nested_column() : input_column.get(); + const IColumn* character_set_nested = character_set_nullable + ? &character_set_nullable->get_nested_column() + : character_set_column.get(); + const auto& character_sets = assert_cast(*character_set_nested); + const NullMap* input_null_map = + input_nullable ? &input_nullable->get_null_map_data() : nullptr; + const NullMap* character_set_null_map = + character_set_nullable ? &character_set_nullable->get_null_map_data() : nullptr; + const bool has_nullable = input_null_map != nullptr || character_set_null_map != nullptr; + auto result_column = create_result_column(); + result_column->reserve(input_rows_count); + ColumnUInt8::MutablePtr result_null_column; + if (has_nullable) { + result_null_column = ColumnUInt8::create(input_rows_count, 0); + } + ConverterCache converters; + std::string converted; + CharacterSet constant_character_set = CharacterSet::UTF_8; + if (character_set_is_const && input_rows_count != 0 && + !(character_set_null_map && (*character_set_null_map)[0])) { + RETURN_IF_ERROR( + parse_character_set(character_sets.get_data_at(0), constant_character_set)); + } + + for (size_t row = 0; row < input_rows_count; ++row) { + const size_t input_index = index_check_const(row, input_is_const); + const size_t character_set_index = index_check_const(row, character_set_is_const); + const bool input_is_null = input_null_map && (*input_null_map)[input_index]; + const bool character_set_is_null = + character_set_null_map && (*character_set_null_map)[character_set_index]; + if (input_is_null || character_set_is_null) { + result_column->insert_default(); + result_null_column->get_data()[row] = 1; + continue; + } + + CharacterSet character_set = constant_character_set; + if (!character_set_is_const) { + RETURN_IF_ERROR(parse_character_set(character_sets.get_data_at(character_set_index), + character_set)); + } + + const StringRef input = input_nested->get_data_at(input_index); + RETURN_IF_ERROR(convert_input(input, character_set, converters, converted)); + result_column->insert_data(converted.data(), converted.size()); + } + + if (has_nullable) { + block.replace_by_position(result, + ColumnNullable::create(std::move(result_column), + std::move(result_null_column))); + } else { + block.replace_by_position(result, std::move(result_column)); + } + return Status::OK(); + } + +private: + using ResultColumn = std::conditional_t; + static constexpr auto UTF16_LITTLE_ENDIAN_CONVERTER = static_cast(CharacterSet::SIZE); + using ConverterCache = + std::array, UTF16_LITTLE_ENDIAN_CONVERTER + 1>; + + struct ConversionSpec { + StringRef input; + size_t converter_index; + std::string_view converter_character_set; + }; + + static typename ResultColumn::MutablePtr create_result_column() { + return ResultColumn::create(); + } + + static ConversionSpec get_conversion_spec(StringRef input, CharacterSet character_set) { + const auto character_set_index = static_cast(character_set); + ConversionSpec spec {input, character_set_index, + SUPPORTED_CHARACTER_SETS[character_set_index]}; + if (character_set != CharacterSet::UTF_16) { + return spec; + } + + // Java's UTF-16 encoder always emits a big-endian BOM. ICU's generic UTF-16 converter + // follows the host byte order, so encode with UTF-16BE and add the BOM explicitly. + spec.converter_character_set = + SUPPORTED_CHARACTER_SETS[static_cast(CharacterSet::UTF_16BE)]; + if constexpr (Encode) { + return spec; + } + + // Java's UTF-16 decoder honors either BOM and defaults to big endian without a BOM. + if (input.size < 2) { + return spec; + } + const auto first = static_cast(input.data[0]); + const auto second = static_cast(input.data[1]); + if (first == 0xFE && second == 0xFF) { + spec.input = input.substring(2); + } else if (first == 0xFF && second == 0xFE) { + spec.input = input.substring(2); + spec.converter_index = UTF16_LITTLE_ENDIAN_CONVERTER; + spec.converter_character_set = + SUPPORTED_CHARACTER_SETS[static_cast(CharacterSet::UTF_16LE)]; + } + return spec; + } + + static Status convert_input(StringRef input, CharacterSet character_set, + ConverterCache& converters, std::string& converted) { + const ConversionSpec spec = get_conversion_spec(input, character_set); + if (converters[spec.converter_index] == nullptr) { + converters[spec.converter_index] = std::make_unique(); + if constexpr (Encode) { + RETURN_IF_ERROR(converters[spec.converter_index]->open( + "UTF-8", spec.converter_character_set)); + } else { + RETURN_IF_ERROR(converters[spec.converter_index]->open(spec.converter_character_set, + "UTF-8")); + } + } + + RETURN_IF_ERROR(converters[spec.converter_index]->convert( + spec.input, SUPPORTED_CHARACTER_SETS[static_cast(character_set)], + converted)); + if constexpr (Encode) { + if (character_set == CharacterSet::UTF_16 && input.size != 0) { + converted.insert(0, "\xFE\xFF", 2); + } + } + return Status::OK(); + } +}; + +using FunctionEncode = FunctionCharacterEncoding; +using FunctionDecode = FunctionCharacterEncoding; + +} // namespace + +void register_function_character_encoding(SimpleFunctionFactory& factory) { + factory.register_function(); + factory.register_function(); +} + +} // namespace doris diff --git a/be/src/exprs/function/simple_function_factory.h b/be/src/exprs/function/simple_function_factory.h index 5dd09e0847dba5..60d5bb14827758 100644 --- a/be/src/exprs/function/simple_function_factory.h +++ b/be/src/exprs/function/simple_function_factory.h @@ -123,6 +123,7 @@ void register_function_binary(SimpleFunctionFactory& factory); void register_function_levenshtein(SimpleFunctionFactory& factory); void register_function_hamming_distance(SimpleFunctionFactory& factory); void register_function_soundex(SimpleFunctionFactory& factory); +void register_function_character_encoding(SimpleFunctionFactory& factory); #if defined(BE_TEST) && !defined(BE_BENCHMARK) void register_function_throw_exception(SimpleFunctionFactory& factory); @@ -366,6 +367,7 @@ class SimpleFunctionFactory { register_function_levenshtein(instance); register_function_hamming_distance(instance); register_function_soundex(instance); + register_function_character_encoding(instance); register_function_json_transform(instance); register_function_json_hash(instance); #if defined(BE_TEST) && !defined(BE_BENCHMARK) diff --git a/be/test/exprs/function/function_character_encoding_test.cpp b/be/test/exprs/function/function_character_encoding_test.cpp new file mode 100644 index 00000000000000..908665f0517dd0 --- /dev/null +++ b/be/test/exprs/function/function_character_encoding_test.cpp @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" +#include "exprs/function/function_test_util.h" + +namespace doris { + +using namespace ut_type; + +TEST(function_character_encoding_test, encode_supported_charsets) { + // The UTF-16 byte pairs 0x4E2D and 0x2D4E are "N-" and "-N" as raw bytes. + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}; + DataSet data_set = { + {{std::string("A"), std::string("US-ASCII")}, VARBINARY("A")}, + {{std::string("é"), std::string("ISO-8859-1")}, VARBINARY("\xE9")}, + {{std::string("中"), std::string("UTF-8")}, VARBINARY("\xE4\xB8\xAD")}, + {{std::string("中"), std::string("UTF-16BE")}, VARBINARY("N-")}, + {{std::string("中"), std::string("UTF-16LE")}, VARBINARY("-N")}, + {{std::string("中"), std::string("UTF-16")}, VARBINARY("\xFE\xFF\x4E\x2D")}, + {{std::string("😀"), std::string("utf-16be")}, + VARBINARY(std::string_view("\xD8\x3D\xDE\x00", 4))}, + {{std::string("A\0中", 5), std::string("UTF-8")}, + VARBINARY(std::string_view("A\0\xE4\xB8\xAD", 5))}, + {{std::string(""), std::string("UTF-16")}, VARBINARY("")}, + {{Null(), std::string("UTF-8")}, Null()}, + {{std::string("text"), Null()}, Null()}, + }; + + check_function_all_arg_comb("encode", input_types, data_set); +} + +TEST(function_character_encoding_test, decode_supported_charsets) { + // The UTF-16 byte pairs 0x4E2D and 0x2D4E are "N-" and "-N" as raw bytes. + InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARCHAR}; + DataSet data_set = { + {{VARBINARY("A"), std::string("US-ASCII")}, std::string("A")}, + {{VARBINARY("\xE9"), std::string("ISO-8859-1")}, std::string("é")}, + {{VARBINARY("\xE4\xB8\xAD"), std::string("UTF-8")}, std::string("中")}, + {{VARBINARY("N-"), std::string("UTF-16BE")}, std::string("中")}, + {{VARBINARY("-N"), std::string("UTF-16LE")}, std::string("中")}, + {{VARBINARY("\xFE\xFF\x4E\x2D"), std::string("UTF-16")}, std::string("中")}, + {{VARBINARY("\xFF\xFE\x2D\x4E"), std::string("utf-16")}, std::string("中")}, + {{VARBINARY("N-"), std::string("UTF-16")}, std::string("中")}, + {{VARBINARY("\xFE\xFF"), std::string("UTF-16")}, std::string("")}, + {{VARBINARY(std::string_view("\xD8\x3D\xDE\x00", 4)), std::string("UTF-16BE")}, + std::string("😀")}, + {{VARBINARY(std::string_view("A\0\xE4\xB8\xAD", 5)), std::string("UTF-8")}, + std::string("A\0中", 5)}, + {{VARBINARY(""), std::string("UTF-16")}, std::string("")}, + {{Null(), std::string("UTF-8")}, Null()}, + {{VARBINARY("text"), Null()}, Null()}, + }; + + check_function_all_arg_comb("decode", input_types, data_set); +} + +TEST(function_character_encoding_test, rejects_invalid_conversions) { + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}; + DataSet data_set = { + {{std::string("text"), std::string("GBK")}, VARBINARY("")}, + }; + + Status status = check_function("encode", input_types, data_set, -1, + -1, true); + ASSERT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Unsupported character set"), std::string::npos); + } + + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}; + DataSet data_set = { + {{std::string("中"), std::string("US-ASCII")}, VARBINARY("")}, + }; + + Status status = check_function("encode", input_types, data_set, -1, + -1, true); + ASSERT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Character conversion using 'US-ASCII' failed"), + std::string::npos); + } + + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARCHAR}; + DataSet data_set = { + {{VARBINARY("\xE4\xB8"), std::string("UTF-8")}, std::string("")}, + }; + + Status status = + check_function("decode", input_types, data_set, -1, -1, true); + ASSERT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Character conversion using 'UTF-8' failed"), + std::string::npos); + } +} + +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index af2826d7b116f1..00ba6a61d9233e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -183,6 +183,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; import org.apache.doris.nereids.trees.expressions.functions.scalar.Dceil; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Decode; import org.apache.doris.nereids.trees.expressions.functions.scalar.DecodeAsVarchar; import org.apache.doris.nereids.trees.expressions.functions.scalar.DeduplicateMap; import org.apache.doris.nereids.trees.expressions.functions.scalar.Degrees; @@ -200,6 +201,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.E; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Elt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Encode; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsBigInt; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsInt; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsLargeInt; @@ -778,6 +780,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(DaysDiff.class, "days_diff"), scalar(DaysSub.class, "days_sub", "date_sub", "subdate"), scalar(Dceil.class, "dceil"), + scalar(Decode.class, "decode"), scalar(DecodeAsVarchar.class, "decode_as_varchar"), scalar(DeduplicateMap.class, "deduplicate_map"), scalar(Degrees.class, "degrees"), @@ -797,6 +800,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(ElementAt.class, "element_at", "struct_element"), scalar(Elt.class, "elt"), scalar(Embed.class, "embed"), + scalar(Encode.class, "encode"), scalar(EncodeAsSmallInt.class, "encode_as_smallint"), scalar(EncodeAsInt.class, "encode_as_int"), scalar(EncodeAsBigInt.class, "encode_as_bigint"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java index f670ab0a2d8a2c..46f605f8fc7b53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java @@ -40,6 +40,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.types.ArrayType; @@ -52,7 +53,9 @@ import java.net.URLDecoder; import java.net.URLEncoder; import java.nio.ByteBuffer; +import java.nio.CharBuffer; import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -1145,6 +1148,69 @@ public static Expression urlEncode(StringLikeLiteral first) { } } + /** + * Executable arithmetic function encode + */ + @ExecFunction(name = "encode") + public static Expression encode(StringLikeLiteral source, StringLikeLiteral characterSet) { + Charset charset = supportedCharacterSet(characterSet.getValue()); + try { + ByteBuffer encoded = charset.newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .encode(CharBuffer.wrap(source.getValue())); + byte[] bytes = new byte[encoded.remaining()]; + encoded.get(bytes); + return new VarBinaryLiteral(bytes); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("Failed to encode value using " + characterSet.getValue(), e); + } + } + + /** + * Executable arithmetic function decode + */ + @ExecFunction(name = "decode") + public static Expression decode(VarBinaryLiteral binary, StringLikeLiteral characterSet) { + Charset charset = supportedCharacterSet(characterSet.getValue()); + try { + CharBuffer decoded = charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap((byte[]) binary.getValue())); + return new StringLiteral(decoded.toString()); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("Failed to decode value using " + characterSet.getValue(), e); + } + } + + private static Charset supportedCharacterSet(String name) { + String canonicalName; + switch (name.toUpperCase(Locale.ROOT)) { + case "US-ASCII": + canonicalName = "US-ASCII"; + break; + case "ISO-8859-1": + canonicalName = "ISO-8859-1"; + break; + case "UTF-8": + canonicalName = "UTF-8"; + break; + case "UTF-16BE": + canonicalName = "UTF-16BE"; + break; + case "UTF-16LE": + canonicalName = "UTF-16LE"; + break; + case "UTF-16": + canonicalName = "UTF-16"; + break; + default: + throw new IllegalArgumentException("Unsupported character set: " + name); + } + return Charset.forName(canonicalName); + } + /** * Executable arithmetic functions append_trailing_char_if_absent */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java new file mode 100644 index 00000000000000..86153192f8b988 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'decode'. This class is generated by GenerateFunction. + */ +public class Decode extends ScalarFunction + implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(StringType.INSTANCE).args(VarBinaryType.INSTANCE, StringType.INSTANCE) + ); + + /** + * constructor with 2 arguments. + */ + public Decode(Expression binary, Expression characterSet) { + super("decode", binary, characterSet); + } + + /** constructor for withChildren and reuse signature */ + private Decode(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public Decode withChildren(List children) { + Preconditions.checkArgument(children.size() == 2); + return new Decode(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitDecode(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java new file mode 100644 index 00000000000000..3758a195e7d408 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'encode'. This class is generated by GenerateFunction. + */ +public class Encode extends ScalarFunction + implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(VarBinaryType.INSTANCE).args(StringType.INSTANCE, StringType.INSTANCE) + ); + + /** + * constructor with 2 arguments. + */ + public Encode(Expression source, Expression characterSet) { + super("encode", source, characterSet); + } + + /** constructor for withChildren and reuse signature */ + private Encode(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public Encode withChildren(List children) { + Preconditions.checkArgument(children.size() == 2); + return new Encode(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitEncode(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 26930ccd27a84a..030890135d3f75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -200,6 +200,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; import org.apache.doris.nereids.trees.expressions.functions.scalar.Dceil; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Decode; import org.apache.doris.nereids.trees.expressions.functions.scalar.DecodeAsVarchar; import org.apache.doris.nereids.trees.expressions.functions.scalar.Degrees; import org.apache.doris.nereids.trees.expressions.functions.scalar.Dexp; @@ -216,6 +217,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.E; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Elt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Encode; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsBigInt; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsInt; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsLargeInt; @@ -1085,6 +1087,10 @@ default R visitCutToFirstSignificantSubdomain(CutToFirstSignificantSubdomain cut return visitScalarFunction(cutToFirstSignificantSubdomain, context); } + default R visitEncode(Encode encode, C context) { + return visitScalarFunction(encode, context); + } + default R visitEncodeAsSmallInt(EncodeAsSmallInt encode, C context) { return visitScalarFunction(encode, context); } @@ -1290,6 +1296,10 @@ default R visitDigitalMasking(DigitalMasking digitalMasking, C context) { return visitScalarFunction(digitalMasking, context); } + default R visitDecode(Decode decode, C context) { + return visitScalarFunction(decode, context); + } + default R visitDecodeAsVarchar(DecodeAsVarchar decode, C context) { return visitScalarFunction(decode, context); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java index 5381bc8e062acc..79aee87e9104cd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java @@ -19,6 +19,8 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.ExpressionEvaluator; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Decode; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Encode; import org.apache.doris.nereids.trees.expressions.functions.scalar.UrlDecode; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; @@ -26,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -95,8 +98,59 @@ void testUrlDecodeStillFoldsValidUtf8() { assertUrlDecodeValue("%EF%BF%BD", "�"); } + @Test + void testEncodeFoldsSupportedCharsets() { + assertEncodeValue("A", "US-ASCII", new byte[] {0x41}); + assertEncodeValue("é", "ISO-8859-1", new byte[] {(byte) 0xE9}); + assertEncodeValue("中", "UTF-8", new byte[] {(byte) 0xE4, (byte) 0xB8, (byte) 0xAD}); + assertEncodeValue("中", "UTF-16BE", new byte[] {0x4E, 0x2D}); + assertEncodeValue("中", "UTF-16LE", new byte[] {0x2D, 0x4E}); + assertEncodeValue("中", "utf-16", new byte[] {(byte) 0xFE, (byte) 0xFF, 0x4E, 0x2D}); + assertEncodeValue("😀", "UTF-16BE", new byte[] {(byte) 0xD8, 0x3D, (byte) 0xDE, 0x00}); + assertEncodeValue("", "UTF-16", new byte[] {}); + } + + @Test + void testDecodeFoldsSupportedCharsets() { + assertDecodeValue(new byte[] {0x41}, "US-ASCII", "A"); + assertDecodeValue(new byte[] {(byte) 0xE9}, "ISO-8859-1", "é"); + assertDecodeValue(new byte[] {(byte) 0xE4, (byte) 0xB8, (byte) 0xAD}, "UTF-8", "中"); + assertDecodeValue(new byte[] {0x4E, 0x2D}, "UTF-16BE", "中"); + assertDecodeValue(new byte[] {0x2D, 0x4E}, "UTF-16LE", "中"); + assertDecodeValue(new byte[] {(byte) 0xFE, (byte) 0xFF, 0x4E, 0x2D}, "UTF-16", "中"); + assertDecodeValue(new byte[] {(byte) 0xFF, (byte) 0xFE, 0x2D, 0x4E}, "utf-16", "中"); + assertDecodeValue(new byte[] {0x4E, 0x2D}, "UTF-16", "中"); + assertDecodeValue(new byte[] {(byte) 0xFE, (byte) 0xFF}, "UTF-16", ""); + assertDecodeValue(new byte[] {(byte) 0xD8, 0x3D, (byte) 0xDE, 0x00}, "UTF-16BE", "😀"); + assertDecodeValue(new byte[] {}, "UTF-16", ""); + } + + @Test + void testInvalidCharacterConversionDoesNotFold() { + Encode unmappable = new Encode(new StringLiteral("中"), new StringLiteral("US-ASCII")); + Decode malformed = new Decode(new VarBinaryLiteral(new byte[] {(byte) 0xE4, (byte) 0xB8}), + new StringLiteral("UTF-8")); + Encode unsupported = new Encode(new StringLiteral("text"), new StringLiteral("GBK")); + + Assertions.assertSame(unmappable, ExpressionEvaluator.INSTANCE.eval(unmappable)); + Assertions.assertSame(malformed, ExpressionEvaluator.INSTANCE.eval(malformed)); + Assertions.assertSame(unsupported, ExpressionEvaluator.INSTANCE.eval(unsupported)); + } + private void assertUrlDecodeValue(String encoded, String expected) { Expression result = ExpressionEvaluator.INSTANCE.eval(new UrlDecode(new StringLiteral(encoded))); Assertions.assertEquals(expected, ((StringLikeLiteral) result).getValue()); } + + private void assertEncodeValue(String value, String characterSet, byte[] expected) { + Expression result = ExpressionEvaluator.INSTANCE.eval( + new Encode(new StringLiteral(value), new StringLiteral(characterSet))); + Assertions.assertArrayEquals(expected, (byte[]) ((VarBinaryLiteral) result).getValue()); + } + + private void assertDecodeValue(byte[] value, String characterSet, String expected) { + Expression result = ExpressionEvaluator.INSTANCE.eval( + new Decode(new VarBinaryLiteral(value), new StringLiteral(characterSet))); + Assertions.assertEquals(expected, ((StringLikeLiteral) result).getValue()); + } } diff --git a/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy new file mode 100644 index 00000000000000..79aa656c9bb414 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_encode_decode") { + test { + sql "select encode('text', 'GBK')" + exception "Unsupported character set" + } + + test { + sql "select encode('中', 'US-ASCII')" + exception "Character conversion using 'US-ASCII' failed" + } + + test { + sql "select decode(X'E4B8', 'UTF-8')" + exception "Character conversion using 'UTF-8' failed" + } +} From 4b7d759925ebd689a1c57d870a4b1f6f749d8f35 Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Thu, 17 Sep 2026 16:29:04 +0800 Subject: [PATCH 02/11] [test](regression) Add encode and decode regression coverage ### What problem does this PR solve? Add positive regression coverage for Hive-compatible encode and decode across supported character sets, BOM handling, empty strings, and null propagation. ### How does this PR solve the problem? Exercise table-driven encode/decode queries and record runner-generated expected output while retaining the existing invalid-conversion checks. ### Check List - [x] Regression test - [x] FE unit test - [x] BE unit test --- .../binary_functions/test_encode_decode.out | 29 ++++++++++++++ .../test_encode_decode.groovy | 40 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out diff --git a/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out b/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out new file mode 100644 index 00000000000000..9f4f10dd67b474 --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out @@ -0,0 +1,29 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !encode_supported_charsets -- +1 41 +10 FEFF4E2D +11 \N +12 \N +2 E9 +3 E4B8AD +4 4E2D +5 2D4E +6 FEFF4E2D +7 D83DDE00 +8 +9 FEFF4E2D + +-- !decode_supported_charsets -- +1 A +10 中 +11 \N +12 \N +2 é +3 中 +4 中 +5 中 +6 中 +7 😀 +8 +9 中 + diff --git a/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy index 79aa656c9bb414..64806ff937ff1a 100644 --- a/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy +++ b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy @@ -16,6 +16,46 @@ // under the License. suite("test_encode_decode") { + sql "drop table if exists test_encode_decode" + sql """ + create table test_encode_decode ( + id int, + plain_text string, + binary_value string, + charset varchar(32) + ) duplicate key(id) + distributed by hash(id) buckets 1 + properties ("replication_num" = "1") + """ + + sql """ + insert into test_encode_decode values + (1, 'A', unhex('41'), 'US-ASCII'), + (2, 'é', unhex('E9'), 'ISO-8859-1'), + (3, '中', unhex('E4B8AD'), 'UTF-8'), + (4, '中', unhex('4E2D'), 'UTF-16BE'), + (5, '中', unhex('2D4E'), 'UTF-16LE'), + (6, '中', unhex('FEFF4E2D'), 'UTF-16'), + (7, '😀', unhex('D83DDE00'), 'UTF-16BE'), + (8, '', unhex(''), 'UTF-16'), + (9, '中', unhex('FFFE2D4E'), 'utf-16'), + (10, '中', unhex('4E2D'), 'UTF-16'), + (11, null, null, 'UTF-8'), + (12, 'text', unhex('74657874'), null) + """ + + order_qt_encode_supported_charsets """ + select id, hex(encode(plain_text, charset)) + from test_encode_decode + order by id + """ + + order_qt_decode_supported_charsets """ + select id, decode(cast(binary_value as varbinary), charset) + from test_encode_decode + order by id + """ + test { sql "select encode('text', 'GBK')" exception "Unsupported character set" From 7a9f82b620e68885d50a0ba7410ee2a53a4643fb Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Fri, 18 Sep 2026 10:45:44 +0800 Subject: [PATCH 03/11] [test](fe) Cover encode and decode expression contracts ### What problem does this PR solve? Issue Number: #48203 Related PR: #68131 Problem Summary: The FE incremental coverage gate reported 75.44% because the encode and decode scalar expression contracts were not exercised. Add focused unit tests for signatures, child rewriting, argument validation, and scalar visitor delegation. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeDecodeTest (3 tests passed) - ./run-fe-ut.sh --coverage --run org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeDecodeTest (3 tests passed; affected lines fully covered) - Behavior changed: No - Does this need documentation: No --- .../functions/scalar/EncodeDecodeTest.java | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java new file mode 100644 index 00000000000000..306b3fac3c88ec --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class EncodeDecodeTest { + + @Test + public void testEncodeExpressionContract() { + StringLiteral source = new StringLiteral("hello"); + StringLiteral characterSet = new StringLiteral("UTF-8"); + Encode encode = new Encode(source, characterSet); + + Assertions.assertEquals("encode", encode.getName()); + Assertions.assertEquals(2, encode.arity()); + Assertions.assertSame(source, encode.child(0)); + Assertions.assertSame(characterSet, encode.child(1)); + + FunctionSignature signature = encode.getSignatures().get(0); + Assertions.assertEquals(VarBinaryType.INSTANCE, signature.returnType); + Assertions.assertEquals(StringType.INSTANCE, signature.getArgType(0)); + Assertions.assertEquals(StringType.INSTANCE, signature.getArgType(1)); + + StringLiteral replacementSource = new StringLiteral("world"); + StringLiteral replacementCharacterSet = new StringLiteral("UTF-16"); + Encode rewritten = encode.withChildren( + ImmutableList.of(replacementSource, replacementCharacterSet)); + Assertions.assertNotSame(encode, rewritten); + Assertions.assertSame(replacementSource, rewritten.child(0)); + Assertions.assertSame(replacementCharacterSet, rewritten.child(1)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> encode.withChildren(ImmutableList.of(replacementSource))); + } + + @Test + public void testDecodeExpressionContract() { + VarBinaryLiteral binary = new VarBinaryLiteral(new byte[] {0x68, 0x69}); + StringLiteral characterSet = new StringLiteral("UTF-8"); + Decode decode = new Decode(binary, characterSet); + + Assertions.assertEquals("decode", decode.getName()); + Assertions.assertEquals(2, decode.arity()); + Assertions.assertSame(binary, decode.child(0)); + Assertions.assertSame(characterSet, decode.child(1)); + + FunctionSignature signature = decode.getSignatures().get(0); + Assertions.assertEquals(StringType.INSTANCE, signature.returnType); + Assertions.assertEquals(VarBinaryType.INSTANCE, signature.getArgType(0)); + Assertions.assertEquals(StringType.INSTANCE, signature.getArgType(1)); + + VarBinaryLiteral replacementBinary = new VarBinaryLiteral(new byte[] {0x41}); + StringLiteral replacementCharacterSet = new StringLiteral("US-ASCII"); + Decode rewritten = decode.withChildren( + ImmutableList.of(replacementBinary, replacementCharacterSet)); + Assertions.assertNotSame(decode, rewritten); + Assertions.assertSame(replacementBinary, rewritten.child(0)); + Assertions.assertSame(replacementCharacterSet, rewritten.child(1)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> decode.withChildren(ImmutableList.of(replacementBinary))); + } + + @Test + public void testVisitorDelegatesToScalarFunction() { + Encode encode = new Encode(new StringLiteral("hello"), new StringLiteral("UTF-8")); + Decode decode = new Decode(new VarBinaryLiteral(new byte[] {0x68, 0x69}), + new StringLiteral("UTF-8")); + ExpressionVisitor visitor = new ExpressionVisitor() { + @Override + public Expression visit(Expression expression, Void context) { + return expression; + } + }; + + Assertions.assertSame(encode, encode.accept(visitor, null)); + Assertions.assertSame(decode, decode.accept(visitor, null)); + } +} From 3196b6e3906fb5c3d6b59c09e008af76997b20df Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Fri, 18 Sep 2026 16:08:10 +0800 Subject: [PATCH 04/11] [fix](fe) Align charset matching with backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What problem does this PR solve? Issue Number: close #48203 Related PR: #68131 Problem Summary: FE constant folding normalized character set names with Unicode uppercasing, while BE only performs ASCII case-insensitive matching. This allowed names such as Uſ-ASCII to fold successfully in FE but fail when supplied through a runtime column. Match supported names with ASCII-only comparison and add literal-versus-runtime regression coverage. ### Release note Fix inconsistent character set validation between FE constant folding and BE runtime execution for encode and decode. ### Check List (For Author) - Test: Unit Test / Regression test - ./run-fe-ut.sh --run org.apache.doris.nereids.trees.expressions.functions.executable.StringArithmeticTest (13 tests passed) - Added regression coverage for literal and runtime character set arguments; not run locally because it requires a Linux Doris cluster - Behavior changed: Yes. Non-ASCII names that only become supported through Unicode case conversion are now rejected consistently. - Does this need documentation: No --- .../executable/StringArithmetic.java | 46 +++++++++---------- .../executable/StringArithmeticTest.java | 10 ++++ .../test_encode_decode.groovy | 12 +++++ 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java index caa53ef7099f01..0277e680600f1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java @@ -73,6 +73,8 @@ */ public class StringArithmetic { private static final long MAX_DAMERAU_LEVENSHTEIN_MATRIX_CELLS = 16L * 1024L * 1024L; + private static final List SUPPORTED_CHARACTER_SETS = ImmutableList.of( + "US-ASCII", "ISO-8859-1", "UTF-8", "UTF-16BE", "UTF-16LE", "UTF-16"); private static Literal castStringLikeLiteral(StringLikeLiteral first, String value) { if (first instanceof StringLiteral) { @@ -1191,30 +1193,28 @@ public static Expression decode(VarBinaryLiteral binary, StringLikeLiteral chara } private static Charset supportedCharacterSet(String name) { - String canonicalName; - switch (name.toUpperCase(Locale.ROOT)) { - case "US-ASCII": - canonicalName = "US-ASCII"; - break; - case "ISO-8859-1": - canonicalName = "ISO-8859-1"; - break; - case "UTF-8": - canonicalName = "UTF-8"; - break; - case "UTF-16BE": - canonicalName = "UTF-16BE"; - break; - case "UTF-16LE": - canonicalName = "UTF-16LE"; - break; - case "UTF-16": - canonicalName = "UTF-16"; - break; - default: - throw new IllegalArgumentException("Unsupported character set: " + name); + for (String supportedCharacterSet : SUPPORTED_CHARACTER_SETS) { + if (equalsIgnoreAsciiCase(name, supportedCharacterSet)) { + return Charset.forName(supportedCharacterSet); + } + } + throw new IllegalArgumentException("Unsupported character set: " + name); + } + + private static boolean equalsIgnoreAsciiCase(String value, String expected) { + if (value.length() != expected.length()) { + return false; + } + for (int i = 0; i < value.length(); i++) { + char current = value.charAt(i); + if (current >= 'a' && current <= 'z') { + current -= 'a' - 'A'; + } + if (current != expected.charAt(i)) { + return false; + } } - return Charset.forName(canonicalName); + return true; } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java index e71b8265a5d249..de577e507e27c5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java @@ -138,6 +138,16 @@ void testInvalidCharacterConversionDoesNotFold() { Assertions.assertSame(unsupported, ExpressionEvaluator.INSTANCE.eval(unsupported)); } + @Test + void testUnicodeCaseFoldedCharacterSetDoesNotFold() { + Encode encode = new Encode(new StringLiteral("A"), new StringLiteral("U\u017F-ASCII")); + Decode decode = new Decode(new VarBinaryLiteral(new byte[] {0x41}), + new StringLiteral("U\u017F-ASCII")); + + Assertions.assertSame(encode, ExpressionEvaluator.INSTANCE.eval(encode)); + Assertions.assertSame(decode, ExpressionEvaluator.INSTANCE.eval(decode)); + } + private void assertUrlDecodeValue(String encoded, String expected) { Expression result = ExpressionEvaluator.INSTANCE.eval(new UrlDecode(new StringLiteral(encoded))); Assertions.assertEquals(expected, ((StringLikeLiteral) result).getValue()); diff --git a/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy index 64806ff937ff1a..5449fd0835edb1 100644 --- a/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy +++ b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy @@ -70,4 +70,16 @@ suite("test_encode_decode") { sql "select decode(X'E4B8', 'UTF-8')" exception "Character conversion using 'UTF-8' failed" } + + test { + sql "select encode('A', 'Uſ-ASCII')" + exception "Unsupported character set" + } + + sql "insert into test_encode_decode values (13, 'A', unhex('41'), 'Uſ-ASCII')" + + test { + sql "select encode(plain_text, charset) from test_encode_decode where id = 13" + exception "Unsupported character set" + } } From c980c3094ad272bd3944ead39f8769a1d8ce662b Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Fri, 18 Sep 2026 20:17:23 +0800 Subject: [PATCH 05/11] [fix](fe) Satisfy Unicode escape checkstyle ### What problem does this PR solve? Issue Number: #48203 Related PR: #68131 Problem Summary: The clean TeamCity compile build rejects the U+017F test literal because its Unicode escape lacks the trailing code point explanation required by FE Checkstyle. Define the charset once with a trailing U+017F comment and reuse it for encode and decode. ### Release note None ### Check List (For Author) - Test: Unit Test - mvn -pl fe-core -DskipTests -Dcheckstyle.cache.file=/tmp/doris-checkstyle-green-cache-68131 checkstyle:check (passed) - ./run-fe-ut.sh --run org.apache.doris.nereids.trees.expressions.functions.executable.StringArithmeticTest (13 tests passed) - Behavior changed: No - Does this need documentation: No --- .../functions/executable/StringArithmeticTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java index de577e507e27c5..cf143f7a226e24 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java @@ -140,9 +140,10 @@ void testInvalidCharacterConversionDoesNotFold() { @Test void testUnicodeCaseFoldedCharacterSetDoesNotFold() { - Encode encode = new Encode(new StringLiteral("A"), new StringLiteral("U\u017F-ASCII")); + String unicodeCaseFoldedCharset = "U\u017F-ASCII"; // U+017F LATIN SMALL LETTER LONG S + Encode encode = new Encode(new StringLiteral("A"), new StringLiteral(unicodeCaseFoldedCharset)); Decode decode = new Decode(new VarBinaryLiteral(new byte[] {0x41}), - new StringLiteral("U\u017F-ASCII")); + new StringLiteral(unicodeCaseFoldedCharset)); Assertions.assertSame(encode, ExpressionEvaluator.INSTANCE.eval(encode)); Assertions.assertSame(decode, ExpressionEvaluator.INSTANCE.eval(decode)); From ad2aba0e47a27f77f128b391344f206c429221b0 Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Mon, 21 Sep 2026 15:18:16 +0800 Subject: [PATCH 06/11] [improvement](be) Streamline encode and decode conversion ### What problem does this PR solve? Issue Number: close #48203 Related PR: #68131 Problem Summary: The BE character conversion path preflighted and converted every value twice through a full UTF-16 intermediate buffer. This repeatedly scanned input and pivot data and copied decode output once more into the result column. Stream conversion through ICU ucnv_convertEx with a bounded pivot, reuse block buffers, append decode output directly to ColumnString, and preserve strict conversion errors and Hive-compatible UTF-16 BOM behavior. On the same Linux Release build, the final implementation improved all 40 column execution cases by 1.96x to 3.92x. Extended reruns for every case whose initial CV exceeded 5% reduced CV to 1.0% to 2.8% and measured 2.17x to 3.93x. ### Release note Improve the query performance of encode and decode without changing their SQL behavior. ### Check List (For Author) - Test: Unit Test / Manual performance test - Linux Release build with ./build.sh --benchmark (passed) - ./run-be-ut.sh --run --filter='function_character_encoding_test.*' (5 tests passed) - 40-case Release benchmark correctness smoke test (40/40 passed) - Fixed-CPU ABBA performance test with 10 samples per case (1.96x to 3.92x) - Extended ABBA rerun for every case with initial CV above 5% (2.17x to 3.93x, CV 1.0% to 2.8%) - build-support/check-format.sh (passed) - build-support/check-build-hygiene.sh (passed) - clang-tidy attempted but blocked by the pre-existing unmatched NOLINTEND in be/src/core/types.h:576 - Behavior changed: No - Does this need documentation: Yes. https://github.com/apache/doris-website/pull/4151 --- be/benchmark/benchmark_character_encoding.hpp | 181 ++++++++++++++++++ be/benchmark/benchmark_main.cpp | 1 + .../function/function_character_encoding.cpp | 115 ++++++----- .../function_character_encoding_test.cpp | 72 +++++++ 4 files changed, 326 insertions(+), 43 deletions(-) create mode 100644 be/benchmark/benchmark_character_encoding.hpp diff --git a/be/benchmark/benchmark_character_encoding.hpp b/be/benchmark/benchmark_character_encoding.hpp new file mode 100644 index 00000000000000..91476ed2cdc29d --- /dev/null +++ b/be/benchmark/benchmark_character_encoding.hpp @@ -0,0 +1,181 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "core/block/block.h" +#include "core/column/column_const.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" +#include "exprs/function/simple_function_factory.h" +#include "exprs/function_context.h" +#include "util/defer_op.h" + +namespace doris { +namespace { + +struct CharacterEncodingData { + std::string input; + std::string expected; + std::string charset; +}; + +template +CharacterEncodingData make_character_encoding_data(size_t length) { + CharacterEncodingData data; + auto& [input, expected, charset] = data; + if constexpr (Scenario == 0) { + charset = "UTF-16BE"; + input.assign(length, 'A'); + for (size_t i = 0; i < length; ++i) { + expected.append("\0A", 2); + } + } else if constexpr (Scenario == 1) { + charset = "ISO-8859-1"; + input.assign(length, '\xE9'); + for (size_t i = 0; i < length; ++i) { + expected += "é"; + } + } else if constexpr (Scenario == 2) { + charset = "UTF-8"; + input.assign(length - 7, 'A'); + input += "中😀"; + expected = input; + } else { + charset = "UTF-16BE"; + for (size_t i = 0; i < length / 2; ++i) { + input += "N-"; // The UTF-16BE byte pair for 中. + expected += "中"; + } + } + return data; +} + +// Benchmark actual block execution, including result allocation and converter setup, not just +// ICU calls. Inputs are materialized columns so constant folding cannot eliminate conversion. +// Args: input size, constant charset (0/1). Large rows use smaller blocks to bound working memory. +template +void BM_character_encoding(benchmark::State& state) { + const size_t length = state.range(0); + const size_t rows = std::min(4096, (1 << 20) / length); + const auto [input, expected, charset] = make_character_encoding_data(length); + DataTypePtr string_type = std::make_shared(); + DataTypePtr binary_type = std::make_shared(); + DataTypePtr input_type = Encode ? string_type : binary_type; + DataTypePtr result_type = Encode ? binary_type : string_type; + auto values = input_type->create_column(); + for (size_t i = 0; i < rows; ++i) { + values->insert_data(input.data(), input.size()); + } + auto charsets = ColumnString::create(); + const bool constant_charset = state.range(1) != 0; + for (size_t i = 0; i < (constant_charset ? 1 : rows); ++i) { + charsets->insert_data(charset.data(), charset.size()); + } + ColumnPtr charset_column; + if (constant_charset) { + charset_column = ColumnConst::create(std::move(charsets), rows); + } else { + charset_column = std::move(charsets); + } + Block block {{std::move(values), input_type, "input"}, + {std::move(charset_column), string_type, "charset"}}; + auto function = SimpleFunctionFactory::instance().get_function( + Encode ? "encode" : "decode", block.get_columns_with_type_and_name(), result_type); + if (function == nullptr) { + state.SkipWithError("Character encoding function not registered"); + return; + } + auto context = FunctionContext::create_context(nullptr, result_type, {input_type, string_type}); + auto status = function->open(context.get(), FunctionContext::FRAGMENT_LOCAL); + if (!status.ok()) { + state.SkipWithError(status.to_string()); + return; + } + Defer close_fragment {[&] { + auto close_status = function->close(context.get(), FunctionContext::FRAGMENT_LOCAL); + if (!close_status.ok()) { + state.SkipWithError(close_status.to_string()); + } + }}; + status = function->open(context.get(), FunctionContext::THREAD_LOCAL); + if (!status.ok()) { + state.SkipWithError(status.to_string()); + return; + } + Defer close_thread {[&] { + auto close_status = function->close(context.get(), FunctionContext::THREAD_LOCAL); + if (!close_status.ok()) { + state.SkipWithError(close_status.to_string()); + } + }}; + block.insert({nullptr, result_type, "result"}); + + // Verify every output outside the measured loop. + status = function->execute(context.get(), block, {0, 1}, 2, rows); + if (!status.ok()) { + state.SkipWithError(status.to_string()); + return; + } + for (size_t i = 0; i < rows; ++i) { + auto actual = block.get_by_position(2).column->get_data_at(i); + if (std::string_view(actual.data, actual.size) != expected) { + state.SkipWithError("Character encoding result mismatch"); + return; + } + } + for (auto _ : state) { + status = function->execute(context.get(), block, {0, 1}, 2, rows); + if (!status.ok()) { + state.SkipWithError(status.to_string()); + break; + } + benchmark::DoNotOptimize(block.get_by_position(2).column); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * rows); + state.SetBytesProcessed(state.iterations() * rows * input.size()); +} + +BENCHMARK_TEMPLATE(BM_character_encoding, true, 0) + ->Name("encode_utf16be_ascii") + ->ArgsProduct({{15, 63, 1023, 65535}, {0, 1}}); +BENCHMARK_TEMPLATE(BM_character_encoding, false, 1) + ->Name("decode_latin1_nonascii") + ->ArgsProduct({{15, 63, 1023, 65535}, {0, 1}}); +BENCHMARK_TEMPLATE(BM_character_encoding, true, 2) + ->Name("encode_utf8_mixed") + ->ArgsProduct({{15, 63, 1023, 65535}, {0, 1}}); +BENCHMARK_TEMPLATE(BM_character_encoding, false, 2) + ->Name("decode_utf8_mixed") + ->ArgsProduct({{15, 63, 1023, 65535}, {0, 1}}); +BENCHMARK_TEMPLATE(BM_character_encoding, false, 3) + ->Name("decode_utf16be_cjk") + ->ArgsProduct({{16, 64, 1024, 65536}, {0, 1}}); + +} // namespace +} // namespace doris diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp index acbc591effd4e6..d82efa2a11f711 100644 --- a/be/benchmark/benchmark_main.cpp +++ b/be/benchmark/benchmark_main.cpp @@ -27,6 +27,7 @@ #include "benchmark_binary_arithmetic.hpp" #include "benchmark_bit_pack.hpp" #include "benchmark_case_expr.hpp" +#include "benchmark_character_encoding.hpp" #include "benchmark_column_array_view.hpp" #include "benchmark_column_array_view_distance.hpp" #include "benchmark_fastunion.hpp" diff --git a/be/src/exprs/function/function_character_encoding.cpp b/be/src/exprs/function/function_character_encoding.cpp index edbaa72843a58f..c7897395674433 100644 --- a/be/src/exprs/function/function_character_encoding.cpp +++ b/be/src/exprs/function/function_character_encoding.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -26,7 +27,6 @@ #include #include #include -#include #include "common/status.h" #include "core/assert_cast.h" @@ -127,8 +127,8 @@ class ConverterPair { return Status::OK(); } - Status convert(StringRef input, std::string_view character_set_name, std::string& output) { - output.clear(); + Status convert(StringRef input, std::string_view character_set_name, + ColumnString::Chars& output) { if (input.size == 0) { return Status::OK(); } @@ -137,36 +137,51 @@ class ConverterPair { character_set_name); } - UErrorCode error = U_ZERO_ERROR; - int32_t utf16_size = ucnv_toUChars(_source.get(), nullptr, 0, input.data, - static_cast(input.size), &error); - if (error != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(error)) { - return conversion_error(character_set_name, error); - } - - _utf16.resize(static_cast(utf16_size)); - error = U_ZERO_ERROR; - ucnv_toUChars(_source.get(), _utf16.data(), utf16_size, input.data, - static_cast(input.size), &error); - if (U_FAILURE(error)) { - return conversion_error(character_set_name, error); - } - - error = U_ZERO_ERROR; - int32_t output_size = - ucnv_fromUChars(_target.get(), nullptr, 0, _utf16.data(), utf16_size, &error); - if (error != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(error)) { - return conversion_error(character_set_name, error); - } - - output.resize(static_cast(output_size)); - error = U_ZERO_ERROR; - ucnv_fromUChars(_target.get(), output.data(), output_size, _utf16.data(), utf16_size, - &error); - if (U_FAILURE(error)) { - return conversion_error(character_set_name, error); + // Keep only a bounded Unicode pivot, rather than materializing the entire UTF-16 + // string and scanning both the source and the pivot twice to preflight sizes. + UChar pivot[1024]; + UChar* pivot_source = pivot; + UChar* pivot_target = pivot; + const char* source = input.data; + const char* source_limit = input.data + input.size; + bool reset = true; + size_t available = input.size; + while (true) { + const size_t written = output.size(); + constexpr size_t MAX_OUTPUT_SIZE = std::numeric_limits::max(); + if (UNLIKELY(written == MAX_OUTPUT_SIZE)) { + ColumnString::check_chars_length(written + 1, 0); + } + output.reserve(written + std::min(available, MAX_OUTPUT_SIZE - written)); + // Reuse the spare capacity of the block's result/scratch buffer. In particular, + // expanding rows should not need an overflow/retry on every conversion. Never + // reserve beyond ColumnString's UInt32 offset limit before it can report overflow. + const size_t target_size = + std::min({output.capacity() - written, MAX_OUTPUT_SIZE - written, + static_cast(std::numeric_limits::max())}); + output.resize_assume_reserved(written + target_size); + char* target = reinterpret_cast(output.data()) + written; + const char* target_limit = reinterpret_cast(output.data()) + output.size(); + UErrorCode error = U_ZERO_ERROR; + ucnv_convertEx(_target.get(), _source.get(), &target, target_limit, &source, + source_limit, pivot, &pivot_source, &pivot_target, pivot + 1024, reset, + true, &error); + output.resize(target - reinterpret_cast(output.data())); + // Validate bytes actually produced, not the allocation bound: conversion may + // shrink the input, and spare capacity is not part of the result column. + ColumnString::check_chars_length(output.size(), 0); + if (error == U_BUFFER_OVERFLOW_ERROR) { + // Resume this row without resetting either converter or discarding pending + // pivot/output bytes. A new row resets the converters on its first call. + reset = false; + available *= 2; + continue; + } + if (U_FAILURE(error)) { + return conversion_error(character_set_name, error); + } + return Status::OK(); } - return Status::OK(); } private: @@ -177,7 +192,6 @@ class ConverterPair { ConverterPtr _source; ConverterPtr _target; - std::vector _utf16; }; template @@ -224,13 +238,19 @@ class FunctionCharacterEncoding : public IFunction { character_set_nullable ? &character_set_nullable->get_null_map_data() : nullptr; const bool has_nullable = input_null_map != nullptr || character_set_null_map != nullptr; auto result_column = create_result_column(); - result_column->reserve(input_rows_count); + if constexpr (Encode) { + result_column->get_data().reserve(input_rows_count); + } else { + result_column->reserve(input_rows_count); + } ColumnUInt8::MutablePtr result_null_column; if (has_nullable) { result_null_column = ColumnUInt8::create(input_rows_count, 0); } ConverterCache converters; - std::string converted; + // Varbinary owns out-of-line values in an arena and inlines small values. Share one + // tracked scratch buffer across charsets so that its capacity is not retained seven times. + ColumnString::Chars scratch; CharacterSet constant_character_set = CharacterSet::UTF_8; if (character_set_is_const && input_rows_count != 0 && !(character_set_null_map && (*character_set_null_map)[0])) { @@ -257,8 +277,17 @@ class FunctionCharacterEncoding : public IFunction { } const StringRef input = input_nested->get_data_at(input_index); - RETURN_IF_ERROR(convert_input(input, character_set, converters, converted)); - result_column->insert_data(converted.data(), converted.size()); + if constexpr (Encode) { + scratch.clear(); + RETURN_IF_ERROR(convert_input(input, character_set, converters, scratch)); + result_column->insert_data(reinterpret_cast(scratch.data()), + scratch.size()); + } else { + // Write straight into the result column, including when the buffer grows. + auto& chars = result_column->get_chars(); + RETURN_IF_ERROR(convert_input(input, character_set, converters, chars)); + result_column->get_offsets().push_back(chars.size()); + } } if (has_nullable) { @@ -321,7 +350,7 @@ class FunctionCharacterEncoding : public IFunction { } static Status convert_input(StringRef input, CharacterSet character_set, - ConverterCache& converters, std::string& converted) { + ConverterCache& converters, ColumnString::Chars& converted) { const ConversionSpec spec = get_conversion_spec(input, character_set); if (converters[spec.converter_index] == nullptr) { converters[spec.converter_index] = std::make_unique(); @@ -334,15 +363,15 @@ class FunctionCharacterEncoding : public IFunction { } } - RETURN_IF_ERROR(converters[spec.converter_index]->convert( - spec.input, SUPPORTED_CHARACTER_SETS[static_cast(character_set)], - converted)); if constexpr (Encode) { if (character_set == CharacterSet::UTF_16 && input.size != 0) { - converted.insert(0, "\xFE\xFF", 2); + converted.push_back(0xFE); + converted.push_back(0xFF); } } - return Status::OK(); + return converters[spec.converter_index]->convert( + spec.input, SUPPORTED_CHARACTER_SETS[static_cast(character_set)], + converted); } }; diff --git a/be/test/exprs/function/function_character_encoding_test.cpp b/be/test/exprs/function/function_character_encoding_test.cpp index 908665f0517dd0..4931e315a12059 100644 --- a/be/test/exprs/function/function_character_encoding_test.cpp +++ b/be/test/exprs/function/function_character_encoding_test.cpp @@ -113,4 +113,76 @@ TEST(function_character_encoding_test, rejects_invalid_conversions) { } } +TEST(function_character_encoding_test, streaming_boundaries_and_row_reuse) { + // Cross the pivot boundary and force output expansion, including pending surrogate pairs. + for (size_t length : {1, 15, 1023, 1024, 1025, 65535}) { + std::string ascii(length, 'A'); + std::string utf16; + for (size_t i = 0; i < length; ++i) { + utf16.append("\0A", 2); + } + const std::string utf16_bom = std::string("\xFE\xFF", 2) + utf16; + const std::string supplementary = ascii + "😀"; + const std::string supplementary_utf16 = utf16 + std::string("\xD8\x3D\xDE\0", 4); + DataSet encoded = { + {{ascii, std::string("UTF-16BE")}, VARBINARY(utf16)}, + {{supplementary, std::string("UTF-16BE")}, VARBINARY(supplementary_utf16)}, + {{Null(), std::string("UTF-16BE")}, Null()}, + {{std::string(""), std::string("UTF-16BE")}, VARBINARY("")}, + {{ascii, std::string("UTF-16")}, VARBINARY(utf16_bom)}, + {{std::string("A"), std::string("UTF-16BE")}, + VARBINARY(std::string_view("\0A", 2))}, + }; + check_function_all_arg_comb( + "encode", {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}, encoded); + + std::string latin1(length, '\xE9'); + std::string expanded; + for (size_t i = 0; i < length; ++i) { + expanded += "é"; + } + DataSet decoded = { + {{VARBINARY(latin1), std::string("ISO-8859-1")}, expanded}, + {{VARBINARY(supplementary_utf16), std::string("UTF-16BE")}, supplementary}, + {{Null(), std::string("ISO-8859-1")}, Null()}, + {{VARBINARY(""), std::string("ISO-8859-1")}, std::string("")}, + {{VARBINARY(utf16_bom), std::string("UTF-16")}, ascii}, + {{VARBINARY("\xFF\xFE\x2D\x4E"), std::string("UTF-16")}, std::string("中")}, + {{VARBINARY("\xFE\xFF"), std::string("UTF-16")}, std::string("")}, + {{VARBINARY("\xE9"), std::string("ISO-8859-1")}, std::string("é")}, + }; + check_function_all_arg_comb( + "decode", {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARCHAR}, decoded); + } +} + +TEST(function_character_encoding_test, rejects_invalid_input_after_streaming) { + const std::string invalid_utf8 = std::string(4096, 'A') + "\xE4\xB8"; + const std::string unrepresentable = std::string(4096, 'A') + "中"; + for (const auto& input : {invalid_utf8, unrepresentable}) { + DataSet data_set = {{{input, std::string("US-ASCII")}, VARBINARY("")}}; + Status status = check_function( + "encode", {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}, data_set, -1, + -1, true); + ASSERT_TRUE(status.is()) << status; + } + const std::string expanding_invalid_utf8 = std::string(50000, 'A') + "\xE4\xB8"; + DataSet invalid_encode = {{{expanding_invalid_utf8, std::string("UTF-16BE")}, VARBINARY("")}}; + Status encode_status = check_function( + "encode", {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}, invalid_encode, + -1, -1, true); + ASSERT_TRUE(encode_status.is()) << encode_status; + + std::string invalid_utf16; + for (size_t i = 0; i < 25000; ++i) { + invalid_utf16 += "N-"; // The UTF-16BE byte pair for 中. + } + invalid_utf16.append("\xD8\x3D", 2); // An unpaired high surrogate after several pivot fills. + DataSet data_set = {{{VARBINARY(invalid_utf16), std::string("UTF-16BE")}, std::string("")}}; + Status status = check_function( + "decode", {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARCHAR}, data_set, -1, + -1, true); + ASSERT_TRUE(status.is()) << status; +} + } // namespace doris From 632fa2a98c7770f581d2b87611830acafb806baf Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Mon, 21 Sep 2026 19:10:50 +0800 Subject: [PATCH 07/11] [fix](function) Require a constant character set for encode and decode ### What problem does this PR solve? Issue Number: #48203 Related PR: #68131 Problem Summary: encode and decode previously accepted a per-row character set column. Require the second argument to be a constant expression, including folded expressions such as upper('utf-8'). Reject table columns in FE. Do not fold away an invalid character set when the first argument is a null literal, so FE constant nulls and BE column nulls return the same error. Rewrite the regression CASE queries that evaluated every charset against every row. ### Release note encode and decode now require a constant character set expression. An unsupported character set is rejected even when the input is NULL. ### Check List (For Author) - Test: Unit Test / Regression test - ./run-fe-ut.sh --run org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeDecodeTest (4 tests passed) - ./run-be-ut.sh --run --filter='function_character_encoding_test.*' (7 tests passed) - ./run-regression-test.sh --run -d query_p0/sql_functions/binary_functions -s test_encode_decode (1 suite passed) - Behavior changed: Yes. The character set argument must be constant. Invalid character sets error even when the first argument is NULL. - Does this need documentation: Yes. https://github.com/apache/doris-website/pull/4151 --- be/benchmark/benchmark_character_encoding.hpp | 24 ++--- .../function/function_character_encoding.cpp | 16 ++- .../function_character_encoding_test.cpp | 91 +++++++++++++---- .../expressions/functions/scalar/Decode.java | 17 ++++ .../expressions/functions/scalar/Encode.java | 17 ++++ .../functions/scalar/EncodeDecodeTest.java | 28 ++++++ .../binary_functions/test_encode_decode.out | 12 +++ .../test_encode_decode.groovy | 99 +++++++++++++++++-- 8 files changed, 255 insertions(+), 49 deletions(-) diff --git a/be/benchmark/benchmark_character_encoding.hpp b/be/benchmark/benchmark_character_encoding.hpp index 91476ed2cdc29d..f0f27b8a23bd10 100644 --- a/be/benchmark/benchmark_character_encoding.hpp +++ b/be/benchmark/benchmark_character_encoding.hpp @@ -77,7 +77,7 @@ CharacterEncodingData make_character_encoding_data(size_t length) { // Benchmark actual block execution, including result allocation and converter setup, not just // ICU calls. Inputs are materialized columns so constant folding cannot eliminate conversion. -// Args: input size, constant charset (0/1). Large rows use smaller blocks to bound working memory. +// The charset is constant by contract. Large rows use smaller blocks to bound working memory. template void BM_character_encoding(benchmark::State& state) { const size_t length = state.range(0); @@ -92,16 +92,8 @@ void BM_character_encoding(benchmark::State& state) { values->insert_data(input.data(), input.size()); } auto charsets = ColumnString::create(); - const bool constant_charset = state.range(1) != 0; - for (size_t i = 0; i < (constant_charset ? 1 : rows); ++i) { - charsets->insert_data(charset.data(), charset.size()); - } - ColumnPtr charset_column; - if (constant_charset) { - charset_column = ColumnConst::create(std::move(charsets), rows); - } else { - charset_column = std::move(charsets); - } + charsets->insert_data(charset.data(), charset.size()); + ColumnPtr charset_column = ColumnConst::create(std::move(charsets), rows); Block block {{std::move(values), input_type, "input"}, {std::move(charset_column), string_type, "charset"}}; auto function = SimpleFunctionFactory::instance().get_function( @@ -163,19 +155,19 @@ void BM_character_encoding(benchmark::State& state) { BENCHMARK_TEMPLATE(BM_character_encoding, true, 0) ->Name("encode_utf16be_ascii") - ->ArgsProduct({{15, 63, 1023, 65535}, {0, 1}}); + ->ArgsProduct({{15, 63, 1023, 65535}}); BENCHMARK_TEMPLATE(BM_character_encoding, false, 1) ->Name("decode_latin1_nonascii") - ->ArgsProduct({{15, 63, 1023, 65535}, {0, 1}}); + ->ArgsProduct({{15, 63, 1023, 65535}}); BENCHMARK_TEMPLATE(BM_character_encoding, true, 2) ->Name("encode_utf8_mixed") - ->ArgsProduct({{15, 63, 1023, 65535}, {0, 1}}); + ->ArgsProduct({{15, 63, 1023, 65535}}); BENCHMARK_TEMPLATE(BM_character_encoding, false, 2) ->Name("decode_utf8_mixed") - ->ArgsProduct({{15, 63, 1023, 65535}, {0, 1}}); + ->ArgsProduct({{15, 63, 1023, 65535}}); BENCHMARK_TEMPLATE(BM_character_encoding, false, 3) ->Name("decode_utf16be_cjk") - ->ArgsProduct({{16, 64, 1024, 65536}, {0, 1}}); + ->ArgsProduct({{16, 64, 1024, 65536}}); } // namespace } // namespace doris diff --git a/be/src/exprs/function/function_character_encoding.cpp b/be/src/exprs/function/function_character_encoding.cpp index c7897395674433..300b80c87f678a 100644 --- a/be/src/exprs/function/function_character_encoding.cpp +++ b/be/src/exprs/function/function_character_encoding.cpp @@ -215,6 +215,8 @@ class FunctionCharacterEncoding : public IFunction { return have_nullable(arguments) ? make_nullable(result_type) : result_type; } + ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; } + bool use_default_implementation_for_nulls() const override { return false; } Status execute_impl(FunctionContext* /*context*/, Block& block, const ColumnNumbers& arguments, @@ -223,6 +225,7 @@ class FunctionCharacterEncoding : public IFunction { unpack_if_const(block.get_by_position(arguments[0]).column); auto [character_set_column, character_set_is_const] = unpack_if_const(block.get_by_position(arguments[1]).column); + DCHECK(character_set_is_const); const auto* input_nullable = check_and_get_column(input_column.get()); const auto* character_set_nullable = check_and_get_column(character_set_column.get()); @@ -260,32 +263,25 @@ class FunctionCharacterEncoding : public IFunction { for (size_t row = 0; row < input_rows_count; ++row) { const size_t input_index = index_check_const(row, input_is_const); - const size_t character_set_index = index_check_const(row, character_set_is_const); const bool input_is_null = input_null_map && (*input_null_map)[input_index]; const bool character_set_is_null = - character_set_null_map && (*character_set_null_map)[character_set_index]; + character_set_null_map && (*character_set_null_map)[0]; if (input_is_null || character_set_is_null) { result_column->insert_default(); result_null_column->get_data()[row] = 1; continue; } - CharacterSet character_set = constant_character_set; - if (!character_set_is_const) { - RETURN_IF_ERROR(parse_character_set(character_sets.get_data_at(character_set_index), - character_set)); - } - const StringRef input = input_nested->get_data_at(input_index); if constexpr (Encode) { scratch.clear(); - RETURN_IF_ERROR(convert_input(input, character_set, converters, scratch)); + RETURN_IF_ERROR(convert_input(input, constant_character_set, converters, scratch)); result_column->insert_data(reinterpret_cast(scratch.data()), scratch.size()); } else { // Write straight into the result column, including when the buffer grows. auto& chars = result_column->get_chars(); - RETURN_IF_ERROR(convert_input(input, character_set, converters, chars)); + RETURN_IF_ERROR(convert_input(input, constant_character_set, converters, chars)); result_column->get_offsets().push_back(chars.size()); } } diff --git a/be/test/exprs/function/function_character_encoding_test.cpp b/be/test/exprs/function/function_character_encoding_test.cpp index 4931e315a12059..be88acac23d39d 100644 --- a/be/test/exprs/function/function_character_encoding_test.cpp +++ b/be/test/exprs/function/function_character_encoding_test.cpp @@ -26,9 +26,26 @@ namespace doris { using namespace ut_type; +template +void check_character_encoding(const std::string& function_name, PrimitiveType input_type, + const DataSet& data_set) { + for (bool input_is_const : {false, true}) { + for (const auto& line : data_set) { + InputTypeSet input_types; + if (input_is_const) { + input_types.emplace_back(Consted {input_type}); + } else { + input_types.emplace_back(input_type); + } + input_types.emplace_back(Consted {PrimitiveType::TYPE_VARCHAR}); + ASSERT_TRUE( + (check_function(function_name, input_types, {line}).ok())); + } + } +} + TEST(function_character_encoding_test, encode_supported_charsets) { // The UTF-16 byte pairs 0x4E2D and 0x2D4E are "N-" and "-N" as raw bytes. - InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}; DataSet data_set = { {{std::string("A"), std::string("US-ASCII")}, VARBINARY("A")}, {{std::string("é"), std::string("ISO-8859-1")}, VARBINARY("\xE9")}, @@ -45,12 +62,11 @@ TEST(function_character_encoding_test, encode_supported_charsets) { {{std::string("text"), Null()}, Null()}, }; - check_function_all_arg_comb("encode", input_types, data_set); + check_character_encoding("encode", PrimitiveType::TYPE_VARCHAR, data_set); } TEST(function_character_encoding_test, decode_supported_charsets) { // The UTF-16 byte pairs 0x4E2D and 0x2D4E are "N-" and "-N" as raw bytes. - InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARCHAR}; DataSet data_set = { {{VARBINARY("A"), std::string("US-ASCII")}, std::string("A")}, {{VARBINARY("\xE9"), std::string("ISO-8859-1")}, std::string("é")}, @@ -70,12 +86,13 @@ TEST(function_character_encoding_test, decode_supported_charsets) { {{VARBINARY("text"), Null()}, Null()}, }; - check_function_all_arg_comb("decode", input_types, data_set); + check_character_encoding("decode", PrimitiveType::TYPE_VARBINARY, data_set); } TEST(function_character_encoding_test, rejects_invalid_conversions) { { - InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}; + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, + Consted {PrimitiveType::TYPE_VARCHAR}}; DataSet data_set = { {{std::string("text"), std::string("GBK")}, VARBINARY("")}, }; @@ -87,7 +104,8 @@ TEST(function_character_encoding_test, rejects_invalid_conversions) { } { - InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}; + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, + Consted {PrimitiveType::TYPE_VARCHAR}}; DataSet data_set = { {{std::string("中"), std::string("US-ASCII")}, VARBINARY("")}, }; @@ -100,7 +118,8 @@ TEST(function_character_encoding_test, rejects_invalid_conversions) { } { - InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARCHAR}; + InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, + Consted {PrimitiveType::TYPE_VARCHAR}}; DataSet data_set = { {{VARBINARY("\xE4\xB8"), std::string("UTF-8")}, std::string("")}, }; @@ -113,6 +132,46 @@ TEST(function_character_encoding_test, rejects_invalid_conversions) { } } +TEST(function_character_encoding_test, invalid_constant_character_set_precedes_null_input) { + DataSet encode_data = { + {{Null(), std::string("GBK")}, Null()}, + }; + Status encode_status = check_function( + "encode", {PrimitiveType::TYPE_VARCHAR, Consted {PrimitiveType::TYPE_VARCHAR}}, + encode_data, -1, -1, true); + ASSERT_TRUE(encode_status.is()) << encode_status; + EXPECT_NE(encode_status.to_string().find("Unsupported character set"), std::string::npos); + + DataSet decode_data = { + {{Null(), std::string("GBK")}, Null()}, + }; + Status decode_status = check_function( + "decode", {PrimitiveType::TYPE_VARBINARY, Consted {PrimitiveType::TYPE_VARCHAR}}, + decode_data, -1, -1, true); + ASSERT_TRUE(decode_status.is()) << decode_status; + EXPECT_NE(decode_status.to_string().find("Unsupported character set"), std::string::npos); +} + +TEST(function_character_encoding_test, requires_constant_character_set) { + DataSet encode_data = { + {{std::string("A"), std::string("UTF-8")}, VARBINARY("A")}, + }; + Status encode_status = check_function( + "encode", {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}, encode_data, -1, + -1, true); + ASSERT_TRUE(encode_status.is()) << encode_status; + EXPECT_NE(encode_status.to_string().find("must be constant"), std::string::npos); + + DataSet decode_data = { + {{VARBINARY("A"), std::string("UTF-8")}, std::string("A")}, + }; + Status decode_status = check_function( + "decode", {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARCHAR}, decode_data, -1, + -1, true); + ASSERT_TRUE(decode_status.is()) << decode_status; + EXPECT_NE(decode_status.to_string().find("must be constant"), std::string::npos); +} + TEST(function_character_encoding_test, streaming_boundaries_and_row_reuse) { // Cross the pivot boundary and force output expansion, including pending surrogate pairs. for (size_t length : {1, 15, 1023, 1024, 1025, 65535}) { @@ -133,8 +192,7 @@ TEST(function_character_encoding_test, streaming_boundaries_and_row_reuse) { {{std::string("A"), std::string("UTF-16BE")}, VARBINARY(std::string_view("\0A", 2))}, }; - check_function_all_arg_comb( - "encode", {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}, encoded); + check_character_encoding("encode", PrimitiveType::TYPE_VARCHAR, encoded); std::string latin1(length, '\xE9'); std::string expanded; @@ -151,8 +209,7 @@ TEST(function_character_encoding_test, streaming_boundaries_and_row_reuse) { {{VARBINARY("\xFE\xFF"), std::string("UTF-16")}, std::string("")}, {{VARBINARY("\xE9"), std::string("ISO-8859-1")}, std::string("é")}, }; - check_function_all_arg_comb( - "decode", {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARCHAR}, decoded); + check_character_encoding("decode", PrimitiveType::TYPE_VARBINARY, decoded); } } @@ -162,15 +219,15 @@ TEST(function_character_encoding_test, rejects_invalid_input_after_streaming) { for (const auto& input : {invalid_utf8, unrepresentable}) { DataSet data_set = {{{input, std::string("US-ASCII")}, VARBINARY("")}}; Status status = check_function( - "encode", {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}, data_set, -1, - -1, true); + "encode", {PrimitiveType::TYPE_VARCHAR, Consted {PrimitiveType::TYPE_VARCHAR}}, + data_set, -1, -1, true); ASSERT_TRUE(status.is()) << status; } const std::string expanding_invalid_utf8 = std::string(50000, 'A') + "\xE4\xB8"; DataSet invalid_encode = {{{expanding_invalid_utf8, std::string("UTF-16BE")}, VARBINARY("")}}; Status encode_status = check_function( - "encode", {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}, invalid_encode, - -1, -1, true); + "encode", {PrimitiveType::TYPE_VARCHAR, Consted {PrimitiveType::TYPE_VARCHAR}}, + invalid_encode, -1, -1, true); ASSERT_TRUE(encode_status.is()) << encode_status; std::string invalid_utf16; @@ -180,8 +237,8 @@ TEST(function_character_encoding_test, rejects_invalid_input_after_streaming) { invalid_utf16.append("\xD8\x3D", 2); // An unpaired high surrogate after several pivot fills. DataSet data_set = {{{VARBINARY(invalid_utf16), std::string("UTF-16BE")}, std::string("")}}; Status status = check_function( - "decode", {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARCHAR}, data_set, -1, - -1, true); + "decode", {PrimitiveType::TYPE_VARBINARY, Consted {PrimitiveType::TYPE_VARCHAR}}, + data_set, -1, -1, true); ASSERT_TRUE(status.is()) << status; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java index 86153192f8b988..96ca7e95350ee7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; @@ -53,6 +54,22 @@ private Decode(ScalarFunctionParams functionParams) { super(functionParams); } + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!getArgument(1).isConstant()) { + throw new AnalysisException("the second argument of function " + + getName() + " must be constant: " + toSql()); + } + } + + // Invalid character sets must still be rejected when the first argument is a + // null literal. FoldConstantRuleOnFE otherwise rewrites PropagateNullable + // calls with any null child to NULL and skips backend evaluation. + @Override + public boolean foldable() { + return false; + } + /** * withChildren. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java index 3758a195e7d408..917317e1a49b4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; @@ -53,6 +54,22 @@ private Encode(ScalarFunctionParams functionParams) { super(functionParams); } + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!getArgument(1).isConstant()) { + throw new AnalysisException("the second argument of function " + + getName() + " must be constant: " + toSql()); + } + } + + // Invalid character sets must still be rejected when the first argument is a + // null literal. FoldConstantRuleOnFE otherwise rewrites PropagateNullable + // calls with any null child to NULL and skips backend evaluation. + @Override + public boolean foldable() { + return false; + } + /** * withChildren. */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java index 306b3fac3c88ec..c9564457b3705f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java @@ -18,7 +18,9 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -100,4 +102,30 @@ public Expression visit(Expression expression, Void context) { Assertions.assertSame(encode, encode.accept(visitor, null)); Assertions.assertSame(decode, decode.accept(visitor, null)); } + + @Test + public void testCharacterSetMustBeConstant() { + SlotReference characterSetColumn = new SlotReference("charset", StringType.INSTANCE); + Encode encode = new Encode(new StringLiteral("hello"), characterSetColumn); + Decode decode = new Decode(new VarBinaryLiteral(new byte[] {0x68, 0x69}), + characterSetColumn); + + AnalysisException encodeException = Assertions.assertThrows( + AnalysisException.class, encode::checkLegalityBeforeTypeCoercion); + Assertions.assertTrue(encodeException.getMessage().contains( + "second argument of function encode must be constant")); + AnalysisException decodeException = Assertions.assertThrows( + AnalysisException.class, decode::checkLegalityBeforeTypeCoercion); + Assertions.assertTrue(decodeException.getMessage().contains( + "second argument of function decode must be constant")); + + Assertions.assertDoesNotThrow(new Encode(new StringLiteral("hello"), + new Upper(new StringLiteral("utf-8")))::checkLegalityBeforeTypeCoercion); + Assertions.assertDoesNotThrow(new Decode(new VarBinaryLiteral(new byte[] {0x68, 0x69}), + new StringLiteral("UTF-8"))::checkLegalityBeforeTypeCoercion); + Assertions.assertFalse(new Encode(new StringLiteral("hello"), + new StringLiteral("UTF-8")).foldable()); + Assertions.assertFalse(new Decode(new VarBinaryLiteral(new byte[] {0x68, 0x69}), + new StringLiteral("UTF-8")).foldable()); + } } diff --git a/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out b/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out index 9f4f10dd67b474..158be3a23bd405 100644 --- a/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out +++ b/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out @@ -27,3 +27,15 @@ 8 9 中 +-- !encode_constant_expr -- +E4B8AD + +-- !decode_constant_expr -- +中 + +-- !encode_null_valid_charset -- +\N + +-- !decode_null_valid_charset -- +\N + diff --git a/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy index 5449fd0835edb1..86340134a860f7 100644 --- a/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy +++ b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy @@ -44,18 +44,72 @@ suite("test_encode_decode") { (12, 'text', unhex('74657874'), null) """ + // Vectorized CASE evaluates every branch for every row, so mixed-charset + // CASE encode/decode queries fail on strict conversion. Filter each + // constant charset down to compatible rows instead. order_qt_encode_supported_charsets """ - select id, hex(encode(plain_text, charset)) - from test_encode_decode + select * from ( + select id, hex(encode(plain_text, 'US-ASCII')) as encoded from test_encode_decode where id = 1 + union all + select id, hex(encode(plain_text, 'ISO-8859-1')) from test_encode_decode where id = 2 + union all + select id, hex(encode(plain_text, 'UTF-8')) from test_encode_decode where id = 3 + union all + select id, hex(encode(plain_text, 'UTF-16BE')) from test_encode_decode where id = 4 + union all + select id, hex(encode(plain_text, 'UTF-16LE')) from test_encode_decode where id = 5 + union all + select id, hex(encode(plain_text, 'UTF-16')) from test_encode_decode where id = 6 + union all + select id, hex(encode(plain_text, 'UTF-16BE')) from test_encode_decode where id = 7 + union all + select id, hex(encode(plain_text, 'UTF-16')) from test_encode_decode where id = 8 + union all + select id, hex(encode(plain_text, 'utf-16')) from test_encode_decode where id = 9 + union all + select id, hex(encode(plain_text, 'UTF-16')) from test_encode_decode where id = 10 + union all + select id, hex(encode(plain_text, 'UTF-8')) from test_encode_decode where id = 11 + union all + select id, hex(encode(plain_text, cast(null as string))) from test_encode_decode where id = 12 + ) t order by id """ order_qt_decode_supported_charsets """ - select id, decode(cast(binary_value as varbinary), charset) - from test_encode_decode + select * from ( + select id, decode(cast(binary_value as varbinary), 'US-ASCII') as decoded from test_encode_decode where id = 1 + union all + select id, decode(cast(binary_value as varbinary), 'ISO-8859-1') from test_encode_decode where id = 2 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-8') from test_encode_decode where id = 3 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16BE') from test_encode_decode where id = 4 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16LE') from test_encode_decode where id = 5 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16') from test_encode_decode where id = 6 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16BE') from test_encode_decode where id = 7 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16') from test_encode_decode where id = 8 + union all + select id, decode(cast(binary_value as varbinary), 'utf-16') from test_encode_decode where id = 9 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16') from test_encode_decode where id = 10 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-8') from test_encode_decode where id = 11 + union all + select id, decode(cast(binary_value as varbinary), cast(null as string)) from test_encode_decode where id = 12 + ) t order by id """ + qt_encode_constant_expr "select hex(encode('中', upper('utf-8')))" + qt_decode_constant_expr "select decode(X'E4B8AD', upper('utf-8'))" + qt_encode_null_valid_charset "select encode(null, 'UTF-8')" + qt_decode_null_valid_charset "select decode(null, 'UTF-8')" + test { sql "select encode('text', 'GBK')" exception "Unsupported character set" @@ -76,10 +130,43 @@ suite("test_encode_decode") { exception "Unsupported character set" } - sql "insert into test_encode_decode values (13, 'A', unhex('41'), 'Uſ-ASCII')" + test { + sql "select encode(plain_text, charset) from test_encode_decode where id = 1" + exception "second argument of function encode must be constant" + } + + test { + sql "select decode(cast(binary_value as varbinary), charset) from test_encode_decode where id = 1" + exception "second argument of function decode must be constant" + } + + test { + sql "select encode(null, 'GBK')" + exception "Unsupported character set" + } + + test { + sql "select encode(cast(null as string), 'GBK')" + exception "Unsupported character set" + } + + test { + sql "select encode(plain_text, 'GBK') from test_encode_decode where id = 11" + exception "Unsupported character set" + } + + test { + sql "select decode(null, 'GBK')" + exception "Unsupported character set" + } + + test { + sql "select decode(cast(null as varbinary), 'GBK')" + exception "Unsupported character set" + } test { - sql "select encode(plain_text, charset) from test_encode_decode where id = 13" + sql "select decode(cast(binary_value as varbinary), 'GBK') from test_encode_decode where id = 11" exception "Unsupported character set" } } From c4a27db88818c498f07cc26c210411049e095a8c Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Wed, 23 Sep 2026 10:56:16 +0800 Subject: [PATCH 08/11] [improvement](be) Convert encode and decode with simdutf ### What problem does this PR solve? Issue Number: #48203 Related PR: #68131 Problem Summary: Character conversion still goes through ICU even though the supported sets are ASCII, Latin1, UTF-8, and UTF-16. Use simdutf for those transcodes. Keep strict errors, Java/Hive UTF-16 BOM behavior, and the constant character-set contract. ### Release note None ### Check List (For Author) - Test: Unit Test - Existing function_character_encoding_test covers the charset, BOM, null, and invalid-input cases. Not rerun in this commit because the local tree does not have the simdutf third-party build. - Behavior changed: No. Same supported character sets and error conditions. The error suffix now uses simdutf's error name instead of an ICU status name. - Does this need documentation: No --- .../function/function_character_encoding.cpp | 348 ++++++++++-------- 1 file changed, 192 insertions(+), 156 deletions(-) diff --git a/be/src/exprs/function/function_character_encoding.cpp b/be/src/exprs/function/function_character_encoding.cpp index 300b80c87f678a..a40ce1b54d830f 100644 --- a/be/src/exprs/function/function_character_encoding.cpp +++ b/be/src/exprs/function/function_character_encoding.cpp @@ -15,18 +15,18 @@ // specific language governing permissions and limitations // under the License. -#include -#include +#include -#include #include #include #include +#include #include #include #include #include #include +#include #include "common/status.h" #include "core/assert_cast.h" @@ -88,111 +88,43 @@ Status parse_character_set(StringRef value, CharacterSet& character_set) { std::string(value.data, value.size)); } -using ConverterPtr = std::unique_ptr; - -class ConverterPair { -public: - ConverterPair() : _source(nullptr, ucnv_close), _target(nullptr, ucnv_close) {} - - Status open(std::string_view source_name, std::string_view target_name) { - UErrorCode error = U_ZERO_ERROR; - _source.reset(ucnv_open(source_name.data(), &error)); - if (U_FAILURE(error)) { - return Status::InternalError("Failed to open ICU converter '{}': {}", source_name, - u_errorName(error)); - } - - error = U_ZERO_ERROR; - ucnv_setToUCallBack(_source.get(), UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr, nullptr, - &error); - if (U_FAILURE(error)) { - return Status::InternalError("Failed to configure ICU converter '{}': {}", source_name, - u_errorName(error)); - } - - error = U_ZERO_ERROR; - _target.reset(ucnv_open(target_name.data(), &error)); - if (U_FAILURE(error)) { - return Status::InternalError("Failed to open ICU converter '{}': {}", target_name, - u_errorName(error)); - } +Status conversion_error(std::string_view character_set_name, simdutf::error_code error) { + return Status::InvalidArgument("Character conversion using '{}' failed: {}", character_set_name, + simdutf::error_to_string(error)); +} - error = U_ZERO_ERROR; - ucnv_setFromUCallBack(_target.get(), UCNV_FROM_U_CALLBACK_STOP, nullptr, nullptr, nullptr, - &error); - if (U_FAILURE(error)) { - return Status::InternalError("Failed to configure ICU converter '{}': {}", target_name, - u_errorName(error)); - } - return Status::OK(); - } +Status reject_too_large(std::string_view character_set_name) { + return Status::InvalidArgument("Input is too large for character conversion using '{}'", + character_set_name); +} - Status convert(StringRef input, std::string_view character_set_name, - ColumnString::Chars& output) { - if (input.size == 0) { - return Status::OK(); - } - if (input.size > static_cast(std::numeric_limits::max())) { - return Status::InvalidArgument("Input is too large for character conversion using '{}'", - character_set_name); - } +// Grow the byte buffer, then return the address of the newly reserved range. +char* reserve_output(ColumnString::Chars& output, size_t extra) { + const size_t start = output.size(); + ColumnString::check_chars_length(start + extra, 0); + output.resize(start + extra); + return reinterpret_cast(output.data() + start); +} - // Keep only a bounded Unicode pivot, rather than materializing the entire UTF-16 - // string and scanning both the source and the pivot twice to preflight sizes. - UChar pivot[1024]; - UChar* pivot_source = pivot; - UChar* pivot_target = pivot; - const char* source = input.data; - const char* source_limit = input.data + input.size; - bool reset = true; - size_t available = input.size; - while (true) { - const size_t written = output.size(); - constexpr size_t MAX_OUTPUT_SIZE = std::numeric_limits::max(); - if (UNLIKELY(written == MAX_OUTPUT_SIZE)) { - ColumnString::check_chars_length(written + 1, 0); - } - output.reserve(written + std::min(available, MAX_OUTPUT_SIZE - written)); - // Reuse the spare capacity of the block's result/scratch buffer. In particular, - // expanding rows should not need an overflow/retry on every conversion. Never - // reserve beyond ColumnString's UInt32 offset limit before it can report overflow. - const size_t target_size = - std::min({output.capacity() - written, MAX_OUTPUT_SIZE - written, - static_cast(std::numeric_limits::max())}); - output.resize_assume_reserved(written + target_size); - char* target = reinterpret_cast(output.data()) + written; - const char* target_limit = reinterpret_cast(output.data()) + output.size(); - UErrorCode error = U_ZERO_ERROR; - ucnv_convertEx(_target.get(), _source.get(), &target, target_limit, &source, - source_limit, pivot, &pivot_source, &pivot_target, pivot + 1024, reset, - true, &error); - output.resize(target - reinterpret_cast(output.data())); - // Validate bytes actually produced, not the allocation bound: conversion may - // shrink the input, and spare capacity is not part of the result column. - ColumnString::check_chars_length(output.size(), 0); - if (error == U_BUFFER_OVERFLOW_ERROR) { - // Resume this row without resetting either converter or discarding pending - // pivot/output bytes. A new row resets the converters on its first call. - reset = false; - available *= 2; - continue; - } - if (U_FAILURE(error)) { - return conversion_error(character_set_name, error); - } - return Status::OK(); - } +Status copy_validated(StringRef input, std::string_view character_set_name, + simdutf::result validation, ColumnString::Chars& output) { + if (validation.error != simdutf::SUCCESS) { + return conversion_error(character_set_name, validation.error); } + memcpy(reserve_output(output, input.size), input.data, input.size); + return Status::OK(); +} -private: - static Status conversion_error(std::string_view character_set_name, UErrorCode error) { - return Status::InvalidArgument("Character conversion using '{}' failed: {}", - character_set_name, u_errorName(error)); +// Doris string bytes are not guaranteed to be char16_t-aligned. +const char16_t* utf16_units(StringRef input, std::vector& aligned) { + if (reinterpret_cast(input.data) % alignof(char16_t) == 0) { + return reinterpret_cast(input.data); } - - ConverterPtr _source; - ConverterPtr _target; -}; + const size_t units = input.size / 2; + aligned.resize(units); + memcpy(aligned.data(), input.data, input.size); + return aligned.data(); +} template class FunctionCharacterEncoding : public IFunction { @@ -250,10 +182,10 @@ class FunctionCharacterEncoding : public IFunction { if (has_nullable) { result_null_column = ColumnUInt8::create(input_rows_count, 0); } - ConverterCache converters; // Varbinary owns out-of-line values in an arena and inlines small values. Share one - // tracked scratch buffer across charsets so that its capacity is not retained seven times. + // tracked scratch buffer across rows so that its capacity is not retained per row. ColumnString::Chars scratch; + std::vector utf16_scratch; CharacterSet constant_character_set = CharacterSet::UTF_8; if (character_set_is_const && input_rows_count != 0 && !(character_set_null_map && (*character_set_null_map)[0])) { @@ -275,13 +207,14 @@ class FunctionCharacterEncoding : public IFunction { const StringRef input = input_nested->get_data_at(input_index); if constexpr (Encode) { scratch.clear(); - RETURN_IF_ERROR(convert_input(input, constant_character_set, converters, scratch)); + RETURN_IF_ERROR( + convert_input(input, constant_character_set, utf16_scratch, scratch)); result_column->insert_data(reinterpret_cast(scratch.data()), scratch.size()); } else { // Write straight into the result column, including when the buffer grows. auto& chars = result_column->get_chars(); - RETURN_IF_ERROR(convert_input(input, constant_character_set, converters, chars)); + RETURN_IF_ERROR(convert_input(input, constant_character_set, utf16_scratch, chars)); result_column->get_offsets().push_back(chars.size()); } } @@ -298,76 +231,179 @@ class FunctionCharacterEncoding : public IFunction { private: using ResultColumn = std::conditional_t; - static constexpr auto UTF16_LITTLE_ENDIAN_CONVERTER = static_cast(CharacterSet::SIZE); - using ConverterCache = - std::array, UTF16_LITTLE_ENDIAN_CONVERTER + 1>; - - struct ConversionSpec { - StringRef input; - size_t converter_index; - std::string_view converter_character_set; - }; static typename ResultColumn::MutablePtr create_result_column() { return ResultColumn::create(); } - static ConversionSpec get_conversion_spec(StringRef input, CharacterSet character_set) { - const auto character_set_index = static_cast(character_set); - ConversionSpec spec {input, character_set_index, - SUPPORTED_CHARACTER_SETS[character_set_index]}; - if (character_set != CharacterSet::UTF_16) { - return spec; - } + static std::string_view charset_name(CharacterSet character_set) { + return SUPPORTED_CHARACTER_SETS[static_cast(character_set)]; + } - // Java's UTF-16 encoder always emits a big-endian BOM. ICU's generic UTF-16 converter - // follows the host byte order, so encode with UTF-16BE and add the BOM explicitly. - spec.converter_character_set = - SUPPORTED_CHARACTER_SETS[static_cast(CharacterSet::UTF_16BE)]; + static Status convert_input(StringRef input, CharacterSet character_set, + std::vector& utf16_scratch, + ColumnString::Chars& converted) { + if (input.size == 0) { + return Status::OK(); + } + const std::string_view name = charset_name(character_set); + if (input.size > static_cast(std::numeric_limits::max())) { + return reject_too_large(name); + } if constexpr (Encode) { - return spec; + return encode_input(input, character_set, name, utf16_scratch, converted); + } else { + return decode_input(input, character_set, name, utf16_scratch, converted); + } + } + + static Status encode_input(StringRef input, CharacterSet character_set, std::string_view name, + std::vector& utf16_scratch, ColumnString::Chars& output) { + switch (character_set) { + case CharacterSet::US_ASCII: + return copy_validated(input, name, + simdutf::validate_ascii_with_errors(input.data, input.size), + output); + case CharacterSet::UTF_8: + return copy_validated(input, name, + simdutf::validate_utf8_with_errors(input.data, input.size), + output); + case CharacterSet::ISO_8859_1: + return encode_latin1(input, name, output); + case CharacterSet::UTF_16BE: + return encode_utf16(input, name, false, false, utf16_scratch, output); + case CharacterSet::UTF_16LE: + return encode_utf16(input, name, true, false, utf16_scratch, output); + case CharacterSet::UTF_16: + // Java's UTF-16 encoder always emits a big-endian BOM. + return encode_utf16(input, name, false, true, utf16_scratch, output); + default: + return Status::InvalidArgument("Unsupported character set '{}'", name); + } + } + + static Status decode_input(StringRef input, CharacterSet character_set, std::string_view name, + std::vector& utf16_scratch, ColumnString::Chars& output) { + switch (character_set) { + case CharacterSet::US_ASCII: + return copy_validated(input, name, + simdutf::validate_ascii_with_errors(input.data, input.size), + output); + case CharacterSet::UTF_8: + return copy_validated(input, name, + simdutf::validate_utf8_with_errors(input.data, input.size), + output); + case CharacterSet::ISO_8859_1: + return decode_latin1(input, name, output); + case CharacterSet::UTF_16BE: + return decode_utf16(input, name, false, utf16_scratch, output); + case CharacterSet::UTF_16LE: + return decode_utf16(input, name, true, utf16_scratch, output); + case CharacterSet::UTF_16: + return decode_utf16_with_bom(input, name, utf16_scratch, output); + default: + return Status::InvalidArgument("Unsupported character set '{}'", name); } + } + + static Status encode_latin1(StringRef input, std::string_view name, + ColumnString::Chars& output) { + const size_t start = output.size(); + char* dest = reserve_output(output, input.size); + const size_t written = simdutf::convert_utf8_to_latin1(input.data, input.size, dest); + if (written == 0) { + const simdutf::result detail = + simdutf::convert_utf8_to_latin1_with_errors(input.data, input.size, dest); + output.resize(start); + const simdutf::error_code error = + detail.error == simdutf::SUCCESS ? simdutf::OTHER : detail.error; + return conversion_error(name, error); + } + output.resize(start + written); + return Status::OK(); + } + + static Status decode_latin1(StringRef input, std::string_view name, + ColumnString::Chars& output) { + const size_t need = simdutf::utf8_length_from_latin1(input.data, input.size); + char* dest = reserve_output(output, need); + const size_t written = simdutf::convert_latin1_to_utf8(input.data, input.size, dest); + if (written != need) { + output.resize(output.size() - need); + return conversion_error(name, simdutf::OTHER); + } + return Status::OK(); + } - // Java's UTF-16 decoder honors either BOM and defaults to big endian without a BOM. + static Status encode_utf16(StringRef input, std::string_view name, bool little_endian, + bool write_bom, std::vector& utf16_scratch, + ColumnString::Chars& output) { + utf16_scratch.resize(input.size); + const simdutf::result result = + little_endian ? simdutf::convert_utf8_to_utf16le_with_errors(input.data, input.size, + utf16_scratch.data()) + : simdutf::convert_utf8_to_utf16be_with_errors(input.data, input.size, + utf16_scratch.data()); + if (result.error != simdutf::SUCCESS) { + return conversion_error(name, result.error); + } + const size_t payload_bytes = result.count * sizeof(char16_t); + const size_t start = output.size(); + char* dest = reserve_output(output, payload_bytes + (write_bom ? 2 : 0)); + if (write_bom) { + auto* bytes = reinterpret_cast(dest); + bytes[0] = 0xFE; + bytes[1] = 0xFF; + dest += 2; + } + memcpy(dest, utf16_scratch.data(), payload_bytes); + return Status::OK(); + } + + // Java's UTF-16 decoder honors either BOM and defaults to big endian without one. + static Status decode_utf16_with_bom(StringRef input, std::string_view name, + std::vector& utf16_scratch, + ColumnString::Chars& output) { if (input.size < 2) { - return spec; + return conversion_error(name, simdutf::TOO_SHORT); } const auto first = static_cast(input.data[0]); const auto second = static_cast(input.data[1]); + bool little_endian = false; if (first == 0xFE && second == 0xFF) { - spec.input = input.substring(2); + input = input.substring(2); } else if (first == 0xFF && second == 0xFE) { - spec.input = input.substring(2); - spec.converter_index = UTF16_LITTLE_ENDIAN_CONVERTER; - spec.converter_character_set = - SUPPORTED_CHARACTER_SETS[static_cast(CharacterSet::UTF_16LE)]; + input = input.substring(2); + little_endian = true; } - return spec; + if (input.size == 0) { + return Status::OK(); + } + return decode_utf16(input, name, little_endian, utf16_scratch, output); } - static Status convert_input(StringRef input, CharacterSet character_set, - ConverterCache& converters, ColumnString::Chars& converted) { - const ConversionSpec spec = get_conversion_spec(input, character_set); - if (converters[spec.converter_index] == nullptr) { - converters[spec.converter_index] = std::make_unique(); - if constexpr (Encode) { - RETURN_IF_ERROR(converters[spec.converter_index]->open( - "UTF-8", spec.converter_character_set)); - } else { - RETURN_IF_ERROR(converters[spec.converter_index]->open(spec.converter_character_set, - "UTF-8")); - } + static Status decode_utf16(StringRef input, std::string_view name, bool little_endian, + std::vector& utf16_scratch, ColumnString::Chars& output) { + if (input.size % 2 != 0) { + return conversion_error(name, simdutf::TOO_SHORT); } - - if constexpr (Encode) { - if (character_set == CharacterSet::UTF_16 && input.size != 0) { - converted.push_back(0xFE); - converted.push_back(0xFF); - } + const size_t units = input.size / 2; + if (units > (std::numeric_limits::max() / 3)) { + return reject_too_large(name); } - return converters[spec.converter_index]->convert( - spec.input, SUPPORTED_CHARACTER_SETS[static_cast(character_set)], - converted); + const char16_t* units_ptr = utf16_units(input, utf16_scratch); + const size_t start = output.size(); + char* dest = reserve_output(output, units * 3); + const simdutf::result result = + little_endian + ? simdutf::convert_utf16le_to_utf8_with_errors(units_ptr, units, dest) + : simdutf::convert_utf16be_to_utf8_with_errors(units_ptr, units, dest); + if (result.error != simdutf::SUCCESS) { + output.resize(start); + return conversion_error(name, result.error); + } + output.resize(start + result.count); + return Status::OK(); } }; From 951de3ca8d12047ff718ce25c5e1273a2a95bb5b Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Wed, 23 Sep 2026 11:12:07 +0800 Subject: [PATCH 09/11] [fix](function) Require a supported character set literal for encode and decode ### What problem does this PR solve? Issue Number: #48203 Related PR: #68131 Problem Summary: The character set argument must be a string literal, not a general constant expression. Reject columns and expressions such as upper('utf-8') during analysis, and reject literals outside US-ASCII, ISO-8859-1, UTF-8, UTF-16BE, UTF-16LE, and UTF-16. A NULL literal is still allowed. Run the same check again after rewrite. ### Release note encode and decode now require the character set to be a supported string literal or NULL. ### Check List (For Author) - Test: Unit Test / Regression test - EncodeDecodeTest covers literal, NULL, unsupported charset, and non-literal arguments. - regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy updated for the literal contract. - Behavior changed: Yes. upper('utf-8') and other non-literal character set expressions are rejected. - Does this need documentation: Yes. https://github.com/apache/doris-website/pull/4151 --- .../scalar/CharacterSetLiterals.java | 75 +++++++++++++++++++ .../expressions/functions/scalar/Decode.java | 11 +-- .../expressions/functions/scalar/Encode.java | 11 +-- .../functions/scalar/EncodeDecodeTest.java | 38 +++++++--- .../binary_functions/test_encode_decode.out | 6 -- .../test_encode_decode.groovy | 20 +++-- 6 files changed, 128 insertions(+), 33 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CharacterSetLiterals.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CharacterSetLiterals.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CharacterSetLiterals.java new file mode 100644 index 00000000000000..9354feba7d9b5c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CharacterSetLiterals.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** Character sets accepted by encode and decode. */ +final class CharacterSetLiterals { + private static final List SUPPORTED = ImmutableList.of( + "US-ASCII", "ISO-8859-1", "UTF-8", "UTF-16BE", "UTF-16LE", "UTF-16"); + + private CharacterSetLiterals() { + } + + static void checkSecondArgument(ScalarFunction function) { + Expression characterSet = function.getArgument(1); + if (!characterSet.isLiteral()) { + throw new AnalysisException("the second argument of function " + + function.getName() + " must be a literal: " + function.toSql()); + } + if (characterSet.isNullLiteral()) { + return; + } + if (!(characterSet instanceof StringLikeLiteral)) { + throw new AnalysisException("the second argument of function " + + function.getName() + " must be a string literal: " + function.toSql()); + } + String value = ((StringLikeLiteral) characterSet).getValue(); + for (String supported : SUPPORTED) { + if (equalsIgnoreAsciiCase(value, supported)) { + return; + } + } + throw new AnalysisException("Unsupported character set '" + value + + "'. Supported character sets are US-ASCII, ISO-8859-1, UTF-8, " + + "UTF-16BE, UTF-16LE, and UTF-16"); + } + + private static boolean equalsIgnoreAsciiCase(String value, String expected) { + if (value.length() != expected.length()) { + return false; + } + for (int i = 0; i < value.length(); i++) { + char current = value.charAt(i); + if (current >= 'a' && current <= 'z') { + current -= 'a' - 'A'; + } + if (current != expected.charAt(i)) { + return false; + } + } + return true; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java index 96ca7e95350ee7..e1845bcca281bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; @@ -56,10 +55,12 @@ private Decode(ScalarFunctionParams functionParams) { @Override public void checkLegalityBeforeTypeCoercion() { - if (!getArgument(1).isConstant()) { - throw new AnalysisException("the second argument of function " - + getName() + " must be constant: " + toSql()); - } + CharacterSetLiterals.checkSecondArgument(this); + } + + @Override + public void checkLegalityAfterRewrite() { + checkLegalityBeforeTypeCoercion(); } // Invalid character sets must still be rejected when the first argument is a diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java index 917317e1a49b4a..e650ea607ebcf3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; @@ -56,10 +55,12 @@ private Encode(ScalarFunctionParams functionParams) { @Override public void checkLegalityBeforeTypeCoercion() { - if (!getArgument(1).isConstant()) { - throw new AnalysisException("the second argument of function " - + getName() + " must be constant: " + toSql()); - } + CharacterSetLiterals.checkSecondArgument(this); + } + + @Override + public void checkLegalityAfterRewrite() { + checkLegalityBeforeTypeCoercion(); } // Invalid character sets must still be rejected when the first argument is a diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java index c9564457b3705f..d684d06cd88886 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -113,19 +114,34 @@ public void testCharacterSetMustBeConstant() { AnalysisException encodeException = Assertions.assertThrows( AnalysisException.class, encode::checkLegalityBeforeTypeCoercion); Assertions.assertTrue(encodeException.getMessage().contains( - "second argument of function encode must be constant")); + "second argument of function encode must be a literal")); AnalysisException decodeException = Assertions.assertThrows( AnalysisException.class, decode::checkLegalityBeforeTypeCoercion); Assertions.assertTrue(decodeException.getMessage().contains( - "second argument of function decode must be constant")); - - Assertions.assertDoesNotThrow(new Encode(new StringLiteral("hello"), - new Upper(new StringLiteral("utf-8")))::checkLegalityBeforeTypeCoercion); - Assertions.assertDoesNotThrow(new Decode(new VarBinaryLiteral(new byte[] {0x68, 0x69}), - new StringLiteral("UTF-8"))::checkLegalityBeforeTypeCoercion); - Assertions.assertFalse(new Encode(new StringLiteral("hello"), - new StringLiteral("UTF-8")).foldable()); - Assertions.assertFalse(new Decode(new VarBinaryLiteral(new byte[] {0x68, 0x69}), - new StringLiteral("UTF-8")).foldable()); + "second argument of function decode must be a literal")); + + AnalysisException encodeUpper = Assertions.assertThrows(AnalysisException.class, + new Encode(new StringLiteral("hello"), new Upper(new StringLiteral("utf-8"))) + ::checkLegalityBeforeTypeCoercion); + Assertions.assertTrue(encodeUpper.getMessage().contains("must be a literal")); + + Encode literalEncode = new Encode(new StringLiteral("hello"), new StringLiteral("utf-8")); + Decode literalDecode = new Decode(new VarBinaryLiteral(new byte[] {0x68, 0x69}), + new StringLiteral("UTF-8")); + Assertions.assertDoesNotThrow(literalEncode::checkLegalityBeforeTypeCoercion); + Assertions.assertDoesNotThrow(literalEncode::checkLegalityAfterRewrite); + Assertions.assertDoesNotThrow(literalDecode::checkLegalityBeforeTypeCoercion); + Assertions.assertDoesNotThrow(literalDecode::checkLegalityAfterRewrite); + + Encode nullCharset = new Encode(new StringLiteral("hello"), new NullLiteral(StringType.INSTANCE)); + Assertions.assertDoesNotThrow(nullCharset::checkLegalityBeforeTypeCoercion); + + AnalysisException unsupported = Assertions.assertThrows(AnalysisException.class, + new Encode(new StringLiteral("hello"), new StringLiteral("GBK")) + ::checkLegalityBeforeTypeCoercion); + Assertions.assertTrue(unsupported.getMessage().contains("Unsupported character set")); + + Assertions.assertFalse(literalEncode.foldable()); + Assertions.assertFalse(literalDecode.foldable()); } } diff --git a/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out b/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out index 158be3a23bd405..fa44a7072e384e 100644 --- a/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out +++ b/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out @@ -27,12 +27,6 @@ 8 9 中 --- !encode_constant_expr -- -E4B8AD - --- !decode_constant_expr -- -中 - -- !encode_null_valid_charset -- \N diff --git a/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy index 86340134a860f7..1b31ee85286f51 100644 --- a/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy +++ b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy @@ -71,7 +71,7 @@ suite("test_encode_decode") { union all select id, hex(encode(plain_text, 'UTF-8')) from test_encode_decode where id = 11 union all - select id, hex(encode(plain_text, cast(null as string))) from test_encode_decode where id = 12 + select id, hex(encode(plain_text, null)) from test_encode_decode where id = 12 ) t order by id """ @@ -100,13 +100,11 @@ suite("test_encode_decode") { union all select id, decode(cast(binary_value as varbinary), 'UTF-8') from test_encode_decode where id = 11 union all - select id, decode(cast(binary_value as varbinary), cast(null as string)) from test_encode_decode where id = 12 + select id, decode(cast(binary_value as varbinary), null) from test_encode_decode where id = 12 ) t order by id """ - qt_encode_constant_expr "select hex(encode('中', upper('utf-8')))" - qt_decode_constant_expr "select decode(X'E4B8AD', upper('utf-8'))" qt_encode_null_valid_charset "select encode(null, 'UTF-8')" qt_decode_null_valid_charset "select decode(null, 'UTF-8')" @@ -132,12 +130,22 @@ suite("test_encode_decode") { test { sql "select encode(plain_text, charset) from test_encode_decode where id = 1" - exception "second argument of function encode must be constant" + exception "second argument of function encode must be a literal" } test { sql "select decode(cast(binary_value as varbinary), charset) from test_encode_decode where id = 1" - exception "second argument of function decode must be constant" + exception "second argument of function decode must be a literal" + } + + test { + sql "select hex(encode('中', upper('utf-8')))" + exception "must be a literal" + } + + test { + sql "select decode(X'E4B8AD', upper('utf-8'))" + exception "must be a literal" } test { From 39f40f61e65bbea3f5381c883d6a9c9c958b3ac0 Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Wed, 23 Sep 2026 11:40:50 +0800 Subject: [PATCH 10/11] [fix](fe) Enable constant folding for encode and decode --- .../expressions/functions/scalar/Decode.java | 8 --- .../expressions/functions/scalar/Encode.java | 8 --- .../scalar/EncodeDecodePlannerTest.java | 61 +++++++++++++++++++ .../functions/scalar/EncodeDecodeTest.java | 3 - 4 files changed, 61 insertions(+), 19 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodePlannerTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java index e1845bcca281bd..4cb1296e6dcabf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java @@ -63,14 +63,6 @@ public void checkLegalityAfterRewrite() { checkLegalityBeforeTypeCoercion(); } - // Invalid character sets must still be rejected when the first argument is a - // null literal. FoldConstantRuleOnFE otherwise rewrites PropagateNullable - // calls with any null child to NULL and skips backend evaluation. - @Override - public boolean foldable() { - return false; - } - /** * withChildren. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java index e650ea607ebcf3..1bab3f65ed517b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java @@ -63,14 +63,6 @@ public void checkLegalityAfterRewrite() { checkLegalityBeforeTypeCoercion(); } - // Invalid character sets must still be rejected when the first argument is a - // null literal. FoldConstantRuleOnFE otherwise rewrites PropagateNullable - // calls with any null child to NULL and skips backend evaluation. - @Override - public boolean foldable() { - return false; - } - /** * withChildren. */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodePlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodePlannerTest.java new file mode 100644 index 00000000000000..253b4077efa009 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodePlannerTest.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class EncodeDecodePlannerTest extends TestWithFeService implements MemoPatternMatchSupported { + + @Test + void testLiteralCallsFoldDuringPlanRewrite() { + VarBinaryLiteral encoded = new VarBinaryLiteral(new byte[] {0x4E, 0x2D}); + PlanChecker.from(connectContext) + .analyze("select encode('中', 'UTF-16BE')") + .rewrite() + .matches(logicalResultSink( + logicalOneRowRelation().when(oneRow -> + oneRow.getProjects().get(0).child(0).equals(encoded)))); + + StringLiteral decoded = new StringLiteral("中"); + PlanChecker.from(connectContext) + .analyze("select decode(X'E4B8AD', 'UTF-8')") + .rewrite() + .matches(logicalResultSink( + logicalOneRowRelation().when(oneRow -> + oneRow.getProjects().get(0).child(0).equals(decoded)))); + } + + @Test + void testInvalidCharsetRejectedBeforeNullFolding() { + AnalysisException encodeError = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze("select encode(NULL, 'GBK')")); + Assertions.assertTrue(encodeError.getMessage().contains("Unsupported character set")); + + AnalysisException decodeError = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze("select decode(NULL, 'GBK')")); + Assertions.assertTrue(decodeError.getMessage().contains("Unsupported character set")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java index d684d06cd88886..5b28389add3d7f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java @@ -140,8 +140,5 @@ public void testCharacterSetMustBeConstant() { new Encode(new StringLiteral("hello"), new StringLiteral("GBK")) ::checkLegalityBeforeTypeCoercion); Assertions.assertTrue(unsupported.getMessage().contains("Unsupported character set")); - - Assertions.assertFalse(literalEncode.foldable()); - Assertions.assertFalse(literalDecode.foldable()); } } From 62069e53c5f754c31badaafaee67b1a12cbf8609 Mon Sep 17 00:00:00 2001 From: ZhiPing Date: Wed, 23 Sep 2026 12:33:29 +0800 Subject: [PATCH 11/11] [fix](be) Fix character encoding build warnings ### What problem does this PR solve? Issue Number: close #48203 Related PR: #68131 Problem Summary: The strict Linux BE builds treated shadowed name parameters and an unused UTF-16 output offset as errors. Rename the character-set parameters and remove the unused local without changing conversion behavior. ### Release note None ### Check List (For Author) - Test: Strict Clang syntax compilation, clang-format, and build hygiene checks. The local targeted BE UT was blocked during macOS OpenBLAS/OpenMP configuration before Doris compilation. - Behavior changed: No - Does this need documentation: No --- .../function/function_character_encoding.cpp | 76 ++++++++++--------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/be/src/exprs/function/function_character_encoding.cpp b/be/src/exprs/function/function_character_encoding.cpp index a40ce1b54d830f..3e317db4740b7d 100644 --- a/be/src/exprs/function/function_character_encoding.cpp +++ b/be/src/exprs/function/function_character_encoding.cpp @@ -246,67 +246,69 @@ class FunctionCharacterEncoding : public IFunction { if (input.size == 0) { return Status::OK(); } - const std::string_view name = charset_name(character_set); + const std::string_view character_set_name = charset_name(character_set); if (input.size > static_cast(std::numeric_limits::max())) { - return reject_too_large(name); + return reject_too_large(character_set_name); } if constexpr (Encode) { - return encode_input(input, character_set, name, utf16_scratch, converted); + return encode_input(input, character_set, character_set_name, utf16_scratch, converted); } else { - return decode_input(input, character_set, name, utf16_scratch, converted); + return decode_input(input, character_set, character_set_name, utf16_scratch, converted); } } - static Status encode_input(StringRef input, CharacterSet character_set, std::string_view name, + static Status encode_input(StringRef input, CharacterSet character_set, + std::string_view character_set_name, std::vector& utf16_scratch, ColumnString::Chars& output) { switch (character_set) { case CharacterSet::US_ASCII: - return copy_validated(input, name, + return copy_validated(input, character_set_name, simdutf::validate_ascii_with_errors(input.data, input.size), output); case CharacterSet::UTF_8: - return copy_validated(input, name, + return copy_validated(input, character_set_name, simdutf::validate_utf8_with_errors(input.data, input.size), output); case CharacterSet::ISO_8859_1: - return encode_latin1(input, name, output); + return encode_latin1(input, character_set_name, output); case CharacterSet::UTF_16BE: - return encode_utf16(input, name, false, false, utf16_scratch, output); + return encode_utf16(input, character_set_name, false, false, utf16_scratch, output); case CharacterSet::UTF_16LE: - return encode_utf16(input, name, true, false, utf16_scratch, output); + return encode_utf16(input, character_set_name, true, false, utf16_scratch, output); case CharacterSet::UTF_16: // Java's UTF-16 encoder always emits a big-endian BOM. - return encode_utf16(input, name, false, true, utf16_scratch, output); + return encode_utf16(input, character_set_name, false, true, utf16_scratch, output); default: - return Status::InvalidArgument("Unsupported character set '{}'", name); + return Status::InvalidArgument("Unsupported character set '{}'", character_set_name); } } - static Status decode_input(StringRef input, CharacterSet character_set, std::string_view name, + static Status decode_input(StringRef input, CharacterSet character_set, + std::string_view character_set_name, std::vector& utf16_scratch, ColumnString::Chars& output) { switch (character_set) { case CharacterSet::US_ASCII: - return copy_validated(input, name, + return copy_validated(input, character_set_name, simdutf::validate_ascii_with_errors(input.data, input.size), output); case CharacterSet::UTF_8: - return copy_validated(input, name, + return copy_validated(input, character_set_name, simdutf::validate_utf8_with_errors(input.data, input.size), output); case CharacterSet::ISO_8859_1: - return decode_latin1(input, name, output); + return decode_latin1(input, character_set_name, output); case CharacterSet::UTF_16BE: - return decode_utf16(input, name, false, utf16_scratch, output); + return decode_utf16(input, character_set_name, false, utf16_scratch, output); case CharacterSet::UTF_16LE: - return decode_utf16(input, name, true, utf16_scratch, output); + return decode_utf16(input, character_set_name, true, utf16_scratch, output); case CharacterSet::UTF_16: - return decode_utf16_with_bom(input, name, utf16_scratch, output); + return decode_utf16_with_bom(input, character_set_name, utf16_scratch, output); default: - return Status::InvalidArgument("Unsupported character set '{}'", name); + return Status::InvalidArgument("Unsupported character set '{}'", character_set_name); } } - static Status encode_latin1(StringRef input, std::string_view name, + static Status encode_latin1(StringRef input, std::string_view character_set_name, ColumnString::Chars& output) { const size_t start = output.size(); char* dest = reserve_output(output, input.size); @@ -317,27 +319,27 @@ class FunctionCharacterEncoding : public IFunction { output.resize(start); const simdutf::error_code error = detail.error == simdutf::SUCCESS ? simdutf::OTHER : detail.error; - return conversion_error(name, error); + return conversion_error(character_set_name, error); } output.resize(start + written); return Status::OK(); } - static Status decode_latin1(StringRef input, std::string_view name, + static Status decode_latin1(StringRef input, std::string_view character_set_name, ColumnString::Chars& output) { const size_t need = simdutf::utf8_length_from_latin1(input.data, input.size); char* dest = reserve_output(output, need); const size_t written = simdutf::convert_latin1_to_utf8(input.data, input.size, dest); if (written != need) { output.resize(output.size() - need); - return conversion_error(name, simdutf::OTHER); + return conversion_error(character_set_name, simdutf::OTHER); } return Status::OK(); } - static Status encode_utf16(StringRef input, std::string_view name, bool little_endian, - bool write_bom, std::vector& utf16_scratch, - ColumnString::Chars& output) { + static Status encode_utf16(StringRef input, std::string_view character_set_name, + bool little_endian, bool write_bom, + std::vector& utf16_scratch, ColumnString::Chars& output) { utf16_scratch.resize(input.size); const simdutf::result result = little_endian ? simdutf::convert_utf8_to_utf16le_with_errors(input.data, input.size, @@ -345,10 +347,9 @@ class FunctionCharacterEncoding : public IFunction { : simdutf::convert_utf8_to_utf16be_with_errors(input.data, input.size, utf16_scratch.data()); if (result.error != simdutf::SUCCESS) { - return conversion_error(name, result.error); + return conversion_error(character_set_name, result.error); } const size_t payload_bytes = result.count * sizeof(char16_t); - const size_t start = output.size(); char* dest = reserve_output(output, payload_bytes + (write_bom ? 2 : 0)); if (write_bom) { auto* bytes = reinterpret_cast(dest); @@ -361,11 +362,11 @@ class FunctionCharacterEncoding : public IFunction { } // Java's UTF-16 decoder honors either BOM and defaults to big endian without one. - static Status decode_utf16_with_bom(StringRef input, std::string_view name, + static Status decode_utf16_with_bom(StringRef input, std::string_view character_set_name, std::vector& utf16_scratch, ColumnString::Chars& output) { if (input.size < 2) { - return conversion_error(name, simdutf::TOO_SHORT); + return conversion_error(character_set_name, simdutf::TOO_SHORT); } const auto first = static_cast(input.data[0]); const auto second = static_cast(input.data[1]); @@ -379,17 +380,18 @@ class FunctionCharacterEncoding : public IFunction { if (input.size == 0) { return Status::OK(); } - return decode_utf16(input, name, little_endian, utf16_scratch, output); + return decode_utf16(input, character_set_name, little_endian, utf16_scratch, output); } - static Status decode_utf16(StringRef input, std::string_view name, bool little_endian, - std::vector& utf16_scratch, ColumnString::Chars& output) { + static Status decode_utf16(StringRef input, std::string_view character_set_name, + bool little_endian, std::vector& utf16_scratch, + ColumnString::Chars& output) { if (input.size % 2 != 0) { - return conversion_error(name, simdutf::TOO_SHORT); + return conversion_error(character_set_name, simdutf::TOO_SHORT); } const size_t units = input.size / 2; if (units > (std::numeric_limits::max() / 3)) { - return reject_too_large(name); + return reject_too_large(character_set_name); } const char16_t* units_ptr = utf16_units(input, utf16_scratch); const size_t start = output.size(); @@ -400,7 +402,7 @@ class FunctionCharacterEncoding : public IFunction { : simdutf::convert_utf16be_to_utf8_with_errors(units_ptr, units, dest); if (result.error != simdutf::SUCCESS) { output.resize(start); - return conversion_error(name, result.error); + return conversion_error(character_set_name, result.error); } output.resize(start + result.count); return Status::OK();