diff --git a/common/BUILD b/common/BUILD index 27ae6ded9..213a91035 100644 --- a/common/BUILD +++ b/common/BUILD @@ -756,6 +756,7 @@ cc_library( ":unknown", ":value_kind", "//base:attributes", + "//base:function_result_set", "//common/internal:byte_string", "//eval/internal:cel_value_equal", "//eval/public:cel_value", diff --git a/common/legacy_value.cc b/common/legacy_value.cc index 505f2fbf5..622257f43 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -1071,8 +1071,8 @@ absl::Status ModernValue(google::protobuf::Arena* arena, MapValue(common_internal::LegacyMapValue(legacy_value.MapOrDie())); return absl::OkStatus(); case CelValue::Type::kUnknownSet: - result = - common_internal::MakeUnknownValue(*legacy_value.UnknownSetOrDie()); + result = common_internal::MakeUnknownValue( + *legacy_value.UnknownSetOrDie(), arena); return absl::OkStatus(); case CelValue::Type::kCelType: { auto type_name = legacy_value.CelTypeOrDie().value(); @@ -1183,7 +1183,8 @@ absl::StatusOr FromLegacyValue(google::protobuf::Arena* arena, case CelValue::Type::kMap: return MapValue(common_internal::LegacyMapValue(legacy_value.MapOrDie())); case CelValue::Type::kUnknownSet: - return common_internal::MakeUnknownValue(*legacy_value.UnknownSetOrDie()); + return common_internal::MakeUnknownValue(*legacy_value.UnknownSetOrDie(), + arena); case CelValue::Type::kCelType: return CreateTypeValueFromView(arena, legacy_value.CelTypeOrDie().value()); diff --git a/common/value.h b/common/value.h index 9278cf4de..b8801c00e 100644 --- a/common/value.h +++ b/common/value.h @@ -298,103 +298,54 @@ class Value final : private common_internal::ValueMixin { Value() = default; Value(const Value&) = default; Value& operator=(const Value&) = default; - Value(Value&& other) = default; - Value& operator=(Value&&) = default; // NOLINTNEXTLINE(google-explicit-constructor) Value(const ListValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(ListValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const ListValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(ListValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const MapValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(MapValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const MapValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(MapValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const StructValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(StructValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const StructValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(StructValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const MessageValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(MessageValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const MessageValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(MessageValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const OptionalValue& value) : variant_(absl::in_place_type, static_cast(value)) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(OptionalValue&& value) - : variant_(absl::in_place_type, - static_cast(value)) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const OptionalValue& value) { variant_.Assign(static_cast(value)); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(OptionalValue&& value) { - variant_.Assign(static_cast(value)); - return *this; - } - template >>> diff --git a/common/values/unknown_value.cc b/common/values/unknown_value.cc index 1cb8a7674..abfd66625 100644 --- a/common/values/unknown_value.cc +++ b/common/values/unknown_value.cc @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/base/no_destructor.h" #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" +#include "base/function_result_set.h" #include "common/value.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -63,4 +65,13 @@ absl::Status UnknownValue::Equal( return absl::OkStatus(); } +namespace common_internal { + +const FunctionResultSet& GetEmptyFunctionResults() { + static const absl::NoDestructor instance; + return *instance; +} + +} // namespace common_internal + } // namespace cel diff --git a/common/values/unknown_value.h b/common/values/unknown_value.h index 1d17de1ca..dae2aa0b2 100644 --- a/common/values/unknown_value.h +++ b/common/values/unknown_value.h @@ -26,6 +26,7 @@ #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" +#include "base/function_result_set.h" #include "common/type.h" #include "common/unknown.h" #include "common/value_kind.h" @@ -42,7 +43,7 @@ class UnknownValue; namespace common_internal { [[nodiscard]] -UnknownValue MakeUnknownValue(Unknown value); +UnknownValue MakeUnknownValue(Unknown value, google::protobuf::Arena* absl_nonnull arena); [[nodiscard]] Unknown GetUnknown(const UnknownValue& value); [[nodiscard]] @@ -90,7 +91,8 @@ class UnknownValue final : private common_internal::ValueMixin { [[nodiscard]] AttributeSet ToAttributeSet() const { - return unknown_.unknown_attributes(); + return unknown_ != nullptr ? unknown_->unknown_attributes() + : AttributeSet(); } void swap(UnknownValue& other) noexcept { @@ -99,15 +101,16 @@ class UnknownValue final : private common_internal::ValueMixin { } private: - friend UnknownValue common_internal::MakeUnknownValue(Unknown value); + friend UnknownValue common_internal::MakeUnknownValue( + Unknown value, google::protobuf::Arena* absl_nonnull arena); friend Unknown common_internal::GetUnknown(const UnknownValue&); friend const FunctionResultSet& common_internal::GetUnknownFunctionResultSet( const UnknownValue& value); friend class common_internal::ValueMixin; - explicit UnknownValue(Unknown unknown) : unknown_(std::move(unknown)) {} + explicit UnknownValue(const Unknown* unknown) : unknown_(unknown) {} - Unknown unknown_; + const Unknown* unknown_ = nullptr; }; inline void swap(UnknownValue& lhs, UnknownValue& rhs) noexcept { @@ -120,20 +123,24 @@ inline std::ostream& operator<<(std::ostream& out, const UnknownValue& value) { namespace common_internal { +const FunctionResultSet& GetEmptyFunctionResults(); + [[nodiscard]] -inline UnknownValue MakeUnknownValue(Unknown value) { - return UnknownValue(std::move(value)); +inline UnknownValue MakeUnknownValue(Unknown value, + google::protobuf::Arena* absl_nonnull arena) { + return UnknownValue(google::protobuf::Arena::Create(arena, std::move(value))); } [[nodiscard]] inline Unknown GetUnknown(const UnknownValue& value) { - return value.unknown_; + return value.unknown_ != nullptr ? *value.unknown_ : Unknown(); } [[nodiscard]] inline const FunctionResultSet& GetUnknownFunctionResultSet( const UnknownValue& value ABSL_ATTRIBUTE_LIFETIME_BOUND) { - return value.unknown_.unknown_function_results(); + return value.unknown_ != nullptr ? value.unknown_->unknown_function_results() + : GetEmptyFunctionResults(); } } // namespace common_internal diff --git a/common/values/value_variant.cc b/common/values/value_variant.cc deleted file mode 100644 index 5f240f79b..000000000 --- a/common/values/value_variant.cc +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/values/value_variant.h" - -#include -#include -#include -#include - -#include "absl/base/optimization.h" -#include "absl/log/absl_check.h" -#include "common/values/unknown_value.h" -#include "common/values/values.h" - -namespace cel::common_internal { - -void ValueVariant::SlowCopyConstruct(const ValueVariant& other) noexcept { - ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); - - switch (index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - break; - default: - ABSL_UNREACHABLE(); - } -} - -void ValueVariant::SlowMoveConstruct(ValueVariant& other) noexcept { - ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); - - switch (index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - break; - default: - ABSL_UNREACHABLE(); - } -} - -void ValueVariant::SlowDestruct() noexcept { - ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); - - switch (index_) { - case ValueIndex::kUnknown: - At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } -} - -void ValueVariant::SlowCopyAssign(const ValueVariant& other, bool trivial, - bool other_trivial) noexcept { - ABSL_DCHECK(!trivial || !other_trivial); - - if (trivial) { - switch (other.index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - break; - default: - ABSL_UNREACHABLE(); - } - index_ = other.index_; - kind_ = other.kind_; - flags_ = other.flags_; - } else if (other_trivial) { - switch (index_) { - case ValueIndex::kUnknown: - At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - FastCopyAssign(other); - } else { - switch (index_) { - case ValueIndex::kUnknown: - switch (other.index_) { - case ValueIndex::kUnknown: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - default: - ABSL_UNREACHABLE(); - } - flags_ = other.flags_; - } -} - -void ValueVariant::SlowMoveAssign(ValueVariant& other, bool trivial, - bool other_trivial) noexcept { - ABSL_DCHECK(!trivial || !other_trivial); - - if (trivial) { - switch (other.index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - break; - default: - ABSL_UNREACHABLE(); - } - index_ = other.index_; - kind_ = other.kind_; - flags_ = other.flags_; - } else if (other_trivial) { - switch (index_) { - case ValueIndex::kUnknown: - At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - FastMoveAssign(other); - } else { - switch (index_) { - case ValueIndex::kUnknown: - switch (other.index_) { - case ValueIndex::kUnknown: - *At() = std::move(*other.At()); - break; - default: - ABSL_UNREACHABLE(); - } - break; - default: - ABSL_UNREACHABLE(); - } - flags_ = other.flags_; - } -} - -void ValueVariant::SlowSwap(ValueVariant& lhs, ValueVariant& rhs, - bool lhs_trivial, bool rhs_trivial) noexcept { - using std::swap; - ABSL_DCHECK(!lhs_trivial || !rhs_trivial); - - if (lhs_trivial) { - alignas(ValueVariant) std::byte tmp[sizeof(ValueVariant)]; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(tmp, std::addressof(lhs), sizeof(ValueVariant)); - switch (rhs.index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&lhs.raw_[0])) - UnknownValue(*rhs.At()); - rhs.At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - lhs.index_ = rhs.index_; - lhs.kind_ = rhs.kind_; - lhs.flags_ = rhs.flags_; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(rhs), tmp, sizeof(ValueVariant)); - } else if (rhs_trivial) { - alignas(ValueVariant) std::byte tmp[sizeof(ValueVariant)]; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(tmp, std::addressof(rhs), sizeof(ValueVariant)); - switch (lhs.index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&rhs.raw_[0])) - UnknownValue(*lhs.At()); - lhs.At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - rhs.index_ = lhs.index_; - rhs.kind_ = lhs.kind_; - rhs.flags_ = lhs.flags_; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(lhs), tmp, sizeof(ValueVariant)); - } else { - ValueVariant tmp = std::move(lhs); - lhs = std::move(rhs); - rhs = std::move(tmp); - } -} - -} // namespace cel::common_internal diff --git a/common/values/value_variant.h b/common/values/value_variant.h index e7c199834..aaf0766d2 100644 --- a/common/values/value_variant.h +++ b/common/values/value_variant.h @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -27,8 +26,6 @@ #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/meta/type_traits.h" -#include "absl/utility/utility.h" -#include "common/arena.h" #include "common/value_kind.h" #include "common/values/bool_value.h" #include "common/values/bytes_value.h" @@ -89,23 +86,9 @@ enum class ValueIndex : uint8_t { kBytes, kString, kError, - // Keep non-trivial alternatives together to aid in compiling optimizations. kUnknown, }; -// Used by ValueVariant to indicate pre-computed behaviors. -enum class ValueFlags : uint32_t { - kNone = 0, - kNonTrivial = 1, -}; - -ABSL_ATTRIBUTE_ALWAYS_INLINE inline constexpr ValueFlags operator&( - ValueFlags lhs, ValueFlags rhs) { - return static_cast( - static_cast>(lhs) & - static_cast>(rhs)); -} - // Traits specialized by each alternative. // // ValueIndex ValueAlternative::kIndex @@ -130,166 +113,90 @@ template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kNull; static constexpr ValueKind kKind = NullValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const NullValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kBool; static constexpr ValueKind kKind = BoolValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const BoolValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kInt; static constexpr ValueKind kKind = IntValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const IntValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kUint; static constexpr ValueKind kKind = UintValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const UintValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kDouble; static constexpr ValueKind kKind = DoubleValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const DoubleValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kDuration; static constexpr ValueKind kKind = DurationValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const DurationValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kTimestamp; static constexpr ValueKind kKind = TimestampValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const TimestampValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kType; static constexpr ValueKind kKind = TypeValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const TypeValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kLegacyList; static constexpr ValueKind kKind = LegacyListValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const LegacyListValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedJsonList; static constexpr ValueKind kKind = ParsedJsonListValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedJsonListValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedRepeatedField; static constexpr ValueKind kKind = ParsedRepeatedFieldValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags( - const ParsedRepeatedFieldValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kCustomList; static constexpr ValueKind kKind = CustomListValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const CustomListValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kLegacyMap; static constexpr ValueKind kKind = LegacyMapValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const LegacyMapValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedJsonMap; static constexpr ValueKind kKind = ParsedJsonMapValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedJsonMapValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedMapField; static constexpr ValueKind kKind = ParsedMapFieldValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedMapFieldValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> @@ -297,98 +204,54 @@ struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kCustomMap; static constexpr ValueKind kKind = CustomMapValue::kKind; static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const CustomMapValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kLegacyStruct; static constexpr ValueKind kKind = LegacyStructValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const LegacyStructValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedMessage; static constexpr ValueKind kKind = ParsedMessageValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedMessageValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kCustomStruct; static constexpr ValueKind kKind = CustomStructValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const CustomStructValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kOpaque; static constexpr ValueKind kKind = OpaqueValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const OpaqueValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kBytes; static constexpr ValueKind kKind = BytesValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static ValueFlags Flags(const BytesValue* absl_nonnull alternative) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kString; static constexpr ValueKind kKind = StringValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static ValueFlags Flags(const StringValue* absl_nonnull alternative) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kError; static constexpr ValueKind kKind = ErrorValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static ValueFlags Flags(const ErrorValue* absl_nonnull alternative) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kUnknown; static constexpr ValueKind kKind = UnknownValue::kKind; - static constexpr bool kAlwaysTrivial = false; - - static constexpr ValueFlags Flags(const UnknownValue* absl_nonnull) { - return ValueFlags::kNonTrivial; - } }; template @@ -406,93 +269,33 @@ inline constexpr bool IsValueAlternativeV = IsValueAlternative::value; inline constexpr size_t kValueVariantAlign = 8; inline constexpr size_t kValueVariantSize = 24; -// Hand-rolled variant used by cel::Value which exhibits up to a 25% performance -// improvement compared to using std::variant. -// -// The implementation abuses the fact that most alternatives are trivially -// copyable and some are conditionally trivially copyable at runtime. For the -// fast path, we perform raw byte copying. For the slow path, we fallback to a -// non-inlined function. The compiler is typically smart enough to inline the -// fast path and emit efficient instructions for the raw byte copying (usually -// two instructions). It also uses switch for visiting, which most compilers can -// optimize better compared to a function pointer table (which libc++ currently -// uses and Clang currently does not optimize well). -class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI - ValueVariant final { +// Hand-rolled variant used by cel::Value which avoids std::variant and its +// lack of support for an unchecked get. +class alignas(kValueVariantAlign) ValueVariant { public: ValueVariant() = default; - - ValueVariant(const ValueVariant& other) noexcept - : index_(other.index_), kind_(other.kind_), flags_(other.flags_) { - if ((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone) { - std::memcpy(raw_, other.raw_, sizeof(raw_)); - } else { - SlowCopyConstruct(other); - } - } - - ValueVariant(ValueVariant&& other) noexcept - : index_(other.index_), kind_(other.kind_), flags_(other.flags_) { - if ((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone) { - std::memcpy(raw_, other.raw_, sizeof(raw_)); - } else { - SlowMoveConstruct(other); - } - } - - ~ValueVariant() { - if ((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial) { - SlowDestruct(); - } - } - - ValueVariant& operator=(const ValueVariant& other) noexcept { - if (this != &other) { - const bool trivial = - (flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - const bool other_trivial = - (other.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - if (trivial && other_trivial) { - FastCopyAssign(other); - } else { - SlowCopyAssign(other, trivial, other_trivial); - } - } - return *this; - } - - ValueVariant& operator=(ValueVariant&& other) noexcept { - if (this != &other) { - const bool trivial = - (flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - const bool other_trivial = - (other.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - if (trivial && other_trivial) { - FastMoveAssign(other); - } else { - SlowMoveAssign(other, trivial, other_trivial); - } - } - return *this; - } + ValueVariant(const ValueVariant&) = default; + ValueVariant& operator=(const ValueVariant&) = default; template - explicit ValueVariant(absl::in_place_type_t, Args&&... args) + explicit ValueVariant(std::in_place_type_t, Args&&... args) : index_(ValueAlternative::kIndex), kind_(ValueAlternative::kKind) { static_assert(alignof(T) <= kValueVariantAlign); static_assert(sizeof(T) <= kValueVariantSize); - flags_ = ValueAlternative::Flags(::new (static_cast(&raw_[0])) - T(std::forward(args)...)); + ::new (static_cast(&raw_[0])) T(std::forward(args)...); } template >>> explicit ValueVariant(T&& value) - : ValueVariant(absl::in_place_type>, + : ValueVariant(std::in_place_type>, std::forward(value)) {} - ValueKind kind() const { return kind_; } + [[nodiscard]] + ValueKind kind() const { + return kind_; + } template void Assign(T&& value) { @@ -501,67 +304,35 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI static_assert(alignof(U) <= kValueVariantAlign); static_assert(sizeof(U) <= kValueVariantSize); - if constexpr (ValueAlternative::kAlwaysTrivial) { - if ((flags_ & ValueFlags::kNonTrivial) != ValueFlags::kNone) { - SlowDestruct(); - } - index_ = ValueAlternative::kIndex; - kind_ = ValueAlternative::kKind; - flags_ = ValueAlternative::Flags(::new (static_cast(&raw_[0])) - U(std::forward(value))); - } else { - // U is not always trivial. See if the current active alternative is U. If - // it is, we can just do a simple assignment without having to destruct - // first. Otherwise fallback to destruct and construct. - if (index_ == ValueAlternative::kIndex) { - *At() = std::forward(value); - flags_ = ValueAlternative::Flags(At()); - } else { - if ((flags_ & ValueFlags::kNonTrivial) != ValueFlags::kNone) { - SlowDestruct(); - } - index_ = ValueAlternative::kIndex; - kind_ = ValueAlternative::kKind; - flags_ = ValueAlternative::Flags(::new (static_cast(&raw_[0])) - U(std::forward(value))); - } - } + index_ = ValueAlternative::kIndex; + kind_ = ValueAlternative::kKind; + ::new (static_cast(&raw_[0])) U(std::forward(value)); } template + [[nodiscard]] bool Is() const { return index_ == ValueAlternative::kIndex; } template - T& Get() & ABSL_ATTRIBUTE_LIFETIME_BOUND { + [[nodiscard]] + T& Get() ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK(Is()); return *At(); } template - const T& Get() const& ABSL_ATTRIBUTE_LIFETIME_BOUND { + [[nodiscard]] + const T& Get() const ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK(Is()); return *At(); } template - T&& Get() && ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(Is()); - - return std::move(*At()); - } - - template - const T&& Get() const&& ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(Is()); - - return std::move(*At()); - } - - template + [[nodiscard]] T* absl_nullable As() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (Is()) { return At(); @@ -570,6 +341,7 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI } template + [[nodiscard]] const T* absl_nullable As() const ABSL_ATTRIBUTE_LIFETIME_BOUND { if (Is()) { return At(); @@ -578,187 +350,119 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI } template - ABSL_ATTRIBUTE_ALWAYS_INLINE decltype(auto) Visit(Visitor&& visitor) & { - return std::as_const(*this).Visit(std::forward(visitor)); - } - - template - decltype(auto) Visit(Visitor&& visitor) const& { + decltype(auto) Visit(Visitor&& visitor) const { switch (index_) { case ValueIndex::kNull: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBool: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kInt: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUint: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDouble: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDuration: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kTimestamp: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kType: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyList: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonList: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedRepeatedField: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomList: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyMap: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonMap: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMapField: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomMap: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyStruct: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMessage: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomStruct: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kOpaque: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBytes: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kString: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kError: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUnknown: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); } } template - decltype(auto) Visit(Visitor&& visitor) && { + decltype(auto) Visit(Visitor&& visitor) { switch (index_) { case ValueIndex::kNull: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBool: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kInt: - return std::forward(visitor)(std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUint: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDouble: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDuration: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kTimestamp: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kType: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyList: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonList: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedRepeatedField: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomList: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyMap: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonMap: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMapField: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomMap: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyStruct: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMessage: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomStruct: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kOpaque: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBytes: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kString: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kError: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUnknown: - return std::forward(visitor)( - std::move(*this).Get()); - } - } - - template - ABSL_ATTRIBUTE_ALWAYS_INLINE decltype(auto) Visit(Visitor&& visitor) const&& { - return Visit(std::forward(visitor)); - } - - friend void swap(ValueVariant& lhs, ValueVariant& rhs) noexcept { - if (&lhs != &rhs) { - const bool lhs_trivial = - (lhs.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - const bool rhs_trivial = - (rhs.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - if (lhs_trivial && rhs_trivial) { -// We validated the instances can be copied byte-wise at runtime, but compilers -// warn since this is not safe in the general case. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" -#elif defined(__clang__) && __clang_major__ >= 20 -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wnontrivial-memcall" -#endif - alignas(ValueVariant) std::byte tmp[sizeof(ValueVariant)]; - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(tmp, std::addressof(lhs), sizeof(ValueVariant)); - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(lhs), std::addressof(rhs), - sizeof(ValueVariant)); - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(rhs), tmp, sizeof(ValueVariant)); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#elif defined(__clang__) && __clang_major__ >= 20 -#pragma clang diagnostic pop -#endif - } else { - SlowSwap(lhs, rhs, lhs_trivial, rhs_trivial); - } + return std::forward(visitor)(*At()); } } private: - friend struct cel::ArenaTraits; - template + [[nodiscard]] ABSL_ATTRIBUTE_ALWAYS_INLINE T* absl_nonnull At() ABSL_ATTRIBUTE_LIFETIME_BOUND { + static_assert(IsValueAlternativeV); static_assert(alignof(T) <= kValueVariantAlign); static_assert(sizeof(T) <= kValueVariantSize); @@ -766,59 +470,23 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI } template + [[nodiscard]] ABSL_ATTRIBUTE_ALWAYS_INLINE const T* absl_nonnull At() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + static_assert(IsValueAlternativeV); static_assert(alignof(T) <= kValueVariantAlign); static_assert(sizeof(T) <= kValueVariantSize); return std::launder(reinterpret_cast(&raw_[0])); } - ABSL_ATTRIBUTE_ALWAYS_INLINE void FastCopyAssign( - const ValueVariant& other) noexcept { - index_ = other.index_; - kind_ = other.kind_; - flags_ = other.flags_; - std::memcpy(raw_, other.raw_, sizeof(raw_)); - } - - ABSL_ATTRIBUTE_ALWAYS_INLINE void FastMoveAssign( - ValueVariant& other) noexcept { - FastCopyAssign(other); - } - - void SlowCopyConstruct(const ValueVariant& other) noexcept; - - void SlowMoveConstruct(ValueVariant& other) noexcept; - - void SlowDestruct() noexcept; - - void SlowCopyAssign(const ValueVariant& other, bool trivial, - bool other_trivial) noexcept; - - void SlowMoveAssign(ValueVariant& other, bool ntrivial, - bool other_trivial) noexcept; - - static void SlowSwap(ValueVariant& lhs, ValueVariant& rhs, bool lhs_trivial, - bool rhs_trivial) noexcept; - ValueIndex index_ = ValueIndex::kNull; ValueKind kind_ = ValueKind::kNull; - ValueFlags flags_ = ValueFlags::kNone; alignas(kValueVariantAlign) std::byte raw_[kValueVariantSize]; }; } // namespace common_internal -template <> -struct ArenaTraits { - static bool trivially_destructible( - const common_internal::ValueVariant& value) { - return (value.flags_ & common_internal::ValueFlags::kNonTrivial) == - common_internal::ValueFlags::kNone; - } -}; - } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_VALUE_VARIANT_H_ diff --git a/common/values/value_variant_test.cc b/common/values/value_variant_test.cc index 968dca0d9..1c0416d4c 100644 --- a/common/values/value_variant_test.cc +++ b/common/values/value_variant_test.cc @@ -110,6 +110,7 @@ TYPED_TEST(ValueVariantTest, MoveAssign) { } TYPED_TEST(ValueVariantTest, Swap) { + using std::swap; using Left = typename TypeParam::first_type; using Right = typename TypeParam::second_type; diff --git a/eval/compiler/instrumentation_test.cc b/eval/compiler/instrumentation_test.cc index 630f398d1..39edecf6a 100644 --- a/eval/compiler/instrumentation_test.cc +++ b/eval/compiler/instrumentation_test.cc @@ -105,6 +105,7 @@ TEST_F(InstrumentationTest, Basic) { ASSERT_OK_AND_ASSIGN(auto value, plan.EvaluateWithCallback( activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); // AST for the test expression: // + <4> @@ -150,6 +151,7 @@ TEST_F(InstrumentationTest, BasicWithConstFolding) { ASSERT_OK_AND_ASSIGN(auto value, plan.EvaluateWithCallback( activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); // AST for the test expression: // + <4> diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 13efe4c0e..08dacb9b2 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -819,6 +819,7 @@ cc_test( "//eval/public:unknown_set", "//internal:testing", "//runtime/internal:attribute_matcher", + "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_protobuf//:protobuf", ], diff --git a/eval/eval/attribute_utility.cc b/eval/eval/attribute_utility.cc index a0e57161a..b1e75111d 100644 --- a/eval/eval/attribute_utility.cc +++ b/eval/eval/attribute_utility.cc @@ -104,7 +104,7 @@ bool AttributeUtility::CheckForUnknown(const AttributeTrail& trail, // it together with initial_set (if initial_set is not null). // Returns pointer to merged set or nullptr, if there were no sets to merge. absl::optional AttributeUtility::MergeUnknowns( - absl::Span args) const { + absl::Span args, google::protobuf::Arena* arena) const { // Empty unknown value may be used as a sentinel in some tests so need to // distinguish unset (nullopt) and empty(engaged empty value). absl::optional result_set; @@ -129,11 +129,13 @@ absl::optional AttributeUtility::MergeUnknowns( return cel::common_internal::MakeUnknownValue( cel::Unknown(result_set->unknown_attributes(), - result_set->unknown_function_results())); + result_set->unknown_function_results()), + arena); } -UnknownValue AttributeUtility::MergeUnknownValues( - const UnknownValue& left, const UnknownValue& right) const { +UnknownValue AttributeUtility::MergeUnknownValues(const UnknownValue& left, + const UnknownValue& right, + google::protobuf::Arena* arena) const { // Empty unknown value may be used as a sentinel in some tests so need to // distinguish unset (nullopt) and empty(engaged empty value). AttributeSet attributes; @@ -145,7 +147,7 @@ UnknownValue AttributeUtility::MergeUnknownValues( cel::common_internal::GetUnknownFunctionResultSet(right)); return cel::common_internal::MakeUnknownValue( - cel::Unknown(std::move(attributes), std::move(function_results))); + cel::Unknown(std::move(attributes), std::move(function_results)), arena); } // Creates merged UnknownAttributeSet. @@ -174,7 +176,7 @@ AttributeSet AttributeUtility::CheckForUnknowns( // Returns pointer to merged set or nullptr, if there were no sets to merge. absl::optional AttributeUtility::IdentifyAndMergeUnknowns( absl::Span args, absl::Span attrs, - bool use_partial) const { + bool use_partial, google::protobuf::Arena* arena) const { absl::optional result_set; // Identify new unknowns by attribute patterns. @@ -184,7 +186,7 @@ absl::optional AttributeUtility::IdentifyAndMergeUnknowns( } // merge down existing unknown sets - absl::optional arg_unknowns = MergeUnknowns(args); + absl::optional arg_unknowns = MergeUnknowns(args, arena); if (!result_set.has_value()) { // No new unknowns so no need to check for presence of existing unknowns -- @@ -199,12 +201,14 @@ absl::optional AttributeUtility::IdentifyAndMergeUnknowns( return cel::common_internal::MakeUnknownValue( cel::Unknown(result_set->unknown_attributes(), - result_set->unknown_function_results())); + result_set->unknown_function_results()), + arena); } -UnknownValue AttributeUtility::CreateUnknownSet(cel::Attribute attr) const { +UnknownValue AttributeUtility::CreateUnknownSet(cel::Attribute attr, + google::protobuf::Arena* arena) const { return cel::common_internal::MakeUnknownValue( - cel::Unknown(AttributeSet({std::move(attr)}))); + cel::Unknown(AttributeSet({std::move(attr)})), arena); } absl::StatusOr AttributeUtility::CreateMissingAttributeError( @@ -216,9 +220,10 @@ absl::StatusOr AttributeUtility::CreateMissingAttributeError( UnknownValue AttributeUtility::CreateUnknownSet( const cel::FunctionDescriptor& fn_descriptor, int64_t expr_id, - absl::Span args) const { + absl::Span args, google::protobuf::Arena* arena) const { return cel::common_internal::MakeUnknownValue( - cel::Unknown(FunctionResultSet(FunctionResult(fn_descriptor.name())))); + cel::Unknown(FunctionResultSet(FunctionResult(fn_descriptor.name()))), + arena); } void AttributeUtility::Add(Accumulator& a, const cel::UnknownValue& v) const { @@ -257,9 +262,10 @@ bool Accumulator::IsEmpty() const { function_result_set_.empty(); } -cel::UnknownValue Accumulator::Build() && { +cel::UnknownValue Accumulator::Build(google::protobuf::Arena* arena) && { return cel::common_internal::MakeUnknownValue( - cel::Unknown(std::move(attribute_set_), std::move(function_result_set_))); + cel::Unknown(std::move(attribute_set_), std::move(function_result_set_)), + arena); } } // namespace google::api::expr::runtime diff --git a/eval/eval/attribute_utility.h b/eval/eval/attribute_utility.h index 51ffc5d47..251e6c7d2 100644 --- a/eval/eval/attribute_utility.h +++ b/eval/eval/attribute_utility.h @@ -71,7 +71,7 @@ class AttributeUtility { bool IsEmpty() const; - cel::UnknownValue Build() &&; + cel::UnknownValue Build(google::protobuf::Arena* arena) &&; private: explicit Accumulator(const AttributeUtility& parent) @@ -131,11 +131,12 @@ class AttributeUtility { // Scans over the args collection, merges any UnknownValues found. // Returns the merged UnknownValue or nullopt if not found. absl::optional MergeUnknowns( - absl::Span args) const; + absl::Span args, google::protobuf::Arena* arena) const; // Creates a merged UnknownValue from two unknown values. cel::UnknownValue MergeUnknownValues(const cel::UnknownValue& left, - const cel::UnknownValue& right) const; + const cel::UnknownValue& right, + google::protobuf::Arena* arena) const; // Creates merged UnknownValue. // Merges together UnknownValues found in the args @@ -143,10 +144,11 @@ class AttributeUtility { // Returns returns the merged UnknownValue if available or nullopt. absl::optional IdentifyAndMergeUnknowns( absl::Span args, absl::Span attrs, - bool use_partial) const; + bool use_partial, google::protobuf::Arena* arena) const; // Create an initial UnknownSet from a single attribute. - cel::UnknownValue CreateUnknownSet(cel::Attribute attr) const; + cel::UnknownValue CreateUnknownSet(cel::Attribute attr, + google::protobuf::Arena* arena) const; // Factory function for missing attribute errors. absl::StatusOr CreateMissingAttributeError( @@ -155,7 +157,7 @@ class AttributeUtility { // Create an initial UnknownSet from a single missing function call. cel::UnknownValue CreateUnknownSet( const cel::FunctionDescriptor& fn_descriptor, int64_t expr_id, - absl::Span args) const; + absl::Span args, google::protobuf::Arena* arena) const; Accumulator CreateAccumulator() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return Accumulator(*this); diff --git a/eval/eval/attribute_utility_test.cc b/eval/eval/attribute_utility_test.cc index ea42c5b3b..cf65bd8c7 100644 --- a/eval/eval/attribute_utility_test.cc +++ b/eval/eval/attribute_utility_test.cc @@ -3,6 +3,7 @@ #include #include +#include "absl/types/optional.h" #include "absl/types/span.h" #include "base/attribute.h" #include "base/attribute_set.h" @@ -86,9 +87,9 @@ TEST_F(AttributeUtilityTest, UnknownsUtilityMergeUnknownsFromValues) { AttributeUtility utility(unknown_patterns, missing_attribute_patterns); UnknownValue unknown_set0 = cel::common_internal::MakeUnknownValue( - cel::Unknown(AttributeSet({attribute0}))); + cel::Unknown(AttributeSet({attribute0})), &arena_); UnknownValue unknown_set1 = cel::common_internal::MakeUnknownValue( - cel::Unknown(AttributeSet({attribute1}))); + cel::Unknown(AttributeSet({attribute1})), &arena_); std::vector values = { unknown_set0, @@ -97,7 +98,8 @@ TEST_F(AttributeUtilityTest, UnknownsUtilityMergeUnknownsFromValues) { cel::IntValue(1), }; - absl::optional unknown_set = utility.MergeUnknowns(values); + absl::optional unknown_set = + utility.MergeUnknowns(values, &arena_); ASSERT_TRUE(unknown_set.has_value()); EXPECT_THAT((*unknown_set).ToAttributeSet(), UnorderedPointwise( @@ -161,7 +163,7 @@ TEST_F(AttributeUtilityTest, CreateUnknownSet) { std::vector empty_patterns; AttributeUtility utility(empty_patterns, empty_patterns); - UnknownValue set = utility.CreateUnknownSet(trail.attribute()); + UnknownValue set = utility.CreateUnknownSet(trail.attribute(), &arena_); ASSERT_THAT(set.ToAttributeSet(), SizeIs(1)); ASSERT_OK_AND_ASSIGN(auto elem, set.ToAttributeSet().begin()->AsString()); EXPECT_EQ(elem, "destination.ip"); diff --git a/eval/eval/comprehension_step.cc b/eval/eval/comprehension_step.cc index 321e9c213..3532c1419 100644 --- a/eval/eval/comprehension_step.cc +++ b/eval/eval/comprehension_step.cc @@ -69,7 +69,8 @@ void ComprehensionInitStep::Evaluate(ExecutionFrame* frame) const { const AttributeTrail& top_attr = frame->value_stack().PeekAttribute(); if (frame->attribute_utility().CheckForUnknownPartial(top_attr)) { frame->value_stack().PopAndPush( - frame->attribute_utility().CreateUnknownSet(top_attr.attribute())); + frame->attribute_utility().CreateUnknownSet(top_attr.attribute(), + frame->arena())); frame->JumpToOrAbort(error_jump_offset_); return; } @@ -167,7 +168,7 @@ void ComprehensionNextStep::Evaluate1(ExecutionFrame* frame) const { if (frame->attribute_utility().CheckForUnknownExact( iter_slot->attribute())) { *iter_slot->mutable_value() = frame->attribute_utility().CreateUnknownSet( - iter_slot->attribute().attribute()); + iter_slot->attribute().attribute(), frame->arena()); } } else { absl::StatusOr ok = entry.iterator->Next1( @@ -246,7 +247,7 @@ void ComprehensionNextStep::Evaluate2(ExecutionFrame* frame) const { iter2_slot->attribute())) { *iter2_slot->mutable_value() = frame->attribute_utility().CreateUnknownSet( - iter2_slot->attribute().attribute()); + iter2_slot->attribute().attribute(), frame->arena()); } } } diff --git a/eval/eval/container_access_step.cc b/eval/eval/container_access_step.cc index 5f7cf3e2b..03fcb37d8 100644 --- a/eval/eval/container_access_step.cc +++ b/eval/eval/container_access_step.cc @@ -236,14 +236,15 @@ void PerformLookup(ExecutionFrameBase& frame, const Value& container, unknowns.MaybeAdd(key); if (!unknowns.IsEmpty()) { - result = std::move(unknowns).Build(); + result = std::move(unknowns).Build(frame.arena()); return; } trail = container_trail.Step(AttributeQualifierFromValue(key)); if (frame.attribute_utility().CheckForUnknownExact(trail)) { - result = frame.attribute_utility().CreateUnknownSet(trail.attribute()); + result = frame.attribute_utility().CreateUnknownSet(trail.attribute(), + frame.arena()); return; } } diff --git a/eval/eval/create_list_step.cc b/eval/eval/create_list_step.cc index 1d57fafa7..20a9b4f37 100644 --- a/eval/eval/create_list_step.cc +++ b/eval/eval/create_list_step.cc @@ -78,7 +78,7 @@ absl::Status CreateListStep::DoEvaluate(ExecutionFrame* frame, absl::optional unknown_set = frame->attribute_utility().IdentifyAndMergeUnknowns( args, frame->value_stack().GetAttributeSpan(list_size_), - /*use_partial=*/true); + /*use_partial=*/true, frame->arena()); if (unknown_set.has_value()) { *result = std::move(*unknown_set); return absl::OkStatus(); diff --git a/eval/eval/create_map_step.cc b/eval/eval/create_map_step.cc index 8283f1ed8..1b7d994a7 100644 --- a/eval/eval/create_map_step.cc +++ b/eval/eval/create_map_step.cc @@ -72,7 +72,8 @@ absl::StatusOr CreateStructStepForMap::DoEvaluate( if (frame->enable_unknowns()) { absl::optional unknown_set = frame->attribute_utility().IdentifyAndMergeUnknowns( - args, frame->value_stack().GetAttributeSpan(args.size()), true); + args, frame->value_stack().GetAttributeSpan(args.size()), true, + frame->arena()); if (unknown_set.has_value()) { return *unknown_set; } diff --git a/eval/eval/create_struct_step.cc b/eval/eval/create_struct_step.cc index ef05a08f7..5f1aa3f2b 100644 --- a/eval/eval/create_struct_step.cc +++ b/eval/eval/create_struct_step.cc @@ -77,7 +77,7 @@ absl::StatusOr CreateStructStepForStruct::DoEvaluate( absl::optional unknown_set = frame->attribute_utility().IdentifyAndMergeUnknowns( args, frame->value_stack().GetAttributeSpan(entries_size), - /*use_partial=*/true); + /*use_partial=*/true, frame->arena()); if (unknown_set.has_value()) { return *unknown_set; } diff --git a/eval/eval/equality_steps.cc b/eval/eval/equality_steps.cc index 6d8cf5751..5f3a6b58f 100644 --- a/eval/eval/equality_steps.cc +++ b/eval/eval/equality_steps.cc @@ -58,7 +58,7 @@ absl::StatusOr EvaluateEquality( accu.MaybeAdd(lhs, lhs_attr); accu.MaybeAdd(rhs, rhs_attr); if (!accu.IsEmpty()) { - return std::move(accu).Build(); + return std::move(accu).Build(frame.arena()); } } @@ -146,7 +146,7 @@ absl::StatusOr EvaluateIn(ExecutionFrameBase& frame, const Value& item, accu.MaybeAdd(item, item_attr); accu.MaybeAdd(container, container_attr); if (!accu.IsEmpty()) { - return std::move(accu).Build(); + return std::move(accu).Build(frame.arena()); } } if (container.IsList()) { diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 4314f4e38..cbc47d136 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -139,7 +139,7 @@ void ExpressionStep::EvaluateMutableListAppendStep(ExecutionFrame& frame) { std::optional unknown_set = frame.attribute_utility().IdentifyAndMergeUnknowns( args, frame.value_stack().GetAttributeSpan(2), - /*use_partial=*/true); + /*use_partial=*/true, frame.arena()); if (unknown_set.has_value()) { frame.value_stack().PopAndPush(2, std::move(*unknown_set)); return; diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index 4160072fc..37763bc9e 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -115,8 +115,8 @@ std::vector CheckForPartialUnknowns( if (frame->attribute_utility().CheckForUnknown(trail, /*use_partial=*/true)) { - result.push_back( - frame->attribute_utility().CreateUnknownSet(trail.attribute())); + result.push_back(frame->attribute_utility().CreateUnknownSet( + trail.attribute(), frame->arena())); } else { result.push_back(args.at(i)); } @@ -161,8 +161,8 @@ inline absl::StatusOr Invoke( if (frame.unknown_function_results_enabled() && IsUnknownFunctionResultError(*result)) { - return frame.attribute_utility().CreateUnknownSet(overload.descriptor, - expr_id, args); + return frame.attribute_utility().CreateUnknownSet( + overload.descriptor, expr_id, args, frame.arena()); } return result; } @@ -184,7 +184,7 @@ Value NoOverloadResult(absl::string_view name, if (frame.unknown_processing_enabled()) { // Already converted partial unknowns to unknown sets so just merge. absl::optional unknown_set = - frame.attribute_utility().MergeUnknowns(args); + frame.attribute_utility().MergeUnknowns(args, frame.arena()); if (unknown_set.has_value()) { return *unknown_set; } diff --git a/eval/eval/ident_step.cc b/eval/eval/ident_step.cc index 0bfbb9211..db1173101 100644 --- a/eval/eval/ident_step.cc +++ b/eval/eval/ident_step.cc @@ -33,8 +33,8 @@ absl::Status LookupIdent(absl::string_view name, ExecutionFrameBase& frame, } if (frame.unknown_processing_enabled() && frame.attribute_utility().CheckForUnknownExact(attribute)) { - result = - frame.attribute_utility().CreateUnknownSet(attribute.attribute()); + result = frame.attribute_utility().CreateUnknownSet(attribute.attribute(), + frame.arena()); return absl::OkStatus(); } } diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index 3be16e527..cc0a42cf2 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -39,7 +39,7 @@ void EvaluateNotStep(ExecutionFrame& frame) { const AttributeTrail& attribute_trail = frame.value_stack().PeekAttribute(); if (frame.attribute_utility().CheckForUnknownPartial(attribute_trail)) { frame.value_stack().PopAndPush(frame.attribute_utility().CreateUnknownSet( - attribute_trail.attribute())); + attribute_trail.attribute(), frame.arena())); return; } } @@ -123,7 +123,7 @@ void EvaluateBoolLogicStep(BoolLogicKind kind, size_t num_args, if (frame.enable_unknowns()) { // Check if unknown? std::optional unknown_set = - frame.attribute_utility().MergeUnknowns(args); + frame.attribute_utility().MergeUnknowns(args, frame.arena()); if (unknown_set.has_value()) { frame.value_stack().PopAndPush(num_args, *std::move(unknown_set)); return; diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 9bfa5fad4..8f58075af 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -55,7 +55,8 @@ absl::optional CheckForMarkedAttributes(const AttributeTrail& trail, ExecutionFrameBase& frame) { if (frame.unknown_processing_enabled() && frame.attribute_utility().CheckForUnknownExact(trail)) { - return frame.attribute_utility().CreateUnknownSet(trail.attribute()); + return frame.attribute_utility().CreateUnknownSet(trail.attribute(), + frame.arena()); } if (frame.missing_attribute_errors_enabled() && diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 29cb5d6ff..dff02b625 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -688,7 +688,7 @@ absl::StatusOr> CheckForMarkedAttributes( // declared pattern. Follow up will truncate the returned attribute to match // the pattern. return frame.attribute_utility().CreateUnknownSet( - attribute_trail.attribute()); + attribute_trail.attribute(), frame.arena()); } if (frame.missing_attribute_errors_enabled() &&