From 59500e84dec526b2db3ae34dc8c758dfbcd224ef Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 21 Oct 2026 16:04:11 +0900 Subject: [PATCH] GH-54638: [C++][FlightRPC] Upgrade bundled gRPC to 1.85.0-pre1 and Abseil to 20260817.0 gRPC 1.76.0 doesn't compile with OpenSSL 4. gRPC 1.85.0 includes OpenSSL 4 support: https://github.com/grpc/grpc/pull/41932 gRPC 1.85.0 requires a newer Abseil. gRPC 1.76.0 doesn't compile with the newer Abseil. So we need to upgrade both of them together. gRPC 1.85.0 moved its .proto files to the third_party/grpc-proto submodule. Source archives don't include submodules and we disable gRPC_DOWNLOAD_ARCHIVES. So we download grpc-proto by ourselves. We use Abseil 20260817.0 not 20260526.0 that is used by gRPC 1.85.0 because Abseil 20260526.0 can't be built with MSVC with /arch:AVX2: https://github.com/abseil/abseil-cpp/issues/2116 Abseil 20260817.0 removed some libraries and added some new libraries. So we update the list of bundled Abseil libraries. Abseil 20260817.0 uses O_NONBLOCK that isn't available on MinGW. So we define O_NONBLOCK as 0 for MinGW like the upstream fix: https://github.com/abseil/abseil-cpp/commit/ddbdebc6c8 Abseil 20260817.0 includes by itself. So we remove the "-include stdint.h" workaround for GCC 13 or later. gRPC headers don't use deprecated std::iterator. So we remove the "-Wno-error=deprecated-declarations" workaround. We didn't bundle upb static libraries used by gRPC. So static linking with libarrow_bundled_dependencies.a failed with undefined upb symbols. gRPC 1.85.0 also requires the new upb_descriptor_lib. So we bundle all upb static libraries. Protobuf 31.1 can't be built with Abseil 20260817.0 with MSVC. So we upgrade bundled Protobuf to 36.2. Protobuf 36.2 headers include utf8_validity.h in utf8_range. So we add the utf8_range include directory for ORC built by ExternalProject. google-cloud-cpp headers use deprecated Abseil APIs such as absl::visit(). They cause compilation errors with -Werror with Abseil 20260817.0. They are fixed in google-cloud-cpp 3.9.0 or later but google-cloud-cpp 3.9.0 or later requires libcurl 8.7.1 or later. We can't require it because some supported platforms such as Ubuntu 24.04 ship older libcurl. So we use -Wno-error=deprecated-declarations for bundled google-cloud-cpp instead of upgrading it. Co-Authored-By: Claude Opus 5.5 --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 94 ++++++++++++++++----- cpp/thirdparty/versions.txt | 16 ++-- 2 files changed, 81 insertions(+), 29 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 01e1ab147af..200275a3bec 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -678,6 +678,14 @@ else() "${THIRDPARTY_MIRROR_URL}/grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz") endif() +if(DEFINED ENV{ARROW_GRPC_PROTO_URL}) + set(GRPC_PROTO_SOURCE_URL "$ENV{ARROW_GRPC_PROTO_URL}") +else() + set_urls(GRPC_PROTO_SOURCE_URL + "https://github.com/grpc/grpc-proto/archive/${ARROW_GRPC_PROTO_BUILD_VERSION}.tar.gz" + "${THIRDPARTY_MIRROR_URL}/grpc-proto-${ARROW_GRPC_PROTO_BUILD_VERSION}.tar.gz") +endif() + if(DEFINED ENV{ARROW_GTEST_URL}) set(GTEST_SOURCE_URL "$ENV{ARROW_GTEST_URL}") else() @@ -1925,8 +1933,13 @@ function(build_absl) TRUE PARENT_SCOPE) - if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0) - string(APPEND CMAKE_CXX_FLAGS " -include stdint.h") + # Abseil 20260817.0 uses O_NONBLOCK that isn't available on MinGW. + # + # TODO: Remove this workaround once we upgrade to a newer Abseil + # version that includes + # https://github.com/abseil/abseil-cpp/commit/ddbdebc6c8 . + if(MINGW) + string(APPEND CMAKE_CXX_FLAGS " -DO_NONBLOCK=0") endif() fetchcontent_declare(absl @@ -1960,10 +1973,8 @@ function(build_absl) list(PREPEND ARROW_BUNDLED_STATIC_LIBS - absl::bad_any_cast_impl - absl::bad_optional_access - absl::bad_variant_access absl::base + absl::base_cpu_detect absl::city absl::civil_time absl::cord @@ -1974,7 +1985,6 @@ function(build_absl) absl::cordz_sample_token absl::crc32c absl::crc_cord_state - absl::crc_cpu_detect absl::crc_internal absl::debugging_internal absl::decode_rust_punycode @@ -1996,11 +2006,13 @@ function(build_absl) absl::flags_usage absl::flags_usage_internal absl::graphcycles_internal + absl::hardening absl::hash absl::hashtablez_sampler absl::int128 absl::kernel_timeout_internal absl::leak_check + absl::log_entry absl::log_globals absl::log_initialize absl::log_internal_check_op @@ -2015,14 +2027,13 @@ function(build_absl) absl::log_internal_structured_proto absl::log_severity absl::log_sink - absl::low_level_hash absl::malloc_internal absl::periodic_sampler absl::poison absl::random_distributions absl::random_internal_distribution_test_util + absl::random_internal_entropy_pool absl::random_internal_platform - absl::random_internal_pool_urbg absl::random_internal_randen absl::random_internal_randen_hwaes absl::random_internal_randen_hwaes_impl @@ -2033,6 +2044,7 @@ function(build_absl) absl::raw_hash_set absl::raw_logging_internal absl::scoped_set_env + absl::source_location absl::spinlock_wait absl::stacktrace absl::status @@ -2046,6 +2058,7 @@ function(build_absl) absl::throw_delegate absl::time absl::time_zone + absl::tracing_internal absl::utf8_for_code_point absl::vlog_config_internal) set(ARROW_BUNDLED_STATIC_LIBS @@ -3424,10 +3437,21 @@ function(build_grpc) TRUE PARENT_SCOPE) + # gRPC's source archive doesn't include the third_party/grpc-proto + # submodule. We disable gRPC_DOWNLOAD_ARCHIVES, so we download it + # by ourselves. + fetchcontent_declare(grpc_proto + ${FC_DECLARE_COMMON_OPTIONS} + URL ${GRPC_PROTO_SOURCE_URL} + URL_HASH "SHA256=${ARROW_GRPC_PROTO_BUILD_SHA256_CHECKSUM}") + fetchcontent_makeavailable(grpc_proto) + fetchcontent_declare(grpc ${FC_DECLARE_COMMON_OPTIONS} URL ${GRPC_SOURCE_URL} - URL_HASH "SHA256=${ARROW_GRPC_BUILD_SHA256_CHECKSUM}") + URL_HASH "SHA256=${ARROW_GRPC_BUILD_SHA256_CHECKSUM}" + PATCH_COMMAND ${CMAKE_COMMAND} -E copy_directory + ${grpc_proto_SOURCE_DIR} third_party/grpc-proto) prepare_fetchcontent() @@ -3497,16 +3521,6 @@ function(build_grpc) add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin) endif() - # gRPC headers use deprecated std::iterator that causes compilation errors. - # This workaround can be removed once we upgrade to a newer gRPC version. - if(NOT MSVC) - foreach(target ${GRPC_LIBRARY_TARGETS}) - if(TARGET ${target}) - target_compile_options(${target} INTERFACE -Wno-error=deprecated-declarations) - endif() - endforeach() - endif() - # ar -M rejects with the "libgrpc++.a" filename because "+" is a line # continuation character in these scripts, so we have to create a copy of the # static lib that we will bundle later. @@ -3533,7 +3547,19 @@ function(build_grpc) gRPC::address_sorting gRPC::gpr gRPC::grpc - gRPC::grpcpp_for_bundling) + gRPC::grpcpp_for_bundling + upb_base_lib + upb_descriptor_lib + upb_hash_lib + upb_json_lib + upb_lex_lib + upb_mem_lib + upb_message_lib + upb_mini_descriptor_lib + upb_mini_table_lib + upb_reflection_lib + upb_textformat_lib + upb_wire_lib) set(ARROW_BUNDLED_STATIC_LIBS ${ARROW_BUNDLED_STATIC_LIBS} PARENT_SCOPE) @@ -3848,6 +3874,19 @@ function(build_google_cloud_cpp_storage) PUBLIC $) endif() + # google-cloud-cpp headers use deprecated Abseil APIs such as + # absl::visit(). They are fixed in google-cloud-cpp 3.9.0 or later + # but google-cloud-cpp 3.9.0 or later requires libcurl 8.7.1 or + # later. We can't require it because some supported platforms such as + # Ubuntu 24.04 ship older libcurl. + # + # TODO: Remove this workaround once we upgrade to google-cloud-cpp + # 3.9.0 or later. + if(NOT MSVC AND TARGET google_cloud_cpp_common) + target_compile_options(google_cloud_cpp_common + INTERFACE -Wno-error=deprecated-declarations) + endif() + if(CMAKE_VERSION VERSION_LESS 3.28) set_property(DIRECTORY ${google_cloud_cpp_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL TRUE) endif() @@ -4035,6 +4074,13 @@ function(build_orc) if(absl_SOURCE_DIR) set(ORC_ABSL_INCLUDE_DIR "${absl_SOURCE_DIR}") endif() + # Protobuf 36.2 headers include utf8_validity.h in utf8_range. + # ORC build needs the utf8_range include directory if we're using a + # bundled Protobuf. + fetchcontent_getproperties(protobuf) + if(protobuf_SOURCE_DIR) + set(ORC_UTF8_RANGE_INCLUDE_DIR "${protobuf_SOURCE_DIR}/third_party/utf8_range") + endif() get_target_property(ORC_SNAPPY_INCLUDE_DIR ${Snappy_TARGET} INTERFACE_INCLUDE_DIRECTORIES) @@ -4046,10 +4092,12 @@ function(build_orc) get_target_property(ORC_ZLIB_ROOT ZLIB::ZLIB INTERFACE_INCLUDE_DIRECTORIES) get_filename_component(ORC_ZLIB_ROOT "${ORC_ZLIB_ROOT}" DIRECTORY) + set(ORC_CXX_FLAGS "${EP_CXX_FLAGS}") if(ORC_ABSL_INCLUDE_DIR) - set(ORC_CXX_FLAGS "${EP_CXX_FLAGS} -isystem ${ORC_ABSL_INCLUDE_DIR}") - else() - set(ORC_CXX_FLAGS "${EP_CXX_FLAGS}") + string(APPEND ORC_CXX_FLAGS " -isystem ${ORC_ABSL_INCLUDE_DIR}") + endif() + if(ORC_UTF8_RANGE_INCLUDE_DIR) + string(APPEND ORC_CXX_FLAGS " -isystem ${ORC_UTF8_RANGE_INCLUDE_DIR}") endif() set(ORC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index e2f16c588e2..79a14cacb74 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -23,8 +23,8 @@ # `DEPENDENCIES` array (see the comment on top of the declaration for the # format). -ARROW_ABSL_BUILD_VERSION=20250127.0 -ARROW_ABSL_BUILD_SHA256_CHECKSUM=16242f394245627e508ec6bb296b433c90f8d914f73b9c026fddb905e27276e8 +ARROW_ABSL_BUILD_VERSION=20260817.0 +ARROW_ABSL_BUILD_SHA256_CHECKSUM=f7e05179df39c45434cad433f5783840bb3788ef322976f9138bc6b72b3a107d ARROW_AWS_C_AUTH_BUILD_VERSION=v0.10.1 ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=85d737f0f735256f1931e85e4cadbe228d88698f7b59a9b390b49ef5d0778a43 ARROW_AWS_C_CAL_BUILD_VERSION=v0.9.13 @@ -72,8 +72,11 @@ ARROW_GLOG_BUILD_VERSION=v0.5.0 ARROW_GLOG_BUILD_SHA256_CHECKSUM=eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5 ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v3.2.0 ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=79ed6303d5b0c46f627bf8b38b5bd6b007c85e76c88722eb570a495bc2b579a0 -ARROW_GRPC_BUILD_VERSION=v1.76.0 -ARROW_GRPC_BUILD_SHA256_CHECKSUM=0af37b800953130b47c075b56683ee60bdc3eda3c37fc6004193f5b569758204 +ARROW_GRPC_BUILD_VERSION=v1.85.0-pre1 +ARROW_GRPC_BUILD_SHA256_CHECKSUM=598f189d8a187b35e87401d306cd9227008289ba40f1205a2e15f433bc93cc32 +# This must be synchronized with the grpc-proto version in gRPC's CMakeLists.txt. +ARROW_GRPC_PROTO_BUILD_VERSION=dd549fc4ed04b60b0fdd3866b82ef0a7c87c6e4f +ARROW_GRPC_PROTO_BUILD_SHA256_CHECKSUM=6aaa807572edfe307eaddc15161ce684d02d054e3fca36b38993efc85575ddfa ARROW_GTEST_BUILD_VERSION=1.18.0 ARROW_GTEST_BUILD_SHA256_CHECKSUM=6e3191c1455468b3fc35a417fb565c1c5071aee1b7e7f85e30cf48a98d37d8b5 ARROW_JEMALLOC_BUILD_VERSION=5.3.0 @@ -90,8 +93,8 @@ ARROW_OPENTELEMETRY_PROTO_BUILD_VERSION=v1.7.0 ARROW_OPENTELEMETRY_PROTO_BUILD_SHA256_CHECKSUM=11330d850f5e24d34c4246bc8cb21fcd311e7565d219195713455a576bb11bed ARROW_ORC_BUILD_VERSION=2.2.1 ARROW_ORC_BUILD_SHA256_CHECKSUM=52fc762332442e8b05d7182f8c035f9e04d945b9a52be22ab69f28b3f37d4500 -ARROW_PROTOBUF_BUILD_VERSION=v31.1 -ARROW_PROTOBUF_BUILD_SHA256_CHECKSUM=12bfd76d27b9ac3d65c00966901609e020481b9474ef75c7ff4601ac06fa0b82 +ARROW_PROTOBUF_BUILD_VERSION=v36.2 +ARROW_PROTOBUF_BUILD_SHA256_CHECKSUM=3d9642a662d10e68ebae5e53f14dcce5105684212d5078f8e0d47d1ab3ae6b64 # Because of https://github.com/Tencent/rapidjson/pull/1323, we require # a pre-release version of RapidJSON to build with GCC 8 without # warnings. @@ -157,6 +160,7 @@ DEPENDENCIES=( "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz" "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz" "ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz" + "ARROW_GRPC_PROTO_URL grpc-proto-${ARROW_GRPC_PROTO_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc-proto/archive/${ARROW_GRPC_PROTO_BUILD_VERSION}.tar.gz" "ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/releases/download/v${ARROW_GTEST_BUILD_VERSION}/googletest-${ARROW_GTEST_BUILD_VERSION}.tar.gz" "ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2" "ARROW_LZ4_URL lz4-${ARROW_LZ4_BUILD_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${ARROW_LZ4_BUILD_VERSION}.tar.gz"