diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_string.cc b/cpp/src/arrow/compute/kernels/scalar_cast_string.cc index 52b79c8d452..8352ea0b90d 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_string.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_string.cc @@ -200,13 +200,51 @@ struct TemporalToStringCastFunctor { DCHECK(!timezone.empty()); ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone)); ARROW_ASSIGN_OR_RAISE(auto locale, GetLocale("C")); - TimestampFormatter formatter{ - timezone == "UTC" ? kUtcFormatString : kFormatString, tz, locale}; + if (timezone == "UTC") { + TimestampFormatter formatter{kUtcFormatString, tz, locale}; + return VisitArraySpanInline( + input, + [&](value_type v) { + ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(v)); + return builder->Append(std::move(formatted)); + }, + [&]() { + builder->UnsafeAppendNull(); + return Status::OK(); + }); + } + + ARROW_ASSIGN_OR_RAISE(auto utc_tz, LocateZone("UTC")); + TimestampFormatter utc_formatter{kUtcFormatString, utc_tz, locale}; + std::ostringstream bufstream; + bufstream.imbue(locale); + bufstream.exceptions(std::ios::failbit | std::ios::badbit); + return VisitArraySpanInline( input, [&](value_type v) { - ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(v)); - return builder->Append(std::move(formatted)); + bufstream.str(""); + const auto timepoint = sys_time(Duration{v}); + bool subminute_offset = false; + auto format_zoned_time = [&](auto&& zt) { + if (ARROW_PREDICT_FALSE(zt.get_info().offset.count() % 60 != 0)) { + subminute_offset = true; + return Status::OK(); + } + try { + chrono::to_stream(bufstream, kFormatString.c_str(), zt); + return Status::OK(); + } catch (const std::runtime_error& ex) { + bufstream.clear(); + return Status::Invalid("Failed formatting timestamp: ", ex.what()); + } + }; + RETURN_NOT_OK(ApplyTimeZone(tz, timepoint, format_zoned_time)); + if (ARROW_PREDICT_FALSE(subminute_offset)) { + ARROW_ASSIGN_OR_RAISE(auto formatted, utc_formatter(v)); + return builder->Append(std::move(formatted)); + } + return builder->Append(bufstream.str()); }, [&]() { builder->UnsafeAppendNull(); diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc index 864ec4afce0..350cb95514d 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc @@ -2792,6 +2792,16 @@ TEST_F(CastTimezone, TimestampWithZoneToString) { ArrayFromJSON( string_type, R"(["1968-11-30 13:30:44.123456789-0700", "2016-02-29 10:42:23.456789246-0700"])")); + + CheckCast(ArrayFromJSON(timestamp(TimeUnit::MICRO, "Africa/Monrovia"), + "[-302443200000000]"), + ArrayFromJSON(string_type, + R"(["1960-06-01 12:00:00.000000Z"])")); + + CheckCast(ArrayFromJSON(timestamp(TimeUnit::MICRO, "Europe/Paris"), + "[-2195899200000000]"), + ArrayFromJSON(string_type, + R"(["1900-06-01 12:00:00.000000Z"])")); } } diff --git a/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc b/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc index 86a81ffdd38..c10b55c56b5 100644 --- a/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc @@ -2041,6 +2041,12 @@ TEST_F(ScalarTemporalTest, Strftime) { CheckScalarUnary("strftime", timestamp(TimeUnit::NANO, "Pacific/Honolulu"), nanoseconds, utf8(), string_nanoseconds, &options); + auto options_ez = StrftimeOptions("%Y-%m-%dT%H:%M:%S%Ez"); + const char* string_microseconds_ez = R"( + ["1970-01-01T05:30:59.123456+05:30", null])"; + CheckScalarUnary("strftime", timestamp(TimeUnit::MICRO, "Asia/Kolkata"), microseconds, + utf8(), string_microseconds_ez, &options_ez); + auto options_hms = StrftimeOptions("%H:%M:%S"); auto options_ymdhms = StrftimeOptions("%Y-%m-%dT%H:%M:%S"); @@ -2095,6 +2101,15 @@ TEST_F(ScalarTemporalTest, Strftime) { Invalid, testing::HasSubstr("Invalid: Timezone not present, cannot convert to string"), Strftime(arr_ns, StrftimeOptions("%Y-%m-%dT%H:%M:%S%Z"))); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + testing::HasSubstr("Invalid: Timezone not present, cannot convert to string"), + Strftime(arr_s, StrftimeOptions("%Y-%m-%dT%H:%M:%S%Ez"))); + auto arr_ts = ArrayFromJSON(timestamp(TimeUnit::SECOND), seconds); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + testing::HasSubstr("Invalid: Timezone not present, cannot convert to string"), + Strftime(arr_ts, StrftimeOptions("%Y-%m-%dT%H:%M:%S%Ez"))); auto options_ymd = StrftimeOptions("%Y-%m-%d"); @@ -2170,6 +2185,12 @@ TEST_F(ScalarTemporalTest, StrftimeCLocale) { CheckScalarUnary("strftime", timestamp(TimeUnit::NANO, "Pacific/Honolulu"), nanoseconds, utf8(), string_nanoseconds, &options); + auto options_ez = StrftimeOptions("%Y-%m-%dT%H:%M:%S%Ez"); + const char* string_microseconds_ez = R"( + ["1970-01-01T05:30:59.123456+05:30", null])"; + CheckScalarUnary("strftime", timestamp(TimeUnit::MICRO, "Asia/Kolkata"), microseconds, + utf8(), string_microseconds_ez, &options_ez); + CheckScalarUnary("strftime", timestamp(TimeUnit::NANO, "Pacific/Honolulu"), nanoseconds, utf8(), string_locale_specific, &options_locale_specific); } diff --git a/cpp/src/arrow/compute/kernels/scalar_temporal_unary.cc b/cpp/src/arrow/compute/kernels/scalar_temporal_unary.cc index 74a29081ae8..52b96084d23 100644 --- a/cpp/src/arrow/compute/kernels/scalar_temporal_unary.cc +++ b/cpp/src/arrow/compute/kernels/scalar_temporal_unary.cc @@ -1198,6 +1198,28 @@ Result GetLocale(const std::string& locale) { } } +// Check if format string contains %z, %Ez, or (optionally) %Z +static inline bool FormatHasZone(std::string_view format, bool include_zone_name = true) { + size_t cur = 0; + size_t count = 0; + while (cur < format.size() - 1) { + if (format[cur] == '%') { + count++; + if (count % 2 == 1) { + if (format[cur + 1] == 'z' || + (include_zone_name && format[cur + 1] == 'Z') || + (format[cur + 1] == 'E' && cur + 2 < format.size() && format[cur + 2] == 'z')) { + return true; + } + } + } else { + count = 0; + } + cur++; + } + return false; +} + template struct Strftime { const StrftimeOptions& options; @@ -1215,8 +1237,7 @@ struct Strftime { const auto& timezone = GetInputTimezone(type); if (timezone.empty()) { - if ((options.format.find("%z") != std::string::npos) || - (options.format.find("%Z") != std::string::npos)) { + if (FormatHasZone(options.format, /*include_zone_name=*/true)) { return Status::Invalid( "Timezone not present, cannot convert to string with timezone: ", options.format); @@ -1262,23 +1283,7 @@ struct Strftime { // Convert string representations of timestamps in arbitrary format to timestamps const std::string GetZone(const std::string& format) { - // Check for use of %z or %Z - size_t cur = 0; - size_t count = 0; - std::string zone = ""; - while (cur < format.size() - 1) { - if (format[cur] == '%') { - count++; - if (format[cur + 1] == 'z' && count % 2 == 1) { - zone = "UTC"; - break; - } - } else { - count = 0; - } - cur++; - } - return zone; + return FormatHasZone(format, /*include_zone_name=*/false) ? "UTC" : ""; } template diff --git a/cpp/src/arrow/util/value_parsing.cc b/cpp/src/arrow/util/value_parsing.cc index 0cc71f276df..4161c958ce5 100644 --- a/cpp/src/arrow/util/value_parsing.cc +++ b/cpp/src/arrow/util/value_parsing.cc @@ -78,11 +78,14 @@ class StrptimeTimestampParser : public TimestampParser { public: explicit StrptimeTimestampParser(std::string format) : format_(std::move(format)), have_zone_offset_(false) { - // Check for use of %z + // Check for use of %z or %Ez size_t cur = 0; while (cur < format_.size()) { if (format_[cur] == '%') { - if (cur + 1 < format_.size() && format_[cur + 1] == 'z') { + if (cur + 1 < format_.size() && + (format_[cur + 1] == 'z' || + (format_[cur + 1] == 'E' && cur + 2 < format_.size() && + format_[cur + 2] == 'z'))) { have_zone_offset_ = true; break; } diff --git a/cpp/src/arrow/util/value_parsing.h b/cpp/src/arrow/util/value_parsing.h index 195cdc843ac..0097291593a 100644 --- a/cpp/src/arrow/util/value_parsing.h +++ b/cpp/src/arrow/util/value_parsing.h @@ -553,6 +553,35 @@ static inline bool ParseHH_MM_SS(const char* s, Duration* out) { return true; } +template +static inline bool ParseHHMMSS(const char* s, Duration* out) { + uint8_t hours = 0; + uint8_t minutes = 0; + uint8_t seconds = 0; + if (ARROW_PREDICT_FALSE(!ParseUnsigned(s + 0, 2, &hours))) { + return false; + } + if (ARROW_PREDICT_FALSE(!ParseUnsigned(s + 2, 2, &minutes))) { + return false; + } + if (ARROW_PREDICT_FALSE(!ParseUnsigned(s + 4, 2, &seconds))) { + return false; + } + if (ARROW_PREDICT_FALSE(hours >= 24)) { + return false; + } + if (ARROW_PREDICT_FALSE(minutes >= 60)) { + return false; + } + if (ARROW_PREDICT_FALSE(seconds >= 60)) { + return false; + } + *out = std::chrono::duration_cast(std::chrono::hours(hours) + + std::chrono::minutes(minutes) + + std::chrono::seconds(seconds)); + return true; +} + static inline bool ParseSubSeconds(const char* s, size_t length, TimeUnit::type unit, uint32_t* out) { // The decimal point has been peeled off at this point @@ -669,7 +698,7 @@ static inline bool ParseTimestampISO8601(const char* s, size_t length, // We allow the following zone offset formats: // - (none) // - Z - // - [+-]HH(:?MM)? + // - [+-]HH(:?MM(:?SS)?)? // // We allow the following formats for all units: // - "YYYY-MM-DD" @@ -712,23 +741,25 @@ static inline bool ParseTimestampISO8601(const char* s, size_t length, if (s[length - 1] == 'Z') { --length; if (out_zone_offset_present) *out_zone_offset_present = true; - } else if (s[length - 3] == '+' || s[length - 3] == '-') { - // [+-]HH - length -= 3; - if (ARROW_PREDICT_FALSE(!detail::ParseHH(s + length + 1, &zone_offset))) { + } else if (length >= 19 && (s[length - 9] == '+' || s[length - 9] == '-') && + (s[length - 6] == ':') && (s[length - 3] == ':')) { + // [+-]HH:MM:SS + length -= 9; + if (ARROW_PREDICT_FALSE(!detail::ParseHH_MM_SS(s + length + 1, &zone_offset))) { return false; } if (s[length] == '+') zone_offset *= -1; if (out_zone_offset_present) *out_zone_offset_present = true; - } else if (s[length - 5] == '+' || s[length - 5] == '-') { - // [+-]HHMM - length -= 5; - if (ARROW_PREDICT_FALSE(!detail::ParseHHMM(s + length + 1, &zone_offset))) { + } else if (length >= 17 && (s[length - 7] == '+' || s[length - 7] == '-')) { + // [+-]HHMMSS + length -= 7; + if (ARROW_PREDICT_FALSE(!detail::ParseHHMMSS(s + length + 1, &zone_offset))) { return false; } if (s[length] == '+') zone_offset *= -1; if (out_zone_offset_present) *out_zone_offset_present = true; - } else if ((s[length - 6] == '+' || s[length - 6] == '-') && (s[length - 3] == ':')) { + } else if (length >= 16 && (s[length - 6] == '+' || s[length - 6] == '-') && + (s[length - 3] == ':')) { // [+-]HH:MM length -= 6; if (ARROW_PREDICT_FALSE(!detail::ParseHH_MM(s + length + 1, &zone_offset))) { @@ -736,6 +767,22 @@ static inline bool ParseTimestampISO8601(const char* s, size_t length, } if (s[length] == '+') zone_offset *= -1; if (out_zone_offset_present) *out_zone_offset_present = true; + } else if (length >= 15 && (s[length - 5] == '+' || s[length - 5] == '-')) { + // [+-]HHMM + length -= 5; + if (ARROW_PREDICT_FALSE(!detail::ParseHHMM(s + length + 1, &zone_offset))) { + return false; + } + if (s[length] == '+') zone_offset *= -1; + if (out_zone_offset_present) *out_zone_offset_present = true; + } else if (length >= 13 && (s[length - 3] == '+' || s[length - 3] == '-')) { + // [+-]HH + length -= 3; + if (ARROW_PREDICT_FALSE(!detail::ParseHH(s + length + 1, &zone_offset))) { + return false; + } + if (s[length] == '+') zone_offset *= -1; + if (out_zone_offset_present) *out_zone_offset_present = true; } seconds_type seconds_since_midnight; diff --git a/cpp/src/arrow/util/value_parsing_test.cc b/cpp/src/arrow/util/value_parsing_test.cc index b61f777685b..23a7901a8b6 100644 --- a/cpp/src/arrow/util/value_parsing_test.cc +++ b/cpp/src/arrow/util/value_parsing_test.cc @@ -722,6 +722,10 @@ TEST(StringConversion, ToTimestampDateTime_ISO8601) { AssertConversionFails(type, "1970-01-01 00:00:00-00000"); AssertConversionFails(type, "1970-01-01 00:00:00+2400"); AssertConversionFails(type, "1970-01-01 00:00:00+00:99"); + AssertConversionFails(type, "1970-01-01 00:00:00+00:00:60"); + AssertConversionFails(type, "1970-01-01 00:00:00+000060"); + AssertConversionFails(type, "1970-01-01 00:00:00+240000"); + AssertConversionFails(type, "1970-01-01 00:00:00+24:00:00"); } { TimestampType type{TimeUnit::MILLI}; @@ -742,6 +746,10 @@ TEST(StringConversion, ToTimestampDateTime_ISO8601) { AssertConversion(type, "2018-11-13 17:11:10.123-01", 1542129070123LL + 3600000LL); AssertConversion(type, "2018-11-13 17:11:10.123-0117", 1542129070123LL + 4620000LL); AssertConversion(type, "2018-11-13 17:11:10.123-01:17", 1542129070123LL + 4620000LL); + AssertConversion(type, "2018-11-13 17:11:10.123+004430", 1542129070123LL - 2670000LL); + AssertConversion(type, "2018-11-13 17:11:10.123+00:44:30", 1542129070123LL - 2670000LL); + AssertConversion(type, "2018-11-13 17:11:10.123-004430", 1542129070123LL + 2670000LL); + AssertConversion(type, "2018-11-13 17:11:10.123-00:44:30", 1542129070123LL + 2670000LL); // Invalid subseconds AssertConversionFails(type, "1900-02-28 12:34:56.1234");