Skip to content

fix(bigquery-jdbc): format TIMESTAMP as UTC in getString and add multi-zone TIMESTAMP(12) tests #14666

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
keshavdandeva wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(bigquery-jdbc): format TIMESTAMP as UTC in getString and add multi-zone TIMESTAMP(12) tests #14666

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseArray.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Struct;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Base64;
import java.util.Map;
Expand Down Expand Up @@ -176,6 +177,20 @@ public String toString() {
element == null ? "null" : Base64.getEncoder().encodeToString((byte[]) element))
.collect(Collectors.joining(", ", "[", "]"));
}
if (this.schema.getType().getStandardType() == StandardSQLTypeName.TIMESTAMP) {
return Arrays.stream(array)
.map(
element -> {
if (element == null) {
return "null";
}
if (element instanceof Timestamp) {
return BigQueryTemporalUtility.formatTimestampToUtcString((Timestamp) element);
}
return element.toString();
})
.collect(Collectors.joining(", ", "[", "]"));
}
return Arrays.deepToString(array);
} catch (SQLException e) {
LOG.warning("Error converting array to string");
Expand Down
10 changes: 4 additions & 6 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,9 @@ private StandardSQLTypeName getStandardSQLTypeName(int columnIndex) throws SQLEx
throw logAndCreateException("Invalid column index: " + columnIndex);
}
Field field = this.schemaFieldList.get(columnIndex - 1);
if (BigQueryBaseArray.isArray(field)) {
return StandardSQLTypeName.ARRAY;
}
return field.getType().getStandardType();
}
}
Expand Down Expand Up @@ -338,12 +341,7 @@ protected int getColumnIndex(String columnLabel) throws SQLException {
@Override
public String getString(int columnIndex) throws SQLException {
LOG.finestTrace("getString");
try {
Object value = getObject(columnIndex);
return BigQueryTypeRegistry.convert(value, String.class);
} catch (BigQueryJdbcException e) {
throw createCoercionException(columnIndex, String.class, e);
}
return getObject(columnIndex, String.class);
}

@Override
Expand Down
8 changes: 7 additions & 1 deletion java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseStruct.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ public String toString() {
if (i > 0) {
sb.append(",");
}
String fieldName = schema.get(i).getName();
Field field = schema.get(i);
String fieldName = field.getName();
Object value = attributes[i];

sb.append("\"").append(fieldName.replace("\"", "\\\"")).append("\":");
Expand All @@ -89,6 +90,11 @@ public String toString() {
} else if (value instanceof String || value instanceof org.apache.arrow.vector.util.Text) {
String stringValue = value.toString().replace("\"", "\\\"");
sb.append("\"").append(stringValue).append("\"");
} else if (value instanceof Timestamp
&& field.getType().getStandardType() == StandardSQLTypeName.TIMESTAMP) {
sb.append("\"")
.append(BigQueryTemporalUtility.formatTimestampToUtcString((Timestamp) value))
.append("\"");
} else if (value instanceof Timestamp || value instanceof Date || value instanceof Time) {
sb.append("\"").append(value.toString()).append("\"");
} else if (value instanceof byte[]) {
Expand Down
23 changes: 23 additions & 0 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTemporalUtility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.util.Calendar;
import java.util.TimeZone;

Expand All @@ -49,6 +51,12 @@ final class BigQueryTemporalUtility {
private static final BigDecimal MICROS_PER_SECOND = new BigDecimal("1000000");
private static final DateTimeFormatter BASE_FORMATTER =
DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.");
private static final DateTimeFormatter UTC_TIMESTAMP_TO_STRING_FORMATTER =
new DateTimeFormatterBuilder()
.appendPattern("uuuu-MM-dd HH:mm:ss.")
.appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, false)
.toFormatter()
.withZone(ZoneOffset.UTC);
static final String TIMESTAMP_PICOSECONDS_TYPE_NAME = "TIMESTAMP_PICOSECONDS";

private BigQueryTemporalUtility() {}
Expand Down Expand Up @@ -333,6 +341,17 @@ static String formatTimestampStringFromMicroseconds(
return sb.toString();
}

/**
* Formats a {@link Timestamp} in UTC using {@link Timestamp#toString()}'s fractional-second
* convention (1 to 9 digits with trailing zeros stripped after the first decimal place).
*/
static String formatTimestampToUtcString(Timestamp ts) {
if (ts == null) {
return null;
}
return UTC_TIMESTAMP_TO_STRING_FORMATTER.format(ts.toInstant());
}

/**
* Formats a timestamp value (which may be a {@link Long} epoch microsecond, an ISO-8601 string, a
* {@link java.sql.Timestamp}, or an epoch decimal string) into a standard UTC JDBC timestamp
Expand All @@ -346,6 +365,10 @@ static String formatTimestampValue(Object value, boolean enableTimestampPicos)
if (value instanceof Long) {
return formatTimestampStringFromMicroseconds((Long) value, enableTimestampPicos);
}
if (value instanceof Timestamp) {
return formatTimestampStringFromIso(
formatTimestampToUtcString((Timestamp) value), enableTimestampPicos);
}
String str = value.toString();
if (str.indexOf(':') >= 0) {
return formatTimestampStringFromIso(str, enableTimestampPicos);
Expand Down
6 changes: 5 additions & 1 deletion java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,8 @@ static TypeDescriptor<?> createTimestampDescriptor() {
// Modern fast-path: Bypass intermediate object creation for JSR-310 targets
if (targetClass == Instant.class
|| targetClass == OffsetDateTime.class
|| targetClass == ZonedDateTime.class) {
|| targetClass == ZonedDateTime.class
|| targetClass == String.class) {
Instant instant;
if (val instanceof Instant) {
instant = (Instant) val;
Expand Down Expand Up @@ -361,6 +362,9 @@ static TypeDescriptor<?> createTimestampDescriptor() {
if (targetClass == ZonedDateTime.class) {
return instant.atZone(ZoneOffset.UTC);
}
if (targetClass == String.class) {
return TIMESTAMP_FORMATTER.format(instant.atOffset(ZoneOffset.UTC));
}
}

// Legacy path: Box values into java.sql.Timestamp
Expand Down
113 changes: 113 additions & 0 deletions ...igquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.Arrays;
Expand Down Expand Up @@ -75,6 +76,7 @@
import org.apache.arrow.vector.complex.impl.UnionListWriter;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.JsonStringHashMap;
import org.apache.arrow.vector.util.Text;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -658,4 +660,115 @@ public void testCalendarGettersOnTimestampIgnoreCalendar(String jvmZone)
TimeZone.setDefault(originalDefault);
}
}

@ParameterizedTest
@ValueSource(strings = {"UTC", "America/New_York", "Asia/Tokyo"})
public void testGetStringOnTimestampAndDatetimeAcrossJvmZones(String jvmZone)
throws SQLException, IOException {
TimeZone originalDefault = TimeZone.getDefault();
try {
TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of(jvmZone)));
Instant ref = Instant.parse("2025-01-01T12:34:56.123Z");
BigQueryArrowResultSet rs = createTemporalArrowResultSet(ref);
assertThat(rs.next()).isTrue();

assertThat(rs.getString("ts")).isEqualTo("2025-01-01 12:34:56.123000");
assertThat(rs.getObject("ts", String.class)).isEqualTo("2025-01-01 12:34:56.123000");
assertThat(rs.getString("dt")).isEqualTo("2025-01-01 12:34:56.123000");

long tsMicros = ref.getEpochSecond() * 1_000_000L + ref.getNano() / 1_000L;
LocalDateTime dtLocal = LocalDateTime.of(2025, 1, 1, 12, 34, 56, 123_000_000);

Field tsArraySchema =
Field.newBuilder("ts_array", StandardSQLTypeName.TIMESTAMP)
.setMode(Field.Mode.REPEATED)
.build();
JsonStringArrayList<Long> tsList = new JsonStringArrayList<>();
tsList.add(tsMicros);
BigQueryArrowArray tsArrowArray = new BigQueryArrowArray(tsArraySchema, tsList);
assertThat(tsArrowArray.toString()).isEqualTo("[2025-01-01 12:34:56.123]");
try (ResultSet nestedTsRs = tsArrowArray.getResultSet()) {
assertThat(nestedTsRs.next()).isTrue();
assertThat(nestedTsRs.getString(2)).isEqualTo("2025-01-01 12:34:56.123000");
}

Field dtArraySchema =
Field.newBuilder("dt_array", StandardSQLTypeName.DATETIME)
.setMode(Field.Mode.REPEATED)
.build();
JsonStringArrayList<LocalDateTime> dtList = new JsonStringArrayList<>();
dtList.add(dtLocal);
BigQueryArrowArray dtArrowArray = new BigQueryArrowArray(dtArraySchema, dtList);
assertThat(dtArrowArray.toString()).isEqualTo("[2025-01-01 12:34:56.123]");

FieldList structSchema =
FieldList.of(
Field.of("ts", StandardSQLTypeName.TIMESTAMP),
Field.of("dt", StandardSQLTypeName.DATETIME));
JsonStringHashMap<String, Object> structValues = new JsonStringHashMap<>();
structValues.put("ts", tsMicros);
structValues.put("dt", dtLocal);
BigQueryArrowStruct arrowStruct = new BigQueryArrowStruct(structSchema, structValues);
assertThat(arrowStruct.toString())
.isEqualTo("{\"ts\":\"2025-01-01 12:34:56.123\",\"dt\":\"2025-01-01 12:34:56.123\"}");
} finally {
TimeZone.setDefault(originalDefault);
}
}

@ParameterizedTest
@ValueSource(strings = {"UTC", "America/New_York", "Asia/Tokyo"})
public void testTimestampPicosAccessorsAcrossJvmZones(String jvmZone)
throws SQLException, IOException {
TimeZone originalDefault = TimeZone.getDefault();
try {
TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of(jvmZone)));
VectorSchemaRoot root = getPicosVectorSchemaRoot();
ArrowRecordBatch batch =
ArrowRecordBatch.newBuilder()
.setSerializedRecordBatch(serializeVectorSchemaRoot(root))
.build();
BlockingQueue<BigQueryArrowBatchWrapper> picosBuffer = new LinkedBlockingDeque<>();
picosBuffer.add(BigQueryArrowBatchWrapper.of(batch));
picosBuffer.add(BigQueryArrowBatchWrapper.of(null, true));

BigQueryStatement picosStmt = mock(BigQueryStatement.class);
doReturn(true).when(picosStmt).isEnableTimestampPicos();

Schema schema =
Schema.of(
Field.newBuilder("picosTimestamp", StandardSQLTypeName.TIMESTAMP)
.setTimestampPrecision(12L)
.build());
ArrowSchema arrowSchema =
ArrowSchema.newBuilder().setSerializedSchema(serializeSchema(root.getSchema())).build();

BigQueryArrowResultSet rs =
BigQueryArrowResultSet.of(
schema, arrowSchema, 1, picosStmt, picosBuffer, mock(Future.class), null);
assertThat(rs.next()).isTrue();

Instant expectedInstant = Instant.parse("2026-04-08T10:00:00.123456789Z");
ZonedDateTime expectedLocal = expectedInstant.atZone(ZoneId.of(jvmZone));
Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/London"));

assertThat(rs.getString("picosTimestamp")).isEqualTo("2026-04-08 10:00:00.123456789123");
assertThat(rs.getTimestamp("picosTimestamp").toInstant()).isEqualTo(expectedInstant);
assertThat(rs.getTimestamp("picosTimestamp", cal).toInstant()).isEqualTo(expectedInstant);
assertThat(rs.getObject("picosTimestamp", Instant.class)).isEqualTo(expectedInstant);
assertThat(rs.getObject("picosTimestamp", OffsetDateTime.class).toInstant())
.isEqualTo(expectedInstant);
assertThat(rs.getObject("picosTimestamp", LocalDateTime.class))
.isEqualTo(expectedLocal.toLocalDateTime());
assertThat(rs.getDate("picosTimestamp").toLocalDate()).isEqualTo(expectedLocal.toLocalDate());
assertThat(rs.getDate("picosTimestamp", cal).toLocalDate())
.isEqualTo(expectedLocal.toLocalDate());
assertThat(rs.getTime("picosTimestamp").toLocalTime())
.isEqualTo(expectedLocal.toLocalTime().withNano(0));
assertThat(rs.getTime("picosTimestamp", cal).toLocalTime())
.isEqualTo(expectedLocal.toLocalTime().withNano(0));
} finally {
TimeZone.setDefault(originalDefault);
}
}
}
Loading
Loading