Skip to content

Commit

Permalink
Use recommended semantics for getObject of date/time types (#181)
Browse files Browse the repository at this point in the history
  • Loading branch information
@wnob
wnob authored Dec 11, 2023
1 parent 8f9b83b commit d12cdf1
Show file tree
Hide file tree
Showing 4 changed files with 322 additions and 138 deletions.
160 changes: 52 additions & 108 deletions src/main/java/net/starschema/clouddb/jdbc/BQForwardOnlyResultSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,6 @@
import java.sql.Statement;
import java.sql.Time;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.*;
import javax.annotation.Nullable;
import org.slf4j.Logger;
Expand Down Expand Up @@ -127,9 +123,6 @@ public class BQForwardOnlyResultSet implements java.sql.ResultSet {
*/
private int Cursor = -1;

private final DateTimeFormatter TIMESTAMP_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withZone(ZoneId.of("UTC"));

public BQForwardOnlyResultSet(
Bigquery bigquery,
String projectId,
Expand Down Expand Up @@ -248,7 +241,7 @@ public BQForwardOnlyResultSet(
*/
public Object getObject(int columnIndex) throws SQLException {

String result = getString(columnIndex, false);
String result = rawString(columnIndex);

if (this.wasNull()) {
return null;
Expand All @@ -274,16 +267,26 @@ public Object getObject(int columnIndex) throws SQLException {
if (Columntype.equals("INTEGER")) {
return toLong(result);
}
if (Columntype.equals("DATETIME")) {
// A BigQuery DATETIME is essentially defined by its string representation;
// the "clock-calendar parameters" comprising year, month, day, hour, minute, etc.
// On the other hand, a [java.sql.Timestamp] object is defined as a global instant, similar
// to BQ TIMESTAMP. It has a [toString] method that interprets that instant in the system
// default time zone. Thus, in order to produce a [Timestamp] object whose [toString] method
// has the correct result, we must adjust the value of the instant according to the system
// default time zone (passing a null Calendar uses the system default).
return DateTimeUtils.parseDateTime(result, null);
}
if (Columntype.equals("TIMESTAMP")) {
return toTimestamp(result, null);
// A BigQuery TIMESTAMP is defined as a global instant in time, so when we create the
// [java.sql.Timestamp] object to represent it, we must not make any time zone adjustment.
return DateTimeUtils.parseTimestamp(result);
}
if (Columntype.equals("DATE")) {
return toDate(result, null);
return DateTimeUtils.parseDate(result, null);
}
if (Columntype.equals("DATETIME")) {
// Date time represents a "clock face" time and so should NOT be processed into an actual
// time
return result;
if (Columntype.equals("TIME")) {
return DateTimeUtils.parseTime(result, null);
}
if (Columntype.equals("NUMERIC")) {
return toBigDecimal(result);
Expand Down Expand Up @@ -321,71 +324,13 @@ private Long toLong(String value) throws SQLException {
}
}

/** Parse date/time types */
private Timestamp toTimestamp(String value, Calendar cal) throws SQLException {
try {
long dbValue =
new BigDecimal(value)
.movePointRight(3)
.longValue(); // movePointRight(3) = *1000 (done before rounding) - from seconds
// (BigQuery specifications) to milliseconds (required by java).
// Precision under millisecond is discarded (BigQuery supports
// micro-seconds)
if (cal == null) {
cal =
Calendar.getInstance(
TimeZone.getTimeZone(
"UTC")); // The time zone of the server that host the JVM should NOT impact the
// results. Use UTC calendar instead (which wont apply any correction,
// whatever the time zone of the data)
}

Calendar dbCal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
dbCal.setTimeInMillis(dbValue);
cal.set(Calendar.YEAR, dbCal.get(Calendar.YEAR));
cal.set(Calendar.MONTH, dbCal.get(Calendar.MONTH));
cal.set(Calendar.DAY_OF_MONTH, dbCal.get(Calendar.DAY_OF_MONTH));
cal.set(Calendar.HOUR_OF_DAY, dbCal.get(Calendar.HOUR_OF_DAY));
cal.set(Calendar.MINUTE, dbCal.get(Calendar.MINUTE));
cal.set(Calendar.SECOND, dbCal.get(Calendar.SECOND));
cal.set(Calendar.MILLISECOND, dbCal.get(Calendar.MILLISECOND));
return new Timestamp(cal.getTime().getTime());
} catch (NumberFormatException e) {
// before giving up, check to see if we've been given a 'time' value without a
// date, e.g. from current_time(), and if we have, try to parse it
try {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String dateStringFromValue = ("1970-01-01 " + value).substring(0, 19);
java.util.Date parsedDate = dateFormat.parse(dateStringFromValue);
return new java.sql.Timestamp(parsedDate.getTime());
} catch (Exception e2) {
throw new BQSQLException(e);
}
}
}

// Secondary converters

/** Parse integral or floating types with (virtually) infinite precision */
private BigDecimal toBigDecimal(String value) {
return new BigDecimal(value);
}

static Date toDate(String value, Calendar cal) throws SQLException {
// Dates in BigQuery come back in the YYYY-MM-DD format
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
try {
java.util.Date date = sdf.parse(value);
return new java.sql.Date(date.getTime());
} catch (java.text.ParseException e) {
throw new BQSQLException(e);
}
}

private Time toTime(String value, Calendar cal) throws SQLException {
return new java.sql.Time(toTimestamp(value, cal).getTime());
}

@Override
/**
* Returns the current rows Data at the given index as String
Expand All @@ -396,10 +341,18 @@ private Time toTime(String value, Calendar cal) throws SQLException {
* is unsupported
*/
public String getString(int columnIndex) throws SQLException {
return getString(columnIndex, true);
String rawString = rawString(columnIndex);

BQResultsetMetaData metadata = getBQResultsetMetaData();
if (metadata.getColumnTypeName(columnIndex).equals("TIMESTAMP")
&& !metadata.getColumnMode(columnIndex).equals("REPEATED")) {
return DateTimeUtils.formatTimestamp(rawString);
}

return rawString;
}

private String getString(int columnIndex, boolean formatTimestamps) throws SQLException {
private String rawString(int columnIndex) throws SQLException {
// to make the logfiles smaller!
// logger.debug("Function call getString columnIndex is: " + String.valueOf(columnIndex));
this.closestrm();
Expand All @@ -410,21 +363,21 @@ private String getString(int columnIndex, boolean formatTimestamps) throws SQLEx
throw new BQSQLException("ColumnIndex is not valid");
}

if (this.rowsofResult == null) throw new BQSQLException("Invalid position!");
if (this.rowsofResult == null) {
throw new BQSQLException("Invalid position!");
}

// Since the results came from BigQuery's JSON API,
// the only types we'll ever see for resultObject are JSON-supported types,
// i.e. strings, numbers, arrays, booleans, or null.
// Many standard Java types, like timestamps, will be represented as strings.
Object resultObject = this.rowsofResult.get(this.Cursor).getF().get(columnIndex - 1).getV();

if (Data.isNull(resultObject)) {
this.wasnull = true;
return null;
}
this.wasnull = false;
if (!this.getBQResultsetMetaData().getColumnMode(columnIndex).equals("REPEATED")
&& formatTimestamps
&& getMetaData().getColumnTypeName(columnIndex).equals("TIMESTAMP")) {
Instant instant =
Instant.ofEpochMilli(
(new BigDecimal((String) resultObject).movePointRight(3)).longValue());
return TIMESTAMP_FORMATTER.format(instant);
}
if (resultObject instanceof List || resultObject instanceof Map) {
Object resultTransformedWithSchema =
smartTransformResult(resultObject, schema.getFields().get(columnIndex - 1));
Expand Down Expand Up @@ -913,11 +866,7 @@ public String getCursorName() throws SQLException {
/** {@inheritDoc} */
@Override
public Date getDate(int columnIndex) throws SQLException {
String value = this.getString(columnIndex);
if (this.wasNull()) {
return null;
}
return toDate(value, null);
return getDate(columnIndex, null);
}

/** {@inheritDoc} */
Expand All @@ -927,14 +876,13 @@ public Date getDate(int columnIndex, Calendar cal) throws SQLException {
if (this.wasNull()) {
return null;
}
return toDate(value, cal);
return DateTimeUtils.parseDate(value, cal);
}

/** {@inheritDoc} */
@Override
public Date getDate(String columnLabel) throws SQLException {
int columnIndex = this.findColumn(columnLabel);
return this.getDate(columnIndex);
return getDate(columnLabel, null);
}

/** {@inheritDoc} */
Expand Down Expand Up @@ -1336,11 +1284,7 @@ public String getString(String columnLabel) throws SQLException {
/** {@inheritDoc} */
@Override
public Time getTime(int columnIndex) throws SQLException {
String value = this.getString(columnIndex);
if (this.wasNull()) {
return null;
}
return toTime(value, null);
return getTime(columnIndex, null);
}

/** {@inheritDoc} */
Expand All @@ -1350,14 +1294,13 @@ public Time getTime(int columnIndex, Calendar cal) throws SQLException {
if (this.wasNull()) {
return null;
}
return toTime(value, cal);
return DateTimeUtils.parseTime(value, cal);
}

/** {@inheritDoc} */
@Override
public Time getTime(String columnLabel) throws SQLException {
int columnIndex = this.findColumn(columnLabel);
return this.getTime(columnIndex);
return getTime(columnLabel, null);
}

/** {@inheritDoc} */
Expand All @@ -1370,28 +1313,29 @@ public Time getTime(String columnLabel, Calendar cal) throws SQLException {
/** {@inheritDoc} */
@Override
public Timestamp getTimestamp(int columnIndex) throws SQLException {
String value = this.getString(columnIndex);
if (this.wasNull()) {
return null;
}
return toTimestamp(value, null);
return getTimestamp(columnIndex, null);
}

/** {@inheritDoc} */
@Override
public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException {
String value = this.getString(columnIndex);
String value = rawString(columnIndex);
if (this.wasNull()) {
return null;
}
return toTimestamp(value, cal);
// Both the TIMESTAMP and DATETIME objects support JDBC's getTimestamp method.
// DATETIME in BigQuery is analogous to TIMESTAMP in ISO SQL.
// TIMESTAMP in BigQuery is analogous to TIMESTAMP WITH LOCAL TIME ZONE in ISO SQL.
String columnType = this.schema.getFields().get(columnIndex - 1).getType();
return "TIMESTAMP".equals(columnType)
? DateTimeUtils.parseTimestamp(value)
: DateTimeUtils.parseDateTime(value, cal);
}

/** {@inheritDoc} */
@Override
public Timestamp getTimestamp(String columnLabel) throws SQLException {
int columnIndex = this.findColumn(columnLabel);
return this.getTimestamp(columnIndex);
return getTimestamp(columnLabel, null);
}

/** {@inheritDoc} */
Expand Down
31 changes: 19 additions & 12 deletions src/main/java/net/starschema/clouddb/jdbc/BQScrollableResultSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@
*/
package net.starschema.clouddb.jdbc;

import static net.starschema.clouddb.jdbc.BQForwardOnlyResultSet.toDate;

import com.google.api.client.util.Data;
import com.google.api.services.bigquery.model.BiEngineReason;
import com.google.api.services.bigquery.model.GetQueryResultsResponse;
Expand All @@ -36,7 +34,6 @@
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.util.List;
import javax.annotation.Nullable;

Expand Down Expand Up @@ -204,21 +201,31 @@ public Object getObject(int columnIndex) throws SQLException {
if (Columntype.equals("INTEGER")) {
return Long.parseLong(result);
}
if (Columntype.equals("DATETIME")) {
// A BigQuery DATETIME is essentially defined by its string representation;
// the "clock-calendar parameters" comprising year, month, day, hour, minute, etc.
// On the other hand, a [java.sql.Timestamp] object is defined as a global instant,
// similar to BQ TIMESTAMP. It has a [toString] method that interprets that instant in the
// system default time zone. Thus, in order to produce a [Timestamp] object whose
// [toString] method has the correct result, we must adjust the value of the instant
// according to the system default time zone (passing a null Calendar uses the system
// default).
return DateTimeUtils.parseDateTime(result, null);
}
if (Columntype.equals("TIMESTAMP")) {
long val = new BigDecimal(result).longValue() * 1000;
return new Timestamp(val);
// A BigQuery TIMESTAMP is defined as a global instant in time, so when we create the
// [java.sql.Timestamp] object to represent it, we must not make any time zone adjustment.
return DateTimeUtils.parseTimestamp(result);
}
if (Columntype.equals("DATETIME")) {
// Date time represents a "clock face" time and so should NOT be processed into an actual
// time
return result;
if (Columntype.equals("DATE")) {
return DateTimeUtils.parseDate(result, null);
}
if (Columntype.equals("TIME")) {
return DateTimeUtils.parseTime(result, null);
}
if (Columntype.equals("NUMERIC")) {
return new BigDecimal(result);
}
if (Columntype.equals("DATE")) {
return toDate(result, null);
}
throw new BQSQLException("Unsupported Type");
} catch (NumberFormatException e) {
throw new BQSQLException(e);
Expand Down
Loading

0 comments on commit d12cdf1

Please sign in to comment.