diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java index 716ca448c274..d44c5e1626fd 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java @@ -390,7 +390,7 @@ public HiveTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName.getTableName(), table.get().getParameters(), getPartitionKeyColumnHandles(table.get(), typeManager), - getRegularColumnHandles(table.get(), typeManager, getTimestampPrecision(session).getPrecision()), + getRegularColumnHandles(table.get(), typeManager, getTimestampPrecision(session)), getHiveBucketHandle(session, table.get(), typeManager)); } @@ -566,7 +566,7 @@ private ConnectorTableMetadata doGetTableMetadata(ConnectorSession session, Sche Function metadataGetter = columnMetadataGetter(table); ImmutableList.Builder columns = ImmutableList.builder(); - for (HiveColumnHandle columnHandle : hiveColumnHandles(table, typeManager, getTimestampPrecision(session).getPrecision())) { + for (HiveColumnHandle columnHandle : hiveColumnHandles(table, typeManager, getTimestampPrecision(session))) { columns.add(metadataGetter.apply(columnHandle)); } @@ -710,7 +710,7 @@ public Map getColumnHandles(ConnectorSession session, Conn SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName(); Table table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(tableName)); - return hiveColumnHandles(table, typeManager, getTimestampPrecision(session).getPrecision()).stream() + return hiveColumnHandles(table, typeManager, getTimestampPrecision(session)).stream() .collect(toImmutableMap(HiveColumnHandle::getName, identity())); } @@ -849,7 +849,7 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe throw new PrestoException(NOT_SUPPORTED, "Bucketing/Partitioning columns not supported when Avro schema url is set"); } - validateTimestampColumns(tableMetadata.getColumns()); + validateTimestampColumns(tableMetadata.getColumns(), getTimestampPrecision(session)); List columnHandles = getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy)); HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); Map tableProperties = getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsContext(session, schemaName, tableName)); @@ -1236,11 +1236,11 @@ public void finishStatisticsCollection(ConnectorSession session, ConnectorTableH List partitionColumnNames = partitionColumns.stream() .map(Column::getName) .collect(toImmutableList()); - // TODO: revisit when handling write path - List hiveColumnHandles = hiveColumnHandles(table, typeManager, TimestampType.DEFAULT_PRECISION); + HiveTimestampPrecision timestampPrecision = getTimestampPrecision(session); + List hiveColumnHandles = hiveColumnHandles(table, typeManager, timestampPrecision); Map columnTypes = hiveColumnHandles.stream() .filter(columnHandle -> !columnHandle.isHidden()) - .collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager))); + .collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager, timestampPrecision))); Map, ComputedStatistics> computedStatisticsMap = createComputedStatisticsToPartitionMap(computedStatistics, partitionColumnNames, columnTypes); @@ -1568,7 +1568,7 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl } } - List handles = hiveColumnHandles(table, typeManager, getTimestampPrecision(session).getPrecision()).stream() + List handles = hiveColumnHandles(table, typeManager, getTimestampPrecision(session)).stream() .filter(columnHandle -> !columnHandle.isHidden()) .collect(toList()); @@ -2447,7 +2447,7 @@ else if (hiveTableHandle.isInAcidTransaction()) { @Override public Optional getNewTableLayout(ConnectorSession session, ConnectorTableMetadata tableMetadata) { - validateTimestampColumns(tableMetadata.getColumns()); + validateTimestampColumns(tableMetadata.getColumns(), getTimestampPrecision(session)); validatePartitionColumns(tableMetadata); validateBucketColumns(tableMetadata); validateColumns(tableMetadata); @@ -2502,11 +2502,12 @@ public TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession private TableStatisticsMetadata getStatisticsCollectionMetadata(List columns, List partitionedBy, Optional> analyzeColumns, boolean includeRowCount) { - validateTimestampColumns(columns); Set columnStatistics = columns.stream() .filter(column -> !partitionedBy.contains(column.getName())) .filter(column -> !column.isHidden()) .filter(column -> analyzeColumns.isEmpty() || analyzeColumns.get().contains(column.getName())) + // TODO: we only support stats collection at millis precision for now (https://github.com/prestosql/presto/issues/5170) + .filter(column -> !(column.getType() instanceof TimestampType) || column.getType() == TIMESTAMP_MILLIS) .map(this::getColumnStatisticMetadata) .flatMap(List::stream) .collect(toImmutableSet()); @@ -2720,14 +2721,14 @@ private static void validateColumns(ConnectorTableMetadata tableMetadata) } } - // temporary, until variable precision timestamps are supported on write - private static void validateTimestampColumns(List columns) + // TODO validate timestamps in structural types (https://github.com/prestosql/presto/issues/5195) + private static void validateTimestampColumns(List columns, HiveTimestampPrecision timestampPrecision) { for (ColumnMetadata column : columns) { Type type = column.getType(); if (type instanceof TimestampType) { - if (type != TIMESTAMP_MILLIS) { - throw new PrestoException(NOT_SUPPORTED, "CREATE TABLE, INSERT and ANALYZE are not supported with requested timestamp precision: " + type); + if (((TimestampType) type).getPrecision() != timestampPrecision.getPrecision()) { + throw new PrestoException(NOT_SUPPORTED, format("Incorrect timestamp precision for %s; the configured precision is %s", type, timestampPrecision)); } } } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveType.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveType.java index 1dba2200c0c7..a3c4c5dca0b4 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveType.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveType.java @@ -127,12 +127,12 @@ public Type getType(TypeManager typeManager) return typeManager.getType(getTypeSignature()); } - public Type getType(TypeManager typeManager, int timestampPrecision) + public Type getType(TypeManager typeManager, HiveTimestampPrecision timestampPrecision) { Type tentativeType = typeManager.getType(getTypeSignature()); // TODO: handle timestamps in structural types (https://github.com/prestosql/presto/issues/5195) if (tentativeType instanceof TimestampType) { - return TimestampType.createTimestampType(timestampPrecision); + return TimestampType.createTimestampType(timestampPrecision.getPrecision()); } return tentativeType; } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriterFactory.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriterFactory.java index 73c27793a625..3bfaeb603859 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriterFactory.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriterFactory.java @@ -84,6 +84,7 @@ import static io.prestosql.plugin.hive.HiveSessionProperties.getCompressionCodec; import static io.prestosql.plugin.hive.HiveSessionProperties.getInsertExistingPartitionsBehavior; import static io.prestosql.plugin.hive.HiveSessionProperties.getTemporaryStagingDirectoryPath; +import static io.prestosql.plugin.hive.HiveSessionProperties.getTimestampPrecision; import static io.prestosql.plugin.hive.HiveSessionProperties.isTemporaryStagingDirectoryEnabled; import static io.prestosql.plugin.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_EXISTING_DIRECTORY; import static io.prestosql.plugin.hive.metastore.MetastoreUtil.getHiveSchema; @@ -544,7 +545,7 @@ else if (insertExistingPartitionsBehavior == InsertExistingPartitionsBehavior.ER } List types = dataColumns.stream() - .map(column -> column.getHiveType().getType(typeManager)) + .map(column -> column.getHiveType().getType(typeManager, getTimestampPrecision(session))) .collect(toImmutableList()); Map columnIndexes = new HashMap<>(); diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/RcFileFileWriterFactory.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/RcFileFileWriterFactory.java index a3c18c86db97..6c886509d0c4 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/RcFileFileWriterFactory.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/RcFileFileWriterFactory.java @@ -48,6 +48,7 @@ import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_WRITE_VALIDATION_FAILED; import static io.prestosql.plugin.hive.HiveMetadata.PRESTO_QUERY_ID_NAME; import static io.prestosql.plugin.hive.HiveMetadata.PRESTO_VERSION_NAME; +import static io.prestosql.plugin.hive.HiveSessionProperties.getTimestampPrecision; import static io.prestosql.plugin.hive.HiveSessionProperties.isRcfileOptimizedWriterValidate; import static io.prestosql.plugin.hive.rcfile.RcFilePageSourceFactory.createTextVectorEncoding; import static io.prestosql.plugin.hive.util.HiveUtil.getColumnNames; @@ -122,7 +123,7 @@ else if (ColumnarSerDe.class.getName().equals(storageFormat.getSerDe())) { // an index to rearrange columns in the proper order List fileColumnNames = getColumnNames(schema); List fileColumnTypes = getColumnTypes(schema).stream() - .map(hiveType -> hiveType.getType(typeManager)) + .map(hiveType -> hiveType.getType(typeManager, getTimestampPrecision(session))) .collect(toList()); int[] fileInputColumnIndexes = fileColumnNames.stream() diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/RecordFileWriter.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/RecordFileWriter.java index f28c1a175335..91b534f87ad0 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/RecordFileWriter.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/RecordFileWriter.java @@ -45,6 +45,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR; import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_WRITER_DATA_ERROR; +import static io.prestosql.plugin.hive.HiveSessionProperties.getTimestampPrecision; import static io.prestosql.plugin.hive.util.HiveUtil.getColumnNames; import static io.prestosql.plugin.hive.util.HiveUtil.getColumnTypes; import static io.prestosql.plugin.hive.util.HiveWriteUtils.createRecordWriter; @@ -90,7 +91,7 @@ public RecordFileWriter( // existing tables may have columns in a different order List fileColumnNames = getColumnNames(schema); List fileColumnTypes = getColumnTypes(schema).stream() - .map(hiveType -> hiveType.getType(typeManager)) + .map(hiveType -> hiveType.getType(typeManager, getTimestampPrecision(session))) .collect(toList()); fieldCount = fileColumnNames.size(); diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcFileWriterFactory.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcFileWriterFactory.java index 459b372b79a0..17682c913619 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcFileWriterFactory.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcFileWriterFactory.java @@ -64,6 +64,7 @@ import static io.prestosql.plugin.hive.HiveSessionProperties.getOrcOptimizedWriterMinStripeSize; import static io.prestosql.plugin.hive.HiveSessionProperties.getOrcOptimizedWriterValidateMode; import static io.prestosql.plugin.hive.HiveSessionProperties.getOrcStringStatisticsLimit; +import static io.prestosql.plugin.hive.HiveSessionProperties.getTimestampPrecision; import static io.prestosql.plugin.hive.HiveSessionProperties.isOrcOptimizedWriterValidate; import static io.prestosql.plugin.hive.acid.AcidSchema.ACID_COLUMN_NAMES; import static io.prestosql.plugin.hive.acid.AcidSchema.createAcidColumnPrestoTypes; @@ -148,7 +149,7 @@ public Optional createFileWriter( // an index to rearrange columns in the proper order List fileColumnNames = getColumnNames(schema); List fileColumnTypes = getColumnTypes(schema).stream() - .map(hiveType -> hiveType.getType(typeManager)) + .map(hiveType -> hiveType.getType(typeManager, getTimestampPrecision(session))) .collect(toList()); int[] fileInputColumnIndexes = fileColumnNames.stream() diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/parquet/ParquetFileWriterFactory.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/parquet/ParquetFileWriterFactory.java index 96b7f866a7ef..d9f66504faf9 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/parquet/ParquetFileWriterFactory.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/parquet/ParquetFileWriterFactory.java @@ -42,6 +42,7 @@ import java.util.concurrent.Callable; import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_WRITER_OPEN_ERROR; +import static io.prestosql.plugin.hive.HiveSessionProperties.getTimestampPrecision; import static io.prestosql.plugin.hive.util.HiveUtil.getColumnNames; import static io.prestosql.plugin.hive.util.HiveUtil.getColumnTypes; import static java.util.Objects.requireNonNull; @@ -91,7 +92,7 @@ public Optional createFileWriter( List fileColumnNames = getColumnNames(schema); List fileColumnTypes = getColumnTypes(schema).stream() - .map(hiveType -> hiveType.getType(typeManager)) + .map(hiveType -> hiveType.getType(typeManager, getTimestampPrecision(session))) .collect(toList()); int[] fileInputColumnIndexes = fileColumnNames.stream() diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/rcfile/RcFilePageSourceFactory.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/rcfile/RcFilePageSourceFactory.java index c2ac8ab026ac..e2680a3d33a4 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/rcfile/RcFilePageSourceFactory.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/rcfile/RcFilePageSourceFactory.java @@ -24,6 +24,7 @@ import io.prestosql.plugin.hive.HiveColumnHandle; import io.prestosql.plugin.hive.HiveConfig; import io.prestosql.plugin.hive.HivePageSourceFactory; +import io.prestosql.plugin.hive.HiveTimestampPrecision; import io.prestosql.plugin.hive.ReaderProjections; import io.prestosql.plugin.hive.acid.AcidTransaction; import io.prestosql.plugin.hive.util.FSDataInputStreamTail; @@ -184,7 +185,7 @@ else if (deserializerClassName.equals(ColumnarSerDe.class.getName())) { try { ImmutableMap.Builder readColumns = ImmutableMap.builder(); - int timestampPrecision = getTimestampPrecision(session).getPrecision(); + HiveTimestampPrecision timestampPrecision = getTimestampPrecision(session); for (HiveColumnHandle column : projectedReaderColumns) { readColumns.put(column.getBaseHiveColumnIndex(), column.getHiveType().getType(typeManager, timestampPrecision)); } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/FieldSetterFactory.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/FieldSetterFactory.java index 821b85f7453a..3059482830fb 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/FieldSetterFactory.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/FieldSetterFactory.java @@ -24,8 +24,10 @@ import io.prestosql.spi.type.DecimalType; import io.prestosql.spi.type.DoubleType; import io.prestosql.spi.type.IntegerType; +import io.prestosql.spi.type.LongTimestamp; import io.prestosql.spi.type.RealType; import io.prestosql.spi.type.SmallintType; +import io.prestosql.spi.type.TimestampType; import io.prestosql.spi.type.TinyintType; import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarbinaryType; @@ -56,10 +58,13 @@ import static io.prestosql.plugin.hive.util.HiveUtil.isMapType; import static io.prestosql.plugin.hive.util.HiveUtil.isRowType; import static io.prestosql.plugin.hive.util.HiveWriteUtils.getHiveDecimal; -import static io.prestosql.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.prestosql.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; +import static io.prestosql.spi.type.Timestamps.MICROSECONDS_PER_SECOND; +import static io.prestosql.spi.type.Timestamps.MILLISECONDS_PER_SECOND; +import static io.prestosql.spi.type.Timestamps.PICOSECONDS_PER_MICROSECOND; +import static io.prestosql.spi.type.Timestamps.PICOSECONDS_PER_NANOSECOND; import static java.lang.Float.intBitsToFloat; import static java.lang.Math.floorDiv; +import static java.lang.Math.floorMod; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -118,8 +123,8 @@ public FieldSetter create(SettableStructObjectInspector rowInspector, Object row return new DateFieldSetter(rowInspector, row, field); } - if (type.equals(TIMESTAMP_MILLIS)) { - return new TimestampFieldSetter(rowInspector, row, field, timeZone); + if (type instanceof TimestampType) { + return new TimestampFieldSetter(rowInspector, row, field, (TimestampType) type, timeZone); } if (type instanceof DecimalType) { @@ -365,22 +370,49 @@ private static class TimestampFieldSetter extends FieldSetter { private final DateTimeZone timeZone; + private final TimestampType type; private final TimestampWritableV2 value = new TimestampWritableV2(); - public TimestampFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, DateTimeZone timeZone) + public TimestampFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, TimestampType type, DateTimeZone timeZone) { super(rowInspector, row, field); + this.type = requireNonNull(type, "type is null"); this.timeZone = requireNonNull(timeZone, "timeZone is null"); } @Override public void setField(Block block, int position) { - long epochMilli = floorDiv(TIMESTAMP_MILLIS.getLong(block, position), MICROSECONDS_PER_MILLISECOND); - epochMilli = timeZone.convertLocalToUTC(epochMilli, false); - value.set(Timestamp.ofEpochMilli(epochMilli)); + long epochMicros; + int picosOfMicro; + if (type.isShort()) { + epochMicros = type.getLong(block, position); + picosOfMicro = 0; + } + else { + LongTimestamp longTimestamp = (LongTimestamp) type.getObject(block, position); + epochMicros = longTimestamp.getEpochMicros(); + picosOfMicro = longTimestamp.getPicosOfMicro(); + } + + long epochSeconds = floorDiv(epochMicros, MICROSECONDS_PER_SECOND); + long picosOfSecond = (long) floorMod(epochMicros, MICROSECONDS_PER_SECOND) * PICOSECONDS_PER_MICROSECOND + picosOfMicro; + + epochSeconds = convertLocalEpochSecondsToUtc(epochSeconds); + // no rounding since the the data has nanosecond precision, at most + int nanosOfSecond = toIntExact(picosOfSecond / PICOSECONDS_PER_NANOSECOND); + + Timestamp timestamp = Timestamp.ofEpochSecond(epochSeconds, nanosOfSecond); + value.set(timestamp); rowInspector.setStructFieldData(row, field, value); } + + private long convertLocalEpochSecondsToUtc(long epochSeconds) + { + long epochMillis = epochSeconds * MILLISECONDS_PER_SECOND; + epochMillis = timeZone.convertLocalToUTC(epochMillis, false); + return epochMillis / MILLISECONDS_PER_SECOND; + } } private static class DecimalFieldSetter diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveBucketing.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveBucketing.java index e8b94b30abc7..80ed64f6e280 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveBucketing.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveBucketing.java @@ -22,6 +22,7 @@ import io.prestosql.plugin.hive.HiveBucketProperty; import io.prestosql.plugin.hive.HiveColumnHandle; import io.prestosql.plugin.hive.HiveTableHandle; +import io.prestosql.plugin.hive.HiveTimestampPrecision; import io.prestosql.plugin.hive.HiveType; import io.prestosql.plugin.hive.metastore.Column; import io.prestosql.plugin.hive.metastore.Table; @@ -184,7 +185,7 @@ public static Optional getHiveBucketHandle(ConnectorSession se return Optional.empty(); } - int timestampPrecision = getTimestampPrecision(session).getPrecision(); + HiveTimestampPrecision timestampPrecision = getTimestampPrecision(session); Map map = getRegularColumnHandles(table, typeManager, timestampPrecision).stream() .collect(Collectors.toMap(HiveColumnHandle::getName, identity())); diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveTypeTranslator.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveTypeTranslator.java index 39df86d1e117..8c6dac0125c3 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveTypeTranslator.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveTypeTranslator.java @@ -18,6 +18,7 @@ import io.prestosql.spi.type.CharType; import io.prestosql.spi.type.DecimalType; import io.prestosql.spi.type.NamedTypeSignature; +import io.prestosql.spi.type.TimestampType; import io.prestosql.spi.type.Type; import io.prestosql.spi.type.TypeSignatureParameter; import io.prestosql.spi.type.VarcharType; @@ -49,7 +50,6 @@ import static io.prestosql.spi.type.IntegerType.INTEGER; import static io.prestosql.spi.type.RealType.REAL; import static io.prestosql.spi.type.SmallintType.SMALLINT; -import static io.prestosql.spi.type.TimestampType.TIMESTAMP_MILLIS; import static io.prestosql.spi.type.TinyintType.TINYINT; import static io.prestosql.spi.type.VarbinaryType.VARBINARY; import static java.lang.String.format; @@ -113,7 +113,7 @@ public static TypeInfo translate(Type type) if (DATE.equals(type)) { return HIVE_DATE.getTypeInfo(); } - if (TIMESTAMP_MILLIS.equals(type)) { + if (type instanceof TimestampType) { return HIVE_TIMESTAMP.getTypeInfo(); } if (type instanceof DecimalType) { diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveUtil.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveUtil.java index 078c418afb3d..49b44ac73536 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveUtil.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveUtil.java @@ -27,6 +27,7 @@ import io.prestosql.orc.OrcWriterOptions; import io.prestosql.plugin.hive.HiveColumnHandle; import io.prestosql.plugin.hive.HivePartitionKey; +import io.prestosql.plugin.hive.HiveTimestampPrecision; import io.prestosql.plugin.hive.HiveType; import io.prestosql.plugin.hive.avro.PrestoAvroSerDe; import io.prestosql.plugin.hive.metastore.Column; @@ -819,7 +820,7 @@ public static Slice charPartitionKey(String value, String name, Type columnType) return partitionKey; } - public static List hiveColumnHandles(Table table, TypeManager typeManager, int timestampPrecision) + public static List hiveColumnHandles(Table table, TypeManager typeManager, HiveTimestampPrecision timestampPrecision) { ImmutableList.Builder columns = ImmutableList.builder(); @@ -845,7 +846,7 @@ public static List hiveColumnHandles(Table table, TypeManager return columns.build(); } - public static List getRegularColumnHandles(Table table, TypeManager typeManager, int timestampPrecision) + public static List getRegularColumnHandles(Table table, TypeManager typeManager, HiveTimestampPrecision timestampPrecision) { ImmutableList.Builder columns = ImmutableList.builder(); diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveWriteUtils.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveWriteUtils.java index 19b0055501d7..bb26b4aa6ec9 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveWriteUtils.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveWriteUtils.java @@ -47,6 +47,7 @@ import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.RealType; import io.prestosql.spi.type.SmallintType; +import io.prestosql.spi.type.TimestampType; import io.prestosql.spi.type.TinyintType; import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarbinaryType; @@ -695,7 +696,7 @@ public static ObjectInspector getRowColumnInspector(Type type) return writableDateObjectInspector; } - if (type.equals(TIMESTAMP_MILLIS)) { + if (type instanceof TimestampType) { return writableTimestampObjectInspector; } diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestBackgroundHiveSplitLoader.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestBackgroundHiveSplitLoader.java index 765762f4466e..b4eb8fff9e6a 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestBackgroundHiveSplitLoader.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestBackgroundHiveSplitLoader.java @@ -123,7 +123,6 @@ public class TestBackgroundHiveSplitLoader { private static final int BUCKET_COUNT = 2; - private static final int TIMESTAMP_PRECISION = 3; private static final String SAMPLE_PATH = "hdfs://VOL1:9000/db_name/table_name/000000_0"; private static final String SAMPLE_PATH_FILTERED = "hdfs://VOL1:9000/db_name/table_name/000000_1"; @@ -322,7 +321,7 @@ public void testPathFilterBucketedPartitionedTable() PARTITIONED_TABLE, Optional.of( new HiveBucketHandle( - getRegularColumnHandles(PARTITIONED_TABLE, TYPE_MANAGER, TIMESTAMP_PRECISION), + getRegularColumnHandles(PARTITIONED_TABLE, TYPE_MANAGER, HiveTimestampPrecision.MILLISECONDS), BUCKETING_V1, BUCKET_COUNT, BUCKET_COUNT))); diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveFileFormats.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveFileFormats.java index ccb7f97f7644..e26a90ad42e0 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveFileFormats.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveFileFormats.java @@ -292,13 +292,8 @@ public void testRcBinaryOptimizedWriter(int rowCount) public void testOrc(int rowCount, long fileSizePadding) throws Exception { - // Hive binary writers are broken for timestamps - List testColumns = TEST_COLUMNS.stream() - .filter(TestHiveFileFormats::withoutTimestamps) - .collect(toImmutableList()); - assertThatFileFormat(ORC) - .withColumns(testColumns) + .withColumns(TEST_COLUMNS) .withRowsCount(rowCount) .withFileSizePadding(fileSizePadding) .isReadableByPageSource(new OrcPageSourceFactory(new OrcReaderOptions(), HDFS_ENVIRONMENT, STATS, UTC)); diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java index 357f870328a2..05ebe447e469 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java @@ -54,8 +54,6 @@ import io.prestosql.testing.MaterializedRow; import io.prestosql.testing.QueryRunner; import io.prestosql.testing.ResultWithQueryId; -import io.prestosql.testing.sql.SqlExecutor; -import io.prestosql.testing.sql.TestTable; import io.prestosql.type.TypeDeserializer; import org.apache.hadoop.fs.Path; import org.intellij.lang.annotations.Language; @@ -69,6 +67,7 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -161,6 +160,7 @@ public class TestHiveIntegrationSmokeTest extends AbstractTestIntegrationSmokeTest { + private static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"); private final String catalog; private final Session bucketedSession; @@ -3008,7 +3008,7 @@ public void testArrays() @Test(dataProvider = "timestampPrecision") public void testTemporalArrays(HiveTimestampPrecision timestampPrecision) { - Session session = withTimestampPrecision(getSession(), timestampPrecision.name()); + Session session = withTimestampPrecision(getSession(), timestampPrecision); assertUpdate("DROP TABLE IF EXISTS tmp_array11"); assertUpdate("CREATE TABLE tmp_array11 AS SELECT ARRAY[DATE '2014-09-30'] AS col", 1); assertOneNotNullResult("SELECT col[1] FROM tmp_array11"); @@ -3020,7 +3020,7 @@ public void testTemporalArrays(HiveTimestampPrecision timestampPrecision) @Test(dataProvider = "timestampPrecision") public void testMaps(HiveTimestampPrecision timestampPrecision) { - Session session = withTimestampPrecision(getSession(), timestampPrecision.name()); + Session session = withTimestampPrecision(getSession(), timestampPrecision); assertUpdate("DROP TABLE IF EXISTS tmp_map1"); assertUpdate("CREATE TABLE tmp_map1 AS SELECT MAP(ARRAY[0,1], ARRAY[2,NULL]) AS col", 1); assertQuery("SELECT col[0] FROM tmp_map1", "SELECT 2"); @@ -4171,53 +4171,92 @@ public void testPredicatePushDownToTableScan() @DataProvider public Object[][] timestampPrecisionAndValues() { - // TODO: revisit values once we handle write path and are able to write with higher precision, - // make sure push-down happens correctly in the presence of rounding; - // consider using LocalDateTime instead of String return new Object[][] { - {HiveTimestampPrecision.MILLISECONDS, "1965-10-31 01:00:08.123"}, - {HiveTimestampPrecision.MICROSECONDS, "1965-10-31 01:00:08.123000"}, - {HiveTimestampPrecision.NANOSECONDS, "1965-10-31 01:00:08.123000000"}, - {HiveTimestampPrecision.MILLISECONDS, "2012-10-31 01:00:08.123"}, - {HiveTimestampPrecision.MICROSECONDS, "2012-10-31 01:00:08.123000"}, - {HiveTimestampPrecision.NANOSECONDS, "2012-10-31 01:00:08.123000000"}}; + {HiveTimestampPrecision.MILLISECONDS, LocalDateTime.parse("2012-10-31T01:00:08.123")}, + {HiveTimestampPrecision.MICROSECONDS, LocalDateTime.parse("2012-10-31T01:00:08.123456")}, + {HiveTimestampPrecision.NANOSECONDS, LocalDateTime.parse("2012-10-31T01:00:08.123000000")}, + {HiveTimestampPrecision.NANOSECONDS, LocalDateTime.parse("2012-10-31T01:00:08.123000001")}, + {HiveTimestampPrecision.NANOSECONDS, LocalDateTime.parse("2012-10-31T01:00:08.123456789")}, + {HiveTimestampPrecision.MILLISECONDS, LocalDateTime.parse("1965-10-31T01:00:08.123")}, + {HiveTimestampPrecision.MICROSECONDS, LocalDateTime.parse("1965-10-31T01:00:08.123456")}, + {HiveTimestampPrecision.NANOSECONDS, LocalDateTime.parse("1965-10-31T01:00:08.123000000")}, + {HiveTimestampPrecision.NANOSECONDS, LocalDateTime.parse("1965-10-31T01:00:08.123000001")}, + {HiveTimestampPrecision.NANOSECONDS, LocalDateTime.parse("1965-10-31T01:00:08.123456789")}}; } @Test(dataProvider = "timestampPrecisionAndValues") - public void testParquetTimestampPredicatePushdown(HiveTimestampPrecision timestampPrecision, String value) + public void testParquetTimestampPredicatePushdown(HiveTimestampPrecision timestampPrecision, LocalDateTime value) { - Session session = withTimestampPrecision(getSession(), timestampPrecision.name()); + Session session = withTimestampPrecision(getSession(), timestampPrecision); assertUpdate("DROP TABLE IF EXISTS test_parquet_timestamp_predicate_pushdown"); assertUpdate("CREATE TABLE test_parquet_timestamp_predicate_pushdown (t TIMESTAMP) WITH (format = 'PARQUET')"); - assertUpdate(format("INSERT INTO test_parquet_timestamp_predicate_pushdown VALUES (TIMESTAMP '%s')", value), 1); - assertQuery(session, "SELECT * FROM test_parquet_timestamp_predicate_pushdown", format("VALUES (TIMESTAMP '%s')", value)); + assertUpdate(session, format("INSERT INTO test_parquet_timestamp_predicate_pushdown VALUES (%s)", formatTimestamp(value)), 1); + assertQuery(session, "SELECT * FROM test_parquet_timestamp_predicate_pushdown", format("VALUES (%s)", formatTimestamp(value))); DistributedQueryRunner queryRunner = (DistributedQueryRunner) getQueryRunner(); ResultWithQueryId queryResult = queryRunner.executeWithQueryId( session, - format("SELECT * FROM test_parquet_timestamp_predicate_pushdown WHERE t < TIMESTAMP '%s'", value)); + format("SELECT * FROM test_parquet_timestamp_predicate_pushdown WHERE t < %s", formatTimestamp(value))); assertEquals(getQueryInfo(queryRunner, queryResult).getQueryStats().getProcessedInputDataSize().toBytes(), 0); queryResult = queryRunner.executeWithQueryId( session, - format("SELECT * FROM test_parquet_timestamp_predicate_pushdown WHERE t > TIMESTAMP '%s'", value)); + format("SELECT * FROM test_parquet_timestamp_predicate_pushdown WHERE t > %s", formatTimestamp(value))); assertEquals(getQueryInfo(queryRunner, queryResult).getQueryStats().getProcessedInputDataSize().toBytes(), 0); // TODO: replace this with a simple query stats check once we find a way to wait until all pending updates to query stats have been applied - ExponentialSleeper sleeper = new ExponentialSleeper( - new Duration(0, SECONDS), - new Duration(5, SECONDS), - new Duration(100, MILLISECONDS), - 2.0); + // (might be fixed by https://github.com/prestosql/presto/issues/5172) + ExponentialSleeper sleeper = new ExponentialSleeper(); assertEventually(new Duration(30, SECONDS), () -> { ResultWithQueryId result = queryRunner.executeWithQueryId( session, - format("SELECT * FROM test_parquet_timestamp_predicate_pushdown WHERE t = TIMESTAMP '%s'", value)); + format("SELECT * FROM test_parquet_timestamp_predicate_pushdown WHERE t = %s", formatTimestamp(value))); sleeper.sleep(); assertThat(getQueryInfo(queryRunner, result).getQueryStats().getProcessedInputDataSize().toBytes()).isGreaterThan(0); }); } + @Test(dataProvider = "timestampPrecisionAndValues") + public void testOrcTimestampPredicatePushdown(HiveTimestampPrecision timestampPrecision, LocalDateTime value) + { + Session session = withTimestampPrecision(getSession(), timestampPrecision); + assertUpdate("DROP TABLE IF EXISTS test_orc_timestamp_predicate_pushdown"); + assertUpdate("CREATE TABLE test_orc_timestamp_predicate_pushdown (t TIMESTAMP) WITH (format = 'ORC')"); + assertUpdate(session, format("INSERT INTO test_orc_timestamp_predicate_pushdown VALUES (%s)", formatTimestamp(value)), 1); + assertQuery(session, "SELECT * FROM test_orc_timestamp_predicate_pushdown", format("VALUES (%s)", formatTimestamp(value))); + + // to account for the fact that ORC stats are stored at millisecond precision and Presto rounds timestamps, + // we filter by timestamps that differ from the actual value by at least 1ms, to observe pruning + DistributedQueryRunner queryRunner = (DistributedQueryRunner) getQueryRunner(); + ResultWithQueryId queryResult = queryRunner.executeWithQueryId( + session, + format("SELECT * FROM test_orc_timestamp_predicate_pushdown WHERE t < %s", formatTimestamp(value.minusNanos(MILLISECONDS.toNanos(1))))); + assertEquals(getQueryInfo(queryRunner, queryResult).getQueryStats().getProcessedInputDataSize().toBytes(), 0); + + queryResult = queryRunner.executeWithQueryId( + session, + format("SELECT * FROM test_orc_timestamp_predicate_pushdown WHERE t > %s", formatTimestamp(value.plusNanos(MILLISECONDS.toNanos(1))))); + assertEquals(getQueryInfo(queryRunner, queryResult).getQueryStats().getProcessedInputDataSize().toBytes(), 0); + + assertQuery(session, "SELECT * FROM test_orc_timestamp_predicate_pushdown WHERE t < " + formatTimestamp(value.plusNanos(1)), format("VALUES (%s)", formatTimestamp(value))); + + // TODO: replace this with a simple query stats check once we find a way to wait until all pending updates to query stats have been applied + // (might be fixed by https://github.com/prestosql/presto/issues/5172) + ExponentialSleeper sleeper = new ExponentialSleeper(); + assertEventually(new Duration(30, SECONDS), () -> { + ResultWithQueryId result = queryRunner.executeWithQueryId( + session, + format("SELECT * FROM test_orc_timestamp_predicate_pushdown WHERE t = %s", formatTimestamp(value))); + sleeper.sleep(); + assertThat(getQueryInfo(queryRunner, result).getQueryStats().getProcessedInputDataSize().toBytes()).isGreaterThan(0); + }); + } + + private static String formatTimestamp(LocalDateTime timestamp) + { + return format("TIMESTAMP '%s'", TIMESTAMP_FORMATTER.format(timestamp)); + } + private QueryInfo getQueryInfo(DistributedQueryRunner queryRunner, ResultWithQueryId queryResult) { return queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(queryResult.getQueryId()); @@ -5770,26 +5809,6 @@ public void testAnalyzeEmptyTable() assertUpdate("ANALYZE " + tableName, 0); } - @DataProvider - public Object[][] nonDefaultTimestampPrecisions() - { - return new Object[][] { - {HiveTimestampPrecision.MICROSECONDS}, - {HiveTimestampPrecision.NANOSECONDS} - }; - } - - @Test(dataProvider = "nonDefaultTimestampPrecisions") - public void testWriteNonDefaultPrecisionTimestampColumn(HiveTimestampPrecision timestampPrecision) - { - SqlExecutor sqlExecutor = sql -> getQueryRunner().execute(sql); - try (TestTable table = new TestTable(sqlExecutor, "test_analyze_empty_timestamp", "(c_bigint BIGINT, c_timestamp TIMESTAMP)")) { - Session session = withTimestampPrecision(getSession(), timestampPrecision.name()); - assertQueryFails(session, "ANALYZE " + table.getName(), format("\\QCREATE TABLE, INSERT and ANALYZE are not supported with requested timestamp precision: timestamp(%s)\\E", timestampPrecision.getPrecision())); - assertQueryFails(session, format("INSERT INTO %s VALUES (1, TIMESTAMP'2001-02-03 11:22:33.123456789')", table.getName()), format("\\QCREATE TABLE, INSERT and ANALYZE are not supported with requested timestamp precision: timestamp(%s)\\E", timestampPrecision.getPrecision())); - } - } - @Test public void testInvalidAnalyzePartitionedTable() { @@ -7081,6 +7100,106 @@ public void testUnsupportedCsvTable() "\\QHive CSV storage format only supports VARCHAR (unbounded). Unsupported columns: i integer, bound varchar(10)\\E"); } + @Test + public void testWriteInvalidPrecisionTimestamp() + { + Session session = withTimestampPrecision(getSession(), HiveTimestampPrecision.MICROSECONDS); + assertQueryFails( + session, + "CREATE TABLE test_invalid_precision_timestamp(ts) AS SELECT TIMESTAMP '2001-02-03 11:22:33.123456789'", + "\\QIncorrect timestamp precision for timestamp(9); the configured precision is " + HiveTimestampPrecision.MICROSECONDS); + assertQueryFails( + session, + "CREATE TABLE test_invalid_precision_timestamp (ts TIMESTAMP(9))", + "\\QIncorrect timestamp precision for timestamp(9); the configured precision is " + HiveTimestampPrecision.MICROSECONDS); + assertQueryFails( + session, + "CREATE TABLE test_invalid_precision_timestamp(ts) AS SELECT TIMESTAMP '2001-02-03 11:22:33.123'", + "\\QIncorrect timestamp precision for timestamp(3); the configured precision is " + HiveTimestampPrecision.MICROSECONDS); + assertQueryFails( + session, + "CREATE TABLE test_invalid_precision_timestamp (ts TIMESTAMP(3))", + "\\QIncorrect timestamp precision for timestamp(3); the configured precision is " + HiveTimestampPrecision.MICROSECONDS); + } + + @Test + public void testTimestampPrecisionInsert() + { + testWithAllStorageFormats(this::testTimestampPrecisionInsert); + } + + private void testTimestampPrecisionInsert(Session session, HiveStorageFormat storageFormat) + { + if (storageFormat == HiveStorageFormat.AVRO) { + // Avro timestamps are stored with millisecond precision + return; + } + + String createTable = "CREATE TABLE test_timestamp_precision (ts TIMESTAMP) WITH (format = '%s')"; + @Language("SQL") String insert = "INSERT INTO test_timestamp_precision VALUES (TIMESTAMP '%s')"; + + testTimestampPrecisionWrites( + session, + (ts, precision) -> { + assertUpdate("DROP TABLE IF EXISTS test_timestamp_precision"); + assertUpdate(format(createTable, storageFormat)); + assertUpdate(withTimestampPrecision(session, precision), format(insert, ts), 1); + }); + } + + @Test + public void testTimestampPrecisionCtas() + { + testWithAllStorageFormats(this::testTimestampPrecisionCtas); + } + + private void testTimestampPrecisionCtas(Session session, HiveStorageFormat storageFormat) + { + if (storageFormat == HiveStorageFormat.AVRO) { + // Avro timestamps are stored with millisecond precision + return; + } + + String createTableAs = "CREATE TABLE test_timestamp_precision WITH (format = '%s') AS SELECT TIMESTAMP '%s' ts"; + + testTimestampPrecisionWrites( + session, + (ts, precision) -> { + assertUpdate("DROP TABLE IF EXISTS test_timestamp_precision"); + assertUpdate(withTimestampPrecision(session, precision), format(createTableAs, storageFormat, ts), 1); + }); + } + + private void testTimestampPrecisionWrites(Session session, BiConsumer populateData) + { + populateData.accept("2019-02-03 18:30:00.123", HiveTimestampPrecision.MILLISECONDS); + @Language("SQL") String sql = "SELECT ts FROM test_timestamp_precision"; + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.MILLISECONDS), sql, "VALUES ('2019-02-03 18:30:00.123')"); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.MICROSECONDS), sql, "VALUES ('2019-02-03 18:30:00.123')"); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.NANOSECONDS), sql, "VALUES ('2019-02-03 18:30:00.123')"); + + populateData.accept("2019-02-03 18:30:00.456789", HiveTimestampPrecision.MICROSECONDS); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.MILLISECONDS), sql, "VALUES ('2019-02-03 18:30:00.457')"); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.MICROSECONDS), sql, "VALUES ('2019-02-03 18:30:00.456789')"); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.NANOSECONDS), sql, "VALUES ('2019-02-03 18:30:00.456789000')"); + + populateData.accept("2019-02-03 18:30:00.456789876", HiveTimestampPrecision.NANOSECONDS); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.MILLISECONDS), sql, "VALUES ('2019-02-03 18:30:00.457')"); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.MICROSECONDS), sql, "VALUES ('2019-02-03 18:30:00.456790')"); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.NANOSECONDS), sql, "VALUES ('2019-02-03 18:30:00.456789876')"); + + // some rounding edge cases + + populateData.accept("2019-02-03 18:30:00.999999", HiveTimestampPrecision.MICROSECONDS); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.MILLISECONDS), sql, "VALUES ('2019-02-03 18:30:01.000')"); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.MICROSECONDS), sql, "VALUES ('2019-02-03 18:30:00.999999')"); + + populateData.accept("2019-02-03 18:30:00.999999999", HiveTimestampPrecision.NANOSECONDS); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.MILLISECONDS), sql, "VALUES ('2019-02-03 18:30:01.000')"); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.MICROSECONDS), sql, "VALUES ('2019-02-03 18:30:01.000000')"); + assertQuery(withTimestampPrecision(session, HiveTimestampPrecision.NANOSECONDS), sql, "VALUES ('2019-02-03 18:30:00.999999999')"); + } + private Session getParallelWriteSession() { return Session.builder(getSession()) @@ -7237,6 +7356,15 @@ private static class ExponentialSleeper this.sleepIncrementFactor = sleepIncrementFactor; } + ExponentialSleeper() + { + this( + new Duration(0, SECONDS), + new Duration(5, SECONDS), + new Duration(100, MILLISECONDS), + 2.0); + } + public void sleep() { try { @@ -7266,10 +7394,10 @@ public Object[][] timestampPrecision() {HiveTimestampPrecision.NANOSECONDS}}; } - private Session withTimestampPrecision(Session session, String precision) + private Session withTimestampPrecision(Session session, HiveTimestampPrecision precision) { return Session.builder(session) - .setCatalogSessionProperty(catalog, "timestamp_precision", precision) + .setCatalogSessionProperty(catalog, "timestamp_precision", precision.name()) .build(); } } diff --git a/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java b/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java index 4efd914702ff..1e3cba5d5d1d 100644 --- a/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java +++ b/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java @@ -770,7 +770,7 @@ public void finishStatisticsCollection(Session session, AnalyzeTableHandle table { CatalogName catalogName = tableHandle.getCatalogName(); CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); - catalogMetadata.getMetadata().finishStatisticsCollection(session.toConnectorSession(), tableHandle.getConnectorHandle(), computedStatistics); + catalogMetadata.getMetadata().finishStatisticsCollection(session.toConnectorSession(catalogName), tableHandle.getConnectorHandle(), computedStatistics); } @Override diff --git a/presto-main/src/main/java/io/prestosql/testing/MaterializedResult.java b/presto-main/src/main/java/io/prestosql/testing/MaterializedResult.java index af6526900b11..651144abcde7 100644 --- a/presto-main/src/main/java/io/prestosql/testing/MaterializedResult.java +++ b/presto-main/src/main/java/io/prestosql/testing/MaterializedResult.java @@ -27,6 +27,7 @@ import io.prestosql.spi.connector.ConnectorSession; import io.prestosql.spi.type.ArrayType; import io.prestosql.spi.type.CharType; +import io.prestosql.spi.type.LongTimestamp; import io.prestosql.spi.type.MapType; import io.prestosql.spi.type.RowType; import io.prestosql.spi.type.SqlDate; @@ -294,7 +295,12 @@ else if (type instanceof TimeWithTimeZoneType) { } else if (type instanceof TimestampType) { long micros = ((SqlTimestamp) value).getEpochMicros(); - type.writeLong(blockBuilder, micros); + if (((TimestampType) type).getPrecision() <= TimestampType.MAX_SHORT_PRECISION) { + type.writeLong(blockBuilder, micros); + } + else { + type.writeObject(blockBuilder, new LongTimestamp(micros, ((SqlTimestamp) value).getPicosOfMicros())); + } } else if (TIMESTAMP_WITH_TIME_ZONE.equals(type)) { long millisUtc = ((SqlTimestampWithTimeZone) value).getMillisUtc(); diff --git a/presto-orc/src/main/java/io/prestosql/orc/writer/TimestampColumnWriter.java b/presto-orc/src/main/java/io/prestosql/orc/writer/TimestampColumnWriter.java index 971a8cc638ae..790114484090 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/writer/TimestampColumnWriter.java +++ b/presto-orc/src/main/java/io/prestosql/orc/writer/TimestampColumnWriter.java @@ -316,15 +316,6 @@ public void reset() statisticsBuilder = statisticsBuilderSupplier.get(); } - private void writeTimestampMillis(Block block) - { - for (int i = 0; i < block.getPositionCount(); i++) { - if (!block.isNull(i)) { - writeMillis(type.getLong(block, i)); - } - } - } - private void writeTimestampMicros(Block block) { for (int i = 0; i < block.getPositionCount(); i++) { @@ -354,6 +345,7 @@ private void writeTimestampNanos(Block block) long seconds = timestamp.getEpochMicros() / MICROSECONDS_PER_SECOND; long microsFraction = floorMod(timestamp.getEpochMicros(), MICROSECONDS_PER_SECOND); long nanosFraction = (microsFraction * NANOSECONDS_PER_MICROSECOND) + + // no rounding since the the data has nanosecond precision, at most (timestamp.getPicosOfMicro() / PICOSECONDS_PER_NANOSECOND); long millis = floorDiv(timestamp.getEpochMicros(), MICROSECONDS_PER_MILLISECOND); diff --git a/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java b/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java index cfe314a7b015..306299170ab8 100644 --- a/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java +++ b/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java @@ -151,7 +151,6 @@ import static io.prestosql.spi.type.Varchars.truncateToLength; import static io.prestosql.testing.DateTimeTestingUtils.sqlTimestampOf; import static io.prestosql.testing.TestingConnectorSession.SESSION; -import static java.lang.Math.toIntExact; import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; import static org.apache.hadoop.hive.serde2.ColumnProjectionUtils.READ_ALL_COLUMNS; @@ -837,7 +836,7 @@ else if (type.equals(TIMESTAMP_TZ_MILLIS)) { actualValue = SqlTimestampWithTimeZone.newInstance(3, timestamp.toEpochMilli(), 0, UTC_KEY); } else if (type.equals(TIMESTAMP_TZ_MICROS)) { - int picosOfMilli = toIntExact(roundDiv(timestamp.getNanos(), NANOSECONDS_PER_MICROSECOND) * PICOSECONDS_PER_MICROSECOND); + int picosOfMilli = roundDiv(timestamp.getNanos(), NANOSECONDS_PER_MICROSECOND) * PICOSECONDS_PER_MICROSECOND; actualValue = SqlTimestampWithTimeZone.newInstance(3, timestamp.toEpochMilli(), picosOfMilli, UTC_KEY); } else if (type.equals(TIMESTAMP_TZ_NANOS)) { @@ -1041,7 +1040,7 @@ private static Object preprocessWriteValueHive(Type type, Object value) } if (type.equals(TIMESTAMP_TZ_MILLIS) || type.equals(TIMESTAMP_TZ_MICROS) || type.equals(TIMESTAMP_TZ_NANOS)) { SqlTimestampWithTimeZone timestamp = (SqlTimestampWithTimeZone) value; - int nanosOfMilli = toIntExact(roundDiv(timestamp.getPicosOfMilli(), PICOSECONDS_PER_NANOSECOND)); + int nanosOfMilli = roundDiv(timestamp.getPicosOfMilli(), PICOSECONDS_PER_NANOSECOND); return Timestamp.ofEpochMilli(timestamp.getEpochMillis(), nanosOfMilli); } if (type instanceof DecimalType) { diff --git a/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveStorageFormats.java b/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveStorageFormats.java index 527ec0240884..3e25f77886b5 100644 --- a/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveStorageFormats.java +++ b/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveStorageFormats.java @@ -331,47 +331,220 @@ public void testSnappyCompressedParquetTableCreatedInHive() onHive().executeQuery("DROP TABLE " + tableName); } - @Test(dataProvider = "storageFormatsWithNanosecondPrecision", groups = STORAGE_FORMATS) - public void testTimestamp(StorageFormat storageFormat) + @Test(dataProvider = "storageFormatsWithNanosecondPrecision") + public void testTimestampCreatedFromHive(StorageFormat storageFormat) throws Exception { - // only admin user is allowed to change session properties - Connection connection = onPresto().getConnection(); - setAdminRole(connection); - setSessionProperties(connection, storageFormat); - String tableName = "test_timestamp_" + storageFormat.getName().toLowerCase(Locale.ENGLISH); - onPresto().executeQuery("DROP TABLE IF EXISTS " + tableName); - - onPresto().executeQuery(format("CREATE TABLE %s (id BIGINT, ts TIMESTAMP) WITH (%s)", tableName, storageFormat.getStoragePropertiesAsSql())); + setupTimestampData(tableName, storageFormat); + // write precision is not relevant here, as Hive always uses nanos List data = ImmutableList.of( - new TimestampAndPrecision(1, "MILLISECONDS", "2020-01-02 12:34:56.123", "2020-01-02 12:34:56.123"), - new TimestampAndPrecision(2, "MILLISECONDS", "2020-01-02 12:34:56.1234", "2020-01-02 12:34:56.123"), - new TimestampAndPrecision(3, "MILLISECONDS", "2020-01-02 12:34:56.1236", "2020-01-02 12:34:56.124"), - new TimestampAndPrecision(4, "MICROSECONDS", "2020-01-02 12:34:56.123456", "2020-01-02 12:34:56.123456"), - new TimestampAndPrecision(5, "MICROSECONDS", "2020-01-02 12:34:56.1234564", "2020-01-02 12:34:56.123456"), - new TimestampAndPrecision(6, "MICROSECONDS", "2020-01-02 12:34:56.1234567", "2020-01-02 12:34:56.123457"), - new TimestampAndPrecision(7, "NANOSECONDS", "2020-01-02 12:34:56.123456789", "2020-01-02 12:34:56.123456789")); + new TimestampAndPrecision( + "NANOSECONDS", + "1967-01-02 12:34:56.123", + ImmutableMap.of( + "MILLISECONDS", "1967-01-02 12:34:56.123", + "MICROSECONDS", "1967-01-02 12:34:56.123", + "NANOSECONDS", "1967-01-02 12:34:56.123")), + new TimestampAndPrecision( + "NANOSECONDS", + "2020-01-02 12:34:56.123", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123", + "NANOSECONDS", "2020-01-02 12:34:56.123")), + new TimestampAndPrecision( + "NANOSECONDS", + "1967-01-02 12:34:56.1234", + ImmutableMap.of( + "MILLISECONDS", "1967-01-02 12:34:56.123", + "MICROSECONDS", "1967-01-02 12:34:56.1234", + "NANOSECONDS", "1967-01-02 12:34:56.1234")), + new TimestampAndPrecision( + "NANOSECONDS", + "2020-01-02 12:34:56.1234", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.1234", + "NANOSECONDS", "2020-01-02 12:34:56.1234")), + new TimestampAndPrecision( + "NANOSECONDS", + "1967-01-02 12:34:56.1236", + ImmutableMap.of( + "MILLISECONDS", "1967-01-02 12:34:56.124", + "MICROSECONDS", "1967-01-02 12:34:56.1236", + "NANOSECONDS", "1967-01-02 12:34:56.1236")), + new TimestampAndPrecision( + "NANOSECONDS", + "2020-01-02 12:34:56.1236", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.124", + "MICROSECONDS", "2020-01-02 12:34:56.1236", + "NANOSECONDS", "2020-01-02 12:34:56.1236")), + new TimestampAndPrecision( + "NANOSECONDS", + "1967-01-02 12:34:56.123456", + ImmutableMap.of( + "MILLISECONDS", "1967-01-02 12:34:56.123", + "MICROSECONDS", "1967-01-02 12:34:56.123456", + "NANOSECONDS", "1967-01-02 12:34:56.123456")), + new TimestampAndPrecision( + "NANOSECONDS", + "2020-01-02 12:34:56.123456", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123456", + "NANOSECONDS", "2020-01-02 12:34:56.123456")), + new TimestampAndPrecision( + "NANOSECONDS", + "1967-01-02 12:34:56.1234564", + ImmutableMap.of( + "MILLISECONDS", "1967-01-02 12:34:56.123", + "MICROSECONDS", "1967-01-02 12:34:56.123456", + "NANOSECONDS", "1967-01-02 12:34:56.1234564")), + new TimestampAndPrecision( + "NANOSECONDS", + "2020-01-02 12:34:56.1234564", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123456", + "NANOSECONDS", "2020-01-02 12:34:56.1234564")), + new TimestampAndPrecision( + "NANOSECONDS", + "1967-01-02 12:34:56.1234567", + ImmutableMap.of( + "MILLISECONDS", "1967-01-02 12:34:56.123", + "MICROSECONDS", "1967-01-02 12:34:56.123457", + "NANOSECONDS", "1967-01-02 12:34:56.1234567")), + new TimestampAndPrecision( + "NANOSECONDS", + "2020-01-02 12:34:56.1234567", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123457", + "NANOSECONDS", "2020-01-02 12:34:56.1234567")), + new TimestampAndPrecision( + "NANOSECONDS", + "1967-01-02 12:34:56.123456789", + ImmutableMap.of( + "MILLISECONDS", "1967-01-02 12:34:56.123", + "MICROSECONDS", "1967-01-02 12:34:56.123457", + "NANOSECONDS", "1967-01-02 12:34:56.123456789")), + new TimestampAndPrecision( + "NANOSECONDS", + "2020-01-02 12:34:56.123456789", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123457", + "NANOSECONDS", "2020-01-02 12:34:56.123456789"))); // insert records one by one so that we have one file per record, which allows us to exercise predicate push-down in Parquet // (which only works when the value range has a min = max) for (TimestampAndPrecision entry : data) { - onHive().executeQuery(format("INSERT INTO %s VALUES (%s, '%s')", tableName, entry.getId(), entry.getValue())); + onHive().executeQuery(format("INSERT INTO %s VALUES (%s, '%s')", tableName, entry.getId(), entry.getWriteValue())); } + runTimestampQueries(tableName, data); + } + + @Test(dataProvider = "storageFormatsWithNanosecondPrecision") + public void testTimestampCreatedFromPresto(StorageFormat storageFormat) + throws Exception + { + String tableName = "test_timestamp_" + storageFormat.getName().toLowerCase(Locale.ENGLISH); + setupTimestampData(tableName, storageFormat); + + List data = ImmutableList.of( + new TimestampAndPrecision( + "MILLISECONDS", + "2020-01-02 12:34:56.123", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123", + "NANOSECONDS", "2020-01-02 12:34:56.123")), + new TimestampAndPrecision( + "MILLISECONDS", + "2020-01-02 12:34:56.1234", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123", + "NANOSECONDS", "2020-01-02 12:34:56.123")), + new TimestampAndPrecision( + "MILLISECONDS", + "2020-01-02 12:34:56.1236", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.124", + "MICROSECONDS", "2020-01-02 12:34:56.124", + "NANOSECONDS", "2020-01-02 12:34:56.124")), + new TimestampAndPrecision( + "MICROSECONDS", + "2020-01-02 12:34:56.123456", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123456", + "NANOSECONDS", "2020-01-02 12:34:56.123456")), + new TimestampAndPrecision( + "MICROSECONDS", + "2020-01-02 12:34:56.1234564", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123456", + "NANOSECONDS", "2020-01-02 12:34:56.123456")), + new TimestampAndPrecision( + "MICROSECONDS", + "2020-01-02 12:34:56.1234567", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123457", + "NANOSECONDS", "2020-01-02 12:34:56.123457")), + new TimestampAndPrecision( + "NANOSECONDS", + "2020-01-02 12:34:56.123456789", + ImmutableMap.of( + "MILLISECONDS", "2020-01-02 12:34:56.123", + "MICROSECONDS", "2020-01-02 12:34:56.123457", + "NANOSECONDS", "2020-01-02 12:34:56.123456789"))); + for (TimestampAndPrecision entry : data) { - setSessionProperty(connection, "hive.timestamp_precision", entry.getPrecision()); - assertThat(onPresto().executeQuery(format("SELECT ts FROM %s WHERE id = %s", tableName, entry.getId()))) - .containsOnly(row(Timestamp.valueOf(entry.getRoundedValue()))); - assertThat(onPresto().executeQuery(format("SELECT id FROM %s WHERE id = %s AND ts = TIMESTAMP'%s'", tableName, entry.getId(), entry.getRoundedValue()))) - .containsOnly(row(entry.getId())); - if (entry.isRoundedUp()) { - assertThat(onPresto().executeQuery(format("SELECT id FROM %s WHERE id = %s AND ts > TIMESTAMP'%s'", tableName, entry.getId(), entry.getValue()))) - .containsOnly(row(entry.getId())); - } - if (entry.isRoundedDown()) { - assertThat(onPresto().executeQuery(format("SELECT id FROM %s WHERE id = %s AND ts < TIMESTAMP'%s'", tableName, entry.getId(), entry.getValue()))) + // insert timestamps with different precisions + setSessionProperty(onPresto().getConnection(), "hive.timestamp_precision", entry.getPrecision()); + // insert records one by one so that we have one file per record, which allows us to exercise predicate push-down in Parquet + // (which only works when the value range has a min = max) + onPresto().executeQuery(format("INSERT INTO %s VALUES (%s, TIMESTAMP'%s')", tableName, entry.getId(), entry.getWriteValue())); + } + + runTimestampQueries(tableName, data); + } + + private void setupTimestampData(String tableName, StorageFormat storageFormat) + { + // only admin user is allowed to change session properties + Connection connection = onPresto().getConnection(); + setAdminRole(connection); + setSessionProperties(connection, storageFormat); + + onPresto().executeQuery("DROP TABLE IF EXISTS " + tableName); + onPresto().executeQuery(format("CREATE TABLE %s (id BIGINT, ts TIMESTAMP) WITH (%s)", tableName, storageFormat.getStoragePropertiesAsSql())); + } + + private void runTimestampQueries(String tableName, List data) + throws SQLException + { + for (TimestampAndPrecision entry : data) { + for (String precision : List.of("MILLISECONDS", "MICROSECONDS", "NANOSECONDS")) { + setSessionProperty(onPresto().getConnection(), "hive.timestamp_precision", precision); + assertThat(onPresto().executeQuery(format("SELECT ts FROM %s WHERE id = %s", tableName, entry.getId()))) + .containsOnly(row(Timestamp.valueOf(entry.getReadValues(precision)))); + assertThat(onPresto().executeQuery(format("SELECT id FROM %s WHERE id = %s AND ts = TIMESTAMP'%s'", tableName, entry.getId(), entry.getReadValues(precision)))) .containsOnly(row(entry.getId())); + if (entry.isRoundedUp(precision)) { + assertThat(onPresto().executeQuery(format("SELECT id FROM %s WHERE id = %s AND ts > TIMESTAMP'%s'", tableName, entry.getId(), entry.getWriteValue()))) + .containsOnly(row(entry.getId())); + } + if (entry.isRoundedDown(precision)) { + assertThat(onPresto().executeQuery(format("SELECT id FROM %s WHERE id = %s AND ts < TIMESTAMP'%s'", tableName, entry.getId(), entry.getWriteValue()))) + .containsOnly(row(entry.getId())); + } } } onPresto().executeQuery("DROP TABLE " + tableName); @@ -503,17 +676,21 @@ public String toString() private static class TimestampAndPrecision { + private static int counter; private final int id; + // precision used when writing the data private final String precision; - private final String value; - private final String roundedValue; + // inserted value + private final String writeValue; + // expected values to be read back at various precisions + private final Map readValues; - public TimestampAndPrecision(int id, String precision, String value, String roundedValue) + public TimestampAndPrecision(String precision, String writeValue, Map readValues) { - this.id = id; + this.id = counter++; this.precision = precision; - this.value = value; - this.roundedValue = roundedValue; + this.writeValue = writeValue; + this.readValues = readValues; } public int getId() @@ -526,29 +703,29 @@ public String getPrecision() return precision; } - public String getValue() + public String getWriteValue() { - return value; + return writeValue; } - public String getRoundedValue() + public String getReadValues(String precision) { - return roundedValue; + return readValues.get(precision); } - private int roundingSign() + private int roundingSign(String precision) { - return Timestamp.valueOf(roundedValue).compareTo(Timestamp.valueOf(value)); + return Timestamp.valueOf(readValues.get(precision)).compareTo(Timestamp.valueOf(writeValue)); } - public boolean isRoundedUp() + public boolean isRoundedUp(String precision) { - return roundingSign() > 0; + return roundingSign(precision) > 0; } - public boolean isRoundedDown() + public boolean isRoundedDown(String precision) { - return roundingSign() < 0; + return roundingSign(precision) < 0; } } } diff --git a/presto-rcfile/src/main/java/io/prestosql/rcfile/TimestampHolder.java b/presto-rcfile/src/main/java/io/prestosql/rcfile/TimestampHolder.java new file mode 100644 index 000000000000..c8de3e92f3c9 --- /dev/null +++ b/presto-rcfile/src/main/java/io/prestosql/rcfile/TimestampHolder.java @@ -0,0 +1,72 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.rcfile; + +import io.prestosql.spi.block.Block; +import io.prestosql.spi.type.LongTimestamp; +import io.prestosql.spi.type.TimestampType; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.function.BiFunction; + +import static io.prestosql.spi.type.Timestamps.MICROSECONDS_PER_SECOND; +import static io.prestosql.spi.type.Timestamps.PICOSECONDS_PER_MICROSECOND; +import static io.prestosql.spi.type.Timestamps.PICOSECONDS_PER_NANOSECOND; +import static java.lang.Math.floorDiv; +import static java.lang.Math.floorMod; +import static java.lang.Math.toIntExact; + +public final class TimestampHolder +{ + private final long seconds; + private final int nanosOfSecond; + + public TimestampHolder(long epochMicros, int picosOfMicro) + { + this.seconds = floorDiv(epochMicros, MICROSECONDS_PER_SECOND); + long picosOfSecond = (long) floorMod(epochMicros, MICROSECONDS_PER_SECOND) * PICOSECONDS_PER_MICROSECOND + picosOfMicro; + + // no rounding since the the data has nanosecond precision, at most + this.nanosOfSecond = toIntExact(picosOfSecond / PICOSECONDS_PER_NANOSECOND); + } + + public long getSeconds() + { + return seconds; + } + + public int getNanosOfSecond() + { + return nanosOfSecond; + } + + public LocalDateTime toLocalDateTime() + { + return LocalDateTime.ofEpochSecond(seconds, nanosOfSecond, ZoneOffset.UTC); + } + + public static BiFunction getFactory(TimestampType type) + { + if (type.isShort()) { + return (block, position) -> new TimestampHolder(type.getLong(block, position), 0); + } + else { + return (block, position) -> { + LongTimestamp longTimestamp = (LongTimestamp) type.getObject(block, position); + return new TimestampHolder(longTimestamp.getEpochMicros(), longTimestamp.getPicosOfMicro()); + }; + } + } +} diff --git a/presto-rcfile/src/main/java/io/prestosql/rcfile/binary/TimestampEncoding.java b/presto-rcfile/src/main/java/io/prestosql/rcfile/binary/TimestampEncoding.java index 2403fb5341eb..0f84ee62f74f 100644 --- a/presto-rcfile/src/main/java/io/prestosql/rcfile/binary/TimestampEncoding.java +++ b/presto-rcfile/src/main/java/io/prestosql/rcfile/binary/TimestampEncoding.java @@ -19,21 +19,21 @@ import io.prestosql.plugin.base.type.PrestoTimestampEncoder; import io.prestosql.rcfile.ColumnData; import io.prestosql.rcfile.EncodeOutput; +import io.prestosql.rcfile.TimestampHolder; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.BlockBuilder; import io.prestosql.spi.type.TimestampType; import org.joda.time.DateTimeZone; +import java.util.function.BiFunction; + import static io.airlift.slice.SizeOf.SIZE_OF_INT; import static io.prestosql.plugin.base.type.PrestoTimestampEncoderFactory.createTimestampEncoder; import static io.prestosql.rcfile.RcFileDecoderUtils.decodeVIntSize; import static io.prestosql.rcfile.RcFileDecoderUtils.isNegativeVInt; import static io.prestosql.rcfile.RcFileDecoderUtils.readVInt; import static io.prestosql.rcfile.RcFileDecoderUtils.writeVInt; -import static io.prestosql.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; -import static java.lang.Math.floorDiv; -import static java.lang.Math.floorMod; -import static java.lang.Math.toIntExact; +import static io.prestosql.spi.type.Timestamps.MILLISECONDS_PER_SECOND; import static java.util.Objects.requireNonNull; public class TimestampEncoding @@ -53,9 +53,10 @@ public TimestampEncoding(TimestampType type, DateTimeZone timeZone) @Override public void encodeColumn(Block block, SliceOutput output, EncodeOutput encodeOutput) { + BiFunction factory = TimestampHolder.getFactory(type); for (int position = 0; position < block.getPositionCount(); position++) { if (!block.isNull(position)) { - writeTimestamp(output, floorDiv(type.getLong(block, position), MICROSECONDS_PER_MILLISECOND)); + writeTimestamp(output, factory.apply(block, position)); } encodeOutput.closeEntry(); } @@ -64,7 +65,7 @@ public void encodeColumn(Block block, SliceOutput output, EncodeOutput encodeOut @Override public void encodeValueInto(Block block, int position, SliceOutput output) { - writeTimestamp(output, floorDiv(type.getLong(block, position), MICROSECONDS_PER_MILLISECOND)); + writeTimestamp(output, TimestampHolder.getFactory(type).apply(block, position)); } @Override @@ -176,11 +177,11 @@ private static int decodeNanos(int nanos) return nanos; } - private void writeTimestamp(SliceOutput output, long millis) + private void writeTimestamp(SliceOutput output, TimestampHolder timestamp) { - millis = timeZone.convertLocalToUTC(millis, false); - long seconds = floorDiv(millis, 1000); - int nanos = toIntExact(floorMod(millis, 1000) * 1_000_000); + long millis = timeZone.convertLocalToUTC(timestamp.getSeconds() * MILLISECONDS_PER_SECOND, false); + long seconds = millis / MILLISECONDS_PER_SECOND; + int nanos = timestamp.getNanosOfSecond(); writeTimestamp(seconds, nanos, output); } diff --git a/presto-rcfile/src/main/java/io/prestosql/rcfile/text/TimestampEncoding.java b/presto-rcfile/src/main/java/io/prestosql/rcfile/text/TimestampEncoding.java index e1f268b038e4..1cc58465078a 100644 --- a/presto-rcfile/src/main/java/io/prestosql/rcfile/text/TimestampEncoding.java +++ b/presto-rcfile/src/main/java/io/prestosql/rcfile/text/TimestampEncoding.java @@ -19,20 +19,19 @@ import io.prestosql.plugin.base.type.PrestoTimestampEncoder; import io.prestosql.rcfile.ColumnData; import io.prestosql.rcfile.EncodeOutput; +import io.prestosql.rcfile.TimestampHolder; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.BlockBuilder; import io.prestosql.spi.type.TimestampType; -import org.joda.time.format.DateTimeFormat; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.temporal.ChronoField; +import java.util.function.BiFunction; import static io.prestosql.plugin.base.type.PrestoTimestampEncoderFactory.createTimestampEncoder; -import static io.prestosql.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; -import static java.lang.Math.floorDiv; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.util.Objects.requireNonNull; import static org.joda.time.DateTimeZone.UTC; @@ -45,9 +44,7 @@ public class TimestampEncoding .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) .optionalStart().appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, true).optionalEnd() .toFormatter(); - // TODO: switch to java.time when we implement writes with variable precision - private static final org.joda.time.format.DateTimeFormatter HIVE_TIMESTAMP_PRINTER = - new org.joda.time.format.DateTimeFormatterBuilder().append(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS").getPrinter()).toFormatter().withZoneUTC(); + private static final DateTimeFormatter HIVE_TIMESTAMP_PRINTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"); private final TimestampType type; private final Slice nullSequence; @@ -64,14 +61,15 @@ public TimestampEncoding(TimestampType type, Slice nullSequence) @Override public void encodeColumn(Block block, SliceOutput output, EncodeOutput encodeOutput) { + BiFunction factory = TimestampHolder.getFactory(type); for (int position = 0; position < block.getPositionCount(); position++) { if (block.isNull(position)) { output.writeBytes(nullSequence); } else { - long millis = floorDiv(type.getLong(block, position), MICROSECONDS_PER_MILLISECOND); + LocalDateTime localDateTime = factory.apply(block, position).toLocalDateTime(); buffer.setLength(0); - HIVE_TIMESTAMP_PRINTER.printTo(buffer, millis); + HIVE_TIMESTAMP_PRINTER.formatTo(localDateTime, buffer); for (int index = 0; index < buffer.length(); index++) { output.writeByte(buffer.charAt(index)); } @@ -83,9 +81,9 @@ public void encodeColumn(Block block, SliceOutput output, EncodeOutput encodeOut @Override public void encodeValueInto(int depth, Block block, int position, SliceOutput output) { - long millis = floorDiv(type.getLong(block, position), MICROSECONDS_PER_MILLISECOND); + LocalDateTime localDateTime = TimestampHolder.getFactory(type).apply(block, position).toLocalDateTime(); buffer.setLength(0); - HIVE_TIMESTAMP_PRINTER.printTo(buffer, millis); + HIVE_TIMESTAMP_PRINTER.formatTo(localDateTime, buffer); for (int index = 0; index < buffer.length(); index++) { output.writeByte(buffer.charAt(index)); } diff --git a/presto-spi/src/main/java/io/prestosql/spi/type/SqlTimestamp.java b/presto-spi/src/main/java/io/prestosql/spi/type/SqlTimestamp.java index 2d8ce3922b17..1aa44b6800ee 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/type/SqlTimestamp.java +++ b/presto-spi/src/main/java/io/prestosql/spi/type/SqlTimestamp.java @@ -28,7 +28,6 @@ import static io.prestosql.spi.type.Timestamps.roundDiv; import static java.lang.Math.floorDiv; import static java.lang.Math.floorMod; -import static java.lang.Math.toIntExact; import static java.lang.String.format; public final class SqlTimestamp @@ -152,7 +151,7 @@ public LocalDateTime toLocalDateTime() long epochSecond = floorDiv(epochMicros, MICROSECONDS_PER_SECOND); int microOfSecond = floorMod(epochMicros, MICROSECONDS_PER_SECOND); int nanoOfSecond = (microOfSecond * NANOSECONDS_PER_MICROSECOND) + - toIntExact(roundDiv(picosOfMicros, PICOSECONDS_PER_NANOSECOND)); + roundDiv(picosOfMicros, PICOSECONDS_PER_NANOSECOND); return LocalDateTime.ofEpochSecond(epochSecond, nanoOfSecond, ZoneOffset.UTC); } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/type/Timestamps.java b/presto-spi/src/main/java/io/prestosql/spi/type/Timestamps.java index 169c17761151..0671d9bece92 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/type/Timestamps.java +++ b/presto-spi/src/main/java/io/prestosql/spi/type/Timestamps.java @@ -102,6 +102,12 @@ private static long scaleFactor(int fromPrecision, int toPrecision) return POWERS_OF_TEN[toPrecision - fromPrecision]; } + @SuppressWarnings("NumericCastThatLosesPrecision") + public static int roundDiv(int value, long factor) + { + return (int) roundDiv((long) value, factor); + } + public static long roundDiv(long value, long factor) { if (factor <= 0) {