diff --git a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java index 3708dafc4126..f576de9238cf 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java +++ b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java @@ -36,13 +36,14 @@ import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.DateTimeUtil; /** Expression utility methods. */ public class ExpressionUtil { private static final Function HASH_FUNC = Transforms.bucket(Integer.MAX_VALUE).bind(Types.StringType.get()); private static final OffsetDateTime EPOCH = Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC); - private static final long FIVE_MINUTES_IN_MICROS = TimeUnit.MINUTES.toMicros(5); + private static final long FIVE_MINUTES_IN_MILLIS = TimeUnit.MINUTES.toMillis(5); private static final long THREE_DAYS_IN_HOURS = TimeUnit.DAYS.toHours(3); private static final long NINETY_DAYS_IN_HOURS = TimeUnit.DAYS.toHours(90); private static final Pattern DATE = Pattern.compile("\\d{4}-\\d{2}-\\d{2}"); @@ -247,13 +248,12 @@ public static UnboundTerm unbind(Term term) { private static class ExpressionSanitizer extends ExpressionVisitors.ExpressionVisitor { - private final long now; + private final long nowMillis; private final int today; private ExpressionSanitizer() { - long nowMillis = System.currentTimeMillis(); - OffsetDateTime nowDateTime = Instant.ofEpochMilli(nowMillis).atOffset(ZoneOffset.UTC); - this.now = nowMillis * 1000; + this.nowMillis = System.currentTimeMillis(); + OffsetDateTime nowDateTime = Instant.ofEpochMilli(this.nowMillis).atOffset(ZoneOffset.UTC); this.today = (int) ChronoUnit.DAYS.between(EPOCH, nowDateTime); } @@ -293,13 +293,13 @@ public Expression predicate(BoundPredicate pred) { return new UnboundPredicate<>( pred.op(), unbind(pred.term()), - (T) sanitize(bound.term().type(), bound.literal(), now, today)); + (T) sanitize(bound.term().type(), bound.literal(), nowMillis, today)); } else if (pred.isSetPredicate()) { BoundSetPredicate bound = (BoundSetPredicate) pred; Iterable iter = () -> bound.literalSet().stream() - .map(lit -> (T) sanitize(bound.term().type(), lit, now, today)) + .map(lit -> (T) sanitize(bound.term().type(), lit, nowMillis, today)) .iterator(); return new UnboundPredicate<>(pred.op(), unbind(pred.term()), iter); } @@ -326,11 +326,11 @@ public Expression predicate(UnboundPredicate pred) { case STARTS_WITH: case NOT_STARTS_WITH: return new UnboundPredicate<>( - pred.op(), pred.term(), (T) sanitize(pred.literal(), now, today)); + pred.op(), pred.term(), (T) sanitize(pred.literal(), nowMillis, today)); case IN: case NOT_IN: Iterable iter = - () -> pred.literals().stream().map(lit -> sanitize(lit, now, today)).iterator(); + () -> pred.literals().stream().map(lit -> sanitize(lit, nowMillis, today)).iterator(); return new UnboundPredicate<>(pred.op(), pred.term(), (Iterable) iter); default: throw new UnsupportedOperationException( @@ -340,13 +340,12 @@ public Expression predicate(UnboundPredicate pred) { } private static class StringSanitizer extends ExpressionVisitors.ExpressionVisitor { - private final long nowMicros; + private final long nowMillis; private final int today; private StringSanitizer() { - long nowMillis = System.currentTimeMillis(); - OffsetDateTime nowDateTime = Instant.ofEpochMilli(nowMillis).atOffset(ZoneOffset.UTC); - this.nowMicros = nowMillis * 1000; + this.nowMillis = System.currentTimeMillis(); + OffsetDateTime nowDateTime = Instant.ofEpochMilli(this.nowMillis).atOffset(ZoneOffset.UTC); this.today = (int) ChronoUnit.DAYS.between(EPOCH, nowDateTime); } @@ -376,7 +375,7 @@ public String or(String leftResult, String rightResult) { } private String value(BoundLiteralPredicate pred) { - return sanitize(pred.term().type(), pred.literal().value(), nowMicros, today); + return sanitize(pred.term().type(), pred.literal().value(), nowMillis, today); } @Override @@ -408,7 +407,7 @@ public String predicate(BoundPredicate pred) { + " IN " + abbreviateValues( pred.asSetPredicate().literalSet().stream() - .map(lit -> sanitize(pred.term().type(), lit, nowMicros, today)) + .map(lit -> sanitize(pred.term().type(), lit, nowMillis, today)) .collect(Collectors.toList())) .stream() .collect(Collectors.joining(", ", "(", ")")); @@ -417,7 +416,7 @@ public String predicate(BoundPredicate pred) { + " NOT IN " + abbreviateValues( pred.asSetPredicate().literalSet().stream() - .map(lit -> sanitize(pred.term().type(), lit, nowMicros, today)) + .map(lit -> sanitize(pred.term().type(), lit, nowMillis, today)) .collect(Collectors.toList())) .stream() .collect(Collectors.joining(", ", "(", ")")); @@ -444,23 +443,23 @@ public String predicate(UnboundPredicate pred) { case NOT_NAN: return "not_nan(" + term + ")"; case LT: - return term + " < " + sanitize(pred.literal(), nowMicros, today); + return term + " < " + sanitize(pred.literal(), nowMillis, today); case LT_EQ: - return term + " <= " + sanitize(pred.literal(), nowMicros, today); + return term + " <= " + sanitize(pred.literal(), nowMillis, today); case GT: - return term + " > " + sanitize(pred.literal(), nowMicros, today); + return term + " > " + sanitize(pred.literal(), nowMillis, today); case GT_EQ: - return term + " >= " + sanitize(pred.literal(), nowMicros, today); + return term + " >= " + sanitize(pred.literal(), nowMillis, today); case EQ: - return term + " = " + sanitize(pred.literal(), nowMicros, today); + return term + " = " + sanitize(pred.literal(), nowMillis, today); case NOT_EQ: - return term + " != " + sanitize(pred.literal(), nowMicros, today); + return term + " != " + sanitize(pred.literal(), nowMillis, today); case IN: return term + " IN " + abbreviateValues( pred.literals().stream() - .map(lit -> sanitize(lit, nowMicros, today)) + .map(lit -> sanitize(lit, nowMillis, today)) .collect(Collectors.toList())) .stream() .collect(Collectors.joining(", ", "(", ")")); @@ -469,14 +468,14 @@ public String predicate(UnboundPredicate pred) { + " NOT IN " + abbreviateValues( pred.literals().stream() - .map(lit -> sanitize(lit, nowMicros, today)) + .map(lit -> sanitize(lit, nowMillis, today)) .collect(Collectors.toList())) .stream() .collect(Collectors.joining(", ", "(", ")")); case STARTS_WITH: - return term + " STARTS WITH " + sanitize(pred.literal(), nowMicros, today); + return term + " STARTS WITH " + sanitize(pred.literal(), nowMillis, today); case NOT_STARTS_WITH: - return term + " NOT STARTS WITH " + sanitize(pred.literal(), nowMicros, today); + return term + " NOT STARTS WITH " + sanitize(pred.literal(), nowMillis, today); default: throw new UnsupportedOperationException( "Cannot sanitize unsupported predicate type: " + pred.op()); @@ -501,7 +500,7 @@ private static List abbreviateValues(List sanitizedValues) { return sanitizedValues; } - private static String sanitize(Type type, Object value, long now, int today) { + private static String sanitize(Type type, Object value, long nowMillis, int today) { switch (type.typeId()) { case INTEGER: case LONG: @@ -514,9 +513,9 @@ private static String sanitize(Type type, Object value, long now, int today) { case TIME: return "(time)"; case TIMESTAMP: - return sanitizeTimestamp((long) value, now); + return sanitizeTimestamp(((Types.TimestampType) type).unit(), (long) value, nowMillis); case STRING: - return sanitizeString((CharSequence) value, now, today); + return sanitizeString((CharSequence) value, nowMillis, today); case BOOLEAN: case UUID: case DECIMAL: @@ -529,13 +528,16 @@ private static String sanitize(Type type, Object value, long now, int today) { String.format("Cannot sanitize value for unsupported type %s: %s", type, value)); } - private static String sanitize(Literal literal, long now, int today) { + private static String sanitize(Literal literal, long nowMillis, int today) { if (literal instanceof Literals.StringLiteral) { - return sanitizeString(((Literals.StringLiteral) literal).value(), now, today); + return sanitizeString(((Literals.StringLiteral) literal).value(), nowMillis, today); } else if (literal instanceof Literals.DateLiteral) { return sanitizeDate(((Literals.DateLiteral) literal).value(), today); } else if (literal instanceof Literals.TimestampLiteral) { - return sanitizeTimestamp(((Literals.TimestampLiteral) literal).value(), now); + return sanitizeTimestamp( + ((Literals.TimestampLiteral) literal).unit(), + ((Literals.TimestampLiteral) literal).value(), + nowMillis); } else if (literal instanceof Literals.TimeLiteral) { return "(time)"; } else if (literal instanceof Literals.IntegerLiteral) { @@ -564,14 +566,26 @@ private static String sanitizeDate(int days, int today) { return "(date)"; } - private static String sanitizeTimestamp(long micros, long now) { - String isPast = now > micros ? "ago" : "from-now"; - long diff = Math.abs(now - micros); - if (diff < FIVE_MINUTES_IN_MICROS) { + private static String sanitizeTimestamp(ChronoUnit unit, long timeUnits, long nowMillis) { + long timeMillis; + switch (unit) { + case MICROS: + timeMillis = DateTimeUtil.microsToMillis(timeUnits); + break; + case NANOS: + timeMillis = DateTimeUtil.nanosToMillis(timeUnits); + break; + default: + throw new UnsupportedOperationException("Unsupported timestamp unit: " + unit); + } + + long diff = Math.abs(nowMillis - timeMillis); + if (diff < FIVE_MINUTES_IN_MILLIS) { return "(timestamp-about-now)"; } - long hours = TimeUnit.MICROSECONDS.toHours(diff); + String isPast = nowMillis > timeMillis ? "ago" : "from-now"; + long hours = DateTimeUtil.millisToHours(diff); if (hours <= THREE_DAYS_IN_HOURS) { return "(timestamp-" + hours + "-hours-" + isPast + ")"; } else if (hours < NINETY_DAYS_IN_HOURS) { @@ -589,17 +603,17 @@ private static String sanitizeNumber(Number value, String type) { return "(" + numDigits + "-digit-" + type + ")"; } - private static String sanitizeString(CharSequence value, long now, int today) { + private static String sanitizeString(CharSequence value, long nowMillis, int today) { try { if (DATE.matcher(value).matches()) { Literal date = Literal.of(value).to(Types.DateType.get()); return sanitizeDate(date.value(), today); } else if (TIMESTAMP.matcher(value).matches()) { - Literal ts = Literal.of(value).to(Types.TimestampType.withoutZone()); - return sanitizeTimestamp(ts.value(), now); + Literal ts = Literal.of(value).to(Types.TimestampType.nanosWithoutZone()); + return sanitizeTimestamp(ChronoUnit.NANOS, ts.value(), nowMillis); } else if (TIMESTAMPTZ.matcher(value).matches()) { - Literal ts = Literal.of(value).to(Types.TimestampType.withZone()); - return sanitizeTimestamp(ts.value(), now); + Literal ts = Literal.of(value).to(Types.TimestampType.nanosWithZone()); + return sanitizeTimestamp(ChronoUnit.NANOS, ts.value(), nowMillis); } else if (TIME.matcher(value).matches()) { return "(time)"; } else { diff --git a/api/src/main/java/org/apache/iceberg/expressions/Literals.java b/api/src/main/java/org/apache/iceberg/expressions/Literals.java index 79d7190c49df..c0ba36ec0c30 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/Literals.java +++ b/api/src/main/java/org/apache/iceberg/expressions/Literals.java @@ -39,7 +39,9 @@ import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.types.Types.TimestampType; import org.apache.iceberg.util.ByteBuffers; +import org.apache.iceberg.util.DateTimeUtil; import org.apache.iceberg.util.NaNUtil; class Literals { @@ -298,7 +300,7 @@ public Literal to(Type type) { case TIME: return (Literal) new TimeLiteral(value()); case TIMESTAMP: - return (Literal) new TimestampLiteral(value()); + return (Literal) new TimestampLiteral(((TimestampType) type).unit(), value()); case DATE: if ((long) Integer.MAX_VALUE < value()) { return aboveMax(); @@ -426,8 +428,11 @@ protected Type.TypeID typeId() { } static class TimestampLiteral extends ComparableLiteral { - TimestampLiteral(Long value) { + private final ChronoUnit unit; + + TimestampLiteral(ChronoUnit unit, Long value) { super(value); + this.unit = unit; } @Override @@ -435,7 +440,28 @@ static class TimestampLiteral extends ComparableLiteral { public Literal to(Type type) { switch (type.typeId()) { case TIMESTAMP: - return (Literal) this; + ChronoUnit toUnit = ((TimestampType) type).unit(); + switch (unit) { + case MICROS: + switch (toUnit) { + case MICROS: + return (Literal) this; + case NANOS: + return (Literal) + new TimestampLiteral(unit, DateTimeUtil.microsToNanos(value())); + } + break; + case NANOS: + switch (toUnit) { + case MICROS: + return (Literal) + new TimestampLiteral(unit, DateTimeUtil.nanosToMicros(value())); + case NANOS: + return (Literal) this; + } + break; + } + break; case DATE: return (Literal) new DateLiteral( @@ -451,6 +477,10 @@ public Literal to(Type type) { protected Type.TypeID typeId() { return Type.TypeID.TIMESTAMP; } + + protected ChronoUnit unit() { + return unit; + } } static class DecimalLiteral extends ComparableLiteral { @@ -501,18 +531,22 @@ public Literal to(Type type) { return (Literal) new TimeLiteral(timeMicros); case TIMESTAMP: - if (((Types.TimestampType) type).shouldAdjustToUTC()) { - long timestampMicros = - ChronoUnit.MICROS.between( - EPOCH, OffsetDateTime.parse(value(), DateTimeFormatter.ISO_DATE_TIME)); - return (Literal) new TimestampLiteral(timestampMicros); + TimestampType tsType = (TimestampType) type; + if (tsType.shouldAdjustToUTC()) { + long timestampUnits = + tsType + .unit() + .between(EPOCH, OffsetDateTime.parse(value(), DateTimeFormatter.ISO_DATE_TIME)); + return (Literal) new TimestampLiteral(tsType.unit(), timestampUnits); } else { - long timestampMicros = - ChronoUnit.MICROS.between( - EPOCH, - LocalDateTime.parse(value(), DateTimeFormatter.ISO_LOCAL_DATE_TIME) - .atOffset(ZoneOffset.UTC)); - return (Literal) new TimestampLiteral(timestampMicros); + long timestampUnits = + tsType + .unit() + .between( + EPOCH, + LocalDateTime.parse(value(), DateTimeFormatter.ISO_LOCAL_DATE_TIME) + .atOffset(ZoneOffset.UTC)); + return (Literal) new TimestampLiteral(tsType.unit(), timestampUnits); } case STRING: diff --git a/api/src/main/java/org/apache/iceberg/transforms/Days.java b/api/src/main/java/org/apache/iceberg/transforms/Days.java index f69d5d6110ed..d8e377757955 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Days.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Days.java @@ -19,6 +19,7 @@ package org.apache.iceberg.transforms; import java.io.ObjectStreamException; +import java.time.temporal.ChronoUnit; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -37,7 +38,7 @@ protected Transform toEnum(Type type) { case DATE: return (Transform) Dates.DAY; case TIMESTAMP: - return (Transform) Timestamps.DAY; + return (Transform) Timestamps.get((Types.TimestampType) type, ChronoUnit.DAYS); default: throw new IllegalArgumentException("Unsupported type: " + type); } @@ -55,14 +56,14 @@ public boolean satisfiesOrderOf(Transform other) { } if (other instanceof Timestamps) { - return Timestamps.DAY.satisfiesOrderOf(other); + return ((Timestamps) other).getResultTypeUnit() == ChronoUnit.DAYS + || ((Timestamps) other).getResultTypeUnit() == ChronoUnit.MONTHS + || ((Timestamps) other).getResultTypeUnit() == ChronoUnit.YEARS; } else if (other instanceof Dates) { return Dates.DAY.satisfiesOrderOf(other); - } else if (other instanceof Days || other instanceof Months || other instanceof Years) { - return true; + } else { + return other instanceof Days || other instanceof Months || other instanceof Years; } - - return false; } @Override diff --git a/api/src/main/java/org/apache/iceberg/transforms/Hours.java b/api/src/main/java/org/apache/iceberg/transforms/Hours.java index afc14516f3cd..f12d8856cbd8 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Hours.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Hours.java @@ -19,6 +19,7 @@ package org.apache.iceberg.transforms; import java.io.ObjectStreamException; +import java.time.temporal.ChronoUnit; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -34,7 +35,7 @@ static Hours get() { @SuppressWarnings("unchecked") protected Transform toEnum(Type type) { if (type.typeId() == Type.TypeID.TIMESTAMP) { - return (Transform) Timestamps.HOUR; + return (Transform) Timestamps.get((Types.TimestampType) type, ChronoUnit.HOURS); } throw new IllegalArgumentException("Unsupported type: " + type); @@ -57,15 +58,16 @@ public boolean satisfiesOrderOf(Transform other) { } if (other instanceof Timestamps) { - return other == Timestamps.HOUR; - } else if (other instanceof Hours - || other instanceof Days - || other instanceof Months - || other instanceof Years) { - return true; + return ((Timestamps) other).getResultTypeUnit() == ChronoUnit.HOURS + || ((Timestamps) other).getResultTypeUnit() == ChronoUnit.DAYS + || ((Timestamps) other).getResultTypeUnit() == ChronoUnit.MONTHS + || ((Timestamps) other).getResultTypeUnit() == ChronoUnit.YEARS; + } else { + return other instanceof Hours + || other instanceof Days + || other instanceof Months + || other instanceof Years; } - - return false; } @Override diff --git a/api/src/main/java/org/apache/iceberg/transforms/Months.java b/api/src/main/java/org/apache/iceberg/transforms/Months.java index 8fa4d42385f7..a1d60c4fcf58 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Months.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Months.java @@ -19,6 +19,7 @@ package org.apache.iceberg.transforms; import java.io.ObjectStreamException; +import java.time.temporal.ChronoUnit; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -37,7 +38,8 @@ protected Transform toEnum(Type type) { case DATE: return (Transform) Dates.MONTH; case TIMESTAMP: - return (Transform) Timestamps.MONTH; + return (Transform) + Timestamps.get((Types.TimestampType) type, ChronoUnit.MONTHS); default: throw new IllegalArgumentException("Unsupported type: " + type); } @@ -55,14 +57,13 @@ public boolean satisfiesOrderOf(Transform other) { } if (other instanceof Timestamps) { - return Timestamps.MONTH.satisfiesOrderOf(other); + return ((Timestamps) other).getResultTypeUnit() == ChronoUnit.MONTHS + || ((Timestamps) other).getResultTypeUnit() == ChronoUnit.YEARS; } else if (other instanceof Dates) { return Dates.MONTH.satisfiesOrderOf(other); - } else if (other instanceof Months || other instanceof Years) { - return true; + } else { + return other instanceof Months || other instanceof Years; } - - return false; } @Override diff --git a/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java b/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java index e4796478bf28..1c77039b5f40 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java +++ b/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java @@ -121,17 +121,13 @@ static R visit(Schema schema, PartitionField field, PartitionSpecVisitor } else if (transform instanceof Truncate) { int width = ((Truncate) transform).width(); return visitor.truncate(field.fieldId(), sourceName, field.sourceId(), width); - } else if (transform == Dates.YEAR - || transform == Timestamps.YEAR - || transform instanceof Years) { + } else if ("year".equalsIgnoreCase(transform.toString())) { return visitor.year(field.fieldId(), sourceName, field.sourceId()); - } else if (transform == Dates.MONTH - || transform == Timestamps.MONTH - || transform instanceof Months) { + } else if ("month".equalsIgnoreCase(transform.toString())) { return visitor.month(field.fieldId(), sourceName, field.sourceId()); - } else if (transform == Dates.DAY || transform == Timestamps.DAY || transform instanceof Days) { + } else if ("day".equalsIgnoreCase(transform.toString())) { return visitor.day(field.fieldId(), sourceName, field.sourceId()); - } else if (transform == Timestamps.HOUR || transform instanceof Hours) { + } else if ("hour".equalsIgnoreCase(transform.toString())) { return visitor.hour(field.fieldId(), sourceName, field.sourceId()); } else if (transform instanceof VoidTransform) { return visitor.alwaysNull(field.fieldId(), sourceName, field.sourceId()); diff --git a/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java b/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java index 680e095270fb..4712fee60049 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java +++ b/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java @@ -84,22 +84,16 @@ static List visit(SortOrder sortOrder, SortOrderVisitor visitor) { results.add( visitor.truncate( sourceName, field.sourceId(), width, field.direction(), field.nullOrder())); - } else if (transform == Dates.YEAR - || transform == Timestamps.YEAR - || transform instanceof Years) { + } else if ("year".equalsIgnoreCase(transform.toString())) { results.add( visitor.year(sourceName, field.sourceId(), field.direction(), field.nullOrder())); - } else if (transform == Dates.MONTH - || transform == Timestamps.MONTH - || transform instanceof Months) { + } else if ("month".equalsIgnoreCase(transform.toString())) { results.add( visitor.month(sourceName, field.sourceId(), field.direction(), field.nullOrder())); - } else if (transform == Dates.DAY - || transform == Timestamps.DAY - || transform instanceof Days) { + } else if ("day".equalsIgnoreCase(transform.toString())) { results.add( visitor.day(sourceName, field.sourceId(), field.direction(), field.nullOrder())); - } else if (transform == Timestamps.HOUR || transform instanceof Hours) { + } else if ("day".equalsIgnoreCase(transform.toString())) { results.add( visitor.hour(sourceName, field.sourceId(), field.direction(), field.nullOrder())); } else if (transform instanceof UnknownTransform) { diff --git a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java index b5b50e9d42b2..a82d0288c45c 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java @@ -20,6 +20,7 @@ import com.google.errorprone.annotations.Immutable; import java.time.temporal.ChronoUnit; +import java.util.Locale; import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.BoundTransform; import org.apache.iceberg.expressions.Expression; @@ -28,57 +29,129 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.types.Types.TimestampType; import org.apache.iceberg.util.DateTimeUtil; import org.apache.iceberg.util.SerializableFunction; -enum Timestamps implements Transform { - YEAR(ChronoUnit.YEARS, "year"), - MONTH(ChronoUnit.MONTHS, "month"), - DAY(ChronoUnit.DAYS, "day"), - HOUR(ChronoUnit.HOURS, "hour"); +class Timestamps implements Transform { + + static final Timestamps YEAR_FROM_MICROS = new Timestamps(ChronoUnit.MICROS, ChronoUnit.YEARS); + static final Timestamps MONTH_FROM_MICROS = new Timestamps(ChronoUnit.MICROS, ChronoUnit.MONTHS); + static final Timestamps DAY_FROM_MICROS = new Timestamps(ChronoUnit.MICROS, ChronoUnit.DAYS); + static final Timestamps HOUR_FROM_MICROS = new Timestamps(ChronoUnit.MICROS, ChronoUnit.HOURS); + static final Timestamps YEAR_FROM_NANOS = new Timestamps(ChronoUnit.NANOS, ChronoUnit.YEARS); + static final Timestamps MONTH_FROM_NANOS = new Timestamps(ChronoUnit.NANOS, ChronoUnit.MONTHS); + static final Timestamps DAY_FROM_NANOS = new Timestamps(ChronoUnit.NANOS, ChronoUnit.DAYS); + static final Timestamps HOUR_FROM_NANOS = new Timestamps(ChronoUnit.NANOS, ChronoUnit.HOURS); + + static Timestamps get(TimestampType type, String resultTypeUnit) { + switch (resultTypeUnit.toLowerCase(Locale.ENGLISH)) { + case "year": + return get(type, ChronoUnit.YEARS); + case "month": + return get(type, ChronoUnit.MONTHS); + case "day": + return get(type, ChronoUnit.DAYS); + case "hour": + return get(type, ChronoUnit.HOURS); + default: + throw new IllegalArgumentException( + "Unsupported source/result type units: " + type + "->" + resultTypeUnit); + } + } + + static Timestamps get(TimestampType type, ChronoUnit resultTypeUnit) { + switch (type.unit()) { + case MICROS: + switch (resultTypeUnit) { + case YEARS: + return YEAR_FROM_MICROS; + case MONTHS: + return MONTH_FROM_MICROS; + case DAYS: + return DAY_FROM_MICROS; + case HOURS: + return HOUR_FROM_MICROS; + } + break; + case NANOS: + switch (resultTypeUnit) { + case YEARS: + return YEAR_FROM_NANOS; + case MONTHS: + return MONTH_FROM_NANOS; + case DAYS: + return DAY_FROM_NANOS; + case HOURS: + return HOUR_FROM_NANOS; + } + break; + } + throw new IllegalArgumentException( + "Unsupported source/result type units: " + type + "->" + resultTypeUnit); + } @Immutable static class Apply implements SerializableFunction { - private final ChronoUnit granularity; + private final ChronoUnit sourceTypeUnit; + private final ChronoUnit resultTypeUnit; - Apply(ChronoUnit granularity) { - this.granularity = granularity; + Apply(ChronoUnit sourceTypeUnit, ChronoUnit resultTypeUnit) { + this.sourceTypeUnit = sourceTypeUnit; + this.resultTypeUnit = resultTypeUnit; } @Override - public Integer apply(Long timestampMicros) { - if (timestampMicros == null) { + public Integer apply(Long timestampUnits) { + if (timestampUnits == null) { return null; } - switch (granularity) { - case YEARS: - return DateTimeUtil.microsToYears(timestampMicros); - case MONTHS: - return DateTimeUtil.microsToMonths(timestampMicros); - case DAYS: - return DateTimeUtil.microsToDays(timestampMicros); - case HOURS: - return DateTimeUtil.microsToHours(timestampMicros); + switch (sourceTypeUnit) { + case MICROS: + switch (resultTypeUnit) { + case YEARS: + return DateTimeUtil.microsToYears(timestampUnits); + case MONTHS: + return DateTimeUtil.microsToMonths(timestampUnits); + case DAYS: + return DateTimeUtil.microsToDays(timestampUnits); + case HOURS: + return DateTimeUtil.microsToHours(timestampUnits); + default: + throw new UnsupportedOperationException( + "Unsupported result type unit: " + resultTypeUnit); + } + case NANOS: + switch (resultTypeUnit) { + case YEARS: + return DateTimeUtil.nanosToYears(timestampUnits); + case MONTHS: + return DateTimeUtil.nanosToMonths(timestampUnits); + case DAYS: + return DateTimeUtil.nanosToDays(timestampUnits); + case HOURS: + return DateTimeUtil.nanosToHours(timestampUnits); + default: + throw new UnsupportedOperationException( + "Unsupported result type unit: " + resultTypeUnit); + } default: - throw new UnsupportedOperationException("Unsupported time unit: " + granularity); + throw new UnsupportedOperationException( + "Unsupported source type unit: " + sourceTypeUnit); } } } - private final ChronoUnit granularity; - private final String name; private final Apply apply; - Timestamps(ChronoUnit granularity, String name) { - this.granularity = granularity; - this.name = name; - this.apply = new Apply(granularity); + Timestamps(ChronoUnit sourceTypeUnit, ChronoUnit resultTypeUnit) { + this.apply = new Apply(sourceTypeUnit, resultTypeUnit); } @Override - public Integer apply(Long timestampMicros) { - return apply.apply(timestampMicros); + public Integer apply(Long timestampUnits) { + return apply.apply(timestampUnits); } @Override @@ -94,12 +167,16 @@ public boolean canTransform(Type type) { @Override public Type getResultType(Type sourceType) { - if (granularity == ChronoUnit.DAYS) { + if (apply.resultTypeUnit == ChronoUnit.DAYS) { return Types.DateType.get(); } return Types.IntegerType.get(); } + public ChronoUnit getResultTypeUnit() { + return apply.resultTypeUnit; + } + @Override public boolean preservesOrder() { return true; @@ -112,11 +189,11 @@ public boolean satisfiesOrderOf(Transform other) { } if (other instanceof Timestamps) { - // test the granularity, in hours. hour(ts) => 1 hour, day(ts) => 24 hours, and hour satisfies - // the order of day + // test the granularity, in hours. hour(ts) => 1 hour, day(ts) => 24 hours, and + // hour satisfies the order of day Timestamps otherTransform = (Timestamps) other; - return granularity.getDuration().toHours() - <= otherTransform.granularity.getDuration().toHours(); + return apply.resultTypeUnit.getDuration().toHours() + <= otherTransform.apply.resultTypeUnit.getDuration().toHours(); } return false; @@ -174,7 +251,7 @@ public String toHumanString(Type outputType, Integer value) { return "null"; } - switch (granularity) { + switch (apply.resultTypeUnit) { case YEARS: return TransformUtil.humanYear(value); case MONTHS: @@ -184,13 +261,25 @@ public String toHumanString(Type outputType, Integer value) { case HOURS: return TransformUtil.humanHour(value); default: - throw new UnsupportedOperationException("Unsupported time unit: " + granularity); + throw new UnsupportedOperationException("Unsupported time unit: " + apply.resultTypeUnit); } } @Override public String toString() { - return name; + switch (apply.resultTypeUnit) { + case YEARS: + return "year"; + case MONTHS: + return "month"; + case DAYS: + return "day"; + case HOURS: + return "hour"; + default: + throw new UnsupportedOperationException( + "Unsupported result time unit: " + apply.resultTypeUnit); + } } @Override diff --git a/api/src/main/java/org/apache/iceberg/transforms/Transform.java b/api/src/main/java/org/apache/iceberg/transforms/Transform.java index 5a56b672b1b1..0c5e7dd77d0f 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Transform.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Transform.java @@ -24,7 +24,7 @@ import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; +import org.apache.iceberg.types.Types.TimestampType; import org.apache.iceberg.util.SerializableFunction; /** @@ -176,11 +176,7 @@ default String toHumanString(Type type, T value) { case TIME: return TransformUtil.humanTime((Long) value); case TIMESTAMP: - if (((Types.TimestampType) type).shouldAdjustToUTC()) { - return TransformUtil.humanTimestampWithZone((Long) value); - } else { - return TransformUtil.humanTimestampWithoutZone((Long) value); - } + return TransformUtil.humanTimestamp((TimestampType) type, (Long) value); case FIXED: case BINARY: if (value instanceof ByteBuffer) { diff --git a/api/src/main/java/org/apache/iceberg/transforms/TransformUtil.java b/api/src/main/java/org/apache/iceberg/transforms/TransformUtil.java index 53bc23a49888..1f962a79813d 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/TransformUtil.java +++ b/api/src/main/java/org/apache/iceberg/transforms/TransformUtil.java @@ -26,6 +26,7 @@ import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.Base64; +import org.apache.iceberg.types.Types; class TransformUtil { @@ -54,12 +55,26 @@ static String humanTime(Long microsFromMidnight) { return LocalTime.ofNanoOfDay(microsFromMidnight * 1000).toString(); } - static String humanTimestampWithZone(Long timestampMicros) { - return ChronoUnit.MICROS.addTo(EPOCH, timestampMicros).toString(); - } - - static String humanTimestampWithoutZone(Long timestampMicros) { - return ChronoUnit.MICROS.addTo(EPOCH, timestampMicros).toLocalDateTime().toString(); + public static String humanTimestamp(Types.TimestampType tsType, Long value) { + if (tsType.shouldAdjustToUTC()) { + switch (tsType.unit()) { + case MICROS: + return ChronoUnit.MICROS.addTo(EPOCH, value).toString(); + case NANOS: + return ChronoUnit.NANOS.addTo(EPOCH, value).toString(); + default: + throw new IllegalArgumentException("Invalid timestamp unit: " + tsType.unit()); + } + } else { + switch (tsType.unit()) { + case MICROS: + return ChronoUnit.MICROS.addTo(EPOCH, value).toLocalDateTime().toString(); + case NANOS: + return ChronoUnit.NANOS.addTo(EPOCH, value).toLocalDateTime().toString(); + default: + throw new IllegalArgumentException("Invalid timestamp unit: " + tsType.unit()); + } + } } static String humanHour(int hourOrdinal) { diff --git a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java index a1ce33ddd6da..c7ddd8d17039 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java @@ -25,6 +25,7 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types.TimestampType; /** * Factory methods for transforms. @@ -86,8 +87,9 @@ private Transforms() {} try { if (type.typeId() == Type.TypeID.TIMESTAMP) { - return Timestamps.valueOf(transform.toUpperCase(Locale.ENGLISH)); - } else if (type.typeId() == Type.TypeID.DATE) { + return Timestamps.get((TimestampType) type, transform); + } + if (type.typeId() == Type.TypeID.DATE) { return Dates.valueOf(transform.toUpperCase(Locale.ENGLISH)); } } catch (IllegalArgumentException ignored) { @@ -129,10 +131,14 @@ public static Transform year(Type type) { case DATE: return (Transform) Dates.YEAR; case TIMESTAMP: - return (Transform) Timestamps.YEAR; - default: - throw new IllegalArgumentException("Cannot partition type " + type + " by year"); + switch (((TimestampType) type).unit()) { + case MICROS: + return (Transform) Timestamps.YEAR_FROM_MICROS; + case NANOS: + return (Transform) Timestamps.YEAR_FROM_NANOS; + } } + throw new IllegalArgumentException("Cannot partition type " + type + " by year"); } /** @@ -150,10 +156,14 @@ public static Transform month(Type type) { case DATE: return (Transform) Dates.MONTH; case TIMESTAMP: - return (Transform) Timestamps.MONTH; - default: - throw new IllegalArgumentException("Cannot partition type " + type + " by month"); + switch (((TimestampType) type).unit()) { + case MICROS: + return (Transform) Timestamps.MONTH_FROM_MICROS; + case NANOS: + return (Transform) Timestamps.MONTH_FROM_NANOS; + } } + throw new IllegalArgumentException("Cannot partition type " + type + " by month"); } /** @@ -171,10 +181,14 @@ public static Transform day(Type type) { case DATE: return (Transform) Dates.DAY; case TIMESTAMP: - return (Transform) Timestamps.DAY; - default: - throw new IllegalArgumentException("Cannot partition type " + type + " by day"); + switch (((TimestampType) type).unit()) { + case MICROS: + return (Transform) Timestamps.DAY_FROM_MICROS; + case NANOS: + return (Transform) Timestamps.DAY_FROM_NANOS; + } } + throw new IllegalArgumentException("Cannot partition type " + type + " by day"); } /** @@ -188,9 +202,16 @@ public static Transform day(Type type) { @Deprecated @SuppressWarnings("unchecked") public static Transform hour(Type type) { - Preconditions.checkArgument( - type.typeId() == Type.TypeID.TIMESTAMP, "Cannot partition type %s by hour", type); - return (Transform) Timestamps.HOUR; + if (Preconditions.checkNotNull(type.typeId(), "Type ID cannot be null") + == Type.TypeID.TIMESTAMP) { + switch (((TimestampType) type).unit()) { + case MICROS: + return (Transform) Timestamps.HOUR_FROM_MICROS; + case NANOS: + return (Transform) Timestamps.HOUR_FROM_NANOS; + } + } + throw new IllegalArgumentException("Cannot partition type " + type + " by hour"); } /** diff --git a/api/src/main/java/org/apache/iceberg/transforms/Years.java b/api/src/main/java/org/apache/iceberg/transforms/Years.java index 6c1eee578506..de81fabf7ec8 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Years.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Years.java @@ -19,6 +19,7 @@ package org.apache.iceberg.transforms; import java.io.ObjectStreamException; +import java.time.temporal.ChronoUnit; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -37,7 +38,7 @@ protected Transform toEnum(Type type) { case DATE: return (Transform) Dates.YEAR; case TIMESTAMP: - return (Transform) Timestamps.YEAR; + return (Transform) Timestamps.get((Types.TimestampType) type, ChronoUnit.YEARS); default: throw new IllegalArgumentException("Unsupported type: " + type); } @@ -55,14 +56,12 @@ public boolean satisfiesOrderOf(Transform other) { } if (other instanceof Timestamps) { - return Timestamps.YEAR.satisfiesOrderOf(other); + return ((Timestamps) other).getResultTypeUnit() == ChronoUnit.YEARS; } else if (other instanceof Dates) { return Dates.YEAR.satisfiesOrderOf(other); - } else if (other instanceof Years) { - return true; + } else { + return other instanceof Years; } - - return false; } @Override diff --git a/api/src/main/java/org/apache/iceberg/types/Comparators.java b/api/src/main/java/org/apache/iceberg/types/Comparators.java index d09d9f5395ce..ddc52446e041 100644 --- a/api/src/main/java/org/apache/iceberg/types/Comparators.java +++ b/api/src/main/java/org/apache/iceberg/types/Comparators.java @@ -39,8 +39,10 @@ private Comparators() {} .put(Types.DoubleType.get(), Comparator.naturalOrder()) .put(Types.DateType.get(), Comparator.naturalOrder()) .put(Types.TimeType.get(), Comparator.naturalOrder()) - .put(Types.TimestampType.withZone(), Comparator.naturalOrder()) - .put(Types.TimestampType.withoutZone(), Comparator.naturalOrder()) + .put(Types.TimestampType.microsWithZone(), Comparator.naturalOrder()) + .put(Types.TimestampType.microsWithoutZone(), Comparator.naturalOrder()) + .put(Types.TimestampType.nanosWithZone(), Comparator.naturalOrder()) + .put(Types.TimestampType.nanosWithoutZone(), Comparator.naturalOrder()) .put(Types.StringType.get(), Comparators.charSequences()) .put(Types.UUIDType.get(), Comparator.naturalOrder()) .put(Types.BinaryType.get(), Comparators.unsignedBytes()) diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java index da70dd9ac6ab..7dfd16541a04 100644 --- a/api/src/main/java/org/apache/iceberg/types/Types.java +++ b/api/src/main/java/org/apache/iceberg/types/Types.java @@ -19,6 +19,7 @@ package org.apache.iceberg.types; import java.io.Serializable; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.List; import java.util.Locale; @@ -46,8 +47,10 @@ private Types() {} .put(DoubleType.get().toString(), DoubleType.get()) .put(DateType.get().toString(), DateType.get()) .put(TimeType.get().toString(), TimeType.get()) - .put(TimestampType.withZone().toString(), TimestampType.withZone()) - .put(TimestampType.withoutZone().toString(), TimestampType.withoutZone()) + .put(TimestampType.microsWithZone().toString(), TimestampType.microsWithZone()) + .put(TimestampType.microsWithoutZone().toString(), TimestampType.microsWithoutZone()) + .put(TimestampType.nanosWithZone().toString(), TimestampType.nanosWithZone()) + .put(TimestampType.nanosWithoutZone().toString(), TimestampType.nanosWithoutZone()) .put(StringType.get().toString(), StringType.get()) .put(UUIDType.get().toString(), UUIDType.get()) .put(BinaryType.get().toString(), BinaryType.get()) @@ -205,27 +208,56 @@ public String toString() { } public static class TimestampType extends PrimitiveType { - private static final TimestampType INSTANCE_WITH_ZONE = new TimestampType(true); - private static final TimestampType INSTANCE_WITHOUT_ZONE = new TimestampType(false); + + private static final TimestampType INSTANCE_MICROS_WITH_ZONE = + new TimestampType(true, ChronoUnit.MICROS); + private static final TimestampType INSTANCE_MICROS_WITHOUT_ZONE = + new TimestampType(false, ChronoUnit.MICROS); + private static final TimestampType INSTANCE_NANOS_WITH_ZONE = + new TimestampType(true, ChronoUnit.NANOS); + private static final TimestampType INSTANCE_NANOS_WITHOUT_ZONE = + new TimestampType(false, ChronoUnit.NANOS); public static TimestampType withZone() { - return INSTANCE_WITH_ZONE; + return INSTANCE_MICROS_WITH_ZONE; } public static TimestampType withoutZone() { - return INSTANCE_WITHOUT_ZONE; + return INSTANCE_MICROS_WITHOUT_ZONE; + } + + public static TimestampType microsWithZone() { + return INSTANCE_MICROS_WITH_ZONE; + } + + public static TimestampType microsWithoutZone() { + return INSTANCE_MICROS_WITHOUT_ZONE; + } + + public static TimestampType nanosWithZone() { + return INSTANCE_NANOS_WITH_ZONE; + } + + public static TimestampType nanosWithoutZone() { + return INSTANCE_NANOS_WITHOUT_ZONE; } private final boolean adjustToUTC; + private final ChronoUnit unit; - private TimestampType(boolean adjustToUTC) { + private TimestampType(boolean adjustToUTC, ChronoUnit unit) { this.adjustToUTC = adjustToUTC; + this.unit = unit; } public boolean shouldAdjustToUTC() { return adjustToUTC; } + public ChronoUnit unit() { + return unit; + } + @Override public TypeID typeId() { return TypeID.TIMESTAMP; @@ -234,9 +266,23 @@ public TypeID typeId() { @Override public String toString() { if (shouldAdjustToUTC()) { - return "timestamptz"; + switch (unit) { + case MICROS: + return "timestamptz"; + case NANOS: + return "timestamptz_ns"; + default: + throw new IllegalArgumentException("Unsupported unit: " + unit); + } } else { - return "timestamp"; + switch (unit) { + case MICROS: + return "timestamp"; + case NANOS: + return "timestamp_ns"; + default: + throw new IllegalArgumentException("Unsupported unit: " + unit); + } } } @@ -249,12 +295,12 @@ public boolean equals(Object o) { } TimestampType timestampType = (TimestampType) o; - return adjustToUTC == timestampType.adjustToUTC; + return adjustToUTC == timestampType.adjustToUTC && unit == timestampType.unit; } @Override public int hashCode() { - return Objects.hash(TimestampType.class, adjustToUTC); + return Objects.hash(TimestampType.class, adjustToUTC, unit); } } diff --git a/api/src/main/java/org/apache/iceberg/util/DateTimeUtil.java b/api/src/main/java/org/apache/iceberg/util/DateTimeUtil.java index a2f5301f44a9..e9abddf54fad 100644 --- a/api/src/main/java/org/apache/iceberg/util/DateTimeUtil.java +++ b/api/src/main/java/org/apache/iceberg/util/DateTimeUtil.java @@ -33,8 +33,12 @@ private DateTimeUtil() {} public static final OffsetDateTime EPOCH = Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC); public static final LocalDate EPOCH_DAY = EPOCH.toLocalDate(); - public static final long MICROS_PER_MILLIS = 1000L; + public static final long MICROS_PER_MILLIS = 1_000L; + public static final long MILLIS_PER_SECOND = 1_000L; public static final long MICROS_PER_SECOND = 1_000_000L; + public static final long NANOS_PER_SECOND = 1_000_000_000L; + public static final long NANOS_PER_MILLI = 1_000_000L; + public static final long NANOS_PER_MICRO = 1_000L; public static LocalDate dateFromDays(int daysFromEpoch) { return ChronoUnit.DAYS.addTo(EPOCH_DAY, daysFromEpoch); @@ -75,6 +79,22 @@ public static long microsToMillis(long micros) { return Math.floorDiv(micros, MICROS_PER_MILLIS); } + public static long nanosToMillis(long nanos) { + return Math.floorDiv(nanos, NANOS_PER_MILLI); + } + + public static long nanosToMicros(long nanos) { + return Math.floorDiv(nanos, NANOS_PER_MICRO); + } + + public static long microsToNanos(long micros) { + return Math.multiplyExact(micros, NANOS_PER_MICRO); + } + + public static long millisToNanos(long millis) { + return Math.multiplyExact(millis, NANOS_PER_MILLI); + } + public static OffsetDateTime timestamptzFromMicros(long microsFromEpoch) { return ChronoUnit.MICROS.addTo(EPOCH, microsFromEpoch); } @@ -159,28 +179,76 @@ public static int microsToYears(long micros) { return convertMicros(micros, ChronoUnit.YEARS); } + public static int nanosToYears(long nanos) { + return convertNanos(nanos, ChronoUnit.YEARS); + } + public static int microsToMonths(long micros) { return convertMicros(micros, ChronoUnit.MONTHS); } + public static int nanosToMonths(long nanos) { + return convertNanos(nanos, ChronoUnit.MONTHS); + } + public static int microsToDays(long micros) { return convertMicros(micros, ChronoUnit.DAYS); } + public static int nanosToDays(long nanos) { + return convertNanos(nanos, ChronoUnit.DAYS); + } + + public static int millisToHours(long millis) { + return convertMillis(millis, ChronoUnit.HOURS); + } + public static int microsToHours(long micros) { return convertMicros(micros, ChronoUnit.HOURS); } + public static int nanosToHours(long nanos) { + return convertNanos(nanos, ChronoUnit.HOURS); + } + + private static int convertMillis(long millis, ChronoUnit granularity) { + if (millis >= 0) { + long epochSecond = Math.floorDiv(millis, MILLIS_PER_SECOND); + long nanoAdjustment = Math.floorMod(millis, MILLIS_PER_SECOND) * NANOS_PER_MILLI; + return (int) granularity.between(EPOCH, toOffsetDateTime(epochSecond, nanoAdjustment)); + } else { + // add 1 milli to the value to account for the case where there is exactly 1 unit between + // the timestamp and epoch because the result will always be decremented. + long epochSecond = Math.floorDiv(millis, MILLIS_PER_SECOND); + long nanoAdjustment = Math.floorMod(millis + 1, MILLIS_PER_SECOND) * NANOS_PER_MILLI; + return (int) granularity.between(EPOCH, toOffsetDateTime(epochSecond, nanoAdjustment)) - 1; + } + } + private static int convertMicros(long micros, ChronoUnit granularity) { if (micros >= 0) { long epochSecond = Math.floorDiv(micros, MICROS_PER_SECOND); - long nanoAdjustment = Math.floorMod(micros, MICROS_PER_SECOND) * 1000; + long nanoAdjustment = Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICRO; return (int) granularity.between(EPOCH, toOffsetDateTime(epochSecond, nanoAdjustment)); } else { // add 1 micro to the value to account for the case where there is exactly 1 unit between // the timestamp and epoch because the result will always be decremented. long epochSecond = Math.floorDiv(micros, MICROS_PER_SECOND); - long nanoAdjustment = Math.floorMod(micros + 1, MICROS_PER_SECOND) * 1000; + long nanoAdjustment = Math.floorMod(micros + 1, MICROS_PER_SECOND) * NANOS_PER_MICRO; + return (int) granularity.between(EPOCH, toOffsetDateTime(epochSecond, nanoAdjustment)) - 1; + } + } + + private static int convertNanos(long nanos, ChronoUnit granularity) { + if (nanos >= 0) { + long epochSecond = Math.floorDiv(nanos, NANOS_PER_SECOND); + long nanoAdjustment = Math.floorMod(nanos, NANOS_PER_SECOND); + return (int) granularity.between(EPOCH, toOffsetDateTime(epochSecond, nanoAdjustment)); + } else { + // add 1 nano to the value to account for the case where there is exactly 1 unit between + // the timestamp and epoch because the result will always be decremented. + long epochSecond = Math.floorDiv(nanos, NANOS_PER_SECOND); + long nanoAdjustment = Math.floorMod(nanos + 1, NANOS_PER_SECOND); return (int) granularity.between(EPOCH, toOffsetDateTime(epochSecond, nanoAdjustment)) - 1; } } diff --git a/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java b/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java index 5e4ca1fb11be..369a3a842224 100644 --- a/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java +++ b/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java @@ -29,12 +29,13 @@ public class PartitionSpecTestBase { Types.NestedField.required(2, "l", Types.LongType.get()), Types.NestedField.required(3, "d", Types.DateType.get()), Types.NestedField.required(4, "t", Types.TimeType.get()), - Types.NestedField.required(5, "ts", Types.TimestampType.withoutZone()), + Types.NestedField.required(5, "ts", Types.TimestampType.microsWithoutZone()), Types.NestedField.required(6, "dec", Types.DecimalType.of(9, 2)), Types.NestedField.required(7, "s", Types.StringType.get()), Types.NestedField.required(8, "u", Types.UUIDType.get()), Types.NestedField.required(9, "f", Types.FixedType.ofLength(3)), - Types.NestedField.required(10, "b", Types.BinaryType.get())); + Types.NestedField.required(10, "b", Types.BinaryType.get()), + Types.NestedField.required(11, "tsn", Types.TimestampType.nanosWithoutZone())); // a spec with all of the allowed transform/type pairs public static final PartitionSpec[] SPECS = @@ -49,6 +50,7 @@ public class PartitionSpecTestBase { PartitionSpec.builderFor(SCHEMA).identity("u").build(), PartitionSpec.builderFor(SCHEMA).identity("f").build(), PartitionSpec.builderFor(SCHEMA).identity("b").build(), + PartitionSpec.builderFor(SCHEMA).identity("tsn").build(), PartitionSpec.builderFor(SCHEMA).bucket("i", 128).build(), PartitionSpec.builderFor(SCHEMA).bucket("l", 128).build(), PartitionSpec.builderFor(SCHEMA).bucket("d", 128).build(), @@ -59,6 +61,7 @@ public class PartitionSpecTestBase { PartitionSpec.builderFor(SCHEMA).bucket("u", 128).build(), PartitionSpec.builderFor(SCHEMA).bucket("f", 128).build(), PartitionSpec.builderFor(SCHEMA).bucket("b", 128).build(), + PartitionSpec.builderFor(SCHEMA).bucket("tsn", 128).build(), PartitionSpec.builderFor(SCHEMA).year("d").build(), PartitionSpec.builderFor(SCHEMA).month("d").build(), PartitionSpec.builderFor(SCHEMA).day("d").build(), @@ -66,6 +69,10 @@ public class PartitionSpecTestBase { PartitionSpec.builderFor(SCHEMA).month("ts").build(), PartitionSpec.builderFor(SCHEMA).day("ts").build(), PartitionSpec.builderFor(SCHEMA).hour("ts").build(), + PartitionSpec.builderFor(SCHEMA).year("tsn").build(), + PartitionSpec.builderFor(SCHEMA).month("tsn").build(), + PartitionSpec.builderFor(SCHEMA).day("tsn").build(), + PartitionSpec.builderFor(SCHEMA).hour("tsn").build(), PartitionSpec.builderFor(SCHEMA).truncate("i", 10).build(), PartitionSpec.builderFor(SCHEMA).truncate("l", 10).build(), PartitionSpec.builderFor(SCHEMA).truncate("dec", 10).build(), diff --git a/api/src/test/java/org/apache/iceberg/TestAccessors.java b/api/src/test/java/org/apache/iceberg/TestAccessors.java index 332556e474c7..233c8c508239 100644 --- a/api/src/test/java/org/apache/iceberg/TestAccessors.java +++ b/api/src/test/java/org/apache/iceberg/TestAccessors.java @@ -178,8 +178,10 @@ public void testTime() { @Test public void testTimestamp() { - assertAccessorReturns(Types.TimestampType.withoutZone(), 123L); - assertAccessorReturns(Types.TimestampType.withZone(), 123L); + assertAccessorReturns(Types.TimestampType.microsWithoutZone(), 123L); + assertAccessorReturns(Types.TimestampType.microsWithZone(), 123L); + assertAccessorReturns(Types.TimestampType.nanosWithoutZone(), 123L); + assertAccessorReturns(Types.TimestampType.nanosWithZone(), 123L); } @Test diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java b/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java index 2fda247a33c8..5a98806cf291 100644 --- a/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java +++ b/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java @@ -32,7 +32,7 @@ public class TestPartitionPaths { new Schema( Types.NestedField.required(1, "id", Types.IntegerType.get()), Types.NestedField.optional(2, "data", Types.StringType.get()), - Types.NestedField.optional(3, "ts", Types.TimestampType.withoutZone())); + Types.NestedField.optional(3, "ts", Types.TimestampType.microsWithoutZone())); @Test public void testPartitionPath() { @@ -42,8 +42,8 @@ public void testPartitionPath() { Transform bucket = Transforms.bucket(10); Literal ts = - Literal.of("2017-12-01T10:12:55.038194").to(Types.TimestampType.withoutZone()); - Object tsHour = hour.bind(Types.TimestampType.withoutZone()).apply(ts.value()); + Literal.of("2017-12-01T10:12:55.038194").to(Types.TimestampType.microsWithoutZone()); + Object tsHour = hour.bind(Types.TimestampType.microsWithoutZone()).apply(ts.value()); Object idBucket = bucket.bind(Types.IntegerType.get()).apply(1); Row partition = Row.of(tsHour, idBucket); diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java index eb0e74164688..a621ccab81e5 100644 --- a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java +++ b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java @@ -30,10 +30,10 @@ public class TestPartitionSpecValidation { private static final Schema SCHEMA = new Schema( NestedField.required(1, "id", Types.LongType.get()), - NestedField.required(2, "ts", Types.TimestampType.withZone()), - NestedField.required(3, "another_ts", Types.TimestampType.withZone()), - NestedField.required(4, "d", Types.TimestampType.withZone()), - NestedField.required(5, "another_d", Types.TimestampType.withZone()), + NestedField.required(2, "ts", Types.TimestampType.microsWithZone()), + NestedField.required(3, "another_ts", Types.TimestampType.microsWithZone()), + NestedField.required(4, "d", Types.TimestampType.microsWithZone()), + NestedField.required(5, "another_d", Types.TimestampType.microsWithZone()), NestedField.required(6, "s", Types.StringType.get())); @Test diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java index 9a27830543ad..0c580ddf5472 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java @@ -42,7 +42,7 @@ public class TestExpressionUtil { Types.NestedField.required(1, "id", Types.LongType.get()), Types.NestedField.required(2, "val", Types.IntegerType.get()), Types.NestedField.required(3, "val2", Types.IntegerType.get()), - Types.NestedField.required(4, "ts", Types.TimestampType.withoutZone()), + Types.NestedField.required(4, "ts", Types.TimestampType.microsWithoutZone()), Types.NestedField.required(5, "date", Types.DateType.get()), Types.NestedField.required(6, "time", Types.DateType.get()), Types.NestedField.optional(7, "data", Types.StringType.get()), @@ -496,7 +496,7 @@ public void testSanitizeTimestampAboutNow() { Expressions.predicate( Expression.Operation.EQ, "test", - Literal.of(nowLocal).to(Types.TimestampType.withoutZone())))); + Literal.of(nowLocal).to(Types.TimestampType.microsWithoutZone())))); assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", nowLocal))) .as("Sanitized string should be identical except for descriptive literal") @@ -522,7 +522,7 @@ public void testSanitizeTimestampPast() { Expressions.predicate( Expression.Operation.EQ, "test", - Literal.of(ninetyMinutesAgoLocal).to(Types.TimestampType.withoutZone())))); + Literal.of(ninetyMinutesAgoLocal).to(Types.TimestampType.microsWithoutZone())))); assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", ninetyMinutesAgoLocal))) .as("Sanitized string should be identical except for descriptive literal") @@ -548,7 +548,7 @@ public void testSanitizeTimestampLastWeek() { Expressions.predicate( Expression.Operation.EQ, "test", - Literal.of(lastWeekLocal).to(Types.TimestampType.withoutZone())))); + Literal.of(lastWeekLocal).to(Types.TimestampType.microsWithoutZone())))); assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", lastWeekLocal))) .as("Sanitized string should be identical except for descriptive literal") @@ -574,7 +574,8 @@ public void testSanitizeTimestampFuture() { Expressions.predicate( Expression.Operation.EQ, "test", - Literal.of(ninetyMinutesFromNowLocal).to(Types.TimestampType.withoutZone())))); + Literal.of(ninetyMinutesFromNowLocal) + .to(Types.TimestampType.microsWithoutZone())))); assertThat( ExpressionUtil.toSanitizedString(Expressions.equal("test", ninetyMinutesFromNowLocal))) @@ -597,7 +598,7 @@ public void testSanitizeTimestamptzAboutNow() { Expressions.predicate( Expression.Operation.EQ, "test", - Literal.of(nowUtc).to(Types.TimestampType.withZone())))); + Literal.of(nowUtc).to(Types.TimestampType.microsWithZone())))); assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", nowUtc))) .as("Sanitized string should be identical except for descriptive literal") @@ -618,7 +619,7 @@ public void testSanitizeTimestamptzPast() { Expressions.predicate( Expression.Operation.EQ, "test", - Literal.of(ninetyMinutesAgoUtc).to(Types.TimestampType.withZone())))); + Literal.of(ninetyMinutesAgoUtc).to(Types.TimestampType.microsWithZone())))); assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", ninetyMinutesAgoUtc))) .as("Sanitized string should be identical except for descriptive literal") @@ -639,7 +640,7 @@ public void testSanitizeTimestamptzLastWeek() { Expressions.predicate( Expression.Operation.EQ, "test", - Literal.of(lastWeekUtc).to(Types.TimestampType.withZone())))); + Literal.of(lastWeekUtc).to(Types.TimestampType.microsWithZone())))); assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", lastWeekUtc))) .as("Sanitized string should be identical except for descriptive literal") @@ -660,7 +661,7 @@ public void testSanitizeTimestamptzFuture() { Expressions.predicate( Expression.Operation.EQ, "test", - Literal.of(ninetyMinutesFromNowUtc).to(Types.TimestampType.withZone())))); + Literal.of(ninetyMinutesFromNowUtc).to(Types.TimestampType.microsWithZone())))); assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", ninetyMinutesFromNowUtc))) .as("Sanitized string should be identical except for descriptive literal") diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestLiteralSerialization.java b/api/src/test/java/org/apache/iceberg/expressions/TestLiteralSerialization.java index d5aa251ffb50..fcb031c27aa8 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestLiteralSerialization.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestLiteralSerialization.java @@ -38,8 +38,10 @@ public void testLiterals() throws Exception { Literal.of(8.75D), Literal.of("2017-11-29").to(Types.DateType.get()), Literal.of("11:30:07").to(Types.TimeType.get()), - Literal.of("2017-11-29T11:30:07.123").to(Types.TimestampType.withoutZone()), - Literal.of("2017-11-29T11:30:07.123+01:00").to(Types.TimestampType.withZone()), + Literal.of("2017-11-29T11:30:07.123456").to(Types.TimestampType.microsWithoutZone()), + Literal.of("2017-11-29T11:30:07.123456+01:00").to(Types.TimestampType.microsWithZone()), + Literal.of("2017-11-29T11:30:07.123456789").to(Types.TimestampType.nanosWithoutZone()), + Literal.of("2017-11-29T11:30:07.123456789+01:00").to(Types.TimestampType.nanosWithZone()), Literal.of("abc"), Literal.of(UUID.randomUUID()), Literal.of(new byte[] {1, 2, 3}).to(Types.FixedType.ofLength(3)), diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java b/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java index f8d2cd49d969..4c576d9a41fc 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java @@ -42,7 +42,11 @@ public void testIdentityConversions() { Pair.of(Literal.of("34.55"), Types.DecimalType.of(9, 2)), Pair.of(Literal.of("2017-08-18"), Types.DateType.get()), Pair.of(Literal.of("14:21:01.919"), Types.TimeType.get()), - Pair.of(Literal.of("2017-08-18T14:21:01.919"), Types.TimestampType.withoutZone()), + Pair.of( + Literal.of("2017-08-18T14:21:01.919432"), Types.TimestampType.microsWithoutZone()), + Pair.of( + Literal.of("2017-08-18T14:21:01.919432755"), + Types.TimestampType.nanosWithoutZone()), Pair.of(Literal.of("abc"), Types.StringType.get()), Pair.of(Literal.of(UUID.randomUUID()), Types.UUIDType.get()), Pair.of(Literal.of(new byte[] {0, 1, 2}), Types.FixedType.ofLength(3)), @@ -99,8 +103,10 @@ public void testInvalidBooleanConversions() { Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), - Types.TimestampType.withZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), @@ -114,8 +120,10 @@ public void testInvalidIntegerConversions() { Literal.of(34), Types.BooleanType.get(), Types.TimeType.get(), - Types.TimestampType.withZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), @@ -142,8 +150,10 @@ public void testInvalidFloatConversions() { Types.LongType.get(), Types.DateType.get(), Types.TimeType.get(), - Types.TimestampType.withZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), @@ -159,8 +169,10 @@ public void testInvalidDoubleConversions() { Types.LongType.get(), Types.DateType.get(), Types.TimeType.get(), - Types.TimestampType.withZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), @@ -177,8 +189,10 @@ public void testInvalidDateConversions() { Types.FloatType.get(), Types.DoubleType.get(), Types.TimeType.get(), - Types.TimestampType.withZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), @@ -196,8 +210,10 @@ public void testInvalidTimeConversions() { Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), - Types.TimestampType.withZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), @@ -206,9 +222,26 @@ public void testInvalidTimeConversions() { } @Test - public void testInvalidTimestampConversions() { + public void testInvalidTimestampMicrosConversions() { testInvalidConversions( - Literal.of("2017-08-18T14:21:01.919").to(Types.TimestampType.withoutZone()), + Literal.of("2017-08-18T14:21:01.919123").to(Types.TimestampType.microsWithoutZone()), + Types.BooleanType.get(), + Types.IntegerType.get(), + Types.LongType.get(), + Types.FloatType.get(), + Types.DoubleType.get(), + Types.TimeType.get(), + Types.DecimalType.of(9, 4), + Types.StringType.get(), + Types.UUIDType.get(), + Types.FixedType.ofLength(1), + Types.BinaryType.get()); + } + + @Test + public void testInvalidTimestampNanosConversions() { + testInvalidConversions( + Literal.of("2017-08-18T14:21:01.919123456").to(Types.TimestampType.nanosWithoutZone()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), @@ -233,8 +266,10 @@ public void testInvalidDecimalConversions() { Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), - Types.TimestampType.withZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), @@ -267,8 +302,10 @@ public void testInvalidUUIDConversions() { Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), - Types.TimestampType.withZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.FixedType.ofLength(1), @@ -286,8 +323,10 @@ public void testInvalidFixedConversions() { Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), - Types.TimestampType.withZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), @@ -305,8 +344,10 @@ public void testInvalidBinaryConversions() { Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), - Types.TimestampType.withZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestStringLiteralConversions.java b/api/src/test/java/org/apache/iceberg/expressions/TestStringLiteralConversions.java index f35b274eb3d5..0c6348f8af94 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestStringLiteralConversions.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestStringLiteralConversions.java @@ -101,7 +101,7 @@ public void testStringToTimestampLiteral() { // Timestamp with explicit UTC offset, +00:00 Literal timestampStr = Literal.of("2017-08-18T14:21:01.919+00:00"); - Literal timestamp = timestampStr.to(Types.TimestampType.withZone()); + Literal timestamp = timestampStr.to(Types.TimestampType.microsWithZone()); long avroValue = avroConversion.toLong( LocalDateTime.of(2017, 8, 18, 14, 21, 1, 919 * 1000000).toInstant(ZoneOffset.UTC), @@ -112,7 +112,7 @@ public void testStringToTimestampLiteral() { // Timestamp without an explicit zone should be UTC (equal to the previous converted value) timestampStr = Literal.of("2017-08-18T14:21:01.919"); - timestamp = timestampStr.to(Types.TimestampType.withoutZone()); + timestamp = timestampStr.to(Types.TimestampType.microsWithoutZone()); assertThat((long) timestamp.value()) .as("Timestamp without zone should match UTC") @@ -120,7 +120,7 @@ public void testStringToTimestampLiteral() { // Timestamp with an explicit offset should be adjusted to UTC timestampStr = Literal.of("2017-08-18T14:21:01.919-07:00"); - timestamp = timestampStr.to(Types.TimestampType.withZone()); + timestamp = timestampStr.to(Types.TimestampType.microsWithZone()); avroValue = avroConversion.toLong( LocalDateTime.of(2017, 8, 18, 21, 21, 1, 919 * 1000000).toInstant(ZoneOffset.UTC), @@ -141,7 +141,7 @@ public void testNegativeStringToTimestampLiteral() { // Timestamp with explicit UTC offset, +00:00 Literal timestampStr = Literal.of("1969-12-31T23:59:58.999999+00:00"); - Literal timestamp = timestampStr.to(Types.TimestampType.withZone()); + Literal timestamp = timestampStr.to(Types.TimestampType.microsWithZone()); long avroValue = avroConversion.toLong( LocalDateTime.of(1969, 12, 31, 23, 59, 58, 999999 * 1_000).toInstant(ZoneOffset.UTC), @@ -156,7 +156,7 @@ public void testNegativeStringToTimestampLiteral() { // Timestamp without an explicit zone should be UTC (equal to the previous converted value) timestampStr = Literal.of("1969-12-31T23:59:58.999999"); - timestamp = timestampStr.to(Types.TimestampType.withoutZone()); + timestamp = timestampStr.to(Types.TimestampType.microsWithoutZone()); assertThat((long) timestamp.value()) .as("Timestamp without zone should match UTC") @@ -164,7 +164,7 @@ public void testNegativeStringToTimestampLiteral() { // Timestamp with an explicit offset should be adjusted to UTC timestampStr = Literal.of("1969-12-31T16:59:58.999999-07:00"); - timestamp = timestampStr.to(Types.TimestampType.withZone()); + timestamp = timestampStr.to(Types.TimestampType.microsWithZone()); avroValue = avroConversion.toLong( LocalDateTime.of(1969, 12, 31, 23, 59, 58, 999999 * 1_000).toInstant(ZoneOffset.UTC), @@ -181,8 +181,13 @@ public void testNegativeStringToTimestampLiteral() { @Test public void testTimestampWithZoneWithoutZoneInLiteral() { // Zone must be present in literals when converting to timestamp with zone - Literal timestampStr = Literal.of("2017-08-18T14:21:01.919"); - Assertions.assertThatThrownBy(() -> timestampStr.to(Types.TimestampType.withZone())) + Assertions.assertThatThrownBy( + () -> Literal.of("2017-08-18T14:21:01.919123").to(Types.TimestampType.microsWithZone())) + .isInstanceOf(DateTimeException.class) + .hasMessageContaining("could not be parsed"); + Assertions.assertThatThrownBy( + () -> + Literal.of("2017-08-18T14:21:01.919123456").to(Types.TimestampType.nanosWithZone())) .isInstanceOf(DateTimeException.class) .hasMessageContaining("could not be parsed"); } @@ -190,8 +195,16 @@ public void testTimestampWithZoneWithoutZoneInLiteral() { @Test public void testTimestampWithoutZoneWithZoneInLiteral() { // Zone must not be present in literals when converting to timestamp without zone - Literal timestampStr = Literal.of("2017-08-18T14:21:01.919+07:00"); - Assertions.assertThatThrownBy(() -> timestampStr.to(Types.TimestampType.withoutZone())) + Assertions.assertThatThrownBy( + () -> + Literal.of("2017-08-18T14:21:01.919123+07:00") + .to(Types.TimestampType.microsWithoutZone())) + .isInstanceOf(DateTimeException.class) + .hasMessageContaining("could not be parsed"); + Assertions.assertThatThrownBy( + () -> + Literal.of("2017-08-18T14:21:01.919123456+07:00") + .to(Types.TimestampType.nanosWithoutZone())) .isInstanceOf(DateTimeException.class) .hasMessageContaining("could not be parsed"); } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java b/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java index b8a0e40c1110..c76ac98612b9 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java @@ -107,17 +107,65 @@ public void testSpecValues() { .isEqualTo(-662762989); Literal timestampVal = - Literal.of("2017-11-16T22:31:08").to(Types.TimestampType.withoutZone()); + Literal.of("2017-11-16T22:31:08").to(Types.TimestampType.microsWithoutZone()); assertThat(BucketUtil.hash(timestampVal.value())) .as("Spec example: hash(2017-11-16T22:31:08) = -2047944441") .isEqualTo(-2047944441); + timestampVal = + Literal.of("2017-11-16T22:31:08.000001").to(Types.TimestampType.microsWithoutZone()); + assertThat(BucketUtil.hash(timestampVal.value())) + .as("Spec example: hash(2017-11-16T22:31:08.000001) = -1207196810") + .isEqualTo(-1207196810); + Literal timestamptzVal = - Literal.of("2017-11-16T14:31:08-08:00").to(Types.TimestampType.withZone()); + Literal.of("2017-11-16T14:31:08-08:00").to(Types.TimestampType.microsWithZone()); assertThat(BucketUtil.hash(timestamptzVal.value())) .as("Spec example: hash(2017-11-16T14:31:08-08:00) = -2047944441") .isEqualTo(-2047944441); + timestamptzVal = + Literal.of("2017-11-16T14:31:08.000001-08:00").to(Types.TimestampType.microsWithZone()); + assertThat(BucketUtil.hash(timestamptzVal.value())) + .as("Spec example: hash(2017-11-16T14:31:08.000001-08:00) = -1207196810") + .isEqualTo(-1207196810); + + Literal timestampNsVal = + Literal.of("2017-11-16T22:31:08").to(Types.TimestampType.nanosWithoutZone()); + assertThat(BucketUtil.hash(timestampNsVal.value())) + .as("Spec example: hash(2017-11-16T22:31:08) = -737750069") + .isEqualTo(-737750069); + + timestampNsVal = + Literal.of("2017-11-16T22:31:08.000001").to(Types.TimestampType.nanosWithoutZone()); + assertThat(BucketUtil.hash(timestampNsVal.value())) + .as("Spec example: hash(2017-11-16T22:31:08.000001) = -976603392") + .isEqualTo(-976603392); + + timestampNsVal = + Literal.of("2017-11-16T22:31:08.000000001").to(Types.TimestampType.nanosWithoutZone()); + assertThat(BucketUtil.hash(timestampNsVal.value())) + .as("Spec example: hash(2017-11-16T22:31:08.000000001) = -160215926") + .isEqualTo(-160215926); + + Literal timestamptzNsVal = + Literal.of("2017-11-16T14:31:08-08:00").to(Types.TimestampType.nanosWithZone()); + assertThat(BucketUtil.hash(timestamptzNsVal.value())) + .as("Spec example: hash(2017-11-16T14:31:08-08:00) = -737750069") + .isEqualTo(-737750069); + + timestamptzNsVal = + Literal.of("2017-11-16T14:31:08.000001-08:00").to(Types.TimestampType.nanosWithZone()); + assertThat(BucketUtil.hash(timestamptzNsVal.value())) + .as("Spec example: hash(2017-11-16T14:31:08.000001-08:00) = -976603392") + .isEqualTo(-976603392); + + timestamptzNsVal = + Literal.of("2017-11-16T14:31:08.000000001-08:00").to(Types.TimestampType.nanosWithZone()); + assertThat(BucketUtil.hash(timestamptzNsVal.value())) + .as("Spec example: hash(2017-11-16T14:31:08.000000001-08:00) = -160215926") + .isEqualTo(-160215926); + assertThat(BucketUtil.hash("iceberg")) .as("Spec example: hash(\"iceberg\") = 1210000089") .isEqualTo(1210000089); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java b/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java index 6101fdf0986d..8d651bf618dd 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java @@ -98,7 +98,7 @@ public void testTimeHumanString() { @Test public void testTimestampWithZoneHumanString() { - Types.TimestampType timestamptz = Types.TimestampType.withZone(); + Types.TimestampType timestamptz = Types.TimestampType.microsWithZone(); Transform identity = Transforms.identity(); Literal ts = Literal.of("2017-12-01T10:12:55.038194-08:00").to(timestamptz); @@ -111,7 +111,7 @@ public void testTimestampWithZoneHumanString() { @Test public void testTimestampWithoutZoneHumanString() { - Types.TimestampType timestamp = Types.TimestampType.withoutZone(); + Types.TimestampType timestamp = Types.TimestampType.microsWithoutZone(); Transform identity = Transforms.identity(); String tsString = "2017-12-01T10:12:55.038194"; diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestProjection.java index ccfda895f9f1..ffc48fc0e9aa 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestProjection.java @@ -268,10 +268,10 @@ public void testBadSparkPartitionFilter() { public void testProjectionNames() { final Schema schema = new Schema( - required(1, "timestamp1", Types.TimestampType.withoutZone()), - optional(2, "timestamp2", Types.TimestampType.withoutZone()), - optional(3, "timestamp3", Types.TimestampType.withoutZone()), - optional(4, "timestamp4", Types.TimestampType.withoutZone()), + required(1, "timestamp1", Types.TimestampType.microsWithoutZone()), + optional(2, "timestamp2", Types.TimestampType.microsWithoutZone()), + optional(3, "timestamp3", Types.TimestampType.microsWithoutZone()), + optional(4, "timestamp4", Types.TimestampType.microsWithoutZone()), optional(5, "date1", Types.DateType.get()), optional(6, "date2", Types.DateType.get()), optional(7, "date3", Types.DateType.get()), diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java b/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java index fa3436e5701b..87bdd6944a73 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java @@ -199,19 +199,23 @@ public void testIn() { public void testInTimestamp() { Schema schema = new Schema( - Types.NestedField.optional(50, "ts", Types.TimestampType.withoutZone()), + Types.NestedField.optional(50, "ts", Types.TimestampType.microsWithoutZone()), Types.NestedField.optional(51, "dateint", Types.IntegerType.get())); Long date20191201 = (Long) - Literal.of("2019-12-01T00:00:00.00000").to(Types.TimestampType.withoutZone()).value(); + Literal.of("2019-12-01T00:00:00.00000") + .to(Types.TimestampType.microsWithoutZone()) + .value(); Long date20191202 = (Long) - Literal.of("2019-12-02T00:00:00.00000").to(Types.TimestampType.withoutZone()).value(); + Literal.of("2019-12-02T00:00:00.00000") + .to(Types.TimestampType.microsWithoutZone()) + .value(); PartitionSpec spec = PartitionSpec.builderFor(schema).day("ts").build(); - Function day = Transforms.day().bind(Types.TimestampType.withoutZone()); + Function day = Transforms.day().bind(Types.TimestampType.microsWithoutZone()); Integer tsDay = day.apply(date20191201); Expression pred = in("ts", date20191201, date20191202); @@ -307,19 +311,23 @@ public void testNotNaN() { public void testNotInTimestamp() { Schema schema = new Schema( - Types.NestedField.optional(50, "ts", Types.TimestampType.withoutZone()), + Types.NestedField.optional(50, "ts", Types.TimestampType.microsWithoutZone()), Types.NestedField.optional(51, "dateint", Types.IntegerType.get())); Long date20191201 = (Long) - Literal.of("2019-12-01T00:00:00.00000").to(Types.TimestampType.withoutZone()).value(); + Literal.of("2019-12-01T00:00:00.00000") + .to(Types.TimestampType.microsWithoutZone()) + .value(); Long date20191202 = (Long) - Literal.of("2019-12-02T00:00:00.00000").to(Types.TimestampType.withoutZone()).value(); + Literal.of("2019-12-02T00:00:00.00000") + .to(Types.TimestampType.microsWithoutZone()) + .value(); PartitionSpec spec = PartitionSpec.builderFor(schema).day("ts").build(); - Function day = Transforms.day().bind(Types.TimestampType.withoutZone()); + Function day = Transforms.day().bind(Types.TimestampType.microsWithoutZone()); Integer tsDay = day.apply(date20191201); Expression pred = notIn("ts", date20191201, date20191202); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java index 3c37e643eb95..dee5d8265b04 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java @@ -29,7 +29,7 @@ public class TestTimestamps { @Test @SuppressWarnings("deprecation") public void testDeprecatedTimestampTransform() { - Types.TimestampType type = Types.TimestampType.withoutZone(); + Types.TimestampType type = Types.TimestampType.microsWithoutZone(); Literal ts = Literal.of("2017-12-01T10:12:55.038194").to(type); Literal pts = Literal.of("1970-01-01T00:00:01.000001").to(type); Literal nts = Literal.of("1969-12-31T23:59:58.999999").to(type); @@ -61,7 +61,7 @@ public void testDeprecatedTimestampTransform() { @Test public void testTimestampTransform() { - Types.TimestampType type = Types.TimestampType.withoutZone(); + Types.TimestampType type = Types.TimestampType.microsWithoutZone(); Literal ts = Literal.of("2017-12-01T10:12:55.038194").to(type); Literal pts = Literal.of("1970-01-01T00:00:01.000001").to(type); Literal nts = Literal.of("1969-12-31T23:59:58.999999").to(type); @@ -105,7 +105,7 @@ public void testTimestampTransform() { @Test public void testTimestampWithoutZoneToHumanString() { - Types.TimestampType type = Types.TimestampType.withoutZone(); + Types.TimestampType type = Types.TimestampType.microsWithoutZone(); Literal date = Literal.of("2017-12-01T10:12:55.038194").to(type); Transform year = Transforms.year(); @@ -125,7 +125,7 @@ public void testTimestampWithoutZoneToHumanString() { @Test public void testNegativeTimestampWithoutZoneToHumanString() { - Types.TimestampType type = Types.TimestampType.withoutZone(); + Types.TimestampType type = Types.TimestampType.microsWithoutZone(); Literal date = Literal.of("1969-12-30T10:12:55.038194").to(type); Transform year = Transforms.year(); @@ -145,7 +145,7 @@ public void testNegativeTimestampWithoutZoneToHumanString() { @Test public void testNegativeTimestampWithoutZoneToHumanStringLowerBound() { - Types.TimestampType type = Types.TimestampType.withoutZone(); + Types.TimestampType type = Types.TimestampType.microsWithoutZone(); Literal date = Literal.of("1969-12-30T00:00:00.000000").to(type); Transform year = Transforms.year(); @@ -165,7 +165,7 @@ public void testNegativeTimestampWithoutZoneToHumanStringLowerBound() { @Test public void testNegativeTimestampWithoutZoneToHumanStringUpperBound() { - Types.TimestampType type = Types.TimestampType.withoutZone(); + Types.TimestampType type = Types.TimestampType.microsWithoutZone(); Literal date = Literal.of("1969-12-31T23:59:59.999999").to(type); Transform year = Transforms.year(); @@ -185,7 +185,7 @@ public void testNegativeTimestampWithoutZoneToHumanStringUpperBound() { @Test public void testTimestampWithZoneToHumanString() { - Types.TimestampType type = Types.TimestampType.withZone(); + Types.TimestampType type = Types.TimestampType.microsWithZone(); Literal date = Literal.of("2017-12-01T10:12:55.038194-08:00").to(type); Transform year = Transforms.year(); @@ -206,7 +206,7 @@ public void testTimestampWithZoneToHumanString() { @Test public void testNullHumanString() { - Types.TimestampType type = Types.TimestampType.withZone(); + Types.TimestampType type = Types.TimestampType.microsWithZone(); assertThat(Transforms.year().toHumanString(type, null)) .as("Should produce \"null\" for null") .isEqualTo("null"); @@ -223,7 +223,7 @@ public void testNullHumanString() { @Test public void testTimestampsReturnType() { - Types.TimestampType type = Types.TimestampType.withZone(); + Types.TimestampType type = Types.TimestampType.microsWithZone(); Transform year = Transforms.year(); Type yearResultType = year.getResultType(type); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java index cd20868a06eb..dc0f199db132 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -44,7 +44,7 @@ import org.junit.jupiter.api.Test; public class TestTimestampsProjection { - private static final Types.TimestampType TYPE = Types.TimestampType.withoutZone(); + private static final Types.TimestampType TYPE = Types.TimestampType.microsWithoutZone(); private static final Schema SCHEMA = new Schema(optional(1, "timestamp", TYPE)); @SuppressWarnings("unchecked") diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTransformSerialization.java b/api/src/test/java/org/apache/iceberg/transforms/TestTransformSerialization.java index c2330247fa9d..70b5a16e3bb3 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTransformSerialization.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTransformSerialization.java @@ -39,8 +39,8 @@ public void testFunctionSerialization() throws Exception { Types.StringType.get(), Types.DateType.get(), Types.TimeType.get(), - Types.TimestampType.withoutZone(), - Types.TimestampType.withoutZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithoutZone(), Types.BinaryType.get(), Types.FixedType.ofLength(4), Types.DecimalType.of(9, 4), diff --git a/api/src/test/java/org/apache/iceberg/types/TestComparators.java b/api/src/test/java/org/apache/iceberg/types/TestComparators.java index 165d96c029cc..a04b039e5d65 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestComparators.java +++ b/api/src/test/java/org/apache/iceberg/types/TestComparators.java @@ -75,8 +75,10 @@ public void testTime() { @Test public void testTimestamp() { - assertComparesCorrectly(Comparators.forType(Types.TimestampType.withoutZone()), 111, 222); - assertComparesCorrectly(Comparators.forType(Types.TimestampType.withZone()), 111, 222); + assertComparesCorrectly(Comparators.forType(Types.TimestampType.microsWithoutZone()), 111, 222); + assertComparesCorrectly(Comparators.forType(Types.TimestampType.microsWithZone()), 111, 222); + assertComparesCorrectly(Comparators.forType(Types.TimestampType.nanosWithoutZone()), 111, 222); + assertComparesCorrectly(Comparators.forType(Types.TimestampType.nanosWithZone()), 111, 222); } @Test diff --git a/api/src/test/java/org/apache/iceberg/types/TestConversions.java b/api/src/test/java/org/apache/iceberg/types/TestConversions.java index 6c7a884a5839..2b160f01acb0 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestConversions.java +++ b/api/src/test/java/org/apache/iceberg/types/TestConversions.java @@ -93,15 +93,26 @@ public void testByteBufferConversions() { assertThat(Literal.of(10000L).to(TimeType.get()).toByteBuffer().array()) .isEqualTo(new byte[] {16, 39, 0, 0, 0, 0, 0, 0}); - // timestamps are stored as microseconds from 1970-01-01 00:00:00.000000 in an 8-byte + // timestamps are stored as micro|nanoseconds from 1970-01-01 00:00:00 in an + // 8-byte // little-endian long // 400000L is 0...110|00011010|10000000 in binary // 10000000 -> -128, 00011010 -> 26, 00000110 -> 6, ... , 00000000 -> 0 - assertConversion(400000L, TimestampType.withoutZone(), new byte[] {-128, 26, 6, 0, 0, 0, 0, 0}); - assertConversion(400000L, TimestampType.withZone(), new byte[] {-128, 26, 6, 0, 0, 0, 0, 0}); - assertThat(Literal.of(400000L).to(TimestampType.withoutZone()).toByteBuffer().array()) + assertConversion( + 400000L, TimestampType.microsWithoutZone(), new byte[] {-128, 26, 6, 0, 0, 0, 0, 0}); + assertConversion( + 400000L, TimestampType.microsWithZone(), new byte[] {-128, 26, 6, 0, 0, 0, 0, 0}); + assertThat(Literal.of(400000L).to(TimestampType.microsWithoutZone()).toByteBuffer().array()) + .isEqualTo(new byte[] {-128, 26, 6, 0, 0, 0, 0, 0}); + assertThat(Literal.of(400000L).to(TimestampType.microsWithZone()).toByteBuffer().array()) + .isEqualTo(new byte[] {-128, 26, 6, 0, 0, 0, 0, 0}); + assertConversion( + 400000L, TimestampType.nanosWithoutZone(), new byte[] {-128, 26, 6, 0, 0, 0, 0, 0}); + assertConversion( + 400000L, TimestampType.nanosWithZone(), new byte[] {-128, 26, 6, 0, 0, 0, 0, 0}); + assertThat(Literal.of(400000L).to(TimestampType.nanosWithoutZone()).toByteBuffer().array()) .isEqualTo(new byte[] {-128, 26, 6, 0, 0, 0, 0, 0}); - assertThat(Literal.of(400000L).to(TimestampType.withZone()).toByteBuffer().array()) + assertThat(Literal.of(400000L).to(TimestampType.nanosWithZone()).toByteBuffer().array()) .isEqualTo(new byte[] {-128, 26, 6, 0, 0, 0, 0, 0}); // strings are stored as UTF-8 bytes (without length) diff --git a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java index 7f5948bd5838..56a011263995 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java +++ b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java @@ -37,8 +37,10 @@ public class TestReadabilityChecks { Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), - Types.TimestampType.withoutZone(), - Types.TimestampType.withZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(3), diff --git a/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java b/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java index d981b5a26789..52cb95dcba03 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java +++ b/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java @@ -39,8 +39,10 @@ public void testIdentityTypes() throws Exception { Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), - Types.TimestampType.withoutZone(), - Types.TimestampType.withZone(), + Types.TimestampType.microsWithoutZone(), + Types.TimestampType.microsWithZone(), + Types.TimestampType.nanosWithoutZone(), + Types.TimestampType.nanosWithZone(), Types.StringType.get(), Types.UUIDType.get(), Types.BinaryType.get() diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypes.java b/api/src/test/java/org/apache/iceberg/types/TestTypes.java index ca5c6edce16b..088c042de550 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypes.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypes.java @@ -29,7 +29,9 @@ public void fromPrimitiveString() { Assertions.assertThat(Types.fromPrimitiveString("BooLean")).isSameAs(Types.BooleanType.get()); Assertions.assertThat(Types.fromPrimitiveString("timestamp")) - .isSameAs(Types.TimestampType.withoutZone()); + .isSameAs(Types.TimestampType.microsWithoutZone()); + Assertions.assertThat(Types.fromPrimitiveString("timestamp_ns")) + .isSameAs(Types.TimestampType.nanosWithoutZone()); Assertions.assertThat(Types.fromPrimitiveString("Fixed[ 3 ]")) .isEqualTo(Types.FixedType.ofLength(3));