Timestamp serialization in UTC, independent of timezone

rcongiu · Apr 3, 2017 · b7e90e4 · b7e90e4
1 parent bfb14b5
commit b7e90e4
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 5 deletions.
diff --git a/json-serde/src/main/java/org/openx/data/jsonserde/JsonSerDe.java b/json-serde/src/main/java/org/openx/data/jsonserde/JsonSerDe.java
@@ -55,6 +55,8 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.openx.data.jsonserde.objectinspector.primitive.JavaStringTimestampObjectInspector;
+import org.openx.data.jsonserde.objectinspector.primitive.ParsePrimitiveUtils;
 
 /**
  * Properties:
@@ -325,7 +327,7 @@ public Object serializeField(Object obj,
                         result = ((StringObjectInspector)poi).getPrimitiveJavaObject(obj);
                         break;
                     case TIMESTAMP:
-                        result = ((TimestampObjectInspector)poi).getPrimitiveJavaObject(obj);
+                        result = ParsePrimitiveUtils.serializeAsUTC((Timestamp)((TimestampObjectInspector)poi).getPrimitiveJavaObject(obj));
                         break;
                     case UNKNOWN:
                         throw new RuntimeException("Unknown primitive");

diff --git a/...rg/openx/data/jsonserde/objectinspector/primitive/JavaStringTimestampObjectInspector.java b/...rg/openx/data/jsonserde/objectinspector/primitive/JavaStringTimestampObjectInspector.java
@@ -46,12 +46,19 @@ public Object set(Object o, TimestampWritable tw) {
 
     @Override
     public Object create(byte[] bytes, int offset) {
-       return new TimestampWritable(bytes, offset).toString();
+       return formatTimeStamp(new TimestampWritable(bytes, offset));
     }
 
     @Override
     public Object create(Timestamp tmstmp) {
-        return tmstmp.toString();
+        return formatTimeStamp(tmstmp);
+    }
+
+    private String formatTimeStamp(Timestamp ts) {
+        return ParsePrimitiveUtils.serializeAsUTC(ts);
+    }
+    private String formatTimeStamp(TimestampWritable tsw) {
+        return formatTimeStamp(tsw.getTimestamp());
     }
 
     @Override

diff --git a/...src/main/java/org/openx/data/jsonserde/objectinspector/primitive/ParsePrimitiveUtils.java b/...src/main/java/org/openx/data/jsonserde/objectinspector/primitive/ParsePrimitiveUtils.java
@@ -9,6 +9,8 @@
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.TimeZone;
 
 /**
  *
@@ -59,6 +61,11 @@ public static long parseLong(String s) {
         }
     }
 
+    static TimeZone defaultZone = TimeZone.getDefault();
+    public static String serializeAsUTC(Timestamp ts) {
+        return UTC_FORMAT.format(ts.getTime() - defaultZone.getOffset(ts.getTime()) );
+    }
+
     public static Timestamp parseTimestamp(String s) {
         final String sampleUnixTimestampInMs = "1454612111000";
 

diff --git a/json-serde/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java b/json-serde/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java
@@ -191,7 +191,12 @@ public void testSerializeTimestamp() throws SerDeException, JSONException {
             ObjectInspectorFactory.ObjectInspectorOptions.JAVA));
 
 
-    row.add( new Timestamp(1326439500L));
+
+    java.sql.Timestamp ts = new Timestamp(1326439500L);
+    Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+    cal.set(2015,10,12,22,33,44); // month is zero-based!
+
+    row.add( new Timestamp(cal.getTime().getTime())); // see http://docs.oracle.com/javase/7/docs/api/java/util/Date.html#UTC(int,%20int,%20int,%20int,%20int,%20int)
     fieldNames.add("three");
     lOi.add(ObjectInspectorFactory
             .getReflectionObjectInspector(Timestamp.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA));
@@ -207,7 +212,7 @@ public void testSerializeTimestamp() throws SerDeException, JSONException {
     // one by one
     assertTrue(serialized.contains("\"one\":true"));
     assertTrue(serialized.contains("\"two\":\"field\""));
-    assertTrue(serialized.contains("\"three\":\"1970-01-16 00:27:19.5\""));
+    assertTrue(serialized.contains("\"three\":\"2015-11-12T22:33:44Z\"")); // UTC format
 
 
   }