cleaned up profiles, made it so for one profile it only compiles its …

…needed shim
rcongiu · Apr 3, 2017 · 520d62f · 520d62f
1 parent b7e90e4
commit 520d62f
Show file tree

Hide file tree

Showing 9 changed files with 54 additions and 11 deletions.
diff --git a/README.md b/README.md
@@ -287,6 +287,9 @@ OK
 {"name":"roberto"}
 ```
 
+The SerDe must also be in the classpath for the UDF to work. If not installed
+as a hive extra library, you should also `ADD JAR` theSerDe Jar
+
 ### Timestamps
 
 Note that the system default timezone is used to convert timestamps.

diff --git a/json-serde-cdh4-shim/pom.xml b/json-serde-cdh4-shim/pom.xml
@@ -46,7 +46,7 @@
 	</dependency>
 	<dependency>
 	    <groupId>org.apache.hadoop</groupId>
-	    <artifactId>hadoop-common</artifactId>
+	    <artifactId>${hadoop.dependency}</artifactId>
 	    <version>${cdh4.hadoop.version}</version>
 	    <scope>provided</scope>
 	</dependency>

diff --git a/json-serde-cdh4-shim/src/main/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java b/json-serde-cdh4-shim/src/main/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
@@ -0,0 +1,32 @@
+package org.apache.hadoop.hive.serde2;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
+
+import java.util.Properties;
+
+import java.util.Properties;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeStats;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
+
+public abstract class AbstractSerDe implements SerDe {
+    public AbstractSerDe() {
+    }
+
+    public abstract void initialize(Configuration var1, Properties var2) throws SerDeException;
+
+    public abstract Class<? extends Writable> getSerializedClass();
+
+    public abstract Writable serialize(Object var1, ObjectInspector var2) throws SerDeException;
+
+    public abstract SerDeStats getSerDeStats();
+
+    public abstract Object deserialize(Writable var1) throws SerDeException;
+
+    public abstract ObjectInspector getObjectInspector() throws SerDeException;
+}
diff --git a/...-shim/src/main/java/org/openx/data/jsonserde/objectinspector/primitive/TypeEntryShim.java b/...-shim/src/main/java/org/openx/data/jsonserde/objectinspector/primitive/TypeEntryShim.java
@@ -39,7 +39,7 @@ public class TypeEntryShim {
     // this won't actually be used since datetype was added in 1.2, but we have to add it anyway
     public static PrimitiveTypeEntry dateType = PrimitiveObjectInspectorUtils.stringTypeEntry;
 
-    // no specific OIs in this shimt
+    // no specific OIs in this shim
     public static void addObjectInspectors(Map<PrimitiveObjectInspector.PrimitiveCategory, AbstractPrimitiveJavaObjectInspector> primitiveOICache) {
 
     }

diff --git a/json-serde-cdh5-shim/pom.xml b/json-serde-cdh5-shim/pom.xml
@@ -48,7 +48,7 @@
 
 	<dependency>
 	    <groupId>org.apache.hadoop</groupId>
-	    <artifactId>hadoop-common</artifactId>
+	    <artifactId>${hadoop.dependency}</artifactId>
 	    <version>${cdh5.hadoop.version}</version>
 	    <scope>provided</scope>
 	</dependency>

diff --git a/json-serde/pom.xml b/json-serde/pom.xml
@@ -101,7 +101,7 @@
 
         <dependency>
             <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
+            <artifactId>${hadoop.dependency}</artifactId>
             <version>${cdh.hadoop.version}</version>
             <scope>provided</scope>
         </dependency>

diff --git a/json-serde/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java b/json-serde/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java
@@ -18,6 +18,7 @@
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.junit.Before;
@@ -198,8 +199,7 @@ public void testSerializeTimestamp() throws SerDeException, JSONException {
 
     row.add( new Timestamp(cal.getTime().getTime())); // see http://docs.oracle.com/javase/7/docs/api/java/util/Date.html#UTC(int,%20int,%20int,%20int,%20int,%20int)
     fieldNames.add("three");
-    lOi.add(ObjectInspectorFactory
-            .getReflectionObjectInspector(Timestamp.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA));
+    lOi.add(PrimitiveObjectInspectorFactory.javaTimestampObjectInspector);
 
     StructObjectInspector soi = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, lOi);
 

diff --git a/json-udf/pom.xml b/json-udf/pom.xml
@@ -74,7 +74,7 @@
             <groupId>org.openx.data</groupId>
             <artifactId>json-serde</artifactId>
             <version>${project.version}</version>
-            <scope>compile</scope>
+            <scope>provided</scope>
         </dependency>
 
         <dependency>
@@ -92,7 +92,7 @@
 
         <dependency>
             <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
+            <artifactId>${hadoop.dependency}</artifactId>
             <version>${cdh.hadoop.version}</version>
             <scope>provided</scope>
         </dependency>

diff --git a/pom.xml b/pom.xml
@@ -28,7 +28,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven-jar-plugin.version>2.4</maven-jar-plugin.version>
         <!-- cdh4 versions -->
-        <cdh4.version>4.6.0</cdh4.version>
+        <cdh4.version>4.7.1</cdh4.version>
         <cdh4.hive.version>0.10.0-cdh${cdh4.version}</cdh4.hive.version>
         <cdh4.hadoop.version>2.0.0-cdh${cdh4.version}</cdh4.hadoop.version>
         <!-- cdh5 versions -->
@@ -43,6 +43,7 @@
         <hdp23.version>2.3.0</hdp23.version>
         <hdp23.hive.version>1.2.1</hdp23.hive.version>
         <hdp23.hadoop.version>2.7.1</hdp23.hadoop.version>
+        <hadoop.dependency>hadoop-common</hadoop.dependency>
     </properties>
 
     <profiles>
@@ -51,6 +52,9 @@
             <activation>
                 <activeByDefault>false</activeByDefault>
             </activation>
+            <modules>
+                <module>json-serde-cdh4-shim</module>
+            </modules>
             <properties>
                 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
                 <serde.shim>json-serde-cdh4-shim</serde.shim>
@@ -65,6 +69,9 @@
             <activation>
                 <activeByDefault>true</activeByDefault>
             </activation>
+            <modules>
+                <module>json-serde-cdh5-shim</module>
+            </modules>
             <properties>
                 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
                 <serde.shim>json-serde-cdh5-shim</serde.shim>
@@ -79,6 +86,9 @@
             <activation>
                 <activeByDefault>false</activeByDefault>
             </activation>
+            <modules>
+                <module>json-serde-cdh5-shim</module>
+            </modules>
             <properties>
                 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
                 <serde.shim>json-serde-cdh5-shim</serde.shim>
@@ -92,8 +102,6 @@
 
     <modules>
         <module>json</module>
-        <module>json-serde-cdh4-shim</module>
-        <module>json-serde-cdh5-shim</module>
         <module>json-serde</module>
         <module>json-udf</module>
     </modules>