diff --git a/chill-avro/src/main/scala/com/twitter/chill/avro/AvroSerializer.scala b/chill-avro/src/main/scala/com/twitter/chill/avro/AvroSerializer.scala new file mode 100644 index 00000000..521c2ad8 --- /dev/null +++ b/chill-avro/src/main/scala/com/twitter/chill/avro/AvroSerializer.scala @@ -0,0 +1,45 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.twitter.chill.avro + +import org.apache.avro.specific.SpecificRecordBase +import com.twitter.chill.{InjectiveSerializer, KSerializer} +import com.twitter.bijection.avro.SpecificAvroCodecs +import org.apache.avro.Schema +import com.twitter.bijection.Injection + +/** + * @author Mansur Ashraf + * @since 2/9/14. + */ +object AvroSerializer { + + def SpecificRecordSerializer[T <: SpecificRecordBase : Manifest]: KSerializer[T] = { + implicit val inj = SpecificAvroCodecs[T] + InjectiveSerializer.asKryo + } + + def SpecificRecordBinarySerializer[T <: SpecificRecordBase : Manifest]: KSerializer[T] = { + implicit val inj = SpecificAvroCodecs.toBinary[T] + InjectiveSerializer.asKryo + } + + def SpecificRecordJsonSerializer[T <: SpecificRecordBase : Manifest](schema: Schema): KSerializer[T] = { + import com.twitter.bijection.StringCodec.utf8 + implicit val inj = SpecificAvroCodecs.toJson[T](schema) + implicit val avroToArray = Injection.connect[T, String, Array[Byte]] + InjectiveSerializer.asKryo + } +} diff --git a/chill-avro/src/test/java/avro/FiscalRecord.java b/chill-avro/src/test/java/avro/FiscalRecord.java new file mode 100644 index 00000000..79d57ba2 --- /dev/null +++ b/chill-avro/src/test/java/avro/FiscalRecord.java @@ -0,0 +1,293 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package avro; + +@SuppressWarnings("all") +public class FiscalRecord extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"FiscalRecord\",\"namespace\":\"avro\",\"fields\":[{\"name\":\"calendarDate\",\"type\":\"string\"},{\"name\":\"fiscalWeek\",\"type\":[\"int\",\"null\"]},{\"name\":\"fiscalYear\",\"type\":[\"int\",\"null\"]}]}"); + @Deprecated + public java.lang.CharSequence calendarDate; + @Deprecated + public java.lang.Integer fiscalWeek; + @Deprecated + public java.lang.Integer fiscalYear; + + /** + * Default constructor. + */ + public FiscalRecord() { + } + + /** + * All-args constructor. + */ + public FiscalRecord(java.lang.CharSequence calendarDate, java.lang.Integer fiscalWeek, java.lang.Integer fiscalYear) { + this.calendarDate = calendarDate; + this.fiscalWeek = fiscalWeek; + this.fiscalYear = fiscalYear; + } + + public org.apache.avro.Schema getSchema() { + return SCHEMA$; + } + + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: + return calendarDate; + case 1: + return fiscalWeek; + case 2: + return fiscalYear; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value = "unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: + calendarDate = (java.lang.CharSequence) value$; + break; + case 1: + fiscalWeek = (java.lang.Integer) value$; + break; + case 2: + fiscalYear = (java.lang.Integer) value$; + break; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'calendarDate' field. + */ + public java.lang.CharSequence getCalendarDate() { + return calendarDate; + } + + /** + * Sets the value of the 'calendarDate' field. + * + * @param value the value to set. + */ + public void setCalendarDate(java.lang.CharSequence value) { + this.calendarDate = value; + } + + /** + * Gets the value of the 'fiscalWeek' field. + */ + public java.lang.Integer getFiscalWeek() { + return fiscalWeek; + } + + /** + * Sets the value of the 'fiscalWeek' field. + * + * @param value the value to set. + */ + public void setFiscalWeek(java.lang.Integer value) { + this.fiscalWeek = value; + } + + /** + * Gets the value of the 'fiscalYear' field. + */ + public java.lang.Integer getFiscalYear() { + return fiscalYear; + } + + /** + * Sets the value of the 'fiscalYear' field. + * + * @param value the value to set. + */ + public void setFiscalYear(java.lang.Integer value) { + this.fiscalYear = value; + } + + /** + * Creates a new FiscalRecord RecordBuilder + */ + public static avro.FiscalRecord.Builder newBuilder() { + return new avro.FiscalRecord.Builder(); + } + + /** + * Creates a new FiscalRecord RecordBuilder by copying an existing Builder + */ + public static avro.FiscalRecord.Builder newBuilder(avro.FiscalRecord.Builder other) { + return new avro.FiscalRecord.Builder(other); + } + + /** + * Creates a new FiscalRecord RecordBuilder by copying an existing FiscalRecord instance + */ + public static avro.FiscalRecord.Builder newBuilder(avro.FiscalRecord other) { + return new avro.FiscalRecord.Builder(other); + } + + /** + * RecordBuilder for FiscalRecord instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.lang.CharSequence calendarDate; + private java.lang.Integer fiscalWeek; + private java.lang.Integer fiscalYear; + + /** + * Creates a new Builder + */ + private Builder() { + super(avro.FiscalRecord.SCHEMA$); + } + + /** + * Creates a Builder by copying an existing Builder + */ + private Builder(avro.FiscalRecord.Builder other) { + super(other); + } + + /** + * Creates a Builder by copying an existing FiscalRecord instance + */ + private Builder(avro.FiscalRecord other) { + super(avro.FiscalRecord.SCHEMA$); + if (isValidValue(fields()[0], other.calendarDate)) { + this.calendarDate = (java.lang.CharSequence) data().deepCopy(fields()[0].schema(), other.calendarDate); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.fiscalWeek)) { + this.fiscalWeek = (java.lang.Integer) data().deepCopy(fields()[1].schema(), other.fiscalWeek); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.fiscalYear)) { + this.fiscalYear = (java.lang.Integer) data().deepCopy(fields()[2].schema(), other.fiscalYear); + fieldSetFlags()[2] = true; + } + } + + /** + * Gets the value of the 'calendarDate' field + */ + public java.lang.CharSequence getCalendarDate() { + return calendarDate; + } + + /** + * Sets the value of the 'calendarDate' field + */ + public avro.FiscalRecord.Builder setCalendarDate(java.lang.CharSequence value) { + validate(fields()[0], value); + this.calendarDate = value; + fieldSetFlags()[0] = true; + return this; + } + + /** + * Checks whether the 'calendarDate' field has been set + */ + public boolean hasCalendarDate() { + return fieldSetFlags()[0]; + } + + /** + * Clears the value of the 'calendarDate' field + */ + public avro.FiscalRecord.Builder clearCalendarDate() { + calendarDate = null; + fieldSetFlags()[0] = false; + return this; + } + + /** + * Gets the value of the 'fiscalWeek' field + */ + public java.lang.Integer getFiscalWeek() { + return fiscalWeek; + } + + /** + * Sets the value of the 'fiscalWeek' field + */ + public avro.FiscalRecord.Builder setFiscalWeek(java.lang.Integer value) { + validate(fields()[1], value); + this.fiscalWeek = value; + fieldSetFlags()[1] = true; + return this; + } + + /** + * Checks whether the 'fiscalWeek' field has been set + */ + public boolean hasFiscalWeek() { + return fieldSetFlags()[1]; + } + + /** + * Clears the value of the 'fiscalWeek' field + */ + public avro.FiscalRecord.Builder clearFiscalWeek() { + fiscalWeek = null; + fieldSetFlags()[1] = false; + return this; + } + + /** + * Gets the value of the 'fiscalYear' field + */ + public java.lang.Integer getFiscalYear() { + return fiscalYear; + } + + /** + * Sets the value of the 'fiscalYear' field + */ + public avro.FiscalRecord.Builder setFiscalYear(java.lang.Integer value) { + validate(fields()[2], value); + this.fiscalYear = value; + fieldSetFlags()[2] = true; + return this; + } + + /** + * Checks whether the 'fiscalYear' field has been set + */ + public boolean hasFiscalYear() { + return fieldSetFlags()[2]; + } + + /** + * Clears the value of the 'fiscalYear' field + */ + public avro.FiscalRecord.Builder clearFiscalYear() { + fiscalYear = null; + fieldSetFlags()[2] = false; + return this; + } + + @Override + public FiscalRecord build() { + try { + FiscalRecord record = new FiscalRecord(); + record.calendarDate = fieldSetFlags()[0] ? this.calendarDate : (java.lang.CharSequence) defaultValue(fields()[0]); + record.fiscalWeek = fieldSetFlags()[1] ? this.fiscalWeek : (java.lang.Integer) defaultValue(fields()[1]); + record.fiscalYear = fieldSetFlags()[2] ? this.fiscalYear : (java.lang.Integer) defaultValue(fields()[2]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } +} diff --git a/chill-avro/src/test/scala/com/twitter/chill/avro/AvroSerializerSpec.scala b/chill-avro/src/test/scala/com/twitter/chill/avro/AvroSerializerSpec.scala new file mode 100644 index 00000000..f46c9100 --- /dev/null +++ b/chill-avro/src/test/scala/com/twitter/chill/avro/AvroSerializerSpec.scala @@ -0,0 +1,63 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.twitter.chill.avro + +import org.specs.Specification +import com.twitter.chill.{KSerializer, ScalaKryoInstantiator, KryoPool} +import org.apache.avro.specific.SpecificRecordBase +import avro.FiscalRecord + +/** + * @author Mansur Ashraf + * @since 2/9/14. + */ +object AvroSerializerSpec extends Specification { + + def getKryo[T <: SpecificRecordBase : Manifest](k: KSerializer[T]) = { + val inst = { + () => (new ScalaKryoInstantiator).newKryo.forClass(k) + } + KryoPool.withByteArrayOutputStream(1, inst) + } + + val testRrecord = FiscalRecord.newBuilder().setCalendarDate("2012-01-01").setFiscalWeek(1).setFiscalYear(2012).build() + + "SpecificRecordSerializer" should { + "Serialize and Deserialize Avro Record" in { + val kryo = getKryo(AvroSerializer.SpecificRecordSerializer[FiscalRecord]) + val bytes = kryo.toBytesWithClass(testRrecord) + val result = kryo.fromBytes(bytes).asInstanceOf[FiscalRecord] + testRrecord must_== result + } + } + + "SpecificRecordBinarySerializer" should { + "Serialize and Deserialize Avro Record" in { + val kryo = getKryo(AvroSerializer.SpecificRecordBinarySerializer[FiscalRecord]) + val bytes = kryo.toBytesWithClass(testRrecord) + val result = kryo.fromBytes(bytes).asInstanceOf[FiscalRecord] + testRrecord must_== result + } + } + + "SpecificRecordJsonSerializer" should { + "Serialize and Deserialize Avro Record" in { + val kryo = getKryo(AvroSerializer.SpecificRecordJsonSerializer[FiscalRecord](FiscalRecord.SCHEMA$)) + val bytes = kryo.toBytesWithClass(testRrecord) + val result = kryo.fromBytes(bytes).asInstanceOf[FiscalRecord] + testRrecord must_== result + } + } +} diff --git a/project/Build.scala b/project/Build.scala index bdcdf363..79febb3b 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -92,8 +92,9 @@ object ChillBuild extends Build { chillHadoop, chillThrift, chillProtobuf, - chillAkka - ) + chillAkka, + chillAvro + ) /** * This returns the youngest jar we released that is compatible @@ -153,7 +154,7 @@ object ChillBuild extends Build { lazy val chillBijection = module("bijection").settings( libraryDependencies ++= Seq( - "com.twitter" %% "bijection-core" % "0.5.2" + "com.twitter" %% "bijection-core" % "0.6.2" ) ).dependsOn(chill % "test->test;compile->compile") @@ -202,4 +203,12 @@ object ChillBuild extends Build { "com.google.protobuf" % "protobuf-java" % "2.3.0" % "provided" ) ).dependsOn(chillJava) + + lazy val chillAvro = module("avro").settings( + crossPaths := false, + autoScalaLibrary := false, + libraryDependencies ++= Seq( + "com.twitter" %% "bijection-avro" % "0.6.2" + ) + ).dependsOn(chill,chillJava, chillBijection) }