Skip to content

Commit

Permalink
Merge pull request #197 from stephanh/generated-tuple-aggregators
Browse files Browse the repository at this point in the history 
Added code to combine tuple of aggregators into one aggregator.
  • Loading branch information
@johnynek
johnynek committed Sep 18, 2013
2 parents 8623270 + 41f7484 commit d887b85
Show file tree
Hide file tree
Showing 3 changed files with 170 additions and 1 deletion.
121 changes: 121 additions & 0 deletions algebird-test/src/test/scala/com/twitter/algebird/TupleAggregatorsTest.scala
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package com.twitter.algebird

import org.specs._

class TupleAggregatorsTest extends Specification {
import GeneratedTupleAggregators._

val data = List(1, 3, 2, 0, 5, 6)
val MinAgg = new Aggregator[Int, Int, Int] {
def prepare(v: Int) = v
def reduce(v1: Int, v2: Int) = Math.min(v1, v2)
def present(v: Int) = v
}

"GeneratedTupleAggregators" should {
"Create an aggregator from a tuple of 2 aggregators" in {
val agg: Aggregator[Int, Tuple2[Int,Int], Tuple2[Int,Int]] = Tuple2(MinAgg,MinAgg)
agg(data) must be_==(Tuple2(0,0))
}

"Create an aggregator from a tuple of 3 aggregators" in {
val agg: Aggregator[Int, Tuple3[Int,Int,Int], Tuple3[Int,Int,Int]] = Tuple3(MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple3(0,0,0))
}

"Create an aggregator from a tuple of 4 aggregators" in {
val agg: Aggregator[Int, Tuple4[Int,Int,Int,Int], Tuple4[Int,Int,Int,Int]] = Tuple4(MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple4(0,0,0,0))
}

"Create an aggregator from a tuple of 5 aggregators" in {
val agg: Aggregator[Int, Tuple5[Int,Int,Int,Int,Int], Tuple5[Int,Int,Int,Int,Int]] = Tuple5(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple5(0,0,0,0,0))
}

"Create an aggregator from a tuple of 6 aggregators" in {
val agg: Aggregator[Int, Tuple6[Int,Int,Int,Int,Int,Int], Tuple6[Int,Int,Int,Int,Int,Int]] = Tuple6(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple6(0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 7 aggregators" in {
val agg: Aggregator[Int, Tuple7[Int,Int,Int,Int,Int,Int,Int], Tuple7[Int,Int,Int,Int,Int,Int,Int]] = Tuple7(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple7(0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 8 aggregators" in {
val agg: Aggregator[Int, Tuple8[Int,Int,Int,Int,Int,Int,Int,Int], Tuple8[Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple8(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple8(0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 9 aggregators" in {
val agg: Aggregator[Int, Tuple9[Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple9[Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple9(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple9(0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 10 aggregators" in {
val agg: Aggregator[Int, Tuple10[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple10[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple10(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple10(0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 11 aggregators" in {
val agg: Aggregator[Int, Tuple11[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple11[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple11(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple11(0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 12 aggregators" in {
val agg: Aggregator[Int, Tuple12[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple12[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple12(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple12(0,0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 13 aggregators" in {
val agg: Aggregator[Int, Tuple13[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple13[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple13(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple13(0,0,0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 14 aggregators" in {
val agg: Aggregator[Int, Tuple14[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple14[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple14(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple14(0,0,0,0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 15 aggregators" in {
val agg: Aggregator[Int, Tuple15[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple15[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple15(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple15(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 16 aggregators" in {
val agg: Aggregator[Int, Tuple16[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple16[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple16(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple16(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 17 aggregators" in {
val agg: Aggregator[Int, Tuple17[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple17[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple17(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple17(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 18 aggregators" in {
val agg: Aggregator[Int, Tuple18[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple18[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple18(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple18(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 19 aggregators" in {
val agg: Aggregator[Int, Tuple19[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple19[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple19(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple19(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 20 aggregators" in {
val agg: Aggregator[Int, Tuple20[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple20[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple20(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple20(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 21 aggregators" in {
val agg: Aggregator[Int, Tuple21[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple21[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple21(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple21(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0))
}

"Create an aggregator from a tuple of 22 aggregators" in {
val agg: Aggregator[Int, Tuple22[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int], Tuple22[Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int,Int]] = Tuple22(MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg,MinAgg)
agg(data) must be_==(Tuple22(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0))
}
}
}
5 changes: 4 additions & 1 deletion project/Build.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,10 @@ object AlgebirdBuild extends Build {
initialCommands := """
import com.twitter.algebird._
""".stripMargin('|'),
libraryDependencies += "com.googlecode.javaewah" % "JavaEWAH" % "0.6.6"
libraryDependencies += "com.googlecode.javaewah" % "JavaEWAH" % "0.6.6",
sourceGenerators in Compile <+= sourceManaged in Compile map { outDir: File =>
GenTupleAggregators.gen(outDir)
}
)

lazy val algebirdTest = module("test").settings(
Expand Down
45 changes: 45 additions & 0 deletions project/GenTupleAggregators.scala
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package algebird

import sbt._

object GenTupleAggregators {
def gen(dir: File) = {
val place = dir / "com" / "twitter" / "algebird" / "GeneratedTupleAggregators.scala"
IO.write(place,
"""package com.twitter.algebird
object GeneratedTupleAggregators extends GeneratedTupleAggregators
trait GeneratedTupleAggregators {
""" + genAggregators(22) + "\n" + "}")

Seq(place)
}

def genAggregators(max: Int): String = {
(2 to max).map(i => {
val nums = (1 to i)
val vs = nums.map("V" + _).mkString(", ")
val ts = nums.map("T" + _).mkString(", ")
val aggs = nums.map(x => "Aggregator[A, V%s, T%s]".format(x, x)).mkString(", ")
val prepares = nums.map(x => "aggs._%s.prepare(v)".format(x)).mkString(", ")
val reduces = nums.map(x => "aggs._%s.reduce(v1._%s, v2._%s)".format(x, x, x)).mkString(", ")
val present = nums.map(x => "aggs._%s.present(v._%s)".format(x, x)).mkString(", ")
val tupleVs = "Tuple%d[%s]".format(i, vs)
val tupleTs = "Tuple%d[%s]".format(i, ts)

"""
implicit def Tuple%dAggregator[A, %s, %s](aggs: Tuple%d[%s]): Aggregator[A, %s, %s] = {
new Aggregator[A, %s, %s] {
def prepare(v: A) = (%s)
def reduce(v1: %s, v2: %s) = (%s)
def present(v: %s) = (%s)
}
}""".format(i, vs, ts, i, aggs, tupleVs, tupleTs,
tupleVs, tupleTs,
prepares,
tupleVs, tupleVs, reduces,
tupleVs, present)
}).mkString("\n")
}
}

0 comments on commit d887b85

Please sign in to comment.