Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Arithmetic/Logical/String expressions implementation #2

Closed
wants to merge 2 commits into from
Closed

Arithmetic/Logical/String expressions implementation #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5071f61
And more arithmetic / logical CG Expressions & their unittest
chenghao-intel Mar 24, 2014
f1d4766
implement the like & rlike
chenghao-intel Mar 24, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 50 additions & 14 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/GeneratedRow.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,22 +128,11 @@ class CodeGenerator extends Logging {
${getColumn(inputTuple, b.dataType, ordinal)}
""".children

case expressions.Literal(value: String, dataType) =>
case expressions.Literal(value, dataType) =>
q"""
val $nullTerm = ${value == null}
val $primitiveTerm: ${termForType(dataType)} = $value
val $primitiveTerm: ${termForType(dataType)} = ${value.toString}
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this is doing the right thing for all data types. The string here is getting lifted to a Scala AST String Literal, which means that independent of the dataType, you are splicing a String literal into the AST.

""".children
case expressions.Literal(value: Int, dataType) =>
q"""
val $nullTerm = ${value == null}
val $primitiveTerm: ${termForType(dataType)} = $value
""".children
case expressions.Literal(value: Long, dataType) =>
q"""
val $nullTerm = ${value == null}
val $primitiveTerm: ${termForType(dataType)} = $value
""".children

case Cast(e, StringType) =>
val eval = expressionEvaluator(e)
eval.code ++
Expand Down Expand Up @@ -188,19 +177,55 @@ class CodeGenerator extends Logging {
$primitiveTerm = true
}
""".children
case Or(e1, e2) =>
val eval1 = expressionEvaluator(e1)
val eval2 = expressionEvaluator(e2)

eval1.code ++ eval2.code ++
q"""
var $nullTerm = false
var $primitiveTerm: ${termForType(BooleanType)} = false

if ((!${eval1.nullTerm} && ${eval1.primitiveTerm}) ||
(!${eval2.nullTerm} && ${eval2.primitiveTerm})) {
$nullTerm = false
$primitiveTerm = true
} else if (${eval1.nullTerm} || ${eval2.nullTerm} ) {
$nullTerm = true
} else {
$nullTerm = false
$primitiveTerm = false
}
""".children

case Add(e1, e2) => (e1, e2) evaluate { case (eval1, eval2) => q"$eval1 + $eval2" }
case Subtract(e1, e2) => (e1, e2) evaluate { case (eval1, eval2) => q"$eval1 - $eval2" }
case Multiply(e1, e2) => (e1, e2) evaluate { case (eval1, eval2) => q"$eval1 * $eval2" }
case Divide(e1, e2) => (e1, e2) evaluate { case (eval1, eval2) => q"$eval1 / $eval2" }
case Remainder(e1, e2) =>(e1, e2) evaluate { case (eval1, eval2) => q"$eval1 % $eval2" }

case UnaryMinus(e) =>
val eval = expressionEvaluator(e)
q"""
..${eval.code}
val $nullTerm = ${eval.nullTerm}
val $primitiveTerm: ${termForType(e.dataType)} = -${eval.primitiveTerm}
""".children

case IsNotNull(e) =>
val eval = expressionEvaluator(e)
q"""
..${eval.code}
var $nullTerm = false
var $primitiveTerm: ${termForType(BooleanType)} = !${eval.nullTerm}
var $primitiveTerm: ${termForType(BooleanType)} = ${eval.nullTerm}.unary_!
""".children

case IsNull(e) =>
val eval = expressionEvaluator(e)
q"""
..${eval.code}
var $nullTerm = false
var $primitiveTerm: ${termForType(BooleanType)} = ${eval.nullTerm}
""".children

case c @ Coalesce(children) =>
Expand All @@ -221,6 +246,17 @@ class CodeGenerator extends Logging {
"""
}

case Not(e) =>
val eval = expressionEvaluator(e)
q"""
..${eval.code}
var $nullTerm = ${eval.nullTerm}
var $primitiveTerm: ${termForType(BooleanType)} = ${eval.primitiveTerm}.unary_!
""".children

// TODO transform the In to If
// case In(v, list) =>

case i @ expressions.If(condition, trueValue, falseValue) =>
val condEval = expressionEvaluator(condition)
val trueEval = expressionEvaluator(trueValue)
Expand Down
97 changes: 94 additions & 3 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,102 @@ package org.apache.spark.sql
package catalyst
package expressions

import scala.util.matching.Regex

import catalyst.types.StringType
import catalyst.types.BooleanType
import analysis.UnresolvedException
import catalyst.errors.`package`.TreeNodeException


abstract class BinaryString extends BinaryExpression {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks good, but I was initially confused by the name. Maybe BinaryStringExpression? BinaryString had me thinking that this was a string encoded in binary or something. A short sentence for scaladoc would also be helpful.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or maybe BinaryStringPredicate, since it always returns a boolean.

self: Product =>

type EvaluatedType = Any

def nullable = left.nullable || right.nullable

override lazy val resolved =
left.resolved && right.resolved && left.dataType == StringType && right.dataType == StringType

def dataType = {
if (!resolved) {
throw new UnresolvedException(this,
s"datatype. Can not resolve due to non string types ${left.dataType}, ${right.dataType}")
}

BooleanType
}

@inline
protected final def s2(
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we really need these functions? In the case of of using Numerics you have to cast back and forth multiple times, but I think the resulting code here is actually more verbose than if you just did this inline.

i: Row,
e1: Expression,
e2: Expression,
f: ((String, String) => Boolean)): Any = {

if (e1.dataType != StringType) {
throw new TreeNodeException(this, s"Types do not match ${e1.dataType} != StringType")
}

case class Like(left: Expression, right: Expression) extends BinaryExpression {
def dataType = BooleanType
def nullable = left.nullable // Right cannot be null.
if (e2.dataType != StringType) {
throw new TreeNodeException(this, s"Types do not match ${e2.dataType} != StringType")
}

val evalE1 = e1.apply(i)
if(evalE1 == null) {
null
} else {
val evalE2 = e2.apply(i)
if (evalE2 == null) {
null
} else {
f.apply(evalE1.asInstanceOf[String], evalE1.asInstanceOf[String])
}
}
}

@inline
protected final def s1(
i: Row,
e1: Expression,
f: ((String) => Boolean)): Any = {

if (e1.dataType != StringType) {
throw new TreeNodeException(this, s"Types do not match ${e1.dataType} != StringType")
}

val evalE1 = e1.apply(i)
if(evalE1 == null) {
null
} else {
f.apply(evalE1.asInstanceOf[String])
}
}
}

case class Like(left: Expression, right: Literal) extends BinaryString {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Independent of what is required by the SQL spec, I think both of these should be Expressions. This will give us a lot more flexibility and make it easier to integrate with the parser. You can still get the string value by calling right.evaluate(EmptyRow).asInstanceOf[String].

def symbol = "LIKE"
// replace the _ with .{1} exactly match 1 time of any character
// replace the % with .*, match 0 or more times with any character
def regex(v: String) = v.replaceAll("_", ".{1}").replaceAll("%", ".*")
lazy val r = regex(right.value.asInstanceOf[String]).r

override def apply(input: Row): Any = if(right.value == null) {
null
} else {
s1(input, left, r.findFirstIn(_) != None)
}
}

case class RLike(left: Expression, right: Literal) extends BinaryString {
def symbol = "RLIKE"

lazy val r = right.value.asInstanceOf[String].r

override def apply(input: Row): Any = if(right.value == null) {
null
} else {
s1(input, left, r.findFirstIn(_) != None)
}
}
68 changes: 0 additions & 68 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CodeGenerationSuite.scala
View file
Open in desktop

This file was deleted.

Loading