From 25fe97f05aac5c4f44bea9d1356722e6735cc940 Mon Sep 17 00:00:00 2001 From: Jacek Laskowski Date: Tue, 29 May 2018 09:32:12 +0200 Subject: [PATCH 1/3] [SPARK-24408][SQL][DOC] Move abs, bitwiseNOT, isnan, nanvl functions to math_funcs group --- .../org/apache/spark/sql/functions.scala | 70 ++++++++++--------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index 5ab9cb3fb86a5..a211f9a09f281 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -1031,14 +1031,6 @@ object functions { // Non-aggregate functions ////////////////////////////////////////////////////////////////////////////////////////////// - /** - * Computes the absolute value. - * - * @group normal_funcs - * @since 1.3.0 - */ - def abs(e: Column): Column = withExpr { Abs(e.expr) } - /** * Creates a new array column. The input columns must all have the same data type. * @@ -1107,14 +1099,6 @@ object functions { */ def input_file_name(): Column = withExpr { InputFileName() } - /** - * Return true iff the column is NaN. - * - * @group normal_funcs - * @since 1.6.0 - */ - def isnan(e: Column): Column = withExpr { IsNaN(e.expr) } - /** * Return true iff the column is null. * @@ -1164,16 +1148,6 @@ object functions { */ def monotonically_increasing_id(): Column = withExpr { MonotonicallyIncreasingID() } - /** - * Returns col1 if it is not NaN, or col2 if col1 is NaN. - * - * Both inputs should be floating point columns (DoubleType or FloatType). - * - * @group normal_funcs - * @since 1.5.0 - */ - def nanvl(col1: Column, col2: Column): Column = withExpr { NaNvl(col1.expr, col2.expr) } - /** * Unary minus, i.e. negate the expression. * {{{ @@ -1324,14 +1298,6 @@ object functions { CaseWhen(Seq((condition.expr, lit(value).expr))) } - /** - * Computes bitwise NOT. - * - * @group normal_funcs - * @since 1.4.0 - */ - def bitwiseNOT(e: Column): Column = withExpr { BitwiseNot(e.expr) } - /** * Parses the expression string into the column that it represents, similar to * [[Dataset#selectExpr]]. @@ -1353,6 +1319,14 @@ object functions { // Math Functions ////////////////////////////////////////////////////////////////////////////////////////////// + /** + * Computes the absolute value of a numeric value. + * + * @group math_funcs + * @since 1.3.0 + */ + def abs(e: Column): Column = withExpr { Abs(e.expr) } + /** * @return inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos` * @@ -1532,6 +1506,14 @@ object functions { */ def bin(columnName: String): Column = bin(Column(columnName)) + /** + * Computes bitwise NOT (~) of a number. + * + * @group math_funcs + * @since 1.4.0 + */ + def bitwiseNOT(e: Column): Column = withExpr { BitwiseNot(e.expr) } + /** * Computes the cube-root of the given value. * @@ -1696,6 +1678,26 @@ object functions { */ def hex(column: Column): Column = withExpr { Hex(column.expr) } + /** + * Return true iff the column is NaN. + * + * Both inputs should be floating point columns (DoubleType or FloatType). + * + * @group math_funcs + * @since 1.6.0 + */ + def isnan(e: Column): Column = withExpr { IsNaN(e.expr) } + + /** + * Returns col1 if it is not NaN, or col2 if col1 is NaN. + * + * Both inputs should be floating point columns (DoubleType or FloatType). + * + * @group math_funcs + * @since 1.5.0 + */ + def nanvl(col1: Column, col2: Column): Column = withExpr { NaNvl(col1.expr, col2.expr) } + /** * Inverse of hex. Interprets each pair of characters as a hexadecimal number * and converts to the byte representation of number. From 487a467219014ea2e322861c3053d2a374740058 Mon Sep 17 00:00:00 2001 From: Jacek Laskowski Date: Tue, 29 May 2018 12:11:58 +0200 Subject: [PATCH 2/3] Fixing Scala style tests --- .../org/apache/spark/sql/functions.scala | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index a211f9a09f281..6917214528417 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -1320,11 +1320,11 @@ object functions { ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Computes the absolute value of a numeric value. - * - * @group math_funcs - * @since 1.3.0 - */ + * Computes the absolute value of a numeric value. + * + * @group math_funcs + * @since 1.3.0 + */ def abs(e: Column): Column = withExpr { Abs(e.expr) } /** @@ -1507,11 +1507,11 @@ object functions { def bin(columnName: String): Column = bin(Column(columnName)) /** - * Computes bitwise NOT (~) of a number. - * - * @group math_funcs - * @since 1.4.0 - */ + * Computes bitwise NOT (~) of a number. + * + * @group math_funcs + * @since 1.4.0 + */ def bitwiseNOT(e: Column): Column = withExpr { BitwiseNot(e.expr) } /** @@ -1679,23 +1679,23 @@ object functions { def hex(column: Column): Column = withExpr { Hex(column.expr) } /** - * Return true iff the column is NaN. - * - * Both inputs should be floating point columns (DoubleType or FloatType). - * - * @group math_funcs - * @since 1.6.0 - */ + * Return true iff the column is NaN. + * + * Both inputs should be floating point columns (DoubleType or FloatType). + * + * @group math_funcs + * @since 1.6.0 + */ def isnan(e: Column): Column = withExpr { IsNaN(e.expr) } /** - * Returns col1 if it is not NaN, or col2 if col1 is NaN. - * - * Both inputs should be floating point columns (DoubleType or FloatType). - * - * @group math_funcs - * @since 1.5.0 - */ + * Returns col1 if it is not NaN, or col2 if col1 is NaN. + * + * Both inputs should be floating point columns (DoubleType or FloatType). + * + * @group math_funcs + * @since 1.5.0 + */ def nanvl(col1: Column, col2: Column): Column = withExpr { NaNvl(col1.expr, col2.expr) } /** From 2bbc7500bd1b9360759c9924638d8457a86c7455 Mon Sep 17 00:00:00 2001 From: Jacek Laskowski Date: Thu, 28 Jun 2018 22:13:55 +0200 Subject: [PATCH 3/3] Reverting some changes after code review --- .../org/apache/spark/sql/functions.scala | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index 6917214528417..4a60a17f471f8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -1099,6 +1099,14 @@ object functions { */ def input_file_name(): Column = withExpr { InputFileName() } + /** + * Return true iff the column is NaN. + * + * @group normal_funcs + * @since 1.6.0 + */ + def isnan(e: Column): Column = withExpr { IsNaN(e.expr) } + /** * Return true iff the column is null. * @@ -1148,6 +1156,16 @@ object functions { */ def monotonically_increasing_id(): Column = withExpr { MonotonicallyIncreasingID() } + /** + * Returns col1 if it is not NaN, or col2 if col1 is NaN. + * + * Both inputs should be floating point columns (DoubleType or FloatType). + * + * @group normal_funcs + * @since 1.5.0 + */ + def nanvl(col1: Column, col2: Column): Column = withExpr { NaNvl(col1.expr, col2.expr) } + /** * Unary minus, i.e. negate the expression. * {{{ @@ -1298,6 +1316,14 @@ object functions { CaseWhen(Seq((condition.expr, lit(value).expr))) } + /** + * Computes bitwise NOT (~) of a number. + * + * @group normal_funcs + * @since 1.4.0 + */ + def bitwiseNOT(e: Column): Column = withExpr { BitwiseNot(e.expr) } + /** * Parses the expression string into the column that it represents, similar to * [[Dataset#selectExpr]]. @@ -1506,14 +1532,6 @@ object functions { */ def bin(columnName: String): Column = bin(Column(columnName)) - /** - * Computes bitwise NOT (~) of a number. - * - * @group math_funcs - * @since 1.4.0 - */ - def bitwiseNOT(e: Column): Column = withExpr { BitwiseNot(e.expr) } - /** * Computes the cube-root of the given value. * @@ -1678,26 +1696,6 @@ object functions { */ def hex(column: Column): Column = withExpr { Hex(column.expr) } - /** - * Return true iff the column is NaN. - * - * Both inputs should be floating point columns (DoubleType or FloatType). - * - * @group math_funcs - * @since 1.6.0 - */ - def isnan(e: Column): Column = withExpr { IsNaN(e.expr) } - - /** - * Returns col1 if it is not NaN, or col2 if col1 is NaN. - * - * Both inputs should be floating point columns (DoubleType or FloatType). - * - * @group math_funcs - * @since 1.5.0 - */ - def nanvl(col1: Column, col2: Column): Column = withExpr { NaNvl(col1.expr, col2.expr) } - /** * Inverse of hex. Interprets each pair of characters as a hexadecimal number * and converts to the byte representation of number.