Skip to content

Commit

Permalink
Fixes #93 conflicts
Browse files Browse the repository at this point in the history
  • Loading branch information
@TebaleloS
TebaleloS committed May 4, 2023
1 parent 6dbe960 commit 7d82796
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 73 deletions.
30 changes: 7 additions & 23 deletions .../scala/za/co/absa/spark/commons/errorhandling/implementations/ErrorMessageArrayTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,14 @@ import za.co.absa.spark.commons.errorhandling.types.ErrorWhen
import za.co.absa.spark.commons.test.SparkTestBase
import org.apache.spark.sql.functions.{col, length}
import za.co.absa.spark.commons.errorhandling.ErrorMessage
<<<<<<< HEAD
import za.co.absa.spark.commons.errorhandling.implementations.submits.{ErrorMessageSubmitOnColumn, ErrorMessageSubmitWithoutColumn}
=======
import za.co.absa.spark.commons.errorhandling.implementations.submits.{ErrorMessageSubmitJustErrorValue, ErrorMessageSubmitOnColumn, ErrorMessageSubmitOnMoreColumns, ErrorMessageSubmitWithoutColumn}
import za.co.absa.spark.commons.errorhandling.implementations.submits
.{
ErrorMessageSubmitJustErrorValue,
ErrorMessageSubmitOnColumn,
ErrorMessageSubmitOnMoreColumns,
ErrorMessageSubmitWithoutColumn
}
import za.co.absa.spark.commons.errorhandling.types.ColumnOrValue.CoV
>>>>>>> master


class ErrorMessageArrayTest extends AnyFunSuite with SparkTestBase {
import spark.implicits._
Expand All @@ -48,12 +49,8 @@ class ErrorMessageArrayTest extends AnyFunSuite with SparkTestBase {
(None, ""),
(Some(1), "a"),
(Some(2), "bb"),
<<<<<<< HEAD
(Some(3), "ccc")
=======
(Some(3), "ccc"),
(Some(0), "X")
>>>>>>> master
).toDF(col1Name, col2Name)

private type ResultDfRecordType = (Option[Integer], String, List[ErrorMessage])
Expand All @@ -69,14 +66,11 @@ class ErrorMessageArrayTest extends AnyFunSuite with SparkTestBase {
ErrorMessage("Test error 2", 2, "This is a test error", Map("Col2" -> "")),
ErrorMessage("Test error 3", 3, "This is a test error", Map.empty)
)),
<<<<<<< HEAD
=======
(Some(0), "X", List(
ErrorMessage("Test error 1", 1, "This is a test error", Map("Col1" -> "0")),
ErrorMessage("Test error 2", 2, "This is a test error", Map("Col2" -> "X")),
ErrorMessage("Test error 3", 3, "This is a test error", Map.empty)
)),
>>>>>>> master
(Some(1), "a", List(
ErrorMessage("Test error 1", 1, "This is a test error", Map("Col1" -> "1")),
ErrorMessage("Test error 2", 2, "This is a test error", Map("Col2" -> "a")),
Expand Down Expand Up @@ -114,10 +108,7 @@ class ErrorMessageArrayTest extends AnyFunSuite with SparkTestBase {
(None, "", List(
ErrorMessage("WrongLine", 0, "This line is wrong", Map.empty)
)),
<<<<<<< HEAD
=======
(Some(0), "X", List.empty),
>>>>>>> master
(Some(1), "a", List.empty),
(Some(2), "bb", List(
ErrorMessage("ValueStillTooBig", 2, "The value of the field is too big", Map("Col1" -> "2"))
Expand Down Expand Up @@ -145,10 +136,7 @@ class ErrorMessageArrayTest extends AnyFunSuite with SparkTestBase {
(None, "", List(
ErrorMessage("WrongLine", 0, "This line is wrong", Map.empty)
)),
<<<<<<< HEAD
=======
(Some(0), "X", List.empty),
>>>>>>> master
(Some(1), "a", List.empty),
(Some(2), "bb", List(
ErrorMessage("ValueStillTooBig", 2, "The value of the field is too big", Map("Col1" -> "2"))
Expand All @@ -171,13 +159,10 @@ class ErrorMessageArrayTest extends AnyFunSuite with SparkTestBase {

assert(result == expected)
}
<<<<<<< HEAD
=======

test("Various error submits combined") {
val errorMessageArray = ErrorMessageArray("MyErrCol")


case class NullError(errColName: String) extends ErrorMessageSubmitOnColumn(
CoV.withValue("Null Error"),
CoV.withValue(1L),
Expand Down Expand Up @@ -221,5 +206,4 @@ class ErrorMessageArrayTest extends AnyFunSuite with SparkTestBase {
assert(result == expected)

}
>>>>>>> master
}
8 changes: 0 additions & 8 deletions ...mmons/src/test/scala/za/co/absa/spark/commons/errorhandling/types/ColumnOrValueForm.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,21 +21,13 @@ import org.scalatest.Assertions

case class ColumnOrValueForm[T] (
column: Column,
<<<<<<< HEAD
isColumn: Boolean,
isValue: Boolean,
=======
>>>>>>> master
columnNames: Set[String],
value: Option[T]
) extends Assertions {
def assertTo(columnOrValue: ColumnOrValue[T]): Unit ={
assert(column == columnOrValue.column)
<<<<<<< HEAD
assert(isColumn == columnOrValue.isColumn)
assert(isValue == columnOrValue.isValue)
=======
>>>>>>> master
assert(columnNames == columnOrValue.columnNames)
assert(value == columnOrValue.getValue)
}
Expand Down
54 changes: 12 additions & 42 deletions ...mmons/src/test/scala/za/co/absa/spark/commons/errorhandling/types/ColumnOrValueTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,6 @@

package za.co.absa.spark.commons.errorhandling.types

<<<<<<< HEAD
import net.bytebuddy.dynamic.scaffold.MethodGraph.Empty
=======
>>>>>>> master
import org.apache.spark.sql.Column
import org.apache.spark.sql.functions.{array, col, current_date, lit, map_from_arrays}
import org.apache.spark.sql.types.StringType
Expand All @@ -29,22 +25,17 @@ import za.co.absa.spark.commons.sql.functions.null_col
class ColumnOrValueTest extends AnyFunSuite {
test("Creation of column based on its name"){
val colName = "my_column"
<<<<<<< HEAD
val expected = ColumnOrValueForm(col(colName), isColumn = true, isValue = false, Set(colName), None)
=======

val expected = ColumnOrValueForm(col(colName), Set(colName), None)
>>>>>>> master
val result = ColumnOrValue(colName)

expected assertTo result
}

test("Creation of column based on its definition") {
val myColumn = current_date
<<<<<<< HEAD
val expected = ColumnOrValueForm(myColumn, isColumn = true, isValue = false, Set(), None)
=======
val expected = ColumnOrValueForm(myColumn, Set(), None)
>>>>>>> master

val result = ColumnOrValue(myColumn)
expected assertTo result
}
Expand All @@ -58,57 +49,41 @@ class ColumnOrValueTest extends AnyFunSuite {
), array(
col("Col1"), col("Col2"), col("Col3")
))
<<<<<<< HEAD
val expected = ColumnOrValueForm[Map[String, Any]](expectedColumn, isColumn = true, isValue = false, colNames, None)
val result = ColumnOrValue[Any](colNames, colTransformer)
expected assertTo(result)
=======

val expected = ColumnOrValueForm[Map[String, Any]](expectedColumn, colNames, None)
val result = ColumnOrValue[Any](colNames, colTransformer)
expected assertTo result
>>>>>>> master
}

test("Creating ColumnOrValue from a defined Option") {
val value = "Foo"
<<<<<<< HEAD
val expected = ColumnOrValueForm(lit(value), isColumn = false, isValue = true, Set(), Option(Option(value)))
=======

val expected = ColumnOrValueForm(lit(value), Set(), Option(Option(value)))
>>>>>>> master

val result = ColumnOrValue.withOption(Option(value))
expected assertTo result
}

test("Creating ColumnOrValue from an empty Option") {
<<<<<<< HEAD
val expected = ColumnOrValueForm[Option[String]](null_col(StringType), isColumn = false, isValue = true, Set(), None)
=======
val expected = ColumnOrValueForm[Option[String]](null_col(StringType), Set(), None)
>>>>>>> master

val result = ColumnOrValue.withOption(None)
expected assertTo result
}

test("Creating ColumnOrValue from a given value") {
val value = 42
<<<<<<< HEAD
val expected = ColumnOrValueForm(lit(value), isColumn = false, isValue = true, Set(), Some(value))
=======

val expected = ColumnOrValueForm(lit(value), Set(), Some(value))
>>>>>>> master

val result = ColumnOrValue.withValue(value)
expected assertTo result
}

test("Creating ColumnOrValue as an undefined (empty) value") {

val myColumn = null_col(StringType)
<<<<<<< HEAD
val expected = ColumnOrValueForm[Option[String]](myColumn, isColumn = false, isValue = true, Set(), None)
=======
val expected = ColumnOrValueForm[Option[String]](myColumn, Set(), None)
>>>>>>> master

val result = ColumnOrValue.asEmpty
expected assertTo result
}
Expand All @@ -121,15 +96,10 @@ class ColumnOrValueTest extends AnyFunSuite {
), array(
col("Col1").cast(StringType), col("Col2").cast(StringType), col("Col3").cast(StringType)
))
<<<<<<< HEAD
val expected = ColumnOrValueForm[Map[String, String]](expectedColumn, isColumn = true, isValue = false, colNames, None)
val result = ColumnOrValue.asMapOfStringColumns(colNames)
expected assertTo(result)
=======
val expected = ColumnOrValueForm[Map[String, String]](expectedColumn, colNames, None)

val expected = ColumnOrValueForm[Map[String, String]](expectedColumn, colNames, String)
val result = ColumnOrValue.asMapOfStringColumns(colNames)
expected assertTo result
>>>>>>> master
}


Expand Down

0 comments on commit 7d82796

Please sign in to comment.