Merge pull request #403 from ashelkovnykov/total

Remove dependency on photon-diagnostics module from photon-client

photon-all/build.gradle

@@ -24,6 +24,7 @@ dependencies {
 configurations {
   runtime {
     exclude group: 'log4j'
+    exclude group: 'org.apache.hadoop'
     exclude group: 'org.apache.spark'
     exclude group: 'org.scala-lang'
     exclude group: 'org.slf4j'
@@ -54,7 +55,10 @@ sourcesJar {
 
 configurations {
   all {
+    // Remove Snappy JAR which causes downstream issues
     exclude group: 'org.xerial.snappy'
+    // Remove JUnit testing dependencies that end up in the jar through Breeze
+    exclude group: 'junit'
   }
 }
 

photon-api/build.gradle

@@ -17,7 +17,6 @@ apply from: '../build-scripts/integration-test.gradle'
 
 dependencies {
   compile(project(":photon-lib$scalaSuffix"))
-  compile(project(":photon-test-utils$scalaSuffix"))
   compile("org.scalanlp:breeze$scalaSuffix:0.11.2")
   compile('com.linkedin.paldb:paldb:1.1.0')
 

photon-client/build.gradle

@@ -18,7 +18,6 @@ apply from: '../build-scripts/integration-test.gradle'
 dependencies {
   compile(project(':photon-avro-schemas'))
   compile(project(":photon-core$scalaSuffix"))
-  compile(project(":photon-diagnostics$scalaSuffix"))
   compile('joda-time:joda-time:2.7')
   // Stub warnings without this
   compile('org.joda:joda-convert:1.8.1')

photon-client/src/integTest/scala/com/linkedin/photon/ml/MockDriver.scala

@@ -18,13 +18,6 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.SparkContext
 import org.testng.Assert._
 
-import com.linkedin.photon.ml.diagnostics.DiagnosticMode.DiagnosticMode
-import com.linkedin.photon.ml.diagnostics.DiagnosticStatus
-import com.linkedin.photon.ml.diagnostics.bootstrap.BootstrapReport
-import com.linkedin.photon.ml.diagnostics.featureimportance.FeatureImportanceReport
-import com.linkedin.photon.ml.diagnostics.fitting.FittingReport
-import com.linkedin.photon.ml.diagnostics.hl.HosmerLemeshowReport
-import com.linkedin.photon.ml.diagnostics.independence.PredictionErrorIndependenceReport
 import com.linkedin.photon.ml.stat.FeatureDataStatistics
 import com.linkedin.photon.ml.util.PhotonLogger
 
@@ -40,10 +33,6 @@ class MockDriver(
     override val seed: Long)
   extends Driver(params: Params, sc: SparkContext, logger: PhotonLogger, seed) {
 
-  /**
-   * Diagnostic status for current run
-   */
-  private val diagnosticStatus = DiagnosticStatus(trainDiagnosed = false, validateDiagnosed = false)
   /**
    * Have the input features been summarized
    */
@@ -65,36 +54,6 @@ class MockDriver(
    */
   def metrics: Map[Double, Map[String, Double]] = perModelMetrics
 
-  /**
-   *
-   */
-  override protected def initializeDiagnosticReport(): Unit = {
-    diagnosticStatus.trainDiagnosed = false
-    diagnosticStatus.validateDiagnosed = false
-    super.initializeDiagnosticReport()
-  }
-
-  /**
-   *
-   * @return
-   */
-  override protected def trainDiagnostic(): (Map[Double, FittingReport], Map[Double, BootstrapReport]) = {
-    diagnosticStatus.trainDiagnosed = true
-    super.trainDiagnostic()
-  }
-
-  /**
-   *
-   * @return
-   */
-  override protected def validateDiagnostic(): (
-      Map[Double, (FeatureImportanceReport, FeatureImportanceReport, PredictionErrorIndependenceReport)],
-      Map[Double, Option[HosmerLemeshowReport]]) = {
-
-    diagnosticStatus.validateDiagnosed = true
-    super.validateDiagnostic()
-  }
-
   /**
    *
    * @param outputDir
@@ -120,16 +79,14 @@ object MockDriver {
    * @param expectedNumFeatures The expected number of features in the input data
    * @param expectedNumTrainingData The expected number of training records
    * @param expectedIsSummarized Whether feature summarization was expected or not
-   * @param expectedDiagnosticMode The expected levels of diagnostics run
    */
   def runLocally(
       args: Array[String],
       sc: SparkContext,
       expectedStages: Array[DriverStage],
       expectedNumFeatures: Int,
       expectedNumTrainingData: Int,
-      expectedIsSummarized: Boolean,
-      expectedDiagnosticMode: DiagnosticMode): MockDriver = {
+      expectedIsSummarized: Boolean): MockDriver = {
 
     // Parse the parameters from command line, should always be the 1st line in main
     val params = PhotonMLCmdLineParser.parseFromCommandLine(args)
@@ -148,7 +105,6 @@ object MockDriver {
     assertEquals(job.numTrainingData(), expectedNumTrainingData,
       "The number of training data points " + job.numTrainingData() + " do not meet the expectation")
     assertEquals(job.isSummarized, expectedIsSummarized)
-    assertEquals(job.diagnosticStatus.getDiagnosticMode, expectedDiagnosticMode)
 
     // Closing up
     logger.close()