From d2ef58d622ab60592eea453ea1cb292b369ddadb Mon Sep 17 00:00:00 2001 From: Imran Rashid Date: Wed, 8 Apr 2015 13:48:51 -0500 Subject: [PATCH] revert changes that had HistoryServer refresh the application listing more often --- .../deploy/history/ApplicationHistoryProvider.scala | 2 +- .../spark/deploy/history/FsHistoryProvider.scala | 2 +- .../org/apache/spark/deploy/history/HistoryPage.scala | 2 +- .../apache/spark/deploy/history/HistoryServer.scala | 6 +++--- .../spark/deploy/history/FsHistoryProviderSuite.scala | 10 +++++----- .../spark/deploy/history/HistoryServerSuite.scala | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala index 93e5a08980ab3..9de865177f432 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala @@ -35,7 +35,7 @@ private[history] abstract class ApplicationHistoryProvider { * * @return List of all know applications. */ - def getListing(refresh: Boolean): Iterable[ApplicationHistoryInfo] + def getListing(): Iterable[ApplicationHistoryInfo] /** * Returns the Spark UI for a specific application. diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index 0cdc3ff43b40c..e46672265d07a 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -139,7 +139,7 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis } } - override def getListing(refresh: Boolean): Iterable[FsApplicationHistoryInfo] = { + override def getListing(): Iterable[FsApplicationHistoryInfo] = { applications.values } diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala index 9b7b0cb0b035f..4e9ff5a443e7d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala @@ -34,7 +34,7 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") val requestedIncomplete = Option(request.getParameter("showIncomplete")).getOrElse("false").toBoolean - val allApps = parent.getApplicationList(false).filter(_.completed != requestedIncomplete) + val allApps = parent.getApplicationList().filter(_.completed != requestedIncomplete) val actualFirst = if (requestedFirst < allApps.size) requestedFirst else 0 val apps = allApps.slice(actualFirst, Math.min(actualFirst + pageSize, allApps.size)) diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index df49c79944aa1..4ad56bba22d35 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -159,12 +159,12 @@ class HistoryServer( * * @return List of all known applications. */ - def getApplicationList(refresh: Boolean): Iterable[ApplicationHistoryInfo] = { - provider.getListing(refresh) + def getApplicationList(): Iterable[ApplicationHistoryInfo] = { + provider.getListing() } def getApplicationInfoList: Seq[ApplicationInfo] = { - getApplicationList(true).map { ApplicationsListResource.appHistoryInfoToPublicAppInfo }.toSeq + getApplicationList().map { ApplicationsListResource.appHistoryInfoToPublicAppInfo }.toSeq } /** diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala index ae953fe04833c..e908ba604ebed 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala @@ -103,7 +103,7 @@ class FsHistoryProviderSuite extends FunSuite with BeforeAndAfter with Matchers // Take the opportunity to check that the offset checks work as expected. provider.checkForLogs() - val list = provider.getListing(false).toSeq + val list = provider.getListing().toSeq list should not be (null) list.size should be (5) list.count(_.completed) should be (3) @@ -174,7 +174,7 @@ class FsHistoryProviderSuite extends FunSuite with BeforeAndAfter with Matchers val provider = new FsHistoryProvider(createTestConf()) provider.checkForLogs() - val list = provider.getListing(false).toSeq + val list = provider.getListing().toSeq list should not be (null) list.size should be (1) } @@ -188,13 +188,13 @@ class FsHistoryProviderSuite extends FunSuite with BeforeAndAfter with Matchers SparkListenerApplicationEnd(2L) ) provider.checkForLogs() - val appListBeforeRename = provider.getListing(false) + val appListBeforeRename = provider.getListing() appListBeforeRename.size should be (1) appListBeforeRename.head.logPath should endWith(EventLoggingListener.IN_PROGRESS) logFile1.renameTo(newLogFile("app1", inProgress = false)) provider.checkForLogs() - val appListAfterRename = provider.getListing(false) + val appListAfterRename = provider.getListing() appListAfterRename.size should be (1) appListAfterRename.head.logPath should not endWith(EventLoggingListener.IN_PROGRESS) } @@ -211,7 +211,7 @@ class FsHistoryProviderSuite extends FunSuite with BeforeAndAfter with Matchers oldLog.mkdir() provider.checkForLogs() - val appListAfterRename = provider.getListing(false) + val appListAfterRename = provider.getListing() appListAfterRename.size should be (1) } diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 3f71918926f58..06180add40a62 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -184,7 +184,7 @@ class HistoryServerSuite extends FunSuite with BeforeAndAfter with Matchers with val ui = mock[SparkUI] val link = "/history/app1" val info = new ApplicationHistoryInfo("app1", "app1", 0, 2, 1, "xxx", true) - when(historyServer.getApplicationList(false)).thenReturn(Seq(info)) + when(historyServer.getApplicationList()).thenReturn(Seq(info)) when(ui.basePath).thenReturn(link) when(historyServer.getProviderConfig()).thenReturn(Map[String, String]()) val page = new HistoryPage(historyServer)