diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index d277f1ef63027..b1d3c8ad9b88c 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -19,8 +19,8 @@ package org.apache.spark import java.io._ import java.net.URI -import java.util.{Properties, UUID} import java.util.concurrent.atomic.AtomicInteger +import java.util.{Properties, UUID} import java.util.UUID.randomUUID import scala.collection.{Map, Set} import scala.collection.generic.Growable @@ -125,7 +125,7 @@ class SparkContext( val master = conf.get("spark.master") val appName = conf.get("spark.app.name") - + // Generate the random name for a temp folder in Tachyon // Add a timestamp as the suffix here to make it more safe val tachyonFolderName = "spark-" + randomUUID.toString() diff --git a/core/src/main/scala/org/apache/spark/storage/TachyonBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/TachyonBlockManager.scala index 0b4082de04162..b0b9674856568 100644 --- a/core/src/main/scala/org/apache/spark/storage/TachyonBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/TachyonBlockManager.scala @@ -36,8 +36,8 @@ import org.apache.spark.util.Utils * @param rootDirs The directories to use for storing block files. Data will be hashed among these. */ private[spark] class TachyonBlockManager( - shuffleManager: ShuffleBlockManager, - rootDirs: String, + shuffleManager: ShuffleBlockManager, + rootDirs: String, val master: String) extends Logging { @@ -49,21 +49,21 @@ private[spark] class TachyonBlockManager( } private val MAX_DIR_CREATION_ATTEMPTS = 10 - private val subDirsPerTachyonDir = + private val subDirsPerTachyonDir = shuffleManager.conf.get("spark.tachyonStore.subDirectories", "64").toInt - // Create one Tachyon directory for each path mentioned in spark.tachyonStore.folderName.dir; - // then, inside this directory, create multiple subdirectories that we will hash files into, + // Create one Tachyon directory for each path mentioned in spark.tachyonStore.folderName; + // then, inside this directory, create multiple subdirectories that we will hash files into, // in order to avoid having really large inodes at the top level in Tachyon. private val tachyonDirs: Array[TachyonFile] = createTachyonDirs() private val subDirs = Array.fill(tachyonDirs.length)(new Array[TachyonFile](subDirsPerTachyonDir)) addShutdownHook() - + def removeFile(file: TachyonFile): Boolean = { client.delete(file.getPath(), false) } - + def fileExists(file: TachyonFile): Boolean = { client.exist(file.getPath()) }