Unfork Optional.java.

Just package the class from the guava dependency instead. Note the
dependency-fu needed in core/pom.xml and assembly/pom.xml to not
expose the guava dependency in the user's compilation classpath.

assembly/pom.xml

@@ -101,12 +101,6 @@
             </includes>
           </artifactSet>
           <filters>
-            <filter>
-              <artifact>com.google.guava:guava</artifact>
-              <excludes>
-                <exclude>com/google/common/base/Optional*</exclude>
-              </excludes>
-            </filter>
             <filter>
               <artifact>*:*</artifact>
               <excludes>

core/pom.xml

@@ -68,9 +68,15 @@
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-server</artifactId>
     </dependency>
+    <!--
+      Promote Guava to "compile" so that maven-shade-plugin picks it up (for packaging the Optional
+      class exposed in the Java API). The plugin will then remove this dependency from the published
+      pom, so that Guava does not pollute the client's compilation classpath.
+    -->
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
+      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
@@ -322,6 +328,34 @@
           </arguments>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <shadedArtifactAttached>false</shadedArtifactAttached>
+              <artifactSet>
+                <includes>
+                  <include>com.google.guava:guava</include>
+                </includes>
+              </artifactSet>
+              <filters>
+                <filter>
+                  <artifact>com.google.guava:guava</artifact>
+                  <includes>
+                    <include>com/google/common/base/Optional*</include>
+                  </includes>
+                </filter>
+              </filters>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
 
     <resources>

project/Relocator.scala

@@ -88,18 +88,9 @@ class RelocatorRemapper(relocators: List[Relocator]) extends Remapper {
  * should be relocated are moved to a new location, and all classes are passed through the
  * remapper so that references to relocated classes are fixed.
  *
- * Note about `preferLocal`: this is a hack to make sure that we always ship the Spark-compiled
- * version of Guava's `Optional` class. Unlike maven, sbt-assembly doesn't seem to allow filtering
- * of specific entries in dependencies. It also does not provide information about where does
- * a particular file come from. The only hint is that the temp path for the file ends with a "_dir"
- * when it comes from a directory, and with a hash when it comes from a jar file. So for classes
- * that match a regex in the `preferLocal` list, we choose the first class file in a local
- * directory.
- *
  * @param relocators List of relocators to apply to classes being shaded.
- * @param preferLocal List of regexes that match classes for which a local version is preferred.
  */
-class ShadeStrategy(relocators: List[Relocator], preferLocal: List[Regex]) extends MergeStrategy {
+class ShadeStrategy(relocators: List[Relocator]) extends MergeStrategy {
 
   private val remapper = new RelocatorRemapper(relocators)
 
@@ -111,7 +102,7 @@ class ShadeStrategy(relocators: List[Relocator], preferLocal: List[Regex]) exten
         (files.head, path)
       } else {
         val className = path.substring(0, path.length() - ".class".length())
-        (remap(chooseFile(path, files), tempDir), remapper.rename(className) + ".class")
+        (remap(files.head, tempDir), remapper.rename(className) + ".class")
       }
     Right(Seq(file -> newPath))
   }
@@ -139,18 +130,4 @@ class ShadeStrategy(relocators: List[Relocator], preferLocal: List[Regex]) exten
     }
   }
 
-  private def chooseFile(path: String, files: Seq[File]): File = {
-    if (!preferLocal.filter { r => r.pattern.matcher(path).matches() }.isEmpty) {
-      def isLocal(f: File) = {
-        val abs = f.getAbsolutePath()
-        val dir = abs.substring(0, abs.length() - path.length())
-        dir.endsWith("_dir")
-      }
-
-      files.filter(isLocal).orElse(files)(0)
-    } else {
-      files.head
-    }
-  }
-
 }