From ef671d4680ffd4d26416040d5c9dd56767453b87 Mon Sep 17 00:00:00 2001 From: Matei Zaharia Date: Tue, 27 May 2014 23:11:49 -0700 Subject: [PATCH] Keep frames in JavaDoc links, and other small tweaks --- docs/js/api-docs.js | 23 ++++++++++++++++--- docs/js/main.js | 21 ++++++++++++++++++ docs/mllib-guide.md | 4 ++-- docs/programming-guide.md | 30 ++++++++++++------------- docs/streaming-programming-guide.md | 34 ++++++++++++++--------------- 5 files changed, 75 insertions(+), 37 deletions(-) diff --git a/docs/js/api-docs.js b/docs/js/api-docs.js index 1414b6d0b81a1..ce89d8943b431 100644 --- a/docs/js/api-docs.js +++ b/docs/js/api-docs.js @@ -1,10 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /* Dynamically injected post-processing code for the API docs */ $(document).ready(function() { var annotations = $("dt:contains('Annotations')").next("dd").children("span.name"); - addBadges(annotations, "AlphaComponent", ":: AlphaComponent ::", "Alpha Component"); - addBadges(annotations, "DeveloperApi", ":: DeveloperApi ::", "Developer API"); - addBadges(annotations, "Experimental", ":: Experimental ::", "Experimental"); + addBadges(annotations, "AlphaComponent", ":: AlphaComponent ::", 'Alpha Component'); + addBadges(annotations, "DeveloperApi", ":: DeveloperApi ::", 'Developer API'); + addBadges(annotations, "Experimental", ":: Experimental ::", 'Experimental'); }); function addBadges(allAnnotations, name, tag, html) { diff --git a/docs/js/main.js b/docs/js/main.js index 59055467110bf..f1a90e47e89a7 100755 --- a/docs/js/main.js +++ b/docs/js/main.js @@ -1,3 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Custom JavaScript code in the MarkDown docs */ + +// Enable language-specific code tabs function codeTabs() { var counter = 0; var langImages = { @@ -62,6 +82,7 @@ function makeCollapsable(elt, accordionClass, accordionBodyId, title) { ); } +// Enable "view solution" sections (for exercises) function viewSolution() { var counter = 0 $("div.solution").each(function() { diff --git a/docs/mllib-guide.md b/docs/mllib-guide.md index 74395a98f4335..95ee6bc96801f 100644 --- a/docs/mllib-guide.md +++ b/docs/mllib-guide.md @@ -84,9 +84,9 @@ val vector: Vector = Vectors.dense(array) // a dense vector
We used to represent a feature vector by `double[]`, which is replaced by -[`Vector`](api/scala/index.html#org.apache.spark.mllib.linalg.Vector) in v1.0. Algorithms that used +[`Vector`](api/java/index.html?org/apache/spark/mllib/linalg/Vector.html) in v1.0. Algorithms that used to accept `RDD` now take -`RDD`. [`LabeledPoint`](api/scala/index.html#org.apache.spark.mllib.regression.LabeledPoint) +`RDD`. [`LabeledPoint`](api/java/index.html?org/apache/spark/mllib/regression/LabeledPoint.html) is now a wrapper of `(double, Vector)` instead of `(double, double[])`. Converting `double[]` to `Vector` is straightforward: diff --git a/docs/programming-guide.md b/docs/programming-guide.md index 49d5efcac2213..7d77e640d0e4b 100644 --- a/docs/programming-guide.md +++ b/docs/programming-guide.md @@ -55,7 +55,7 @@ import org.apache.spark.SparkConf Spark {{site.SPARK_VERSION}} works with Java 6 and higher. If you are using Java 8, Spark supports [lambda expressions](http://docs.oracle.com/javase/tutorial/java/javaOO/lambdaexpressions.html) for concisely writing functions, otherwise you can use the classes in the -[org.apache.spark.api.java.function](api/java/org/apache/spark/api/java/function/package-summary.html) package. +[org.apache.spark.api.java.function](api/java/index.html?org/apache/spark/api/java/function/package-summary.html) package. To write a Spark application in Java, you need to add a dependency on Spark. Spark is available through Maven Central at: @@ -126,8 +126,8 @@ new SparkContext(conf)
-The first thing a Spark program must do is to create a [JavaSparkContext](api/java/org/apache/spark/api/java/JavaSparkContext.html) object, which tells Spark -how to access a cluster. To create a `SparkContext` you first need to build a [SparkConf](api/java/org/apache/spark/SparkConf.html) object +The first thing a Spark program must do is to create a [JavaSparkContext](api/java/index.html?org/apache/spark/api/java/JavaSparkContext.html) object, which tells Spark +how to access a cluster. To create a `SparkContext` you first need to build a [SparkConf](api/java/index.html?org/apache/spark/SparkConf.html) object that contains information about your application. {% highlight java %} @@ -265,7 +265,7 @@ We describe operations on distributed datasets later on. **Note:** *In this guide, we'll often use the concise Java 8 lambda syntax to specify Java functions, but in older versions of Java you can implement the interfaces in the -[org.apache.spark.api.java.function](api/java/org/apache/spark/api/java/function/package-summary.html) package. +[org.apache.spark.api.java.function](api/java/index.html?org/apache/spark/api/java/function/package-summary.html) package. We describe [passing functions to Spark](#passing-functions-to-spark) in more detail below.*
@@ -546,7 +546,7 @@ def doStuff(rdd: RDD[String]): RDD[String] = { Spark's API relies heavily on passing functions in the driver program to run on the cluster. In Java, functions are represented by classes implementing the interfaces in the -[org.apache.spark.api.java.function](api/java/org/apache/spark/api/java/function/package-summary.html) package. +[org.apache.spark.api.java.function](api/java/index.html?org/apache/spark/api/java/function/package-summary.html) package. There are two ways to create such functions: * Implement the Function interfaces in your own class, either as an anonymous inner class or a named one, @@ -697,7 +697,7 @@ from the Scala standard library. You can simply call `new Tuple2(a, b)` to creat its fields later with `tuple._1()` and `tuple._2()`. RDDs of key-value pairs are represented by the -[JavaPairRDD](api/java/org/apache/spark/api/java/JavaPairRDD.html) class. You can construct +[JavaPairRDD](api/java/index.html?org/apache/spark/api/java/JavaPairRDD.html) class. You can construct JavaPairRDDs from JavaRDDs using special versions of the `map` operations, like `mapToPair` and `flatMapToPair`. The JavaPairRDD will have both standard RDD functions and special key-value ones. @@ -749,11 +749,11 @@ We could also use `counts.sortByKey()`, for example, to sort the pairs alphabeti The following table lists some of the common transformations supported by Spark. Refer to the RDD API doc ([Scala](api/scala/index.html#org.apache.spark.rdd.RDD), - [Java](api/java/org/apache/spark/api/java/JavaRDD.html), + [Java](api/java/index.html?org/apache/spark/api/java/JavaRDD.html), [Python](api/python/pyspark.rdd.RDD-class.html)) and pair RDD functions doc ([Scala](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions), - [Java](api/java/org/apache/spark/api/java/JavaPairRDD.html)) + [Java](api/java/index.html?org/apache/spark/api/java/JavaPairRDD.html)) for details. @@ -852,11 +852,11 @@ for details. The following table lists some of the common actions supported by Spark. Refer to the RDD API doc ([Scala](api/scala/index.html#org.apache.spark.rdd.RDD), - [Java](api/java/org/apache/spark/api/java/JavaRDD.html), + [Java](api/java/index.html?org/apache/spark/api/java/JavaRDD.html), [Python](api/python/pyspark.rdd.RDD-class.html)) and pair RDD functions doc ([Scala](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions), - [Java](api/java/org/apache/spark/api/java/JavaPairRDD.html)) + [Java](api/java/index.html?org/apache/spark/api/java/JavaPairRDD.html)) for details.
@@ -931,7 +931,7 @@ to persist the dataset on disk, persist it in memory but as serialized Java obje replicate it across nodes, or store it off-heap in [Tachyon](http://tachyon-project.org/). These levels are set by passing a `StorageLevel` object ([Scala](api/scala/index.html#org.apache.spark.storage.StorageLevel), -[Java](api/java/org/apache/spark/storage/StorageLevel.html), +[Java](api/java/index.html?org/apache/spark/storage/StorageLevel.html), [Python](api/python/pyspark.storagelevel.StorageLevel-class.html)) to `persist()`. The `cache()` method is a shorthand for using the default storage level, which is `StorageLevel.MEMORY_ONLY` (store deserialized objects in memory). The full set of @@ -1150,7 +1150,7 @@ accum.value(); {% endhighlight %} While this code used the built-in support for accumulators of type Integer, programmers can also -create their own types by subclassing [AccumulatorParam](api/java/org/apache/spark/AccumulatorParam.html). +create their own types by subclassing [AccumulatorParam](api/java/index.html?org/apache/spark/AccumulatorParam.html). The AccumulatorParam interface has two methods: `zero` for providing a "zero value" for your data type, and `addInPlace` for adding two values together. For example, supposing we had a `Vector` class representing mathematical vectors, we could write: @@ -1166,10 +1166,10 @@ class VectorAccumulatorParam implements AccumulatorParam { } // Then, create an Accumulator of this type: -Accumulator vecAccum = sc.accumulator(new Vector(...))(new VectorAccumulatorParam()); +Accumulator vecAccum = sc.accumulator(new Vector(...), new VectorAccumulatorParam()); {% endhighlight %} -In Java, Spark also supports the more general [Accumulable](api/java/org/apache/spark/Accumulable.html) +In Java, Spark also supports the more general [Accumulable](api/java/index.html?org/apache/spark/Accumulable.html) interface to accumulate data where the resulting type is not the same as the elements added (e.g. build a list by collecting together elements). @@ -1205,7 +1205,7 @@ class VectorAccumulatorParam(AccumulatorParam): return v1 # Then, create an Accumulator of this type: -vecAccum = sc.accumulator(Vector(...))(VectorAccumulatorParam()) +vecAccum = sc.accumulator(Vector(...), VectorAccumulatorParam()) {% endhighlight %} diff --git a/docs/streaming-programming-guide.md b/docs/streaming-programming-guide.md index 3d02e010b3f3d..b95f818d0fcbb 100644 --- a/docs/streaming-programming-guide.md +++ b/docs/streaming-programming-guide.md @@ -136,7 +136,7 @@ The complete code can be found in the Spark Streaming example
First, we create a -[JavaStreamingContext](api/java/org/apache/spark/streaming/api/java/JavaStreamingContext.html) object, +[JavaStreamingContext](api/java/index.html?org/apache/spark/streaming/api/java/JavaStreamingContext.html) object, which is the main entry point for all streaming functionality. Besides Spark's configuration, we specify that any DStream would be processed in 1 second batches. @@ -215,7 +215,7 @@ jssc.awaitTermination(); // Wait for the computation to terminate {% endhighlight %} The complete code can be found in the Spark Streaming example -[JavaNetworkWordCount]({{site.SPARK_GITHUB_URL}}/blob/master/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java). +[JavaNetworkWordCount]({{site.SPARK_GITHUB_URL}}/blob/master/examples/src/main/java/index.html?org/apache/spark/examples/streaming/JavaNetworkWordCount.java).
@@ -813,8 +813,8 @@ output operators are defined: The complete list of DStream operations is available in the API documentation. For the Scala API, see [DStream](api/scala/index.html#org.apache.spark.streaming.dstream.DStream) and [PairDStreamFunctions](api/scala/index.html#org.apache.spark.streaming.dstream.PairDStreamFunctions). -For the Java API, see [JavaDStream](api/java/org/apache/spark/streaming/api/java/JavaDStream.html) -and [JavaPairDStream](api/java/org/apache/spark/streaming/api/java/JavaPairDStream.html). +For the Java API, see [JavaDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaDStream.html) +and [JavaPairDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaPairDStream.html). ## Persistence Similar to RDDs, DStreams also allow developers to persist the stream's data in memory. That is, @@ -876,7 +876,7 @@ sending the data to two destinations (i.e., the earlier and upgraded application - The existing application is shutdown gracefully (see [`StreamingContext.stop(...)`](api/scala/index.html#org.apache.spark.streaming.StreamingContext) -or [`JavaStreamingContext.stop(...)`](api/java/org/apache/spark/streaming/api/java/JavaStreamingContext.html) +or [`JavaStreamingContext.stop(...)`](api/java/index.html?org/apache/spark/streaming/api/java/JavaStreamingContext.html) for graceful shutdown options) which ensure data that have been received is completely processed before shutdown. Then the upgraded application can be started, which will start processing from the same point where the earlier @@ -1311,10 +1311,10 @@ This section elaborates the steps required to migrate your existing code to 1.0. `FlumeUtils.createStream`, etc.) now returns [InputDStream](api/scala/index.html#org.apache.spark.streaming.dstream.InputDStream) / [ReceiverInputDStream](api/scala/index.html#org.apache.spark.streaming.dstream.ReceiverInputDStream) -(instead of DStream) for Scala, and [JavaInputDStream](api/java/org/apache/spark/streaming/api/java/JavaInputDStream.html) / -[JavaPairInputDStream](api/java/org/apache/spark/streaming/api/java/JavaPairInputDStream.html) / -[JavaReceiverInputDStream](api/java/org/apache/spark/streaming/api/java/JavaReceiverInputDStream.html) / -[JavaPairReceiverInputDStream](api/java/org/apache/spark/streaming/api/java/JavaPairReceiverInputDStream.html) +(instead of DStream) for Scala, and [JavaInputDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaInputDStream.html) / +[JavaPairInputDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaPairInputDStream.html) / +[JavaReceiverInputDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaReceiverInputDStream.html) / +[JavaPairReceiverInputDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaPairReceiverInputDStream.html) (instead of JavaDStream) for Java. This ensures that functionality specific to input streams can be added to these classes in the future without breaking binary compatibility. Note that your existing Spark Streaming applications should not require any change @@ -1365,14 +1365,14 @@ package and renamed for better clarity. [ZeroMQUtils](api/scala/index.html#org.apache.spark.streaming.zeromq.ZeroMQUtils$), and [MQTTUtils](api/scala/index.html#org.apache.spark.streaming.mqtt.MQTTUtils$) - Java docs - * [JavaStreamingContext](api/java/org/apache/spark/streaming/api/java/JavaStreamingContext.html), - [JavaDStream](api/java/org/apache/spark/streaming/api/java/JavaDStream.html) and - [PairJavaDStream](api/java/org/apache/spark/streaming/api/java/PairJavaDStream.html) - * [KafkaUtils](api/java/org/apache/spark/streaming/kafka/KafkaUtils.html), - [FlumeUtils](api/java/org/apache/spark/streaming/flume/FlumeUtils.html), - [TwitterUtils](api/java/org/apache/spark/streaming/twitter/TwitterUtils.html), - [ZeroMQUtils](api/java/org/apache/spark/streaming/zeromq/ZeroMQUtils.html), and - [MQTTUtils](api/java/org/apache/spark/streaming/mqtt/MQTTUtils.html) + * [JavaStreamingContext](api/java/index.html?org/apache/spark/streaming/api/java/JavaStreamingContext.html), + [JavaDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaDStream.html) and + [PairJavaDStream](api/java/index.html?org/apache/spark/streaming/api/java/PairJavaDStream.html) + * [KafkaUtils](api/java/index.html?org/apache/spark/streaming/kafka/KafkaUtils.html), + [FlumeUtils](api/java/index.html?org/apache/spark/streaming/flume/FlumeUtils.html), + [TwitterUtils](api/java/index.html?org/apache/spark/streaming/twitter/TwitterUtils.html), + [ZeroMQUtils](api/java/index.html?org/apache/spark/streaming/zeromq/ZeroMQUtils.html), and + [MQTTUtils](api/java/index.html?org/apache/spark/streaming/mqtt/MQTTUtils.html) * More examples in [Scala]({{site.SPARK_GITHUB_URL}}/tree/master/examples/src/main/scala/org/apache/spark/examples/streaming) and [Java]({{site.SPARK_GITHUB_URL}}/tree/master/examples/src/main/java/org/apache/spark/examples/streaming)