HadoopRDD

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

def ++(other: RDD[(K, V)]): RDD[(K, V)]

Return the union of this RDD and another one.

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

def aggregate[U](zeroValue: U)(seqOp: (U, (K, V)) ⇒ U, combOp: (U, U) ⇒ U)(implicit arg0: ClassTag[U]): U

Aggregate the elements of each partition, and then the results for all the partitions, using given combine functions and a neutral "zero value".

final def asInstanceOf[T0]: T0

def cache(): HadoopRDD.this.type

Persist this RDD with the default storage level (MEMORY_ONLY).

def cartesian[U](other: RDD[U])(implicit arg0: ClassTag[U]): RDD[((K, V), U)]

Return the Cartesian product of this RDD and another one, that is, the RDD of all pairs of elements (a, b) where a is in this and b is in other.

def checkpoint(): Unit

Mark this RDD for checkpointing.

def clearDependencies(): Unit

Clears the dependencies of this RDD.

def clone(): AnyRef

def coalesce(numPartitions: Int, shuffle: Boolean = false)(implicit ord: Ordering[(K, V)] = null): RDD[(K, V)]

Return a new RDD that is reduced into numPartitions partitions.

def collect[U](f: PartialFunction[(K, V), U])(implicit arg0: ClassTag[U]): RDD[U]

Return an RDD that contains all matching values by applying f.

def collect(): Array[(K, V)]

Return an array that contains all of the elements in this RDD.

def compute(theSplit: Partition, context: TaskContext): InterruptibleIterator[(K, V)]

:: DeveloperApi :: Implemented by subclasses to compute a given partition.

def context: SparkContext

The org.apache.spark.SparkContext that this RDD was created on.

def count(): Long

Return the number of elements in the RDD.

def countApprox(timeout: Long, confidence: Double = 0.95): PartialResult[BoundedDouble]

:: Experimental :: Approximate version of count() that returns a potentially incomplete result within a timeout, even if not all tasks have finished.

def countApproxDistinct(relativeSD: Double = 0.05): Long

:: Experimental :: Return approximate number of distinct elements in the RDD.

def countByValue()(implicit ord: Ordering[(K, V)] = null): Map[(K, V), Long]

Return the count of each unique value in this RDD as a map of (value, count) pairs.

def countByValueApprox(timeout: Long, confidence: Double = 0.95)(implicit ord: Ordering[(K, V)] = null): PartialResult[Map[(K, V), BoundedDouble]]

:: Experimental :: Approximate version of countByValue().

final def dependencies: Seq[Dependency[_]]

Get the list of dependencies of this RDD, taking into account whether the RDD is checkpointed or not.

def distinct(): RDD[(K, V)]

Return a new RDD containing the distinct elements in this RDD.

def distinct(numPartitions: Int)(implicit ord: Ordering[(K, V)] = null): RDD[(K, V)]

Return a new RDD containing the distinct elements in this RDD.

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def filter(f: ((K, V)) ⇒ Boolean): RDD[(K, V)]

Return a new RDD containing only the elements that satisfy a predicate.

def finalize(): Unit

def first(): (K, V)

Return the first element in this RDD.

def firstParent[U](implicit arg0: ClassTag[U]): RDD[U]

Returns the first parent RDD

def flatMap[U](f: ((K, V)) ⇒ TraversableOnce[U])(implicit arg0: ClassTag[U]): RDD[U]

Return a new RDD by first applying a function to all elements of this RDD, and then flattening the results.

def fold(zeroValue: (K, V))(op: ((K, V), (K, V)) ⇒ (K, V)): (K, V)

Aggregate the elements of each partition, and then the results for all the partitions, using a given associative function and a neutral "zero value".

def foreach(f: ((K, V)) ⇒ Unit): Unit

Applies a function f to all elements of this RDD.

def foreachPartition(f: (Iterator[(K, V)]) ⇒ Unit): Unit

Applies a function f to each partition of this RDD.

def getCheckpointFile: Option[String]

Gets the name of the file to which this RDD was checkpointed

final def getClass(): Class[_]

def getConf: Configuration

def getDependencies: Seq[Dependency[_]]

Implemented by subclasses to return how this RDD depends on parent RDDs.

def getInputFormat(conf: JobConf): InputFormat[K, V]

def getJobConf(): JobConf

def getPartitions: Array[Partition]

Implemented by subclasses to return the set of partitions in this RDD.

def getPreferredLocations(split: Partition): Seq[String]

Optionally overridden by subclasses to specify placement preferences.

def getStorageLevel: StorageLevel

Get the RDD's current storage level, or StorageLevel.

def glom(): RDD[Array[(K, V)]]

Return an RDD created by coalescing all elements within each partition into an array.

def groupBy[K](f: ((K, V)) ⇒ K, p: Partitioner)(implicit kt: ClassTag[K], ord: Ordering[K] = null): RDD[(K, Iterable[(K, V)])]

Return an RDD of grouped items.

def groupBy[K](f: ((K, V)) ⇒ K, numPartitions: Int)(implicit kt: ClassTag[K]): RDD[(K, Iterable[(K, V)])]

Return an RDD of grouped elements.

def groupBy[K](f: ((K, V)) ⇒ K)(implicit kt: ClassTag[K]): RDD[(K, Iterable[(K, V)])]

Return an RDD of grouped items.

def hashCode(): Int

val id: Int

A unique ID for this RDD (within its SparkContext).

val inputFormatCacheKey: String

def intersection(other: RDD[(K, V)], numPartitions: Int): RDD[(K, V)]

Return the intersection of this RDD and another one.

def intersection(other: RDD[(K, V)], partitioner: Partitioner)(implicit ord: Ordering[(K, V)] = null): RDD[(K, V)]

Return the intersection of this RDD and another one.

def intersection(other: RDD[(K, V)]): RDD[(K, V)]

Return the intersection of this RDD and another one.

def isCheckpointed: Boolean

Return whether this RDD has been checkpointed or not

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

final def iterator(split: Partition, context: TaskContext): Iterator[(K, V)]

Internal method to this RDD; will read from cache if applicable, or otherwise compute it.

val jobConfCacheKey: String

def keyBy[K](f: ((K, V)) ⇒ K): RDD[(K, (K, V))]

Creates tuples of the elements in this RDD by applying f.

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

def map[U](f: ((K, V)) ⇒ U)(implicit arg0: ClassTag[U]): RDD[U]

Return a new RDD by applying a function to all elements of this RDD.

def mapPartitions[U](f: (Iterator[(K, V)]) ⇒ Iterator[U], preservesPartitioning: Boolean = false)(implicit arg0: ClassTag[U]): RDD[U]

Return a new RDD by applying a function to each partition of this RDD.

def mapPartitionsWithContext[U](f: (TaskContext, Iterator[(K, V)]) ⇒ Iterator[U], preservesPartitioning: Boolean = false)(implicit arg0: ClassTag[U]): RDD[U]

:: DeveloperApi :: Return a new RDD by applying a function to each partition of this RDD.

def mapPartitionsWithIndex[U](f: (Int, Iterator[(K, V)]) ⇒ Iterator[U], preservesPartitioning: Boolean = false)(implicit arg0: ClassTag[U]): RDD[U]

Return a new RDD by applying a function to each partition of this RDD, while tracking the index of the original partition.

def max()(implicit ord: Ordering[(K, V)]): (K, V)

Returns the max of this RDD as defined by the implicit Ordering[T].

def min()(implicit ord: Ordering[(K, V)]): (K, V)

Returns the min of this RDD as defined by the implicit Ordering[T].

var name: String

A friendly name for this RDD

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val partitioner: Option[Partitioner]

Optionally overridden by subclasses to specify how they are partitioned.

final def partitions: Array[Partition]

Get the array of partitions of this RDD, taking into account whether the RDD is checkpointed or not.

def persist(): HadoopRDD.this.type

Persist this RDD with the default storage level (MEMORY_ONLY).

def persist(newLevel: StorageLevel): HadoopRDD.this.type

Set this RDD's storage level to persist its values across operations after the first time it is computed.

def pipe(command: Seq[String], env: Map[String, String] = Map(), printPipeContext: ((String) ⇒ Unit) ⇒ Unit = null, printRDDElement: ((K, V), (String) ⇒ Unit) ⇒ Unit = null, separateWorkingDir: Boolean = false): RDD[String]

Return an RDD created by piping elements to a forked external process.

def pipe(command: String, env: Map[String, String]): RDD[String]

Return an RDD created by piping elements to a forked external process.

def pipe(command: String): RDD[String]

Return an RDD created by piping elements to a forked external process.

final def preferredLocations(split: Partition): Seq[String]

Get the preferred locations of a partition (as hostnames), taking into account whether the RDD is checkpointed.

def randomSplit(weights: Array[Double], seed: Long = Utils.random.nextLong): Array[RDD[(K, V)]]

Randomly splits this RDD with the provided weights.

def reduce(f: ((K, V), (K, V)) ⇒ (K, V)): (K, V)

Reduces the elements of this RDD using the specified commutative and associative binary operator.

def repartition(numPartitions: Int)(implicit ord: Ordering[(K, V)] = null): RDD[(K, V)]

Return a new RDD that has exactly numPartitions partitions.

def sample(withReplacement: Boolean, fraction: Double, seed: Long = Utils.random.nextLong): RDD[(K, V)]

Return a sampled subset of this RDD.

def saveAsObjectFile(path: String): Unit

Save this RDD as a SequenceFile of serialized objects.

def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]): Unit

Save this RDD as a compressed text file, using string representations of elements.

def saveAsTextFile(path: String): Unit

Save this RDD as a text file, using string representations of elements.

def setName(_name: String): HadoopRDD.this.type

Assign a name to this RDD

def sparkContext: SparkContext

The SparkContext that created this RDD.

def subtract(other: RDD[(K, V)], p: Partitioner)(implicit ord: Ordering[(K, V)] = null): RDD[(K, V)]

Return an RDD with the elements from this that are not in other.

def subtract(other: RDD[(K, V)], numPartitions: Int): RDD[(K, V)]

Return an RDD with the elements from this that are not in other.

def subtract(other: RDD[(K, V)]): RDD[(K, V)]

Return an RDD with the elements from this that are not in other.

final def synchronized[T0](arg0: ⇒ T0): T0

def take(num: Int): Array[(K, V)]

Take the first num elements of the RDD.

def takeOrdered(num: Int)(implicit ord: Ordering[(K, V)]): Array[(K, V)]

Returns the first K (smallest) elements from this RDD as defined by the specified implicit Ordering[T] and maintains the ordering.

sc.parallelize([10, 4, 2, 12, 3]).takeOrdered(1)
// returns [12]

sc.parallelize([2, 3, 4, 5, 6]).takeOrdered(2)
// returns [2, 3]

def takeSample(withReplacement: Boolean, num: Int, seed: Long = Utils.random.nextLong): Array[(K, V)]

def toDebugString: String

A description of this RDD and its recursive dependencies for debugging.

def toJavaRDD(): JavaRDD[(K, V)]

def toLocalIterator: Iterator[(K, V)]

Return an iterator that contains all of the elements in this RDD.

def toString(): String

def top(num: Int)(implicit ord: Ordering[(K, V)]): Array[(K, V)]

Returns the top K (largest) elements from this RDD as defined by the specified implicit Ordering[T].

sc.parallelize([10, 4, 2, 12, 3]).top(1)
// returns [12]

sc.parallelize([2, 3, 4, 5, 6]).top(2)
// returns [6, 5]

def union(other: RDD[(K, V)]): RDD[(K, V)]

Return the union of this RDD and another one.

def unpersist(blocking: Boolean = true): HadoopRDD.this.type

Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def zip[U](other: RDD[U])(implicit arg0: ClassTag[U]): RDD[((K, V), U)]

Zips this RDD with another one, returning key-value pairs with the first element in each RDD, second element in each RDD, etc.

def zipPartitions[B, C, D, V](rdd2: RDD[B], rdd3: RDD[C], rdd4: RDD[D])(f: (Iterator[(K, V)], Iterator[B], Iterator[C], Iterator[D]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[D], arg3: ClassTag[V]): RDD[V]

def zipPartitions[B, C, D, V](rdd2: RDD[B], rdd3: RDD[C], rdd4: RDD[D], preservesPartitioning: Boolean)(f: (Iterator[(K, V)], Iterator[B], Iterator[C], Iterator[D]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[D], arg3: ClassTag[V]): RDD[V]

def zipPartitions[B, C, V](rdd2: RDD[B], rdd3: RDD[C])(f: (Iterator[(K, V)], Iterator[B], Iterator[C]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[V]): RDD[V]

def zipPartitions[B, C, V](rdd2: RDD[B], rdd3: RDD[C], preservesPartitioning: Boolean)(f: (Iterator[(K, V)], Iterator[B], Iterator[C]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[V]): RDD[V]

def zipPartitions[B, V](rdd2: RDD[B])(f: (Iterator[(K, V)], Iterator[B]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[V]): RDD[V]

def zipPartitions[B, V](rdd2: RDD[B], preservesPartitioning: Boolean)(f: (Iterator[(K, V)], Iterator[B]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[V]): RDD[V]

Zip this RDD's partitions with one (or more) RDD(s) and return a new RDD by applying a function to the zipped partitions.

def zipWithIndex(): RDD[((K, V), Long)]

Zips this RDD with its element indices.

def zipWithUniqueId(): RDD[((K, V), Long)]

Zips this RDD with generated unique Long ids.

def filterWith[A](constructA: (Int) ⇒ A)(p: ((K, V), A) ⇒ Boolean): RDD[(K, V)]

Filters this RDD with p, where p takes an additional parameter of type A.

def flatMapWith[A, U](constructA: (Int) ⇒ A, preservesPartitioning: Boolean = false)(f: ((K, V), A) ⇒ Seq[U])(implicit arg0: ClassTag[U]): RDD[U]

FlatMaps f over this RDD, where f takes an additional parameter of type A.

def foreachWith[A](constructA: (Int) ⇒ A)(f: ((K, V), A) ⇒ Unit): Unit

Applies f to each element of this RDD, where f takes an additional parameter of type A.

def mapPartitionsWithSplit[U](f: (Int, Iterator[(K, V)]) ⇒ Iterator[U], preservesPartitioning: Boolean = false)(implicit arg0: ClassTag[U]): RDD[U]

Return a new RDD by applying a function to each partition of this RDD, while tracking the index of the original partition.

def mapWith[A, U](constructA: (Int) ⇒ A, preservesPartitioning: Boolean = false)(f: ((K, V), A) ⇒ U)(implicit arg0: ClassTag[U]): RDD[U]

Maps f over this RDD, where f takes an additional parameter of type A.

def toArray(): Array[(K, V)]

Return an array that contains all of the elements in this RDD.