org.apache.spark.streaming.util

Class HdfsUtils

java.lang.Object

org.apache.spark.streaming.util.HdfsUtils

public class HdfsUtils
extends java.lang.Object

Constructor Summary

Constructors

Constructor and Description
HdfsUtils()

Method Summary

Methods

Modifier and Type	Method and Description
static boolean	checkFileExists(java.lang.String path, org.apache.hadoop.conf.Configuration conf) Check if the file exists at the given path.
static void	checkState(boolean state, scala.Function0<java.lang.String> errorMsg)
static java.lang.String[]	getFileSegmentLocations(java.lang.String path, long offset, long length, org.apache.hadoop.conf.Configuration conf) Get the locations of the HDFS blocks containing the given file segment.
static org.apache.hadoop.fs.FileSystem	getFileSystemForPath(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf)
static org.apache.hadoop.fs.FSDataInputStream	getInputStream(java.lang.String path, org.apache.hadoop.conf.Configuration conf)
static org.apache.hadoop.fs.FSDataOutputStream	getOutputStream(java.lang.String path, org.apache.hadoop.conf.Configuration conf)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

HdfsUtils

public HdfsUtils()

Method Detail

getOutputStream

public static org.apache.hadoop.fs.FSDataOutputStream getOutputStream(java.lang.String path,
                                                      org.apache.hadoop.conf.Configuration conf)

getInputStream

public static org.apache.hadoop.fs.FSDataInputStream getInputStream(java.lang.String path,
                                                    org.apache.hadoop.conf.Configuration conf)

checkState

public static void checkState(boolean state,
              scala.Function0<java.lang.String> errorMsg)

getFileSegmentLocations

public static java.lang.String[] getFileSegmentLocations(java.lang.String path,
                                         long offset,
                                         long length,
                                         org.apache.hadoop.conf.Configuration conf)

Get the locations of the HDFS blocks containing the given file segment.

getFileSystemForPath

public static org.apache.hadoop.fs.FileSystem getFileSystemForPath(org.apache.hadoop.fs.Path path,
                                                   org.apache.hadoop.conf.Configuration conf)

checkFileExists

public static boolean checkFileExists(java.lang.String path,
                      org.apache.hadoop.conf.Configuration conf)

Check if the file exists at the given path.