cascading.platform.hadoop
Class BaseHadoopPlatform
java.lang.Object
cascading.platform.TestPlatform
cascading.platform.hadoop.BaseHadoopPlatform
- Direct Known Subclasses:
- Hadoop2MR1Platform
public abstract class BaseHadoopPlatform
- extends cascading.platform.TestPlatform
Fields inherited from class cascading.platform.TestPlatform |
CLUSTER_TESTING_PROPERTY |
Method Summary |
void |
copyFromLocal(String inputFile)
|
void |
copyToLocal(String outputFile)
|
Tap |
getDelimitedFile(Fields fields,
boolean skipHeader,
boolean writeHeader,
String delimiter,
String quote,
Class[] types,
String filename,
SinkMode mode)
|
Tap |
getDelimitedFile(Fields fields,
boolean hasHeader,
String delimiter,
String quote,
Class[] types,
String filename,
SinkMode mode)
|
Tap |
getDelimitedFile(String delimiter,
String quote,
FieldTypeResolver fieldTypeResolver,
String filename,
SinkMode mode)
|
FlowProcess |
getFlowProcess()
|
String |
getHiddenTemporaryPath()
|
JobConf |
getJobConf()
|
Comparator |
getLongComparator(boolean reverseSort)
|
Tap |
getPartitionTap(Tap sink,
Partition partition,
int openThreshold)
|
Map<Object,Object> |
getProperties()
|
Comparator |
getStringComparator(boolean reverseSort)
|
Tap |
getTap(Scheme scheme,
String filename,
SinkMode mode)
|
Tap |
getTemplateTap(Tap sink,
String pathTemplate,
Fields fields,
int openThreshold)
|
Tap |
getTemplateTap(Tap sink,
String pathTemplate,
int openThreshold)
|
Scheme |
getTestConfigDefScheme()
|
Scheme |
getTestFailScheme()
|
Tap |
getTextFile(Fields sourceFields,
Fields sinkFields,
String filename,
SinkMode mode)
|
boolean |
isHDFSAvailable()
|
boolean |
isMapReduce()
|
boolean |
remoteExists(String outputFile)
|
boolean |
remoteRemove(String outputFile,
boolean recursive)
|
protected String |
safeFileName(String filename)
Replaces characters, that are not allowed by HDFS with an "_". |
void |
setNumMapTasks(int numMapTasks)
|
void |
setNumReduceTasks(int numReduceTasks)
|
void |
tearDown()
|
Methods inherited from class cascading.platform.TestPlatform |
getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getFlowConnector, getFlowConnector, getGlobalProperties, getName, getNumMappers, getNumMapTasks, getNumReducers, getNumReduceTasks, getTabDelimitedFile, getTabDelimitedFile, getTextFile, getTextFile, getTextFile, getTextFile, isUseCluster, setNumMappers, setNumMapTasks, setNumReducers, setNumReduceTasks, setUp, setUseCluster |
Methods inherited from class java.lang.Object |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
fileSys
public static transient FileSystem fileSys
jobConf
public static transient Configuration jobConf
properties
public static transient Map<Object,Object> properties
numMapTasks
public int numMapTasks
numReduceTasks
public int numReduceTasks
logger
protected String logger
BaseHadoopPlatform
public BaseHadoopPlatform()
isMapReduce
public boolean isMapReduce()
- Overrides:
isMapReduce
in class cascading.platform.TestPlatform
setNumMapTasks
public void setNumMapTasks(int numMapTasks)
setNumReduceTasks
public void setNumReduceTasks(int numReduceTasks)
getProperties
public Map<Object,Object> getProperties()
- Specified by:
getProperties
in class cascading.platform.TestPlatform
tearDown
public void tearDown()
- Specified by:
tearDown
in class cascading.platform.TestPlatform
getJobConf
public JobConf getJobConf()
isHDFSAvailable
public boolean isHDFSAvailable()
getFlowProcess
public FlowProcess getFlowProcess()
- Specified by:
getFlowProcess
in class cascading.platform.TestPlatform
copyFromLocal
public void copyFromLocal(String inputFile)
throws IOException
- Specified by:
copyFromLocal
in class cascading.platform.TestPlatform
- Throws:
IOException
copyToLocal
public void copyToLocal(String outputFile)
throws IOException
- Specified by:
copyToLocal
in class cascading.platform.TestPlatform
- Throws:
IOException
remoteExists
public boolean remoteExists(String outputFile)
throws IOException
- Specified by:
remoteExists
in class cascading.platform.TestPlatform
- Throws:
IOException
remoteRemove
public boolean remoteRemove(String outputFile,
boolean recursive)
throws IOException
- Specified by:
remoteRemove
in class cascading.platform.TestPlatform
- Throws:
IOException
getTap
public Tap getTap(Scheme scheme,
String filename,
SinkMode mode)
- Specified by:
getTap
in class cascading.platform.TestPlatform
getTextFile
public Tap getTextFile(Fields sourceFields,
Fields sinkFields,
String filename,
SinkMode mode)
- Specified by:
getTextFile
in class cascading.platform.TestPlatform
getDelimitedFile
public Tap getDelimitedFile(Fields fields,
boolean hasHeader,
String delimiter,
String quote,
Class[] types,
String filename,
SinkMode mode)
- Specified by:
getDelimitedFile
in class cascading.platform.TestPlatform
getDelimitedFile
public Tap getDelimitedFile(Fields fields,
boolean skipHeader,
boolean writeHeader,
String delimiter,
String quote,
Class[] types,
String filename,
SinkMode mode)
- Specified by:
getDelimitedFile
in class cascading.platform.TestPlatform
getDelimitedFile
public Tap getDelimitedFile(String delimiter,
String quote,
FieldTypeResolver fieldTypeResolver,
String filename,
SinkMode mode)
- Specified by:
getDelimitedFile
in class cascading.platform.TestPlatform
getTemplateTap
public Tap getTemplateTap(Tap sink,
String pathTemplate,
int openThreshold)
- Specified by:
getTemplateTap
in class cascading.platform.TestPlatform
getTemplateTap
public Tap getTemplateTap(Tap sink,
String pathTemplate,
Fields fields,
int openThreshold)
- Specified by:
getTemplateTap
in class cascading.platform.TestPlatform
getPartitionTap
public Tap getPartitionTap(Tap sink,
Partition partition,
int openThreshold)
- Specified by:
getPartitionTap
in class cascading.platform.TestPlatform
getTestConfigDefScheme
public Scheme getTestConfigDefScheme()
- Specified by:
getTestConfigDefScheme
in class cascading.platform.TestPlatform
getTestFailScheme
public Scheme getTestFailScheme()
- Specified by:
getTestFailScheme
in class cascading.platform.TestPlatform
getLongComparator
public Comparator getLongComparator(boolean reverseSort)
- Specified by:
getLongComparator
in class cascading.platform.TestPlatform
getStringComparator
public Comparator getStringComparator(boolean reverseSort)
- Specified by:
getStringComparator
in class cascading.platform.TestPlatform
getHiddenTemporaryPath
public String getHiddenTemporaryPath()
- Specified by:
getHiddenTemporaryPath
in class cascading.platform.TestPlatform
safeFileName
protected String safeFileName(String filename)
- Replaces characters, that are not allowed by HDFS with an "_".
- Parameters:
filename
- The filename to make safe
- Returns:
- The filename with all non-supported characters removed.
Copyright © 2007-2014 Concurrent, Inc. All Rights Reserved.