public class HadoopUtil extends Object
Modifier and Type | Field and Description |
---|---|
static String |
CASCADING_FLOW_EXECUTING |
Constructor and Description |
---|
HadoopUtil() |
Modifier and Type | Method and Description |
---|---|
static void |
addComparators(Configuration conf,
String property,
Map<String,Fields> map,
BaseFlowStep flowStep,
Group group) |
static void |
addComparators(Configuration conf,
String property,
Map<String,Fields> map,
Fields resolvedFields) |
static void |
addFields(Configuration conf,
String property,
Map<Integer,Fields> fields) |
static void |
addInputPath(Configuration conf,
Path path) |
static JobConf |
asJobConfInstance(Configuration configuration) |
protected static <C extends Configuration> |
callCopyConstructor(Class type,
Configuration parent) |
static <C> C |
copyConfiguration(C parent) |
static <C extends Configuration> |
copyConfiguration(Map<Object,Object> srcProperties,
C dstConfiguration) |
static <C> C |
copyJobConf(C parentJobConf) |
static JobConf |
copyJobConf(JobConf parentJobConf) |
static JobConf |
createJobConf(Map<Object,Object> properties,
JobConf defaultJobconf) |
static PlatformInfo |
createPlatformInfo(Class type,
String attributePath,
String platformName) |
static Map<Object,Object> |
createProperties(Configuration jobConf) |
static byte[] |
decodeBytes(String string) |
static <T> T |
deserializeBase64(String string,
Configuration conf,
Class<T> type)
This method deserializes the Base64 encoded String into an Object instance.
|
static <T> T |
deserializeBase64(String string,
Configuration conf,
Class<T> type,
boolean decompress) |
static String |
encodeBytes(byte[] bytes) |
static Class |
findMainClass(Class defaultType) |
static Map<Path,Path> |
getCommonPaths(Map<String,Path> localPaths,
Map<String,Path> remotePaths) |
static Map<String,String> |
getConfig(Configuration defaultConf,
Configuration updatedConf) |
static FileSystem |
getDefaultFS(Configuration config) |
static Map<Integer,Fields> |
getFields(Configuration conf,
String property) |
static Thread |
getHDFSShutdownHook() |
static JobConf[] |
getJobConfs(Configuration job,
List<Map<String,String>> configs) |
static LocalFileSystem |
getLocalFS(Configuration config) |
static Path |
getOutputPath(Configuration conf) |
static PlatformInfo |
getPlatformInfo() |
static PlatformInfo |
getPlatformInfo(Class type,
String attributePath,
String platformName) |
static void |
initLog4j(Configuration configuration) |
static void |
initLog4j(JobConf configuration) |
static <T> ObjectSerializer |
instantiateSerializer(Configuration conf,
Class<T> type) |
static boolean |
isInflow(Configuration conf) |
static boolean |
isLocal(Configuration conf) |
static boolean |
isYARN(Configuration conf) |
static <J extends Configuration> |
mergeConf(J job,
Map<String,String> config,
boolean directly) |
static String |
pack(Object object,
Configuration conf) |
static Configuration |
removePropertiesFrom(Configuration jobConf,
String... keys) |
static boolean |
removeStateFromDistCache(Configuration conf,
String path) |
static void |
resolvePaths(Configuration config,
Collection<String> classpath,
String remoteRoot,
String resourceSubPath,
Map<String,Path> localPaths,
Map<String,Path> remotePaths) |
static <T> String |
serializeBase64(T object,
Configuration conf) |
static <T> String |
serializeBase64(T object,
Configuration conf,
boolean compress) |
static void |
setIsInflow(Configuration conf) |
static void |
setLocal(Configuration conf) |
static boolean |
setNewApi(Configuration conf,
String className) |
static void |
setOutputPath(Configuration conf,
Path path) |
static Map<String,Long> |
syncPaths(Configuration config,
Map<Path,Path> commonPaths,
boolean syncTimes)
Copies paths from one local path to a remote path.
|
public static final String CASCADING_FLOW_EXECUTING
public HadoopUtil()
public static void setIsInflow(Configuration conf)
public static boolean isInflow(Configuration conf)
public static void initLog4j(Configuration configuration)
public static JobConf asJobConfInstance(Configuration configuration)
public static <C> C copyJobConf(C parentJobConf)
public static JobConf copyJobConf(JobConf parentJobConf)
public static JobConf createJobConf(Map<Object,Object> properties, JobConf defaultJobconf)
public static <C> C copyConfiguration(C parent)
protected static <C extends Configuration> C callCopyConstructor(Class type, Configuration parent)
public static <C extends Configuration> C copyConfiguration(Map<Object,Object> srcProperties, C dstConfiguration)
public static Map<Object,Object> createProperties(Configuration jobConf)
public static Thread getHDFSShutdownHook()
public static String encodeBytes(byte[] bytes)
public static byte[] decodeBytes(String string)
public static <T> ObjectSerializer instantiateSerializer(Configuration conf, Class<T> type) throws ClassNotFoundException
ClassNotFoundException
public static <T> String serializeBase64(T object, Configuration conf) throws IOException
IOException
public static <T> String serializeBase64(T object, Configuration conf, boolean compress) throws IOException
IOException
public static <T> T deserializeBase64(String string, Configuration conf, Class<T> type) throws IOException
string
- IOException
public static <T> T deserializeBase64(String string, Configuration conf, Class<T> type, boolean decompress) throws IOException
IOException
public static Class findMainClass(Class defaultType)
public static Map<String,String> getConfig(Configuration defaultConf, Configuration updatedConf)
public static JobConf[] getJobConfs(Configuration job, List<Map<String,String>> configs)
public static <J extends Configuration> J mergeConf(J job, Map<String,String> config, boolean directly)
public static Configuration removePropertiesFrom(Configuration jobConf, String... keys)
public static boolean removeStateFromDistCache(Configuration conf, String path) throws IOException
IOException
public static PlatformInfo getPlatformInfo()
public static PlatformInfo getPlatformInfo(Class type, String attributePath, String platformName)
public static PlatformInfo createPlatformInfo(Class type, String attributePath, String platformName)
public static Map<String,Long> syncPaths(Configuration config, Map<Path,Path> commonPaths, boolean syncTimes)
config
- commonPaths
- syncTimes
- public static Map<Path,Path> getCommonPaths(Map<String,Path> localPaths, Map<String,Path> remotePaths)
public static void resolvePaths(Configuration config, Collection<String> classpath, String remoteRoot, String resourceSubPath, Map<String,Path> localPaths, Map<String,Path> remotePaths)
public static LocalFileSystem getLocalFS(Configuration config)
public static FileSystem getDefaultFS(Configuration config)
public static boolean isLocal(Configuration conf)
public static boolean isYARN(Configuration conf)
public static void setLocal(Configuration conf)
public static boolean setNewApi(Configuration conf, String className)
public static void addInputPath(Configuration conf, Path path)
public static void setOutputPath(Configuration conf, Path path)
public static Path getOutputPath(Configuration conf)
public static String pack(Object object, Configuration conf)
public static void addFields(Configuration conf, String property, Map<Integer,Fields> fields)
public static Map<Integer,Fields> getFields(Configuration conf, String property) throws IOException
IOException
public static void addComparators(Configuration conf, String property, Map<String,Fields> map, BaseFlowStep flowStep, Group group)
public static void addComparators(Configuration conf, String property, Map<String,Fields> map, Fields resolvedFields)
Copyright © 2007-2015 Concurrent, Inc. All Rights Reserved.