001/* 002 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved. 003 * 004 * Project and contact information: http://www.cascading.org/ 005 * 006 * This file is part of the Cascading project. 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020 021package cascading.flow.hadoop; 022 023import java.beans.ConstructorProperties; 024import java.util.Map; 025import java.util.Properties; 026 027import cascading.flow.FlowConnector; 028import cascading.flow.hadoop.planner.HadoopPlanner; 029import cascading.flow.hadoop.planner.MapReduceHadoopRuleRegistry; 030import cascading.flow.planner.FlowPlanner; 031import cascading.flow.planner.rule.RuleRegistrySet; 032import cascading.pipe.Pipe; 033import cascading.scheme.Scheme; 034import cascading.scheme.hadoop.SequenceFile; 035import cascading.tap.Tap; 036 037/** 038 * Use the HadoopFlowConnector to link source and sink {@link Tap} instances with an assembly of {@link Pipe} instances into 039 * an executable {@link HadoopFlow} for execution on an Apache Hadoop cluster. 040 * 041 * @see cascading.property.AppProps 042 * @see cascading.flow.FlowConnectorProps 043 * @see cascading.flow.FlowDef 044 * @see cascading.flow.hadoop.MapReduceFlow 045 */ 046public class HadoopFlowConnector extends FlowConnector 047 { 048 /** 049 * Constructor HadoopFlowConnector creates a new HadoopFlowConnector instance. 050 * <p/> 051 * All properties passed to Hadoop are retrieved from a default instantiation of the Hadoop 052 * {@link org.apache.hadoop.mapred.JobConf} which pulls all properties from the local CLASSPATH. 053 */ 054 public HadoopFlowConnector() 055 { 056 } 057 058 /** 059 * Constructor HadoopFlowConnector creates a new HadoopFlowConnector instance using the given {@link Properties} instance as 060 * default value for the underlying jobs. All properties are copied to a new native configuration instance. 061 * 062 * @param properties of type Map 063 */ 064 @ConstructorProperties({"properties"}) 065 public HadoopFlowConnector( Map<Object, Object> properties ) 066 { 067 super( properties ); 068 } 069 070 /** 071 * Constructor HadoopFlowConnector creates a new HadoopFlowConnector instance. 072 * <p/> 073 * All properties passed to Hadoop are retrieved from a default instantiation of the Hadoop 074 * {@link org.apache.hadoop.mapred.JobConf} which pulls all properties from the local CLASSPATH. 075 * 076 * @param ruleRegistrySet of type RuleRegistry 077 */ 078 @ConstructorProperties({"ruleRegistrySet"}) 079 public HadoopFlowConnector( RuleRegistrySet ruleRegistrySet ) 080 { 081 super( ruleRegistrySet ); 082 } 083 084 /** 085 * Constructor HadoopFlowConnector creates a new HadoopFlowConnector instance using the given {@link Properties} instance as 086 * default value for the underlying jobs. All properties are copied to a new native configuration instance. 087 * 088 * @param properties of type Map 089 * @param ruleRegistrySet of type RuleRegistry 090 */ 091 @ConstructorProperties({"properties", "ruleRegistrySet"}) 092 public HadoopFlowConnector( Map<Object, Object> properties, RuleRegistrySet ruleRegistrySet ) 093 { 094 super( properties, ruleRegistrySet ); 095 } 096 097 @Override 098 protected Class<? extends Scheme> getDefaultIntermediateSchemeClass() 099 { 100 return SequenceFile.class; 101 } 102 103 @Override 104 protected FlowPlanner createFlowPlanner() 105 { 106 return new HadoopPlanner(); 107 } 108 109 @Override 110 protected RuleRegistrySet createDefaultRuleRegistrySet() 111 { 112 return new RuleRegistrySet( new MapReduceHadoopRuleRegistry() ); 113 } 114 }