001/* 002 * Copyright (c) 2007-2016 Concurrent, Inc. All Rights Reserved. 003 * 004 * Project and contact information: http://www.cascading.org/ 005 * 006 * This file is part of the Cascading project. 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020 021package cascading.flow.hadoop.planner; 022 023import cascading.flow.hadoop.planner.rule.assertion.DualStreamedAccumulatedMergePipelineAssert; 024import cascading.flow.hadoop.planner.rule.partitioner.ConsecutiveTapsNodePartitioner; 025import cascading.flow.hadoop.planner.rule.partitioner.ConsecutiveTapsStepPartitioner; 026import cascading.flow.hadoop.planner.rule.partitioner.GroupTapNodePartitioner; 027import cascading.flow.hadoop.planner.rule.partitioner.MultiTapGroupNodePartitioner; 028import cascading.flow.hadoop.planner.rule.partitioner.StreamedAccumulatedTapsPipelinePartitioner; 029import cascading.flow.hadoop.planner.rule.partitioner.StreamedOnlySourcesPipelinePartitioner; 030import cascading.flow.hadoop.planner.rule.partitioner.StreamedSelfJoinSourcesPipelinePartitioner; 031import cascading.flow.hadoop.planner.rule.partitioner.TapGroupTapStepPartitioner; 032import cascading.flow.hadoop.planner.rule.transformer.RemoveMalformedHashJoinPipelineTransformer; 033import cascading.flow.hadoop.planner.rule.transformer.TapBalanceCheckpointTransformer; 034import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupBlockingHashJoinTransformer; 035import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupGroupTransformer; 036import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupMergeGroupTransformer; 037import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupNonBlockingHashJoinTransformer; 038import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitJoinTransformer; 039import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitMergeGroupTransformer; 040import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitMergeTransformer; 041import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitTransformer; 042import cascading.flow.hadoop.planner.rule.transformer.TapBalanceHashJoinBlockingHashJoinTransformer; 043import cascading.flow.hadoop.planner.rule.transformer.TapBalanceHashJoinSameSourceTransformer; 044import cascading.flow.hadoop.planner.rule.transformer.TapBalanceNonSafePipeSplitTransformer; 045import cascading.flow.hadoop.planner.rule.transformer.TapBalanceNonSafeSplitTransformer; 046import cascading.flow.hadoop.planner.rule.transformer.TapBalanceSameSourceStreamedAccumulatedTransformer; 047import cascading.flow.planner.rule.RuleRegistry; 048import cascading.flow.planner.rule.assertion.BufferAfterEveryAssert; 049import cascading.flow.planner.rule.assertion.EveryAfterBufferAssert; 050import cascading.flow.planner.rule.assertion.LoneGroupAssert; 051import cascading.flow.planner.rule.assertion.MissingGroupAssert; 052import cascading.flow.planner.rule.assertion.SplitBeforeEveryAssert; 053import cascading.flow.planner.rule.transformer.ApplyAssertionLevelTransformer; 054import cascading.flow.planner.rule.transformer.ApplyDebugLevelTransformer; 055import cascading.flow.planner.rule.transformer.RemoveNoOpPipeTransformer; 056 057/** 058 * 059 */ 060public class MapReduceHadoopRuleRegistry extends RuleRegistry 061 { 062 public MapReduceHadoopRuleRegistry() 063 { 064// enableDebugLogging(); 065 066 // PreBalance 067 addRule( new LoneGroupAssert() ); 068 addRule( new MissingGroupAssert() ); 069 addRule( new BufferAfterEveryAssert() ); 070 addRule( new EveryAfterBufferAssert() ); 071 addRule( new SplitBeforeEveryAssert() ); 072 073 // Balance with temporary Taps 074 addRule( new TapBalanceGroupSplitTransformer() ); 075 addRule( new TapBalanceGroupSplitJoinTransformer() ); 076 addRule( new TapBalanceGroupSplitMergeGroupTransformer() ); 077 addRule( new TapBalanceGroupSplitMergeTransformer() ); 078 addRule( new TapBalanceGroupMergeGroupTransformer() ); 079 addRule( new TapBalanceGroupGroupTransformer() ); 080 addRule( new TapBalanceCheckpointTransformer() ); 081 addRule( new TapBalanceHashJoinSameSourceTransformer() ); 082 addRule( new TapBalanceHashJoinBlockingHashJoinTransformer() ); 083 addRule( new TapBalanceGroupBlockingHashJoinTransformer() ); 084 addRule( new TapBalanceGroupNonBlockingHashJoinTransformer() ); 085 addRule( new TapBalanceSameSourceStreamedAccumulatedTransformer() ); 086 addRule( new TapBalanceNonSafeSplitTransformer() ); 087 addRule( new TapBalanceNonSafePipeSplitTransformer() ); 088 089 // PreResolve 090 addRule( new RemoveNoOpPipeTransformer() ); 091 addRule( new ApplyAssertionLevelTransformer() ); 092 addRule( new ApplyDebugLevelTransformer() ); 093 094 // PostResolve 095// addRule( new CombineAdjacentTapTransformer() ); 096 097 // PartitionSteps 098 addRule( new ConsecutiveTapsStepPartitioner() ); 099 addRule( new TapGroupTapStepPartitioner() ); 100 101 // PartitionNodes 102 addRule( new ConsecutiveTapsNodePartitioner() ); 103 addRule( new MultiTapGroupNodePartitioner() ); 104 addRule( new GroupTapNodePartitioner() ); 105 106 // PartitionPipelines 107 addRule( new StreamedAccumulatedTapsPipelinePartitioner() ); 108 addRule( new StreamedSelfJoinSourcesPipelinePartitioner() ); 109 addRule( new StreamedOnlySourcesPipelinePartitioner() ); 110 111 // PostPipelines 112 addRule( new RemoveMalformedHashJoinPipelineTransformer() ); 113 114 // remove when GraphFinder supports captured edges 115 addRule( new DualStreamedAccumulatedMergePipelineAssert() ); 116 117 // enable when GraphFinder supports captured edges 118// addRule( new RemoveStreamedBranchTransformer() ); 119 120 } 121 }