001/* 002* Copyright (c) 2016-2018 Chris K Wensel <chris@wensel.net>. All Rights Reserved. 003* Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved. 004 * 005 * Project and contact information: http://www.cascading.org/ 006 * 007 * This file is part of the Cascading project. 008 * 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 */ 021 022package cascading.flow.stream.element; 023 024import java.io.IOException; 025 026import cascading.CascadingException; 027import cascading.flow.FlowProcess; 028import cascading.flow.SliceCounters; 029import cascading.flow.StepCounters; 030import cascading.flow.stream.duct.Duct; 031import cascading.flow.stream.duct.DuctException; 032import cascading.flow.stream.graph.StreamGraph; 033import cascading.tap.Tap; 034import cascading.tuple.Fields; 035import cascading.tuple.TupleEntry; 036import cascading.tuple.TupleEntryCollector; 037 038/** 039 * 040 */ 041public class SinkStage extends ElementStage<TupleEntry, Void> 042 { 043 private final Tap sink; 044 private TupleEntryCollector collector; 045 046 public SinkStage( FlowProcess flowProcess, Tap sink ) 047 { 048 super( flowProcess, sink ); 049 this.sink = sink; 050 } 051 052 public Tap getSink() 053 { 054 return sink; 055 } 056 057 @Override 058 public void bind( StreamGraph streamGraph ) 059 { 060 // do not bind 061 } 062 063 @Override 064 public void prepare() 065 { 066 try 067 { 068 // todo: pass the resolved fields down 069 collector = sink.openForWrite( flowProcess, getOutput() ); 070 071 if( sink.getSinkFields().isAll() ) 072 { 073 Fields fields = getIncomingScopes().get( 0 ).getIncomingTapFields(); 074 collector.setFields( fields ); 075 } 076 } 077 catch( IOException exception ) 078 { 079 throw new DuctException( "failed opening sink", exception ); 080 } 081 } 082 083 protected Object getOutput() 084 { 085 return null; 086 } 087 088 @Override 089 public void start( Duct previous ) 090 { 091 // do nothing 092 } 093 094 @Override 095 public void receive( Duct previous, int ordinal, TupleEntry tupleEntry ) 096 { 097 try 098 { 099 timedAdd( StepCounters.Write_Duration, tupleEntry ); 100 flowProcess.increment( StepCounters.Tuples_Written, 1 ); 101 flowProcess.increment( SliceCounters.Tuples_Written, 1 ); 102 } 103 catch( OutOfMemoryError error ) 104 { 105 handleReThrowableException( "out of memory, try increasing task memory allocation", error ); 106 } 107 catch( CascadingException exception ) 108 { 109 handleException( exception, tupleEntry ); 110 } 111 catch( Throwable throwable ) 112 { 113 handleException( new DuctException( "internal error: " + tupleEntry.getTuple().print(), throwable ), tupleEntry ); 114 } 115 } 116 117 protected void timedAdd( StepCounters durationCounter, TupleEntry tupleEntry ) 118 { 119 long start = System.currentTimeMillis(); 120 121 try 122 { 123 collector.add( tupleEntry ); 124 } 125 finally 126 { 127 flowProcess.increment( durationCounter, System.currentTimeMillis() - start ); 128 } 129 } 130 131 @Override 132 public void complete( Duct previous ) 133 { 134 // do nothing 135 } 136 137 @Override 138 public void cleanup() 139 { 140 try 141 { 142 if( collector != null ) 143 { 144 long start = System.currentTimeMillis(); 145 146 try 147 { 148 collector.close(); // may flush underlying system 149 } 150 finally 151 { 152 flowProcess.increment( StepCounters.Write_Duration, System.currentTimeMillis() - start ); 153 } 154 } 155 156 collector = null; 157 } 158 finally 159 { 160 super.cleanup(); 161 } 162 } 163 }