001/*
002* Copyright (c) 2016-2018 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
003* Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
004 *
005 * Project and contact information: http://www.cascading.org/
006 *
007 * This file is part of the Cascading project.
008 *
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 *     http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 */
021
022package cascading.flow.stream.element;
023
024import java.io.IOException;
025
026import cascading.CascadingException;
027import cascading.flow.FlowProcess;
028import cascading.flow.SliceCounters;
029import cascading.flow.StepCounters;
030import cascading.flow.stream.duct.Duct;
031import cascading.flow.stream.duct.DuctException;
032import cascading.flow.stream.graph.StreamGraph;
033import cascading.tap.Tap;
034import cascading.tuple.Fields;
035import cascading.tuple.TupleEntry;
036import cascading.tuple.TupleEntryCollector;
037
038/**
039 *
040 */
041public class SinkStage extends ElementStage<TupleEntry, Void>
042  {
043  private final Tap sink;
044  private TupleEntryCollector collector;
045
046  public SinkStage( FlowProcess flowProcess, Tap sink )
047    {
048    super( flowProcess, sink );
049    this.sink = sink;
050    }
051
052  public Tap getSink()
053    {
054    return sink;
055    }
056
057  @Override
058  public void bind( StreamGraph streamGraph )
059    {
060    // do not bind
061    }
062
063  @Override
064  public void prepare()
065    {
066    try
067      {
068      // todo: pass the resolved fields down
069      collector = sink.openForWrite( flowProcess, getOutput() );
070
071      if( sink.getSinkFields().isAll() )
072        {
073        Fields fields = getIncomingScopes().get( 0 ).getIncomingTapFields();
074        collector.setFields( fields );
075        }
076      }
077    catch( IOException exception )
078      {
079      throw new DuctException( "failed opening sink", exception );
080      }
081    }
082
083  protected Object getOutput()
084    {
085    return null;
086    }
087
088  @Override
089  public void start( Duct previous )
090    {
091    // do nothing
092    }
093
094  @Override
095  public void receive( Duct previous, int ordinal, TupleEntry tupleEntry )
096    {
097    try
098      {
099      timedAdd( StepCounters.Write_Duration, tupleEntry );
100      flowProcess.increment( StepCounters.Tuples_Written, 1 );
101      flowProcess.increment( SliceCounters.Tuples_Written, 1 );
102      }
103    catch( OutOfMemoryError error )
104      {
105      handleReThrowableException( "out of memory, try increasing task memory allocation", error );
106      }
107    catch( CascadingException exception )
108      {
109      handleException( exception, tupleEntry );
110      }
111    catch( Throwable throwable )
112      {
113      handleException( new DuctException( "internal error: " + tupleEntry.getTuple().print(), throwable ), tupleEntry );
114      }
115    }
116
117  protected void timedAdd( StepCounters durationCounter, TupleEntry tupleEntry )
118    {
119    long start = System.currentTimeMillis();
120
121    try
122      {
123      collector.add( tupleEntry );
124      }
125    finally
126      {
127      flowProcess.increment( durationCounter, System.currentTimeMillis() - start );
128      }
129    }
130
131  @Override
132  public void complete( Duct previous )
133    {
134    // do nothing
135    }
136
137  @Override
138  public void cleanup()
139    {
140    try
141      {
142      if( collector != null )
143        {
144        long start = System.currentTimeMillis();
145
146        try
147          {
148          collector.close(); // may flush underlying system
149          }
150        finally
151          {
152          flowProcess.increment( StepCounters.Write_Duration, System.currentTimeMillis() - start );
153          }
154        }
155
156      collector = null;
157      }
158    finally
159      {
160      super.cleanup();
161      }
162    }
163  }