cascading.flow
Class BaseFlow<Config>

java.lang.Object
  extended by cascading.flow.BaseFlow<Config>
All Implemented Interfaces:
Flow<Config>, UnitOfWork<FlowStats>

@Process
public abstract class BaseFlow<Config>
extends Object
implements Flow<Config>


Nested Class Summary
static class BaseFlow.FlowHolder
          Class FlowHolder is a helper class for wrapping Flow instances.
 
Field Summary
protected  FlowStats flowStats
          Field flowStats
protected  ShutdownUtil.Hook shutdownHook
           
protected  Map<String,Tap> sinks
          Field sinks
protected  Map<String,Tap> sources
          Field sources
protected  boolean stop
          Field stop
protected  boolean stopJobsOnExit
          Field stopJobsOnExit
protected  Thread thread
          Field thread
 
Fields inherited from interface cascading.flow.Flow
CASCADING_FLOW_ID
 
Constructor Summary
protected BaseFlow()
          Used for testing.
protected BaseFlow(PlatformInfo platformInfo, Map<Object,Object> properties, Config defaultConfig, FlowDef flowDef)
           
protected BaseFlow(PlatformInfo platformInfo, Map<Object,Object> properties, Config defaultConfig, String name)
           
protected BaseFlow(PlatformInfo platformInfo, Map<Object,Object> properties, Config defaultConfig, String name, Map<String,String> flowDescriptor)
           
 
Method Summary
 void addListener(FlowListener flowListener)
          Method addListener registers the given flowListener with this instance.
 void addStepListener(FlowStepListener flowStepListener)
          Method addStepListener registers the given flowStepListener with this instance.
 boolean areSinksStale()
          Method areSinksStale returns true if any of the sinks referenced are out of date in relation to the sources.
 boolean areSourcesNewer(long sinkModified)
          Method areSourcesNewer returns true if any source is newer than the given sinkModified date value.
 void cleanup()
           
 void complete()
          Method complete starts the current Flow instance if it has not be previously started, then block until completion.
 Config createConfig(Map<Object,Object> properties, Config defaultConfig)
           
protected  Thread createFlowThread(String threadName)
           
 void deleteCheckpointsIfNotUpdate()
           
 void deleteCheckpointsIfReplace()
           
 void deleteSinks()
          Method deleteSinks deletes all sinks, whether or not they are configured for SinkMode.UPDATE.
 void deleteSinksIfNotUpdate()
          Method deleteSinksIfNotUpdate deletes all sinks if they are not configured with the SinkMode.UPDATE flag.
 void deleteSinksIfReplace()
           
 void deleteTrapsIfNotUpdate()
           
 void deleteTrapsIfReplace()
           
protected  void fireOnCompleted()
           
protected  void fireOnStarting()
           
protected  void fireOnStopping()
           
protected  void fireOnThrowable()
           
protected  void fireOnThrowable(Throwable throwable)
           
 String getCascadeID()
           
 CascadingServices getCascadingServices()
           
 List<String> getCheckpointNames()
           
 Map<String,Tap> getCheckpoints()
          Method getCheckpoints returns the checkpoint taps of this Flow object.
 Collection<Tap> getCheckpointsCollection()
          Method getCheckpointsCollection returns a Collection of checkpoint Taps for this Flow object.
protected  List<String> getClassPath()
           
protected  Fields getFieldsFor(ElementGraph pipeGraph, Tap tap)
           
 Map<String,String> getFlowDescriptor()
          Returns an immutable map of properties giving more details about the Flow object.
 FlowSession getFlowSession()
           
 FlowSkipStrategy getFlowSkipStrategy()
          Method getFlowSkipStrategy returns the current FlowSkipStrategy used by this Flow.
 FlowStats getFlowStats()
          Method getFlowStats returns the flowStats of this Flow object.
 List<FlowStep<Config>> getFlowSteps()
          Method getFlowSteps returns the flowSteps of this Flow object.
 FlowStepStrategy getFlowStepStrategy()
          Returns the current FlowStepStrategy instance.
 BaseFlow.FlowHolder getHolder()
          Used to return a simple wrapper for use as an edge in a graph where there can only be one instance of every edge.
 String getID()
          Method getID returns the ID of this Flow object.
protected abstract  int getMaxNumParallelSteps()
           
 String getName()
          Method getName returns the name of this Flow object.
 PlatformInfo getPlatformInfo()
           
 String getRunID()
           
 Tap getSink()
          Method getSink returns the first sink of this Flow object.
 Tap getSink(String name)
           
 long getSinkModified()
          Method getSinkModified returns the youngest modified date of any sink Tap managed by this Flow instance.
 List<String> getSinkNames()
           
 Map<String,Tap> getSinks()
          Method getSinks returns the sinks of this Flow object.
 Collection<Tap> getSinksCollection()
          Method getSinksCollection returns a Collection of sink Taps for this Flow object.
 Tap getSource(String name)
           
 List<String> getSourceNames()
           
 Map<String,Tap> getSources()
          Method getSources returns the sources of this Flow object.
 Collection<Tap> getSourcesCollection()
          Method getSourcesCollection returns a Collection of source Taps for this Flow object.
 UnitOfWorkSpawnStrategy getSpawnStrategy()
           
 FlowStats getStats()
           
 int getSubmitPriority()
          Method getSubmitPriority returns the submitPriority of this Flow object.
 String getTags()
           
 List<String> getTrapNames()
           
 Map<String,Tap> getTraps()
          Method getTraps returns the traps of this Flow object.
 Collection<Tap> getTrapsCollection()
          Method getTrapsCollection returns a Collection of trap Taps for this Flow object.
protected  void handleExecutorShutdown()
           
 boolean hasListeners()
          Method hasListeners returns true if FlowListener instances have been registered.
 boolean hasStepListeners()
          Method hasStepListeners returns true if FlowStepListener instances have been registered with any of the FlowSteps belonging to this instance
protected abstract  void initConfig(Map<Object,Object> properties, Config parentConfig)
          This method creates a new internal Config with the parentConfig as defaults using the properties to override the defaults.
protected  void initFromProperties(Map<Object,Object> properties)
           
 void initialize(ElementGraph pipeGraph, FlowStepGraph<Config> flowStepGraph)
           
protected  void initializeNewJobsMap()
           
protected  void initSteps()
           
protected abstract  void internalClean(boolean stop)
           
protected abstract  void internalShutdown()
           
protected abstract  void internalStart()
           
protected  void internalStopAllJobs()
           
 boolean isSkipFlow()
          Method isSkipFlow returns true if the parent Cascade should skip this Flow instance.
 boolean isStopJobsOnExit()
          Method isStopJobsOnExit returns the stopJobsOnExit of this Flow object.
protected  void logInfo(String message)
           
protected abstract  Config newConfig(Config defaultConfig)
           
 TupleEntryIterator openSink()
          Method openSink opens the first sink Tap.
 TupleEntryIterator openSink(String name)
          Method openSink opens the named sink Tap.
 TupleEntryIterator openSource()
          Method openSource opens the first source Tap.
 TupleEntryIterator openSource(String name)
          Method openSource opens the named source Tap.
 TupleEntryIterator openTapForRead(Tap tap)
          Method openTapForRead return a TupleEntryIterator for the given Tap instance.
 TupleEntryCollector openTapForWrite(Tap tap)
          Method openTapForWrite returns a (@link TupleCollector} for the given Tap instance.
 TupleEntryIterator openTrap()
          Method openTrap opens the first trap Tap.
 TupleEntryIterator openTrap(String name)
          Method openTrap opens the named trap Tap.
 void prepare()
          Method prepare is used by a Cascade to notify the given Flow it should initialize or clear any resources necessary for Flow.start() to be called successfully.
protected  void presentSinkFields(ElementGraph pipeGraph)
          Present the current resolved fields for the Tap
protected  void presentSourceFields(ElementGraph pipeGraph)
          Present the current resolved fields for the Tap
protected  void registerShutdownHook()
           
 boolean removeListener(FlowListener flowListener)
          Method removeListener removes the given flowListener from this instance.
 boolean removeStepListener(FlowStepListener flowStepListener)
          Method removeStepListener removes the given flowStepListener from this instance.
 boolean resourceExists(Tap tap)
          Method resourceExists returns true if the resource represented by the given Tap instance exists.
protected  void retrieveSinkFields()
          Force a Scheme to fetch any fields from a meta-data store
protected  void retrieveSourceFields()
          Force a Scheme to fetch any fields from a meta-data store
 void setCascade(Cascade cascade)
           
protected  void setCheckpoints(Map<String,Tap> checkpoints)
           
protected abstract  void setConfigProperty(Config config, Object key, Object value)
           
 FlowSkipStrategy setFlowSkipStrategy(FlowSkipStrategy flowSkipStrategy)
          Method setFlowSkipStrategy sets a new FlowSkipStrategy, the current strategy is returned.
protected  void setFlowStepGraph(FlowStepGraph flowStepGraph)
           
 void setFlowStepStrategy(FlowStepStrategy flowStepStrategy)
          Sets a default FlowStepStrategy instance.
protected  void setName(String name)
           
protected  void setSinks(Map<String,Tap> sinks)
           
protected  void setSources(Map<String,Tap> sources)
           
 void setSpawnStrategy(UnitOfWorkSpawnStrategy spawnStrategy)
           
 void setSubmitPriority(int submitPriority)
          Method setSubmitPriority sets the submitPriority of this Flow object.
protected  void setTraps(Map<String,Tap> traps)
           
 void start()
          Method start begins the execution of this Flow instance.
 void stop()
          Method stop stops all running jobs, killing any currently executing.
 String toString()
           
 ElementGraph updateSchemes(ElementGraph pipeGraph)
           
 void writeDOT(String filename)
          Method writeDOT writes this Flow instance to the given filename as a DOT file for import into a graphics package.
 void writeStepsDOT(String filename)
          Method writeStepsDOT writes this Flow step graph to the given filename as a DOT file for import into a graphics package.
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
 
Methods inherited from interface cascading.flow.Flow
getConfig, getConfigAsProperties, getConfigCopy, getFlowProcess, getProperty, stepsAreLocal
 

Field Detail

flowStats

protected FlowStats flowStats
Field flowStats


sources

protected Map<String,Tap> sources
Field sources


sinks

protected Map<String,Tap> sinks
Field sinks


stopJobsOnExit

protected boolean stopJobsOnExit
Field stopJobsOnExit


thread

protected transient Thread thread
Field thread


stop

protected boolean stop
Field stop


shutdownHook

protected ShutdownUtil.Hook shutdownHook
Constructor Detail

BaseFlow

protected BaseFlow()
Used for testing.


BaseFlow

protected BaseFlow(PlatformInfo platformInfo,
                   Map<Object,Object> properties,
                   Config defaultConfig,
                   String name)

BaseFlow

protected BaseFlow(PlatformInfo platformInfo,
                   Map<Object,Object> properties,
                   Config defaultConfig,
                   String name,
                   Map<String,String> flowDescriptor)

BaseFlow

protected BaseFlow(PlatformInfo platformInfo,
                   Map<Object,Object> properties,
                   Config defaultConfig,
                   FlowDef flowDef)
Method Detail

getPlatformInfo

public PlatformInfo getPlatformInfo()
Specified by:
getPlatformInfo in interface Flow<Config>

initialize

public void initialize(ElementGraph pipeGraph,
                       FlowStepGraph<Config> flowStepGraph)

updateSchemes

public ElementGraph updateSchemes(ElementGraph pipeGraph)

retrieveSourceFields

protected void retrieveSourceFields()
Force a Scheme to fetch any fields from a meta-data store


presentSourceFields

protected void presentSourceFields(ElementGraph pipeGraph)
Present the current resolved fields for the Tap

Parameters:
pipeGraph -

retrieveSinkFields

protected void retrieveSinkFields()
Force a Scheme to fetch any fields from a meta-data store


presentSinkFields

protected void presentSinkFields(ElementGraph pipeGraph)
Present the current resolved fields for the Tap

Parameters:
pipeGraph -

getFieldsFor

protected Fields getFieldsFor(ElementGraph pipeGraph,
                              Tap tap)

getCascadingServices

public CascadingServices getCascadingServices()

initSteps

protected void initSteps()

getName

public String getName()
Description copied from interface: Flow
Method getName returns the name of this Flow object.

Specified by:
getName in interface Flow<Config>
Specified by:
getName in interface UnitOfWork<FlowStats>
Returns:
the name (type String) of this Flow object.

setName

protected void setName(String name)

getID

public String getID()
Description copied from interface: Flow
Method getID returns the ID of this Flow object.

The ID value is a long HEX String used to identify this instance globally. Subsequent Flow instances created with identical parameters will not return the same ID.

Specified by:
getID in interface Flow<Config>
Specified by:
getID in interface UnitOfWork<FlowStats>
Returns:
the ID (type String) of this Flow object.

getTags

public String getTags()
Specified by:
getTags in interface Flow<Config>
Specified by:
getTags in interface UnitOfWork<FlowStats>

getSubmitPriority

public int getSubmitPriority()
Description copied from interface: Flow
Method getSubmitPriority returns the submitPriority of this Flow object.

10 is lowest, 1 is the highest, 5 is the default.

Specified by:
getSubmitPriority in interface Flow<Config>
Returns:
the submitPriority (type int) of this FlowStep object.

setSubmitPriority

public void setSubmitPriority(int submitPriority)
Description copied from interface: Flow
Method setSubmitPriority sets the submitPriority of this Flow object.

10 is lowest, 1 is the highest, 5 is the default.

Specified by:
setSubmitPriority in interface Flow<Config>
Parameters:
submitPriority - the submitPriority of this FlowStep object.

setSources

protected void setSources(Map<String,Tap> sources)

setSinks

protected void setSinks(Map<String,Tap> sinks)

setTraps

protected void setTraps(Map<String,Tap> traps)

setCheckpoints

protected void setCheckpoints(Map<String,Tap> checkpoints)

setFlowStepGraph

protected void setFlowStepGraph(FlowStepGraph flowStepGraph)

initConfig

protected abstract void initConfig(Map<Object,Object> properties,
                                   Config parentConfig)
This method creates a new internal Config with the parentConfig as defaults using the properties to override the defaults.

Parameters:
properties - of type Map
parentConfig - of type Config

createConfig

public Config createConfig(Map<Object,Object> properties,
                           Config defaultConfig)

setConfigProperty

protected abstract void setConfigProperty(Config config,
                                          Object key,
                                          Object value)

newConfig

protected abstract Config newConfig(Config defaultConfig)

initFromProperties

protected void initFromProperties(Map<Object,Object> properties)

getFlowSession

public FlowSession getFlowSession()

getFlowStats

public FlowStats getFlowStats()
Description copied from interface: Flow
Method getFlowStats returns the flowStats of this Flow object.

Specified by:
getFlowStats in interface Flow<Config>
Returns:
the flowStats (type FlowStats) of this Flow object.

getFlowDescriptor

public Map<String,String> getFlowDescriptor()
Description copied from interface: Flow
Returns an immutable map of properties giving more details about the Flow object.

See FlowDef.addDescription(String, String) to set values on a given Flow.

Flow descriptions provide meta-data to monitoring systems describing the workload a given Flow represents. For known description types, see FlowDescriptors.

Specified by:
getFlowDescriptor in interface Flow<Config>
Returns:
Map

getStats

public FlowStats getStats()
Specified by:
getStats in interface UnitOfWork<FlowStats>

hasListeners

public boolean hasListeners()
Description copied from interface: Flow
Method hasListeners returns true if FlowListener instances have been registered.

Specified by:
hasListeners in interface Flow<Config>
Returns:
boolean

addListener

public void addListener(FlowListener flowListener)
Description copied from interface: Flow
Method addListener registers the given flowListener with this instance.

Specified by:
addListener in interface Flow<Config>
Parameters:
flowListener - of type FlowListener

removeListener

public boolean removeListener(FlowListener flowListener)
Description copied from interface: Flow
Method removeListener removes the given flowListener from this instance.

Specified by:
removeListener in interface Flow<Config>
Parameters:
flowListener - of type FlowListener
Returns:
true if the listener was removed

hasStepListeners

public boolean hasStepListeners()
Description copied from interface: Flow
Method hasStepListeners returns true if FlowStepListener instances have been registered with any of the FlowSteps belonging to this instance

Specified by:
hasStepListeners in interface Flow<Config>
Returns:
boolean

addStepListener

public void addStepListener(FlowStepListener flowStepListener)
Description copied from interface: Flow
Method addStepListener registers the given flowStepListener with this instance.

Specified by:
addStepListener in interface Flow<Config>
Parameters:
flowStepListener - of type addStepListener

removeStepListener

public boolean removeStepListener(FlowStepListener flowStepListener)
Description copied from interface: Flow
Method removeStepListener removes the given flowStepListener from this instance.

Specified by:
removeStepListener in interface Flow<Config>
Parameters:
flowStepListener - of type FlowStepListener
Returns:
true if the listener was removed from all the FlowStep belonging to this instance

getSources

public Map<String,Tap> getSources()
Description copied from interface: Flow
Method getSources returns the sources of this Flow object.

Specified by:
getSources in interface Flow<Config>
Returns:
the sources (type Map) of this Flow object.

getSourceNames

public List<String> getSourceNames()
Specified by:
getSourceNames in interface Flow<Config>

getSource

public Tap getSource(String name)
Specified by:
getSource in interface Flow<Config>

getSourcesCollection

@DependencyIncoming
public Collection<Tap> getSourcesCollection()
Description copied from interface: Flow
Method getSourcesCollection returns a Collection of source Taps for this Flow object.

Specified by:
getSourcesCollection in interface Flow<Config>
Returns:
the sourcesCollection (type Collection) of this Flow object.

getSinks

public Map<String,Tap> getSinks()
Description copied from interface: Flow
Method getSinks returns the sinks of this Flow object.

Specified by:
getSinks in interface Flow<Config>
Returns:
the sinks (type Map) of this Flow object.

getSinkNames

public List<String> getSinkNames()
Specified by:
getSinkNames in interface Flow<Config>

getSink

public Tap getSink(String name)
Specified by:
getSink in interface Flow<Config>

getSinksCollection

@DependencyOutgoing
public Collection<Tap> getSinksCollection()
Description copied from interface: Flow
Method getSinksCollection returns a Collection of sink Taps for this Flow object.

Specified by:
getSinksCollection in interface Flow<Config>
Returns:
the sinkCollection (type Collection) of this Flow object.

getSink

public Tap getSink()
Description copied from interface: Flow
Method getSink returns the first sink of this Flow object.

Specified by:
getSink in interface Flow<Config>
Returns:
the sink (type Tap) of this Flow object.

getTraps

public Map<String,Tap> getTraps()
Description copied from interface: Flow
Method getTraps returns the traps of this Flow object.

Specified by:
getTraps in interface Flow<Config>
Returns:
the traps (type Map) of this Flow object.

getTrapNames

public List<String> getTrapNames()
Specified by:
getTrapNames in interface Flow<Config>

getTrapsCollection

public Collection<Tap> getTrapsCollection()
Description copied from interface: Flow
Method getTrapsCollection returns a Collection of trap Taps for this Flow object.

Specified by:
getTrapsCollection in interface Flow<Config>
Returns:
the trapsCollection (type Collection) of this Flow object.

getCheckpoints

public Map<String,Tap> getCheckpoints()
Description copied from interface: Flow
Method getCheckpoints returns the checkpoint taps of this Flow object.

Specified by:
getCheckpoints in interface Flow<Config>
Returns:
the traps (type Map) of this Flow object.

getCheckpointNames

public List<String> getCheckpointNames()
Specified by:
getCheckpointNames in interface Flow<Config>

getCheckpointsCollection

public Collection<Tap> getCheckpointsCollection()
Description copied from interface: Flow
Method getCheckpointsCollection returns a Collection of checkpoint Taps for this Flow object.

Specified by:
getCheckpointsCollection in interface Flow<Config>
Returns:
the trapsCollection (type Collection) of this Flow object.

isStopJobsOnExit

public boolean isStopJobsOnExit()
Description copied from interface: Flow
Method isStopJobsOnExit returns the stopJobsOnExit of this Flow object. Defaults to true.

Specified by:
isStopJobsOnExit in interface Flow<Config>
Returns:
the stopJobsOnExit (type boolean) of this Flow object.

getFlowSkipStrategy

public FlowSkipStrategy getFlowSkipStrategy()
Description copied from interface: Flow
Method getFlowSkipStrategy returns the current FlowSkipStrategy used by this Flow.

Specified by:
getFlowSkipStrategy in interface Flow<Config>
Returns:
FlowSkipStrategy

setFlowSkipStrategy

public FlowSkipStrategy setFlowSkipStrategy(FlowSkipStrategy flowSkipStrategy)
Description copied from interface: Flow
Method setFlowSkipStrategy sets a new FlowSkipStrategy, the current strategy is returned.

FlowSkipStrategy instances define when a Flow instance should be skipped. The default strategy is FlowSkipIfSinkNotStale. An alternative strategy would be FlowSkipIfSinkExists.

A FlowSkipStrategy will not be consulted when executing a Flow directly through Flow.start() or Flow.complete(). Only when the Flow is executed through a Cascade instance.

Specified by:
setFlowSkipStrategy in interface Flow<Config>
Parameters:
flowSkipStrategy - of type FlowSkipStrategy
Returns:
FlowSkipStrategy

isSkipFlow

public boolean isSkipFlow()
                   throws IOException
Description copied from interface: Flow
Method isSkipFlow returns true if the parent Cascade should skip this Flow instance. True is returned if the current FlowSkipStrategy returns true.

Specified by:
isSkipFlow in interface Flow<Config>
Returns:
the skipFlow (type boolean) of this Flow object.
Throws:
IOException - when

areSinksStale

public boolean areSinksStale()
                      throws IOException
Description copied from interface: Flow
Method areSinksStale returns true if any of the sinks referenced are out of date in relation to the sources. Or if any sink method Tap.isReplace() returns true.

Specified by:
areSinksStale in interface Flow<Config>
Returns:
boolean
Throws:
IOException - when

areSourcesNewer

public boolean areSourcesNewer(long sinkModified)
                        throws IOException
Description copied from interface: Flow
Method areSourcesNewer returns true if any source is newer than the given sinkModified date value.

Specified by:
areSourcesNewer in interface Flow<Config>
Parameters:
sinkModified - of type long
Returns:
boolean
Throws:
IOException - when

getSinkModified

public long getSinkModified()
                     throws IOException
Description copied from interface: Flow
Method getSinkModified returns the youngest modified date of any sink Tap managed by this Flow instance.

If zero (0) is returned, at least one of the sink resources does not exist. If minus one (-1) is returned, atleast one of the sinks are marked for delete (returns true).

Specified by:
getSinkModified in interface Flow<Config>
Returns:
the sinkModified (type long) of this Flow object.
Throws:
IOException - when

getFlowStepStrategy

public FlowStepStrategy getFlowStepStrategy()
Description copied from interface: Flow
Returns the current FlowStepStrategy instance.

Specified by:
getFlowStepStrategy in interface Flow<Config>
Returns:
FlowStepStrategy

setFlowStepStrategy

public void setFlowStepStrategy(FlowStepStrategy flowStepStrategy)
Description copied from interface: Flow
Sets a default FlowStepStrategy instance.

Use a FlowStepStrategy to change FlowStep configuration properties before the properties are submitted to the underlying platform for the step unit of work.

Specified by:
setFlowStepStrategy in interface Flow<Config>
Parameters:
flowStepStrategy - The FlowStepStrategy to use.

getFlowSteps

public List<FlowStep<Config>> getFlowSteps()
Description copied from interface: Flow
Method getFlowSteps returns the flowSteps of this Flow object. They will be in topological order.

Specified by:
getFlowSteps in interface Flow<Config>
Returns:
the steps (type List) of this Flow object.

prepare

@ProcessPrepare
public void prepare()
Description copied from interface: Flow
Method prepare is used by a Cascade to notify the given Flow it should initialize or clear any resources necessary for Flow.start() to be called successfully.

Specifically, this implementation calls deleteSinksIfNotUpdate() && deleteTrapsIfNotUpdate().

Specified by:
prepare in interface Flow<Config>
Specified by:
prepare in interface UnitOfWork<FlowStats>

start

@ProcessStart
public void start()
Description copied from interface: Flow
Method start begins the execution of this Flow instance. It will return immediately. Use the method Flow.complete() to block until this Flow completes.

Specified by:
start in interface Flow<Config>
Specified by:
start in interface UnitOfWork<FlowStats>

createFlowThread

protected Thread createFlowThread(String threadName)

internalStart

protected abstract void internalStart()

stop

@ProcessStop
public void stop()
Description copied from interface: Flow
Method stop stops all running jobs, killing any currently executing.

Specified by:
stop in interface Flow<Config>
Specified by:
stop in interface UnitOfWork<FlowStats>

internalClean

protected abstract void internalClean(boolean stop)

complete

@ProcessComplete
public void complete()
Description copied from interface: Flow
Method complete starts the current Flow instance if it has not be previously started, then block until completion.

Specified by:
complete in interface Flow<Config>
Specified by:
complete in interface UnitOfWork<FlowStats>

cleanup

@ProcessCleanup
public void cleanup()
Specified by:
cleanup in interface Flow<Config>
Specified by:
cleanup in interface UnitOfWork<FlowStats>

openSource

public TupleEntryIterator openSource()
                              throws IOException
Description copied from interface: Flow
Method openSource opens the first source Tap.

Specified by:
openSource in interface Flow<Config>
Returns:
TupleIterator
Throws:
IOException - when

openSource

public TupleEntryIterator openSource(String name)
                              throws IOException
Description copied from interface: Flow
Method openSource opens the named source Tap.

Specified by:
openSource in interface Flow<Config>
Parameters:
name - of type String
Returns:
TupleIterator
Throws:
IOException - when

openSink

public TupleEntryIterator openSink()
                            throws IOException
Description copied from interface: Flow
Method openSink opens the first sink Tap.

Specified by:
openSink in interface Flow<Config>
Returns:
TupleIterator
Throws:
IOException - when

openSink

public TupleEntryIterator openSink(String name)
                            throws IOException
Description copied from interface: Flow
Method openSink opens the named sink Tap.

Specified by:
openSink in interface Flow<Config>
Parameters:
name - of type String
Returns:
TupleIterator
Throws:
IOException - when

openTrap

public TupleEntryIterator openTrap()
                            throws IOException
Description copied from interface: Flow
Method openTrap opens the first trap Tap.

Specified by:
openTrap in interface Flow<Config>
Returns:
TupleIterator
Throws:
IOException - when

openTrap

public TupleEntryIterator openTrap(String name)
                            throws IOException
Description copied from interface: Flow
Method openTrap opens the named trap Tap.

Specified by:
openTrap in interface Flow<Config>
Parameters:
name - of type String
Returns:
TupleIterator
Throws:
IOException - when

deleteSinks

public void deleteSinks()
                 throws IOException
Method deleteSinks deletes all sinks, whether or not they are configured for SinkMode.UPDATE.

Use with caution.

Throws:
IOException - when
See Also:
deleteSinksIfNotUpdate()

deleteSinksIfNotUpdate

public void deleteSinksIfNotUpdate()
                            throws IOException
Method deleteSinksIfNotUpdate deletes all sinks if they are not configured with the SinkMode.UPDATE flag.

Typically used by a Cascade before executing the flow if the sinks are stale.

Use with caution.

Throws:
IOException - when

deleteSinksIfReplace

public void deleteSinksIfReplace()
                          throws IOException
Throws:
IOException

deleteTrapsIfNotUpdate

public void deleteTrapsIfNotUpdate()
                            throws IOException
Throws:
IOException

deleteCheckpointsIfNotUpdate

public void deleteCheckpointsIfNotUpdate()
                                  throws IOException
Throws:
IOException

deleteTrapsIfReplace

public void deleteTrapsIfReplace()
                          throws IOException
Throws:
IOException

deleteCheckpointsIfReplace

public void deleteCheckpointsIfReplace()
                                throws IOException
Throws:
IOException

resourceExists

public boolean resourceExists(Tap tap)
                       throws IOException
Description copied from interface: Flow
Method resourceExists returns true if the resource represented by the given Tap instance exists.

Specified by:
resourceExists in interface Flow<Config>
Parameters:
tap - of type Tap
Returns:
boolean
Throws:
IOException - when

openTapForRead

public TupleEntryIterator openTapForRead(Tap tap)
                                  throws IOException
Description copied from interface: Flow
Method openTapForRead return a TupleEntryIterator for the given Tap instance.

Note the returned iterator will return the same instance of TupleEntry on every call, thus a copy must be made of either the TupleEntry or the underlying Tuple instance if they are to be stored in a Collection.

Specified by:
openTapForRead in interface Flow<Config>
Parameters:
tap - of type Tap
Returns:
TupleIterator
Throws:
IOException - when there is an error opening the resource

openTapForWrite

public TupleEntryCollector openTapForWrite(Tap tap)
                                    throws IOException
Description copied from interface: Flow
Method openTapForWrite returns a (@link TupleCollector} for the given Tap instance.

Specified by:
openTapForWrite in interface Flow<Config>
Parameters:
tap - of type Tap
Returns:
TupleCollector
Throws:
IOException - when there is an error opening the resource

getMaxNumParallelSteps

protected abstract int getMaxNumParallelSteps()

internalShutdown

protected abstract void internalShutdown()

initializeNewJobsMap

protected void initializeNewJobsMap()

internalStopAllJobs

protected void internalStopAllJobs()

handleExecutorShutdown

protected void handleExecutorShutdown()

fireOnCompleted

protected void fireOnCompleted()

fireOnThrowable

protected void fireOnThrowable(Throwable throwable)

fireOnThrowable

protected void fireOnThrowable()

fireOnStopping

protected void fireOnStopping()

fireOnStarting

protected void fireOnStarting()

toString

public String toString()
Overrides:
toString in class Object

logInfo

protected void logInfo(String message)

writeDOT

public void writeDOT(String filename)
Description copied from interface: Flow
Method writeDOT writes this Flow instance to the given filename as a DOT file for import into a graphics package.

Specified by:
writeDOT in interface Flow<Config>
Parameters:
filename - of type String

writeStepsDOT

public void writeStepsDOT(String filename)
Description copied from interface: Flow
Method writeStepsDOT writes this Flow step graph to the given filename as a DOT file for import into a graphics package.

Specified by:
writeStepsDOT in interface Flow<Config>
Parameters:
filename - of type String

getHolder

public BaseFlow.FlowHolder getHolder()
Used to return a simple wrapper for use as an edge in a graph where there can only be one instance of every edge.

Returns:
FlowHolder

setCascade

public void setCascade(Cascade cascade)

getCascadeID

public String getCascadeID()
Specified by:
getCascadeID in interface Flow<Config>

getRunID

public String getRunID()
Specified by:
getRunID in interface Flow<Config>

getClassPath

protected List<String> getClassPath()

setSpawnStrategy

public void setSpawnStrategy(UnitOfWorkSpawnStrategy spawnStrategy)
Specified by:
setSpawnStrategy in interface UnitOfWork<FlowStats>

getSpawnStrategy

public UnitOfWorkSpawnStrategy getSpawnStrategy()
Specified by:
getSpawnStrategy in interface UnitOfWork<FlowStats>

registerShutdownHook

protected void registerShutdownHook()


Copyright © 2007-2015 Concurrent, Inc. All Rights Reserved.