001    /*
002     * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
003     *
004     * Project and contact information: http://www.cascading.org/
005     *
006     * This file is part of the Cascading project.
007     *
008     * Licensed under the Apache License, Version 2.0 (the "License");
009     * you may not use this file except in compliance with the License.
010     * You may obtain a copy of the License at
011     *
012     *     http://www.apache.org/licenses/LICENSE-2.0
013     *
014     * Unless required by applicable law or agreed to in writing, software
015     * distributed under the License is distributed on an "AS IS" BASIS,
016     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017     * See the License for the specific language governing permissions and
018     * limitations under the License.
019     */
020    
021    package cascading.stats.hadoop;
022    
023    import java.util.HashMap;
024    import java.util.Map;
025    
026    import cascading.stats.CascadingStats;
027    import org.apache.hadoop.mapred.Counters;
028    import org.apache.hadoop.mapred.TaskCompletionEvent;
029    import org.apache.hadoop.mapred.TaskReport;
030    
031    import static cascading.stats.CascadingStats.Status.*;
032    
033    /** Class HadoopTaskStats tracks individual task stats. */
034    public class HadoopSliceStats
035      {
036      private final CascadingStats.Status parentStatus;
037    
038      public static class HadoopAttempt
039        {
040        private final TaskCompletionEvent event;
041    
042        public HadoopAttempt( TaskCompletionEvent event )
043          {
044          this.event = event;
045          }
046    
047        public int getEventId()
048          {
049          return event.getEventId();
050          }
051    
052        public int getTaskRunTime()
053          {
054          return event.getTaskRunTime();
055          }
056    
057        public String getTaskStatus()
058          {
059          return event.getTaskStatus().toString();
060          }
061    
062        public String getTaskTrackerHttp()
063          {
064          return event.getTaskTrackerHttp();
065          }
066    
067        public CascadingStats.Status getStatusFor()
068          {
069          CascadingStats.Status status = null;
070    
071          switch( event.getTaskStatus() )
072            {
073            case FAILED:
074              status = FAILED;
075              break;
076            case KILLED:
077              status = STOPPED;
078              break;
079            case SUCCEEDED:
080              status = SUCCESSFUL;
081              break;
082            case OBSOLETE:
083              status = SKIPPED;
084              break;
085            case TIPFAILED:
086              status = FAILED;
087              break;
088            }
089          return status;
090          }
091        }
092    
093      public enum Kind
094        {
095          SETUP, MAPPER, REDUCER, CLEANUP
096        }
097    
098      private String id;
099      private Kind kind;
100      private final boolean parentStepHasReducers;
101      private TaskReport taskReport;
102      private Map<String, Map<String, Long>> counters;
103    
104      private Map<Integer, HadoopAttempt> attempts = new HashMap<Integer, HadoopAttempt>();
105    
106      HadoopSliceStats( String id, CascadingStats.Status parentStatus, Kind kind, boolean parentStepHasReducers, TaskReport taskReport )
107        {
108        this.parentStatus = parentStatus;
109        this.id = id;
110        this.kind = kind;
111        this.parentStepHasReducers = parentStepHasReducers;
112        this.taskReport = taskReport;
113        }
114    
115      public String getID()
116        {
117        return id;
118        }
119    
120      public Kind getKind()
121        {
122        return kind;
123        }
124    
125      /**
126       * Method getId returns the Hadoop task id.
127       *
128       * @return the id (type String) of this HadoopTaskStats object.
129       */
130      public String getTaskID()
131        {
132        return taskReport.getTaskID().toString();
133        }
134    
135      public int getTaskIDNum()
136        {
137        return taskReport.getTaskID().getId();
138        }
139    
140      public String getJobID()
141        {
142        return taskReport.getTaskID().getJobID().toString();
143        }
144    
145      public boolean parentStepHasReducers()
146        {
147        return parentStepHasReducers;
148        }
149    
150      public float getProgress()
151        {
152        return taskReport.getProgress();
153        }
154    
155      public String getState()
156        {
157        return taskReport.getState();
158        }
159    
160      public long getStartTime()
161        {
162        return taskReport.getStartTime();
163        }
164    
165      public long getFinishTime()
166        {
167        return taskReport.getFinishTime();
168        }
169    
170      public CascadingStats.Status getParentStatus()
171        {
172        return parentStatus;
173        }
174    
175      public CascadingStats.Status getStatus()
176        {
177        CascadingStats.Status status = null;
178    
179        switch( taskReport.getCurrentStatus() )
180          {
181          case PENDING:
182            status = PENDING;
183            break;
184          case RUNNING:
185            status = RUNNING;
186            break;
187          case COMPLETE:
188            status = SUCCESSFUL;
189            break;
190          case KILLED:
191            status = STOPPED;
192            break;
193          case FAILED:
194            status = FAILED;
195            break;
196          }
197    
198        return status;
199        }
200    
201      public String[] getDiagnostics()
202        {
203        return taskReport.getDiagnostics();
204        }
205    
206      public Map<String, Map<String, Long>> getCounters()
207        {
208        if( counters == null )
209          setCounters( taskReport );
210    
211        return counters;
212        }
213    
214      public Map<Integer, HadoopAttempt> getAttempts()
215        {
216        return attempts;
217        }
218    
219      private void setCounters( TaskReport taskReport )
220        {
221        this.counters = new HashMap<String, Map<String, Long>>();
222    
223        Counters hadoopCounters = taskReport.getCounters();
224    
225        for( Counters.Group group : hadoopCounters )
226          {
227          Map<String, Long> values = new HashMap<String, Long>();
228    
229          this.counters.put( group.getName(), values );
230    
231          for( Counters.Counter counter : group )
232            values.put( counter.getName(), counter.getCounter() );
233          }
234        }
235    
236      /**
237       * Method getCounterValue returns the raw Hadoop counter value.
238       *
239       * @param counter of Enum
240       * @return long
241       */
242      public long getCounterValue( Enum counter )
243        {
244        return getCounterValue( counter.getDeclaringClass().getName(), counter.name() );
245        }
246    
247      /**
248       * Method getCounterValue returns the raw Hadoop counter value.
249       *
250       * @param group of String
251       * @param name  of String
252       * @return long
253       */
254      public long getCounterValue( String group, String name )
255        {
256        if( getCounters() == null || getCounters().get( group ) == null )
257          return 0;
258    
259        Long value = getCounters().get( group ).get( name );
260    
261        if( value == null )
262          return 0;
263    
264        return value;
265        }
266    
267      public void addAttempt( TaskCompletionEvent event )
268        {
269        attempts.put( event.getEventId(), new HadoopAttempt( event ) );
270        }
271    
272      @Override
273      public String toString()
274        {
275        final StringBuilder sb = new StringBuilder();
276        sb.append( "HadoopTaskStats" );
277        sb.append( "{id='" ).append( id ).append( '\'' );
278        sb.append( ", kind=" ).append( kind );
279        sb.append( '}' );
280        return sb.toString();
281        }
282      }