Examples of AttemptStatsValue

        
        long nextMillis = Math.min(currentUnitEnd.getTimeInMillis(),end);
        
        double percentOfTotal = (nextMillis - currentTime)/((double)(end - start));   
        
        AttemptStatsValue value = new AttemptStatsValue();
                
        value.setElapsedMinutes((nextMillis - currentTime)/1000.0/60.0);
        
        if (attempt.getCounters().get(CPU_MILLISECONDS) != null)
        {
          value.setCpuMinutes(percentOfTotal * attempt.getCounters().get(CPU_MILLISECONDS)/1000.0/60.0);
        }
        
        if (attempt.getCounters().get(SPILLED_RECORDS) != null)
        {
          value.setSpilledRecords((long)(percentOfTotal * attempt.getCounters().get(SPILLED_RECORDS)));
        }
        
        if (attempt.getCounters().get(REDUCE_SHUFFLE_BYTES) != null)
        {
          value.setReduceShuffleBytes(attempt.getCounters().get(REDUCE_SHUFFLE_BYTES));
        }
                
        key.setTime(currentUnitStart.getTimeInMillis());
        
        if ((key.getTime() + unit.toMillis(1)) >= start && start >= key.getTime())
        {
          value.setStarted(1);
        }
        
        if ((key.getTime() + unit.toMillis(1)) >= end && end >= key.getTime())
        {
          value.setFinished(1);
        }
        
        currentTime = nextMillis;
        
        context.write(new AvroKey<AttemptStatsKey>(key), new AvroValue<AttemptStatsValue>(value));

    }
    
    @Override
    protected void reduce(AvroKey<AttemptStatsKey> key, Iterable<AvroValue<AttemptStatsValue>> values, final Context context) throws IOException, InterruptedException 
    { 
      AttemptStatsValue merged = new AttemptStatsValue();
            
      merged.setElapsedMinutes(0.0);
      merged.setStarted(0);
      merged.setFinished(0);
      
      for (AvroValue<AttemptStatsValue> valueWrapped : values)
      {
        AttemptStatsValue value = valueWrapped.datum();
        merged.setElapsedMinutes(value.getElapsedMinutes() + merged.getElapsedMinutes());
        merged.setStarted(value.getStarted() + merged.getStarted());
        merged.setFinished(value.getFinished() + merged.getFinished());
        
        if (value.getCpuMinutes() != null)
        {
          if (merged.getCpuMinutes() == null)
          {
            merged.setCpuMinutes(value.getCpuMinutes());
          }
          else
          {
            merged.setCpuMinutes(merged.getCpuMinutes() + value.getCpuMinutes());
          }
        }
        
        if (value.getSpilledRecords() != null)
        {
          if (merged.getSpilledRecords() == null)
          {
            merged.setSpilledRecords(value.getSpilledRecords());
          }
          else
          {
            merged.setSpilledRecords(merged.getSpilledRecords() + value.getSpilledRecords());
          }
        }
        
        if (value.getReduceShuffleBytes() != null)
        {
          if (merged.getReduceShuffleBytes() == null)
          {
            merged.setReduceShuffleBytes(value.getReduceShuffleBytes());
          }
          else 
          {
            merged.setReduceShuffleBytes(merged.getReduceShuffleBytes() + value.getReduceShuffleBytes());
          }
        }
      }
      
      context.write(key, new AvroWrapper<AttemptStatsValue>(merged));

Examples of AttemptStatsValue

Examples of com.linkedin.whiteelephant.analysis.AttemptStatsValue

Examples of com.linkedin.whiteelephant.analysis.AttemptStatsValue