Examples of DataStatistics

eu.stratosphere.compiler.DataStatistics
The collection of access methods that can be used to retrieve statistical information about the data processed in a job. Currently this method acts as an entry point only for obtaining cached statistics.
mil.nga.giat.geowave.store.adapter.statistics.DataStatistics
org.apache.flink.compiler.DataStatistics
The collection of access methods that can be used to retrieve statistical information about the data processed in a job. Currently this method acts as an entry point only for obtaining cached statistics.
org.apache.hadoop.mapred.JobInProgress.DataStatistics
org.apache.hadoop.mapred.gridmix.GenerateData.DataStatistics

Examples of org.apache.flink.compiler.DataStatistics

    output.setInput(testReducer);
    testReducer.setInput(input);


    Plan plan = new Plan(output);


    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(plan);


    NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
    return jgg.compileJobGraph(op);

View Full Code Here

Examples of org.apache.flink.compiler.DataStatistics

  
  // ------------------------------------------------------------------------  
  
  @Before
  public void setup() {
    this.dataStats = new DataStatistics();
    this.withStatsCompiler = new PactCompiler(this.dataStats, new DefaultCostEstimator());
    this.withStatsCompiler.setDefaultDegreeOfParallelism(DEFAULT_PARALLELISM);
    
    this.noStatsCompiler = new PactCompiler(null, new DefaultCostEstimator());
    this.noStatsCompiler.setDefaultDegreeOfParallelism(DEFAULT_PARALLELISM);

View Full Code Here

Examples of org.apache.flink.compiler.DataStatistics

    
    
    private OptimizedPlan compileProgram(String jobName) {
      Plan p = createProgramPlan(jobName);
      
      PactCompiler pc = new PactCompiler(new DataStatistics());
      return pc.compile(p);
    }

View Full Code Here

Examples of org.apache.flink.compiler.DataStatistics

    Plan p = getTestJob();
    if (p == null) {
      Assert.fail("Error: Cannot obtain Pact plan. Did the thest forget to override either 'getPactPlan()' or 'getJobGraph()' ?");
    }
    
    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(p);
    
    if (printPlan) {
      System.out.println(new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op)); 
    }

View Full Code Here

Examples of org.apache.flink.compiler.DataStatistics

        // we use the existing session
        shutDownAtEnd = false;
      }


      try {
        PactCompiler pc = new PactCompiler(new DataStatistics());
        OptimizedPlan op = pc.compile(plan);
        
        NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
        JobGraph jobGraph = jgg.compileJobGraph(op);

View Full Code Here

Examples of org.apache.flink.compiler.DataStatistics

   *            The program's plan.
   * @return JSON dump of the optimized plan.
   * @throws Exception
   */
  public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(plan);
    PlanJSONDumpGenerator gen = new PlanJSONDumpGenerator();
  
    return gen.getOptimizerPlanAsJSON(op);
  }

View Full Code Here

Examples of org.apache.flink.compiler.DataStatistics

   */
  public static String optimizerPlanAsJSON(Plan plan) throws Exception {
    LocalExecutor exec = new LocalExecutor();
    try {
      exec.start();
      PactCompiler pc = new PactCompiler(new DataStatistics());
      OptimizedPlan op = pc.compile(plan);
      PlanJSONDumpGenerator gen = new PlanJSONDumpGenerator();


      return gen.getOptimizerPlanAsJSON(op);
    } finally {

View Full Code Here

Examples of org.apache.hadoop.mapred.JobInProgress.DataStatistics

   * 
   * Added for use by queue scheduling algorithms.
   * @param currentTime 
   */
  boolean canBeSpeculated(long currentTime) {
    DataStatistics taskStats = job.getRunningTaskStatistics(isMapTask());
    if (LOG.isDebugEnabled()) {
      LOG.debug("activeTasks.size(): " + activeTasks.size() + " "
          + activeTasks.firstKey() + " task's progressrate: " + 
          getCurrentProgressRate(currentTime) + 
          " taskStats : " + taskStats);
    }
    return (!skipping && isRunnable() && isRunning() &&
        activeTasks.size() <= MAX_TASK_EXECS &&
        currentTime - lastDispatchTime >= SPECULATIVE_LAG &&
        completes == 0 && !isOnlyCommitPending() &&
        (taskStats.mean() - getCurrentProgressRate(currentTime) >
              taskStats.std() * job.getSlowTaskThreshold()));
  }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobInProgress.DataStatistics

   * 
   * Added for use by queue scheduling algorithms.
   * @param currentTime 
   */
  boolean canBeSpeculated(long currentTime) {
    DataStatistics taskStats = job.getRunningTaskStatistics(isMapTask());
    if (LOG.isDebugEnabled()) {
      LOG.debug("activeTasks.size(): " + activeTasks.size() + " "
          + activeTasks.firstKey() + " task's progressrate: " + 
          getCurrentProgressRate(currentTime) + 
          " taskStats : " + taskStats);
    }
    return (!skipping && isRunnable() && isRunning() &&
        activeTasks.size() <= MAX_TASK_EXECS &&
        currentTime - lastDispatchTime >= SPECULATIVE_LAG &&
        completes == 0 && !isOnlyCommitPending() &&
        (taskStats.mean() - getCurrentProgressRate(currentTime) >
              taskStats.std() * job.getSlowTaskThreshold()));
  }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobInProgress.DataStatistics

   * 
   * Added for use by queue scheduling algorithms.
   * @param currentTime 
   */
  boolean canBeSpeculated(long currentTime) {
    DataStatistics taskStats = job.getRunningTaskStatistics(isMapTask());
    if (LOG.isDebugEnabled()) {
      LOG.debug("activeTasks.size(): " + activeTasks.size() + " "
          + activeTasks.firstKey() + " task's progressrate: " + 
          getCurrentProgressRate(currentTime) + 
          " taskStats : " + taskStats);
    }
    return (!skipping && isRunnable() && isRunning() &&
        activeTasks.size() <= MAX_TASK_EXECS &&
        currentTime - lastDispatchTime >= SPECULATIVE_LAG &&
        completes == 0 && !isOnlyCommitPending() &&
        (taskStats.mean() - getCurrentProgressRate(currentTime) >
              taskStats.std() * job.getSlowTaskThreshold()));
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.