Examples of org.apache.hadoop.mapred.PoolManager

org.apache.hadoop.mapred.PoolManager
Maintains a list of pools as well as scheduling parameters for each pool, such as guaranteed share allocations, from the fair scheduler config file.

    out.println("<defaultMaxTotalInitedTasks>50000" +
                "</defaultMaxTotalInitedTasks>");
    out.println("</allocations>");
    out.close();


    PoolManager poolManager = scheduler.getPoolManager();
    poolManager.reloadAllocs();


    assertEquals(6, poolManager.getPools().size()); // 5 in file + default pool
    assertEquals(0, poolManager.getMinSlots(Pool.DEFAULT_POOL_NAME,
        TaskType.MAP));
    assertEquals(0, poolManager.getMinSlots(Pool.DEFAULT_POOL_NAME,
        TaskType.REDUCE));
    assertEquals(1, poolManager.getMinSlots("pool_a", TaskType.MAP));
    assertEquals(2, poolManager.getMinSlots("pool_a", TaskType.REDUCE));
    assertEquals(2, poolManager.getMinSlots("pool_b", TaskType.MAP));
    assertEquals(1, poolManager.getMinSlots("pool_b", TaskType.REDUCE));
    assertEquals(2, poolManager.getMinSlots("pool_c", TaskType.MAP));
    assertEquals(0, poolManager.getMinSlots("pool_c", TaskType.REDUCE));
    assertEquals(0, poolManager.getMinSlots("pool_d", TaskType.MAP));
    assertEquals(0, poolManager.getMinSlots("pool_d", TaskType.REDUCE));
    assertEquals(0, poolManager.getMinSlots("pool_e", TaskType.MAP));
    assertEquals(0, poolManager.getMinSlots("pool_e", TaskType.REDUCE));
    assertEquals(10, poolManager.getUserMaxJobs("user1"));
    assertEquals(5, poolManager.getUserMaxJobs("user2"));
    assertEquals(4, poolManager.getMaxSlots("pool_b", TaskType.MAP));
    assertEquals(2, poolManager.getMaxSlots("pool_b", TaskType.REDUCE));
    assertEquals(4, poolManager.getMaxSlots("pool_c", TaskType.MAP));
    assertEquals(100000, poolManager.getPoolMaxInitedTasks("pool_a"));
    assertEquals(20000, poolManager.getPoolMaxInitedTasks("pool_b"));
    assertEquals(50000, poolManager.getPoolMaxInitedTasks("pool_c"));
    assertEquals(50000, poolManager.getPoolMaxInitedTasks("pool_d"));
    assertEquals(50000, poolManager.getPoolMaxInitedTasks("pool_e"));
    assertEquals(Integer.MAX_VALUE,
                 poolManager.getMaxSlots("pool_c", TaskType.REDUCE));
    assertEquals(120000, poolManager.getMinSharePreemptionTimeout(
        Pool.DEFAULT_POOL_NAME));
    assertEquals(120000, poolManager.getMinSharePreemptionTimeout("pool_a"));
    assertEquals(120000, poolManager.getMinSharePreemptionTimeout("pool_b"));
    assertEquals(120000, poolManager.getMinSharePreemptionTimeout("pool_c"));
    assertEquals(120000, poolManager.getMinSharePreemptionTimeout("pool_d"));
    assertEquals(120000, poolManager.getMinSharePreemptionTimeout("pool_a"));
    assertEquals(60000, poolManager.getMinSharePreemptionTimeout("pool_e"));
    assertEquals(300000, poolManager.getFairSharePreemptionTimeout());
  }

View Full Code Here

    // Create two jobs with ten maps
    String poolName = "pool_limited";
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 5, poolName);
    advanceTime(10);


    PoolManager poolMgr = scheduler.getPoolManager();
    assertEquals(2, poolMgr.getMaxSlots(poolName, TaskType.MAP));
    assertEquals(1, poolMgr.getMaxSlots(poolName, TaskType.REDUCE));


    // Check scheduler variables
    JobInfo info1 = scheduler.infos.get(job1);
    assertEquals(0,    info1.runningMaps);
    assertEquals(0,    info1.runningReduces);

View Full Code Here

                              "pool@root",
                              "ứnicȍḏe",
                              "\0",
                              "" };


    PoolManager mgr = scheduler.getPoolManager();
    JobConf jobConf = new JobConf(conf);
    JobInProgress job = new FakeJobInProgress(jobConf, taskTrackerManager,
        jobTracker);


    // Check that each good pool name is permitted
    for (String goodPoolName : goodPoolNames) {
      job.conf.set(PoolManager.EXPLICIT_POOL_PROPERTY, goodPoolName);
      mgr.addJob(job);
      assertEquals("PoolManager rejected good pool name",
          goodPoolName, mgr.getPoolName(job));
    }


    // Check that each bad pool name is rejected
    for (String badPoolName : badPoolNames) {
      job.conf.set(PoolManager.EXPLICIT_POOL_PROPERTY, badPoolName);
      mgr.addJob(job);
      assertFalse("PoolManager accepted bad pool name",
          badPoolName.equals(mgr.getPoolName(job)));
    }
  }

View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.PoolManager

org.apache.hadoop.conf.Configuration

org.apache.hadoop.mapred.TestFairScheduler

java.net.URLConnection

javax.xml.parsers.DocumentBuilderFactory

org.w3c.dom.Node

javax.xml.parsers.DocumentBuilder

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.