Package org.apache.hadoop.mapred.FairScheduler

Examples of org.apache.hadoop.mapred.FairScheduler.JobInfo

858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
   * these tasks and assign a second wave, which should continue to be
   * allocated in this manner.
   */
  public void testLargeJobs() throws IOException {
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info1 = scheduler.infos.get(job1);
   
    // Check scheduler variables
    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
    assertEquals(10,   info1.mapSchedulable.getDemand());
    assertEquals(10,   info1.reduceSchedulable.getDemand());
    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
   
    // Advance time before submitting another job j2, to make j1 run before j2
    // deterministically.
    advanceTime(100);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info2 = scheduler.infos.get(job2);
   
    // Check scheduler variables
    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
    assertEquals(10,   info1.mapSchedulable.getDemand());
View Full Code Here
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
   */
  public void testLargeJobsWithAssignMultiple() throws IOException {
    setUpCluster(1, 2, true);
   
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info1 = scheduler.infos.get(job1);
   
    // Check scheduler variables
    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
    assertEquals(10,   info1.mapSchedulable.getDemand());
    assertEquals(10,   info1.reduceSchedulable.getDemand());
    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
   
    // Advance time before submitting another job j2, to make j1 run before j2
    // deterministically.
    advanceTime(100);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info2 = scheduler.infos.get(job2);
   
    // Check scheduler variables; the fair shares should now have been allocated
    // equally between j1 and j2, but j1 should have (4 slots)*(100 ms) deficit
    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
View Full Code Here
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
   */
  public void testJobsWithPriorities() throws IOException {
    setUpCluster(1, 3, false);
   
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info1 = scheduler.infos.get(job1);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info2 = scheduler.infos.get(job2);
    job2.setPriority(JobPriority.HIGH);
    scheduler.update();
   
    // Check scheduler variables
    assertEquals(0,   info1.mapSchedulable.getRunningTasks());
View Full Code Here
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
    Pool defaultPool = scheduler.getPoolManager().getPool("default");
    Pool poolA = scheduler.getPoolManager().getPool("poolA");
    Pool poolB = scheduler.getPoolManager().getPool("poolB");
   
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info1 = scheduler.infos.get(job1);
   
    // Check scheduler variables
    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
    assertEquals(10,   info1.mapSchedulable.getDemand());
    assertEquals(10,   info1.reduceSchedulable.getDemand());
    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
   
    // Advance time 200ms and submit jobs 2 and 3
    advanceTime(200);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
    JobInfo info2 = scheduler.infos.get(job2);
    advanceTime(100);
    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
    JobInfo info3 = scheduler.infos.get(job3);
   
    // Check that minimum and fair shares have been allocated
    assertEquals(0,    defaultPool.getMapSchedulable().getMinShare());
    assertEquals(0,    defaultPool.getReduceSchedulable().getMinShare());
    assertEquals(1.0,  info1.mapSchedulable.getFairShare());
View Full Code Here
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
    out.close();
    scheduler.getPoolManager().reloadAllocs();
    Pool poolA = scheduler.getPoolManager().getPool("poolA");
   
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info1 = scheduler.infos.get(job1);
   
    // Check scheduler variables
    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
    assertEquals(10,   info1.mapSchedulable.getDemand());
    assertEquals(10,   info1.reduceSchedulable.getDemand());
    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
   
    // Advance time 200ms and submit job 2
    advanceTime(200);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
    JobInfo info2 = scheduler.infos.get(job2);
   
    // Check that minimum and fair shares have been allocated
    assertEquals(2,    poolA.getMapSchedulable().getMinShare());
    assertEquals(2,    poolA.getReduceSchedulable().getMinShare());
    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
    assertEquals(2.0,  info2.mapSchedulable.getFairShare());
    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
   
    // Advance time 100ms and submit job 3
    advanceTime(100);
    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
    JobInfo info3 = scheduler.infos.get(job3);
   
    // Check that minimum and fair shares have been allocated
    assertEquals(2,    poolA.getMapSchedulable().getMinShare());
    assertEquals(2,    poolA.getReduceSchedulable().getMinShare());
    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
View Full Code Here
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
    out.close();
    scheduler.getPoolManager().reloadAllocs();
    Pool poolA = scheduler.getPoolManager().getPool("poolA");
   
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info1 = scheduler.infos.get(job1);
   
    // Check scheduler variables
    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
    assertEquals(10,   info1.mapSchedulable.getDemand());
    assertEquals(10,   info1.reduceSchedulable.getDemand());
    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
   
    // Advance time 200ms and submit job 2
    advanceTime(200);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
    JobInfo info2 = scheduler.infos.get(job2);
   
    // Check that minimum and fair shares have been allocated
    assertEquals(2,    poolA.getMapSchedulable().getMinShare());
    assertEquals(2,    poolA.getReduceSchedulable().getMinShare());
    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
    assertEquals(2.0,  info2.mapSchedulable.getFairShare());
    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
   
    // Advance time 100ms and submit job 3
    advanceTime(100);
    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
    JobInfo info3 = scheduler.infos.get(job3);
   
    // Check that minimum and fair shares have been allocated
    assertEquals(2,    poolA.getMapSchedulable().getMinShare());
    assertEquals(2,    poolA.getReduceSchedulable().getMinShare());
    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
View Full Code Here
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
    out.println("</allocations>");
    out.close();
    scheduler.getPoolManager().reloadAllocs();
   
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info1 = scheduler.infos.get(job1);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 1, 1, "poolA");
    JobInfo info2 = scheduler.infos.get(job2);
   
    // Check scheduler variables
    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
    assertEquals(10,   info1.mapSchedulable.getDemand());
View Full Code Here
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
    // Submit jobs, advancing time in-between to make sure that they are
    // all submitted at distinct times.
    JobInProgress job1 = submitJobNotInitialized(JobStatus.PREP, 10, 10);
    assertTrue(((FakeJobInProgress)job1).inited());
    job1.getStatus().setRunState(JobStatus.RUNNING);
    JobInfo info1 = scheduler.infos.get(job1);
    advanceTime(10);
    JobInProgress job2 = submitJobNotInitialized(JobStatus.PREP, 10, 10);
    assertTrue(((FakeJobInProgress)job2).inited());
    job2.getStatus().setRunState(JobStatus.RUNNING);
    JobInfo info2 = scheduler.infos.get(job2);
    advanceTime(10);
    JobInProgress job3 = submitJobNotInitialized(JobStatus.PREP, 10, 10);
    JobInfo info3 = scheduler.infos.get(job3);
    advanceTime(10);
    JobInProgress job4 = submitJobNotInitialized(JobStatus.PREP, 10, 10);
    JobInfo info4 = scheduler.infos.get(job4);
   
    // Only two of the jobs should be initialized.
    assertTrue(((FakeJobInProgress)job1).inited());
    assertTrue(((FakeJobInProgress)job2).inited());
    assertFalse(((FakeJobInProgress)job3).inited());
View Full Code Here
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
   
    // Submit jobs, advancing time in-between to make sure that they are
    // all submitted at distinct times.
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
    job1.getJobConf().set("user.name", "user1");
    JobInfo info1 = scheduler.infos.get(job1);
    advanceTime(10);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10);
    job2.getJobConf().set("user.name", "user1");
    JobInfo info2 = scheduler.infos.get(job2);
    advanceTime(10);
    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10);
    job3.getJobConf().set("user.name", "user2");
    JobInfo info3 = scheduler.infos.get(job3);
    advanceTime(10);
    JobInProgress job4 = submitJob(JobStatus.RUNNING, 10, 10);
    job4.getJobConf().set("user.name", "user2");
    JobInfo info4 = scheduler.infos.get(job4);
   
    // Check scheduler variables
    assertEquals(1.33,  info1.mapSchedulable.getFairShare(), 0.1);
    assertEquals(1.33,  info1.reduceSchedulable.getFairShare(), 0.1);
    assertEquals(0.0,   info2.mapSchedulable.getFairShare());
View Full Code Here
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
    // all submitted at distinct times.
   
    // Two jobs for user1; only one should get to run
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
    job1.getJobConf().set("user.name", "user1");
    JobInfo info1 = scheduler.infos.get(job1);
    advanceTime(10);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10);
    job2.getJobConf().set("user.name", "user1");
    JobInfo info2 = scheduler.infos.get(job2);
    advanceTime(10);
   
    // Three jobs for user2; all should get to run
    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10);
    job3.getJobConf().set("user.name", "user2");
    JobInfo info3 = scheduler.infos.get(job3);
    advanceTime(10);
    JobInProgress job4 = submitJob(JobStatus.RUNNING, 10, 10);
    job4.getJobConf().set("user.name", "user2");
    JobInfo info4 = scheduler.infos.get(job4);
    advanceTime(10);
    JobInProgress job5 = submitJob(JobStatus.RUNNING, 10, 10);
    job5.getJobConf().set("user.name", "user2");
    JobInfo info5 = scheduler.infos.get(job5);
    advanceTime(10);
   
    // Three jobs for user3; only two should get to run
    JobInProgress job6 = submitJob(JobStatus.RUNNING, 10, 10);
    job6.getJobConf().set("user.name", "user3");
    JobInfo info6 = scheduler.infos.get(job6);
    advanceTime(10);
    JobInProgress job7 = submitJob(JobStatus.RUNNING, 10, 10);
    job7.getJobConf().set("user.name", "user3");
    JobInfo info7 = scheduler.infos.get(job7);
    advanceTime(10);
    JobInProgress job8 = submitJob(JobStatus.RUNNING, 10, 10);
    job8.getJobConf().set("user.name", "user3");
    JobInfo info8 = scheduler.infos.get(job8);
    advanceTime(10);
   
    // Two jobs for user4, in poolA; only one should get to run
    JobInProgress job9 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
    job9.getJobConf().set("user.name", "user4");
    JobInfo info9 = scheduler.infos.get(job9);
    advanceTime(10);
    JobInProgress job10 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
    job10.getJobConf().set("user.name", "user4");
    JobInfo info10 = scheduler.infos.get(job10);
    advanceTime(10);
   
    // Check scheduler variables. The jobs in poolA should get half
    // the total share, while those in the default pool should get
    // the other half. This works out to 2 slots each for the jobs
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.FairScheduler.JobInfo

  • org.apache.hadoop.mapred.FairSchedulerServlet
  • org.apache.hadoop.mapred.JobSchedulable
  • org.apache.hadoop.mapred.PoolSchedulable
  • org.apache.hadoop.mapred.TestFairScheduler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.