Package org.apache.hadoop.mapreduce.util

Examples of org.apache.hadoop.mapreduce.util.LinuxResourceCalculatorPlugin

257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
    if (!System.getProperty("os.name").startsWith("Linux")) {
      return;
    }

    JobConf conf = new JobConf();
    LinuxResourceCalculatorPlugin plugin = new LinuxResourceCalculatorPlugin();
    // In this case, we only check these three fields because they are static
    conf.setLong("totalVmemOnTT", plugin.getVirtualMemorySize());
    conf.setLong("totalPmemOnTT", plugin.getPhysicalMemorySize());
    conf.setLong("numProcessors", plugin.getNumProcessors());

    try {
      setUpCluster(conf);
      runSleepJob(miniMRCluster.createJobConf());
      verifyTestResults(true);
View Full Code Here
532
533
534
535
536
537
538
539
540
541
542
543
544
    // very small value, so that no task escapes to successful completion.
    fConf.set("mapred.tasktracker.taskmemorymanager.monitoring-interval",
        String.valueOf(300));
   
    // reserve all memory on TT so that the job will exceed memory limits
    LinuxResourceCalculatorPlugin memoryCalculatorPlugin =
            new LinuxResourceCalculatorPlugin();
    long totalPhysicalMemory = memoryCalculatorPlugin.getPhysicalMemorySize();
    long reservedPhysicalMemory = totalPhysicalMemory / (1024 * 1024) + 1;
    fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB,
                  reservedPhysicalMemory);
    long maxRssMemoryAllowedForAllTasks = totalPhysicalMemory -
                                          reservedPhysicalMemory * 1024 * 1024L;
 
View Full Code Here
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
    if (!System.getProperty("os.name").startsWith("Linux")) {
      return;
    }

    JobConf conf = new JobConf();
    LinuxResourceCalculatorPlugin plugin = new LinuxResourceCalculatorPlugin();
    // In this case, we only check these three fields because they are static
    conf.setLong("totalVmemOnTT", plugin.getVirtualMemorySize());
    conf.setLong("totalPmemOnTT", plugin.getPhysicalMemorySize());
    conf.setLong("numProcessors", plugin.getNumProcessors());

    try {
      setUpCluster(conf);
      runSleepJob(miniMRCluster.createJobConf());
      verifyTestResults(true);
View Full Code Here
541
542
543
544
545
546
547
548
549
550
551
552
553
    // very small value, so that no task escapes to successful completion.
    fConf.setInt(TTConfig.TT_MEMORY_MANAGER_MONITORING_INTERVAL, 100);
   
    // reserve all memory on TT so that the job will exceed memory limits
    LinuxResourceCalculatorPlugin memoryCalculatorPlugin =
            new LinuxResourceCalculatorPlugin();
    long totalPhysicalMemory = memoryCalculatorPlugin.getPhysicalMemorySize();
    long reservedPhysicalMemory = totalPhysicalMemory / (1024 * 1024) + 1;
    fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB,
                  reservedPhysicalMemory);
    long maxRssMemoryAllowedForAllTasks = totalPhysicalMemory -
                                          reservedPhysicalMemory * 1024 * 1024L;
 
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.util.LinuxResourceCalculatorPlugin

  • org.apache.hadoop.mapred.TestTaskTrackerMemoryManager
  • org.apache.hadoop.mapred.TestTTResourceReporting
  • java.io.InputStreamReader
  • java.io.FileInputStream
  • java.io.FileReader
  • java.io.BufferedReader
  • java.util.regex.Matcher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.