Examples of org.apache.hadoop.mapreduce.Job

org.apache.hadoop.mapreduce.Job
The job submitter's view of the Job. It allows the user to configure the job, submit it, control its execution, and query the state. The set methods only work until the job is submitted, afterwards they will throw an IllegalStateException.

    conf.set(STATE_IN_KEY, stateIn);
    conf.set(NUM_TOPICS_KEY, Integer.toString(numTopics));
    conf.set(NUM_WORDS_KEY, Integer.toString(numWords));
    conf.set(TOPIC_SMOOTHING_KEY, Double.toString(topicSmoothing));
    
    Job job = new Job(conf);
    
    job.setOutputKeyClass(IntPairWritable.class);
    job.setOutputValueClass(DoubleWritable.class);
    FileInputFormat.addInputPaths(job, input);
    Path outPath = new Path(stateOut);
    FileOutputFormat.setOutputPath(job, outPath);
    
    job.setMapperClass(LDAMapper.class);
    job.setReducerClass(LDAReducer.class);
    job.setCombinerClass(LDAReducer.class);
    job.setNumReduceTasks(numReducers);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    job.setInputFormatClass(SequenceFileInputFormat.class);
    job.setJarByClass(LDADriver.class);
    
    job.waitForCompletion(true);
    return findLL(stateOut, conf);
  }

View Full Code Here

  public static Job createSubmittableJob(Configuration conf, String[] args)
  throws IOException {
    if (!doCommandLine(args)) {
      return null;
    }
    Job job = new Job(conf, NAME + "_" + tableName);
    job.setJarByClass(CopyTable.class);
    Scan scan = new Scan();
    scan.setCacheBlocks(false);
    if (startTime != 0) {
      scan.setTimeRange(startTime,
          endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
    }
    if (allCells) {
      scan.setRaw(true);
    }
    if (versions >= 0) {
      scan.setMaxVersions(versions);
    }
    
    if (startRow != null) {
      scan.setStartRow(Bytes.toBytes(startRow));
    }
    
    if (stopRow != null) {
      scan.setStopRow(Bytes.toBytes(stopRow));
    }
    
    if(families != null) {
      String[] fams = families.split(",");
      Map<String,String> cfRenameMap = new HashMap<String,String>();
      for(String fam : fams) {
        String sourceCf;
        if(fam.contains(":")) { 
            // fam looks like "sourceCfName:destCfName"
            String[] srcAndDest = fam.split(":", 2);
            sourceCf = srcAndDest[0];
            String destCf = srcAndDest[1];
            cfRenameMap.put(sourceCf, destCf);
        } else {
            // fam is just "sourceCf"
            sourceCf = fam; 
        }
        scan.addFamily(Bytes.toBytes(sourceCf));
      }
      Import.configureCfRenaming(job.getConfiguration(), cfRenameMap);
    }
    TableMapReduceUtil.initTableMapperJob(tableName, scan,
        Import.Importer.class, null, null, job);
    TableMapReduceUtil.initTableReducerJob(
        newTableName == null ? tableName : newTableName, null, job,
        null, peerAddress, null, null);
    job.setNumReduceTasks(0);
    return job;
  }

View Full Code Here

  }


  @Override
  public int run(String[] args) throws Exception {
    String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
    Job job = createSubmittableJob(getConf(), otherArgs);
    if (job == null) return 1;
    return job.waitForCompletion(true) ? 0 : 1;
  }

View Full Code Here

    Class mapperClass = mapperClassName != null ?
        Class.forName(mapperClassName) : DEFAULT_MAPPER;


    String tableName = args[0];
    Path inputDir = new Path(args[1]);
    Job job = new Job(conf, NAME + "_" + tableName);
    job.setJarByClass(mapperClass);
    FileInputFormat.setInputPaths(job, inputDir);
    job.setInputFormatClass(TextInputFormat.class);
    job.setMapperClass(mapperClass);


    String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY);
    if (hfileOutPath != null) {
      if (!doesTableExist(tableName)) {
        createTable(conf, tableName);
      }
      HTable table = new HTable(conf, tableName);
      job.setReducerClass(PutSortReducer.class);
      Path outputDir = new Path(hfileOutPath);
      FileOutputFormat.setOutputPath(job, outputDir);
      job.setMapOutputKeyClass(ImmutableBytesWritable.class);
      job.setMapOutputValueClass(Put.class);
      HFileOutputFormat.configureIncrementalLoad(job, table);
    } else {
      // No reducers.  Just write straight to table.  Call initTableReducerJob
      // to set up the TableOutputFormat.
      TableMapReduceUtil.initTableReducerJob(tableName, null, job);
      job.setNumReduceTasks(0);
    }


    TableMapReduceUtil.addDependencyJars(job);
    TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
        com.google.common.base.Function.class /* Guava used by TsvParser */);
    return job;
  }

View Full Code Here

    conf.setBoolean("mapreduce.map.speculative", false);
    conf.setBoolean("mapreduce.reduce.speculative", false);
    conf.setBoolean("mapred.map.tasks.speculative.execution", false);
    conf.setBoolean("mapred.reduce.tasks.speculative.execution", false);


    Job job = new Job(conf);
    job.setJobName("ExportSnapshot");
    job.setJarByClass(ExportSnapshot.class);
    job.setMapperClass(ExportMapper.class);
    job.setInputFormatClass(SequenceFileInputFormat.class);
    job.setOutputFormatClass(NullOutputFormat.class);
    job.setNumReduceTasks(0);
    for (Path path: createInputFiles(conf, snapshotFiles, mappers)) {
      LOG.debug("Add Input Path=" + path);
      SequenceFileInputFormat.addInputPath(job, path);
    }


    // Run the MR Job
    if (!job.waitForCompletion(true)) {
      // TODO: Replace the fixed string with job.getStatus().getFailureInfo()
      // when it will be available on all the supported versions.
      throw new ExportSnapshotException("Copy Files Map-Reduce Job failed");
    }
  }

View Full Code Here

      System.err.println("  job. By default, includes any jars detected by TableMapReduceUtils. Provide");
      System.err.println("  additional entries by specifying a comma-separated list in tmpjars.");
      return 0;
    }


    Job job = new Job(getConf());
    TableMapReduceUtil.addDependencyJars(job);
    System.out.println(TableMapReduceUtil.buildDependencyClasspath(job.getConfiguration()));
    return 0;
  }

View Full Code Here

        }
        
        // Pass signature of the loader to rightLoader
        // make a copy of the conf to use in calls to rightLoader.
        rightLoader.setUDFContextSignature(signature);
        Job job = new Job(new Configuration(PigMapReduce.sJobConf));
        rightLoader.setLocation(rightInputFileName, job);
        ((IndexableLoadFunc)rightLoader).initialize(job.getConfiguration());
        ((IndexableLoadFunc)rightLoader).seekNear(
                firstLeftKey instanceof Tuple ? (Tuple)firstLeftKey : mTupleFactory.newTuple(firstLeftKey));
    }

View Full Code Here


        for(int i=0; i < relationCnt-1; i ++){


            LoadFunc loadfunc = (LoadFunc)PigContext.instantiateFuncFromSpec(sidFuncSpecs.get(i));
            loadfunc.setUDFContextSignature(loaderSignatures.get(i));
            Job dummyJob = new Job(new Configuration(PigMapReduce.sJobConf));
            loadfunc.setLocation(sideFileSpecs.get(i), dummyJob);
            ((IndexableLoadFunc)loadfunc).initialize(dummyJob.getConfiguration());
            sideLoaders.add(loadfunc);
            Tuple rearranged;


            if ( index.get(0).first.equals(curSplitIdx)){ 
                // This is a first split, bind at very first record in all side relations.

View Full Code Here

    // Set it back to replace invalid timestamp (non-numeric) with current
    // system time
    conf.setLong(TIMESTAMP_CONF_KEY, timstamp); 
    
    hbaseAdmin = new HBaseAdmin(conf);
    Job job = createSubmittableJob(conf, otherArgs);
    System.exit(job.waitForCompletion(true) ? 0 : 1);
  }

View Full Code Here

        sb.append(args[i]);
        sb.append(" ");
      }
    }


    Job job = new Job(conf, NAME + "_" + tableName);
    job.setJarByClass(RowCounter.class);
    Scan scan = new Scan();
    scan.setCacheBlocks(false);
    if (startKey != null && !startKey.equals("")) {
      scan.setStartRow(Bytes.toBytes(startKey));
    }
    if (endKey != null && !endKey.equals("")) {
      scan.setStopRow(Bytes.toBytes(endKey));
    }
    scan.setFilter(new FirstKeyOnlyFilter());
    if (sb.length() > 0) {
      for (String columnName : sb.toString().trim().split(" ")) {
        String family = StringUtils.substringBefore(columnName, ":");
        String qualifier = StringUtils.substringAfter(columnName, ":");
        if (StringUtils.isBlank(qualifier)) {
          scan.addFamily(Bytes.toBytes(family));
        } else {
          scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
        }
      }
    }
    job.setOutputFormatClass(NullOutputFormat.class);
    TableMapReduceUtil.initTableMapperJob(tableName, scan,
      RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job);
    job.setNumReduceTasks(0);
    return job;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.Job

bulkimport.BulkImportJobExample

co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService

com.alimama.quanjingmonitor.kmeans.KMeansDriver

com.asakusafw.runtime.stage.AbstractStageClient

com.asakusafw.thundergate.runtime.cache.mapreduce.CacheBuildClient

com.datasalt.pangool.tuplemr.mapred.lib.output.PangoolMultipleOutputs

com.datasalt.pangool.tuplemr.mapred.lib.output.TestTupleTextInputOutputFormat

com.datasalt.pangool.tuplemr.mapred.TestRollup

com.datasalt.pangool.tuplemr.mapred.TestTupleMRJob

com.hadoop.compression.lzo.DistributedLzoIndexer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.