Examples of GenericOptionsParser

org.apache.hadoop.util.GenericOptionsParser

GenericOptionsParser is a utility to parse command line arguments generic to the Hadoop framework. GenericOptionsParser recognizes several standarad command line arguments, enabling applications to easily specify a namenode, a jobtracker, additional configuration resources etc.

Generic Options

The supported generic options are:

 -conf <configuration file>     specify a configuration file -D <property=value>            use value for given property -fs <local|namenode:port>      specify a namenode -jt <local|jobtracker:port>    specify a job tracker -files <comma separated list of files>    specify comma separated files to be copied to the map reduce cluster -libjars <comma separated list of jars>   specify comma separated jar files to include in the classpath. -archives <comma separated list of archives>    specify comma separated archives to be unarchived on the compute machines.

The general command line syntax is:

 bin/hadoop command [genericOptions] [commandOptions]

Generic command line arguments might modify Configuration objects, given to constructors.

The functionality is implemented using Commons CLI.

Examples:

 $ bin/hadoop dfs -fs darwin:8020 -ls /data list /data directory in dfs with namenode darwin:8020 $ bin/hadoop dfs -D fs.default.name=darwin:8020 -ls /data list /data directory in dfs with namenode darwin:8020 $ bin/hadoop dfs -conf hadoop-site.xml -ls /data list /data directory in dfs with conf specified in hadoop-site.xml $ bin/hadoop job -D mapred.job.tracker=darwin:50020 -submit job.xml submit a job to job tracker darwin:50020 $ bin/hadoop job -jt darwin:50020 -submit job.xml submit a job to job tracker darwin:50020 $ bin/hadoop job -jt local -submit job.xml submit a job to local runner $ bin/hadoop jar -libjars testlib.jar  -archives test.tgz -files file.txt inputjar args job submission with libjars, files and archives

@see Tool @see ToolRunner

Examples of org.apache.hadoop.util.GenericOptionsParser

   * @param args  The command line parameters.
   * @throws Exception When running the job fails.
   */
  public static void main(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 1) {
      printUsage("Wrong number of parameters: " + args.length);
      System.exit(-1);
    }
    Job job = createSubmittableJob(conf, otherArgs);

View Full Code Here

Examples of org.apache.hadoop.util.GenericOptionsParser

    boolean verbose = false;
    boolean gruntCalled = false;
    String logFileName = null;
    try {
        Configuration conf = new Configuration(false);
        GenericOptionsParser parser = new GenericOptionsParser(conf, args);
        conf = parser.getConfiguration();    
        
        Properties properties = new Properties();
        PropertiesUtil.loadDefaultProperties(properties);
        properties.putAll(ConfigurationUtil.toProperties(conf));
        
        String[] pigArgs = parser.getRemainingArgs();
        
        
        boolean userSpecifiedLog = false;
        
        boolean checkScriptOnly = false;

View Full Code Here

Examples of org.apache.hadoop.util.GenericOptionsParser

  }


  @Override
  public int run(String[] args) throws Exception {
    setConf(HBaseConfiguration.create(getConf()));
    String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
    if (otherArgs.length < 2) {
      usage("Wrong number of arguments: " + otherArgs.length);
      return -1;
    }

View Full Code Here

Examples of org.apache.hadoop.util.GenericOptionsParser

   * @param args  The command line parameters.
   * @throws Exception When running the job fails.
   */
  public static void main(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
      usage("Wrong number of arguments: " + otherArgs.length);
      System.exit(-1);
    }
    Job job = createSubmittableJob(conf, otherArgs);

View Full Code Here

Examples of org.apache.hadoop.util.GenericOptionsParser

    Iface getClient(String controllerConnectionStr);
  }


  public static void main(String... args) throws Exception {
    Configuration configuration = new Configuration();
    String[] otherArgs = new GenericOptionsParser(configuration, args).getRemainingArgs();
    Job job = setupJob(configuration, new ControllerPool() {
      @Override
      public Iface getClient(String controllerConnectionStr) {
        return BlurClient.getClient(controllerConnectionStr);
      }

View Full Code Here

Examples of org.apache.hadoop.util.GenericOptionsParser

   * @param args  The command line parameters.
   * @throws Exception When running the job fails.
   */
  public static void main(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
      usage("Wrong number of arguments: " + otherArgs.length);
      System.exit(-1);
    }
    Job job = createSubmittableJob(conf, otherArgs);

View Full Code Here

Examples of org.apache.hadoop.util.GenericOptionsParser

    String[] args = new String[] {
        EXPORT_TABLE,
        OUTPUT_DIR,
        "1000"
    };
    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());




    String IMPORT_TABLE = "importTableSimpleCase";
    t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
    args = new String[] {
        "-D" + Import.CF_RENAME_PROP + "="+FAMILYA_STRING+":"+FAMILYB_STRING,
        IMPORT_TABLE,
        OUTPUT_DIR
    };


    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());

View Full Code Here

Examples of org.apache.hadoop.util.GenericOptionsParser

   */
  @Test
  public void testMetaExport() throws Exception {
    String EXPORT_TABLE = ".META.";
    String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
        UTIL.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());

View Full Code Here

Examples of org.apache.hadoop.util.GenericOptionsParser

        EXPORT_TABLE,
        OUTPUT_DIR,
        "1000"
    };


    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());




    String IMPORT_TABLE = "importWithDeletes";
    desc = new HTableDescriptor(IMPORT_TABLE);
    desc.addFamily(new HColumnDescriptor(FAMILYA)
        .setMaxVersions(5)
        .setKeepDeletedCells(true)
    );
    UTIL.getHBaseAdmin().createTable(desc);
    t.close();
    t = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
    args = new String[] {
        IMPORT_TABLE,
        OUTPUT_DIR
    };


    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());

View Full Code Here

Examples of org.apache.hadoop.util.GenericOptionsParser

    p.add(FAMILYA, QUAL, now + 4, QUAL);
    exportTable.put(p);


    String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };


    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
        UTIL.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());


    String IMPORT_TABLE = "importWithFilter";
    desc = new HTableDescriptor(IMPORT_TABLE);
    desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
    UTIL.getHBaseAdmin().createTable(desc);


    HTable importTable = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
    args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + PrefixFilter.class.getName(),
        "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
        "1000" };


    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());


    // get the count of the source table for that time range
    PrefixFilter filter = new PrefixFilter(ROW1);
    int count = getCount(exportTable, filter);


    Assert.assertEquals("Unexpected row count between export and import tables", count,
      getCount(importTable, null));


    // and then test that a broken command doesn't bork everything - easier here because we don't
    // need to re-run the export job


    args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + Filter.class.getName(),
        "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
        OUTPUT_DIR, "1000" };


    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.