Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.GenericOptionsParser


   * @param args  The command line parameters.
   * @throws Exception When running the job fails.
   */
  public static void main(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 1) {
      printUsage("Wrong number of parameters: " + args.length);
      System.exit(-1);
    }
    Job job = createSubmittableJob(conf, otherArgs);
View Full Code Here


    boolean verbose = false;
    boolean gruntCalled = false;
    String logFileName = null;
    try {
        Configuration conf = new Configuration(false);
        GenericOptionsParser parser = new GenericOptionsParser(conf, args);
        conf = parser.getConfiguration();   
       
        Properties properties = new Properties();
        PropertiesUtil.loadDefaultProperties(properties);
        properties.putAll(ConfigurationUtil.toProperties(conf));
       
        String[] pigArgs = parser.getRemainingArgs();
       
       
        boolean userSpecifiedLog = false;
       
        boolean checkScriptOnly = false;
View Full Code Here

  }

  @Override
  public int run(String[] args) throws Exception {
    setConf(HBaseConfiguration.create(getConf()));
    String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
    if (otherArgs.length < 2) {
      usage("Wrong number of arguments: " + otherArgs.length);
      return -1;
    }
View Full Code Here

   * @param args  The command line parameters.
   * @throws Exception When running the job fails.
   */
  public static void main(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
      usage("Wrong number of arguments: " + otherArgs.length);
      System.exit(-1);
    }
    Job job = createSubmittableJob(conf, otherArgs);
View Full Code Here

    Iface getClient(String controllerConnectionStr);
  }

  public static void main(String... args) throws Exception {
    Configuration configuration = new Configuration();
    String[] otherArgs = new GenericOptionsParser(configuration, args).getRemainingArgs();
    Job job = setupJob(configuration, new ControllerPool() {
      @Override
      public Iface getClient(String controllerConnectionStr) {
        return BlurClient.getClient(controllerConnectionStr);
      }
View Full Code Here

   * @param args  The command line parameters.
   * @throws Exception When running the job fails.
   */
  public static void main(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
      usage("Wrong number of arguments: " + otherArgs.length);
      System.exit(-1);
    }
    Job job = createSubmittableJob(conf, otherArgs);
View Full Code Here

    String[] args = new String[] {
        EXPORT_TABLE,
        OUTPUT_DIR,
        "1000"
    };
    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();

    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());


    String IMPORT_TABLE = "importTableSimpleCase";
    t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
    args = new String[] {
        "-D" + Import.CF_RENAME_PROP + "="+FAMILYA_STRING+":"+FAMILYB_STRING,
        IMPORT_TABLE,
        OUTPUT_DIR
    };

    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();

    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());
View Full Code Here

   */
  @Test
  public void testMetaExport() throws Exception {
    String EXPORT_TABLE = ".META.";
    String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
        UTIL.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();

    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());
View Full Code Here

        EXPORT_TABLE,
        OUTPUT_DIR,
        "1000"
    };

    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();

    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());


    String IMPORT_TABLE = "importWithDeletes";
    desc = new HTableDescriptor(IMPORT_TABLE);
    desc.addFamily(new HColumnDescriptor(FAMILYA)
        .setMaxVersions(5)
        .setKeepDeletedCells(true)
    );
    UTIL.getHBaseAdmin().createTable(desc);
    t.close();
    t = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
    args = new String[] {
        IMPORT_TABLE,
        OUTPUT_DIR
    };

    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();

    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());
View Full Code Here

    p.add(FAMILYA, QUAL, now + 4, QUAL);
    exportTable.put(p);

    String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };

    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
        UTIL.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();

    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());

    String IMPORT_TABLE = "importWithFilter";
    desc = new HTableDescriptor(IMPORT_TABLE);
    desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
    UTIL.getHBaseAdmin().createTable(desc);

    HTable importTable = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
    args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + PrefixFilter.class.getName(),
        "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
        "1000" };

    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();

    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());

    // get the count of the source table for that time range
    PrefixFilter filter = new PrefixFilter(ROW1);
    int count = getCount(exportTable, filter);

    Assert.assertEquals("Unexpected row count between export and import tables", count,
      getCount(importTable, null));

    // and then test that a broken command doesn't bork everything - easier here because we don't
    // need to re-run the export job

    args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + Filter.class.getName(),
        "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
        OUTPUT_DIR, "1000" };

    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();

    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());
View Full Code Here

TOP

Related Classes of org.apache.hadoop.util.GenericOptionsParser

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.