Examples of SqoopOptions


Examples of com.cloudera.sqoop.SqoopOptions


  @Override
  protected void propagateOptionsToJob(Job job) {
    super.propagateOptionsToJob(job);
    SqoopOptions opts = context.getOptions();
    Configuration conf = job.getConfiguration();
    conf.setIfUnset("pgbulkload.bin", "pg_bulkload");
    if (opts.getNullStringValue() != null) {
      conf.set("pgbulkload.null.string", opts.getNullStringValue());
    }
    setDelimiter("pgbulkload.input.field.delim",
                 opts.getInputFieldDelim(),
                 conf);
    setDelimiter("pgbulkload.input.record.delim",
                 opts.getInputRecordDelim(),
                 conf);
    setDelimiter("pgbulkload.input.enclosedby",
                 opts.getInputEnclosedBy(),
                 conf);
    setDelimiter("pgbulkload.input.escapedby",
                 opts.getInputEscapedBy(),
                 conf);
    conf.setBoolean("pgbulkload.input.encloserequired",
                    opts.isInputEncloseRequired());
    conf.setIfUnset("pgbulkload.check.constraints", "YES");
    conf.setIfUnset("pgbulkload.parse.errors", "INFINITE");
    conf.setIfUnset("pgbulkload.duplicate.errors", "INFINITE");
    conf.set("mapred.jar", context.getJarFile());
    conf.setBoolean("mapred.map.tasks.speculative.execution", false);
View Full Code Here

Examples of com.cloudera.sqoop.SqoopOptions

    if (null == data) {
      LOG.error("No such job: " + jobName);
      return 1;
    }

    SqoopOptions childOpts = data.getSqoopOptions();
    SqoopTool childTool = data.getSqoopTool();

    // Don't overwrite the original SqoopOptions with the
    // arguments; make a child options.

    SqoopOptions clonedOpts = (SqoopOptions) childOpts.clone();
    clonedOpts.setParent(childOpts);

    int dashPos = getDashPosition(extraArguments);
    String [] childArgv;
    if (dashPos >= extraArguments.length) {
      childArgv = new String[0];
View Full Code Here

Examples of com.cloudera.sqoop.SqoopOptions

    if (null == data) {
      LOG.error("No such job: " + jobName);
      return 1;
    }

    SqoopOptions childOpts = data.getSqoopOptions();
    SqoopTool childTool = data.getSqoopTool();

    System.out.println("Job: " + jobName);
    System.out.println("Tool: " + childTool.getToolName());

    System.out.println("Options:");
    System.out.println("----------------------------");
    Properties props = childOpts.writeProperties();
    for (Map.Entry<Object, Object> entry : props.entrySet()) {
      System.out.println(entry.getKey().toString() + " = " + entry.getValue());
    }

    // TODO: This does not show entries in the Configuration
View Full Code Here

Examples of com.cloudera.sqoop.SqoopOptions


  @Override
  public void runExport() throws ExportException, IOException {
    ConnManager cmgr = context.getConnManager();
    SqoopOptions options = context.getOptions();
    Configuration conf = options.getConf();
    DBConfiguration dbConf = null;
    String outputTableName = context.getTableName();
    String tableName = outputTableName;
    String tableClassName =
        new TableClassName(options).getClassForTable(outputTableName);
View Full Code Here

Examples of com.cloudera.sqoop.SqoopOptions

  }


  @Override
  protected int configureNumTasks(Job job) throws IOException {
    SqoopOptions options = context.getOptions();
    int numMapTasks = options.getNumMappers();
    if (numMapTasks < 1) {
      numMapTasks = SqoopOptions.DEFAULT_NUM_MAPPERS;
      LOG.warn("Invalid mapper count; using " + numMapTasks + " mappers.");
    }
View Full Code Here

Examples of com.cloudera.sqoop.SqoopOptions

    int ret;
    List<String> generatedJars = null;
    try {
      ExportTool exporter = new ExportTool();
      Configuration conf = getConf();
      SqoopOptions opts = getSqoopOptions(conf);
      Sqoop sqoop = new Sqoop(exporter, conf, opts);
      ret = Sqoop.runSqoop(sqoop, argv);
      generatedJars = exporter.getGeneratedJarFiles();
    } catch (Exception e) {
      LOG.error("Got exception running Sqoop: "
View Full Code Here

Examples of com.cloudera.sqoop.SqoopOptions

  @Before
  public void setUp() {
    super.setUp();

    SqoopOptions options = new SqoopOptions(CONNECT_STRING,
      DBO_TABLE_NAME);
    options.setUsername(DATABASE_USER);
    options.setPassword(DATABASE_PASSWORD);

    manager = new SQLServerManager(options);

    createTableAndPopulateData(SCHEMA_DBO, DBO_TABLE_NAME);
    createTableAndPopulateData(SCHEMA_SCH, SCH_TABLE_NAME);
View Full Code Here

Examples of com.cloudera.sqoop.SqoopOptions

      String newQuery = options.getSqlQuery().replace(
        "$CONDITIONS", sb.toString());
      options.setSqlQuery(newQuery);
    }
    // Save this state for next time.
    SqoopOptions recordOptions = options.getParent();
    if (null == recordOptions) {
      recordOptions = options;
    }
    recordOptions.setIncrementalLastValue(
        (nextVal == null) ? null : nextVal.toString());

    return true;
  }
View Full Code Here

Examples of com.cloudera.sqoop.SqoopOptions

  @Before
  public void setUp() {
    super.setUp();

    SqoopOptions options = new SqoopOptions(CONNECT_STRING, getTableName());
    options.setUsername(DATABASE_USER);
    options.setPassword(DATABASE_PASSWORD);

    manager = new Db2Manager(options);

    // Drop the existing table, if there is one.
    Connection conn = null;
View Full Code Here

Examples of com.cloudera.sqoop.SqoopOptions

   * @throws ExportException if the job fails unexpectedly or is misconfigured.
   */
  public void runExport() throws ExportException, IOException {

    ConnManager cmgr = context.getConnManager();
    SqoopOptions options = context.getOptions();
    Configuration conf = options.getConf();

    String outputTableName = context.getTableName();
    String stagingTableName = context.getOptions().getStagingTableName();

    String tableName = outputTableName;
    boolean stagingEnabled = false;
    if (stagingTableName != null) { // user has specified the staging table
      if (cmgr.supportsStagingForExport()) {
        LOG.info("Data will be staged in the table: " + stagingTableName);
        tableName = stagingTableName;
        stagingEnabled = true;
      } else {
        throw new ExportException("The active connection manager ("
            + cmgr.getClass().getCanonicalName()
            + ") does not support staging of data for export. "
            + "Please retry without specifying the --staging-table option.");
      }
    }

    String tableClassName =
        new TableClassName(options).getClassForTable(outputTableName);
    String ormJarFile = context.getJarFile();

    LOG.info("Beginning export of " + outputTableName);
    loadJars(conf, ormJarFile, tableClassName);

    if (stagingEnabled) {
      // Prepare the staging table
      if (options.doClearStagingTable()) {
        try {
          // Delete all records from staging table
          cmgr.deleteAllRecords(stagingTableName);
        } catch (SQLException ex) {
          throw new ExportException(
              "Failed to empty staging table before export run", ex);
        }
      } else {
        // User has not explicitly specified the clear staging table option.
        // Assert that the staging table is empty.
        try {
          long rowCount = cmgr.getTableRowCount(stagingTableName);
          if (rowCount != 0L) {
            throw new ExportException("The specified staging table ("
                + stagingTableName + ") is not empty. To force deletion of "
                + "its data, please retry with --clear-staging-table option.");
          }
        } catch (SQLException ex) {
          throw new ExportException(
              "Failed to count data rows in staging table: "
                  + stagingTableName, ex);
        }
      }
    }

    Job job = new Job(conf);
    try {
      // Set the external jar to use for the job.
      job.getConfiguration().set("mapred.jar", ormJarFile);
      if (options.getMapreduceJobName() != null) {
        job.setJobName(options.getMapreduceJobName());
      }

      propagateOptionsToJob(job);
      configureInputFormat(job, tableName, tableClassName, null);
      configureOutputFormat(job, tableName, tableClassName);
      configureMapper(job, tableName, tableClassName);
      configureNumTasks(job);
      cacheJars(job, context.getConnManager());

      jobSetup(job);
      setJob(job);
      boolean success = runJob(job);
      if (!success) {
        throw new ExportException("Export job failed!");
      }

      if (options.isValidationEnabled()) {
        validateExport(tableName, conf, job);
      }
    } catch (InterruptedException ie) {
      throw new IOException(ie);
    } catch (ClassNotFoundException cnfe) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.