/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.tool;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.Properties;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Category;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import com.cloudera.sqoop.ConnFactory;
import com.cloudera.sqoop.Sqoop;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
import com.cloudera.sqoop.cli.RelatedOptions;
import com.cloudera.sqoop.cli.ToolOptions;
import com.cloudera.sqoop.lib.DelimiterSet;
import com.cloudera.sqoop.manager.ConnManager;
import com.cloudera.sqoop.metastore.JobData;
/**
* Layer on top of SqoopTool that provides some basic common code
* that most SqoopTool implementations will use.
*
* Subclasses should call init() at the top of their run() method,
* and call destroy() at the end in a finally block.
*/
public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
public static final Log LOG = LogFactory.getLog(
BaseSqoopTool.class.getName());
public static final String HELP_STR = "\nTry --help for usage instructions.";
// Here are all the arguments that are used by the standard sqoop tools.
// Their names are recorded here so that tools can share them and their
// use consistently. The argument parser applies the leading '--' to each
// string.
public static final String CONNECT_STRING_ARG = "connect";
public static final String CONN_MANAGER_CLASS_NAME =
"connection-manager";
public static final String CONNECT_PARAM_FILE = "connection-param-file";
public static final String DRIVER_ARG = "driver";
public static final String USERNAME_ARG = "username";
public static final String PASSWORD_ARG = "password";
public static final String PASSWORD_PROMPT_ARG = "P";
public static final String DIRECT_ARG = "direct";
public static final String BATCH_ARG = "batch";
public static final String TABLE_ARG = "table";
public static final String STAGING_TABLE_ARG = "staging-table";
public static final String CLEAR_STAGING_TABLE_ARG = "clear-staging-table";
public static final String COLUMNS_ARG = "columns";
public static final String SPLIT_BY_ARG = "split-by";
public static final String WHERE_ARG = "where";
public static final String HADOOP_HOME_ARG = "hadoop-home";
public static final String HIVE_HOME_ARG = "hive-home";
public static final String WAREHOUSE_DIR_ARG = "warehouse-dir";
public static final String TARGET_DIR_ARG = "target-dir";
public static final String APPEND_ARG = "append";
public static final String NULL_STRING = "null-string";
public static final String INPUT_NULL_STRING = "input-null-string";
public static final String NULL_NON_STRING = "null-non-string";
public static final String INPUT_NULL_NON_STRING = "input-null-non-string";
public static final String MAP_COLUMN_JAVA = "map-column-java";
public static final String MAP_COLUMN_HIVE = "map-column-hive";
public static final String FMT_SEQUENCEFILE_ARG = "as-sequencefile";
public static final String FMT_TEXTFILE_ARG = "as-textfile";
public static final String FMT_AVRODATAFILE_ARG = "as-avrodatafile";
public static final String HIVE_IMPORT_ARG = "hive-import";
public static final String HIVE_TABLE_ARG = "hive-table";
public static final String HIVE_OVERWRITE_ARG = "hive-overwrite";
public static final String HIVE_DROP_DELIMS_ARG = "hive-drop-import-delims";
public static final String HIVE_DELIMS_REPLACEMENT_ARG =
"hive-delims-replacement";
public static final String HIVE_PARTITION_KEY_ARG = "hive-partition-key";
public static final String HIVE_PARTITION_VALUE_ARG = "hive-partition-value";
public static final String CREATE_HIVE_TABLE_ARG =
"create-hive-table";
public static final String NUM_MAPPERS_ARG = "num-mappers";
public static final String NUM_MAPPERS_SHORT_ARG = "m";
public static final String COMPRESS_ARG = "compress";
public static final String COMPRESSION_CODEC_ARG = "compression-codec";
public static final String COMPRESS_SHORT_ARG = "z";
public static final String DIRECT_SPLIT_SIZE_ARG = "direct-split-size";
public static final String INLINE_LOB_LIMIT_ARG = "inline-lob-limit";
public static final String FETCH_SIZE_ARG = "fetch-size";
public static final String EXPORT_PATH_ARG = "export-dir";
public static final String FIELDS_TERMINATED_BY_ARG = "fields-terminated-by";
public static final String LINES_TERMINATED_BY_ARG = "lines-terminated-by";
public static final String OPTIONALLY_ENCLOSED_BY_ARG =
"optionally-enclosed-by";
public static final String ENCLOSED_BY_ARG = "enclosed-by";
public static final String ESCAPED_BY_ARG = "escaped-by";
public static final String MYSQL_DELIMITERS_ARG = "mysql-delimiters";
public static final String INPUT_FIELDS_TERMINATED_BY_ARG =
"input-fields-terminated-by";
public static final String INPUT_LINES_TERMINATED_BY_ARG =
"input-lines-terminated-by";
public static final String INPUT_OPTIONALLY_ENCLOSED_BY_ARG =
"input-optionally-enclosed-by";
public static final String INPUT_ENCLOSED_BY_ARG = "input-enclosed-by";
public static final String INPUT_ESCAPED_BY_ARG = "input-escaped-by";
public static final String CODE_OUT_DIR_ARG = "outdir";
public static final String BIN_OUT_DIR_ARG = "bindir";
public static final String PACKAGE_NAME_ARG = "package-name";
public static final String CLASS_NAME_ARG = "class-name";
public static final String JAR_FILE_NAME_ARG = "jar-file";
public static final String SQL_QUERY_ARG = "query";
public static final String SQL_QUERY_BOUNDARY = "boundary-query";
public static final String SQL_QUERY_SHORT_ARG = "e";
public static final String VERBOSE_ARG = "verbose";
public static final String HELP_ARG = "help";
public static final String UPDATE_KEY_ARG = "update-key";
public static final String UPDATE_MODE_ARG = "update-mode";
// Arguments for incremental imports.
public static final String INCREMENT_TYPE_ARG = "incremental";
public static final String INCREMENT_COL_ARG = "check-column";
public static final String INCREMENT_LAST_VAL_ARG = "last-value";
// HBase arguments.
public static final String HBASE_TABLE_ARG = "hbase-table";
public static final String HBASE_COL_FAM_ARG = "column-family";
public static final String HBASE_ROW_KEY_ARG = "hbase-row-key";
public static final String HBASE_CREATE_TABLE_ARG = "hbase-create-table";
// Arguments for the saved job management system.
public static final String STORAGE_METASTORE_ARG = "meta-connect";
public static final String JOB_CMD_CREATE_ARG = "create";
public static final String JOB_CMD_DELETE_ARG = "delete";
public static final String JOB_CMD_EXEC_ARG = "exec";
public static final String JOB_CMD_LIST_ARG = "list";
public static final String JOB_CMD_SHOW_ARG = "show";
// Arguments for the metastore.
public static final String METASTORE_SHUTDOWN_ARG = "shutdown";
// Arguments for merging datasets.
public static final String NEW_DATASET_ARG = "new-data";
public static final String OLD_DATASET_ARG = "onto";
public static final String MERGE_KEY_ARG = "merge-key";
public BaseSqoopTool() {
}
public BaseSqoopTool(String toolName) {
super(toolName);
}
protected ConnManager manager;
public ConnManager getManager() {
return manager;
}
public void setManager(ConnManager mgr) {
this.manager = mgr;
}
/**
* Should be called at the beginning of the run() method to initialize
* the connection manager, etc. If this succeeds (returns true), it should
* be paired with a call to destroy().
* @return true on success, false on failure.
*/
protected boolean init(SqoopOptions sqoopOpts) {
// Get the connection to the database.
try {
JobData data = new JobData(sqoopOpts, this);
this.manager = new ConnFactory(sqoopOpts.getConf()).getManager(data);
return true;
} catch (Exception e) {
LOG.error("Got error creating database manager: "
+ StringUtils.stringifyException(e));
if (System.getProperty(Sqoop.SQOOP_RETHROW_PROPERTY) != null) {
throw new RuntimeException(e);
}
}
return false;
}
/**
* Should be called in a 'finally' block at the end of the run() method.
*/
protected void destroy(SqoopOptions sqoopOpts) {
if (null != manager) {
try {
manager.close();
} catch (SQLException sqlE) {
LOG.warn("Error while closing connection: " + sqlE);
}
}
}
/**
* Examines a subset of the arrray presented, and determines if it
* contains any non-empty arguments. If so, logs the arguments
* and returns true.
*
* @param argv an array of strings to check.
* @param offset the first element of the array to check
* @param len the number of elements to check
* @return true if there are any non-null, non-empty argument strings
* present.
*/
protected boolean hasUnrecognizedArgs(String [] argv, int offset, int len) {
if (argv == null) {
return false;
}
boolean unrecognized = false;
boolean printedBanner = false;
for (int i = offset; i < Math.min(argv.length, offset + len); i++) {
if (argv[i] != null && argv[i].length() > 0) {
if (!printedBanner) {
LOG.error("Error parsing arguments for " + getToolName() + ":");
printedBanner = true;
}
LOG.error("Unrecognized argument: " + argv[i]);
unrecognized = true;
}
}
return unrecognized;
}
protected boolean hasUnrecognizedArgs(String [] argv) {
if (null == argv) {
return false;
}
return hasUnrecognizedArgs(argv, 0, argv.length);
}
/**
* If argv contains an entry "--", return an array containing all elements
* after the "--" separator. Otherwise, return null.
* @param argv a set of arguments to scan for the subcommand arguments.
*/
protected String [] getSubcommandArgs(String [] argv) {
if (null == argv) {
return null;
}
for (int i = 0; i < argv.length; i++) {
if (argv[i].equals("--")) {
return Arrays.copyOfRange(argv, i + 1, argv.length);
}
}
return null;
}
/**
* @return RelatedOptions used by job management tools.
*/
protected RelatedOptions getJobOptions() {
RelatedOptions relatedOpts = new RelatedOptions(
"Job management arguments");
relatedOpts.addOption(OptionBuilder.withArgName("jdbc-uri")
.hasArg()
.withDescription("Specify JDBC connect string for the metastore")
.withLongOpt(STORAGE_METASTORE_ARG)
.create());
// Create an option-group surrounding the operations a user
// can perform on jobs.
OptionGroup group = new OptionGroup();
group.addOption(OptionBuilder.withArgName("job-id")
.hasArg()
.withDescription("Create a new saved job")
.withLongOpt(JOB_CMD_CREATE_ARG)
.create());
group.addOption(OptionBuilder.withArgName("job-id")
.hasArg()
.withDescription("Delete a saved job")
.withLongOpt(JOB_CMD_DELETE_ARG)
.create());
group.addOption(OptionBuilder.withArgName("job-id")
.hasArg()
.withDescription("Show the parameters for a saved job")
.withLongOpt(JOB_CMD_SHOW_ARG)
.create());
Option execOption = OptionBuilder.withArgName("job-id")
.hasArg()
.withDescription("Run a saved job")
.withLongOpt(JOB_CMD_EXEC_ARG)
.create();
group.addOption(execOption);
group.addOption(OptionBuilder
.withDescription("List saved jobs")
.withLongOpt(JOB_CMD_LIST_ARG)
.create());
relatedOpts.addOptionGroup(group);
// Since the "common" options aren't used in the job tool,
// add these settings here.
relatedOpts.addOption(OptionBuilder
.withDescription("Print more information while working")
.withLongOpt(VERBOSE_ARG)
.create());
relatedOpts.addOption(OptionBuilder
.withDescription("Print usage instructions")
.withLongOpt(HELP_ARG)
.create());
return relatedOpts;
}
/**
* @return RelatedOptions used by most/all Sqoop tools.
*/
protected RelatedOptions getCommonOptions() {
// Connection args (common)
RelatedOptions commonOpts = new RelatedOptions("Common arguments");
commonOpts.addOption(OptionBuilder.withArgName("jdbc-uri")
.hasArg().withDescription("Specify JDBC connect string")
.withLongOpt(CONNECT_STRING_ARG)
.create());
commonOpts.addOption(OptionBuilder.withArgName("class-name")
.hasArg().withDescription("Specify connection manager class name")
.withLongOpt(CONN_MANAGER_CLASS_NAME)
.create());
commonOpts.addOption(OptionBuilder.withArgName("properties-file")
.hasArg().withDescription("Specify connection parameters file")
.withLongOpt(CONNECT_PARAM_FILE)
.create());
commonOpts.addOption(OptionBuilder.withArgName("class-name")
.hasArg().withDescription("Manually specify JDBC driver class to use")
.withLongOpt(DRIVER_ARG)
.create());
commonOpts.addOption(OptionBuilder.withArgName("username")
.hasArg().withDescription("Set authentication username")
.withLongOpt(USERNAME_ARG)
.create());
commonOpts.addOption(OptionBuilder.withArgName("password")
.hasArg().withDescription("Set authentication password")
.withLongOpt(PASSWORD_ARG)
.create());
commonOpts.addOption(OptionBuilder
.withDescription("Read password from console")
.create(PASSWORD_PROMPT_ARG));
commonOpts.addOption(OptionBuilder.withArgName("dir")
.hasArg().withDescription("Override $HADOOP_HOME")
.withLongOpt(HADOOP_HOME_ARG)
.create());
// misc (common)
commonOpts.addOption(OptionBuilder
.withDescription("Print more information while working")
.withLongOpt(VERBOSE_ARG)
.create());
commonOpts.addOption(OptionBuilder
.withDescription("Print usage instructions")
.withLongOpt(HELP_ARG)
.create());
return commonOpts;
}
/**
* @param explicitHiveImport true if the user has an explicit --hive-import
* available, or false if this is implied by the tool.
* @return options governing interaction with Hive
*/
protected RelatedOptions getHiveOptions(boolean explicitHiveImport) {
RelatedOptions hiveOpts = new RelatedOptions("Hive arguments");
if (explicitHiveImport) {
hiveOpts.addOption(OptionBuilder
.withDescription("Import tables into Hive "
+ "(Uses Hive's default delimiters if none are set.)")
.withLongOpt(HIVE_IMPORT_ARG)
.create());
}
hiveOpts.addOption(OptionBuilder.withArgName("dir")
.hasArg().withDescription("Override $HIVE_HOME")
.withLongOpt(HIVE_HOME_ARG)
.create());
hiveOpts.addOption(OptionBuilder
.withDescription("Overwrite existing data in the Hive table")
.withLongOpt(HIVE_OVERWRITE_ARG)
.create());
hiveOpts.addOption(OptionBuilder
.withDescription("Fail if the target hive table exists")
.withLongOpt(CREATE_HIVE_TABLE_ARG)
.create());
hiveOpts.addOption(OptionBuilder.withArgName("table-name")
.hasArg()
.withDescription("Sets the table name to use when importing to hive")
.withLongOpt(HIVE_TABLE_ARG)
.create());
hiveOpts.addOption(OptionBuilder
.withDescription("Drop Hive record \\0x01 and row delimiters "
+ "(\\n\\r) from imported string fields")
.withLongOpt(HIVE_DROP_DELIMS_ARG)
.create());
hiveOpts.addOption(OptionBuilder
.hasArg()
.withDescription("Replace Hive record \\0x01 and row delimiters "
+ "(\\n\\r) from imported string fields with user-defined string")
.withLongOpt(HIVE_DELIMS_REPLACEMENT_ARG)
.create());
hiveOpts.addOption(OptionBuilder.withArgName("partition-key")
.hasArg()
.withDescription("Sets the partition key to use when importing to hive")
.withLongOpt(HIVE_PARTITION_KEY_ARG)
.create());
hiveOpts.addOption(OptionBuilder.withArgName("partition-value")
.hasArg()
.withDescription("Sets the partition value to use when importing "
+ "to hive")
.withLongOpt(HIVE_PARTITION_VALUE_ARG)
.create());
hiveOpts.addOption(OptionBuilder
.hasArg()
.withDescription("Override mapping for specific column to hive"
+ " types.")
.withLongOpt(MAP_COLUMN_HIVE)
.create());
return hiveOpts;
}
/**
* @return options governing output format delimiters
*/
protected RelatedOptions getOutputFormatOptions() {
RelatedOptions formatOpts = new RelatedOptions(
"Output line formatting arguments");
formatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets the field separator character")
.withLongOpt(FIELDS_TERMINATED_BY_ARG)
.create());
formatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets the end-of-line character")
.withLongOpt(LINES_TERMINATED_BY_ARG)
.create());
formatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets a field enclosing character")
.withLongOpt(OPTIONALLY_ENCLOSED_BY_ARG)
.create());
formatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets a required field enclosing character")
.withLongOpt(ENCLOSED_BY_ARG)
.create());
formatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets the escape character")
.withLongOpt(ESCAPED_BY_ARG)
.create());
formatOpts.addOption(OptionBuilder
.withDescription("Uses MySQL's default delimiter set: "
+ "fields: , lines: \\n escaped-by: \\ optionally-enclosed-by: '")
.withLongOpt(MYSQL_DELIMITERS_ARG)
.create());
return formatOpts;
}
/**
* @return options governing input format delimiters.
*/
protected RelatedOptions getInputFormatOptions() {
RelatedOptions inputFormatOpts =
new RelatedOptions("Input parsing arguments");
inputFormatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets the input field separator")
.withLongOpt(INPUT_FIELDS_TERMINATED_BY_ARG)
.create());
inputFormatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets the input end-of-line char")
.withLongOpt(INPUT_LINES_TERMINATED_BY_ARG)
.create());
inputFormatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets a field enclosing character")
.withLongOpt(INPUT_OPTIONALLY_ENCLOSED_BY_ARG)
.create());
inputFormatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets a required field encloser")
.withLongOpt(INPUT_ENCLOSED_BY_ARG)
.create());
inputFormatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets the input escape character")
.withLongOpt(INPUT_ESCAPED_BY_ARG)
.create());
return inputFormatOpts;
}
/**
* @param multiTable true if these options will be used for bulk code-gen.
* @return options related to code generation.
*/
protected RelatedOptions getCodeGenOpts(boolean multiTable) {
RelatedOptions codeGenOpts =
new RelatedOptions("Code generation arguments");
codeGenOpts.addOption(OptionBuilder.withArgName("dir")
.hasArg()
.withDescription("Output directory for generated code")
.withLongOpt(CODE_OUT_DIR_ARG)
.create());
codeGenOpts.addOption(OptionBuilder.withArgName("dir")
.hasArg()
.withDescription("Output directory for compiled objects")
.withLongOpt(BIN_OUT_DIR_ARG)
.create());
codeGenOpts.addOption(OptionBuilder.withArgName("name")
.hasArg()
.withDescription("Put auto-generated classes in this package")
.withLongOpt(PACKAGE_NAME_ARG)
.create());
codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
.hasArg()
.withDescription("Null string representation")
.withLongOpt(NULL_STRING)
.create());
codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
.hasArg()
.withDescription("Input null string representation")
.withLongOpt(INPUT_NULL_STRING)
.create());
codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
.hasArg()
.withDescription("Null non-string representation")
.withLongOpt(NULL_NON_STRING)
.create());
codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
.hasArg()
.withDescription("Input null non-string representation")
.withLongOpt(INPUT_NULL_NON_STRING)
.create());
codeGenOpts.addOption(OptionBuilder
.hasArg()
.withDescription("Override mapping for specific columns to java types")
.withLongOpt(MAP_COLUMN_JAVA)
.create());
if (!multiTable) {
codeGenOpts.addOption(OptionBuilder.withArgName("name")
.hasArg()
.withDescription("Sets the generated class name. "
+ "This overrides --" + PACKAGE_NAME_ARG + ". When combined "
+ "with --" + JAR_FILE_NAME_ARG + ", sets the input class.")
.withLongOpt(CLASS_NAME_ARG)
.create());
}
return codeGenOpts;
}
protected RelatedOptions getHBaseOptions() {
RelatedOptions hbaseOpts =
new RelatedOptions("HBase arguments");
hbaseOpts.addOption(OptionBuilder.withArgName("table")
.hasArg()
.withDescription("Import to <table> in HBase")
.withLongOpt(HBASE_TABLE_ARG)
.create());
hbaseOpts.addOption(OptionBuilder.withArgName("family")
.hasArg()
.withDescription("Sets the target column family for the import")
.withLongOpt(HBASE_COL_FAM_ARG)
.create());
hbaseOpts.addOption(OptionBuilder.withArgName("col")
.hasArg()
.withDescription("Specifies which input column to use as the row key")
.withLongOpt(HBASE_ROW_KEY_ARG)
.create());
hbaseOpts.addOption(OptionBuilder
.withDescription("If specified, create missing HBase tables")
.withLongOpt(HBASE_CREATE_TABLE_ARG)
.create());
return hbaseOpts;
}
/**
* Apply common command-line to the state.
*/
protected void applyCommonOptions(CommandLine in, SqoopOptions out)
throws InvalidOptionsException {
// common options.
if (in.hasOption(VERBOSE_ARG)) {
// Immediately switch into DEBUG logging.
Category sqoopLogger = Logger.getLogger(
Sqoop.class.getName()).getParent();
sqoopLogger.setLevel(Level.DEBUG);
LOG.debug("Enabled debug logging.");
}
if (in.hasOption(HELP_ARG)) {
ToolOptions toolOpts = new ToolOptions();
configureOptions(toolOpts);
printHelp(toolOpts);
throw new InvalidOptionsException("");
}
if (in.hasOption(CONNECT_STRING_ARG)) {
out.setConnectString(in.getOptionValue(CONNECT_STRING_ARG));
}
if (in.hasOption(CONN_MANAGER_CLASS_NAME)) {
out.setConnManagerClassName(in.getOptionValue(CONN_MANAGER_CLASS_NAME));
}
if (in.hasOption(CONNECT_PARAM_FILE)) {
File paramFile = new File(in.getOptionValue(CONNECT_PARAM_FILE));
if (!paramFile.exists()) {
throw new InvalidOptionsException(
"Specified connection parameter file not found: " + paramFile);
}
InputStream inStream = null;
Properties connectionParams = new Properties();
try {
inStream = new FileInputStream(
new File(in.getOptionValue(CONNECT_PARAM_FILE)));
connectionParams.load(inStream);
} catch (IOException ex) {
LOG.warn("Failed to load connection parameter file", ex);
throw new InvalidOptionsException(
"Error while loading connection parameter file: "
+ ex.getMessage());
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException ex) {
LOG.warn("Failed to close input stream", ex);
}
}
}
LOG.debug("Loaded connection parameters: " + connectionParams);
out.setConnectionParams(connectionParams);
}
if (in.hasOption(NULL_STRING)) {
out.setNullStringValue(in.getOptionValue(NULL_STRING));
}
if (in.hasOption(INPUT_NULL_STRING)) {
out.setInNullStringValue(in.getOptionValue(INPUT_NULL_STRING));
}
if (in.hasOption(NULL_NON_STRING)) {
out.setNullNonStringValue(in.getOptionValue(NULL_NON_STRING));
}
if (in.hasOption(INPUT_NULL_NON_STRING)) {
out.setInNullNonStringValue(in.getOptionValue(INPUT_NULL_NON_STRING));
}
if (in.hasOption(DRIVER_ARG)) {
out.setDriverClassName(in.getOptionValue(DRIVER_ARG));
}
if (in.hasOption(USERNAME_ARG)) {
out.setUsername(in.getOptionValue(USERNAME_ARG));
if (null == out.getPassword()) {
// Set password to empty if the username is set first,
// to ensure that they're either both null or neither is.
out.setPassword("");
}
}
if (in.hasOption(PASSWORD_ARG)) {
LOG.warn("Setting your password on the command-line is insecure. "
+ "Consider using -" + PASSWORD_PROMPT_ARG + " instead.");
out.setPassword(in.getOptionValue(PASSWORD_ARG));
}
if (in.hasOption(PASSWORD_PROMPT_ARG)) {
out.setPasswordFromConsole();
}
if (in.hasOption(HADOOP_HOME_ARG)) {
out.setHadoopHome(in.getOptionValue(HADOOP_HOME_ARG));
}
}
protected void applyHiveOptions(CommandLine in, SqoopOptions out)
throws InvalidOptionsException {
if (in.hasOption(HIVE_HOME_ARG)) {
out.setHiveHome(in.getOptionValue(HIVE_HOME_ARG));
}
if (in.hasOption(HIVE_IMPORT_ARG)) {
out.setHiveImport(true);
}
if (in.hasOption(HIVE_OVERWRITE_ARG)) {
out.setOverwriteHiveTable(true);
}
if (in.hasOption(CREATE_HIVE_TABLE_ARG)) {
out.setFailIfHiveTableExists(true);
}
if (in.hasOption(HIVE_TABLE_ARG)) {
out.setHiveTableName(in.getOptionValue(HIVE_TABLE_ARG));
}
if (in.hasOption(HIVE_DROP_DELIMS_ARG)) {
out.setHiveDropDelims(true);
}
if (in.hasOption(HIVE_DELIMS_REPLACEMENT_ARG)) {
out.setHiveDelimsReplacement(
in.getOptionValue(HIVE_DELIMS_REPLACEMENT_ARG));
}
if (in.hasOption(HIVE_PARTITION_KEY_ARG)) {
out.setHivePartitionKey(in.getOptionValue(HIVE_PARTITION_KEY_ARG));
}
if (in.hasOption(HIVE_PARTITION_VALUE_ARG)) {
out.setHivePartitionValue(in.getOptionValue(HIVE_PARTITION_VALUE_ARG));
}
if (in.hasOption(MAP_COLUMN_HIVE)) {
out.setMapColumnHive(in.getOptionValue(MAP_COLUMN_HIVE));
}
}
protected void applyOutputFormatOptions(CommandLine in, SqoopOptions out)
throws InvalidOptionsException {
if (in.hasOption(FIELDS_TERMINATED_BY_ARG)) {
out.setFieldsTerminatedBy(SqoopOptions.toChar(
in.getOptionValue(FIELDS_TERMINATED_BY_ARG)));
out.setExplicitDelims(true);
}
if (in.hasOption(LINES_TERMINATED_BY_ARG)) {
out.setLinesTerminatedBy(SqoopOptions.toChar(
in.getOptionValue(LINES_TERMINATED_BY_ARG)));
out.setExplicitDelims(true);
}
if (in.hasOption(OPTIONALLY_ENCLOSED_BY_ARG)) {
out.setEnclosedBy(SqoopOptions.toChar(
in.getOptionValue(OPTIONALLY_ENCLOSED_BY_ARG)));
out.setOutputEncloseRequired(false);
out.setExplicitDelims(true);
}
if (in.hasOption(ENCLOSED_BY_ARG)) {
out.setEnclosedBy(SqoopOptions.toChar(
in.getOptionValue(ENCLOSED_BY_ARG)));
out.setOutputEncloseRequired(true);
out.setExplicitDelims(true);
}
if (in.hasOption(ESCAPED_BY_ARG)) {
out.setEscapedBy(SqoopOptions.toChar(
in.getOptionValue(ESCAPED_BY_ARG)));
out.setExplicitDelims(true);
}
if (in.hasOption(MYSQL_DELIMITERS_ARG)) {
out.setOutputEncloseRequired(false);
out.setFieldsTerminatedBy(',');
out.setLinesTerminatedBy('\n');
out.setEscapedBy('\\');
out.setEnclosedBy('\'');
out.setExplicitDelims(true);
}
}
protected void applyInputFormatOptions(CommandLine in, SqoopOptions out)
throws InvalidOptionsException {
if (in.hasOption(INPUT_FIELDS_TERMINATED_BY_ARG)) {
out.setInputFieldsTerminatedBy(SqoopOptions.toChar(
in.getOptionValue(INPUT_FIELDS_TERMINATED_BY_ARG)));
}
if (in.hasOption(INPUT_LINES_TERMINATED_BY_ARG)) {
out.setInputLinesTerminatedBy(SqoopOptions.toChar(
in.getOptionValue(INPUT_LINES_TERMINATED_BY_ARG)));
}
if (in.hasOption(INPUT_OPTIONALLY_ENCLOSED_BY_ARG)) {
out.setInputEnclosedBy(SqoopOptions.toChar(
in.getOptionValue(INPUT_OPTIONALLY_ENCLOSED_BY_ARG)));
out.setInputEncloseRequired(false);
}
if (in.hasOption(INPUT_ENCLOSED_BY_ARG)) {
out.setInputEnclosedBy(SqoopOptions.toChar(
in.getOptionValue(INPUT_ENCLOSED_BY_ARG)));
out.setInputEncloseRequired(true);
}
if (in.hasOption(INPUT_ESCAPED_BY_ARG)) {
out.setInputEscapedBy(SqoopOptions.toChar(
in.getOptionValue(INPUT_ESCAPED_BY_ARG)));
}
}
protected void applyCodeGenOptions(CommandLine in, SqoopOptions out,
boolean multiTable) throws InvalidOptionsException {
if (in.hasOption(CODE_OUT_DIR_ARG)) {
out.setCodeOutputDir(in.getOptionValue(CODE_OUT_DIR_ARG));
}
if (in.hasOption(BIN_OUT_DIR_ARG)) {
out.setJarOutputDir(in.getOptionValue(BIN_OUT_DIR_ARG));
}
if (in.hasOption(PACKAGE_NAME_ARG)) {
out.setPackageName(in.getOptionValue(PACKAGE_NAME_ARG));
}
if (in.hasOption(MAP_COLUMN_JAVA)) {
out.setMapColumn(in.getOptionValue(MAP_COLUMN_JAVA));
}
if (!multiTable && in.hasOption(CLASS_NAME_ARG)) {
out.setClassName(in.getOptionValue(CLASS_NAME_ARG));
}
}
protected void applyHBaseOptions(CommandLine in, SqoopOptions out) {
if (in.hasOption(HBASE_TABLE_ARG)) {
out.setHBaseTable(in.getOptionValue(HBASE_TABLE_ARG));
}
if (in.hasOption(HBASE_COL_FAM_ARG)) {
out.setHBaseColFamily(in.getOptionValue(HBASE_COL_FAM_ARG));
}
if (in.hasOption(HBASE_ROW_KEY_ARG)) {
out.setHBaseRowKeyColumn(in.getOptionValue(HBASE_ROW_KEY_ARG));
}
if (in.hasOption(HBASE_CREATE_TABLE_ARG)) {
out.setCreateHBaseTable(true);
}
}
protected void validateCommonOptions(SqoopOptions options)
throws InvalidOptionsException {
if (options.getConnectString() == null) {
throw new InvalidOptionsException(
"Error: Required argument --connect is missing."
+ HELP_STR);
}
}
protected void validateCodeGenOptions(SqoopOptions options)
throws InvalidOptionsException {
if (options.getClassName() != null && options.getPackageName() != null) {
throw new InvalidOptionsException(
"--class-name overrides --package-name. You cannot use both."
+ HELP_STR);
}
}
protected void validateOutputFormatOptions(SqoopOptions options)
throws InvalidOptionsException {
if (options.doHiveImport()) {
if (!options.explicitDelims()) {
// user hasn't manually specified delimiters, and wants to import
// straight to Hive. Use Hive-style delimiters.
LOG.info("Using Hive-specific delimiters for output. You can override");
LOG.info("delimiters with --fields-terminated-by, etc.");
options.setOutputDelimiters(DelimiterSet.HIVE_DELIMITERS);
}
if (options.getOutputEscapedBy() != DelimiterSet.NULL_CHAR) {
LOG.warn("Hive does not support escape characters in fields;");
LOG.warn("parse errors in Hive may result from using --escaped-by.");
}
if (options.getOutputEnclosedBy() != DelimiterSet.NULL_CHAR) {
LOG.warn("Hive does not support quoted strings; parse errors");
LOG.warn("in Hive may result from using --enclosed-by.");
}
}
}
protected void validateHiveOptions(SqoopOptions options)
throws InvalidOptionsException {
// Empty; this method is present to maintain API consistency, and
// is reserved for future constraints on Hive options.
if (options.getHiveDelimsReplacement() != null
&& options.doHiveDropDelims()) {
throw new InvalidOptionsException("The " + HIVE_DROP_DELIMS_ARG
+ " option conflicts with the " + HIVE_DELIMS_REPLACEMENT_ARG
+ " option." + HELP_STR);
}
}
protected void validateHBaseOptions(SqoopOptions options)
throws InvalidOptionsException {
if ((options.getHBaseColFamily() != null && options.getHBaseTable() == null)
|| (options.getHBaseColFamily() == null
&& options.getHBaseTable() != null)) {
throw new InvalidOptionsException(
"Both --hbase-table and --column-family must be set together."
+ HELP_STR);
}
}
/**
* Given an array of extra arguments (usually populated via
* this.extraArguments), determine the offset of the first '--'
* argument in the list. Return 'extra.length' if there is none.
*/
protected int getDashPosition(String [] extra) {
int dashPos = extra.length;
for (int i = 0; i < extra.length; i++) {
if (extra[i].equals("--")) {
dashPos = i;
break;
}
}
return dashPos;
}
}