Package com.cloudera.sqoop

Examples of com.cloudera.sqoop.SqoopOptions


      return;
    }

    String tableName = context.getTableName();
    String jarFile = context.getJarFile();
    SqoopOptions options = context.getOptions();

    MySQLDumpImportJob importer = null;
    try {
      importer = new MySQLDumpImportJob(options, context);
    } catch (ClassNotFoundException cnfe) {
      throw new IOException("Could not load required classes", cnfe);
    }

    String splitCol = getSplitColumn(options, tableName);
    if (null == splitCol && options.getNumMappers() > 1) {
      // Can't infer a primary key.
      throw new ImportException("No primary key could be found for table "
          + tableName + ". Please specify one with --split-by or perform "
          + "a sequential import with '-m 1'.");
    }

    LOG.info("Beginning mysqldump fast path import");

    if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile) {
      // TODO(aaron): Support SequenceFile-based load-in.
      LOG.warn("File import layout " + options.getFileLayout()
          + " is not supported by");
      LOG.warn("MySQL direct import; import will proceed as text files.");
    }

    importer.runImport(tableName, jarFile, splitCol, options.getConf());
  }
View Full Code Here


   */
  public void importTable(com.cloudera.sqoop.manager.ImportJobContext context)
    throws IOException, ImportException {

    String tableName = context.getTableName();
    SqoopOptions options = context.getOptions();

    LOG.info("Beginning psql fast path import");

    if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile) {
      // TODO(aaron): Support SequenceFile-based load-in
      LOG.warn("File import layout" + options.getFileLayout()
          + " is not supported by");
      LOG.warn("Postgresql direct import; import will proceed as text files.");
    }

    if (!StringUtils.equals(options.getNullStringValue(),
      options.getNullNonStringValue())) {
      throw new ImportException(
        "Detected different values of --input-string and --input-non-string " +
        "parameters. PostgreSQL direct manager do not support that. Please " +
        "either use the same values or omit the --direct parameter.");
    }

    String commandFilename = null;
    String passwordFilename = null;
    Process p = null;
    AsyncSink sink = null;
    AsyncSink errSink = null;
    PerfCounters counters = new PerfCounters();

    try {
      // Get the COPY TABLE command to issue, write this to a file, and pass
      // it in to psql with -f filename.  Then make sure we delete this file
      // in our finally block.
      String copyCmd = getCopyCommand(tableName);
      commandFilename = writeCopyCommand(copyCmd);

      // Arguments to pass to psql on the command line.
      ArrayList<String> args = new ArrayList<String>();

      // Environment to pass to psql.
      List<String> envp = Executor.getCurEnvpStrings();

      // We need to parse the connect string URI to determine the database
      // name and the host and port. If the host is localhost and the port is
      // not specified, we don't want to pass this to psql, because we want to
      // force the use of a UNIX domain socket, not a TCP/IP socket.
      String connectString = options.getConnectString();
      String databaseName = JdbcUrl.getDatabaseName(connectString);
      String hostname = JdbcUrl.getHostName(connectString);
      int port = JdbcUrl.getPort(connectString);

      if (null == databaseName) {
        throw new ImportException("Could not determine database name");
      }

      LOG.info("Performing import of table " + tableName + " from database "
          + databaseName);
      args.add(PSQL_CMD); // requires that this is on the path.
      args.add("--tuples-only");
      args.add("--quiet");

      String username = options.getUsername();
      if (username != null) {
        args.add("--username");
        args.add(username);
        String password = options.getPassword();
        if (null != password) {
          passwordFilename =
            PostgreSQLUtils.writePasswordFile(options.getTempDir(), password);
          // Need to send PGPASSFILE environment variable specifying
          // location of our postgres file.
          envp.add("PGPASSFILE=" + passwordFilename);
        }
      }

      args.add("--host");
      args.add(hostname);

      if (port != -1) {
        args.add("--port");
        args.add(Integer.toString(port));
      }

      if (null != databaseName && databaseName.length() > 0) {
        args.add(databaseName);
      }

      // The COPY command is in a script file.
      args.add("-f");
      args.add(commandFilename);

      // begin the import in an external process.
      LOG.debug("Starting psql with arguments:");
      for (String arg : args) {
        LOG.debug("  " + arg);
      }

      // This writer will be closed by AsyncSink.
      SplittableBufferedWriter w = DirectImportUtils.createHdfsSink(
          options.getConf(), options, context);

      // Actually start the psql dump.
      p = Runtime.getRuntime().exec(args.toArray(new String[0]),
          envp.toArray(new String[0]));

View Full Code Here

    } catch (SQLException e) {
      LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
      fail("Setup fail with SQLException: " + e.toString());
    }
    Configuration conf = getConf();
    SqoopOptions opts = getSqoopOptions(conf);
    String username = MSSQLTestUtils.getDBUserName();
    String password = MSSQLTestUtils.getDBPassWord();
    opts.setUsername(username);
    opts.setPassword(password);
    opts.setConnectString(getConnectString());
    ConnFactory f = new ConnFactory(conf);
    try {
      this.manager = f.getManager(new JobData(opts, new ImportTool()));
      System.out.println("Manger : " + this.manager);
    } catch (IOException ioe) {
View Full Code Here

      statement.close();
    }
  }

  protected SqoopOptions getSqoopOptions(Configuration conf) {
    SqoopOptions opt = new SqoopOptions(conf);
    String username = MSSQLTestUtils.getDBUserName();
    String password = MSSQLTestUtils.getDBPassWord();
    SqoopOptions opts = new SqoopOptions(conf);
    opts.setUsername(username);
    opts.setPassword(password);

    return opt;
  }
View Full Code Here

    return opt;
  }

  SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
    SqoopOptions opts = null;
    try {
      opts = tool.parseArguments(args, null, null, true);
      String username = MSSQLTestUtils.getDBUserName();
      String password = MSSQLTestUtils.getDBPassWord();
      opts.setUsername(username);
      opts.setPassword(password);

    } catch (Exception e) {
      LOG.error(StringUtils.stringifyException(e));
      fail("Invalid options: " + e.toString());
    }
View Full Code Here

  public String[] codeGen(DATATYPES dt) throws Exception {

    CodeGenTool codeGen = new CodeGenTool();

    String[] codeGenArgs = getCodeGenArgv(dt);
    SqoopOptions options = codeGen.parseArguments(codeGenArgs, null, null,
      true);
    String username = MSSQLTestUtils.getDBUserName();
    String password = MSSQLTestUtils.getDBPassWord();

    options.setUsername(username);
    options.setPassword(password);
    codeGen.validateOptions(options);

    int ret = codeGen.run(options);
    assertEquals(0, ret);
    List<String> generatedJars = codeGen.getGeneratedJarFiles();
View Full Code Here

    job.setMapOutputValueClass(NullWritable.class);
  }

  protected void propagateOptionsToJob(Job job) {
    super.propagateOptionsToJob(job);
    SqoopOptions opts = context.getOptions();
    Configuration conf = job.getConfiguration();
    if (opts.getNullStringValue() != null) {
      conf.set("postgresql.null.string", opts.getNullStringValue());
    }
    setDelimiter("postgresql.input.field.delim",
                 opts.getInputFieldDelim(), conf);
    setDelimiter("postgresql.input.record.delim",
                 opts.getInputRecordDelim(), conf);
    setDelimiter("postgresql.input.enclosedby",
                 opts.getInputEnclosedBy(), conf);
    setDelimiter("postgresql.input.escapedby",
                 opts.getInputEscapedBy(), conf);
    conf.setBoolean("postgresql.input.encloserequired",
                    opts.isInputEncloseRequired());
  }
View Full Code Here

    SequenceFile.Reader reader = null;

    String[] argv = getArgv(true, query, targetDir, false);
    runImport(argv);
    try {
      SqoopOptions opts = new ImportTool().parseArguments(getArgv(false,
          query, targetDir, false), null, null, true);

      CompilationManager compileMgr = new CompilationManager(opts);
      String jarFileName = compileMgr.getJarFilename();
View Full Code Here

    context.setConnManager(this);

    String tableName = context.getTableName();
    String jarFile = context.getJarFile();
    SqoopOptions options = context.getOptions();

    if (null == tableName) {
      LOG.
        error("Netezza external table import does not support query imports.");
      LOG.
        error("Do not use --direct and --query together for Netezza.");
      throw
        new IOException("Null tableName for Netezza external table import.");
    }

    checkNullValueStrings(options.getNullStringValue(),
        options.getNullNonStringValue());

    char qc = options.getOutputEnclosedBy();
    char ec = options.getOutputEscapedBy();

    if (qc > 0 && !(qc == '"' || qc == '\'')) {
      throw new ImportException("Output enclosed-by character must be '\"' "
         + "or ''' for netezza direct mode imports");
    }
    if (ec > 0 && ec != '\\') {
      throw new ImportException("Output escaped-by character must be '\\' "
          + "for netezza direct mode exports");
    }

    NetezzaExternalTableImportJob importer = null;

    importer = new NetezzaExternalTableImportJob(options, context);

    // Direct Netezza Manager will use the datasliceid so no split columns
    // will be used.

    LOG.info("Beginning netezza fast path import");

    if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile) {
      LOG.warn("File import layout " + options.getFileLayout()
          + " is not supported by");
      LOG.warn("Netezza direct import; import will proceed as text files.");
    }

    importer.runImport(tableName, jarFile, null, options.getConf());
  }
View Full Code Here

      statement.close();
    }
  }

  protected SqoopOptions getSqoopOptions(Configuration conf) {
    SqoopOptions opt = new SqoopOptions(conf);
    String username = MSSQLTestUtils.getDBUserName();
    String password = MSSQLTestUtils.getDBPassWord();
    SqoopOptions opts = new SqoopOptions(conf);
    opts.setUsername(username);
    opts.setPassword(password);

    return opt;
  }
View Full Code Here

TOP

Related Classes of com.cloudera.sqoop.SqoopOptions

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.