Examples of org.apache.sqoop.job.etl.Partitioner

org.apache.sqoop.job.etl.Partitioner
This allows connector to define how input data to be partitioned. The number of data partitions also determines the degree of parallelism.

  public List<InputSplit> getSplits(JobContext context)
      throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();


    String partitionerName = conf.get(JobConstants.JOB_ETL_PARTITIONER);
    Partitioner partitioner = (Partitioner) ClassUtils.instantiate(partitionerName);


    PrefixContext connectorContext = new PrefixContext(conf, JobConstants.PREFIX_CONNECTOR_CONTEXT);
    Object connectorConnection = ConfigurationUtils.getConfigConnectorConnection(conf);
    Object connectorJob = ConfigurationUtils.getConfigConnectorJob(conf);
    Schema schema = ConfigurationUtils.getConnectorSchema(conf);


    long maxPartitions = conf.getLong(JobConstants.JOB_ETL_EXTRACTOR_NUM, 10);
    PartitionerContext partitionerContext = new PartitionerContext(connectorContext, maxPartitions, schema);


    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connectorConnection, connectorJob);
    List<InputSplit> splits = new LinkedList<InputSplit>();
    for (Partition partition : partitions) {
      LOG.debug("Partition: " + partition);
      SqoopSplit split = new SqoopSplit();
      split.setPartition(partition);

View Full Code Here

        String.valueOf(START + NUMBER_OF_ROWS - 1));


    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();


    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);


    verifyResult(partitions, new String[] {
        "-5 <= ICOL AND ICOL < -3",
        "-3 <= ICOL AND ICOL < -1",
        "-1 <= ICOL AND ICOL < 1",

View Full Code Here

        String.valueOf(START + NUMBER_OF_ROWS - 1));


    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();


    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);


    verifyResult(partitions, new String[] {
        "-5 <= ICOL AND ICOL < -1",
        "-1 <= ICOL AND ICOL < 2",
        "2 <= ICOL AND ICOL <= 5"

View Full Code Here

        String.valueOf(START + NUMBER_OF_ROWS - 1));


    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();


    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 13, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);


    verifyResult(partitions, new String[] {
        "-5 <= ICOL AND ICOL < -4",
        "-4 <= ICOL AND ICOL < -3",
        "-3 <= ICOL AND ICOL < -2",

View Full Code Here

        String.valueOf((double)(START + NUMBER_OF_ROWS - 1)));


    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();


    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);


    verifyResult(partitions, new String[] {
        "-5.0 <= DCOL AND DCOL < -3.0",
        "-3.0 <= DCOL AND DCOL < -1.0",
        "-1.0 <= DCOL AND DCOL < 1.0",

View Full Code Here

        String.valueOf((double)(START + NUMBER_OF_ROWS - 1)));


    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();


    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);


    verifyResult(partitions, new String[] {
        "-5.0 <= DCOL AND DCOL < -1.6666666666666665",
        "-1.6666666666666665 <= DCOL AND DCOL < 1.666666666666667",
        "1.666666666666667 <= DCOL AND DCOL <= 5.0"

View Full Code Here

    context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(START + NUMBER_OF_ROWS - 1));


    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();


    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);


    verifyResult(partitions, new String[] {
        "-5 <= ICOL AND ICOL < -3",
        "-3 <= ICOL AND ICOL < -1",
        "-1 <= ICOL AND ICOL < 1",

View Full Code Here

    context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(new BigDecimal(START + NUMBER_OF_ROWS - 1)));


    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();


    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);


    verifyResult(partitions, new String[]{
      "-5 <= DCOL AND DCOL < -2",
      "-2 <= DCOL AND DCOL < 1",
      "1 <= DCOL AND DCOL <= 5"

View Full Code Here

    context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(new BigDecimal(START)));


    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();


    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);


    verifyResult(partitions, new String[]{
      "DCOL = -5",
    });
  }

View Full Code Here




    ConnectionConfiguration connConf = new ConnectionConfiguration();
    ImportJobConfiguration jobConf = new ImportJobConfiguration();


    Partitioner partitioner = new GenericJdbcImportPartitioner();
    PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
    List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);




    verifyResult(partitions, new String[]{
        "'2004-10-20' <= DCOL AND DCOL < '2007-10-19'",
        "'2007-10-19' <= DCOL AND DCOL < '2010-10-18'",

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.sqoop.job.etl.Partitioner

org.apache.sqoop.connector.jdbc.TestImportPartitioner

org.apache.sqoop.job.mr.SqoopInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.