Examples of InputSplit


Examples of org.apache.hadoop.mapred.InputSplit

    InputSplit[] splits = aif.getSplits(job, 1);

    Assert.assertEquals(1, splits.length);

    InputSplit split = splits[0];

    Assert.assertEquals(RangeInputSplit.class, split.getClass());

    RangeInputSplit risplit = (RangeInputSplit) split;

    Assert.assertEquals(username, risplit.getPrincipal());
    Assert.assertEquals(table, risplit.getTable());
View Full Code Here

Examples of org.apache.hadoop.mapred.InputSplit

    {
      JobConf job = HadoopPlugin.getJobFactory(BitPostingIndexInputFormat.class.getSimpleName()).newJob();
      setStructures(job, "linksin", "linksin-lookup");
      HadoopUtility.toHConfiguration(index, job);
      index.close();
      InputSplit s = new BitPostingIndexInputSplit(
          new Path(args[3]), Long.parseLong(args[4]), Long.parseLong(args[5]),
          new String[0], Integer.parseInt(args[6]), Integer.parseInt(args[7]));
      RecordReader<IntWritable, IntObjectWrapper<IterablePosting>> rr = new BitPostingIndexInputFormat().getRecordReader(s, job, new Reporter(){
        public InputSplit getInputSplit() throws UnsupportedOperationException {return null;}
        @SuppressWarnings("unchecked")
View Full Code Here

Examples of org.apache.hadoop.mapred.InputSplit

      } catch (HiveException e) {
        throw new RuntimeException(
            "Unable to get metadata for input table split" + split.getPath());
      }
    }
    InputSplit retA[] = newSplits.toArray((new FileSplit[newSplits.size()]));
    l4j.info("Number of input splits: " + splits.length + " new input splits: "
        + retA.length + ", sum of split lengths: " + sumSplitLengths);
    return retA;
  }
View Full Code Here

Examples of org.apache.hadoop.mapred.InputSplit

  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {

    HiveInputSplit hsplit = (HiveInputSplit) split;
   
    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);
View Full Code Here

Examples of org.apache.hadoop.mapred.InputSplit

  public final ExpectedSuppliedException thrown = ExpectedSuppliedException
      .none();

  @Test
  public void testGetInputSplitForMapper() {
    final InputSplit split = new MockReporter(MockReporter.ReporterType.Mapper,
        null, new Path("/")).getInputSplit();
    assertTrue(null != split);
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit

      HRegionLocation regLoc = table.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY, false);
      if (null == regLoc) {
        throw new IOException("Expecting at least one region.");
      }
      List<InputSplit> splits = new ArrayList<InputSplit>(1);
      InputSplit split = new TableSplit(table.getName(),
          HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc
              .getHostnamePort().split(Addressing.HOSTNAME_PORT_SEPARATOR)[0]);
      splits.add(split);
      return splits;
    }
    List<InputSplit> splits = new ArrayList<InputSplit>(keys.getFirst().length);
    for (int i = 0; i < keys.getFirst().length; i++) {
      if ( !includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
        continue;
      }
      HRegionLocation location = table.getRegionLocation(keys.getFirst()[i], false);
      // The below InetSocketAddress creation does a name resolution.
      InetSocketAddress isa = new InetSocketAddress(location.getHostname(), location.getPort());
      if (isa.isUnresolved()) {
        LOG.warn("Failed resolve " + isa);
      }
      InetAddress regionAddress = isa.getAddress();
      String regionLocation;
      try {
        regionLocation = reverseDNS(regionAddress);
      } catch (NamingException e) {
        LOG.error("Cannot resolve the host name for " + regionAddress + " because of " + e);
        regionLocation = location.getHostname();
      }

      byte[] startRow = scan.getStartRow();
      byte[] stopRow = scan.getStopRow();
      // determine if the given start an stop key fall into the region
      if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
          Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
          (stopRow.length == 0 ||
           Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
        byte[] splitStart = startRow.length == 0 ||
          Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ?
            keys.getFirst()[i] : startRow;
        byte[] splitStop = (stopRow.length == 0 ||
          Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) &&
          keys.getSecond()[i].length > 0 ?
            keys.getSecond()[i] : stopRow;
        InputSplit split = new TableSplit(table.getName(),
          splitStart, splitStop, regionLocation);
        splits.add(split);
        if (LOG.isDebugEnabled()) {
          LOG.debug("getSplits: split -> " + i + " -> " + split);
        }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit

  @Override
  protected void setup(Mapper.Context context) throws IOException,
      InterruptedException {

    Configuration conf = context.getConfiguration();
    InputSplit is = context.getInputSplit();
    FileSplit fs = (FileSplit) is;
    Path splitPath = fs.getPath();

    if (splitPath.toString().contains(conf.get(HIHOConf.MERGE_OLD_PATH))) {
      isOld = true;
View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit

  @Override
  protected void setup(Mapper.Context context) throws IOException,
      InterruptedException {

    Configuration conf = context.getConfiguration();
    InputSplit is = context.getInputSplit();
    FileSplit fs = (FileSplit) is;
    Path splitPath = fs.getPath();

    if (splitPath.toString().contains(conf.get(HIHOConf.MERGE_OLD_PATH))) {
      isOld = true;
View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit

    }
    split=MakeIndex.parseSplit(conf.get("higo.column.split",split));
    String custfields=conf.get("higo.column.custfields","");
    usedthedate=conf.getBoolean("higo.column.userthedate", usedthedate);
    this.thedate=null;
     InputSplit inputSplit = context.getInputSplit();
       Path filepath = ((FileSplit) inputSplit).getPath();
      
       if(filepath!=null)
       {
         String pash=filepath.toString();
View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit

        List<Source> newSources = new ArrayList<Source>();
        for (int i = 0; i < sourceCount; i++) {
            Class<? extends InputFormat<?, ?>> formatClass =
                (Class<? extends InputFormat<?, ?>>) readClassByName(InputFormat.class, in);
            Class<? extends InputSplit> splitClass = readClassByName(InputSplit.class, in);
            InputSplit inputSplit = ReflectionUtils.newInstance(splitClass, getConf());
            ((Writable) inputSplit).readFields(in);
            newSources.add(new Source(inputSplit, formatClass));
        }
        this.sources = newSources;
        int locationCount = WritableUtils.readVInt(in);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.