Examples of InputBundle

Examples of org.apache.crunch.io.impl.InputBundle

48
49
50
51
52
53
54
55
56
57
58
  public static Map<InputBundle, Map<Integer, List<Path>>> getFormatNodeMap(JobContext job) {
    Map<InputBundle, Map<Integer, List<Path>>> formatNodeMap = Maps.newHashMap();
    Configuration conf = job.getConfiguration();
    for (String input : Splitter.on(RECORD_SEP).split(conf.get(RuntimeParameters.MULTI_INPUTS))) {
      List<String> fields = Lists.newArrayList(SPLITTER.split(input));
      InputBundle inputBundle = InputBundle.fromSerialized(fields.get(0));
      if (!formatNodeMap.containsKey(inputBundle)) {
        formatNodeMap.put(inputBundle, Maps.<Integer, List<Path>> newHashMap());
      }
      Integer nodeIndex = Integer.valueOf(fields.get(1));
      if (!formatNodeMap.get(inputBundle).containsKey(nodeIndex)) {
View Full Code Here

Examples of org.apache.crunch.io.impl.InputBundle

43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    Configuration conf = job.getConfiguration();
    Map<InputBundle, Map<Integer, List<Path>>> formatNodeMap = CrunchInputs.getFormatNodeMap(job);

    // First, build a map of InputFormats to Paths
    for (Map.Entry<InputBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
      InputBundle inputBundle = entry.getKey();
      Job jobCopy = new Job(conf);
      InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getInputFormatClass(),
          jobCopy.getConfiguration());
      for (Map.Entry<Integer, List<Path>> nodeEntry : entry.getValue().entrySet()) {
        Integer nodeIndex = nodeEntry.getKey();
        List<Path> paths = nodeEntry.getValue();
        FileInputFormat.setInputPaths(jobCopy, paths.toArray(new Path[paths.size()]));

        // Get splits for each input path and tag with InputFormat
        // and Mapper types by wrapping in a TaggedInputSplit.
        List<InputSplit> pathSplits = format.getSplits(jobCopy);
        for (InputSplit pathSplit : pathSplits) {
          splits.add(new CrunchInputSplit(pathSplit, inputBundle.getInputFormatClass(), inputBundle
              .getExtraConfiguration(), nodeIndex, jobCopy.getConfiguration()));
        }
      }
    }
    return splits;
View Full Code Here

Examples of org.apache.crunch.io.impl.InputBundle

32
33
34
35
36
37
38
39
40
41
import org.apache.hadoop.fs.Path;

public class AvroFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T> {

  public AvroFileSource(Path path, AvroType<T> ptype) {
    super(path, ptype, new InputBundle(AvroInputFormat.class)
        .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect()))
        .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString())
        .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName()));
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.