Package org.apache.crunch.io

Examples of org.apache.crunch.io.FormatBundle

100
101
102
103
104
105
106
107
108
109
110
  public void readFields(DataInput in) throws IOException {
    if (conf == null) {
      conf = new Configuration();
    }
    nodeIndex = in.readInt();
    bundle = new FormatBundle();
    bundle.setConf(conf);
    bundle.readFields(in);
    bundle.configure(conf); // yay bootstrap!
    Class<? extends InputSplit> inputSplitClass = readClass(in);
    inputSplit = (InputSplit) ReflectionUtils.newInstance(inputSplitClass, conf);
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
import org.apache.hadoop.fs.Path;

public class AvroFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T> {

  private static <S> FormatBundle getBundle(AvroType<S> ptype) {
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class)
        .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect()))
        .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString())
        .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName());
    AvroMode.fromType(ptype).configure(bundle);
    return bundle;
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
  }

  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    AvroType<?> atype = (AvroType) ((PTableType) ptype).getValueType();
    FormatBundle bundle = FormatBundle.forOutput(AvroPathPerKeyOutputFormat.class);
    String schemaParam;
    if (name == null) {
      schemaParam = "avro.output.schema";
    } else {
      schemaParam = "avro.output.schema." + name;
    }
    bundle.set(schemaParam, atype.getSchema().toString());
    AvroMode.fromType(atype).configure(bundle);
    configureForMapReduce(job, AvroWrapper.class, NullWritable.class, bundle, outputPath, name);
  }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
    assertThat(returnedMode, is(AvroMode.GENERIC));
  }

  @Test
  public void configureBundleSpecific(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.SPECIFIC.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(SpecificData.class)));
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
    assertThat(returnedMode.getData(), is(instanceOf(SpecificData.class)));
  }

  @Test
  public void configureBundleGeneric(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.GENERIC.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(GenericData.class)));
  }
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
    assertThat(returnedMode.getData(), is(instanceOf(GenericData.class)));
  }

  @Test
  public void configureBundleReflect(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.REFLECT.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(ReflectData.class)));
  }
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
  @Test
  public void configureBundleCustomWithFactory(){
    ReaderWriterFactory fakeFactory = new FakeReaderWriterFactory();
    AvroMode mode = AvroMode.SPECIFIC.withFactory(fakeFactory);
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    mode.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getFactory(), is(instanceOf(FakeReaderWriterFactory.class)));
  }
View Full Code Here
35
36
37
38
39
40
41
42
43
* run.
*/
public class NLineFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T> {

  private static FormatBundle getBundle(int linesPerTask) {
    FormatBundle bundle = FormatBundle.forInput(NLineInputFormat.class);
    bundle.set(NLineInputFormat.LINES_PER_MAP, String.valueOf(linesPerTask));
    return bundle;
  }
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
import org.apache.hadoop.fs.Path;

public class AvroFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T> {

  private static <S> FormatBundle getBundle(AvroType<S> ptype) {
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class)
        .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect()))
        .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString())
        .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName());
    return bundle;
  }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
  // configuration field for specifying the separator character.
  private static final String OLD_KV_SEP = "key.value.separator.in.input.line";
  private static final String NEW_KV_SEP = "mapreduce.input.keyvaluelinerecordreader.key.value.separator";
 
  private static FormatBundle getBundle(String sep) {
    FormatBundle bundle = FormatBundle.forInput(KeyValueTextInputFormat.class);
    bundle.set(OLD_KV_SEP, sep);
    bundle.set(NEW_KV_SEP, sep);
    return bundle;
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.io.FormatBundle

  • org.apache.commons.lang.builder.HashCodeBuilder
  • org.apache.crunch.impl.mr.run.CrunchInputFormat
  • org.apache.crunch.impl.mr.run.CrunchInputSplit
  • org.apache.crunch.io.avro.AvroFileSource
  • org.apache.crunch.io.avro.AvroFileSourceTest
  • org.apache.crunch.io.avro.AvroFileTarget
  • org.apache.crunch.io.avro.AvroPathPerKeyTarget
  • org.apache.crunch.io.avro.trevni.TrevniKeySource
  • org.apache.crunch.io.parquet.AvroParquetFileTarget
  • org.apache.crunch.io.text.NLineFileSource
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.