Examples of FormatBundle

org.apache.crunch.io.FormatBundle
A combination of an {@link InputFormat} or {@link OutputFormat} and any extra configuration information that format class needs to run.
The {@code FormatBundle} allow us to let different formats act asif they are the only format that exists in a particular MapReduce job, even when we have multiple types of inputs and outputs within a single job.

Examples of org.apache.crunch.io.FormatBundle


  @Test
  public void configureBundleCustomWithFactory(){
    ReaderWriterFactory fakeFactory = new FakeReaderWriterFactory();
    AvroMode mode = AvroMode.SPECIFIC.withFactory(fakeFactory);
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    mode.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getFactory(), is(instanceOf(FakeReaderWriterFactory.class)));
  }

View Full Code Here

Examples of org.apache.crunch.io.FormatBundle

    if (name == null) {
      schemaParam = PARQUET_AVRO_SCHEMA_PARAMETER;
    } else {
      schemaParam = PARQUET_AVRO_SCHEMA_PARAMETER + "." + name;
    }
    FormatBundle fb = FormatBundle.forOutput(CrunchAvroParquetOutputFormat.class);
    for (Map.Entry<String, String> e : extraConf.entrySet()) {
      fb.set(e.getKey(), e.getValue());
    }
    fb.set(schemaParam, atype.getSchema().toString());
    configureForMapReduce(job, Void.class, atype.getTypeClass(), fb, outputPath, name);
  }

View Full Code Here

Examples of org.apache.crunch.io.FormatBundle

    Configuration base = job.getConfiguration();
    Map<FormatBundle, Map<Integer, List<Path>>> formatNodeMap = CrunchInputs.getFormatNodeMap(job);


    // First, build a map of InputFormats to Paths
    for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
      FormatBundle inputBundle = entry.getKey();
      Configuration conf = new Configuration(base);
      inputBundle.configure(conf);
      Job jobCopy = new Job(conf);
      InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
          jobCopy.getConfiguration());
      if (format instanceof FileInputFormat && !conf.getBoolean(RuntimeParameters.DISABLE_COMBINE_FILE, true)) {
        format = new CrunchCombineFileInputFormat<Object, Object>(jobCopy);
      }
      for (Map.Entry<Integer, List<Path>> nodeEntry : entry.getValue().entrySet()) {

View Full Code Here

Examples of org.apache.crunch.io.FormatBundle

  public void testConfigureJob_SpecificData() throws IOException {
    AvroType<Person> avroSpecificType = Avros.records(Person.class);
    AvroFileSource<Person> personFileSource = new AvroFileSource<Person>(new Path(tempFile.getAbsolutePath()),
        avroSpecificType);


    FormatBundle bundle = personFileSource.getBundle();
    bundle.configure(job.getConfiguration());


    assertFalse(job.getConfiguration().getBoolean(AvroJob.INPUT_IS_REFLECT, true));
    assertEquals(Person.SCHEMA$.toString(), job.getConfiguration().get(AvroJob.INPUT_SCHEMA));
  }

View Full Code Here

Examples of org.apache.crunch.io.FormatBundle

  public void testConfigureJob_GenericData() throws IOException {
    AvroType<Record> avroGenericType = Avros.generics(Person.SCHEMA$);
    AvroFileSource<Record> personFileSource = new AvroFileSource<Record>(new Path(tempFile.getAbsolutePath()),
        avroGenericType);


    FormatBundle bundle = personFileSource.getBundle();
    bundle.configure(job.getConfiguration());


    assertFalse(job.getConfiguration().getBoolean(AvroJob.INPUT_IS_REFLECT, true));


  }

View Full Code Here

Examples of org.apache.crunch.io.FormatBundle

  public void testConfigureJob_ReflectData() throws IOException {
    AvroType<StringWrapper> avroReflectType = Avros.reflects(StringWrapper.class);
    AvroFileSource<StringWrapper> personFileSource = new AvroFileSource<StringWrapper>(new Path(
        tempFile.getAbsolutePath()), avroReflectType);


    FormatBundle bundle = personFileSource.getBundle();
    bundle.configure(job.getConfiguration());
    assertTrue(job.getConfiguration().getBoolean(AvroJob.INPUT_IS_REFLECT, false));


  }

View Full Code Here

Examples of org.apache.crunch.io.FormatBundle

  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = ptype.getConverter();
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    FormatBundle fb = FormatBundle.forOutput(getOutputFormat(ptype));
    for (Map.Entry<String, String> e : extraConf.entrySet()) {
      fb.set(e.getKey(), e.getValue());
    }
    configureForMapReduce(job, keyClass, valueClass, fb, outputPath, name);
  }

View Full Code Here

Examples of org.apache.crunch.io.FormatBundle

 * run.
 */
public class NLineFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T> {


  private static FormatBundle getBundle(int linesPerTask) {
    FormatBundle bundle = FormatBundle.forInput(NLineInputFormat.class);
    bundle.set(NLineInputFormat.LINES_PER_MAP, String.valueOf(linesPerTask));
    bundle.set(RuntimeParameters.DISABLE_COMBINE_FILE, "true");
    return bundle;
  }

View Full Code Here

Examples of org.apache.crunch.io.FormatBundle

import org.apache.hadoop.fs.Path;


public class AvroFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T> {


  private static <S> FormatBundle getBundle(AvroType<S> ptype, AvroMode mode) {
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class)
          .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect()))
          .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString())
          .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName())
          .set(RuntimeParameters.DISABLE_COMBINE_FILE, Boolean.FALSE.toString());
    mode.configure(bundle);

View Full Code Here

Examples of org.apache.crunch.io.FormatBundle

  }


  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    AvroType<?> atype = (AvroType) ((PTableType) ptype).getValueType();
    FormatBundle bundle = FormatBundle.forOutput(AvroPathPerKeyOutputFormat.class);
    String schemaParam;
    if (name == null) {
      schemaParam = "avro.output.schema";
    } else {
      schemaParam = "avro.output.schema." + name;
    }
    bundle.set(schemaParam, atype.getSchema().toString());
    AvroMode.fromType(atype).configure(bundle);
    configureForMapReduce(job, AvroWrapper.class, NullWritable.class, bundle, outputPath, name);
  }

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.