Package org.apache.crunch

Examples of org.apache.crunch.MapFn

257
258
259
260
261
262
263
264
          "Cannot create Writable records from non-Writable class"+ clazz.getCanonicalName());
    }
  }

  public static <W extends Writable> WritableType<W, W> writables(Class<W> clazz) {
    MapFn wIdentity = IdentityFn.getInstance();
    return new WritableType<W, W>(clazz, clazz, wIdentity, wIdentity);
  }
View Full Code Here
455
456
457
458
459
460
461
462
463
    return new WritableType(clazz, TupleWritable.class, input, output, ptypes);
  }

  public static <S, T> PType<T> derived(Class<T> clazz, MapFn<S, T> inputFn, MapFn<T, S> outputFn, PType<S> base) {
    WritableType<S, ?> wt = (WritableType<S, ?>) base;
    MapFn input = new CompositeMapFn(wt.getInputMapFn(), inputFn);
    MapFn output = new CompositeMapFn(outputFn, wt.getOutputMapFn());
    return new WritableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new PType[0]));
  }
View Full Code Here
338
339
340
341
342
343
344
345
          "Cannot create Writable records from non-Writable class"+ clazz.getCanonicalName());
    }
  }

  public static <W extends Writable> WritableType<W, W> writables(Class<W> clazz) {
    MapFn wIdentity = IdentityFn.getInstance();
    return new WritableType(clazz, clazz, wIdentity, wIdentity);
  }
View Full Code Here
675
676
677
678
679
680
681
682
683
    return new WritableType(Union.class, UnionWritable.class, input, output, ptypes);
  }

  public static <S, T> PType<T> derived(Class<T> clazz, MapFn<S, T> inputFn, MapFn<T, S> outputFn, PType<S> base) {
    WritableType<S, ?> wt = (WritableType<S, ?>) base;
    MapFn input = new CompositeMapFn(wt.getInputMapFn(), inputFn);
    MapFn output = new CompositeMapFn(outputFn, wt.getOutputMapFn());
    return new WritableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new PType[0]));
  }
View Full Code Here
682
683
684
685
686
687
688
689
690
    return new WritableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new PType[0]));
  }

  public static <S, T> PType<T> derivedImmutable(Class<T> clazz, MapFn<S, T> inputFn, MapFn<T, S> outputFn, PType<S> base) {
    WritableType<S, ?> wt = (WritableType<S, ?>) base;
    MapFn input = new CompositeMapFn(wt.getInputMapFn(), inputFn);
    MapFn output = new CompositeMapFn(outputFn, wt.getOutputMapFn());
    return WritableType.immutableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new PType[0]));
  }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
          job.getConfiguration(),
          CrunchInputFormat.class,
          converter.getKeyClass(),
          converter.getValueClass());
      input.rdd().setName(source.toString());
      MapFn mapFn = converter.applyPTypeTransforms() ? source.getType().getInputMapFn() : IdentityFn.getInstance();
      return input
          .map(new InputConverterFunction(source.getConverter()))
          .map(new PairMapFunction(mapFn, runtime.getRuntimeContext()));
    } catch (IOException e) {
      throw new RuntimeException(e);
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
          job.getConfiguration(),
          CrunchInputFormat.class,
          converter.getKeyClass(),
          converter.getValueClass());
      input.rdd().setName(source.toString());
      MapFn mapFn = converter.applyPTypeTransforms() ? source.getType().getInputMapFn() : IdentityFn.getInstance();
      return input
          .map(new InputConverterFunction(source.getConverter()))
          .map(new MapFunction(mapFn, runtime.getRuntimeContext()));
    } catch (IOException e) {
      throw new RuntimeException(e);
View Full Code Here
97
98
99
100
101
102
103
104
105
 
  // derived, but override subtypes
  static <S, T> PType<T> derived(Class<T> clazz, MapFn<S, T> inputFn, MapFn<T, S> outputFn,
      PType<S> base, PType[] subTypes) {
    WritableType<S, ?> wt = (WritableType<S, ?>) base;
    MapFn input = new CompositeMapFn(wt.getInputMapFn(), inputFn);
    MapFn output = new CompositeMapFn(outputFn, wt.getOutputMapFn());
    return new WritableType(clazz, wt.getSerializationClass(), input, output, subTypes);
  }
View Full Code Here
337
338
339
340
341
342
343
344
          "Cannot create Writable records from non-Writable class"+ clazz.getCanonicalName());
    }
  }

  public static <W extends Writable> WritableType<W, W> writables(Class<W> clazz) {
    MapFn wIdentity = IdentityFn.getInstance();
    return new WritableType(clazz, clazz, wIdentity, wIdentity);
  }
View Full Code Here
674
675
676
677
678
679
680
681
682
    return new WritableType(Union.class, UnionWritable.class, input, output, ptypes);
  }

  public static <S, T> PType<T> derived(Class<T> clazz, MapFn<S, T> inputFn, MapFn<T, S> outputFn, PType<S> base) {
    WritableType<S, ?> wt = (WritableType<S, ?>) base;
    MapFn input = new CompositeMapFn(wt.getInputMapFn(), inputFn);
    MapFn output = new CompositeMapFn(outputFn, wt.getOutputMapFn());
    return new WritableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new PType[0]));
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.MapFn

  • org.apache.crunch.impl.spark.collect.InputCollection
  • org.apache.crunch.impl.spark.collect.InputTable
  • org.apache.crunch.io.text.LineParser
  • org.apache.crunch.io.text.TextFileReaderFactory
  • org.apache.crunch.types.orc.Orcs
  • org.apache.crunch.types.writable.Writables
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.