Package org.apache.crunch

Examples of org.apache.crunch.ParallelDoOptions


  private PTable<K, Pair<U,V>> joinInternal(PTable<K, U> left, PTable<K, V> right, boolean includeUnmatchedLeftValues) {
    PTypeFamily tf = left.getTypeFamily();
    ReadableData<Pair<K, V>> rightReadable = right.asReadable(materialize);
    MapsideJoinDoFn<K, U, V> mapJoinDoFn = new MapsideJoinDoFn<K, U, V>(
              rightReadable, right.getPTableType(), includeUnmatchedLeftValues);
    ParallelDoOptions options = ParallelDoOptions.builder()
        .sourceTargets(rightReadable.getSourceTargets())
        .build();
    return left.parallelDo("mapjoin", mapJoinDoFn,
        tf.tableOf(left.getKeyType(), tf.pairs(left.getValueType(), right.getValueType())),
        options);
View Full Code Here


  @Test
  public void testFormatNodePath() {
    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);
    ParallelDoOptions doOptions = ParallelDoOptions.builder().build();

    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(doOptions);
    when(head.getParallelDoOptions()).thenReturn(doOptions);
View Full Code Here

    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    SourceTarget<?> srcTarget = mock(SourceTarget.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);

    ParallelDoOptions tailOptions = ParallelDoOptions.builder().sourceTargets(srcTarget).build();
    ParallelDoOptions headOptions = ParallelDoOptions.builder().build();
    when(srcTarget.toString()).thenReturn("target");
    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(tailOptions);
    when(head.getParallelDoOptions()).thenReturn(headOptions);
View Full Code Here

  @Test
  public void testFormatNodePath() {
    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);
    ParallelDoOptions doOptions = ParallelDoOptions.builder().build();

    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(doOptions);
    when(head.getParallelDoOptions()).thenReturn(doOptions);
View Full Code Here

    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    SourceTarget<?> srcTarget = mock(SourceTarget.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);

    ParallelDoOptions tailOptions = ParallelDoOptions.builder().sourceTargets(srcTarget).build();
    ParallelDoOptions headOptions = ParallelDoOptions.builder().build();
    when(srcTarget.toString()).thenReturn("target");
    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(tailOptions);
    when(head.getParallelDoOptions()).thenReturn(headOptions);
View Full Code Here

  private PTable<K, Pair<U,V>> joinInternal(PTable<K, U> left, PTable<K, V> right, boolean includeUnmatchedLeftValues) {
    PTypeFamily tf = left.getTypeFamily();
    ReadableData<Pair<K, V>> rightReadable = right.asReadable(materialize);
    MapsideJoinDoFn<K, U, V> mapJoinDoFn = new MapsideJoinDoFn<K, U, V>(
              rightReadable, right.getPTableType(), includeUnmatchedLeftValues);
    ParallelDoOptions options = ParallelDoOptions.builder()
        .sourceTargets(rightReadable.getSourceTargets())
        .build();
    return left.parallelDo("mapjoin", mapJoinDoFn,
        tf.tableOf(left.getKeyType(), tf.pairs(left.getValueType(), right.getValueType())),
        options);
View Full Code Here

  private PTable<K, Pair<U,V>> joinInternal(PTable<K, U> left, PTable<K, V> right, boolean includeUnmatchedLeftValues) {
    PTypeFamily tf = left.getTypeFamily();
    ReadableData<Pair<K, V>> rightReadable = right.asReadable(materialize);
    MapsideJoinDoFn<K, U, V> mapJoinDoFn = new MapsideJoinDoFn<K, U, V>(rightReadable, includeUnmatchedLeftValues);
    ParallelDoOptions options = ParallelDoOptions.builder()
        .sourceTargets(rightReadable.getSourceTargets())
        .build();
    return left.parallelDo("mapjoin", mapJoinDoFn,
        tf.tableOf(left.getKeyType(), tf.pairs(left.getValueType(), right.getValueType())),
        options);
View Full Code Here

  @Test
  public void testFormatNodePath() {
    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);
    ParallelDoOptions doOptions = ParallelDoOptions.builder().build();

    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(doOptions);
    when(head.getParallelDoOptions()).thenReturn(doOptions);
View Full Code Here

    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    SourceTarget<?> srcTarget = mock(SourceTarget.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);

    ParallelDoOptions tailOptions = ParallelDoOptions.builder().sourceTargets(srcTarget).build();
    ParallelDoOptions headOptions = ParallelDoOptions.builder().build();
    when(srcTarget.toString()).thenReturn("target");
    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(tailOptions);
    when(head.getParallelDoOptions()).thenReturn(headOptions);
View Full Code Here

TOP

Related Classes of org.apache.crunch.ParallelDoOptions

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.