Package org.apache.crunch.impl.mr

Examples of org.apache.crunch.impl.mr.MRPipeline.done()

49
50
51
52
53
54
55
56
57
58
59
            return Pair.of(p[0], p[1]);
          }
        }, Avros.tableOf(Avros.strings(), Avros.strings()))
        .groupByKey()
        .write(new AvroPathPerKeyTarget(outDir));
    p.done();

    Set<String> names = Sets.newHashSet();
    FileSystem fs = outDir.getFileSystem(tempDir.getDefaultConfiguration());
    for (FileStatus fstat : fs.listStatus(outDir)) {
      names.add(fstat.getPath().getName());
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
    PCollection<Put> resultPut = createPut(result);

    // We write the puts in hbase, in the target table
    pipeline.write(resultPut, new HBaseTarget(TABLE_TARGET));

    pipeline.done();
    return 0;
  }

  /**
   * Put the puts in HBase
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    assertEquals(Lists.newArrayList(expected), result);
   
    OrcFileTarget target = new OrcFileTarget(outputPath);
    pipeline.write(rows, target);
   
    assertTrue(pipeline.done().succeeded());
   
    OrcFileReaderFactory<T> reader = new OrcFileReaderFactory<T>(ptype);
    List<T> newResult = Lists.newArrayList(reader.read(fs, inputPath));
   
    assertEquals(Lists.newArrayList(expected), newResult);
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    File outputDirA = tmpDir.getFile("output_a");
    File outputDirB = tmpDir.getFile("output_b");
   
    pipeline.writeTextFile(ungroupedTableA, outputDirA.getAbsolutePath());
    pipeline.writeTextFile(ungroupedTableB, outputDirB.getAbsolutePath());
    PipelineResult result = pipeline.done();
    for(StageResult stageResult : result.getStageResults()){
      assertTrue(stageResult.getStageName().length() > 1);
      assertTrue(stageResult.getStageId().length() > 1);
    }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
        Writables.tableOf(Writables.strings(), Writables.doubles()));

    // write the result to a text file
    pipeline.writeTextFile(avgs, args[1]);
    // Execute the pipeline as a MapReduce.
    PipelineResult result = pipeline.done();

    return result.succeeded() ? 0 : 1;
  }

  // Function to calculate the average response size for a given ip address
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
    // Execute the pipeline as a MapReduce.
    PipelineResult result = pipeline.run();

    System.out.println("Total number of words: " + totalCount.getValue());
   
    pipeline.done();

    return result.succeeded() ? 0 : 1;
  }

  public static void main(String[] args) throws Exception {
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
    PCollection<Put> resultPut = createPut(result);

    // We write the puts in hbase, in the target table
    pipeline.write(resultPut, new HBaseTarget(TABLE_TARGET));

    pipeline.done();
    return 0;
  }

  /**
   * Put the puts in HBase
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
            emitter.emit(sb.toString());
          }
        }, Writables.strings()).write(To.textFile(args[1]));

    // Execute the pipeline as a MapReduce.
    return pipeline.done().succeeded() ? 0 : 1;
  }

  public static void main(String[] args) throws Exception {
    int exitCode = -1;
    try {
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
      }

    }, Writables.strings());

    List<String> nameList = Lists.newArrayList(names.materialize());
    pipeline.done();

    assertEquals(2, nameList.size());
    assertEquals(Sets.newHashSet("Hello", "World"), Sets.newHashSet(nameList));

  }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
        Writables.tableOf(Writables.strings(), Writables.doubles()));

    // write the result to a text file
    pipeline.writeTextFile(avgs, args[1]);
    // Execute the pipeline as a MapReduce.
    PipelineResult result = pipeline.done();

    return result.succeeded() ? 0 : 1;
  }

  // Function to calculate the average response size for a given ip address
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.