Package com.datasalt.pangool.tuplemr.mapred.lib.output

Examples of com.datasalt.pangool.tuplemr.mapred.lib.output.HadoopOutputFormat

156
157
158
159
160
161
162
163
164
165
166
    builder.setSpecificOrderBy("user", new OrderBy().add("money", Order.ASC));

    builder.addInput(new Path("test-input"), new HadoopInputFormat(TextInputFormat.class),
        new FirstInputProcessor());
    builder.setTupleReducer(new MyGroupHandler());
    builder.setOutput(new Path("test-output"), new HadoopOutputFormat(TextOutputFormat.class),
        NullWritable.class, NullWritable.class);

    Job job = builder.createJob();
    try {
      assertRun(job);
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
    builder.setGroupByFields("country", "age", "name");
    builder.setOrderBy(new OrderBy().add("country", Order.ASC).add("age", Order.ASC)
        .add("name", Order.ASC).add("height", Order.DESC));
    builder.setRollupFrom("country");
    builder.setTupleReducer(new IdentityRed());
    builder.setOutput(outputPath, new HadoopOutputFormat(SequenceFileOutputFormat.class), Text.class,
        Text.class);
    builder.addInput(new Path(input), new HadoopInputFormat(SequenceFileInputFormat.class), new Map());

    Job job = builder.createJob();
    try {
View Full Code Here
271
272
273
274
275
276
277
278
279
280
281
    builder.setGroupByFields("age", "name", "country");
    builder.setOrderBy(new OrderBy().add("country", Order.ASC).add("age", Order.ASC)
        .add("name", Order.ASC).add("height", Order.DESC));
    builder.setRollupFrom("age");
    builder.setTupleReducer(new IdentityRed());
    builder.setOutput(outputPath, new HadoopOutputFormat(SequenceFileOutputFormat.class), Text.class,
        Text.class);
    builder.addInput(new Path(input), new HadoopInputFormat(SequenceFileInputFormat.class), new Map());

    Job job = builder.createJob();
    try {
View Full Code Here
450
451
452
453
454
455
456
457
458
459
460
    builder.setGroupByFields("age", "name", "country");
    builder.setOrderBy(new OrderBy().add("country", Order.ASC).add("age", Order.ASC)
        .add("name", Order.ASC));
    builder.setRollupFrom("age");
    builder.setTupleReducer(new IdentityRed());
    builder.setOutput(outputPath, new HadoopOutputFormat(SequenceFileOutputFormat.class), Text.class,
        Text.class);
    builder.addInput(new Path(input), new HadoopInputFormat(SequenceFileInputFormat.class),
        new DoNothingMap());

    Job job = builder.createJob();
View Full Code Here
184
185
186
187
188
189
190
191
192
193
194
        assertEquals(4.6, ((ITuple)currentTuple.get("tuple2")).get("c"));
        assertEquals(false, ((ITuple)currentTuple.get("tuple2")).get("d"));
      };
    });
    builder.setGroupByFields("partitionId");
    builder.setOutput(new Path(OUTPUT), new HadoopOutputFormat(TextOutputFormat.class), Text.class, NullWritable.class);
    Job job = builder.createJob();
    try {
      job.waitForCompletion(true);
    } finally {
      builder.cleanUpInstanceFiles();
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
      protected void map(LongWritable key, Text value, Context context) throws IOException ,InterruptedException {
        value.set(value.toString() + "\t" + classify(value.toString()));
        context.write(value, NullWritable.get());
      }
    });
    job.setOutput(new Path(output), new HadoopOutputFormat(TextOutputFormat.class), Text.class, NullWritable.class);
    job.addInput(new Path(input), new HadoopInputFormat(TextInputFormat.class));
    job.createJob().waitForCompletion(true);
   
    return 1;
  }
View Full Code Here
325
326
327
328
329
330
331
332
333
334
335
    }
    // Set output path
    Path outReservoirPath = new Path(outputPath + "-reservoir");
    builder.setOutput(
        outReservoirPath,
        new HadoopOutputFormat(SequenceFileOutputFormat.class),
        Text.class,
        NullWritable.class);
    builder.setJarByClass(callingClass);

    try {
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
    delete(output);
   
    init(conf, new Path(modelFolder));
   
    MapOnlyJobBuilder job = new MapOnlyJobBuilder(conf);
    job.setOutput(new Path(output), new HadoopOutputFormat(TextOutputFormat.class), Text.class, NullWritable.class);
    job.addInput(new Path(input), new HadoopInputFormat(TextInputFormat.class), new MapOnlyMapper<LongWritable, Text, Text, NullWritable>() {
      protected void map(LongWritable key, Text value, Context context) throws IOException ,InterruptedException {
        value.set(value.toString() + "\t" + classify(value.toString()));
        context.write(value, NullWritable.get());
      }
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.tuplemr.mapred.lib.output.HadoopOutputFormat

  • com.datasalt.pangool.examples.gameoflife.GameOfLifeJob
  • com.datasalt.pangool.examples.Grep
  • com.datasalt.pangool.examples.HCatalogIntegrationTest
  • com.datasalt.pangool.examples.mongo.QypeScrapper
  • com.datasalt.pangool.examples.movingaverage.MovingAverage
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesClassifier
  • com.datasalt.pangool.examples.secondarysort.SecondarySort
  • com.datasalt.pangool.examples.simplesecondarysort.SimpleSecondarySort
  • com.datasalt.pangool.examples.solr.MultiShakespeareIndexer
  • com.datasalt.pangool.examples.topnhashtags.TopNHashTags
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.