Package com.datasalt.pangool.io

Examples of com.datasalt.pangool.io.Tuple

56
57
58
59
60
61
62
      this.n = n;
    }

    public void setup(TupleMRContext context, Collector collector) throws IOException, InterruptedException,
        TupleMRException {
      outputCountTuple = new Tuple(getOutputCountSchema());
    };
View Full Code Here
80
81
82
83
84
85
86
  private static class UserActivityProcessor extends TupleMapper<LongWritable, Text> {

    private Tuple tuple;

    public void setup(TupleMRContext context, Collector collector) throws IOException, InterruptedException {
      this.tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("my_schema"));
    }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    @Override
    public void map(LongWritable key, Text value, TupleMRContext context, Collector collector) throws IOException,
        InterruptedException {
      if(tuple == null) {
        tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema(0));
      }

      String[] fields = value.toString().trim().split("\t");
      tuple.set("intField", Integer.parseInt(fields[0]));
      tuple.set("strField", fields[1]);
View Full Code Here
110
111
112
113
114
115
116
  public static class CountCombinerHandler extends TupleReducer<ITuple, NullWritable> {

    private Tuple tuple;

    public void setup(TupleMRContext context, Collector collector) throws IOException, InterruptedException {
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("my_schema"));
    }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
    @Override
    public void map(LongWritable key, Text value, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {

      if(tuple == null) {
        tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("urlRegister"));
      }
      String[] fields = value.toString().split("\t");
      tuple.set("url", fields[0]);
      tuple.set("timestamp", Long.parseLong(fields[1]));
      tuple.set("ip", fields[2]);
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    @Override
    public void map(LongWritable key, Text value, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {
      if(tuple == null) {
        tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("urlMap"));
      }

      String[] fields = value.toString().split("\t");
      tuple.set("url", fields[0]);
      tuple.set("canonicalUrl", fields[1]);
View Full Code Here
54
55
56
57
58
59
60
    protected Tuple tuple;
    protected ObjectMapper mapper;

    public void setup(TupleMRContext context, Collector collector) throws IOException, InterruptedException {
      this.mapper = new ObjectMapper();
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema(0));
    };
View Full Code Here
23
24
25
26
27
28
29
30
31
32
33
      fS = FileSystem.getLocal(conf);
    }
   
    for(FileStatus fStatus: fS.globStatus(path)) {
      TupleFile.Reader reader = new TupleFile.Reader(fS,  conf, fStatus.getPath());
      Tuple tuple = new Tuple(reader.getSchema());
      while(reader.next(tuple)) {
        System.out.println(fStatus.getPath() + "\t" + tuple);
      }
      reader.close();
    }
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
    job.setOutput(new Path(output), new HadoopOutputFormat(NullOutputFormat.class), ITuple.class,
        NullWritable.class);
    // The reducer will just emit the tuple to the corresponding Category output
    job.setTupleReducer(new TupleReducer<ITuple, NullWritable>() {

      ITuple outTuple = new Tuple(OUT_SCHEMA);

      public void reduce(ITuple group, Iterable<ITuple> tuples, TupleMRContext context,
          Collector collector) throws IOException, InterruptedException, TupleMRException {

        for(ITuple tuple : tuples) {
          Category category = (Category) tuple.get("category");
          outTuple.set("line", tuple.get("line"));
          outTuple.set("text", tuple.get("text"));
          outTuple.set("title", tuple.get("title"));
          collector.getNamedOutput(category.toString().toLowerCase())
              .write(outTuple, NullWritable.get());
        }
      }
    });
View Full Code Here
81
82
83
84
85
86
87
    private Tuple tuple;

    public void setup(TupleMRContext context, Collector collector) throws IOException,
        InterruptedException {
      this.tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("my_schema"));
    }
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.io.Tuple

  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$RetweetsMapper
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$TweetsMapper
  • com.datasalt.pangool.examples.avro.TestAvroTopicalWordCount
  • com.datasalt.pangool.examples.gameoflife.GameOfLifeJob
  • com.datasalt.pangool.examples.movingaverage.MovingAverage$URLVisitsProcessor
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesClassifier
  • com.datasalt.pangool.examples.naivebayes.NaiveBayesGenerate
  • com.datasalt.pangool.examples.secondarysort.SecondarySort$IProcessor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.