Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.IntWritable

134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
   
    List<Integer> keys = new ArrayList<Integer>();
    List<Step0Output> values = new ArrayList<Step0Output>();
   
    // read all the outputs
    IntWritable key = new IntWritable();
    Step0Output value = new Step0Output(0L, 0);
   
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, conf);
     
      try {
        while (reader.next(key, value)) {
          keys.add(key.get());
          values.add(value.clone());
        }
      } finally {
        reader.close();
      }
View Full Code Here
231
232
233
234
235
236
237
      size++;
    }
   
    @Override
    protected void cleanup(Context context) throws IOException, InterruptedException {
      context.write(new IntWritable(partition), new Step0Output(firstId, size));
    }
View Full Code Here
200
201
202
203
204
205
206
        firstId = new LongWritable(key.get());
      }
     
      Instance instance = converter.convert((int) key.get(), value.toString());
     
      context.write(firstId, new IntWritable(instance.label));
    }
View Full Code Here
153
154
155
156
157
158
159
160
      return value;
    }
   
    @Override
    public void initialize(InputSplit arg0, TaskAttemptContext arg1) throws IOException, InterruptedException {
      key = new IntWritable();
      value = NullWritable.get();
    }
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
   
    int[] keys = new int[numMaps];
    Step0Output[] values = new Step0Output[numMaps];
   
    // read all the outputs
    IntWritable key = new IntWritable();
    Step0Output value = new Step0Output(0L, 0);
   
    int index = 0;
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, job);
     
      try {
        while (reader.next(key, value)) {
          keys[index] = key.get();
          values[index] = value.clone();
         
          index++;
        }
      } finally {
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
    Path path = new Path(outPath, "largestCleanEigens");
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
    VectorWritable vw = new VectorWritable();
    IntWritable iw = new IntWritable();
    for(Map.Entry<MatrixSlice,EigenStatus> pruneSlice : prunedEigenMeta) {
      MatrixSlice s = pruneSlice.getKey();
      EigenStatus meta = pruneSlice.getValue();
      EigenVector ev = new EigenVector((DenseVector)s.vector(),
                                       meta.getEigenValue(),
                                       Math.abs(1-meta.getCosAngle()),
                                       s.index());
      log.info("appending " + ev.getName() + " to " + path.toString());
      vw.set(ev);
      iw.set(s.index());
      seqWriter.append(iw, vw);
    }
    seqWriter.close();
  }
View Full Code Here
235
236
237
238
239
240
241
      size++;
    }
   
    @Override
    public void close() throws IOException {
      collector.collect(new IntWritable(partition), new Step0Output(firstId, size));
    }
View Full Code Here
129
130
131
132
133
134
135
    public void close() throws IOException {
    }
   
    @Override
    public IntWritable createKey() {
      return new IntWritable();
    }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
    FileSystem fs = outputPath.getFileSystem(conf);
   
    Path[] outfiles = DFUtils.listOutputFiles(fs, outputPath);
   
    // import the InMemOutputs
    IntWritable key = new IntWritable();
    MapredOutput value = new MapredOutput();
   
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, conf);
     
      try {
        while (reader.next(key, value)) {
          output.put(key.get(), value.clone());
        }
      } finally {
        reader.close();
      }
    }
View Full Code Here
50
51
52
53
54
55
56
57
58
                                                         path,
                                                         intWritable ? IntWritable.class : LongWritable.class,
                                                         VectorWritable.class);
    int recNum = 0;
    for (VectorWritable point : points) {
      writer.append(intWritable ? new IntWritable(recNum++) : new LongWritable(recNum++), point);
    }
    writer.close();
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.IntWritable

  • bbejeck.mapred.aggregation.AverageTemperatureMapReduceTest
  • bbejeck.mapred.aggregation.TemperatureAveragingPair
  • bbejeck.mapred.coocurrance.PairsRelativeOccurrenceReducerTest
  • brickhouse.udf.collect.CollectMaxTest
  • co.nubetech.hiho.dedup.TestDedupJob
  • co.nubetech.hiho.merge.TestMergeJob
  • co.nubetech.hiho.similarity.ngram.TestScoreMapper
  • co.nubetech.hiho.similarity.ngram.TestScoreReducer
  • com.alimama.mdrill.index.IndexReducer
  • com.datasalt.pangool.utils.HadoopUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.