Package com.senseidb.indexing.hadoop.keyvalueformat

Examples of com.senseidb.indexing.hadoop.keyvalueformat.IntermediateForm

113
114
115
116
117
118
119
120
121
122
123
124
125
126
                      doc.add(new Field(AbstractZoieIndexable.DOCUMENT_STORE_FIELD,bytes));
                    }
                  }
             
              //now we have uid and lucene Doc;
              IntermediateForm form = new IntermediateForm();
              form.configure(_conf);
              form.process(doc, analyzer);
              form.closeWriter();

              int chosenShard = -1;
          try {
            chosenShard = _shardingStategy.caculateShard(_shards.length, json);
          } catch (JSONException e) {
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
  public void reduce(Shard key, Iterator<IntermediateForm> values,
      OutputCollector<Shard, IntermediateForm> output, Reporter reporter)
      throws IOException {

    String message = key.toString();
    IntermediateForm form = null;

    while (values.hasNext()) {
      IntermediateForm singleDocForm = values.next();
      long formSize = form == null ? 0 : form.totalSizeInBytes();
      long singleDocFormSize = singleDocForm.totalSizeInBytes();

      if (form != null && formSize + singleDocFormSize > maxSizeInBytes) {
        closeForm(form, message);
        output.collect(key, form);
        form = null;
View Full Code Here
84
85
86
87
88
89
90
91
92
    }
  }

  private IntermediateForm createForm(String message) throws IOException {
  logger.info("Construct a form writer for " + message);
    IntermediateForm form = new IntermediateForm();
    form.configure(iconf);
    return form;
  }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
        logger.info("mapredTempDir is: "+ mapredTempDir);
        final ShardWriter writer = new ShardWriter(fs, key, temp, iconf);

        // update the shard
        while (values.hasNext()) {
          IntermediateForm form = values.next();
          writer.process(form);
          reporter.progress();
        }

        // close the shard
View Full Code Here
TOP

Related Classes of com.senseidb.indexing.hadoop.keyvalueformat.IntermediateForm

  • com.senseidb.indexing.hadoop.map.SenseiMapper
  • com.senseidb.indexing.hadoop.reduce.SenseiCombiner
  • com.senseidb.indexing.hadoop.reduce.SenseiReducer
  • org.apache.lucene.index.IndexWriter
  • org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy
  • org.apache.lucene.store.RAMDirectory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.