Examples of WeightedIntDocVector

ivory.core.data.document.WeightedIntDocVector
Implementation of {@link IntDocVector} with term weights @author Earl J. Wagner

Examples of ivory.core.data.document.WeightedIntDocVector

    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);


    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();


    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(opennlpIndex + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    System.out.println("opennlpInt1\n"+key+";"+value);

View Full Code Here

Examples of ivory.core.data.document.WeightedIntDocVector

    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);


    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();


    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(enwikiEn + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    verifyIntDocVector(enIntDocVector1, value);

View Full Code Here

Examples of ivory.core.data.document.WeightedIntDocVector

    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);


    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();


    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(dewikiEn + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    verifyIntDocVector(deIntDocVector1, value);

View Full Code Here

Examples of ivory.core.data.document.WeightedIntDocVector

        for (MapIF.Entry e : vectorWeights.entrySet()) {
          float score = vectorWeights.get(e.getKey());
          vectorWeights.put(e.getKey(), score / sum2);
        }
      }
      WeightedIntDocVector weightedVector = new WeightedIntDocVector(docLen, vectorWeights);
      output.collect(mDocno, weightedVector);
      reporter.incrCounter(Docs.Total, 1);
    }

View Full Code Here

Examples of ivory.core.data.document.WeightedIntDocVector

    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);


    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();


    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(galagoIndex + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    System.out.println("galagoInt1\n"+key+";"+value);

View Full Code Here

Examples of ivory.core.data.document.WeightedIntDocVector

    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);


    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();


    reader = new SequenceFile.Reader(fs.getConf(),
        SequenceFile.Reader.file(new Path(opennlpIndex + "/test_wt-int-doc-vectors/part-00000")));
    reader.next(key, value);
    System.out.println("opennlpInt1\n"+key+";"+value);

View Full Code Here

Examples of ivory.core.data.document.WeightedIntDocVector

        for (MapIF.Entry e : vectorWeights.entrySet()) {
          float score = vectorWeights.get(e.getKey());
          vectorWeights.put(e.getKey(), score / sum2);
        }
      }
      WeightedIntDocVector weightedVector = new WeightedIntDocVector(docLen, vectorWeights);
      output.collect(mDocno, weightedVector);
      reporter.incrCounter(Docs.Total, 1);
    }

View Full Code Here

Examples of ivory.core.data.document.WeightedIntDocVector

  public void verifyIntDocVectorsEn() throws Exception {
    System.out.println("verifyIntDocVectorsEn");
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);


    WeightedIntDocVector value = findIntDocVector(enIntDocVector1Id, 2, fs, enwikiEn + "/wt-int-doc-vectors");
    System.out.println("enIntDocVector1\n"+value);
    verifyIntDocVector(enIntDocVector1, value);


    value = findIntDocVector(enIntDocVector2Id, 11, fs, enwikiEn + "/wt-int-doc-vectors");
    System.out.println("enIntDocVector2\n"+value);

View Full Code Here

Examples of ivory.core.data.document.WeightedIntDocVector

  public void verifyIntDocVectorsDe() throws Exception {
    System.out.println("verifyIntDocVectorsDe");
    Configuration conf = IntegrationUtils.getBespinConfiguration();
    FileSystem fs = FileSystem.get(conf);


    WeightedIntDocVector value = findIntDocVector(deIntDocVector1Id, 2, fs, dewikiEn + "/wt-int-doc-vectors");
    System.out.println("deIntDocVector1\n"+value);
    verifyIntDocVector(deIntDocVector1, value);


    value = findIntDocVector(deIntDocVector2Id, 11, fs, dewikiEn + "/wt-int-doc-vectors");
    System.out.println("deIntDocVector2\n"+value);

View Full Code Here

Examples of ivory.core.data.document.WeightedIntDocVector

  }
  
  private WeightedIntDocVector findIntDocVector(int docno, int startFrom, FileSystem fs, String dir) {
    SequenceFile.Reader reader;
    IntWritable key = new IntWritable();
    WeightedIntDocVector value = new WeightedIntDocVector();
    try {
      FileStatus[] paths = fs.listStatus(new Path(dir));
      System.err.println("length " + paths.length);
      for (FileStatus path : paths) {
        if (!path.getPath().getName().contains("part"))  continue;

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.