Package edu.umd.cloud9.collection.trec

Examples of edu.umd.cloud9.collection.trec.TrecDocnoMappingBuilder

658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
  public DocnoMapping getDocnoMapping() throws IOException {
    return loadDocnoMapping(indexPath, fs);
  }

  public static DocnoMapping loadDocnoMapping(String indexPath, FileSystem fs) throws IOException {
    DocnoMapping mDocMapping = null;
    // load the docid to docno mappings
    try {
      LOG.info("Loading DocnoMapping file...");
      RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);

      String className = env.readDocnoMappingClass();
      LOG.info(" - Class name: " + className);
      mDocMapping = (DocnoMapping) Class.forName(className).newInstance();

      Path mappingFile = env.getDocnoMappingData();
      LOG.info(" - File name: " + mappingFile);
      mDocMapping.loadMapping(mappingFile, fs);
      LOG.info("Done!");
    } catch (Exception e) {
      throw new IOException("Error initializing DocnoMapping!");
    }
    return mDocMapping;
View Full Code Here
359
360
361
362
363
364
365
366
367
368
369
    if (list.getLength() > 0) {  conf.set(Constants.StemmedStopwordListQ, list.item(0).getTextContent());  } 
  }

  static float eval(QueryEngine qe, Configuration conf, String setting){
    Qrels qrels = new Qrels(conf.get(Constants.QrelsPath));
    DocnoMapping mapping = qe.getDocnoMapping();
    float apSum = 0, p10Sum = 0;
    Map<String, Accumulator[]> results = qe.getResults();
    for (String qid : results.keySet()) {
      float ap = (float) RankedListEvaluator.computeAP(results.get(qid), mapping,
          qrels.getReldocsForQid(qid));
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
      return -1;
    }

    RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);
    Path mappingFile = env.getDocnoMappingData();
    new ClueWarcDocnoMappingBuilder().build(new Path(collection), mappingFile, conf);

    conf.set(Constants.CollectionName, "ClueWeb:English:Segment" + segment);
    conf.set(Constants.CollectionPath, collection);
    conf.set(Constants.IndexPath, indexPath);
    conf.set(Constants.InputFormat, SequenceFileInputFormat.class.getCanonicalName());
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
    if (!fs.exists(p)) {
      sLogger.info("docno-mapping.dat doesn't exist, creating...");
      String[] arr = new String[] { collection, mappingDir.toString(), mappingFile.toString(),
          new Integer(numMappers).toString() };
      NumberTextDocuments tool = new NumberTextDocuments();
      tool.setConf(conf);
      tool.run(arr);

      fs.delete(mappingDir, true);
    }

    // Now we're ready to start the preprocessing pipeline... set
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
    // (sequentially-number integer). If it doesn't exist create it.
    Path mappingFile = env.getDocnoMappingData();
    if (!fs.exists(mappingFile)) {
      sLogger.info(mappingFile + " doesn't exist, creating...");
      String[] arr = new String[] { collection, indexPath + "/medline-docid-tmp",  mappingFile.toString(), new Integer(numMappers).toString() };
      NumberMedlineCitations tool = new NumberMedlineCitations();
      tool.setConf(conf);
      tool.run(arr);

      fs.delete(new Path(indexPath + "/medline-docid-tmp"), true);
    }

    // Now we're ready to start the preprocessing pipeline... set
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
    if (!fs.exists(mappingFile)) {
      LOG.info("docno-mapping.dat doesn't exist, creating...");
      String[] arr = new String[] { collection, mappingDir.toString(),
              mappingFile.toString() };
      NumberTrecDocuments2 tool = new NumberTrecDocuments2();
      tool.setConf(conf);
      tool.run(arr);

      fs.delete(mappingDir, true);
    }

    conf.set(Constants.CollectionName, "TREC_vol45");
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
    String indexRootPath = conf.get(Constants.IndexPath);
    String collection = conf.get(Constants.CollectionPath);

    RetrievalEnvironment env = new RetrievalEnvironment(indexRootPath, fs);
    Path mappingFile = env.getDocnoMappingData();
    new TrecDocnoMappingBuilder().build(new Path(collection), mappingFile, conf);

    conf.set(Constants.DocnoMappingClass, TrecDocnoMapping.class.getCanonicalName());
    conf.set(Constants.DocnoMappingFile, env.getDocnoMappingData().toString());
    conf.setInt(Constants.DocnoOffset, 0); // docnos start at 1
    conf.setInt(Constants.MinDf, 2); // toss away singleton terms
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
      return -1;
    }

    RetrievalEnvironment env = new RetrievalEnvironment(indexRootPath, fs);
    Path mappingFile = env.getDocnoMappingData();
    new TrecDocnoMappingBuilder().build(new Path(collection), mappingFile, conf);

    conf.set(Constants.CollectionName, "TREC_vol45");
    conf.set(Constants.CollectionPath, collection);
    conf.set(Constants.IndexPath, indexRootPath);
    conf.set(Constants.InputFormat, TrecDocumentInputFormat.class.getCanonicalName());
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
    String indexRootPath = conf.get(Constants.IndexPath);
    String collection = conf.get(Constants.CollectionPath);

    RetrievalEnvironment env = new RetrievalEnvironment(indexRootPath, fs);
    Path mappingFile = env.getDocnoMappingData();
    new TrecDocnoMappingBuilder().build(new Path(collection), mappingFile, conf);

    conf.set(Constants.DocnoMappingClass, TrecDocnoMapping.class.getCanonicalName());
    conf.set(Constants.DocnoMappingFile, env.getDocnoMappingData().toString());
    conf.setInt(Constants.DocnoOffset, 0); // docnos start at 1
    conf.setInt(Constants.MinDf, 2); // toss away singleton terms
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
    String indexRootPath = conf.get(Constants.IndexPath);
    String collection = conf.get(Constants.CollectionPath);

    RetrievalEnvironment env = new RetrievalEnvironment(indexRootPath, fs);
    Path mappingFile = env.getDocnoMappingData();
    new TrecDocnoMappingBuilder().build(new Path(collection), mappingFile, conf);

    conf.set(Constants.DocnoMappingClass, TrecDocnoMapping.class.getCanonicalName());
    conf.set(Constants.DocnoMappingFile, env.getDocnoMappingData().toString());
    conf.setInt(Constants.DocnoOffset, 0); // docnos start at 1
    conf.setInt(Constants.MinDf, 2); // toss away singleton terms
View Full Code Here
TOP

Related Classes of edu.umd.cloud9.collection.trec.TrecDocnoMappingBuilder

  • bak.pcj.IntIterator
  • edu.umd.cloud9.example.bfs.BfsNodeTest
  • edu.umd.cloud9.example.pagerank.PageRankNodeTest
  • edu.umd.cloud9.io.array.ArrayListOfIntsWritable
  • edu.umd.cloud9.io.fastutil.Int2FloatOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.Int2IntOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.String2FloatOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.String2IntOpenHashMapWritableTest
  • edu.umd.cloud9.io.map.HMapIDW
  • edu.umd.cloud9.io.map.HMapIDWTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.