Package uk.ac.cam.ch.wwmm.oscar3.flow

Examples of uk.ac.cam.ch.wwmm.oscar3.flow.OscarFlow


  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexSearcher is = lis.getIndexSearcher();
    IndexReader ir = lis.getIndexReader();
   
    TermEnum textEnum = ir.terms();
    Map<String,Integer> docFreqs = new HashMap<String,Integer>();
    float nd = ir.numDocs() * 1.0f;
    while(textEnum.next()) {
View Full Code Here


 
  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexSearcher is = lis.getIndexSearcher();

    TreeNode tn = new TreeNode();
   
    tn.subCluster(is);
   
View Full Code Here

  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexReader ir = lis.getIndexReader();
    Directory dir = new RAMDirectory(ir.directory());
    ir.close();
    IndexSearcher is = new IndexSearcher(dir);
    ir = is.getIndexReader();

View Full Code Here

  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexSearcher is = lis.getIndexSearcher();
    IndexReader ir = lis.getIndexReader();
    Bag<String> tfBag = new Bag<String>();
    for(int i=0;i<ir.maxDoc();i++) {
      TermFreqVector tv = ir.getTermFreqVector(i, "txt");
      String [] terms = tv.getTerms();
      int [] freqs = tv.getTermFrequencies();
View Full Code Here

 
  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexReader ir = lis.getIndexReader();
   
    long allTime = System.currentTimeMillis();
   
    int numDocs = ir.numDocs();
    TermEnum textEnum = ir.terms();
View Full Code Here

  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexSearcher is = lis.getIndexSearcher();
    IndexReader ir = lis.getIndexReader();
   
    List<String> docFiles = new ArrayList<String>();
    for(int i=0;i<ir.numDocs();i++) {
      docFiles.add(ir.document(i).getField("filename").stringValue().replaceAll("markedup", "source"));
    }
   
    TermEnum textEnum = ir.terms();
    Map<String,Integer> docFreqs = new HashMap<String,Integer>();
    while(textEnum.next()) {
      Term t = textEnum.term();
      if("txt".equals(t.field())) {
        String text = t.text();
        if(TermSets.getClosedClass().contains(text)) continue;
        if(!text.matches(".*[A-Za-z].*")) continue;
        int docFreq = ir.docFreq(t);
        if(docFreq > 1) {
          docFreqs.put(text, ir.docFreq(t));
        }
      }     
    }
    List<String> dfl = StringTools.getSortedList(docFreqs);

    List<Query> queries = new ArrayList<Query>();
    for(int i=0;i<50;i++) {
      queries.add(new TermQuery(new Term("txt", dfl.get(i))));
      System.out.println(dfl.get(i));
    }
   
    for(int i=0;i<10;i++) {
      Map<Integer,Integer> bestClusters = new HashMap<Integer,Integer>();
      Map<Integer,Float> bestClusterScores = new HashMap<Integer,Float>();
      List<Map<Integer,Float>> clusters = new ArrayList<Map<Integer,Float>>();
      for(int j=0;j<queries.size();j++) {
        clusters.add(new HashMap<Integer,Float>());
        VectorCollector vc = new VectorCollector();
        is.search(queries.get(j), vc);
        //System.out.println(vc.getResultsVector());
        for(Integer k : vc.getResultsVector().keySet()) {
          float score = vc.getResultsVector().get(k);
          if(score < 0.001) continue;
          if(!bestClusterScores.containsKey(k) || bestClusterScores.get(k) < score) {
            bestClusters.put(k, j);
            bestClusterScores.put(k, score);
          }
        }
      }
      for(Integer j : bestClusters.keySet()) {
        clusters.get(bestClusters.get(j)).put(j, bestClusterScores.get(j));
      }
      //for(Map<Integer,Float> cluster : clusters) System.out.println(cluster);
      queries.clear();
      for(int j=0;j<clusters.size();j++) {
        System.out.println("Size: " + clusters.get(j).size());
        /*if(i == 9) {
          for(Integer k : clusters.get(j).keySet()) {
            System.out.println(docFiles.get(k) + "\t" + bestClusterScores.get(k));
          }
        }*/
        //if(i == 9) ClusterAnalyser.excessAnalyseCluster(clusters.get(j), lis.getIndexReader(), 0.2, true);
        Map<String,Double> scores = ClusterAnalyser.simpleExcessAnalyseCluster(clusters.get(j), lis.getIndexReader(), 0.1);
        BooleanQuery bq = new BooleanQuery(false);
        List<String> terms = StringTools.getSortedList(scores);
        if(terms.size() > 20) terms = terms.subList(0, 20);
        for(String s : terms) {
          System.out.println(s + "\t" + scores.get(s));
          TermQuery tq = new TermQuery(new Term("txt", s));
          tq.setBoost(scores.get(s).floatValue());
          bq.add(new BooleanClause(tq, Occur.SHOULD));
        }
        System.out.println();
        queries.add(bq);
      }
      System.out.println();
    }
    List<Map<Integer,Float>> clusters = new ArrayList<Map<Integer,Float>>();
    final Map<Integer,Integer> clusterSizes = new HashMap<Integer,Integer>();
    for(int j=0;j<queries.size();j++) {
      VectorCollector vc = new VectorCollector();
      is.search(queries.get(j), vc);
      final Map<Integer,Float> cluster = new HashMap<Integer,Float>();
      //System.out.println(vc.getResultsVector());
      for(Integer k : vc.getResultsVector().keySet()) {
        float score = vc.getResultsVector().get(k);
        if(score < 0.2) continue;
        cluster.put(k, score);
      }
      clusters.add(cluster);
      clusterSizes.put(j, cluster.size());
    }
    List<Integer> clustersBySize = new ArrayList<Integer>(clusterSizes.keySet());
    Collections.sort(clustersBySize, Collections.reverseOrder(new Comparator<Integer>() {
      @SuppressWarnings("unchecked")
      public int compare(Integer o1, Integer o2) {
        return clusterSizes.get(o1).compareTo(clusterSizes.get(o2));
      }
    }));
   
    for(Integer j : clustersBySize) {
      final Map<Integer,Float> cluster = clusters.get(j);
      System.out.println("Size:\t" + cluster.size());
      List<Integer> list = new ArrayList<Integer>(cluster.keySet());
      Collections.sort(list, Collections.reverseOrder(new Comparator<Integer>() {
        @SuppressWarnings("unchecked")
        public int compare(Integer o1, Integer o2) {
          return cluster.get(o1).compareTo(cluster.get(o2));
        }
      }));
      for(Integer k : list) {
        System.out.println(docFiles.get(k) + "\t" + cluster.get(k));
      }
      ClusterAnalyser.excessAnalyseCluster(cluster, lis.getIndexReader(), 0.2, true);
      System.out.println();
    }


  }
View Full Code Here

      System.out.println(s + "\t" + tfIdf.get(s));
   
  }
 
  public static Map<String,Double> excessAnalyseCluster(Map<Integer,Float> cluster, IndexReader ir, double threshold, boolean enriched) throws Exception {
    LuceneChemicalIndex lci = new LuceneIndexerSearcher(false).getLci();
    Set<String> inchis = new HashSet<String>();
    Set<String> onts = new HashSet<String>();
   
    List<File> clusterFiles = new ArrayList<File>();
    for(Integer i : cluster.keySet()) {
View Full Code Here

    return scores;
  }

 
  public static void analyseCluster(Map<Integer,Float> cluster, IndexReader ir, DocVectorSimilarity similarity, double threshold) throws Exception {
    LuceneChemicalIndex lci = new LuceneIndexerSearcher(false).getLci();
    List<File> clusterFiles = new ArrayList<File>();
    Bag<String> dfs = new Bag<String>();
    Set<String> inchis = new HashSet<String>();
    Set<String> onts = new HashSet<String>();
    for(Integer i : cluster.keySet()) {
View Full Code Here

      System.out.println(term + "\t" + scores.get(term) + "\t" + overlaps.get(term));
    }
  }

  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexSearcher is = lis.getIndexSearcher();
   
    Stemmer stemmerTools = new Stemmer(new EnglishStemmer());
   
    //QueryParser qp = new Oscar3QueryParser("txt", new Oscar3Analyzer(), lis, false);
    //Query q = qp.parse("NaCl");
   
    String queryTerm = "content";
    //PhraseQuery pq = new PhraseQuery();
    //pq.add(new Term("txt", "aromatase"));
    //pq.add(new Term("txt", "inhibitors"));
    //Query q = new TermQuery(new Term("txt", queryTerm));
    Query q = new StemQuery(new Term("txt", queryTerm), stemmerTools);
    //q = pq;
    VectorCollector vc = new VectorCollector();
    is.search(q, vc);
    Map<String,Double> scores = simpleExcessAnalyseCluster(vc.getResultsVector(), lis.getIndexReader(), 0.01);
    for(String s : StringTools.getSortedList(scores)) {
      System.out.println(s + "\t" + scores.get(s));
    }

    //tfIdfAnalyseCluster(vc.getResultsVector(), lis.getIndexReader());
View Full Code Here

    //  return;
    //}
   
    ChiSquaredDistribution csd = new ChiSquaredDistributionImpl(1);
    List<Query> lq = new ArrayList<Query>();
    TextMiner tm = new TextMiner(new LuceneIndexerSearcher(false));

    System.out.println(tm.corpusBags.txtBag.totalCount());
    System.out.println(tm.corpusBags.txtBag.getCount("dextromethorphan"));
   
    q = new TermQuery(new Term("txt", "CYP2D6"));
View Full Code Here

TOP

Related Classes of uk.ac.cam.ch.wwmm.oscar3.flow.OscarFlow

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.