Package joshua.decoder.hypergraph

Examples of joshua.decoder.hypergraph.KBestExtractor


    int baseline_lm_feat_id = 0;
    //??????????????????????????????????????
   
    SymbolTable p_symbolTable = new BuildinSymbol(null);
   
    KBestExtractor kbest_extractor = null;
    int topN = 300;//TODO
    boolean extract_unique_nbest = true;//TODO
    boolean do_ngram_clip_nbest = true; //TODO
    if (orc_extract_nbest) {
      System.out.println("oracle extraction from nbest list");
      kbest_extractor = new KBestExtractor(p_symbolTable, extract_unique_nbest, false, false, false,  false, true);
    }
   
    BufferedWriter orc_out = FileUtility.getWriteFileStream(f_orc_out);
   
    long start_time0 = System.currentTimeMillis();
View Full Code Here


         
    int topN=3;
    boolean useUniqueNbest =true;
    boolean useTreeNbest = false;
    boolean addCombinedCost = true
    KBestExtractor kbestExtractor = new KBestExtractor(symbolTbl, useUniqueNbest, useTreeNbest, false, addCombinedCost, false, true);
   
   
    DiskHyperGraph diskHG = new DiskHyperGraph(symbolTbl, ngramStateID, saveModelCosts, null);
    diskHG.initRead(testNodesFile, testRulesFile, null);
    for(int sentID=0; sentID < numSent; sentID ++){
      System.out.println("#Process sentence " + sentID);
      HyperGraph testHG = diskHG.readHyperGraph();
      baselineFeature.collectTransitionLogPs(testHG);
      reranker.rankHG(testHG);
   
      try{
        kbestExtractor.lazyKBestExtractOnHG(testHG, features, topN, sentID, out1best);
      } catch (IOException e) {
        e.printStackTrace();
      }
             
    }
View Full Code Here

    this.approximatorMap = new HashMap<VariationalNgramApproximator, FeatureTemplateBasedFF>();   
    VariationalDecoderConfiguration.initializeModels(configFile, this.symbolTbl, this.featFunctions, this.approximatorMap);   
    this.insideOutsideScalingFactor =  VariationalDecoderConfiguration.insideoutsideScalingFactor;

    //this.kbestExtractor = new KBestExtractor(p_symbol); 
    this.kbestExtractor = new KBestExtractor(this.symbolTbl, this.useUniqueNbest, this.useTreeNbest, false, this.addCombinedCost, false, true);
  }
View Full Code Here

    int baseline_lm_feat_id = 0;//???????
   
    int max_num_words =25;
   
    SymbolTable p_symbol = new BuildinSymbol(null);
    KBestExtractor kbest_extractor = new KBestExtractor(p_symbol, true, false, false, false,  false, true);//????????????
    ApproximateFilterHGByOneString filter = new ApproximateFilterHGByOneString(p_symbol,baseline_lm_feat_id,baseline_lm_order);
    StringSumInHG p_sumer = new StringSumInHG(p_symbol, kbest_extractor, filter);
   
    //#### process test set
    BufferedWriter t_writer_nbest =  FileUtilityOld.getWriteFileStream(f_nbest)
    BufferedWriter t_writer_1best =  FileUtilityOld.getWriteFileStream(f_1best);
    System.out.println("############Process file  " + f_test_items);
    DiskHyperGraph dhg_test = new DiskHyperGraph(p_symbol, baseline_lm_feat_id, true, null); //have model costs stored
    dhg_test.initRead(f_test_items, f_test_rules,null);
     
    for(int sent_id=0; sent_id < num_sents; sent_id ++){
      System.out.println("#Process sentence " + sent_id);
      HyperGraph hg_test = dhg_test.readHyperGraph();     
      //if(sent_id==1)System.exit(1);
      //generate a unique nbest of strings based on viterbi cost
      ArrayList<String> nonUniqueNbestStrings = new ArrayList<String>();
      kbest_extractor.lazyKBestExtractOnHG(hg_test, null, topN, sent_id, nonUniqueNbestStrings);
     
      double max_prob = Double.NEGATIVE_INFINITY;
      String max_string = "";
     
      //chech if the sentence is too long
View Full Code Here

    this.testFile        = testFile;
    this.nbestFile       = nbestFile;
    this.oracleFile      = oracleFile;
    this.startSentenceID = startSentenceID;
   
    this.kbestExtractor = new KBestExtractor(
      this.symbolTable,
      JoshuaConfiguration.use_unique_nbest,
      JoshuaConfiguration.use_tree_nbest,
      JoshuaConfiguration.include_align_index,
      JoshuaConfiguration.add_combined_cost,
View Full Code Here

    int baseline_lm_feat_id = 0;
    //??????????????????????????????????????
   
    SymbolTable p_symbolTable = new BuildinSymbol(null);
   
    KBestExtractor kbestExtractor =null;
    int topN=300;//TODO
    boolean extract_unique_nbest = true;//TODO
    boolean do_ngram_clip_nbest = true; //TODO
    if(orc_extract_nbest==true){
      System.out.println("oracle extraction from nbest list");
      kbestExtractor = new KBestExtractor(p_symbolTable, extract_unique_nbest, false, false, false,  false, true);
    }
   
    BufferedWriter orc_out = FileUtility.getWriteFileStream(f_orc_out);
    boolean rerankKbestOracles = true;
    BufferedWriter rerankOrcOut=null;
    if(rerankKbestOracles==true){
      rerankOrcOut = FileUtility.getWriteFileStream(f_orc_out+".rerank");
    }
   
    long start_time0 = System.currentTimeMillis();
    long time_on_reading = 0;
    long time_on_orc_extract = 0;
    BufferedReader t_reader_ref = FileUtility.getReadFileStream(f_ref_files);
   
    DiskHyperGraph dhg_read  = new DiskHyperGraph(p_symbolTable, baseline_lm_feat_id, saveModelScores, null);
   
 
    dhg_read.initRead(f_hypergraphs, f_rule_tbl, null);
   
    KBestExtractor oracleKbestExtractor = new KBestExtractor(p_symbolTable, extract_unique_nbest, false, false, true,  false, true);//extract kbest oracles
    KBestExtractor rerankOracleKbestExtractor = new KBestExtractor(p_symbolTable, extract_unique_nbest, false, false, false,  false, true);//extract kbest oracles
    int topKOracles= 500;//TODO
    //OracleExtractionOnHGV2 orc_extractor = new OracleExtractionOnHGV2(p_symbolTable, baseline_lm_feat_id);
    OracleExtractionOnHGV3 orc_extractor = new OracleExtractionOnHGV3(p_symbolTable);
    String ref_sent= null;
    int sent_id=0;
View Full Code Here

    HGRanker reranker = new HGRanker(features)
   
   
    //==== kbest
    boolean addCombinedCost = false
    KBestExtractor kbestExtractor = new KBestExtractor(symbolTbl, MRConfig.use_unique_nbest, MRConfig.use_tree_nbest, false, addCombinedCost, false, true);
   
    //==== loop
    HyperGraphFactory hgFactory = new HyperGraphFactory(curHypFilePrefix, referenceFiles, MRConfig.ngramStateID,  symbolTbl, true);
    hgFactory.startLoop();
    for(int sentID=0; sentID< this.numTrainingSentence; sentID ++){     
      HGAndReferences res = hgFactory.nextHG();
      reranker.rankHG(res.hg);//reset best pointer and transition prob
   
      String hypSent = kbestExtractor.getKthHyp(res.hg.goalNode, 1, -1, null, null);
      double bleu = BLEU.computeSentenceBleu(res.referenceSentences, hypSent);
      bleuSum  += bleu;
     
      double googleGain = BLEU.computeLinearCorpusGain(MRConfig.linearCorpusGainThetas, res.referenceSentences, hypSent);
      googleGainSum += googleGain;
View Full Code Here

    this.nbestFile      = nbest_file_in;

    t_writer_nbest =  FileUtility.getWriteFileStream(nbestFile);
   
   
    this.kbestExtractor = new KBestExtractor(this.symbolTable, JoshuaConfiguration.use_unique_nbest, JoshuaConfiguration.use_tree_nbest,
        JoshuaConfiguration.include_align_index, JoshuaConfiguration.add_combined_cost,  true, true);
   
  }
View Full Code Here

  public NbestCrunching(SymbolTable symbolTbl, double insideOutsideScalingFactor, int topN){
    this.scalingFactor = insideOutsideScalingFactor;
    this.topN = topN;
   
    this.symbolTbl = symbolTbl;
    this.kbestExtractor = new KBestExtractor(this.symbolTbl, this.useUniqueNbest, this.useTreeNbest, false, this.addCombinedCost,  false, true);
  }
View Full Code Here

  public EstimateEntropyGapOnNbest(SymbolTable symbol_, double scalingFactor, int topn){
    this.scalingFactor = scalingFactor;
    this.topN = topn;
   
    this.symbolTbl = symbol_;
    this.kbestExtractor = new KBestExtractor(this.symbolTbl, this.useUniqueNbest, this.useTreeNbest, false, this.addCombinedCost,  false, true);
  }
View Full Code Here

TOP

Related Classes of joshua.decoder.hypergraph.KBestExtractor

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.