Examples of Vocabulary

Examples of joshua.corpus.vocab.Vocabulary

168
169
170
171
172
173
174
175
   * @param base a double. The base of the logarithm for quantization.
   */
  private BloomFilterLanguageModel(String filename, int order, int size, double base) {
    super(null, order);
    quantizationBase = base;
    vocabulary = new Vocabulary();
    populateBloomFilter(size, filename);
  }
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

563
564
565
566
567
568
569
570
571
572
573
   *
   * @param in an ObjectInput stream to read from
   */
  public void readExternal(ObjectInput in)
  throws IOException, ClassNotFoundException {
    vocabulary = new Vocabulary();
    int vocabSize = in.readInt();
    for (int i = 0; i < vocabSize; i++) {
      String line = in.readUTF();
      vocabulary.addTerminal(line);
    }
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

87
88
89
90
91
92
93
94
95
96
97
98
99
100
       
    }

  private static ParallelCorpus getParallelCorpus(String joshDir, int cacheSize) throws IOException, ClassNotFoundException {
   
    Vocabulary commonVocab = new Vocabulary();
      String binaryVocabFileName = joshDir + "/common.vocab";
      ObjectInput in = BinaryIn.vocabulary(binaryVocabFileName);
    commonVocab.readExternal(in);
   
    String sourceFileName = joshDir + "/source.corpus";
    Corpus sourceCorpusArray = new MemoryMappedCorpusArray(commonVocab, sourceFileName);

    String targetFileName = joshDir + "/target.corpus";
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

43
44
45
46
47
48
49
50
51
52
53
  SymbolTable vocab;
 
  @Test
  public void setup() {
   
    vocab = new Vocabulary();
    vocab.addTerminal("a");
    vocab.addTerminal("because");
    vocab.addTerminal("boycott");
    vocab.addTerminal("of");
    vocab.addTerminal("parliament");
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

548
549
550
551
552
553
554
555
556
557
558
559
      //      accidentally used anywhere.
     
      if (logger.isLoggable(Level.INFO))
        logger.info("Reading common vocabulary from " +
            binaryVocabFileName);
      Vocabulary commonVocab = new Vocabulary();
      commonVocab.readExternal(
          BinaryIn.vocabulary(binaryVocabFileName));

      // Initialize symbol table using suffix array's vocab
      this.initializeSymbolTable(commonVocab);
    }
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

49
50
51
52
53
54
55
56
57
58
59
  PrefixTree tree;
 
  @Test(dependsOnMethods = {"prefixTreeNodes","suffixLinks"})
  public void setup() {
   
    vocab = new Vocabulary();
    it = vocab.addTerminal("it");
    persuades = vocab.addTerminal("persuades");
    him = vocab.addTerminal("him");
    and = vocab.addTerminal("and");
    disheartens = vocab.addTerminal("disheartens");
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
    //String alignmentsType = alignmentsType;
 
    int maxCacheSize = 100000;//12566;
   
    int numSourceWords, numSourceSentences;
    Vocabulary sourceVocab = new Vocabulary();
    int[] sourceWordsSentences = Vocabulary.initializeVocabulary(sourceFileName, sourceVocab, true);
    numSourceWords = sourceWordsSentences[0];
    numSourceSentences = sourceWordsSentences[1];
   
    Corpus sourceCorpusArray = SuffixArrayFactory.createCorpusArray(sourceFileName, sourceVocab, numSourceWords, numSourceSentences);
    Suffixes sourceSuffixArray = SuffixArrayFactory.createSuffixArray(sourceCorpusArray, maxCacheSize);
   
    int numTargetWords, numTargetSentences;
    Vocabulary targetVocab = new Vocabulary();
    int[] targetWordsSentences = Vocabulary.initializeVocabulary(targetFileName, targetVocab, true);
    numTargetWords = targetWordsSentences[0];
    numTargetSentences = targetWordsSentences[1];
   
    Corpus targetCorpusArray = SuffixArrayFactory.createCorpusArray(targetFileName, targetVocab, numTargetWords, numTargetSentences);
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
      for (char c2='a'; c2<='z'; c2++) {
        words.add(new String(new char[]{c1,c2}));
      } 
    }
   
    Vocabulary vocab = new Vocabulary(words);
   
    try {
     
      File tempFile = File.createTempFile(BinaryTest.class.getName(), "vocab");
      FileOutputStream outputStream = new FileOutputStream(tempFile);
      ObjectOutput out = new BinaryOut(outputStream, true);
      vocab.writeExternal(out);
     
      ObjectInput in = new BinaryIn<Vocabulary>(tempFile.getAbsolutePath(), Vocabulary.class);
      Object o = in.readObject();
      Assert.assertTrue(o instanceof Vocabulary);
     
      Vocabulary newVocab = (Vocabulary) o;
     
      Assert.assertNotNull(newVocab);
      Assert.assertEquals(newVocab.size(), vocab.size());     
     
      Assert.assertEquals(newVocab, vocab);
     

     
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
//    refFile.close();


    // Source language vocabulary
    println("Creating src vocabulary @ " + (new Date()));
    srcVocab = new Vocabulary();
    int[] sourceWordsSentences = Vocabulary.initializeVocabulary(trainSrc_fileName, srcVocab, true);

    int numSourceWords = sourceWordsSentences[0];
    int numSourceSentences = sourceWordsSentences[1];

    // Source language corpus array
    println("Reading src corpus @ " + (new Date()));
    srcCorpusArray = SuffixArrayFactory.createCorpusArray(trainSrc_fileName, srcVocab, numSourceWords, numSourceSentences);

    // Source language suffix array
    println("Creating src SA @ " + (new Date()));
    srcSA = SuffixArrayFactory.createSuffixArray(srcCorpusArray, maxCacheSize);


    // Target language vocabulary
    println("Creating tgt vocabulary @ " + (new Date()));
    tgtVocab = new Vocabulary();
    int[] targetWordsSentences = Vocabulary.initializeVocabulary(trainTgt_fileName, tgtVocab, true);

    int numTargetWords = targetWordsSentences[0];
    int numTargetSentences = targetWordsSentences[1];
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    Set<String> sourceWords = new HashSet<String>();
    for (String word : corpusString.split("\\s+")) {
      sourceWords.add(word);
    }

    sourceVocab = new Vocabulary(sourceWords);
   

    corpusSentence = new BasicPhrase(corpusString, sourceVocab);
   
    targetCorpusString = "das macht ihn und es beschädigt ihn , es setzt ihn auf und es führt ihn aus .";
    Set<String> targetWords = new HashSet<String>();
    for (String targetWord : targetCorpusString.split("\\s+")) {
      targetWords.add(targetWord);
    }
   
    targetVocab = new Vocabulary(targetWords);
   
    ntVocab = new HashMap<Integer,String>();
    ntVocab.put(-1, "X");
   
    {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.