Examples of opennlp.ccg.lexicon.Tokenizer.tokenize()

opennlp.ccg.lexicon.Tokenizer.tokenize()
Parses an input string into a list of words, including any explicitly given factors, and the semantic class of special tokens. Tokens are parsed into words using parseToken.

  lm.debugScore = true;
        int secs = (int) (System.currentTimeMillis() - start) / 1000;
        System.out.println("secs: " + secs);
        System.out.println();
        Tokenizer tokenizer = new DefaultTokenizer();
        List<Word> words = tokenizer.tokenize(tokens);
        System.out.println("scoring: " + tokens);
        System.out.println();
        lm.setWordsToScore(words, true);
        lm.prepareToScoreWords();
        double logprob = lm.logprob();

View Full Code Here

        // System.out.println("trie map: ");
        // System.out.println(lm.trieMapRoot.toString());
        // System.out.println();
        
        Tokenizer tokenizer = new DefaultTokenizer();
        List<Word> words = tokenizer.tokenize(tokens);
        System.out.println("scoring: " + tokens);
        System.out.println();
        lm.debugScore = true;
        lm.setWordsToScore(words, true);
        lm.prepareToScoreWords();

View Full Code Here

        // System.out.println("trie map: ");
        // System.out.println(lm.trieMapRoot.toString());
        // System.out.println();
        
        Tokenizer tokenizer = new DefaultTokenizer();
        List<Word> words = tokenizer.tokenize(tokens);
        System.out.println("scoring: " + tokens);
        System.out.println();
        lm.debugScore = true;
        lm.setWordsToScore(words, true);
        lm.prepareToScoreWords();

View Full Code Here

            rCount++;
            totalScore += score; 
            if (gramcomplete) totalScoreComplete += score;
            int itemRank = 1;
            Tokenizer tokenizer = grammar.lexicon.tokenizer;
            String itemOrth = tokenizer.getOrthography(tokenizer.tokenize(testItem.sentence));
            if (!bestRealization.equals(itemOrth)) {
                itemRank = 0;
                List<Edge> bestEdges = chart.bestEdges();
                for (int j = 0; j < bestEdges.size(); j++) {
                    Edge edge = bestEdges.get(j);

View Full Code Here

              if (semClassReplacement || withFactors) {
                  // use words from sign or pre-parsed full words if available
                if (testItem.sign != null) 
                  words = testItem.sign.getWords();
                else if (testItem.fullWords != null) 
                      words = tokenizer.tokenize(testItem.fullWords, true);
                  // otherwise parse
                  else words = grammar.getParsedWords(s);
              }
              else words = tokenizer.tokenize(s);
              // reverse, if apropos

View Full Code Here

                else if (testItem.fullWords != null) 
                      words = tokenizer.tokenize(testItem.fullWords, true);
                  // otherwise parse
                  else words = grammar.getParsedWords(s);
              }
              else words = tokenizer.tokenize(s);
              // reverse, if apropos
              if (reverse) {
                  List<Word> tmp = words;
                  words = new ArrayList<Word>(words.size());
                  words.add(Word.createWord("<s>"));

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.