Package opennlp.tools.util

Examples of opennlp.tools.util.ObjectIntPair

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
          if (foundTrainingTokens) {
            for (int ti = firstTrainingToken; ti <= lastTrainingToken; ti++) {
              Span tSpan = tokens[ti];
              int cStart = cSpan.getStart();
              for (int i = tSpan.getStart() + 1; i < tSpan.getEnd(); i++) {
                String[] context = cg.getContext(new ObjectIntPair(ctok, i - cStart));
                events.add(new Event(TokContextGenerator.NO_SPLIT, context));
              }
              if (tSpan.getEnd() != cSpan.getEnd()) {
                String[] context = cg.getContext(new ObjectIntPair(ctok, tSpan.getEnd() - cStart));
                events.add(new Event(TokContextGenerator.SPLIT, context));
              }
            }
          }
        }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
        for (int tok = 0; tok < spaceToks.length; tok++) {
          StringBuffer sb = new StringBuffer(spaceToks[tok]);
          if (TokenizerME.alphaNumeric.matcher(spaceToks[tok]).matches()) {
            int lastIndex = sb.length() - 1;
            for (int id = 0; id < sb.length(); id++) {
              String[] context = cg.getContext(new ObjectIntPair(sb, id));
              if (id == lastIndex) {
                elist.add(new Event("T", context));
              }
              else {
                elist.add(new Event("F", context));
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
        int end = s.getEnd();
        final int origStart = s.getStart();
        double tokenProb = 1.0;
        for (int j = origStart + 1; j < end; j++) {
          double[] probs =
            model.eval(cg.getContext(new ObjectIntPair(tok, j - origStart)));
          String best = model.getBestOutcome(probs);
          //System.err.println("TokenizerME: "+tok.substring(0,j-origStart)+"^"+tok.substring(j-origStart)+" "+best+" "+probs[model.getIndex(best)]);
          tokenProb *= probs[model.getIndex(best)];
          if (best.equals(TokContextGenerator.SPLIT)) {
            newTokens.add(new Span(start, j));
View Full Code Here
TOP

Related Classes of opennlp.tools.util.ObjectIntPair

  • opennlp.tools.tokenize.TokenizerME
  • opennlp.tools.tokenize.TokEventCollector
  • opennlp.tools.tokenize.TokSpanEventStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.