Package opennlp.tools.tokenize

Examples of opennlp.tools.tokenize.SimpleTokenizer.tokenize()

62
63
64
65
66
67
68
69
70
71
72
  public void testSingleTokeNameAtSentenceStart() {

    String sentence = "Max a b c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 0 && names[0].getEnd() == 1);
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
  @Test
  public void testSingleTokeNameInsideSentence() {
    String sentence = "a b  Max c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 2 && names[0].getEnd() == 3);
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
  @Test
  public void testSingleTokeNameAtSentenceEnd() {
    String sentence = "a b c Max";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 3 && names[0].getEnd() == 4);
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
  public void testSingleTokeNameAtSentenceStart() {

    String sentence = "Max a b c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 0 && names[0].getEnd() == 1);
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
  @Test
  public void testSingleTokeNameInsideSentence() {
    String sentence = "a b  Max c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 2 && names[0].getEnd() == 3);
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
  @Test
  public void testSingleTokeNameAtSentenceEnd() {
    String sentence = "a b c Max";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 3 && names[0].getEnd() == 4);
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
        for (int i = 0; i < sentences.length; i++) {
            String sentence = sentences[i];
            //LOG.debug("Sentence: " + sentence);

            // extract the names in the current sentence
            String[] tokens = tokenizer.tokenize(sentence);
            Span[] tokenspan = tokenizer.tokenizePos(sentence);
            Span[] nameSpans = finder.find(tokens);
            double[] probs = finder.probs();

            if (nameSpans != null && nameSpans.length > 0) {
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
  public void testSingleTokeNameAtSentenceStart() {

    String sentence = "Max a b c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 0 && names[0].getEnd() == 1);
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
  @Test
  public void testSingleTokeNameInsideSentence() {
    String sentence = "a b  Max c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 2 && names[0].getEnd() == 3);
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
  @Test
  public void testSingleTokeNameAtSentenceEnd() {
    String sentence = "a b c Max";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 3 && names[0].getEnd() == 4);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.