Package com.github.pmerienne.trident.ml.preprocessing

Examples of com.github.pmerienne.trident.ml.preprocessing.TwitterTokenizer.tokenize()

31
32
33
34
35
36
37
38
39
40
41
    // Given
    String tweet = "@PrincessSuperC Hey Cici";
    TwitterTokenizer tokenizer = new TwitterTokenizer();

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("hei", "cici");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    // Given
    String tweet = "arg #word!";
    TwitterTokenizer tokenizer = new TwitterTokenizer();

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("arg", "word");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    // Given
    String tweet = "so huuunggryy!";
    TwitterTokenizer tokenizer = new TwitterTokenizer();

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("so", "hungri");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    // Given
    String tweet = "it's 15PM we're in the 20th century";
    TwitterTokenizer tokenizer = new TwitterTokenizer();

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("we'r", "centuri");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
    // Given
    String tweet = "fb &gt; tw";
    TwitterTokenizer tokenizer = new TwitterTokenizer();

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("fb", "tw");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
    // Given
    String tweet = "it's not bad movie";
    TwitterTokenizer tokenizer = new TwitterTokenizer(2, 2);

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("_ bad", "bad", "bad movi", "movi");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
258
259
260
261
262
263
264
265
266
267
268
          String[] values = line.split(",");

          Boolean label = !values[0].equals("0");
          String text = line.substring(line.indexOf(",") + 1);

          TWITTER_SAMPLES.add(new TextInstance<Boolean>(label, tokenizer.tokenize(text)));
        } catch (Exception ex) {
          System.err.println("Skipped twitter sample because it can't be parsed : " + line);
        }
      }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.