}
System.out.println("AbstractTokenizer ready. Provide input text and press RET.");
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
String line;
while ((line = reader.readLine()) != null) {
List<Token> result = tokenizer.tokenize(line);
for (Token token : result) {
System.out.println(token.getSurfaceForm() + "\t"
+ token.getAllFeatures());
}
}