Package com.atilika.kuromoji

Examples of com.atilika.kuromoji.AbstractTokenizer$Builder


import com.atilika.kuromoji.AbstractTokenizer.Mode;

public class TokenizerExample {

  public static void main(String[] args) throws IOException {
    AbstractTokenizer tokenizer;
    if (args.length == 1) {
      Mode mode = Mode.valueOf(args[0].toUpperCase());
      tokenizer = AbstractTokenizer.builder().mode(mode).build();
    } else if (args.length == 2) {
      Mode mode = Mode.valueOf(args[0].toUpperCase());
      tokenizer = AbstractTokenizer.builder().mode(mode).userDictionary(args[1]).build();
    } else {
      tokenizer = AbstractTokenizer.builder().build();
    }
    System.out.println("AbstractTokenizer ready.  Provide input text and press RET.");
    BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
    String line;
    while ((line = reader.readLine()) != null) {
      List<Token> result = tokenizer.tokenize(line);
      for (Token token : result) {
        System.out.println(token.getSurfaceForm() + "\t"
            + token.getAllFeatures());
      }
    }
View Full Code Here

TOP

Related Classes of com.atilika.kuromoji.AbstractTokenizer$Builder

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.