Examples of Detokenizer

Examples of opennlp.tools.tokenize.Detokenizer

53
54
55
56
57
58
59
60
61
62
63
    Parameters params = ArgumentParser.parse(args, Parameters.class);

    ObjectStream<POSSample> posSampleStream = new WordTagSampleStreamFactory()
        .create(params);

    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
View Full Code Here

Examples of opennlp.tools.tokenize.Detokenizer

52
53
54
55
56
57
58
59
60
61
62
   
    Parameters params = ArgumentParser.parse(args, Parameters.class);
   
    ObjectStream<POSSample> samples = new ConllXPOSSampleStreamFactory().create(params);
   
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: " + e.getMessage());
      throw new TerminateToolException(-1);
View Full Code Here

Examples of opennlp.tools.tokenize.Detokenizer

92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
    if (args.length != 1) {
      System.out.println(getHelp());
      throw new TerminateToolException(1);
    }
   
    Detokenizer detokenizer = new DictionaryDetokenizer(
        new DetokenizationDictionaryLoader().load(new File(args[0])));
   
    ObjectStream<String> tokenizedLineStream =
      new PlainTextByLineStream(new InputStreamReader(System.in));
   
    PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
    perfMon.start();
   
    try {
      String tokenizedLine;
      while ((tokenizedLine = tokenizedLineStream.read()) != null) {
       
        // white space tokenize line
        String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);
       
        DetokenizationOperation operations[] = detokenizer.detokenize(tokens);
       
        System.out.println(detokenize(tokens, operations));
       
        perfMon.incrementCounter();
      }
View Full Code Here

Examples of opennlp.tools.tokenize.Detokenizer

54
55
56
57
58
59
60
61
62
63
64
    ObjectStream<NameSample> nameSampleStream = new NameSampleStreamFactory()
        .create(params);

    // TODO: Move this to a factory method
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
View Full Code Here

Examples of opennlp.tools.tokenize.Detokenizer

53
54
55
56
57
58
59
60
61
62
63
    ObjectStream<NameSample> nameSampleStream = new NameSampleStreamFactory()
        .create(params);

    // TODO: Move this to a factory method
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
View Full Code Here

Examples of opennlp.tools.tokenize.Detokenizer

53
54
55
56
57
58
59
60
61
62
63
    ObjectStream<POSSample> posSampleStream = new WordTagSampleStreamFactory()
        .create(params);

    // TODO: Move this to a factory method
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
View Full Code Here

Examples of opennlp.tools.tokenize.Detokenizer

57
58
59
60
61
62
63
64
65
66
67
    // TODO: Compare code to ConllXTokenSampleStream, maybe it can be shared somehow
   
    ObjectStream<POSSample> posSampleStream =
        new ConllXPOSSampleStreamFactory().create(params);
   
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: " + e.getMessage());
      throw new TerminateToolException(-1);
View Full Code Here

Examples of opennlp.tools.tokenize.Detokenizer

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
   
    if (args.length != 1) {
      System.out.println(getHelp());
    } else {
   
      Detokenizer detokenizer = new DictionaryDetokenizer(
          new DetokenizationDictionaryLoader().load(new File(args[0])));

      ObjectStream<String> tokenizedLineStream =
        new PlainTextByLineStream(new InputStreamReader(System.in));

      PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
      perfMon.start();

      try {
        String tokenizedLine;
        while ((tokenizedLine = tokenizedLineStream.read()) != null) {

          // white space tokenize line
          String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);

          System.out.println(detokenizer.detokenize(tokens, null));

          perfMon.incrementCounter();
        }
      }
      catch (IOException e) {
View Full Code Here

Examples of opennlp.tools.tokenize.Detokenizer

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
    if (args.length != 1) {
      System.out.println(getHelp());
    } else {
          try {
      Detokenizer detokenizer = new DictionaryDetokenizer(
          new DetokenizationDictionaryLoader().load(new File(args[0])));

      ObjectStream<String> tokenizedLineStream =
        new PlainTextByLineStream(new SystemInputStreamFactory(), SystemInputStreamFactory.encoding());

      PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
      perfMon.start();


        String tokenizedLine;
        while ((tokenizedLine = tokenizedLineStream.read()) != null) {

          // white space tokenize line
          String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);

          System.out.println(detokenizer.detokenize(tokens, null));

          perfMon.incrementCounter();
        }
              perfMon.stopAndPrintFinalResult();
      }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.