Examples of TokenizerModel

Examples of opennlp.tools.tokenize.TokenizerModel

68
69
70
71
72
73
74
75
     *
     * @throws IOException
     */
    public ApacheExtractor() throws IOException {
        nameFinder = new NameFinderME(new TokenNameFinderModel(ApacheExtractor.class.getResourceAsStream(pathToNERModel)));
        tokenizer = new TokenizerME(new TokenizerModel(ApacheExtractor.class.getResourceAsStream(pathToTokenizerModel)));
        sentenceDetector = new SentenceDetectorME(new SentenceModel(ApacheExtractor.class.getResourceAsStream(pathToSentenceDetectorModel)));
    }
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerModel

139
140
141
142
143
144
145
146
147
148
149
150
151
152
    }

    protected Tokenizer loadTokenizer(String pathPrefix, FileSystem fs) throws IOException {
        Path tokenizerHdfsPath = new Path(pathPrefix + "/en-token.bin");

        TokenizerModel tokenizerModel = null;
        InputStream tokenizerModelInputStream = fs.open(tokenizerHdfsPath);
        try {
            tokenizerModel = new TokenizerModel(tokenizerModelInputStream);
        } finally {
            tokenizerModelInputStream.close();
        }

        return new TokenizerME(tokenizerModel);
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerModel

135
136
137
138
139
140
141
142
143
144
145
146
147
148
    }

    protected Tokenizer loadTokenizer(String pathPrefix, FileSystem fs) throws IOException {
        Path tokenizerHdfsPath = new Path(pathPrefix + "/en-token.bin");

        TokenizerModel tokenizerModel = null;
        InputStream tokenizerModelInputStream = fs.open(tokenizerHdfsPath);
        try {
            tokenizerModel = new TokenizerModel(tokenizerModelInputStream);
        } finally {
            tokenizerModelInputStream.close();
        }

        return new TokenizerME(tokenizerModel);
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerModel

229
230
231
232
233
234
235
236
237
238
239
    // if trace file
    // serialize events ...
   
    InputStream additionalTrainingDataIn = null;
    Writer samplesOut = null;
    TokenizerModel tokenModel;
   
    try {
      if (additionalTrainingDataFile != null) {
       
        if (mLogger.isLoggable(Level.INFO)) {
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerModel

30
31
32
33
34
35
36
    return model;
  }

  @Override
  protected TokenizerModel loadModel(InputStream in) throws IOException {
    return new TokenizerModel(in);
  }
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerModel

81
82
83
84
85
86
87
88
89
90
91
  public void initialize(UimaContext context)
      throws ResourceInitializationException {

    super.initialize(context);

    TokenizerModel model;

    try {
      TokenizerModelResource modelResource = (TokenizerModelResource) context
          .getResourceObject(UimaUtil.MODEL_PARAMETER);
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerModel

109
110
111
112
113
114
115
116
117
118
119
        trainingDataInFile, params.getEncoding());
   
    if(mlParams == null)
      mlParams = createTrainingParameters(params.getIterations(), params.getCutoff());

    TokenizerModel model;
    try {
      Dictionary dict = loadDict(params.getAbbDict());
      model = opennlp.tools.tokenize.TokenizerME.train(params.getLang(),
          sampleStream, dict, params.getAlphaNumOpt(), mlParams);
    } catch (IOException e) {
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerModel

36
37
38
39
40
41
42
  }
 
  @Override
  protected TokenizerModel loadModel(InputStream modelIn) throws IOException,
      InvalidFormatException {
    return new TokenizerModel(modelIn);
  }
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerModel

80
81
82
83
84
85
86
87
88
89
90
  public void initialize(UimaContext context)
      throws ResourceInitializationException {

    super.initialize(context);

    TokenizerModel model;

    try {
      TokenizerModelResource modelResource = (TokenizerModelResource) context
          .getResourceObject(UimaUtil.MODEL_PARAMETER);
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerModel

214
215
216
217
218
219
220
221
222
223
224
    GIS.PRINT_MESSAGES = false;
  
    ObjectStream<TokenSample> samples = ObjectStreamUtils.createObjectStream(tokenSamples);
   
    InputStream additionalTrainingDataIn = null;
    TokenizerModel tokenModel;
   
    try {
      if (additionalTrainingDataFile != null) {
       
        if (mLogger.isLoggable(Level.INFO)) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.