Package opennlp.tools.tokenize

Examples of opennlp.tools.tokenize.TokenizerModel

60
61
62
63
64
65
66
67
68
69
70
71
  }

  public void setTokenizer(FileSystem fs, Path p){
    try {
      FSDataInputStream in = fs.open(p);
      TokenizerModel model;
      model = new TokenizerModel(in);
      tokenizer = new TokenizerME(model);
    }
    catch (IOException e) {
      e.printStackTrace();
    }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
        openNLP = new OpenNLP(new ClasspathDataFileProvider("DUMMY"));
    }

    @Test
    public void testLoadEnTokenizer() throws IOException{
        TokenizerModel model = openNLP.getTokenizerModel("en");
        Assert.assertNotNull(model);
        Tokenizer tokenizer = openNLP.getTokenizer("en");
        Assert.assertNotNull(tokenizer);
    }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
        Assert.assertNotNull(tokenizer);
    }
   
    @Test
    public void testLoadMissingTokenizerModel() throws IOException{
        TokenizerModel model = openNLP.getTokenizerModel("ru");
        //there is not Russian model ...
        //so it is expected that the model is NULL
        Assert.assertNull(model);
    }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
        ner = openNLP.getNameFinder("person", "ru");
        Assert.assertNull(ner);
    }
    @Test
    public void testLoadModelByName() throws IOException{
        TokenizerModel tokenModel = openNLP.getModel(TokenizerModel.class, "en-token.bin", null);
        Assert.assertNotNull(tokenModel);
        SentenceModel sentModel = openNLP.getModel(SentenceModel.class, "en-sent.bin", null);
        Assert.assertNotNull(sentModel);
        POSModel posModel = openNLP.getModel(POSModel.class, "en-pos-maxent.bin", null);
        Assert.assertNotNull(posModel);
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
     */
    public Tokenizer getTokenizer(String language) {
        Tokenizer tokenizer = null;
        if(language != null){
            try {
                TokenizerModel model = getTokenizerModel(language);
                if(model != null){
                    tokenizer = new TokenizerME(getTokenizerModel(language));
                }
            } catch (InvalidFormatException e) {
                log.warn("Unable to load Tokenizer Model for "+language+": " +
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
        if(modelName == null){
            return openNLP.getTokenizer(language);
        } else if(SIMPLE_MODEL_NAME.equals(modelName)){
            return SimpleTokenizer.INSTANCE;
        } else { //try to load the configured model
            TokenizerModel model;
            try {
                model = openNLP.getModel(TokenizerModel.class, modelName, null);
            } catch (Exception e) {
                throw new EngineException("Error while loading the configured OpenNLP "
                    + "TokenizerModel '"+modelName+"' ("+getClass().getSimpleName()+" | name="
View Full Code Here
54
55
56
57
58
59
60
61
        getDefaultPosTagger();
        getDefaultChunker();
    }

    public static Tokenizer getDefaultTokenizer() throws IOException {
        return new TokenizerME(new TokenizerModel(
                getResourceAsStream(tokenizerModelFile)));
    }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
    protected static BaseModel loadgivenmodeltype(OpenNlpModels m, InputStream in) throws InvalidFormatException, IOException {
        BaseModel mdl = null;
        switch(m) {
            case TokenizerModel: {
                mdl = new TokenizerModel(in);
                LOG.debug("OpenNLP5 Tokenizer Model loaded: " + mdl);
                break;
            }
            case POSModel: {
                mdl = new POSModel(in);
View Full Code Here
68
69
70
71
72
73
74
75
     *
     * @throws IOException
     */
    public ApacheExtractor() throws IOException {
        nameFinder = new NameFinderME(new TokenNameFinderModel(ApacheExtractor.class.getResourceAsStream(pathToNERModel)));
        tokenizer = new TokenizerME(new TokenizerModel(ApacheExtractor.class.getResourceAsStream(pathToTokenizerModel)));
        sentenceDetector = new SentenceDetectorME(new SentenceModel(ApacheExtractor.class.getResourceAsStream(pathToSentenceDetectorModel)));
    }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
  private final EnglishChunkFilter chunkFilter;

  public EnglishChunker() {
    try {
      if (tokenModel == null) {
        tokenModel = new TokenizerModel(Tools.getStream(TOKENIZER_MODEL));
      }
      if (posModel == null) {
        posModel = new POSModel(Tools.getStream(POS_TAGGER_MODEL));
      }
      if (chunkerModel == null) {
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.TokenizerModel

  • com.bericotech.clavin.extractor.ApacheExtractor
  • edu.washington.cs.knowitall.util.DefaultObjects
  • functionality.Models
  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorker
  • io.lumify.opennlpme.OpenNLPMaximumEntropyExtractorGraphPropertyWorker
  • ivory.core.tokenize.OpenNLPTokenizer
  • net.sf.nlpshell.Main
  • opennlp.maxent.io.BinaryGISModelReader
  • opennlp.model.AbstractModel
  • opennlp.tools.cmdline.tokenizer.TokenizerMEEvaluatorTool
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.