Examples of TokenizerFactory

Examples of com.aliasi.tokenizer.TokenizerFactory

137
138
139
140
141
142
143
144
145
146
147
  //*-- create the spell checker
  if (crawlConfig.isSpellCheck())
  { FixedWeightEditDistance fixedEdit = new FixedWeightEditDistance( MATCH_WEIGHT, DELETE_WEIGHT, INSERT_WEIGHT,
      SUBSTITUTE_WEIGHT, TRANSPOSE_WEIGHT);
    NGramProcessLM lm = new NGramProcessLM(NGRAM_LENGTH);
    TokenizerFactory tokenizerFactory = new StandardBgramTokenizerFactory(false);  //*-- do not extract entities
    try { if ( (new File(Constants.SPELL_TRAIN_MODEL).exists() ) && !crawlConfig.isFreshCrawl() )
             lm = readModel(Constants.SPELL_TRAIN_MODEL); 
          sc = new TrainSpellChecker(lm, fixedEdit, tokenizerFactory); }
    catch (IOException ie) { logger.error("IO Error: Could not read spell train file " + ie.getMessage()); }
    catch (ClassNotFoundException ce) { logger.error("Class error: " + ce.getMessage()); }
View Full Code Here

Examples of com.google.gwt.place.testplaces.TokenizerFactory

82
83
84
85
86
87
88
89
90
91
    doTest(subject, null);
  }

  public void testTopLevelWithFactory() {
    AbstractPlaceHistoryMapper<TokenizerFactory> subject = GWT.create(WithFactory.class);
    TokenizerFactory factory = new TokenizerFactory();
    subject.setFactory(factory);

    doTest(subject, factory);
  }
View Full Code Here

Examples of com.google.gwt.place.testplaces.TokenizerFactory

96
97
98
99
100
101
102
103
104
105
    doTest(subject, null);
  }

  public void testNestedWithFactory() {
    AbstractPlaceHistoryMapper<TokenizerFactory> subject = GWT.create(LocalWithFactory.class);
    TokenizerFactory factory = new TokenizerFactory();
    subject.setFactory(factory);

    doTest(subject, factory);
  }
View Full Code Here

Examples of edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenizerFactory

378
379
380
381
382
383
384
385
386
387
388
389
     
    }
   
    private static Map<INDEXFIELD, Tokenizer> initMap(Properties props) {
      HashMap<INDEXFIELD, Tokenizer> map = new HashMap<INDEXFIELD, Tokenizer>(INDEXFIELD.values().length);
      TokenizerFactory fact = TokenizerFactory.getInstance(props);
      for (INDEXFIELD fld : INDEXFIELD.values()) {
        map.put(fld, fact.getTokenizer(fld));
      }
     
      return map;
    }
View Full Code Here

Examples of edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenizerFactory

208
209
210
211
212
213
214
215
216
217
218
219
   
  }
 
  private static Map<INDEXFIELD, Tokenizer> initMap(Properties props) {
    HashMap<INDEXFIELD, Tokenizer> map = new HashMap<INDEXFIELD, Tokenizer>(INDEXFIELD.values().length);
    TokenizerFactory fact = TokenizerFactory.getInstance(props);
    for (INDEXFIELD fld : INDEXFIELD.values()) {
      map.put(fld, fact.getTokenizer(fld));
    }
   
    return map;
  }
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerFactory

81
82
83
84
85
86
87
88
89
90
91
    TokenizerModel model;
    try {
      Dictionary dict = loadDict(params.getAbbDict());

      TokenizerFactory tokFactory = TokenizerFactory.create(
          params.getFactory(), params.getLang(), dict,
          params.getAlphaNumOpt(), null);
      model = opennlp.tools.tokenize.TokenizerME.train(sampleStream,
          tokFactory, mlParams);
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerFactory

62
63
64
65
66
67
68
69
70
71
72
    }
   
    try {
      Dictionary dict = TokenizerTrainerTool.loadDict(params.getAbbDict());

      TokenizerFactory tokFactory = TokenizerFactory.create(
          params.getFactory(), params.getLang(), dict,
          params.getAlphaNumOpt(), null);
      validator = new opennlp.tools.tokenize.TokenizerCrossValidator(mlParams,
          tokFactory, listener);
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerFactory

82
83
84
85
86
87
88
89
90
91
92
    TokenizerModel model;
    try {
      Dictionary dict = loadDict(params.getAbbDict());

      TokenizerFactory tokFactory = TokenizerFactory.create(
          params.getFactory(), params.getLang(), dict,
          params.getAlphaNumOpt(), null);
      model = opennlp.tools.tokenize.TokenizerME.train(sampleStream,
          tokFactory, mlParams);
View Full Code Here

Examples of opennlp.tools.tokenize.TokenizerFactory

62
63
64
65
66
67
68
69
70
71
72
    }

    try {
      Dictionary dict = TokenizerTrainerTool.loadDict(params.getAbbDict());

      TokenizerFactory tokFactory = TokenizerFactory.create(
          params.getFactory(), params.getLang(), dict,
          params.getAlphaNumOpt(), null);
      validator = new opennlp.tools.tokenize.TokenizerCrossValidator(mlParams,
          tokFactory, listener);
View Full Code Here

Examples of org.apache.lucene.analysis.util.TokenizerFactory

250
251
252
253
254
255
256
257
258
259
260
    // prepare bi-gram tokenizer factory
    Map<String, String> args = new HashMap<String, String>();
    args.put(AbstractAnalysisFactory.LUCENE_MATCH_VERSION_PARAM, "4.4");
    args.put("minGramSize","2");
    args.put("maxGramSize","2");
    TokenizerFactory tf = new NGramTokenizerFactory(args);
   
    // (ab)->(bc)->(cd)->[ef][fg][gh]
    List<String> rules = new ArrayList<String>();
    rules.add( "abcd=>efgh" );
    synMap = new SlowSynonymMap( true );
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.