Examples of TokenizerFactory

com.aliasi.tokenizer.TokenizerFactory
com.google.gwt.place.testplaces.TokenizerFactory
Used by tests of {@link com.google.gwt.place.rebind.PlaceHistoryMapperGenerator}.
edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenizerFactory
Factory class to instantiate a Tokenizer instance The expectation is that you need to decide which rules to apply for which field Thus, given a field type, initialize the applicable rules and create the tokenizer @author nikhillo
opennlp.tools.tokenize.TokenizerFactory
The factory that provides {@link Tokenizer} default implementations andresources. Users can extend this class if their application requires overriding the {@link TokenContextGenerator}, {@link Dictionary} etc.
org.apache.lucene.analysis.util.TokenizerFactory
Abstract parent class for analysis factories that create {@link Tokenizer}instances.
org.apache.solr.analysis.TokenizerFactory
A TokenizerFactory breaks up a stream of characters into tokens.
TokenizerFactories are registered for FieldTypes with the IndexSchema through the schema.xml file.

Example schema.xml entry to register a TokenizerFactory implementation to tokenize fields of type "cool"
```
 <fieldtype name="cool" class="solr.TextField"> <analyzer> <tokenizer class="solr.StandardTokenizerFactory"/> ... 
```
A single instance of any registered TokenizerFactory is created via the default constructor and is reused for each FieldType.
@version $Id: TokenizerFactory.java 929782 2010-04-01 02:15:27Z rmuir $

Examples of org.apache.solr.analysis.TokenizerFactory

   * @return a Lucene <code>Analyzer</code>
   */
  public static Analyzer buildAnalyzer(AnalyzerDef analyzerDef, Version luceneMatchVersion) {
    ResourceLoader defaultResourceLoader = new HibernateSearchResourceLoader();
    TokenizerDef token = analyzerDef.tokenizer();
    TokenizerFactory tokenFactory = ( TokenizerFactory ) instantiate( token.factory() );
    final Map<String, String> tokenMapsOfParameters = getMapOfParameters( token.params(), luceneMatchVersion );
    tokenFactory.init( tokenMapsOfParameters );
    injectResourceLoader( tokenFactory, defaultResourceLoader, tokenMapsOfParameters );


    final int length = analyzerDef.filters().length;
    final int charLength = analyzerDef.charFilters().length;
    TokenFilterFactory[] filters = new TokenFilterFactory[length];

View Full Code Here

Examples of org.apache.solr.analysis.TokenizerFactory

   * @return a Lucene <code>Analyzer</code>
   */
  public static Analyzer buildAnalyzer(AnalyzerDef analyzerDef, Version luceneMatchVersion) {
    ResourceLoader defaultResourceLoader = new HibernateSearchResourceLoader();
    TokenizerDef token = analyzerDef.tokenizer();
    TokenizerFactory tokenFactory = instanceFromClass( TokenizerFactory.class, token.factory(), "Tokenizer factory" );
    final Map<String, String> tokenMapsOfParameters = getMapOfParameters( token.params(), luceneMatchVersion );
    tokenFactory.init( tokenMapsOfParameters );
    injectResourceLoader( tokenFactory, defaultResourceLoader, tokenMapsOfParameters );


    final int length = analyzerDef.filters().length;
    final int charLength = analyzerDef.charFilters().length;
    TokenFilterFactory[] filters = new TokenFilterFactory[length];

View Full Code Here

Examples of org.apache.solr.analysis.TokenizerFactory

   * @return a Lucene <code>Analyzer</code>
   */
  public static Analyzer buildAnalyzer(AnalyzerDef analyzerDef, Version luceneMatchVersion) {
    ResourceLoader defaultResourceLoader = new HibernateSearchResourceLoader();
    TokenizerDef token = analyzerDef.tokenizer();
    TokenizerFactory tokenFactory = instanceFromClass( TokenizerFactory.class, token.factory(), "Tokenizer factory" );
    final Map<String, String> tokenMapsOfParameters = getMapOfParameters( token.params(), luceneMatchVersion );
    tokenFactory.init( tokenMapsOfParameters );
    injectResourceLoader( tokenFactory, defaultResourceLoader, tokenMapsOfParameters );


    final int length = analyzerDef.filters().length;
    final int charLength = analyzerDef.charFilters().length;
    TokenFilterFactory[] filters = new TokenFilterFactory[length];

View Full Code Here

Examples of org.apache.solr.analysis.TokenizerFactory

      return namedList;
    }


    TokenizerChain tokenizerChain = (TokenizerChain) analyzer;
    CharFilterFactory[] cfiltfacs = tokenizerChain.getCharFilterFactories();
    TokenizerFactory tfac = tokenizerChain.getTokenizerFactory();
    TokenFilterFactory[] filtfacs = tokenizerChain.getTokenFilterFactories();


    NamedList<List<NamedList>> namedList = new NamedList<List<NamedList>>();


    if( cfiltfacs != null ){
      String source = value;
      for(CharFilterFactory cfiltfac : cfiltfacs ){
        CharStream reader = CharReader.get(new StringReader(source));
        reader = cfiltfac.create(reader);
        source = writeCharStream(namedList, reader);
      }
    }


    TokenStream tokenStream = tfac.create(tokenizerChain.charStream(new StringReader(value)));
    List<AttributeSource> tokens = analyzeTokenStream(tokenStream);


    namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));


    ListBasedTokenStream listBasedTokenStream = new ListBasedTokenStream(tokens);

View Full Code Here

Examples of org.apache.solr.analysis.TokenizerFactory

      if (cfilters.size() > 0) {
        aninfo.add("charFilters", cfilters);
      }
      
      SimpleOrderedMap<Object> tokenizer = new SimpleOrderedMap<Object>();
      TokenizerFactory tfac = tchain.getTokenizerFactory();
      tokenizer.add("className", tfac.getClass().getName());
      tokenizer.add("args", tfac.getArgs());
      aninfo.add("tokenizer", tokenizer);


      TokenFilterFactory[] filtfacs = tchain.getTokenFilterFactories();
      SimpleOrderedMap<Map<String, Object>> filters = new SimpleOrderedMap<Map<String, Object>>();
      for (TokenFilterFactory filtfac : filtfacs) {

View Full Code Here

Examples of org.apache.solr.analysis.TokenizerFactory

    SimpleOrderedMap<Object> aninfo = new SimpleOrderedMap<Object>();
    aninfo.add("className", analyzer.getClass().getName());
    if (analyzer instanceof TokenizerChain) {
       SimpleOrderedMap<Object> tokenizer = new SimpleOrderedMap<Object>();
         TokenizerChain tchain = (TokenizerChain)analyzer;
         TokenizerFactory tfac = tchain.getTokenizerFactory();
         tokenizer.add("className", tfac.getClass().getName());
         tokenizer.add("args", tfac.getArgs());
         aninfo.add("tokenizer", tokenizer);
         TokenFilterFactory[] filtfacs = tchain.getTokenFilterFactories();
         
        SimpleOrderedMap<Map<String, Object>> filters = new SimpleOrderedMap<Map<String, Object>>();
        for (TokenFilterFactory filtfac : filtfacs) {

View Full Code Here

Examples of org.apache.solr.analysis.TokenizerFactory

      return namedList;
    }


    TokenizerChain tokenizerChain = (TokenizerChain) analyzer;
    CharFilterFactory[] cfiltfacs = tokenizerChain.getCharFilterFactories();
    TokenizerFactory tfac = tokenizerChain.getTokenizerFactory();
    TokenFilterFactory[] filtfacs = tokenizerChain.getTokenFilterFactories();


    NamedList<List<NamedList>> namedList = new NamedList<List<NamedList>>();


    if( cfiltfacs != null ){
      String source = value;
      for(CharFilterFactory cfiltfac : cfiltfacs ){
        CharStream reader = CharReader.get(new StringReader(source));
        reader = cfiltfac.create(reader);
        source = writeCharStream(namedList, reader);
      }
    }


    TokenStream tokenStream = tfac.create(tokenizerChain.charStream(new StringReader(value)));
    List<Token> tokens = analyzeTokenStream(tokenStream);


    namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));


    ListBasedTokenStream listBasedTokenStream = new ListBasedTokenStream(tokens);

View Full Code Here

Examples of org.apache.solr.analysis.TokenizerFactory

    SimpleOrderedMap<Object> aninfo = new SimpleOrderedMap<Object>();
    aninfo.add("className", analyzer.getClass().getName());
    if (analyzer instanceof TokenizerChain) {
       SimpleOrderedMap<Object> tokenizer = new SimpleOrderedMap<Object>();
         TokenizerChain tchain = (TokenizerChain)analyzer;
         TokenizerFactory tfac = tchain.getTokenizerFactory();
         tokenizer.add("className", tfac.getClass().getName());
         tokenizer.add("args", tfac.getArgs());
         aninfo.add("tokenizer", tokenizer);
         TokenFilterFactory[] filtfacs = tchain.getTokenFilterFactories();
         
        SimpleOrderedMap<Map<String, Object>> filters = new SimpleOrderedMap<Map<String, Object>>();
        for (TokenFilterFactory filtfac : filtfacs) {

View Full Code Here

Examples of org.apache.solr.analysis.TokenizerFactory

    NodeList nList = (NodeList)xpath.evaluate("./filter", node, XPathConstants.NODESET);


    if (tokNode==null){
      throw new SolrException(1,"analyzer without class or tokenizer & filter list");
    }
    TokenizerFactory tfac = readTokenizerFactory(tokNode);


    /******
    // oops, getChildNodes() includes text (newlines, etc) in addition
    // to the actual child elements
    NodeList nList = node.getChildNodes();

View Full Code Here

Examples of org.apache.solr.analysis.TokenizerFactory

  // <tokenizer class="solr.StandardFilterFactory"/>
  private TokenizerFactory readTokenizerFactory(Node node) {
    // if (node.getNodeName() != "tokenizer") return null;
    NamedNodeMap attrs = node.getAttributes();
    String className = DOMUtil.getAttr(attrs,"class","tokenizer");
    TokenizerFactory tfac = (TokenizerFactory)Config.newInstance(className);
    tfac.init(DOMUtil.toMapExcept(attrs,"class"));
    return tfac;
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.