Examples of Analyzer

Examples of org.apache.lucene.analysis.Analyzer

30
31
32
33
34
35
36
37
38
39
40
      e.printStackTrace();
    }
  }
 
  public void testComplex() {
    Analyzer analyzer = new ComplexAnalyzer();
    try {
      //txt = "1884年,中法战争时被派福建会办海疆事务";
      //txt = "1999年12345日报道了一条新闻,2000年中法国足球比赛";
      /*txt = "第一卷 云天落日圆 第一节 偷欢不成倒大霉";
      txt = "中国人民银行";
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer

49
50
51
52
53
54
55
56
57
58
59
      e.printStackTrace();
    }
  }
 
  public void testMaxWord() {
    Analyzer analyzer = new MaxWordAnalyzer();
    try {
      //txt = "1884年,中法战争时被派福建会办海疆事务";
      //txt = "1999年12345日报道了一条新闻,2000年中法国足球比赛";
      //txt = "第一卷 云天落日圆 第一节 偷欢不成倒大霉";
      //txt = "中国人民银行";
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer

89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
    }
    Properties analyzers = new Properties();
    analyzers.load(new FileInputStream(new File("analyzer.properties")));
    String mode = System.getProperty("mode", "complex");
    String a = System.getProperty("analyzer", "mmseg4j");
    Analyzer analyzer = null;
    String an = (String) analyzers.get(a);
    if(an != null) {
      analyzer = (Analyzer)Class.forName(an).newInstance();
      mode = a;
    } else {
      usage();
      return;
    }
    if(args.length > 1) {
      try {
        n = Integer.parseInt(args[1]);
      } catch (NumberFormatException e) {
       
      }
    }
    File path = new File(args[0]);
    System.out.println("analyzer="+analyzer.getClass().getName());
    Effect ef = new Effect(path, analyzer);
    ef.run(mode, n);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer

71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
    }
    Properties analyzers = new Properties();
    analyzers.load(new FileInputStream(new File("analyzer.properties")));
    String mode = System.getProperty("mode", "complex");
    String a = System.getProperty("analyzer", "mmseg4j");
    Analyzer analyzer = null;
    String an = (String) analyzers.get(a);
    if(an != null) {
      analyzer = (Analyzer)Class.forName(an).newInstance();
      mode = a;
    } else {
      usage();
      return;
    }
    if(args.length > 1) {
      try {
        n = Integer.parseInt(args[1]);
      } catch (NumberFormatException e) {
       
      }
    }
    File path = new File(txtPath);
    System.out.println("analyzer="+analyzer.getClass().getName());
    PerformanceAnalyzer pa = new PerformanceAnalyzer(path, analyzer);
    pa.run(mode, n);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer

31
32
33
34
35
36
37
38
39
40
41
42
43
44
public class TestSmartChineseAnalyzer extends TestCase {

  public void testChineseAnalyzer() throws IOException {
    Token nt = new Token();
    Analyzer ca = new SmartChineseAnalyzer(true);
    Reader sentence = new StringReader("我购买了道具和服装。");
    String[] result = { "我", "购买", "了", "道具", "和", "服装" };
    TokenStream ts = ca.tokenStream("sentence", sentence);
    int i = 0;
    nt = ts.next(nt);
    while (nt != null) {
      assertEquals(result[i], nt.term());
      i++;
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
   * @throws IOException
   */
  private void sampleMethod() throws UnsupportedEncodingException,
      FileNotFoundException, IOException {
    Token nt = new Token();
    Analyzer ca = new SmartChineseAnalyzer(true);
    Reader sentence = new StringReader(
        "我从小就不由自主地认为自己长大以后一定得成为一个象我父亲一样的画家, 可能是父母潜移默化的影响。其实我根本不知道作为画家意味着什么，我是否喜欢，最重要的是否适合我，我是否有这个才华。其实人到中年的我还是不确定我最喜欢什么，最想做的是什么？我相信很多人和我一样有同样的烦恼。毕竟不是每个人都能成为作文里的宇航员，科学家和大教授。知道自己适合做什么，喜欢做什么，能做好什么其实是个非常困难的问题。"
            + "幸运的是，我想我的孩子不会为这个太过烦恼。通过老大，我慢慢发现美国高中的一个重要功能就是帮助学生分析他们的专长和兴趣，从而帮助他们选择大学的专业和未来的职业。我觉得帮助一个未成形的孩子找到她未来成长的方向是个非常重要的过程。"
            + "美国高中都有专门的职业顾问，通过接触不同的课程，和各种心理，个性，兴趣很多方面的问答来帮助每个学生找到最感兴趣的专业。这样的教育一般是要到高年级才开始， 可老大因为今年上计算机的课程就是研究一个职业走向的软件项目，所以她提前做了这些考试和面试。看来以后这样的教育会慢慢由电脑来测试了。老大带回家了一些试卷，我挑出一些给大家看看。这门课她花了2个多月才做完，这里只是很小的一部分。"
            + "在测试里有这样的一些问题："
            + "你是个喜欢动手的人吗？ 你喜欢修东西吗？你喜欢体育运动吗？你喜欢在室外工作吗？你是个喜欢思考的人吗？你喜欢数学和科学课吗？你喜欢一个人工作吗？你对自己的智力自信吗？你的创造能力很强吗？你喜欢艺术，音乐和戏剧吗？  你喜欢自由自在的工作环境吗？你喜欢尝试新的东西吗？ 你喜欢帮助别人吗？你喜欢教别人吗？你喜欢和机器和工具打交道吗？你喜欢当领导吗？你喜欢组织活动吗？你什么和数字打交道吗？");
    TokenStream ts = ca.tokenStream("sentence", sentence);

    System.out.println("start: " + (new Date()));
    long before = System.currentTimeMillis();
    nt = ts.next(nt);
    while (nt != null) {
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer

907
908
909
910
911
912
913
914
915
916
917
 
  public void testIndexingThenDeleting() throws Exception {
    final Random r = random;
    Directory dir = newDirectory();
    // note this test explicitly disables payloads
    final Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
      }
    };
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer

105
106
107
108
109
110
111
112
113
114
115
    }
    reader.close();
  }

  public void testPositionIncrementGap() throws IOException {
    Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
      }
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer

139
140
141
142
143
144
145
146
147
148
149
    assertEquals(502, termPositions.nextPosition());
    reader.close();
  }

  public void testTokenReuse() throws IOException {
    Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new TokenFilter(new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader)) {
          boolean first=true;
          AttributeSource.State state;
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer

109
110
111
112
113
114
115
116
117
118
119
      "JFK has been shot", "John Kennedy has been shot",
      "This text has a typo in referring to Keneddy",
      "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets" };

  public void testQueryScorerHits() throws Exception {
    Analyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
    QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, FIELD_NAME, analyzer);
    query = qp.parse("\"very long\"");
    searcher = new IndexSearcher(ramDir, true);
    TopDocs hits = searcher.search(query, 10);
   
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.