Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.WhitespaceAnalyzer


  @Override
  protected void setUp() throws Exception {
    super.setUp();
    dir = new RAMDirectory();
    appAnalyzer = new WhitespaceAnalyzer();
    IndexWriter writer = new IndexWriter(dir, appAnalyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);
    int numDocs = 200;
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      String variedFieldValue = variedFieldValues[i % variedFieldValues.length];
View Full Code Here


    numHits = search(a, "repetitiveField:vaguelyboring");
    assertTrue(numHits == 0);
  }
 
  public void testTokenStream() throws Exception {
    QueryAutoStopWordAnalyzer a = new QueryAutoStopWordAnalyzer(Version.LUCENE_CURRENT, new WhitespaceAnalyzer());
    a.addStopWords(reader, 10);
    TokenStream ts = a.tokenStream("repetitiveField", new StringReader("this boring"));
    TermAttribute termAtt = ts.getAttribute(TermAttribute.class);
    assertTrue(ts.incrementToken());
    assertEquals("this", termAtt.term());
View Full Code Here

  /**
   * subclass that acts just like whitespace analyzer for testing
   */
  private class QueryAutoStopWordSubclassAnalyzer extends QueryAutoStopWordAnalyzer {
    public QueryAutoStopWordSubclassAnalyzer(Version matchVersion) {
      super(matchVersion, new WhitespaceAnalyzer());
    }
View Full Code Here

 
  @Override
  public void setUp() throws Exception {
    super.setUp();
    RAMDirectory directory = new RAMDirectory();
    IndexWriter writer= new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
    for (int i = 0; i < docFields.length; i++) {
      Document doc = new Document();
      doc.add(new Field(FIELD, docFields[i], Field.Store.NO, Field.Index.ANALYZED));
      writer.addDocument(doc);
    }
View Full Code Here

  @Override
  public void setUp() throws Exception {
    super.setUp();
    RAMDirectory directory = new RAMDirectory();
    IndexWriter writer= new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
    for (int i = 0; i < docFields.length; i++) {
      Document doc = new Document();
      doc.add(new Field(field, docFields[i], Field.Store.YES, Field.Index.ANALYZED));
      writer.addDocument(doc);
    }
View Full Code Here

  public void testUpdateDocument() throws Exception {
    boolean optimize = true;

    Directory dir1 = new MockRAMDirectory();
    IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(),
                                         IndexWriter.MaxFieldLength.LIMITED);

    // create the index
    createIndexNoClose(!optimize, "index1", writer);

    // writer.flush(false, true, true);

    // get a reader
    IndexReader r1 = writer.getReader();
    assertTrue(r1.isCurrent());

    String id10 = r1.document(10).getField("id").stringValue();
   
    Document newDoc = r1.document(10);
    newDoc.removeField("id");
    newDoc.add(new Field("id", Integer.toString(8000), Store.YES, Index.NOT_ANALYZED));
    writer.updateDocument(new Term("id", id10), newDoc);
    assertFalse(r1.isCurrent());

    IndexReader r2 = writer.getReader();
    assertTrue(r2.isCurrent());
    assertEquals(0, count(new Term("id", id10), r2));
    assertEquals(1, count(new Term("id", Integer.toString(8000)), r2));
   
    r1.close();
    writer.close();
    assertTrue(r2.isCurrent());
   
    IndexReader r3 = IndexReader.open(dir1, true);
    assertTrue(r3.isCurrent());
    assertTrue(r2.isCurrent());
    assertEquals(0, count(new Term("id", id10), r3));
    assertEquals(1, count(new Term("id", Integer.toString(8000)), r3));

    writer = new IndexWriter(dir1, new WhitespaceAnalyzer(),
                             IndexWriter.MaxFieldLength.LIMITED);
    Document doc = new Document();
    doc.add(new Field("field", "a b c", Field.Store.NO, Field.Index.ANALYZED));
    writer.addDocument(doc);
    assertTrue(r2.isCurrent());
View Full Code Here

   */
  public void testAddIndexes() throws Exception {
    boolean optimize = false;

    Directory dir1 = new MockRAMDirectory();
    IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(),
        IndexWriter.MaxFieldLength.LIMITED);
    writer.setInfoStream(infoStream);
    // create the index
    createIndexNoClose(!optimize, "index1", writer);
    writer.flush(false, true, true);

    // create a 2nd index
    Directory dir2 = new MockRAMDirectory();
    IndexWriter writer2 = new IndexWriter(dir2, new WhitespaceAnalyzer(),
        IndexWriter.MaxFieldLength.LIMITED);
    writer2.setInfoStream(infoStream);
    createIndexNoClose(!optimize, "index2", writer2);
    writer2.close();

View Full Code Here

 
  public void testAddIndexes2() throws Exception {
    boolean optimize = false;

    Directory dir1 = new MockRAMDirectory();
    IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(),
        IndexWriter.MaxFieldLength.LIMITED);
    writer.setInfoStream(infoStream);

    // create a 2nd index
    Directory dir2 = new MockRAMDirectory();
    IndexWriter writer2 = new IndexWriter(dir2, new WhitespaceAnalyzer(),
        IndexWriter.MaxFieldLength.LIMITED);
    writer2.setInfoStream(infoStream);
    createIndexNoClose(!optimize, "index2", writer2);
    writer2.close();
View Full Code Here

   */
  public void testDeleteFromIndexWriter() throws Exception {
    boolean optimize = true;

    Directory dir1 = new MockRAMDirectory();
    IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(),
        IndexWriter.MaxFieldLength.LIMITED);
    writer.setReaderTermsIndexDivisor(2);
    writer.setInfoStream(infoStream);
    // create the index
    createIndexNoClose(!optimize, "index1", writer);
    writer.flush(false, true, true);
    // get a reader
    IndexReader r1 = writer.getReader();

    String id10 = r1.document(10).getField("id").stringValue();

    // deleted IW docs should not show up in the next getReader
    writer.deleteDocuments(new Term("id", id10));
    IndexReader r2 = writer.getReader();
    assertEquals(1, count(new Term("id", id10), r1));
    assertEquals(0, count(new Term("id", id10), r2));
   
    String id50 = r1.document(50).getField("id").stringValue();
    assertEquals(1, count(new Term("id", id50), r1));
   
    writer.deleteDocuments(new Term("id", id50));
   
    IndexReader r3 = writer.getReader();
    assertEquals(0, count(new Term("id", id10), r3));
    assertEquals(0, count(new Term("id", id50), r3));
   
    String id75 = r1.document(75).getField("id").stringValue();
    writer.deleteDocuments(new TermQuery(new Term("id", id75)));
    IndexReader r4 = writer.getReader();
    assertEquals(1, count(new Term("id", id75), r3));
    assertEquals(0, count(new Term("id", id75), r4));
   
    r1.close();
    r2.close();
    r3.close();
    r4.close();
    writer.close();
       
    // reopen the writer to verify the delete made it to the directory
    writer = new IndexWriter(dir1, new WhitespaceAnalyzer(),
        IndexWriter.MaxFieldLength.LIMITED);
    writer.setInfoStream(infoStream);
    IndexReader w2r1 = writer.getReader();
    assertEquals(0, count(new Term("id", id10), w2r1));
    w2r1.close();
View Full Code Here

  public void testAddIndexesAndDoDeletesThreads() throws Throwable {
    final int numIter = 5;
    int numDirs = 3;
   
    Directory mainDir = new MockRAMDirectory();
    IndexWriter mainWriter = new IndexWriter(mainDir, new WhitespaceAnalyzer(),
        IndexWriter.MaxFieldLength.LIMITED);
    mainWriter.setInfoStream(infoStream);
    AddDirectoriesThreads addDirThreads = new AddDirectoriesThreads(numIter, mainWriter);
    addDirThreads.launchThreads(numDirs);
    addDirThreads.joinThreads();
View Full Code Here

TOP

Related Classes of org.apache.lucene.analysis.WhitespaceAnalyzer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.