Package edu.buffalo.cse.ir.wikiindexer.tokenizer

Examples of edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenStream.previous()

704
705
706
707
708
709
710
711
712
713
714
    //valid new token, invalid position
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");
    stream.seekEnd();
    stream.set("valid");
    assertEquals("stream", stream.previous());
    stream = null;
   
    //correct set, single token
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");
View Full Code Here
719
720
721
722
723
724
725
726
727
728
729
   
    //correct set, multiple tokens at the end
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");
    stream.seekEnd();
    stream.previous();
    stream.set("of", "the", "set", "method");
    assertEquals(8, stream.getAllTokens().size());
    assertEquals("method", stream.next());
    stream = null;
   
View Full Code Here
737
738
739
740
741
742
743
744
745
746
747
   
    //correct set, multiple tokens in the middle
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");
    stream.seekEnd();
    stream.previous();
    stream.previous();
    stream.set("really", "interesting");
    assertEquals(6, stream.getAllTokens().size());
    assertEquals("interesting", stream.next());
    assertEquals("stream", stream.next());
View Full Code Here
738
739
740
741
742
743
744
745
746
747
748
    //correct set, multiple tokens in the middle
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");
    stream.seekEnd();
    stream.previous();
    stream.previous();
    stream.set("really", "interesting");
    assertEquals(6, stream.getAllTokens().size());
    assertEquals("interesting", stream.next());
    assertEquals("stream", stream.next());
    assertFalse(stream.hasNext());
View Full Code Here
792
793
794
795
796
797
798
799
800
801
802
   
    //positive run
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");
    stream.seekEnd();
    assertEquals("stream", stream.previous());
    stream = null;
  }

  /**
   * Test method for {@link edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenStream#merge(edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenStream)}.
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
    stream.remove();
    smap = getSortedMap(stream.getTokenMap());
    assertEquals("[be, not, or, to]", smap.keySet().toString());
    assertEquals("[2, 1, 1, 1]", smap.values().toString());
    stream.seekEnd();
    stream.previous(); //be
    stream.previous(); //to
    stream.remove();
    stream.previous();
    stream.remove();
    smap = getSortedMap(stream.getTokenMap());
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
    smap = getSortedMap(stream.getTokenMap());
    assertEquals("[be, not, or, to]", smap.keySet().toString());
    assertEquals("[2, 1, 1, 1]", smap.values().toString());
    stream.seekEnd();
    stream.previous(); //be
    stream.previous(); //to
    stream.remove();
    stream.previous();
    stream.remove();
    smap = getSortedMap(stream.getTokenMap());
    assertEquals("[be, or]", smap.keySet().toString());
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
    assertEquals("[2, 1, 1, 1]", smap.values().toString());
    stream.seekEnd();
    stream.previous(); //be
    stream.previous(); //to
    stream.remove();
    stream.previous();
    stream.remove();
    smap = getSortedMap(stream.getTokenMap());
    assertEquals("[be, or]", smap.keySet().toString());
    assertEquals("[2, 1]", smap.values().toString());
    stream = null;
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
    stream = new TokenStream("to");
    stream.append("be", "or", "not", "to", "be");
    stream.next(); //at be
    stream.mergeWithPrevious();
    stream.seekEnd();
    stream.previous();
    stream.mergeWithPrevious();
    smap = getSortedMap(stream.getTokenMap());
    assertEquals("[not, or, to be]", smap.keySet().toString());
    assertEquals("[1, 1, 2]", smap.values().toString());
    stream = null;
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
    //with merge with next
    stream = new TokenStream("to");
    stream.append("be", "or", "not", "to", "be");
    stream.mergeWithNext();
    stream.seekEnd();
    stream.previous();
    stream.previous();
    stream.mergeWithNext();
    smap = getSortedMap(stream.getTokenMap());
    assertEquals("[not, or, to be]", smap.keySet().toString());
    assertEquals("[1, 1, 2]", smap.values().toString());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.