Package org.apache.accumulo.core.iterators.system

Examples of org.apache.accumulo.core.iterators.system.MultiIterator

71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    try {
      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(mapfiles.length);
      for (int i = 0; i < mapfiles.length; i++)
        iters.add(mapfiles[i]);
     
      MultiIterator mmfi = new MultiIterator(iters, new KeyExtent(new Text(""), endRow, prevEndRow));
      mmfi.seek(new Range(new Key(startRow), null), LocalityGroupUtil.EMPTY_CF_SET, false);
     
      int count = 0;
     
      long t1 = System.currentTimeMillis();
     
      Key lastKey = new Key();
     
      while (mmfi.hasTop()) {
        Key key = mmfi.getTopKey();
       
        if (lastKey.compareTo(key) > 0) {
          String msg = "Not sorted : " + lastKey + " " + key;
          System.err.println(msg);
          throw new RuntimeException(msg);
        }
       
        lastKey.set(key);
       
        System.out.println(" " + key);
       
        mmfi.next();
        count++;
      }
     
      long t2 = System.currentTimeMillis();
     
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(readers.length);
    for (int i = 0; i < readers.length; i++) {
      iters.add(readers[i]);
    }
   
    MultiIterator mmfi = new MultiIterator(iters, new KeyExtent(new Text(""), null, null));
    mmfi.seek(new Range(new Key(), null), LocalityGroupUtil.EMPTY_CF_SET, false);
   
    long t1 = System.currentTimeMillis();
    int count = 0;
    int misses = 0;
    for (Key key : queries) {
      mmfi.seek(new Range(key, null), LocalityGroupUtil.EMPTY_CF_SET, false);
      if (mmfi.getTopKey().compareTo(key) != 0)
        misses++;
      count++;
    }
   
    long t2 = System.currentTimeMillis();
View Full Code Here
92
93
94
95
96
97
98
    SortedKeyValueIterator<Key,Value> getTopLevelIterator(SortedKeyValueIterator<Key,Value> iter) {
      if (topLevelIterators.isEmpty())
        return iter;
      ArrayList<SortedKeyValueIterator<Key,Value>> allIters = new ArrayList<SortedKeyValueIterator<Key,Value>>(topLevelIterators);
      allIters.add(iter);
      return new MultiIterator(allIters, false);
    }
View Full Code Here
433
434
435
436
437
438
439
440
441
442
443
    List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(mapfiles.size() + 1);
   
    iters.addAll(mapfiles);
    iters.add(smi);
   
    MultiIterator multiIter = new MultiIterator(iters, ke);
    DeletingIterator delIter = new DeletingIterator(multiIter, false);
    ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
    ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, columnSet);
    VisibilityFilter visFilter = new VisibilityFilter(colFilter, authorizations, defaultLabels);
   
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
    return readers;
  }
 
  private SortedKeyValueIterator<Key,Value> createSystemIter(Range r, List<SortedKeyValueIterator<Key,Value>> readers, HashSet<Column> columns)
      throws IOException {
    MultiIterator multiIterator = new MultiIterator(readers, false);
    DeletingIterator delIter = new DeletingIterator(multiIterator, false);
    ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
    ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, columns);
    VisibilityFilter visFilter = new VisibilityFilter(colFilter, Constants.NO_AUTHS, new byte[0]);
   
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
        for (String s : inFiles) {
          reader = FileOperations.getInstance().openIndex(s, fs, conf, acuConf);
          iters.add(reader);
        }
       
        MultiIterator mmfi = new MultiIterator(iters, true);
       
        while (mmfi.hasTop()) {
          Key key = mmfi.getTopKey();
         
          boolean gtPrevEndRow = prevEndRow == null || key.compareRow(prevEndRow) > 0;
          boolean lteEndRow = endRow == null || key.compareRow(endRow) <= 0;
         
          if (gtPrevEndRow && lteEndRow)
            writer.append(key, new LongWritable(0));
         
          if (!lteEndRow)
            break;
         
          mmfi.next();
        }
      } finally {
        try {
          if (reader != null)
            reader.close();
View Full Code Here
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
        // the data just punt and return .5
        return .5;
      }
     
      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(readers);
      MultiIterator mmfi = new MultiIterator(iters, true);
     
      // skip the prevendrow
      while (mmfi.hasTop() && mmfi.getTopKey().compareRow(prevEndRow) <= 0) {
        mmfi.next();
      }
     
      int numLte = 0;
     
      while (mmfi.hasTop() && mmfi.getTopKey().compareRow(splitRow) <= 0) {
        numLte++;
        mmfi.next();
      }
     
      if (numLte > numKeys) {
        // something went wrong
        throw new RuntimeException("numLte > numKeys " + numLte + " " + numKeys + " " + prevEndRow + " " + endRow + " " + splitRow + " " + mapFiles);
View Full Code Here
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
        }
        throw new IOException("Failed to find mid point, no entries between " + prevEndRow + " and " + endRow + " for " + mapFiles);
      }
     
      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(readers);
      MultiIterator mmfi = new MultiIterator(iters, true);
     
      // skip the prevendrow
      while (mmfi.hasTop() && mmfi.getTopKey().compareRow(prevEndRow) <= 0)
        mmfi.next();
     
      // read half of the keys in the index
      TreeMap<Double,Key> ret = new TreeMap<Double,Key>();
      Key lastKey = null;
      long keysRead = 0;
     
      Key keyBeforeMidPoint = null;
      long keyBeforeMidPointPosition = 0;
     
      while (keysRead < numKeys / 2) {
        if (lastKey != null && !lastKey.equals(mmfi.getTopKey(), PartialKey.ROW) && (keysRead - 1) / (double) numKeys >= minSplit) {
          keyBeforeMidPoint = new Key(lastKey);
          keyBeforeMidPointPosition = keysRead - 1;
        }
       
        if (lastKey == null)
          lastKey = new Key();
       
        lastKey.set(mmfi.getTopKey());
       
        keysRead++;
       
        // consume minimum
        mmfi.next();
      }
     
      if (keyBeforeMidPoint != null)
        ret.put(keyBeforeMidPointPosition / (double) numKeys, keyBeforeMidPoint);
     
      long t2 = System.currentTimeMillis();
     
      log.debug(String.format("Found midPoint from indexes in %6.2f secs.\n", ((t2 - t1) / 1000.0)));
     
      ret.put(.5, mmfi.getTopKey());
     
      // sanity check
      for (Key key : ret.values()) {
        boolean inRange = (key.compareRow(prevEndRow) > 0 && (endRow == null || key.compareRow(endRow) < 1));
        if (!inRange) {
View Full Code Here
216
217
218
219
220
221
222
223
224
225
226
    SortedKeyValueIterator<Key,Value> source = createIteratorStack(NUM_ROWS, NUM_DOCIDS, columnFamilies, otherColumnFamilies, docs);
    SortedKeyValueIterator<Key,Value> source2 = createIteratorStack(NUM_ROWS, NUM_DOCIDS, columnFamilies, otherColumnFamilies, docs);
    ArrayList<SortedKeyValueIterator<Key,Value>> sourceIters = new ArrayList<SortedKeyValueIterator<Key,Value>>();
    sourceIters.add(source);
    sourceIters.add(source2);
    MultiIterator mi = new MultiIterator(sourceIters, false);
    IteratorSetting is = new IteratorSetting(1, IntersectingIterator.class);
    IntersectingIterator.setColumnFamilies(is, columnFamilies);
    IntersectingIterator iter = new IntersectingIterator();
    iter.init(mi, is.getOptions(), env);
    iter.seek(new Range(), EMPTY_COL_FAMS, false);
View Full Code Here
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
      iters.addAll(memIters);
     
      for (SortedKeyValueIterator<Key,Value> skvi : iters)
        ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
     
      MultiIterator multiIter = new MultiIterator(iters, extent);
     
      TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(IteratorScope.scan, acuTableConf, fileManager, files);
     
      DeletingIterator delIter = new DeletingIterator(multiIter, false);
     
View Full Code Here
TOP

Related Classes of org.apache.accumulo.core.iterators.system.MultiIterator

  • org.apache.accumulo.core.client.impl.OfflineIterator
  • org.apache.accumulo.core.client.impl.OfflineIterator$OfflineIteratorEnvironment
  • org.apache.accumulo.core.client.mock.MockScannerBase$MockIteratorEnvironment
  • org.apache.accumulo.core.data.Range
  • org.apache.accumulo.core.file.FileUtil
  • org.apache.accumulo.core.iterators.AggregatingIteratorTest
  • org.apache.accumulo.core.iterators.system.MultiIteratorTest
  • org.apache.accumulo.core.iterators.user.CombinerTest
  • org.apache.accumulo.core.iterators.user.IndexedDocIteratorTest
  • org.apache.accumulo.core.iterators.user.IntersectingIteratorTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.