Package org.apache.accumulo.core.file

Examples of org.apache.accumulo.core.file.FileSKVIterator

1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
    trf.writer.close();

    trf.openReader();

    FileSKVIterator indexIter = trf.reader.getIndex();
    int count = 0;
    while (indexIter.hasTop()) {
      count++;
      indexIter.next();
    }

    assertTrue(count > 4);

    trf.iter.seek(new Range(nk("r0000", "cf1", "cq1", "", 1), true, nk("r0001", "cf1", "cq1", "", 1), false), EMPTY_COL_FAMS, false);
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
    }

  }

  private static void checkIndex(Reader reader) throws IOException {
    FileSKVIterator indexIter = reader.getIndex();

    if (indexIter.hasTop()) {
      Key lastKey = new Key(indexIter.getTopKey());

      if (reader.getFirstKey().compareTo(lastKey) > 0)
        throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);

      indexIter.next();

      while (indexIter.hasTop()) {
        if (lastKey.compareTo(indexIter.getTopKey()) > 0)
          throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());

        lastKey = new Key(indexIter.getTopKey());
        indexIter.next();

      }

      if (!reader.getLastKey().equals(lastKey)) {
        throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
View Full Code Here
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
    List<TabletLocation> result = new ArrayList<TabletLocation>();
    Collection<ByteSequence> columnFamilies = Collections.emptyList();
    String filename = file.toString();
    // log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
    FileSystem fs = vm.getVolumeByPath(file).getFileSystem();
    FileSKVIterator reader = FileOperations.getInstance().openReader(filename, true, fs, fs.getConf(), acuConf);
    try {
      Text row = startRow;
      if (row == null)
        row = new Text();
      while (true) {
        // log.debug(filename + " Seeking to row " + row);
        reader.seek(new Range(row, null), columnFamilies, false);
        if (!reader.hasTop()) {
          // log.debug(filename + " not found");
          break;
        }
        row = reader.getTopKey().getRow();
        TabletLocation tabletLocation = locator.locateTablet(credentials, row, false, true);
        // log.debug(filename + " found row " + row + " at location " + tabletLocation);
        result.add(tabletLocation);
        row = tabletLocation.tablet_extent.getEndRow();
        if (row != null && (endRow == null || row.compareTo(endRow) < 0)) {
          row = new Text(row);
          row.append(byte0, 0, byte0.length);
        } else
          break;
      }
    } finally {
      reader.close();
    }
    // log.debug(filename + " to be sent to " + result);
    return result;
  }
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
      FileSystem ns = fs.getVolumeByPath(new Path(newMapFile)).getFileSystem();
      FileSKVWriter writer = new RFileOperations().openWriter(newMapFile.toString(), ns, ns.getConf(), acuConf);
      writer.startDefaultLocalityGroup();
      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(inFiles.size());
     
      FileSKVIterator reader = null;
      try {
        for (String s : inFiles) {
          ns = fs.getVolumeByPath(new Path(s)).getFileSystem();
          reader = FileOperations.getInstance().openIndex(s, ns, ns.getConf(), acuConf);
          iters.add(reader);
        }
       
        MultiIterator mmfi = new MultiIterator(iters, true);
       
        while (mmfi.hasTop()) {
          Key key = mmfi.getTopKey();
         
          boolean gtPrevEndRow = prevEndRow == null || key.compareRow(prevEndRow) > 0;
          boolean lteEndRow = endRow == null || key.compareRow(endRow) <= 0;
         
          if (gtPrevEndRow && lteEndRow)
            writer.append(key, new Value(new byte[0]));
         
          if (!lteEndRow)
            break;
         
          mmfi.next();
        }
      } finally {
        try {
          if (reader != null)
            reader.close();
        } catch (IOException e) {
          log.error(e, e);
        }
       
        for (SortedKeyValueIterator<Key,Value> r : iters)
View Full Code Here
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
   
    long numKeys = 0;
   
    // count the total number of index entries
    for (String ref : mapFiles) {
      FileSKVIterator reader = null;
      Path path = new Path(ref);
      FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
      try {
        if (useIndex)
          reader = FileOperations.getInstance().openIndex(path.toString(), ns, ns.getConf(), acuConf);
        else
          reader = FileOperations.getInstance().openReader(path.toString(), new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET, false, ns, ns.getConf(),
              acuConf);
       
        while (reader.hasTop()) {
          Key key = reader.getTopKey();
          if (endRow != null && key.compareRow(endRow) > 0)
            break;
          else if (prevEndRow == null || key.compareRow(prevEndRow) > 0)
            numKeys++;
         
          reader.next();
        }
      } finally {
        try {
          if (reader != null)
            reader.close();
        } catch (IOException e) {
          log.error(e, e);
        }
      }
     
View Full Code Here
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
   
    long t1 = System.currentTimeMillis();
   
    for (FileRef mapfile : mapfiles) {
     
      FileSKVIterator reader = null;
      FileSystem ns = fs.getVolumeByPath(mapfile.path()).getFileSystem();
      try {
        reader = FileOperations.getInstance().openReader(mapfile.toString(), false, ns, ns.getConf(), acuConf);
       
        Key firstKey = reader.getFirstKey();
        if (firstKey != null) {
          mapFilesInfo.put(mapfile, new FileInfo(firstKey, reader.getLastKey()));
        }
       
      } catch (IOException ioe) {
        log.warn("Failed to read map file to determine first and last key : " + mapfile, ioe);
      } finally {
        if (reader != null) {
          try {
            reader.close();
          } catch (IOException ioe) {
            log.warn("failed to close " + mapfile, ioe);
          }
        }
      }
View Full Code Here
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
    Key lastKey = null;
   
    for (FileRef ref : mapFiles) {
      Path path = ref.path();
      FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
      FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), acuConf);
     
      try {
        if (!reader.hasTop())
          // file is empty, so there is no last key
          continue;
       
        Key key = reader.getLastKey();
       
        if (lastKey == null || key.compareTo(lastKey) > 0)
          lastKey = key;
      } finally {
        try {
          if (reader != null)
            reader.close();
        } catch (IOException e) {
          log.error(e, e);
        }
      }
    }
View Full Code Here
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
    for (KeyExtent keyExtent : extents)
      counts.put(keyExtent, new MLong(0));
   
    Text row = new Text();
    FileSystem ns = fs.getVolumeByPath(mapFile).getFileSystem();
    FileSKVIterator index = FileOperations.getInstance().openIndex(mapFile.toString(), ns, ns.getConf(), acuConf);
   
    try {
      while (index.hasTop()) {
        Key key = index.getTopKey();
        totalIndexEntries++;
        key.getRow(row);
       
        for (Entry<KeyExtent,MLong> entry : counts.entrySet())
          if (entry.getKey().contains(row))
            entry.getValue().l++;
       
        index.next();
      }
    } finally {
      try {
        if (index != null)
          index.close();
      } catch (IOException e) {
        // continue with next file
        log.error(e, e);
      }
    }
View Full Code Here
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
   
    HashSet<ByteSequence> columnSet = createColumnBSS(columns);
   
    for (FileRef file : files) {
      FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
      FileSKVIterator reader = FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf);
      Range range = new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true);
      reader.seek(range, columnSet, columnSet.size() == 0 ? false : true);
      while (reader.hasTop() && !range.afterEndKey(reader.getTopKey())) {
        count++;
        reader.next();
      }
      reader.close();
    }
   
    return count;
  }
View Full Code Here
308
309
310
311
312
313
314
315
316
317
318
    readers.clear();
   
    // TODO need to close files - ACCUMULO-1303
    for (String file : absFiles) {
      FileSystem fs = VolumeConfiguration.getVolume(file, conf, config).getFileSystem();
      FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuTableConf, null, null);
      readers.add(reader);
    }
   
    MultiIterator multiIter = new MultiIterator(readers, extent);
   
View Full Code Here
TOP

Related Classes of org.apache.accumulo.core.file.FileSKVIterator

  • org.apache.accumulo.core.client.impl.OfflineIterator
  • org.apache.accumulo.core.client.mock.MockTableOperations
  • org.apache.accumulo.core.client.mock.MockTableOperationsImpl
  • org.apache.accumulo.core.file.map.MapFileOperations
  • org.apache.accumulo.core.file.rfile.RFileOperations
  • org.apache.accumulo.core.file.rfile.RFileTest
  • org.apache.accumulo.core.util.BulkImportHelper
  • org.apache.accumulo.server.client.BulkImporter
  • org.apache.accumulo.server.tabletserver.Compactor
  • org.apache.accumulo.server.tabletserver.FileManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.