Package org.apache.accumulo.core.file

Examples of org.apache.accumulo.core.file.FileSKVIterator


   
    readers.clear();

    // TODO need to close files
    for (String file : absFiles) {
      FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuTableConf, null, null);
      readers.add(reader);
    }
   
    MultiIterator multiIter = new MultiIterator(readers, extent);
   
View Full Code Here


    }
  }
 
  @Override
  public FileSKVIterator openReader(String file, boolean seekToBeginning, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
    FileSKVIterator iter = new FileCFSkippingIterator(new RangeIterator(MapFileUtil.openMapFile(acuconf, fs, file, conf)));
   
    if (seekToBeginning)
      iter.seek(new Range(new Key(), null), new ArrayList<ByteSequence>(), false);
   
    return iter;
  }
View Full Code Here

  @Override
  public FileSKVIterator openReader(String file, Range range, Set<ByteSequence> columnFamilies, boolean inclusive, FileSystem fs, Configuration conf,
      AccumuloConfiguration tableConf) throws IOException {
    MyMapFile.Reader mfIter = MapFileUtil.openMapFile(tableConf, fs, file, conf);
   
    FileSKVIterator iter = new RangeIterator(mfIter);
   
    if (columnFamilies.size() != 0 || inclusive) {
      iter = new FileCFSkippingIterator(iter);
    }
   
    iter.seek(range, columnFamilies, inclusive);
    mfIter.dropIndex();
   
    return iter;
  }
View Full Code Here

  private Range range = new Range();
 
  private List<SortedKeyValueIterator<Key,Value>> openMapFiles(Collection<String> files, FileSystem fs, Configuration conf) throws IOException {
    List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<SortedKeyValueIterator<Key,Value>>();
    for (String file : files) {
      FileSKVIterator reader = FileOperations.getInstance().openReader(file, true, fs, conf, ServerConfiguration.getSystemConfiguration());
      readers.add(reader);
    }
    return readers;
  }
View Full Code Here

    }
   
  }
 
  private static void checkIndex(Reader reader) throws IOException {
    FileSKVIterator indexIter = reader.getIndex();
   
    if (indexIter.hasTop()) {
      Key lastKey = new Key(indexIter.getTopKey());
     
      if (reader.getFirstKey().compareTo(lastKey) > 0)
        throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
     
      indexIter.next();
     
      while (indexIter.hasTop()) {
        if (lastKey.compareTo(indexIter.getTopKey()) > 0)
          throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
       
        lastKey = new Key(indexIter.getTopKey());
        indexIter.next();
       
      }
     
      if (reader.getLastKey().compareTo(lastKey) != 0) {
        throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
View Full Code Here

   
    trf.writer.close();
   
    trf.openReader();
   
    FileSKVIterator indexIter = trf.reader.getIndex();
    int count = 0;
    while (indexIter.hasTop()) {
      count++;
      indexIter.next();
    }
   
    assert (count > 4);
   
    trf.reader.seek(new Range(nk("r0000", "cf1", "cq1", "", 1), true, nk("r0001", "cf1", "cq1", "", 1), false), EMPTY_COL_FAMS, false);
View Full Code Here

      Text endRow) throws Exception {
    List<TabletLocation> result = new ArrayList<TabletLocation>();
    Collection<ByteSequence> columnFamilies = Collections.emptyList();
    String filename = file.toString();
    // log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
    FileSKVIterator reader = FileOperations.getInstance().openReader(filename, true, fs, fs.getConf(), acuConf);
    try {
      Text row = startRow;
      if (row == null)
        row = new Text();
      while (true) {
        // log.debug(filename + " Seeking to row " + row);
        reader.seek(new Range(row, null), columnFamilies, false);
        if (!reader.hasTop()) {
          // log.debug(filename + " not found");
          break;
        }
        row = reader.getTopKey().getRow();
        TabletLocation tabletLocation = locator.locateTablet(row, false, true);
        // log.debug(filename + " found row " + row + " at location " + tabletLocation);
        result.add(tabletLocation);
        row = tabletLocation.tablet_extent.getEndRow();
        if (row != null && (endRow == null || row.compareTo(endRow) < 0)) {
          row = new Text(row);
          row.append(byte0, 0, byte0.length);
        } else
          break;
      }
    } finally {
      reader.close();
    }
    // log.debug(filename + " to be sent to " + result);
    return result;
  }
View Full Code Here

        else {
         
          Configuration conf = CachedConfiguration.getInstance();
          FileSystem fs = TraceFileSystem.wrap(FileSystem.getLocal(conf));
         
          @SuppressWarnings("deprecation")
          FileSKVIterator reader = new MapFileOperations.RangeIterator(new MyMapFile.Reader(fs, memDumpFile, conf));
         
          readers.add(reader);
         
          iter = reader;
View Full Code Here

      mfw = null; // set this to null so we do not try to close it again in finally if the close fails
      mfwTmp.close(); // if the close fails it will cause the compaction to fail
     
      // Verify the file, since hadoop 0.20.2 sometimes lies about the success of close()
      try {
        FileSKVIterator openReader = fileFactory.openReader(outputFile, false, fs, conf, tableConf);
        openReader.close();
      } catch (IOException ex) {
        log.error("Verification of successful compaction fails!!! " + extent + " " + outputFile, ex);
        throw ex;
      }
     
View Full Code Here

    for (String mapFile : filesToCompact.keySet()) {
      try {
       
        FileOperations fileFactory = FileOperations.getInstance();
       
        FileSKVIterator reader;
       
        AccumuloConfiguration tableConf = ServerConfiguration.getTableConfiguration(extent.getTableId().toString());
       
        reader = fileFactory.openReader(mapFile, false, fs, conf, tableConf);
       
        readers.add(reader);
       
        SortedKeyValueIterator<Key,Value> iter = new ProblemReportingIterator(extent.getTableId().toString(), mapFile, false, reader);
       
        if (filesToCompact.get(mapFile).isTimeSet()) {
          iter = new TimeSettingIterator(iter, filesToCompact.get(mapFile).getTime());
        }
       
        iters.add(iter);
       
      } catch (Throwable e) {
       
        ProblemReports.getInstance().report(new ProblemReport(extent.getTableId().toString(), ProblemType.FILE_READ, mapFile, e));
       
        log.warn("Some problem opening map file " + mapFile + " " + e.getMessage(), e);
        // failed to open some map file... close the ones that were opened
        for (FileSKVIterator reader : readers) {
          try {
            reader.close();
          } catch (Throwable e2) {
            log.warn("Failed to close map file", e2);
          }
        }
       
View Full Code Here

TOP

Related Classes of org.apache.accumulo.core.file.FileSKVIterator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.