Examples of org.apache.accumulo.core.file.FileSKVIterator

org.apache.accumulo.core.file.FileSKVIterator

    if (extent.isRootTablet()) {
      long rtime = Long.MIN_VALUE;
      for (FileRef ref : datafiles.keySet()) {
        Path path = ref.path();
        FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
        FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
        long maxTime = -1;
        try {


          while (reader.hasTop()) {
            maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
            reader.next();
          }


        } finally {
          reader.close();
        }


        if (maxTime > rtime) {
          time = TabletTime.LOGICAL_TIME_ID + "" + maxTime;
          rtime = maxTime;

View Full Code Here

    Map<FileRef,Pair<Key,Key>> result = new HashMap<FileRef,Pair<Key,Key>>();
    FileOperations fileFactory = FileOperations.getInstance();
    for (Entry<FileRef,DataFileValue> entry : allFiles.entrySet()) {
      FileRef file = entry.getKey();
      FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
      FileSKVIterator openReader = fileFactory.openReader(file.path().toString(), true, ns, ns.getConf(), this.getTableConfiguration());
      try {
        Key first = openReader.getFirstKey();
        Key last = openReader.getLastKey();
        result.put(file, new Pair<Key,Key>(first, last));
      } finally {
        openReader.close();
      }
    }
    return result;
  }

View Full Code Here

        
        list.add(reader);
      }
      
      for (FileDataSource fds : dataSources) {
        FileSKVIterator reader = map.get(fds.file).remove(0);
        fds.setIterator(reader);
      }
    }

View Full Code Here

        else {
          
          Configuration conf = CachedConfiguration.getInstance();
          FileSystem fs = TraceFileSystem.wrap(FileSystem.getLocal(conf));
          
          FileSKVIterator reader = new RFileOperations().openReader(memDumpFile, true, fs, conf, ServerConfiguration.getSiteConfiguration());


          readers.add(reader);
          
          iter = new MemKeyConversionIterator(reader);
        }

View Full Code Here

    /*
     * Begin the import - iterate the files in the path
     */
    for (FileStatus importStatus : fs.listStatus(importPath)) {
      try {
        FileSKVIterator importIterator = FileOperations.getInstance().openReader(importStatus.getPath().toString(), true, fs, fs.getConf(),
            AccumuloConfiguration.getDefaultConfiguration());
        while (importIterator.hasTop()) {
          Key key = importIterator.getTopKey();
          Value value = importIterator.getTopValue();
          if (setTime) {
            key.setTimestamp(time);
          }
          Mutation mutation = new Mutation(key.getRow());
          if (!key.isDeleted()) {
            mutation.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibilityData().toArray()), key.getTimestamp(),
                value);
          } else {
            mutation.putDelete(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibilityData().toArray()),
                key.getTimestamp());
          }
          table.addMutation(mutation);
          importIterator.next();
        }
      } catch (Exception e) {
        FSDataOutputStream failureWriter = null;
        DataInputStream failureReader = null;
        try {

View Full Code Here

  public FileSKVIterator openReader(FileRef ref) throws IOException {
    // @TODO verify the file isn't some random file in HDFS
    // @TODO ensure these files are always closed?
    FileOperations fileFactory = FileOperations.getInstance();
    FileSystem ns = volumeManager.getVolumeByPath(ref.path()).getFileSystem();
    FileSKVIterator openReader = fileFactory.openReader(ref.path().toString(), true, ns, ns.getConf(), tableConfig);
    return openReader;
  }

View Full Code Here

      mfw = null; // set this to null so we do not try to close it again in finally if the close fails
      mfwTmp.close(); // if the close fails it will cause the compaction to fail


      // Verify the file, since hadoop 0.20.2 sometimes lies about the success of close()
      try {
        FileSKVIterator openReader = fileFactory.openReader(outputFile.path().toString(), false, ns, ns.getConf(), acuTableConf);
        openReader.close();
      } catch (IOException ex) {
        log.error("Verification of successful compaction fails!!! " + extent + " " + outputFile, ex);
        throw ex;
      }

View Full Code Here

    for (FileRef mapFile : filesToCompact.keySet()) {
      try {


        FileOperations fileFactory = FileOperations.getInstance();
        FileSystem fs = this.fs.getVolumeByPath(mapFile.path()).getFileSystem();
        FileSKVIterator reader;


        reader = fileFactory.openReader(mapFile.path().toString(), false, fs, conf, acuTableConf);


        readers.add(reader);


        SortedKeyValueIterator<Key,Value> iter = new ProblemReportingIterator(extent.getTableId().toString(), mapFile.path().toString(), false, reader);


        if (filesToCompact.get(mapFile).isTimeSet()) {
          iter = new TimeSettingIterator(iter, filesToCompact.get(mapFile).getTime());
        }


        iters.add(iter);


      } catch (Throwable e) {


        ProblemReports.getInstance().report(new ProblemReport(extent.getTableId().toString(), ProblemType.FILE_READ, mapFile.path().toString(), e));


        log.warn("Some problem opening map file " + mapFile + " " + e.getMessage(), e);
        // failed to open some map file... close the ones that were opened
        for (FileSKVIterator reader : readers) {
          try {
            reader.close();
          } catch (Throwable e2) {
            log.warn("Failed to close map file", e2);
          }
        }

View Full Code Here

        if (!file.contains(":"))
          throw new IllegalArgumentException("Expected uri, got : " + file);
        Path path = new Path(file);
        FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
        //log.debug("Opening "+file + " path " + path);
        FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), false, ns, ns.getConf(), conf.getTableConfiguration(table.toString()),
            dataCache, indexCache);
        reservedFiles.add(reader);
        readersReserved.put(reader, file);
      } catch (Exception e) {

View Full Code Here

    int count = 0;
    
    HashSet<ByteSequence> columnSet = createColumnBSS(columns);
    
    for (String file : files) {
      FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuconf);
      Range range = new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true);
      reader.seek(range, columnSet, columnSet.size() == 0 ? false : true);
      while (reader.hasTop() && !range.afterEndKey(reader.getTopKey())) {
        count++;
        reader.next();
      }
      reader.close();
    }
    
    return count;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.accumulo.core.file.FileSKVIterator

org.apache.accumulo.core.client.impl.OfflineIterator

org.apache.accumulo.core.client.mock.MockTableOperations

org.apache.accumulo.core.client.mock.MockTableOperationsImpl

org.apache.accumulo.core.file.map.MapFileOperations

org.apache.accumulo.core.file.rfile.RFileOperations

org.apache.accumulo.core.file.rfile.RFileTest

org.apache.accumulo.core.util.BulkImportHelper

org.apache.accumulo.server.client.BulkImporter

org.apache.accumulo.server.tabletserver.Compactor

org.apache.accumulo.server.tabletserver.FileManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.