Package org.apache.accumulo.core.util

Examples of org.apache.accumulo.core.util.OpTimer


        }
       
        try {
          TabletClientService.Iface client = ThriftUtil.getTServerClient(tl.tablet_location, instance.getConfiguration());
          try {
            OpTimer opTimer = null;
            if (log.isTraceEnabled())
              opTimer = new OpTimer(log, Level.TRACE).start("Splitting tablet " + tl.tablet_extent + " on " + tl.tablet_location + " at " + split);
           
            client.splitTablet(null, credentials, tl.tablet_extent.toThrift(), TextUtil.getByteBuffer(split));
           
            // just split it, might as well invalidate it in the cache
            tabLocator.invalidateCache(tl.tablet_extent);
           
            if (opTimer != null)
              opTimer.stop("Split tablet in %DURATION%");
          } finally {
            ThriftUtil.returnClient((TServiceClient) client);
          }
         
        } catch (TApplicationException tae) {
View Full Code Here


  private static List<KeyValue> scan(TabletLocation loc, ScanState scanState, AccumuloConfiguration conf) throws AccumuloSecurityException,
      NotServingTabletException, TException, NoSuchScanIDException, TooManyFilesException {
    if (scanState.finished)
      return null;
   
    OpTimer opTimer = new OpTimer(log, Level.TRACE);
   
    TabletClientService.Iface client = ThriftUtil.getTServerClient(loc.tablet_location, conf);
   
    String old = Thread.currentThread().getName();
    try {
      ScanResult sr;
     
      if (scanState.prevLoc != null && !scanState.prevLoc.equals(loc))
        scanState.scanID = null;
     
      scanState.prevLoc = loc;
     
      if (scanState.scanID == null) {
        String msg = "Starting scan tserver=" + loc.tablet_location + " tablet=" + loc.tablet_extent + " range=" + scanState.range + " ssil="
            + scanState.serverSideIteratorList + " ssio=" + scanState.serverSideIteratorOptions;
        Thread.currentThread().setName(msg);
        opTimer.start(msg);
       
        TabletType ttype = TabletType.type(loc.tablet_extent);
        boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(loc.tablet_location);
        InitialScan is = client.startScan(null, scanState.credentials, loc.tablet_extent.toThrift(), scanState.range.toThrift(),
            Translator.translate(scanState.columns, Translator.CT), scanState.size, scanState.serverSideIteratorList, scanState.serverSideIteratorOptions,
            scanState.authorizations.getAuthorizationsBB(), waitForWrites, scanState.isolated);
        if (waitForWrites)
          serversWaitedForWrites.get(ttype).add(loc.tablet_location);
       
        sr = is.result;
       
        if (sr.more)
          scanState.scanID = is.scanID;
        else
          client.closeScan(null, is.scanID);
       
      } else {
        // log.debug("Calling continue scan : "+scanState.range+"  loc = "+loc);
        String msg = "Continuing scan tserver=" + loc.tablet_location + " scanid=" + scanState.scanID;
        Thread.currentThread().setName(msg);
        opTimer.start(msg);
       
        sr = client.continueScan(null, scanState.scanID);
        if (!sr.more) {
          client.closeScan(null, scanState.scanID);
          scanState.scanID = null;
        }
      }
     
      if (!sr.more) {
        // log.debug("No more : tab end row = "+loc.tablet_extent.getEndRow()+" range = "+scanState.range);
        if (loc.tablet_extent.getEndRow() == null) {
          scanState.finished = true;
          opTimer.stop("Completely finished scan in %DURATION% #results=" + sr.results.size());
        } else if (scanState.range.getEndKey() == null || !scanState.range.afterEndKey(new Key(loc.tablet_extent.getEndRow()).followingKey(PartialKey.ROW))) {
          scanState.startRow = loc.tablet_extent.getEndRow();
          scanState.skipStartRow = true;
          opTimer.stop("Finished scanning tablet in %DURATION% #results=" + sr.results.size());
        } else {
          scanState.finished = true;
          opTimer.stop("Completely finished scan in %DURATION% #results=" + sr.results.size());
        }
      } else {
        opTimer.stop("Finished scan in %DURATION% #results=" + sr.results.size() + " scanid=" + scanState.scanID);
      }
     
      Key.decompress(sr.results);
     
      if (sr.results.size() > 0 && !scanState.finished)
View Full Code Here

 
  @Override
  public void binMutations(List<Mutation> mutations, Map<String,TabletServerMutations> binnedMutations, List<Mutation> failures) throws AccumuloException,
      AccumuloSecurityException, TableNotFoundException {
   
    OpTimer opTimer = null;
    if (log.isTraceEnabled())
      opTimer = new OpTimer(log, Level.TRACE).start("Binning " + mutations.size() + " mutations for table " + tableId);
   
    ArrayList<Mutation> notInCache = new ArrayList<Mutation>();
    Text row = new Text();
   
    rLock.lock();
    try {
      processInvalidated();
     
      // for this to be efficient rows need to be in sorted order, but always sorting is slow... therefore only sort the
      // stuff not in the cache.... it is most efficient to pass _locateTablet rows in sorted order
     
      // For this to be efficient, need to avoid fine grained synchronization and fine grained logging.
      // Therefore methods called by this are not synchronized and should not log.
     
      for (Mutation mutation : mutations) {
        row.set(mutation.getRow());
        TabletLocation tl = locateTabletInCache(row);
        if (tl == null)
          notInCache.add(mutation);
        else
          addMutation(binnedMutations, mutation, tl);
       
      }
    } finally {
      rLock.unlock();
    }
   
    if (notInCache.size() > 0) {
      Collections.sort(notInCache, new Comparator<Mutation>() {
        public int compare(Mutation o1, Mutation o2) {
          return WritableComparator.compareBytes(o1.getRow(), 0, o1.getRow().length, o2.getRow(), 0, o2.getRow().length);
        }
      });
     
      wLock.lock();
      try {
        boolean failed = false;
        for (Mutation mutation : notInCache) {
          if (failed) {
            // when one table does not return a location, something is probably
            // screwy, go ahead and fail everything.
            failures.add(mutation);
            continue;
          }
         
          row.set(mutation.getRow());
         
          TabletLocation tl = _locateTablet(row, false, false, false);
         
          if (tl == null) {
            failures.add(mutation);
            failed = true;
          } else {
            addMutation(binnedMutations, mutation, tl);
          }
        }
      } finally {
        wLock.unlock();
      }
    }
   
    if (opTimer != null)
      opTimer.stop("Binned " + mutations.size() + " mutations for table " + tableId + " to " + binnedMutations.size() + " tservers in %DURATION%");
  }
View Full Code Here

    /*
     * For this to be efficient, need to avoid fine grained synchronization and fine grained logging. Therefore methods called by this are not synchronized and
     * should not log.
     */
   
    OpTimer opTimer = null;
    if (log.isTraceEnabled())
      opTimer = new OpTimer(log, Level.TRACE).start("Binning " + ranges.size() + " ranges for table " + tableId);
   
    List<Range> failures;
    rLock.lock();
    try {
      processInvalidated();
     
      // for this to be optimal, need to look ranges up in sorted order when
      // ranges are not present in cache... however do not want to always
      // sort ranges... therefore try binning ranges using only the cache
      // and sort whatever fails and retry
     
      failures = binRanges(ranges, binnedRanges, true);
    } finally {
      rLock.unlock();
    }
   
    if (failures.size() > 0) {
      // sort failures by range start key
      Collections.sort(failures);
     
      // try lookups again
      wLock.lock();
      try {
        failures = binRanges(failures, binnedRanges, false);
      } finally {
        wLock.unlock();
      }
    }
   
    if (opTimer != null)
      opTimer.stop("Binned " + ranges.size() + " ranges for table " + tableId + " to " + binnedRanges.size() + " tservers in %DURATION%");
   
    return failures;
  }
View Full Code Here

  }
 
  @Override
  public TabletLocation locateTablet(Text row, boolean skipRow, boolean retry) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
   
    OpTimer opTimer = null;
    if (log.isTraceEnabled())
      opTimer = new OpTimer(log, Level.TRACE).start("Locating tablet  table=" + tableId + " row=" + TextUtil.truncate(row) + "  skipRow=" + skipRow + " retry="
          + retry);
   
    while (true) {
     
      TabletLocation tl;
     
      tl = _locateTablet(row, skipRow, retry, true);
     
      if (retry && tl == null) {
        UtilWaitThread.sleep(100);
        if (log.isTraceEnabled())
          log.trace("Failed to locate tablet containing row " + TextUtil.truncate(row) + " in table " + tableId + ", will retry...");
        continue;
      }
     
      if (opTimer != null)
        opTimer.stop("Located tablet " + (tl == null ? null : tl.tablet_extent) + " at " + (tl == null ? null : tl.tablet_location) + " in %DURATION%");
     
      return tl;
    }
  }
View Full Code Here

   
    TTransport transport = null;
    try {
      TabletClientService.Iface client = ThriftUtil.getTServerClient(server, conf);
      try {
        OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Starting multi scan, tserver=" + server + "  #tablets=" + requested.size() + "  #ranges="
            + sumSizes(requested.values()) + " ssil=" + options.serverSideIteratorList + " ssio=" + options.serverSideIteratorOptions);
       
        TabletType ttype = TabletType.type(requested.keySet());
        boolean waitForWrites = !ThriftScanner.serversWaitedForWrites.get(ttype).contains(server);
       
        Map<TKeyExtent,List<TRange>> thriftTabletRanges = Translator.translate(requested, Translator.KET, new Translator.ListTranslator<Range,TRange>(
            Translator.RT));
        InitialMultiScan imsr = client.startMultiScan(null, credentials, thriftTabletRanges, Translator.translate(columns, Translator.CT),
            options.serverSideIteratorList, options.serverSideIteratorOptions, ByteBufferUtil.toByteBuffers(authorizations.getAuthorizations()), waitForWrites);
        if (waitForWrites)
          ThriftScanner.serversWaitedForWrites.get(ttype).add(server);
       
        MultiScanResult scanResult = imsr.result;
       
        opTimer.stop("Got 1st multi scan results, #results=" + scanResult.results.size() + (scanResult.more ? "  scanID=" + imsr.scanID : "")
            + " in %DURATION%");
       
        for (TKeyValue kv : scanResult.results) {
          receiver.receive(new Key(kv.key), new Value(kv.value));
        }
        trackScanning(failures, unscanned, scanResult);
       
        while (scanResult.more) {
         
          opTimer.start("Continuing multi scan, scanid=" + imsr.scanID);
          scanResult = client.continueMultiScan(null, imsr.scanID);
          opTimer.stop("Got more multi scan results, #results=" + scanResult.results.size() + (scanResult.more ? "  scanID=" + imsr.scanID : "")
              + " in %DURATION%");
          for (TKeyValue kv : scanResult.results) {
            receiver.receive(new Key(kv.key), new Value(kv.value));
          }
          trackScanning(failures, unscanned, scanResult);
View Full Code Here

      else
        client = ThriftUtil.getTServerClient(server, conf);
     
      try {
       
        OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Starting multi scan, tserver=" + server + "  #tablets=" + requested.size() + "  #ranges="
            + sumSizes(requested.values()) + " ssil=" + options.serverSideIteratorList + " ssio=" + options.serverSideIteratorOptions);
       
        TabletType ttype = TabletType.type(requested.keySet());
        boolean waitForWrites = !ThriftScanner.serversWaitedForWrites.get(ttype).contains(server);
       
        Map<TKeyExtent,List<TRange>> thriftTabletRanges = Translator.translate(requested, Translator.KET, new Translator.ListTranslator<Range,TRange>(
            Translator.RT));
        InitialMultiScan imsr = client.startMultiScan(Tracer.traceInfo(), credentials, thriftTabletRanges, Translator.translate(columns, Translator.CT),
            options.serverSideIteratorList, options.serverSideIteratorOptions, ByteBufferUtil.toByteBuffers(authorizations.getAuthorizations()), waitForWrites);
        if (waitForWrites)
          ThriftScanner.serversWaitedForWrites.get(ttype).add(server);
       
        MultiScanResult scanResult = imsr.result;
       
        opTimer.stop("Got 1st multi scan results, #results=" + scanResult.results.size() + (scanResult.more ? "  scanID=" + imsr.scanID : "")
            + " in %DURATION%");
       
        ArrayList<Entry<Key,Value>> entries = new ArrayList<Map.Entry<Key,Value>>(scanResult.results.size());
        for (TKeyValue kv : scanResult.results) {
          entries.add(new MyEntry(new Key(kv.key), new Value(kv.value)));
        }
       
        if (entries.size() > 0)
          receiver.receive(entries);
       
        if (entries.size() > 0 || scanResult.fullScans.size() > 0)
          timeoutTracker.madeProgress();
       
        trackScanning(failures, unscanned, scanResult);
       
        while (scanResult.more) {
         
          timeoutTracker.check();
         
          opTimer.start("Continuing multi scan, scanid=" + imsr.scanID);
          scanResult = client.continueMultiScan(Tracer.traceInfo(), imsr.scanID);
          opTimer.stop("Got more multi scan results, #results=" + scanResult.results.size() + (scanResult.more ? "  scanID=" + imsr.scanID : "")
              + " in %DURATION%");
         
          entries = new ArrayList<Map.Entry<Key,Value>>(scanResult.results.size());
          for (TKeyValue kv : scanResult.results) {
            entries.add(new MyEntry(new Key(kv.key), new Value(kv.value)));
View Full Code Here

  private static List<KeyValue> scan(TabletLocation loc, ScanState scanState, AccumuloConfiguration conf) throws AccumuloSecurityException,
      NotServingTabletException, TException, NoSuchScanIDException, TooManyFilesException {
    if (scanState.finished)
      return null;
   
    OpTimer opTimer = new OpTimer(log, Level.TRACE);
   
    TInfo tinfo = Tracer.traceInfo();
    TabletClientService.Client client = ThriftUtil.getTServerClient(loc.tablet_location, conf);
   
    String old = Thread.currentThread().getName();
    try {
      ScanResult sr;
     
      if (scanState.prevLoc != null && !scanState.prevLoc.equals(loc))
        scanState.scanID = null;
     
      scanState.prevLoc = loc;
     
      if (scanState.scanID == null) {
        String msg = "Starting scan tserver=" + loc.tablet_location + " tablet=" + loc.tablet_extent + " range=" + scanState.range + " ssil="
            + scanState.serverSideIteratorList + " ssio=" + scanState.serverSideIteratorOptions;
        Thread.currentThread().setName(msg);
        opTimer.start(msg);
       
        TabletType ttype = TabletType.type(loc.tablet_extent);
        boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(loc.tablet_location);
        InitialScan is = client.startScan(tinfo, scanState.credentials, loc.tablet_extent.toThrift(), scanState.range.toThrift(),
            Translator.translate(scanState.columns, Translator.CT), scanState.size, scanState.serverSideIteratorList, scanState.serverSideIteratorOptions,
            scanState.authorizations.getAuthorizationsBB(), waitForWrites, scanState.isolated);
        if (waitForWrites)
          serversWaitedForWrites.get(ttype).add(loc.tablet_location);
       
        sr = is.result;
       
        if (sr.more)
          scanState.scanID = is.scanID;
        else
          client.closeScan(tinfo, is.scanID);
       
      } else {
        // log.debug("Calling continue scan : "+scanState.range+"  loc = "+loc);
        String msg = "Continuing scan tserver=" + loc.tablet_location + " scanid=" + scanState.scanID;
        Thread.currentThread().setName(msg);
        opTimer.start(msg);
       
        sr = client.continueScan(tinfo, scanState.scanID);
        if (!sr.more) {
          client.closeScan(tinfo, scanState.scanID);
          scanState.scanID = null;
        }
      }
     
      if (!sr.more) {
        // log.debug("No more : tab end row = "+loc.tablet_extent.getEndRow()+" range = "+scanState.range);
        if (loc.tablet_extent.getEndRow() == null) {
          scanState.finished = true;
          opTimer.stop("Completely finished scan in %DURATION% #results=" + sr.results.size());
        } else if (scanState.range.getEndKey() == null || !scanState.range.afterEndKey(new Key(loc.tablet_extent.getEndRow()).followingKey(PartialKey.ROW))) {
          scanState.startRow = loc.tablet_extent.getEndRow();
          scanState.skipStartRow = true;
          opTimer.stop("Finished scanning tablet in %DURATION% #results=" + sr.results.size());
        } else {
          scanState.finished = true;
          opTimer.stop("Completely finished scan in %DURATION% #results=" + sr.results.size());
        }
      } else {
        opTimer.stop("Finished scan in %DURATION% #results=" + sr.results.size() + " scanid=" + scanState.scanID);
      }
     
      Key.decompress(sr.results);
     
      if (sr.results.size() > 0 && !scanState.finished)
View Full Code Here

      throws AccumuloSecurityException, AccumuloException {
   
    try {
      ArrayList<TabletLocation> list = new ArrayList<TabletLocation>();
     
      OpTimer opTimer = null;
      if (log.isTraceEnabled())
        opTimer = new OpTimer(log, Level.TRACE).start("Looking up in " + src.tablet_extent.getTableId() + " row=" + TextUtil.truncate(row) + "  extent="
            + src.tablet_extent + " tserver=" + src.tablet_location);
     
      Range range = new Range(row, true, stopRow, true);
     
      TreeMap<Key,Value> encodedResults = new TreeMap<Key,Value>();
      TreeMap<Key,Value> results = new TreeMap<Key,Value>();
     
      // Use the whole row iterator so that a partial mutations is not read. The code that extracts locations for tablets does a sanity check to ensure there is
      // only one location. Reading a partial mutation could make it appear there are multiple locations when there are not.
      List<IterInfo> serverSideIteratorList = new ArrayList<IterInfo>();
      serverSideIteratorList.add(new IterInfo(10000, WholeRowIterator.class.getName(), "WRI"));
      Map<String,Map<String,String>> serverSideIteratorOptions = Collections.emptyMap();
     
      boolean more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location, encodedResults, locCols,
          serverSideIteratorList, serverSideIteratorOptions, Constants.SCAN_BATCH_SIZE, Constants.NO_AUTHS, false, instance.getConfiguration());
     
      decodeRows(encodedResults, results);
     
      if (more && results.size() == 1) {
        range = new Range(results.lastKey().followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME), true, new Key(stopRow).followingKey(PartialKey.ROW), false);
        encodedResults.clear();
        more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location, encodedResults, locCols, serverSideIteratorList,
            serverSideIteratorOptions, Constants.SCAN_BATCH_SIZE, Constants.NO_AUTHS, false, instance.getConfiguration());
       
        decodeRows(encodedResults, results);
      }
     
      if (opTimer != null)
        opTimer.stop("Got " + results.size() + " results  from " + src.tablet_extent + " in %DURATION%");
     
      // System.out.println("results "+results.keySet());
     
      Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>> metadata = MetadataTable.getMetadataLocationEntries(results);
     
View Full Code Here

 
  @Override
  public void binMutations(List<Mutation> mutations, Map<String,TabletServerMutations> binnedMutations, List<Mutation> failures, TCredentials credentials) throws AccumuloException,
      AccumuloSecurityException, TableNotFoundException {
   
    OpTimer opTimer = null;
    if (log.isTraceEnabled())
      opTimer = new OpTimer(log, Level.TRACE).start("Binning " + mutations.size() + " mutations for table " + tableId);
   
    ArrayList<Mutation> notInCache = new ArrayList<Mutation>();
    Text row = new Text();
   
    rLock.lock();
    try {
      processInvalidated(credentials);
     
      // for this to be efficient rows need to be in sorted order, but always sorting is slow... therefore only sort the
      // stuff not in the cache.... it is most efficient to pass _locateTablet rows in sorted order
     
      // For this to be efficient, need to avoid fine grained synchronization and fine grained logging.
      // Therefore methods called by this are not synchronized and should not log.
     
      for (Mutation mutation : mutations) {
        row.set(mutation.getRow());
        TabletLocation tl = locateTabletInCache(row);
        if (tl == null)
          notInCache.add(mutation);
        else
          addMutation(binnedMutations, mutation, tl);
       
      }
    } finally {
      rLock.unlock();
    }
   
    if (notInCache.size() > 0) {
      Collections.sort(notInCache, new Comparator<Mutation>() {
        public int compare(Mutation o1, Mutation o2) {
          return WritableComparator.compareBytes(o1.getRow(), 0, o1.getRow().length, o2.getRow(), 0, o2.getRow().length);
        }
      });
     
      wLock.lock();
      try {
        boolean failed = false;
        for (Mutation mutation : notInCache) {
          if (failed) {
            // when one table does not return a location, something is probably
            // screwy, go ahead and fail everything.
            failures.add(mutation);
            continue;
          }
         
          row.set(mutation.getRow());
         
          TabletLocation tl = _locateTablet(row, false, false, false, credentials);
         
          if (tl == null) {
            failures.add(mutation);
            failed = true;
          } else {
            addMutation(binnedMutations, mutation, tl);
          }
        }
      } finally {
        wLock.unlock();
      }
    }
   
    if (opTimer != null)
      opTimer.stop("Binned " + mutations.size() + " mutations for table " + tableId + " to " + binnedMutations.size() + " tservers in %DURATION%");
  }
View Full Code Here

TOP

Related Classes of org.apache.accumulo.core.util.OpTimer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.