Package org.apache.accumulo.core.util

Examples of org.apache.accumulo.core.util.OpTimer

66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
        public void run() {
          NativeMap nm = new NativeMap();
         
          Random r = new Random();
         
          OpTimer opTimer = new OpTimer(log, Level.INFO);
         
          opTimer.start("Creating map of size " + mapSizePerThread);
         
          for (int i = 0; i < mapSizePerThread; i++) {
            String row = String.format("r%08d", i);
            String val = row + "v";
            put(nm, row, val, i);
          }
         
          opTimer.stop("Created map of size " + nm.size() + " in %DURATION%");
         
          opTimer.start("Doing " + getsPerThread + " gets()");
         
          for (int i = 0; i < getsPerThread; i++) {
            String row = String.format("r%08d", r.nextInt(mapSizePerThread));
            String val = row + "v";
           
            Value value = nm.get(new Key(new Text(row)));
            if (value == null || !value.toString().equals(val)) {
              log.error("nm.get(" + row + ") failed");
            }
          }
         
          opTimer.stop("Finished " + getsPerThread + " gets in %DURATION%");
         
          int scanned = 0;
         
          opTimer.start("Doing " + getsPerThread + " random iterations");
         
          for (int i = 0; i < getsPerThread; i++) {
            int startRow = r.nextInt(mapSizePerThread);
            String row = String.format("r%08d", startRow);
           
            Iterator<Entry<Key,Value>> iter = nm.iterator(new Key(new Text(row)));
           
            int count = 0;
           
            while (iter.hasNext() && count < 10) {
              String row2 = String.format("r%08d", startRow + count);
              String val2 = row2 + "v";
             
              Entry<Key,Value> entry = iter.next();
              if (!entry.getValue().toString().equals(val2) || !entry.getKey().equals(new Key(new Text(row2)))) {
                log.error("nm.iter(" + row2 + ") failed row = " + row + " count = " + count + " row2 = " + row + " val2 = " + val2);
              }
             
              count++;
            }
           
            scanned += count;
          }
         
          opTimer.stop("Finished " + getsPerThread + " random iterations (scanned = " + scanned + ") in %DURATION%");
         
          nm.delete();
        }
      };
     
View Full Code Here
67
68
69
70
71
72
73
74
75
76
    this.tableOps = tableOps;
  }

  @Override
  public SortedSet<String> list() {
    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Fetching list of namespaces...");
    TreeSet<String> namespaces = new TreeSet<String>(Namespaces.getNameToIdMap(instance).keySet());
    opTimer.stop("Fetched " + namespaces.size() + " namespaces in %DURATION%");
    return namespaces;
  }
View Full Code Here
77
78
79
80
81
82
83
84
85
86
  @Override
  public boolean exists(String namespace) {
    ArgumentChecker.notNull(namespace);

    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Checking if namespace " + namespace + " exists...");
    boolean exists = Namespaces.getNameToIdMap(instance).containsKey(namespace);
    opTimer.stop("Checked existance of " + exists + " in %DURATION%");
    return exists;
  }
View Full Code Here
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
      else
        client = ThriftUtil.getTServerClient(server, conf);
     
      try {
       
        OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Starting multi scan, tserver=" + server + "  #tablets=" + requested.size() + "  #ranges="
            + sumSizes(requested.values()) + " ssil=" + options.serverSideIteratorList + " ssio=" + options.serverSideIteratorOptions);
       
        TabletType ttype = TabletType.type(requested.keySet());
        boolean waitForWrites = !ThriftScanner.serversWaitedForWrites.get(ttype).contains(server);
       
        Map<TKeyExtent,List<TRange>> thriftTabletRanges = Translator.translate(requested, Translators.KET, new Translator.ListTranslator<Range,TRange>(
            Translators.RT));
        InitialMultiScan imsr = client.startMultiScan(Tracer.traceInfo(), credentials.toThrift(instance), thriftTabletRanges,
            Translator.translate(columns, Translators.CT), options.serverSideIteratorList, options.serverSideIteratorOptions,
            ByteBufferUtil.toByteBuffers(authorizations.getAuthorizations()), waitForWrites);
        if (waitForWrites)
          ThriftScanner.serversWaitedForWrites.get(ttype).add(server);
       
        MultiScanResult scanResult = imsr.result;
       
        opTimer.stop("Got 1st multi scan results, #results=" + scanResult.results.size() + (scanResult.more ? "  scanID=" + imsr.scanID : "")
            + " in %DURATION%");
       
        ArrayList<Entry<Key,Value>> entries = new ArrayList<Map.Entry<Key,Value>>(scanResult.results.size());
        for (TKeyValue kv : scanResult.results) {
          entries.add(new MyEntry(new Key(kv.key), new Value(kv.value)));
        }
       
        if (entries.size() > 0)
          receiver.receive(entries);
       
        if (entries.size() > 0 || scanResult.fullScans.size() > 0)
          timeoutTracker.madeProgress();
       
        trackScanning(failures, unscanned, scanResult);
       
        while (scanResult.more) {
         
          timeoutTracker.check();
         
          opTimer.start("Continuing multi scan, scanid=" + imsr.scanID);
          scanResult = client.continueMultiScan(Tracer.traceInfo(), imsr.scanID);
          opTimer.stop("Got more multi scan results, #results=" + scanResult.results.size() + (scanResult.more ? "  scanID=" + imsr.scanID : "")
              + " in %DURATION%");
         
          entries = new ArrayList<Map.Entry<Key,Value>>(scanResult.results.size());
          for (TKeyValue kv : scanResult.results) {
            entries.add(new MyEntry(new Key(kv.key), new Value(kv.value)));
View Full Code Here
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
  private static List<KeyValue> scan(TabletLocation loc, ScanState scanState, AccumuloConfiguration conf) throws AccumuloSecurityException,
      NotServingTabletException, TException, NoSuchScanIDException, TooManyFilesException {
    if (scanState.finished)
      return null;
   
    OpTimer opTimer = new OpTimer(log, Level.TRACE);
   
    TInfo tinfo = Tracer.traceInfo();
    TabletClientService.Client client = ThriftUtil.getTServerClient(loc.tablet_location, conf);
   
    String old = Thread.currentThread().getName();
    try {
      ScanResult sr;
     
      if (scanState.prevLoc != null && !scanState.prevLoc.equals(loc))
        scanState.scanID = null;
     
      scanState.prevLoc = loc;
     
      if (scanState.scanID == null) {
        String msg = "Starting scan tserver=" + loc.tablet_location + " tablet=" + loc.tablet_extent + " range=" + scanState.range + " ssil="
            + scanState.serverSideIteratorList + " ssio=" + scanState.serverSideIteratorOptions;
        Thread.currentThread().setName(msg);
        opTimer.start(msg);
       
        TabletType ttype = TabletType.type(loc.tablet_extent);
        boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(loc.tablet_location);
        InitialScan is = client.startScan(tinfo, scanState.credentials.toThrift(scanState.instance), loc.tablet_extent.toThrift(), scanState.range.toThrift(),
            Translator.translate(scanState.columns, Translators.CT), scanState.size, scanState.serverSideIteratorList, scanState.serverSideIteratorOptions,
            scanState.authorizations.getAuthorizationsBB(), waitForWrites, scanState.isolated, scanState.readaheadThreshold);
        if (waitForWrites)
          serversWaitedForWrites.get(ttype).add(loc.tablet_location);
       
        sr = is.result;
       
        if (sr.more)
          scanState.scanID = is.scanID;
        else
          client.closeScan(tinfo, is.scanID);
       
      } else {
        // log.debug("Calling continue scan : "+scanState.range+"  loc = "+loc);
        String msg = "Continuing scan tserver=" + loc.tablet_location + " scanid=" + scanState.scanID;
        Thread.currentThread().setName(msg);
        opTimer.start(msg);
       
        sr = client.continueScan(tinfo, scanState.scanID);
        if (!sr.more) {
          client.closeScan(tinfo, scanState.scanID);
          scanState.scanID = null;
        }
      }
     
      if (!sr.more) {
        // log.debug("No more : tab end row = "+loc.tablet_extent.getEndRow()+" range = "+scanState.range);
        if (loc.tablet_extent.getEndRow() == null) {
          scanState.finished = true;
          opTimer.stop("Completely finished scan in %DURATION% #results=" + sr.results.size());
        } else if (scanState.range.getEndKey() == null || !scanState.range.afterEndKey(new Key(loc.tablet_extent.getEndRow()).followingKey(PartialKey.ROW))) {
          scanState.startRow = loc.tablet_extent.getEndRow();
          scanState.skipStartRow = true;
          opTimer.stop("Finished scanning tablet in %DURATION% #results=" + sr.results.size());
        } else {
          scanState.finished = true;
          opTimer.stop("Completely finished scan in %DURATION% #results=" + sr.results.size());
        }
      } else {
        opTimer.stop("Finished scan in %DURATION% #results=" + sr.results.size() + " scanid=" + scanState.scanID);
      }
     
      Key.decompress(sr.results);
     
      if (sr.results.size() > 0 && !scanState.finished)
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
 
  @Override
  public <T extends Mutation> void binMutations(Credentials credentials, List<T> mutations, Map<String,TabletServerMutations<T>> binnedMutations, List<T> failures)
      throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
   
    OpTimer opTimer = null;
    if (log.isTraceEnabled())
      opTimer = new OpTimer(log, Level.TRACE).start("Binning " + mutations.size() + " mutations for table " + tableId);
   
    ArrayList<T> notInCache = new ArrayList<T>();
    Text row = new Text();
   
    LockCheckerSession lcSession = new LockCheckerSession();

    rLock.lock();
    try {
      processInvalidated(credentials, lcSession);
     
      // for this to be efficient rows need to be in sorted order, but always sorting is slow... therefore only sort the
      // stuff not in the cache.... it is most efficient to pass _locateTablet rows in sorted order
     
      // For this to be efficient, need to avoid fine grained synchronization and fine grained logging.
      // Therefore methods called by this are not synchronized and should not log.
     
      for (T mutation : mutations) {
        row.set(mutation.getRow());
        TabletLocation tl = locateTabletInCache(row);
        if (tl == null || !addMutation(binnedMutations, mutation, tl, lcSession))
          notInCache.add(mutation);
      }
    } finally {
      rLock.unlock();
    }
   
    if (notInCache.size() > 0) {
      Collections.sort(notInCache, new Comparator<Mutation>() {
        @Override
        public int compare(Mutation o1, Mutation o2) {
          return WritableComparator.compareBytes(o1.getRow(), 0, o1.getRow().length, o2.getRow(), 0, o2.getRow().length);
        }
      });
     
      wLock.lock();
      try {
        boolean failed = false;
        for (T mutation : notInCache) {
          if (failed) {
            // when one table does not return a location, something is probably
            // screwy, go ahead and fail everything.
            failures.add(mutation);
            continue;
          }
         
          row.set(mutation.getRow());
         
          TabletLocation tl = _locateTablet(credentials, row, false, false, false, lcSession);
         
          if (tl == null || !addMutation(binnedMutations, mutation, tl, lcSession)) {
            failures.add(mutation);
            failed = true;
          }
        }
      } finally {
        wLock.unlock();
      }
    }

    if (opTimer != null)
      opTimer.stop("Binned " + mutations.size() + " mutations for table " + tableId + " to " + binnedMutations.size() + " tservers in %DURATION%");
  }
View Full Code Here
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
    /*
     * For this to be efficient, need to avoid fine grained synchronization and fine grained logging. Therefore methods called by this are not synchronized and
     * should not log.
     */
   
    OpTimer opTimer = null;
    if (log.isTraceEnabled())
      opTimer = new OpTimer(log, Level.TRACE).start("Binning " + ranges.size() + " ranges for table " + tableId);
   
    LockCheckerSession lcSession = new LockCheckerSession();

    List<Range> failures;
    rLock.lock();
    try {
      processInvalidated(credentials, lcSession);
     
      // for this to be optimal, need to look ranges up in sorted order when
      // ranges are not present in cache... however do not want to always
      // sort ranges... therefore try binning ranges using only the cache
      // and sort whatever fails and retry
     
      failures = binRanges(credentials, ranges, binnedRanges, true, lcSession);
    } finally {
      rLock.unlock();
    }
   
    if (failures.size() > 0) {
      // sort failures by range start key
      Collections.sort(failures);
     
      // try lookups again
      wLock.lock();
      try {
        failures = binRanges(credentials, failures, binnedRanges, false, lcSession);
      } finally {
        wLock.unlock();
      }
    }
   
    if (opTimer != null)
      opTimer.stop("Binned " + ranges.size() + " ranges for table " + tableId + " to " + binnedRanges.size() + " tservers in %DURATION%");

    return failures;
  }
View Full Code Here
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
 
  @Override
  public TabletLocation locateTablet(Credentials credentials, Text row, boolean skipRow, boolean retry) throws AccumuloException, AccumuloSecurityException,
      TableNotFoundException {
   
    OpTimer opTimer = null;
    if (log.isTraceEnabled())
      opTimer = new OpTimer(log, Level.TRACE).start("Locating tablet  table=" + tableId + " row=" + TextUtil.truncate(row) + "  skipRow=" + skipRow + " retry="
          + retry);
   
    while (true) {
     
      LockCheckerSession lcSession = new LockCheckerSession();
      TabletLocation tl = _locateTablet(credentials, row, skipRow, retry, true, lcSession);

      if (retry && tl == null) {
        UtilWaitThread.sleep(100);
        if (log.isTraceEnabled())
          log.trace("Failed to locate tablet containing row " + TextUtil.truncate(row) + " in table " + tableId + ", will retry...");
        continue;
      }
     
      if (opTimer != null)
        opTimer.stop("Located tablet " + (tl == null ? null : tl.tablet_extent) + " at " + (tl == null ? null : tl.tablet_location) + " in %DURATION%");
     
      return tl;
    }
  }
View Full Code Here
115
116
117
118
119
120
121
122
123
124
   * Retrieve a list of tables in Accumulo.
   *
   * @return List of tables in accumulo
   */
  public SortedSet<String> list() {
    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Fetching list of tables...");
    TreeSet<String> tableNames = new TreeSet<String>(Tables.getNameToIdMap(instance).keySet());
    opTimer.stop("Fetched " + tableNames.size() + " table names in %DURATION%");
    return tableNames;
  }
View Full Code Here
133
134
135
136
137
138
139
140
141
142
  public boolean exists(String tableName) {
    ArgumentChecker.notNull(tableName);
    if (tableName.equals(Constants.METADATA_TABLE_NAME))
      return true;
   
    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Checking if table " + tableName + "exists...");
    boolean exists = Tables.getNameToIdMap(instance).containsKey(tableName);
    opTimer.stop("Checked existance of " + exists + " in %DURATION%");
    return exists;
  }
View Full Code Here
TOP

Related Classes of org.apache.accumulo.core.util.OpTimer

  • org.apache.accumulo.core.client.admin.TableOperationsImpl
  • org.apache.accumulo.core.client.impl.ConnectorImpl
  • org.apache.accumulo.core.client.impl.HdfsZooInstance
  • org.apache.accumulo.core.client.impl.MetadataLocationObtainer
  • org.apache.accumulo.core.client.impl.NamespaceOperationsImpl
  • org.apache.accumulo.core.client.impl.RootTabletLocator
  • org.apache.accumulo.core.client.impl.TableOperationsImpl
  • org.apache.accumulo.core.client.impl.TabletLocatorImpl
  • org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator
  • org.apache.accumulo.core.client.impl.ThriftScanner
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.