Package org.apache.hadoop.hbase.io.hfile.bucket

Examples of org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread


      long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
          * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);

      boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
          DEFAULT_BUCKET_CACHE_COMBINED);
      BucketCache bucketCache = null;
      if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
        int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_THREADS);
        int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
        String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
        float combinedPercentage = conf.getFloat(
            BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
            DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
        if (combinedWithLru) {
          lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
          bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
        }
        try {
          int ioErrorsTolerationDuration = conf.getInt(
              "hbase.bucketcache.ioengine.errors.tolerated.duration",
              BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
          bucketCache = new BucketCache(bucketCacheIOEngineName,
              bucketCacheSize, blockSize, writerThreads, writerQueueLen, persistentPath,
              ioErrorsTolerationDuration);
        } catch (IOException ioex) {
          LOG.error("Can't instantiate bucket cache", ioex);
          throw new RuntimeException(ioex);
View Full Code Here


      long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
          * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);

      boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
          DEFAULT_BUCKET_CACHE_COMBINED);
      BucketCache bucketCache = null;
      if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
        int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_THREADS);
        int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
        String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
        float combinedPercentage = conf.getFloat(
            BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
            DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
        String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
        int[] bucketSizes = null;
        if (configuredBucketSizes != null) {
          bucketSizes = new int[configuredBucketSizes.length];
          for (int i = 0; i < configuredBucketSizes.length; i++) {
            bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i]);
          }
        }
        if (combinedWithLru) {
          lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
          bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
        }
        try {
          int ioErrorsTolerationDuration = conf.getInt(
              "hbase.bucketcache.ioengine.errors.tolerated.duration",
              BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
          bucketCache = new BucketCache(bucketCacheIOEngineName,
              bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
              ioErrorsTolerationDuration);
        } catch (IOException ioex) {
          LOG.error("Can't instantiate bucket cache", ioex);
          throw new RuntimeException(ioex);
View Full Code Here

    assertTrue(bcs[0] instanceof LruBlockCache);
    LruBlockCache lbc = (LruBlockCache)bcs[0];
    assertEquals(CacheConfig.getLruCacheSize(this.conf,
        ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()), lbc.getMaxSize());
    assertTrue(bcs[1] instanceof BucketCache);
    BucketCache bc = (BucketCache)bcs[1];
    // getMaxSize comes back in bytes but we specified size in MB
    assertEquals(bcSize, bc.getMaxSize() / (1024 * 1024));
  }
View Full Code Here

    basicBlockCacheOps(cc, false, false);
    assertTrue(cc.getBlockCache() instanceof LruBlockCache);
    // TODO: Assert sizes allocated are right and proportions.
    LruBlockCache lbc = (LruBlockCache)cc.getBlockCache();
    assertEquals(lruExpectedSize, lbc.getMaxSize());
    BucketCache bc = lbc.getVictimHandler();
    // getMaxSize comes back in bytes but we specified size in MB
    assertEquals(bcExpectedSize, bc.getMaxSize());
    // Test the L1+L2 deploy works as we'd expect with blocks evicted from L1 going to L2.
    long initialL1BlockCount = lbc.getBlockCount();
    long initialL2BlockCount = bc.getBlockCount();
    Cacheable c = new DataCacheEntry();
    BlockCacheKey bck = new BlockCacheKey("bck", 0);
    lbc.cacheBlock(bck, c, false, false);
    assertEquals(initialL1BlockCount + 1, lbc.getBlockCount());
    assertEquals(initialL2BlockCount, bc.getBlockCount());
    // Force evictions by putting in a block too big.
    final long justTooBigSize = lbc.acceptableSize() + 1;
    lbc.cacheBlock(new BlockCacheKey("bck2", 0), new DataCacheEntry() {
      @Override
      public long heapSize() {
        return justTooBigSize;
      }

      @Override
      public int getSerializedLength() {
        return (int)heapSize();
      }
    });
    // The eviction thread in lrublockcache needs to run.
    while (initialL1BlockCount != lbc.getBlockCount()) Threads.sleep(10);
    assertEquals(initialL1BlockCount, lbc.getBlockCount());
    long count = bc.getBlockCount();
    assertTrue(initialL2BlockCount + 1 <= count);
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.