Package org.apache.hadoop.hbase.coprocessor

Examples of org.apache.hadoop.hbase.coprocessor.SecureBulkLoadClient


          } else {
            hasForwardedToken = true;
            LOG.info("Use the existing token: " + userToken);
          }
        }
        bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getTableName());
      }

      // Assumes that region splits can happen while this occurs.
      while (!queue.isEmpty()) {
        // need to reload split keys each iteration.
        final Pair<byte[][], byte[][]> startEndKeys = table.getStartEndKeys();
        if (count != 0) {
          LOG.info("Split occured while grouping HFiles, retry attempt " +
              + count + " with " + queue.size() + " files remaining to group or split");
        }

        int maxRetries = cfg.getInt("hbase.bulkload.retries.number", 0);
        if (maxRetries != 0 && count >= maxRetries) {
          LOG.error("Retry attempted " + count +  " times without completing, bailing out");
          return;
        }
        count++;

        // Using ByteBuffer for byte[] equality semantics
        Multimap<ByteBuffer, LoadQueueItem> regionGroups = groupOrSplitPhase(table,
            pool, queue, startEndKeys);

        bulkLoadPhase(table, conn, pool, queue, regionGroups);

        // NOTE: The next iteration's split / group could happen in parallel to
        // atomic bulkloads assuming that there are splits and no merges, and
        // that we can atomically pull out the groups we want to retry.
      }

    } finally {
      if(useSecure) {
        if(userToken != null && !hasForwardedToken) {
          try {
            userToken.cancel(cfg);
          } catch (Exception e) {
            LOG.warn("Failed to cancel HDFS delegation token.", e);
          }
        }
        if(bulkToken != null) {
          new SecureBulkLoadClient(table).cleanupBulkLoad(bulkToken);
        }
      }
      pool.shutdown();
      if (queue != null && !queue.isEmpty()) {
        StringBuilder err = new StringBuilder();
View Full Code Here


    final ServerCallable<Boolean> svrCallable = new ServerCallable<Boolean>(conn,
        tableName, first) {
      @Override
      public Boolean call() throws Exception {
        SecureBulkLoadClient secureClient = null;
        boolean success = false;

        try {
          LOG.debug("Going to connect to server " + location + " for row "
              + Bytes.toStringBinary(row));
          byte[] regionName = location.getRegionInfo().getRegionName();
          if(!useSecure) {
             success = server.bulkLoadHFiles(famPaths, regionName, assignSeqIds);
          } else {
            HTable table = new HTable(conn.getConfiguration(), tableName);
            secureClient = new SecureBulkLoadClient(table, location.getRegionInfo().getStartKey());
            success = secureClient.bulkLoadHFiles(famPaths, userToken, bulkToken, assignSeqIds);
          }
          return success;
        } finally {
          //Best effort copying of files that might not have been imported
          //from the staging directory back to original location
          //in user directory
          if(secureClient != null && !success) {
            FileSystem fs = FileSystem.get(cfg);
            for(Pair<byte[], String> el : famPaths) {
              Path hfileStagingPath = null;
              Path hfileOrigPath = new Path(el.getSecond());
              try {
                hfileStagingPath= new Path(secureClient.getStagingPath(bulkToken, el.getFirst()),
                    hfileOrigPath.getName());
                if(fs.rename(hfileStagingPath, hfileOrigPath)) {
                  LOG.debug("Moved back file " + hfileOrigPath + " from " +
                      hfileStagingPath);
                } else if(fs.exists(hfileStagingPath)){
View Full Code Here

        //Since delegation token doesn't work in mini cluster
        if (userProvider.isHadoopSecurityEnabled()) {
          FileSystem fs = FileSystem.get(cfg);
          userToken = fs.getDelegationToken("renewer");
        }
        bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getTableName());
      }

      // Assumes that region splits can happen while this occurs.
      while (!queue.isEmpty()) {
        // need to reload split keys each iteration.
        final Pair<byte[][], byte[][]> startEndKeys = table.getStartEndKeys();
        if (count != 0) {
          LOG.info("Split occured while grouping HFiles, retry attempt " +
              + count + " with " + queue.size() + " files remaining to group or split");
        }

        int maxRetries = cfg.getInt("hbase.bulkload.retries.number", 0);
        if (maxRetries != 0 && count >= maxRetries) {
          LOG.error("Retry attempted " + count +  " times without completing, bailing out");
          return;
        }
        count++;

        // Using ByteBuffer for byte[] equality semantics
        Multimap<ByteBuffer, LoadQueueItem> regionGroups = groupOrSplitPhase(table,
            pool, queue, startEndKeys);

        bulkLoadPhase(table, conn, pool, queue, regionGroups);

        // NOTE: The next iteration's split / group could happen in parallel to
        // atomic bulkloads assuming that there are splits and no merges, and
        // that we can atomically pull out the groups we want to retry.
      }

    } finally {
      if(useSecure) {
        if(userToken != null) {
          try {
            userToken.cancel(cfg);
          } catch (Exception e) {
            LOG.warn("Failed to cancel HDFS delegation token.", e);
          }
        }
        if(bulkToken != null) {
          new SecureBulkLoadClient(table).cleanupBulkLoad(bulkToken);
        }
      }
      pool.shutdown();
      if (queue != null && !queue.isEmpty()) {
        StringBuilder err = new StringBuilder();
View Full Code Here

    final ServerCallable<Boolean> svrCallable = new ServerCallable<Boolean>(conn,
        tableName, first) {
      @Override
      public Boolean call() throws Exception {
        SecureBulkLoadClient secureClient = null;
        boolean success = false;

        try {
          LOG.debug("Going to connect to server " + location + " for row "
              + Bytes.toStringBinary(row));
          byte[] regionName = location.getRegionInfo().getRegionName();
          if(!useSecure) {
             success = server.bulkLoadHFiles(famPaths, regionName, assignSeqIds);
          } else {
            HTable table = new HTable(conn.getConfiguration(), tableName);
            secureClient = new SecureBulkLoadClient(table, location.getRegionInfo().getStartKey());
            success = secureClient.bulkLoadHFiles(famPaths, userToken, bulkToken, assignSeqIds);
          }
          return success;
        } finally {
          //Best effort copying of files that might not have been imported
          //from the staging directory back to original location
          //in user directory
          if(secureClient != null && !success) {
            FileSystem fs = FileSystem.get(cfg);
            for(Pair<byte[], String> el : famPaths) {
              Path hfileStagingPath = null;
              Path hfileOrigPath = new Path(el.getSecond());
              try {
                hfileStagingPath= new Path(secureClient.getStagingPath(bulkToken, el.getFirst()),
                    hfileOrigPath.getName());
                if(fs.rename(hfileStagingPath, hfileOrigPath)) {
                  LOG.debug("Moved back file " + hfileOrigPath + " from " +
                      hfileStagingPath);
                } else if(fs.exists(hfileStagingPath)){
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hbase.coprocessor.SecureBulkLoadClient

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.