Examples of org.apache.accumulo.server.zookeeper.DistributedWorkQueue

Package org.apache.accumulo.server.zookeeper

Examples of org.apache.accumulo.server.zookeeper.DistributedWorkQueue

org.apache.accumulo.server.zookeeper.DistributedWorkQueue
Provides a way to push work out to tablet servers via zookeeper and wait for that work to be done. Any tablet server can pick up a work item and process it. Worker processes watch a zookeeper node for tasks to be performed. After getting an exclusive lock on the node, the worker will perform the task.

      fs.rename(orig, dest);
      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
    }
    
    if (loadedFailures.size() > 0) {
      DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
          + Constants.ZBULK_FAILED_COPYQ);
      
      HashSet<String> workIds = new HashSet<String>();
      
      for (String failure : loadedFailures.values()) {
        Path orig = new Path(failure);
        Path dest = new Path(error, orig.getName());
        
        if (fs.exists(dest))
          continue;
        
        bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes());
        workIds.add(orig.getName());
        log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
      }
      
      bifCopyQueue.waitUntilDone(workIds);
    }


    fs.delete(new Path(error, BulkImport.FAILURES_TXT), true);
    return new CleanUpBulkImport(tableId, source, bulk, error);
  }

View Full Code Here

    }
    announceExistence();


    ThreadPoolExecutor distWorkQThreadPool = new SimpleThreadPool(getSystemConfiguration().getCount(Property.TSERV_WORKQ_THREADS), "distributed work queue");


    bulkFailedCopyQ = new DistributedWorkQueue(ZooUtil.getRoot(instance) + Constants.ZBULK_FAILED_COPYQ);
    try {
      bulkFailedCopyQ.startProcessing(new BulkFailedCopyProcessor(), distWorkQThreadPool);
    } catch (Exception e1) {
      throw new RuntimeException("Failed to start distributed work queue for copying ", e1);
    }

View Full Code Here

      fs.rename(orig, dest);
      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
    }


    if (loadedFailures.size() > 0) {
      DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
          + Constants.ZBULK_FAILED_COPYQ);


      HashSet<String> workIds = new HashSet<String>();


      for (String failure : loadedFailures.values()) {
        Path orig = new Path(failure);
        Path dest = new Path(error, orig.getName());


        if (fs.exists(dest))
          continue;


        bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(Constants.UTF8));
        workIds.add(orig.getName());
        log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
      }


      bifCopyQueue.waitUntilDone(workIds);
    }


    fs.deleteRecursively(new Path(error, BulkImport.FAILURES_TXT));
    return new CleanUpBulkImport(tableId, source, bulk, error);
  }

View Full Code Here

    
    ThreadPoolExecutor distWorkQThreadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(
        ServerConfiguration.getSystemConfiguration().getCount(Property.TSERV_WORKQ_THREADS),
        new NamingThreadFactory("distributed work queue"));


    bulkFailedCopyQ = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZBULK_FAILED_COPYQ);
    try {
      bulkFailedCopyQ.startProcessing(new BulkFailedCopyProcessor(), distWorkQThreadPool);
    } catch (Exception e1) {
      throw new RuntimeException("Failed to start distributed work queue for copying ", e1);
    }

View Full Code Here

      fs.rename(orig, dest);
      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": failed");
    }
    
    if (loadedFailures.size() > 0) {
      DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
          + Constants.ZBULK_FAILED_COPYQ);
      
      HashSet<String> workIds = new HashSet<String>();
      
      for (String failure : loadedFailures.values()) {
        Path orig = new Path(failure);
        Path dest = new Path(error, orig.getName());
        
        if (fs.exists(dest))
          continue;
        
        bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes());
        workIds.add(orig.getName());
        log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
      }
      
      bifCopyQueue.waitUntilDone(workIds);
    }


    fs.delete(new Path(error, "failures.txt"), true);
    return new CleanUpBulkImport(tableId, source, bulk, error);
  }

View Full Code Here

      fs.rename(orig, dest);
      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": failed");
    }
    
    if (loadedFailures.size() > 0) {
      DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
          + Constants.ZBULK_FAILED_COPYQ);
      
      HashSet<String> workIds = new HashSet<String>();
      
      for (String failure : loadedFailures.values()) {
        Path orig = new Path(failure);
        Path dest = new Path(error, orig.getName());
        
        if (fs.exists(dest))
          continue;
        
        bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes());
        workIds.add(orig.getName());
        log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
      }
      
      bifCopyQueue.waitUntilDone(workIds);
    }


    fs.delete(new Path(error, "failures.txt"), true);
    return new CleanUpBulkImport(tableId, source, bulk, error);
  }

View Full Code Here

    
    ThreadPoolExecutor distWorkQThreadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(
        ServerConfiguration.getSystemConfiguration().getCount(Property.TSERV_WORKQ_THREADS),
        new NamingThreadFactory("distributed work queue"));


    bulkFailedCopyQ = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZBULK_FAILED_COPYQ);
    try {
      bulkFailedCopyQ.startProcessing(new BulkFailedCopyProcessor(), distWorkQThreadPool);
    } catch (Exception e1) {
      throw new RuntimeException("Failed to start distributed work queue for copying ", e1);
    }

View Full Code Here

      fs.rename(orig, dest);
      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
    }
    
    if (loadedFailures.size() > 0) {
      DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
          + Constants.ZBULK_FAILED_COPYQ);
      
      HashSet<String> workIds = new HashSet<String>();
      
      for (String failure : loadedFailures.values()) {
        Path orig = new Path(failure);
        Path dest = new Path(error, orig.getName());
        
        if (fs.exists(dest))
          continue;
        
        bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(Constants.UTF8));
        workIds.add(orig.getName());
        log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
      }
      
      bifCopyQueue.waitUntilDone(workIds);
    }


    fs.delete(new Path(error, BulkImport.FAILURES_TXT), true);
    return new CleanUpBulkImport(tableId, source, bulk, error);
  }

View Full Code Here

    clientAddress = new InetSocketAddress(clientAddress.getAddress(), clientPort);
    announceExistence();
    
    ThreadPoolExecutor distWorkQThreadPool = new SimpleThreadPool(getSystemConfiguration().getCount(Property.TSERV_WORKQ_THREADS), "distributed work queue");
    
    bulkFailedCopyQ = new DistributedWorkQueue(ZooUtil.getRoot(instance) + Constants.ZBULK_FAILED_COPYQ);
    try {
      bulkFailedCopyQ.startProcessing(new BulkFailedCopyProcessor(), distWorkQThreadPool);
    } catch (Exception e1) {
      throw new RuntimeException("Failed to start distributed work queue for copying ", e1);
    }

View Full Code Here

      fs.rename(orig, dest);
      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": failed");
    }
    
    if (loadedFailures.size() > 0) {
      DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
          + Constants.ZBULK_FAILED_COPYQ);
      
      HashSet<String> workIds = new HashSet<String>();
      
      for (String failure : loadedFailures.values()) {
        Path orig = new Path(failure);
        Path dest = new Path(error, orig.getName());
        
        if (fs.exists(dest))
          continue;
        
        bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes());
        workIds.add(orig.getName());
        log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
      }
      
      bifCopyQueue.waitUntilDone(workIds);
    }


    fs.delete(new Path(error, "failures.txt"), true);
    return new CleanUpBulkImport(tableId, source, bulk, error);
  }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.accumulo.server.zookeeper.DistributedWorkQueue

org.apache.accumulo.master.recovery.RecoveryManager

org.apache.accumulo.master.tableOps.CopyFailed

org.apache.accumulo.server.master.recovery.RecoveryManager

org.apache.accumulo.server.master.tableOps.CopyFailed

org.apache.accumulo.server.tabletserver.log.LogSorter

org.apache.accumulo.server.tabletserver.TabletServer

org.apache.accumulo.tserver.log.LogSorter

org.apache.accumulo.tserver.TabletServer

org.apache.zookeeper.Watcher

java.util.TimerTask

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.