Examples of BatchWriter

com.graphaware.writer.BatchWriter
{@link SingleThreadedWriter} that writes tasks in batches. This is more performant but dangerous,since a single task's failure can roll back the whole batch. This is here for experiments, not for production.
com.rackspacecloud.blueflood.inputs.processors.BatchWriter
org.apache.accumulo.core.client.BatchWriter
Send Mutations to a single Table in Accumulo. When the user uses a Connector to create a BatchWriter, they specify how much memory and how many threads it should use. As the user adds mutations to the batch writer, it buffers them. Once the buffered mutations have used half of the user specified buffer, the mutations are dumped into the background to be written by a thread pool. If the user specified memory completely fills up, then writes are held. When a user calls flush, it does not return until all buffered mutations are written.

Examples of org.apache.accumulo.core.client.BatchWriter

  public void testFilterConjunction() throws Exception {
    MockInstance instance = new MockInstance("rft1");
    Connector conn = instance.getConnector("", new PasswordToken(""));


    conn.tableOperations().create("filter_conjunction");
    BatchWriter bw = conn.createBatchWriter("filter_conjunction", new BatchWriterConfig());
    for (Mutation m : createMutations()) {
      bw.addMutation(m);
    }
    conn.tableOperations().attachIterator("filter_conjunction", new IteratorSetting(40, "rowZeroOrOne", RowZeroOrOneFilter.class));
    conn.tableOperations().attachIterator("filter_conjunction", new IteratorSetting(41, "rowOneOrTwo", RowOneOrTwoFilter.class));
    Scanner scanner = conn.createScanner("filter_conjunction", Constants.NO_AUTHS);
    assertEquals(new HashSet<String>(Arrays.asList("1")), getRows(scanner));

View Full Code Here

Examples of org.apache.accumulo.core.client.BatchWriter

  }
  
  @Override
  public void delete() throws MutationsRejectedException, TableNotFoundException {
    
    BatchWriter writer = new MockBatchWriter(acc, tableName);
    try {
      Iterator<Entry<Key,Value>> iter = super.iterator();
      while (iter.hasNext()) {
        Entry<Key,Value> next = iter.next();
        Key k = next.getKey();
        Mutation m = new Mutation(k.getRow());
        m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp());
        writer.addMutation(m);
      }
    } finally {
      writer.close();
    }
  }

View Full Code Here

Examples of org.apache.accumulo.core.client.BatchWriter

      break;
    }


    String tableId = getId(tableName);


    BatchWriter tbw = tableWriters.get(tableId);
    if (tbw == null) {
      tbw = new TableBatchWriter(tableId);
      tableWriters.put(tableId, tbw);
    }
    return tbw;

View Full Code Here

Examples of org.apache.accumulo.core.client.BatchWriter

        log.info("Simulating adding table: " + tableName);
        return;
      }
      
      log.debug("Adding table: " + tableName);
      BatchWriter bw = null;
      String table = tableName.toString();
      
      if (createTables && !conn.tableOperations().exists(table)) {
        try {
          conn.tableOperations().create(table);

View Full Code Here

Examples of org.apache.accumulo.core.client.BatchWriter

        log.info("Simulating adding table: " + tableName);
        return;
      }
      
      log.debug("Adding table: " + tableName);
      BatchWriter bw = null;
      String table = tableName.toString();
      
      if (createTables && !conn.tableOperations().exists(table)) {
        try {
          conn.tableOperations().create(table);

View Full Code Here

Examples of org.apache.accumulo.core.client.BatchWriter

  private void test2() throws Exception {
    basicTest(PRE_SPLIT_TABLE_NAME, NUM_PRE_SPLITS);
  }
  
  private void basicTest(String table, int expectedSplits) throws Exception {
    BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
    
    Random r = new Random();
    byte rowData[] = new byte[ROW_SIZE];
    
    r.setSeed(SEED);
    
    for (int i = 0; i < NUM_ROWS; i++) {
      
      r.nextBytes(rowData);
      TestIngest.toPrintableChars(rowData);
      
      Mutation mut = new Mutation(new Text(rowData));
      mut.put(new Text(""), new Text(""), new Value(Integer.toString(i).getBytes(Constants.UTF8)));
      bw.addMutation(mut);
    }
    
    bw.close();
    
    checkSplits(table, expectedSplits, expectedSplits);
    
    verify(table);

View Full Code Here

Examples of org.apache.accumulo.core.client.BatchWriter

public class Write extends Test {
  
  @Override
  public void visit(State state, Properties props) throws Exception {
    
    BatchWriter bw = state.getMultiTableBatchWriter().getBatchWriter(state.getString("seqTableName"));
    
    state.set("numWrites", state.getLong("numWrites") + 1);
    
    Long totalWrites = state.getLong("totalWrites") + 1;
    if ((totalWrites % 10000) == 0) {
      log.debug("Total writes: " + totalWrites);
    }
    state.set("totalWrites", totalWrites);
    
    Mutation m = new Mutation(new Text(String.format("%010d", totalWrites)));
    m.put(new Text("cf"), new Text("cq"), new Value("val".getBytes(Constants.UTF8)));
    bw.addMutation(m);
  }

View Full Code Here

Examples of org.apache.accumulo.core.client.BatchWriter

  @Override
  public void run() throws Exception {


    getConnector().tableOperations().create("test");


    BatchWriter bw = getConnector().createBatchWriter("test", new BatchWriterConfig());


    Mutation m1 = new Mutation("r1");
    m1.put("cf1", "cq1", 1, "5");


    bw.addMutation(m1);


    bw.flush();


    Scanner scanner = getConnector().createScanner("test", new Authorizations());


    int count = 0;
    for (Entry<Key,Value> entry : scanner) {
      count++;
      if (!entry.getValue().toString().equals("5")) {
        throw new Exception("Unexpected value " + entry.getValue());
      }
    }


    if (count != 1) {
      throw new Exception("Unexpected count " + count);
    }


    if (countThreads() < 2) {
      printThreadNames();
      throw new Exception("Not seeing expected threads");
    }


    CleanUp.shutdownNow();


    Mutation m2 = new Mutation("r2");
    m2.put("cf1", "cq1", 1, "6");


    try {
      bw.addMutation(m1);
      bw.flush();
      throw new Exception("batch writer did not fail");
    } catch (Exception e) {


    }


    try {
      // expect this to fail also, want to clean up batch writer threads
      bw.close();
      throw new Exception("batch writer close not fail");
    } catch (Exception e) {


    }

View Full Code Here

Examples of org.apache.accumulo.core.client.BatchWriter

    List<String> tableNames = (List<String>) state.get("tables");
    
    String tableName = tableNames.get(rand.nextInt(tableNames.size()));
    
    try {
      BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
      try {
        int numRows = rand.nextInt(100000);
        for (int i = 0; i < numRows; i++) {
          Mutation m = new Mutation(String.format("%016x", (rand.nextLong() & 0x7fffffffffffffffl)));
          long val = (rand.nextLong() & 0x7fffffffffffffffl);
          for (int j = 0; j < 10; j++) {
            m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(Constants.UTF8)));
          }
          
          bw.addMutation(m);
        }
      } finally {
        bw.close();
      }
      
      log.debug("Wrote to " + tableName);
    } catch (TableNotFoundException e) {
      log.debug("BatchWrite " + tableName + " failed, doesnt exist");

View Full Code Here

Examples of org.apache.accumulo.core.client.BatchWriter

    
    String rootDir = "/tmp/shard_bulk/" + dataTableName;
    
    fs.mkdirs(new Path(rootDir));
    
    BatchWriter dataWriter = new SeqfileBatchWriter(conf, fs, rootDir + "/data.seq");
    BatchWriter indexWriter = new SeqfileBatchWriter(conf, fs, rootDir + "/index.seq");
    
    for (int i = 0; i < numToInsert; i++) {
      String docID = Insert.insertRandomDocument(nextDocID++, dataWriter, indexWriter, indexTableName, dataTableName, numPartitions, rand);
      log.debug("Bulk inserting document " + docID);
    }
    
    state.set("nextDocID", Long.valueOf(nextDocID));
    
    dataWriter.close();
    indexWriter.close();
    
    sort(state, fs, dataTableName, rootDir + "/data.seq", rootDir + "/data_bulk", rootDir + "/data_work", maxSplits);
    sort(state, fs, indexTableName, rootDir + "/index.seq", rootDir + "/index_bulk", rootDir + "/index_work", maxSplits);
    
    bulkImport(fs, state, dataTableName, rootDir, "data");

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.