Examples of BatchWriterOpts


Examples of org.apache.accumulo.core.cli.BatchWriterOpts

import org.apache.hadoop.io.Text;

public class GCLotsOfCandidatesTest {
  public static void main(String args[]) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
    ClientOpts opts = new ClientOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(GCLotsOfCandidatesTest.class.getName(), args, bwOpts);
   
    Connector conn = opts.getConnector();
    conn.securityOperations().grantTablePermission(conn.whoami(), Constants.METADATA_TABLE_NAME, TablePermission.WRITE);
    BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, bwOpts.getBatchWriterConfig());
   
    for (int i = 0; i < 100000; ++i) {
      final Text emptyText = new Text("");
      Text row = new Text(String.format("%s/%020d/%s", Constants.METADATA_DELETE_FLAG_PREFIX, i,
          "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj"));
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

    long num = 0;
  }
 
  public static void main(String[] args) {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    opts.parseArgs(TestBinaryRows.class.getName(), args, scanOpts, bwOpts);
   
    try {
      Connector connector = opts.getConnector();
     
      final Text CF = new Text("cf"), CQ = new Text("cq");
      final byte[] CF_BYTES = "cf".getBytes(Constants.UTF8), CQ_BYTES = "cq".getBytes(Constants.UTF8);
     
      if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
        BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
        boolean delete = opts.mode.equals("delete");
       
        for (long i = 0; i < opts.num; i++) {
          byte[] row = encodeLong(i + opts.start);
          String value = "" + (i + opts.start);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

  }
 
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(TestMultiTableIngest.class.getName(), args, scanOpts, bwOpts);
    // create the test table within accumulo
    Connector connector;
    try {
      connector = opts.getConnector();
    } catch (AccumuloException e) {
      throw new RuntimeException(e);
    } catch (AccumuloSecurityException e) {
      throw new RuntimeException(e);
    }
    for (int i = 0; i < opts.tables; i++) {
      tableNames.add(String.format("test_%04d", i));
    }
   
    if (!opts.readonly) {
      for (String table : tableNames)
        connector.tableOperations().create(table);
     
      MultiTableBatchWriter b;
      try {
        b = connector.createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
     
      // populate
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

  }
 
  public static void main(String[] args) throws Exception {
    String program = CreateTestTable.class.getName();
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    opts.parseArgs(program, args, bwOpts, scanOpts);
   
    // create the test table within accumulo
    Connector connector = opts.getConnector();
   
    if (!opts.readOnly) {
      TreeSet<Text> keys = new TreeSet<Text>();
      for (int i = 0; i < opts.count / 100; i++) {
        keys.add(new Text(String.format("%05d", i * 100)));
      }
     
      // presplit
      connector.tableOperations().create(opts.getTableName());
      connector.tableOperations().addSplits(opts.getTableName(), keys);
      BatchWriter b = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
     
      // populate
      for (int i = 0; i < opts.count; i++) {
        Mutation m = new Mutation(new Text(String.format("%05d", i)));
        m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

  }
 
  @Override
  public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(ContinuousMoru.class.getName(), args, bwOpts);
   
    Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());
   
    job.setInputFormatClass(AccumuloInputFormat.class);
    opts.setAccumuloConfigs(job);
   
    // set up ranges
    try {
      Set<Range> ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
      AccumuloInputFormat.setRanges(job, ranges);
      AccumuloInputFormat.setAutoAdjustRanges(job, false);
    } catch (Exception e) {
      throw new IOException(e);
    }
   
    job.setMapperClass(CMapper.class);
   
    job.setNumReduceTasks(0);
   
    job.setOutputFormatClass(AccumuloOutputFormat.class);
    AccumuloOutputFormat.setBatchWriterOptions(job, bwOpts.getBatchWriterConfig());
   
    Configuration conf = job.getConfiguration();
    conf.setLong(MIN, opts.min);
    conf.setLong(MAX, opts.max);
    conf.setInt(MAX_CF, opts.maxColF);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

  }
 
  public static void main(String[] args) throws Exception {
   
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(TestIngest.class.getName(), args, bwOpts);
    opts.getInstance().setConfiguration(ServerConfiguration.getSiteConfiguration());

    createTable(opts);
   
    Instance instance = opts.getInstance();
   
    String name = TestIngest.class.getSimpleName();
    DistributedTrace.enable(instance, new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut()), name, null);
   
    try {
      opts.startTracing(name);
     
      Logger.getLogger(TabletServerBatchWriter.class.getName()).setLevel(Level.TRACE);
     
      // test batch update
     
      long stopTime;
     
      byte[][] bytevals = generateValues(opts);
     
      byte randomValue[] = new byte[opts.dataSize];
      Random random = new Random();
     
      long bytesWritten = 0;
     
      BatchWriter bw = null;
      FileSKVWriter writer = null;
     
      if (opts.outputFile != null) {
        Configuration conf = CachedConfiguration.getInstance();
        FileSystem fs = FileSystem.get(conf);
        writer = FileOperations.getInstance().openWriter(opts.outputFile + "." + RFile.EXTENSION, fs, conf,
            AccumuloConfiguration.getDefaultConfiguration());
        writer.startDefaultLocalityGroup();
      } else {
        Connector connector = opts.getConnector();
        bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
        connector.securityOperations().changeUserAuthorizations(opts.principal, AUTHS);
      }
      Text labBA = new Text(opts.columnVisibility.getExpression());
     
      long startTime = System.currentTimeMillis();
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

public class Write {
 
  public static void main(String[] args) throws Exception {
    WriteOptions opts = new WriteOptions();
    BatchWriterOpts batch_writer_opts = new BatchWriterOpts();
    opts.parseArgs(Write.class.getName(), args, batch_writer_opts);
   
    opts.check();
   
    Connector c = opts.getConnector();
   
    if(opts.clear_table && c.tableOperations().exists(opts.getTableName())) {
      try {
      c.tableOperations().delete(opts.getTableName());
      } catch(TableNotFoundException e) {
        System.err.println("Couldn't delete the table because it doesn't exist any more.");
      }
    }
   
    if(!c.tableOperations().exists(opts.getTableName())) {
      try {
        c.tableOperations().create(opts.getTableName());
      } catch (TableExistsException e) {
        System.err.println("Couldn't create table ourselves, but that's ok. Continuing.");
      }
    }

    long writeDelay = opts.write_delay;
    if (writeDelay < 0) {
      writeDelay = 0;
    }

    DataWriter dw = new DataWriter(c.createBatchWriter(opts.getTableName(), batch_writer_opts.getBatchWriterConfig()),
        new RandomMutations(
            //rows
            new RandomByteArrays(
                new RandomWithinRange(
                    opts.row_seed,
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

  /**
   * Writes a specified number of entries to Accumulo using a {@link BatchWriter}.
   */
  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(RandomBatchWriter.class.getName(), args, bwOpts);
    if ((opts.max - opts.min) < opts.num) {
      System.err.println(String.format("You must specify a min and a max that allow for at least num possible values. For example, you requested %d rows, but a min of %d and a max of %d only allows for %d rows.", opts.num, opts.min, opts.max, (opts.max - opts.min)));
      System.exit(1);
    }
    Random r;
    if (opts.seed == null)
      r = new Random();
    else {
      r = new Random(opts.seed);
    }
    Connector connector = opts.getConnector();
    BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
   
    // reuse the ColumnVisibility object to improve performance
    ColumnVisibility cv = opts.visiblity;
  
    // Generate num unique row ids in the given range
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
      MutationsRejectedException {
   
    ClientOpts opts = new ClientOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(RowOperations.class.getName(), args, scanOpts, bwOpts);
   
    // First the setup work
    connector = opts.getConnector();
   
    // lets create an example table
    connector.tableOperations().create(table);
   
    // lets create 3 rows of information
    Text row1 = new Text("row1");
    Text row2 = new Text("row2");
    Text row3 = new Text("row3");
   
    // Which means 3 different mutations
    Mutation mut1 = new Mutation(row1);
    Mutation mut2 = new Mutation(row2);
    Mutation mut3 = new Mutation(row3);
   
    // And we'll put 4 columns in each row
    Text col1 = new Text("1");
    Text col2 = new Text("2");
    Text col3 = new Text("3");
    Text col4 = new Text("4");
   
    // Now we'll add them to the mutations
    mut1.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut1.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut1.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut1.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
   
    mut2.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut2.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut2.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut2.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
   
    mut3.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut3.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut3.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut3.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
   
    // Now we'll make a Batch Writer
    bw = connector.createBatchWriter(table, bwOpts.getBatchWriterConfig());
   
    // And add the mutations
    bw.addMutation(mut1);
    bw.addMutation(mut2);
    bw.addMutation(mut3);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

  }
 
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(RemoveEntriesForMissingFiles.class.getName(), args, scanOpts, bwOpts);
    FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
    Connector connector = opts.getConnector();
    Scanner metadata = connector.createScanner(Constants.METADATA_TABLE_NAME, opts.auths);
    metadata.setBatchSize(scanOpts.scanBatchSize);
    metadata.setRange(Constants.METADATA_KEYSPACE);
    metadata.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
    int count = 0;
    int missing = 0;
    BatchWriter writer = null;
    if (opts.fix)
      writer = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, bwOpts.getBatchWriterConfig());
    for (Entry<Key,Value> entry : metadata) {
      count++;
      Key key = entry.getKey();
      String table = new String(KeyExtent.tableOfMetadataRow(entry.getKey().getRow()), Constants.UTF8);
      String file = key.getColumnQualifier().toString();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.