Examples of BatchWriterOpts

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

55
56
57
58
59
60
61
62
63
64
65
  public static void runTest(Connector c) throws Exception {
    runTest(c, "bt");
  }

  public static void runTest(Connector c, String tableName) throws Exception {
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    TestBinaryRows.Opts opts = new TestBinaryRows.Opts();
    opts.tableName = tableName;
    opts.start = 0;
    opts.num = 100000;
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
    List<String> files = new ArrayList<String>();
  }
 
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(Index.class.getName(), args, bwOpts);
   
    String splitRegex = "\\W+";
   
    BatchWriter bw = opts.getConnector().createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());   
    for (String filename : opts.files) {
      index(opts.partitions, new File(filename), splitRegex, bw);
    }
    bw.close();
  }
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
  }
 
 
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(FileDataIngest.class.getName(), args, bwOpts);
   
    Connector conn = opts.getConnector();
    if (!conn.tableOperations().exists(opts.tableName)) {
      conn.tableOperations().create(opts.tableName);
      conn.tableOperations().attachIterator(opts.tableName, new IteratorSetting(1, ChunkCombiner.class));
    }
    BatchWriter bw = conn.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
    FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility);
    for (String filename : opts.files) {
      fdi.insertFileData(filename, bw);
    }
    bw.close();
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
   * 
   * @param args
   */
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
   
    final Key rootTableEnd = new Key(Constants.ROOT_TABLET_EXTENT.getEndRow());
    final Range range = new Range(rootTableEnd.followingKey(PartialKey.ROW), true, Constants.METADATA_RESERVED_KEYSPACE_START_KEY, false);
    final Scanner scanner = opts.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
    scanner.setRange(range);
    final Configuration conf = new Configuration();
    final FileSystem fs = FileSystem.get(conf);
   
    KeyExtent last = new KeyExtent();
    String directory = null;
    Set<String> knownFiles = new HashSet<String>();
   
    int count = 0;
    final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
   
    // collect the list of known files and the directory for each extent
    for (Entry<Key,Value> entry : scanner) {
      Key key = entry.getKey();
      KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
  }
 
  @Override
  public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(ContinuousMoru.class.getName(), args, bwOpts);
   
    Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());
   
    job.setInputFormatClass(AccumuloInputFormat.class);
    opts.setAccumuloConfigs(job);
   
    // set up ranges
    try {
      Set<Range> ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
      AccumuloInputFormat.setRanges(job, ranges);
      AccumuloInputFormat.setAutoAdjustRanges(job, false);
    } catch (Exception e) {
      throw new IOException(e);
    }
   
    job.setMapperClass(CMapper.class);
   
    job.setNumReduceTasks(0);
   
    job.setOutputFormatClass(AccumuloOutputFormat.class);
    AccumuloOutputFormat.setBatchWriterOptions(job, bwOpts.getBatchWriterConfig());
   
    Configuration conf = job.getConfiguration();
    conf.setLong(MIN, opts.min);
    conf.setLong(MAX, opts.max);
    conf.setInt(MAX_CF, opts.maxColF);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
  }

  public static void main(String[] args) throws Exception {
   
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(ContinuousIngest.class.getName(), args, bwOpts);
   
    initVisibilities(opts);

    if (opts.min < 0 || opts.max < 0 || opts.max <= opts.min) {
      throw new IllegalArgumentException("bad min and max");
    }
    Connector conn = opts.getConnector();
   
    if (!conn.tableOperations().exists(opts.getTableName()))
      try {
        conn.tableOperations().create(opts.getTableName());
      } catch (TableExistsException tee) {}

    BatchWriter bw = conn.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
    bw = Trace.wrapAll(bw, new CountSampler(1024));
   
    Random r = new Random();
   
    byte[] ingestInstanceId = UUID.randomUUID().toString().getBytes(Constants.UTF8);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
  }
 
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(Reverse.class.getName(), args, scanOpts, bwOpts);
   
    Connector conn = opts.getConnector();
   
    Scanner scanner = conn.createScanner(opts.shardTable, opts.auths);
    scanner.setBatchSize(scanOpts.scanBatchSize);
    BatchWriter bw = conn.createBatchWriter(opts.doc2TermTable, bwOpts.getBatchWriterConfig());
   
    for (Entry<Key,Value> entry : scanner) {
      Key key = entry.getKey();
      Mutation m = new Mutation(key.getColumnQualifier());
      m.put(key.getColumnFamily(), new Text(), new Value(new byte[0]));
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
    List<String> files = new ArrayList<String>();
  }
 
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(Index.class.getName(), args, bwOpts);
   
    String splitRegex = "\\W+";
   
    BatchWriter bw = opts.getConnector().createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());   
    for (String filename : opts.files) {
      index(opts.partitions, new File(filename), splitRegex, bw);
    }
    bw.close();
  }
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
  /**
   * Writes a specified number of entries to Accumulo using a {@link BatchWriter}.
   */
  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(RandomBatchWriter.class.getName(), args, bwOpts);
    if ((opts.max - opts.min) < 1L * opts.num) { // right-side multiplied by 1L to convert to long in a way that doesn't trigger FindBugs
      System.err.println(String.format("You must specify a min and a max that allow for at least num possible values. "
          + "For example, you requested %d rows, but a min of %d and a max of %d (exclusive), which only allows for %d rows.", opts.num, opts.min, opts.max,
          (opts.max - opts.min)));
      System.exit(1);
    }
    Random r;
    if (opts.seed == null)
      r = new Random();
    else {
      r = new Random(opts.seed);
    }
    Connector connector = opts.getConnector();
    BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());

    // reuse the ColumnVisibility object to improve performance
    ColumnVisibility cv = opts.visiblity;

    // Generate num unique row ids in the given range
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
  }
 
 
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(InterferenceTest.class.getName(), args, bwOpts);
   
    if (opts.iterations < 1)
      opts.iterations = Long.MAX_VALUE;
   
    Connector conn = opts.getConnector();
    if (!conn.tableOperations().exists(opts.tableName))
      conn.tableOperations().create(opts.tableName);
   
    Thread writer = new Thread(new Writer(conn.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig()), opts.iterations));
    writer.start();
    Reader r;
    if (opts.isolated)
      r = new Reader(new IsolatedScanner(conn.createScanner(opts.tableName, opts.auths)));
    else
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.