Examples of BatchWriterOpts

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
   * save off files, and recover your instance by re-initializing and importing the existing files.
   * 
   */
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
   
    final Key rootTableEnd = new Key(Constants.ROOT_TABLET_EXTENT.getEndRow());
    final Range range = new Range(rootTableEnd.followingKey(PartialKey.ROW), true, Constants.METADATA_RESERVED_KEYSPACE_START_KEY, false);
    final Scanner scanner = opts.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
    scanner.setRange(range);
    final Configuration conf = new Configuration();
    final FileSystem fs = FileSystem.get(conf);
   
    KeyExtent last = new KeyExtent();
    String directory = null;
    Set<String> knownFiles = new HashSet<String>();
   
    int count = 0;
    final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
   
    // collect the list of known files and the directory for each extent
    for (Entry<Key,Value> entry : scanner) {
      Key key = entry.getKey();
      KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
  }

  public static void main(String[] args) throws Exception {
   
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(ContinuousIngest.class.getName(), args, bwOpts);
   
    initVisibilities(opts);

    if (opts.min < 0 || opts.max < 0 || opts.max <= opts.min) {
      throw new IllegalArgumentException("bad min and max");
    }
    Connector conn = opts.getConnector();
   
    if (!conn.tableOperations().exists(opts.getTableName())) {
      throw new TableNotFoundException(null, opts.getTableName(), "Consult the README and create the table before starting ingest.");
    }

    BatchWriter bw = conn.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
    bw = Trace.wrapAll(bw, new CountSampler(1024));
   
    Random r = new Random();
   
    byte[] ingestInstanceId = UUID.randomUUID().toString().getBytes(Constants.UTF8);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
      MutationsRejectedException {
   
    ClientOpts opts = new ClientOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(RowOperations.class.getName(), args, scanOpts, bwOpts);
   
    // First the setup work
    connector = opts.getConnector();
   
    // lets create an example table
    connector.tableOperations().create(table);
   
    // lets create 3 rows of information
    Text row1 = new Text("row1");
    Text row2 = new Text("row2");
    Text row3 = new Text("row3");
   
    // Which means 3 different mutations
    Mutation mut1 = new Mutation(row1);
    Mutation mut2 = new Mutation(row2);
    Mutation mut3 = new Mutation(row3);
   
    // And we'll put 4 columns in each row
    Text col1 = new Text("1");
    Text col2 = new Text("2");
    Text col3 = new Text("3");
    Text col4 = new Text("4");
   
    // Now we'll add them to the mutations
    mut1.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut1.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut1.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut1.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
   
    mut2.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut2.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut2.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut2.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
   
    mut3.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut3.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut3.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut3.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
   
    // Now we'll make a Batch Writer
    bw = connector.createBatchWriter(table, bwOpts.getBatchWriterConfig());
   
    // And add the mutations
    bw.addMutation(mut1);
    bw.addMutation(mut2);
    bw.addMutation(mut3);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
   * @throws AccumuloSecurityException
   * @throws TableNotFoundException
   */
  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(RandomBatchWriter.class.getName(), args, bwOpts);
   
    Random r;
    if (opts.seed == null)
      r = new Random();
    else {
      r = new Random(opts.seed);
    }
   
    Connector connector = opts.getConnector();
    BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
   
    // reuse the ColumnVisibility object to improve performance
    ColumnVisibility cv = opts.visiblity;
   
    for (int i = 0; i < opts.num; i++) {
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
      MutationsRejectedException {
   
    ClientOpts opts = new ClientOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(RowOperations.class.getName(), args, scanOpts, bwOpts);
   
    // First the setup work
    connector = opts.getConnector();
   
    // lets create an example table
    connector.tableOperations().create(table);
   
    // lets create 3 rows of information
    Text row1 = new Text("row1");
    Text row2 = new Text("row2");
    Text row3 = new Text("row3");
   
    // Which means 3 different mutations
    Mutation mut1 = new Mutation(row1);
    Mutation mut2 = new Mutation(row2);
    Mutation mut3 = new Mutation(row3);
   
    // And we'll put 4 columns in each row
    Text col1 = new Text("1");
    Text col2 = new Text("2");
    Text col3 = new Text("3");
    Text col4 = new Text("4");
   
    // Now we'll add them to the mutations
    mut1.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut1.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut1.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut1.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
   
    mut2.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut2.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut2.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut2.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
   
    mut3.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut3.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut3.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
    mut3.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
   
    // Now we'll make a Batch Writer
    bw = connector.createBatchWriter(table, bwOpts.getBatchWriterConfig());
   
    // And add the mutations
    bw.addMutation(mut1);
    bw.addMutation(mut2);
    bw.addMutation(mut3);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

54
55
56
57
58
59
60
61
62
63
64
65
66
67
   * @throws TableNotFoundException
   * @throws MutationsRejectedException
   */
  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(SequentialBatchWriter.class.getName(), args, bwOpts);
    Connector connector = opts.getConnector();
    BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
   
    long end = opts.start + opts.num;
   
    for (long i = opts.start; i < end; i++) {
      Mutation m = RandomBatchWriter.createMutation(i, opts.valueSize, opts.vis);
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

107
108
109
110
111
112
113
114
115
116
117
 
  static public void main(String[] args) {
   
    ClientOpts opts = new ClientOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(TestRandomDeletes.class.getName(), args, scanOpts, bwOpts);
   
    try {
      long deleted = 0;
     
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
  }
 
  public static void main(String[] args) throws Exception {
   
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(TestIngest.class.getName(), args, bwOpts);
    opts.getInstance().setConfiguration(ServerConfiguration.getSiteConfiguration());

    createTable(opts);
   
    Instance instance = opts.getInstance();
   
    String name = TestIngest.class.getSimpleName();
    DistributedTrace.enable(instance, new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut()), name, null);
   
    try {
      opts.startTracing(name);
     
      Logger.getLogger(TabletServerBatchWriter.class.getName()).setLevel(Level.TRACE);
     
      // test batch update
     
      long stopTime;
     
      byte[][] bytevals = generateValues(opts);
     
      byte randomValue[] = new byte[opts.dataSize];
      Random random = new Random();
     
      long bytesWritten = 0;
     
      BatchWriter bw = null;
      FileSKVWriter writer = null;
     
      if (opts.outputFile != null) {
        Configuration conf = CachedConfiguration.getInstance();
        FileSystem fs = FileSystem.get(conf);
        writer = FileOperations.getInstance().openWriter(opts.outputFile + "." + RFile.EXTENSION, fs, conf,
            AccumuloConfiguration.getDefaultConfiguration());
        writer.startDefaultLocalityGroup();
      } else {
        Connector connector = opts.getConnector();
        bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
        connector.securityOperations().changeUserAuthorizations(opts.principal, AUTHS);
      }
      Text labBA = new Text(opts.columnVisibility.getExpression());
     
      long startTime = System.currentTimeMillis();
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
  }
 
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(TestMultiTableIngest.class.getName(), args, scanOpts, bwOpts);
    // create the test table within accumulo
    Connector connector;
    try {
      connector = opts.getConnector();
    } catch (AccumuloException e) {
      throw new RuntimeException(e);
    } catch (AccumuloSecurityException e) {
      throw new RuntimeException(e);
    }
    for (int i = 0; i < opts.tables; i++) {
      tableNames.add(String.format("test_%04d", i));
    }
   
    if (!opts.readonly) {
      for (String table : tableNames)
        connector.tableOperations().create(table);
     
      MultiTableBatchWriter b;
      try {
        b = connector.createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
     
      // populate
View Full Code Here

Examples of org.apache.accumulo.core.cli.BatchWriterOpts

125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
  }
 
  @Override
  public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(ContinuousMoru.class.getName(), args, bwOpts);
   
    Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());
   
    job.setInputFormatClass(AccumuloInputFormat.class);
    opts.setAccumuloConfigs(job);
   
    // set up ranges
    try {
      Set<Range> ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
      AccumuloInputFormat.setRanges(job, ranges);
      AccumuloInputFormat.setAutoAdjustRanges(job, false);
    } catch (Exception e) {
      throw new IOException(e);
    }
   
    job.setMapperClass(CMapper.class);
   
    job.setNumReduceTasks(0);
   
    job.setOutputFormatClass(AccumuloOutputFormat.class);
    AccumuloOutputFormat.setBatchWriterOptions(job, bwOpts.getBatchWriterConfig());
   
    Configuration conf = job.getConfiguration();
    conf.setLong(MIN, opts.min);
    conf.setLong(MAX, opts.max);
    conf.setInt(MAX_CF, opts.maxColF);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.