Examples of BatchScanner


Examples of org.apache.accumulo.core.client.BatchScanner

    Connector conn = opts.getConnector();
    Scanner scanner = conn.createScanner(opts.getTableName(), auths);
    scanner.setBatchSize(scanOpts.scanBatchSize);
   
    BatchScanner bs = conn.createBatchScanner(opts.getTableName(), auths, bsOpts.scanThreads);
    bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);

    while (true) {
      Set<Text> batch = getBatch(scanner, opts.min, opts.max, scanOpts.scanBatchSize, r);
      List<Range> ranges = new ArrayList<Range>(batch.size());
     
View Full Code Here

Examples of org.apache.accumulo.core.client.BatchScanner

    List<String> tableNames = (List<String>) state.get("tables");
   
    String tableName = tableNames.get(rand.nextInt(tableNames.size()));
   
    try {
      BatchScanner bs = conn.createBatchScanner(tableName, Constants.NO_AUTHS, 3);
      List<Range> ranges = new ArrayList<Range>();
      for (int i = 0; i < rand.nextInt(2000) + 1; i++)
        ranges.add(new Range(String.format("%016x", (rand.nextLong() & 0x7fffffffffffffffl))));
     
      bs.setRanges(ranges);
     
      try {
        Iterator<Entry<Key,Value>> iter = bs.iterator();
        while (iter.hasNext())
          iter.next();
      } finally {
        bs.close();
      }
     
      log.debug("Wrote to " + tableName);
    } catch (TableNotFoundException e) {
      log.debug("BatchScan " + tableName + " failed, doesnt exist");
View Full Code Here

Examples of org.apache.accumulo.core.client.BatchScanner

    Scanner scanner = null;/*
                            * connector.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS); ColumnFQ.fetch(scanner,
                            * Constants.METADATA_LOCATION_COLUMN); ColumnFQ.fetch(scanner, Constants.METADATA_PREV_ROW_COLUMN);
                            */
   
    BatchScanner bs = connector.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 1);
    bs.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
    Constants.METADATA_PREV_ROW_COLUMN.fetch(bs);
   
    bs.setRanges(ranges);
   
    // System.out.println(ranges);
   
    ScanStats stats = new ScanStats();
    for (int i = 0; i < numLoop; i++) {
View Full Code Here

Examples of org.apache.accumulo.core.client.BatchScanner

      columns[index++] = new Text(term);
    }
   
    log.debug("Looking up terms " + searchTerms + " expect to find " + docID);
   
    BatchScanner bs = state.getConnector().createBatchScanner(indexTableName, Constants.NO_AUTHS, 10);
    IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class);
    IntersectingIterator.setColumnFamilies(ii, columns);
    bs.addScanIterator(ii);
    bs.setRanges(Collections.singleton(new Range()));
   
    boolean sawDocID = false;
   
    for (Entry<Key,Value> entry2 : bs) {
      if (entry2.getKey().getColumnQualifier().equals(docID)) {
        sawDocID = true;
        break;
      }
    }
   
    bs.close();
   
    if (!sawDocID)
      throw new Exception("Did not see doc " + docID + " in index.  terms:" + searchTerms + " " + indexTableName + " " + dataTableName);
  }
View Full Code Here

Examples of org.apache.accumulo.core.client.BatchScanner

    int refCount = 0;
   
    try {
      // look for other tables that references this tables files
      Connector conn = master.getConnector();
      BatchScanner bs = conn.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 8);
      try {
        bs.setRanges(Collections.singleton(Constants.NON_ROOT_METADATA_KEYSPACE));
        bs.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
        IteratorSetting cfg = new IteratorSetting(40, "grep", GrepIterator.class);
        GrepIterator.setTerm(cfg, "../" + tableId + "/");
        bs.addScanIterator(cfg);
       
        for (Entry<Key,Value> entry : bs) {
          if (entry.getKey().getColumnQualifier().toString().startsWith("../" + tableId + "/")) {
            refCount++;
          }
        }
      } finally {
        bs.close();
      }
     
    } catch (Exception e) {
      refCount = -1;
      log.error("Failed to scan !METADATA looking for references to deleted table " + tableId, e);
View Full Code Here

Examples of org.apache.accumulo.core.client.BatchScanner

    if (numVerify > (numWrites / 4)) {
      numVerify = numWrites / 4;
    }
   
    Connector conn = state.getConnector();
    BatchScanner scanner = conn.createBatchScanner(state.getString("seqTableName"), new Authorizations(), 2);
   
    try {
      int count = 0;
      List<Range> ranges = new ArrayList<Range>();
      while (count < numVerify) {
        long rangeStart = rand.nextInt((int) numWrites);
        long rangeEnd = rangeStart + 99;
        if (rangeEnd > (numWrites - 1)) {
          rangeEnd = numWrites - 1;
        }
        count += (rangeEnd - rangeStart) + 1;
        ranges.add(new Range(new Text(String.format("%010d", rangeStart)), new Text(String.format("%010d", rangeEnd))));
      }
     
      ranges = Range.mergeOverlapping(ranges);
      Collections.sort(ranges);
     
      if (count == 0 || ranges.size() == 0)
        return;
     
      log.debug(String.format("scanning %d rows in the following %d ranges:", count, ranges.size()));
      for (Range r : ranges) {
        log.debug(r);
      }
     
      scanner.setRanges(ranges);
     
      List<Key> keys = new ArrayList<Key>();
      for (Entry<Key,Value> entry : scanner) {
        keys.add(entry.getKey());
      }
     
      log.debug("scan returned " + keys.size() + " rows. now verifying...");
     
      Collections.sort(keys);
     
      Iterator<Key> iterator = keys.iterator();
      int curKey = Integer.parseInt(iterator.next().getRow().toString());
      boolean done = false;
      for (Range r : ranges) {
        int start = Integer.parseInt(r.getStartKey().getRow().toString());
        int end = Integer.parseInt(String.copyValueOf(r.getEndKey().getRow().toString().toCharArray(), 0, 10));
        for (int i = start; i <= end; i++) {
         
          if (done) {
            log.error("missing key " + i);
            break;
          }
         
          while (curKey < i) {
            log.error("extra key " + curKey);
            if (iterator.hasNext() == false) {
              done = true;
              break;
            }
            curKey = Integer.parseInt(iterator.next().getRow().toString());
          }
         
          if (curKey > i) {
            log.error("missing key " + i);
          }
         
          if (iterator.hasNext()) {
            curKey = Integer.parseInt(iterator.next().getRow().toString());
          } else {
            done = true;
          }
        }
      }
     
      log.debug("verify is now complete");
    } finally {
      scanner.close();
    }
  }
View Full Code Here

Examples of org.apache.accumulo.core.client.BatchScanner

    state.getConnector().tableOperations().compact(indexTableName, null, null, indexFilters, true, true);
    t2 = System.currentTimeMillis();
   
    log.debug("Filtered documents using compaction iterators " + regex + " " + (t3) + " " + (t2 - t1));
   
    BatchScanner bscanner = state.getConnector().createBatchScanner(docTableName, new Authorizations(), 10);
   
    List<Range> ranges = new ArrayList<Range>();
    for (int i = 0; i < 16; i++) {
      ranges.add(Range.prefix(new Text(Integer.toHexString(i) + "" + deleteChar)));
    }
   
    bscanner.setRanges(ranges);
    Iterator<Entry<Key,Value>> iter = bscanner.iterator();
   
    if (iter.hasNext()) {
      throw new Exception("Saw unexpected document " + iter.next().getKey());
    }

    bscanner.close();
  }
View Full Code Here

Examples of org.apache.accumulo.core.client.BatchScanner

   
    for (int i = 0; i < words.length; i++) {
      words[i] = new Text(Insert.generateRandomWord(rand));
    }
   
    BatchScanner bs = state.getConnector().createBatchScanner(indexTableName, Constants.NO_AUTHS, 16);
    IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class.getName());
    IntersectingIterator.setColumnFamilies(ii, words);
    bs.addScanIterator(ii);
    bs.setRanges(Collections.singleton(new Range()));
   
    HashSet<Text> documentsFoundInIndex = new HashSet<Text>();
   
    for (Entry<Key,Value> entry2 : bs) {
      documentsFoundInIndex.add(entry2.getKey().getColumnQualifier());
    }
   
    bs.close();
   
    bs = state.getConnector().createBatchScanner(dataTableName, Constants.NO_AUTHS, 16);
   
    for (int i = 0; i < words.length; i++) {
      IteratorSetting more = new IteratorSetting(20 + i, "ii" + i, RegExFilter.class);
      RegExFilter.setRegexs(more, null, null, null, "(^|(.*\\s))" + words[i] + "($|(\\s.*))", false);
      bs.addScanIterator(more);
    }
   
    bs.setRanges(Collections.singleton(new Range()));
   
    HashSet<Text> documentsFoundByGrep = new HashSet<Text>();
   
    for (Entry<Key,Value> entry2 : bs) {
      documentsFoundByGrep.add(entry2.getKey().getRow());
    }
   
    bs.close();
   
    if (!documentsFoundInIndex.equals(documentsFoundByGrep)) {
      throw new Exception("Set of documents found not equal for words " + Arrays.asList(words).toString() + " " + documentsFoundInIndex + " "
          + documentsFoundByGrep);
    }
View Full Code Here

Examples of org.apache.accumulo.core.client.BatchScanner

    for (Entry<Key,Value> entry : scanner)
      documentsToDelete.add(new Range(entry.getKey().getColumnQualifier()));
   
    if (documentsToDelete.size() > 0) {
      // use a batch scanner to fetch all documents
      BatchScanner bscanner = state.getConnector().createBatchScanner(docTableName, Constants.NO_AUTHS, 8);
      bscanner.setRanges(documentsToDelete);
     
      BatchWriter ibw = state.getMultiTableBatchWriter().getBatchWriter(indexTableName);
      BatchWriter dbw = state.getMultiTableBatchWriter().getBatchWriter(docTableName);
     
      int count = 0;
     
      for (Entry<Key,Value> entry : bscanner) {
        String docID = entry.getKey().getRow().toString();
        String doc = entry.getValue().toString();
       
        Insert.unindexDocument(ibw, doc, docID, numPartitions);
       
        Mutation m = new Mutation(docID);
        m.putDelete("doc", "");
       
        dbw.addMutation(m);
        count++;
      }
     
      bscanner.close();
     
      state.getMultiTableBatchWriter().flush();
     
      if (count != documentsToDelete.size()) {
        throw new Exception("Batch scanner did not return expected number of docs " + count + " " + documentsToDelete.size());
View Full Code Here

Examples of org.apache.accumulo.core.client.BatchScanner

    }
   
    log.debug("Found " + hashes.size() + " hashes starting at " + uuid);
   
    // use batch scanner to verify all of these exist in index
    BatchScanner indexScanner = conn.createBatchScanner(indexTableName, Constants.NO_AUTHS, 3);
    ArrayList<Range> ranges = new ArrayList<Range>();
    for (Text row : hashes.keySet()) {
      ranges.add(new Range(row));
    }
   
    indexScanner.setRanges(ranges);
   
    Map<Text,Text> hashes2 = new HashMap<Text,Text>();
   
    for (Entry<Key,Value> entry : indexScanner)
      hashes2.put(entry.getKey().getRow(), new Text(entry.getValue().get()));
   
    log.debug("Looked up " + ranges.size() + " ranges, found " + hashes2.size());
   
    if (!hashes.equals(hashes2)) {
      log.error("uuids from doc table : " + hashes.values());
      log.error("uuids from index     : " + hashes2.values());
      throw new Exception("Mismatch between document table and index " + indexTableName + " " + imageTableName);
    }
   
    indexScanner.close();
   
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.