* Copyright 2009 The Apache Software Foundation

*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* Tests from client-side of a cluster.

*/

  final Log LOG = LogFactory.getLog(getClass());

  private static byte [] ROW = Bytes.toBytes("testRow");

  private static byte [] FAMILY = Bytes.toBytes("testFamily");

  private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");

  private static byte [] VALUE = Bytes.toBytes("testValue");

 

  private static byte [] EMPTY = new byte[0];

 

  /**

   * Constructor does nothing special, start cluster.

   */

  public TestClient() {

    super();

  }

  /**

   * Test from client side of an involved filter against a multi family that

   * involves deletes.

   *

   * @throws Exception

   */

  public void testWeirdCacheBehaviour() throws Exception {

    byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");

    byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),

        Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),

        Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };

    HTable ht = createTable(TABLE, FAMILIES);

    String value = "this is the value";

    String value2 = "this is some other value";

    String keyPrefix1 = UUID.randomUUID().toString();

    String keyPrefix2 = UUID.randomUUID().toString();

    String keyPrefix3 = UUID.randomUUID().toString();

    putRows(ht, 3, value, keyPrefix1);

    putRows(ht, 3, value, keyPrefix2);

    putRows(ht, 3, value, keyPrefix3);

    ht.flushCommits();

    putRows(ht, 3, value2, keyPrefix1);

    putRows(ht, 3, value2, keyPrefix2);

    putRows(ht, 3, value2, keyPrefix3);

    HTable table = new HTable(conf, Bytes.toBytes("testWeirdCacheBehaviour"));

    System.out.println("Checking values for key: " + keyPrefix1);

    assertEquals("Got back incorrect number of rows from scan", 3,

        getNumberOfRows(keyPrefix1, value2, table));

    System.out.println("Checking values for key: " + keyPrefix2);

    assertEquals("Got back incorrect number of rows from scan", 3,

        getNumberOfRows(keyPrefix2, value2, table));

    System.out.println("Checking values for key: " + keyPrefix3);

    assertEquals("Got back incorrect number of rows from scan", 3,

        getNumberOfRows(keyPrefix3, value2, table));

    deleteColumns(ht, value2, keyPrefix1);

    deleteColumns(ht, value2, keyPrefix2);

    deleteColumns(ht, value2, keyPrefix3);

    System.out.println("Starting important checks.....");

    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,

      0, getNumberOfRows(keyPrefix1, value2, table));

    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,

      0, getNumberOfRows(keyPrefix2, value2, table));

    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,

      0, getNumberOfRows(keyPrefix3, value2, table));

    ht.setScannerCaching(0);

    assertEquals("Got back incorrect number of rows from scan", 0,

      getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100);

    assertEquals("Got back incorrect number of rows from scan", 0,

      getNumberOfRows(keyPrefix2, value2, table));

  }

  private void deleteColumns(HTable ht, String value, String keyPrefix)

  throws IOException {

    ResultScanner scanner = buildScanner(keyPrefix, value, ht);

    Iterator<Result> it = scanner.iterator();

    int count = 0;

    while (it.hasNext()) {

      Result result = it.next();

      Delete delete = new Delete(result.getRow());

      delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));

      ht.delete(delete);

      count++;

    }

    assertEquals("Did not perform correct number of deletes", 3, count);

  }

  private int getNumberOfRows(String keyPrefix, String value, HTable ht)

      throws Exception {

    ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);

    Iterator<Result> scanner = resultScanner.iterator();

    int numberOfResults = 0;

    while (scanner.hasNext()) {

      Result result = scanner.next();

      System.out.println("Got back key: " + Bytes.toString(result.getRow()));

      for (KeyValue kv : result.raw()) {

        System.out.println("kv=" + kv.toString() + ", "

            + Bytes.toString(kv.getValue()));

      }

      numberOfResults++;

    }

    return numberOfResults;

  }

  private ResultScanner buildScanner(String keyPrefix, String value, HTable ht)

      throws IOException {

    // OurFilterList allFilters = new OurFilterList();

    FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);

    allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));

    SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes

        .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes

        .toBytes(value));

    filter.setFilterIfMissing(true);

    allFilters.addFilter(filter);

    // allFilters.addFilter(new

    // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),

    // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));

    Scan scan = new Scan();

    scan.addFamily(Bytes.toBytes("trans-blob"));

    scan.addFamily(Bytes.toBytes("trans-type"));

    scan.addFamily(Bytes.toBytes("trans-date"));

    scan.addFamily(Bytes.toBytes("trans-tags"));

    scan.addFamily(Bytes.toBytes("trans-group"));

    scan.setFilter(allFilters);

    return ht.getScanner(scan);

  }

  private void putRows(HTable ht, int numRows, String value, String key)

      throws IOException {

    for (int i = 0; i < numRows; i++) {

      String row = key + "_" + UUID.randomUUID().toString();

      System.out.println(String.format("Saving row: %s, with value %s", row,

          value));

      Put put = new Put(Bytes.toBytes(row));

      put.add(Bytes.toBytes("trans-blob"), null, Bytes

          .toBytes("value for blob"));

      put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));

      put.add(Bytes.toBytes("trans-date"), null, Bytes

          .toBytes("20090921010101999"));

      put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes

          .toBytes(value));

      put.add(Bytes.toBytes("trans-group"), null, Bytes

          .toBytes("adhocTransactionGroupId"));

      ht.put(put);

    }

  }

  /**

   * Test filters when multiple regions.  It does counts.  Needs eye-balling of

   * logs to ensure that we're not scanning more regions that we're supposed to.

   * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.

   * @throws IOException

   */

  public void testFilterAcrossMutlipleRegions() throws IOException {

    byte [] name = Bytes.toBytes(getName());

    HTable t = createTable(name, FAMILY);

    int rowCount = loadTable(t);

    assertRowCount(t, rowCount);

    // Split the table.  Should split on a reasonable key; 'lqj'

    Map<HRegionInfo, HServerAddress> regions  = splitTable(t);

    assertRowCount(t, rowCount);

    // Get end key of first region.

    byte [] endKey = regions.keySet().iterator().next().getEndKey();

    // Count rows with a filter that stops us before passed 'endKey'.

    // Should be count of rows in first region.

    int endKeyCount = countRows(t, createScanWithRowFilter(endKey));

    assertTrue(endKeyCount < rowCount);

    // How do I know I did not got to second region?  Thats tough.  Can't really

    // do that in client-side region test.  I verified by tracing in debugger.

    // I changed the messages that come out when set to DEBUG so should see

    // when scanner is done. Says "Finished with scanning..." with region name.

    // Check that its finished in right region.

    // New test.  Make it so scan goes into next region by one and then two.

    // Make sure count comes out right.

    byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};

    int plusOneCount = countRows(t, createScanWithRowFilter(key));

    assertEquals(endKeyCount + 1, plusOneCount);

    key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};

    int plusTwoCount = countRows(t, createScanWithRowFilter(key));

    assertEquals(endKeyCount + 2, plusTwoCount);

    // New test.  Make it so I scan one less than endkey.

    key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)};

    int minusOneCount = countRows(t, createScanWithRowFilter(key));

    assertEquals(endKeyCount - 1, minusOneCount);

    // For above test... study logs.  Make sure we do "Finished with scanning.."

    // in first region and that we do not fall into the next region.

   

    key = new byte [] {'a', 'a', 'a'};

    int countBBB = countRows(t,

      createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL));

    assertEquals(1, countBBB);

    int countGreater = countRows(t, createScanWithRowFilter(endKey, null,

      CompareFilter.CompareOp.GREATER_OR_EQUAL));

    // Because started at start of table.

    assertEquals(0, countGreater);

    countGreater = countRows(t, createScanWithRowFilter(endKey, endKey,

      CompareFilter.CompareOp.GREATER_OR_EQUAL));

    assertEquals(rowCount - endKeyCount, countGreater);

  }

 

  /**

   * Load table with rows from 'aaa' to 'zzz'.

   * @param t

   * @return Count of rows loaded.

   * @throws IOException

   */

  private int loadTable(final HTable t) throws IOException {

    // Add data to table.

    byte[] k = new byte[3];

    int rowCount = 0;

    for (byte b1 = 'a'; b1 < 'z'; b1++) {

      for (byte b2 = 'a'; b2 < 'z'; b2++) {

        for (byte b3 = 'a'; b3 < 'z'; b3++) {

          k[0] = b1;

          k[1] = b2;

          k[2] = b3;

          Put put = new Put(k);

          put.add(FAMILY, new byte[0], k);

          t.put(put);

          rowCount++;

        }

      }

    }

    return rowCount;

  }

  /*

   * @param key

   * @return Scan with RowFilter that does LESS than passed key.

   */

  private Scan createScanWithRowFilter(final byte [] key) {

    return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS);

  }

  /*

   * @param key

   * @param op

   * @param startRow

   * @return Scan with RowFilter that does CompareOp op on passed key.

   */

  private Scan createScanWithRowFilter(final byte [] key,

      final byte [] startRow, CompareFilter.CompareOp op) {

    // Make sure key is of some substance... non-null and > than first key.

    assertTrue(key != null && key.length > 0 &&

      Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0);

    LOG.info("Key=" + Bytes.toString(key));

    Scan s = startRow == null? new Scan(): new Scan(startRow);

    Filter f = new RowFilter(op, new BinaryComparator(key));

    f = new WhileMatchFilter(f);

    s.setFilter(f);

    return s;

  }

  /*

   * @param t

   * @param s

   * @return Count of rows in table.

   * @throws IOException

   */

  private int countRows(final HTable t, final Scan s)

  throws IOException {

    // Assert all rows in table.

    ResultScanner scanner = t.getScanner(s);

    int count = 0;

    for (Result result: scanner) {

      count++;

      assertTrue(result.size() > 0);

      // LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow()));

    }

    return count;

  }

  private void assertRowCount(final HTable t, final int expected)

  throws IOException {

    assertEquals(expected, countRows(t, new Scan()));

  }

  /*

   * Split table into multiple regions.

   * @param t Table to split.

   * @return Map of regions to servers.

   * @throws IOException

   */

  private Map<HRegionInfo, HServerAddress> splitTable(final HTable t)

  throws IOException {

    // Split this table in two.

    HBaseAdmin admin = new HBaseAdmin(this.conf);

    admin.split(t.getTableName());

    Map<HRegionInfo, HServerAddress> regions = waitOnSplit(t);

    assertTrue(regions.size() > 1);

    return regions;

  }

  /*

   * Wait on table split.  May return because we waited long enough on the split

   * and it didn't happen.  Caller should check.

   * @param t

   * @return Map of table regions; caller needs to check table actually split.

   */

  private Map<HRegionInfo, HServerAddress> waitOnSplit(final HTable t)

  throws IOException {

    Map<HRegionInfo, HServerAddress> regions = t.getRegionsInfo();

    int originalCount = regions.size();

    for (int i = 0; i < this.conf.getInt("hbase.test.retries", 30); i++) {

      Thread.currentThread();

      try {

        Thread.sleep(this.conf.getInt("hbase.server.thread.wakefrequency", 1000));

      } catch (InterruptedException e) {

        e.printStackTrace();

      }

      regions = t.getRegionsInfo();

      if (regions.size() > originalCount) break;

    }

    return regions;

  }

  public void testSuperSimple() throws Exception {

    byte [] TABLE = Bytes.toBytes(getName());

    HTable ht = createTable(TABLE, FAMILY);

    Put put = new Put(ROW);

    put.add(FAMILY, QUALIFIER, VALUE);

    ht.put(put);

    Scan scan = new Scan();

    scan.addColumn(FAMILY, TABLE);

    ResultScanner scanner = ht.getScanner(scan);

    Result result = scanner.next();

    assertTrue("Expected null result", result == null);

    scanner.close();

    System.out.println("Done.");

  }

  public void testFilters() throws Exception {