* Copyright 2007 The Apache Software Foundation

*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* Basic stand-alone testing of HRegion.

*

* A lot of the meta information for an HRegion now lives inside other

* HRegions or in the HBaseMaster, so only basic testing is possible.

*/

  static final Log LOG = LogFactory.getLog(TestHRegion.class);

  HRegion region = null;

  private final String DIR = "test/build/data/TestHRegion/";

 

  private final int MAX_VERSIONS = 2;

  // Test names

  protected final byte[] tableName = Bytes.toBytes("testtable");;

  protected final byte[] qual1 = Bytes.toBytes("qual1");

  protected final byte[] qual2 = Bytes.toBytes("qual2");

  protected final byte[] qual3 = Bytes.toBytes("qual3");

  protected final byte[] value1 = Bytes.toBytes("value1");

  protected final byte[] value2 = Bytes.toBytes("value2");

  protected final byte [] row = Bytes.toBytes("rowA");

  /**

   * @see org.apache.hadoop.hbase.HBaseTestCase#setUp()

   */

  @Override

  protected void setUp() throws Exception {

    super.setUp();

  }

  //////////////////////////////////////////////////////////////////////////////

  // New tests that doesn't spin up a mini cluster but rather just test the

  // individual code pieces in the HRegion. Putting files locally in

  // /tmp/testtable

  //////////////////////////////////////////////////////////////////////////////

 

  /**

   * An involved filter test.  Has multiple column families and deletes in mix.

   */

  public void testWeirdCacheBehaviour() throws Exception {

    byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");

    byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),

        Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),

        Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };

    initHRegion(TABLE, getName(), FAMILIES);

    String value = "this is the value";

    String value2 = "this is some other value";

    String keyPrefix1 = "prefix1"; // UUID.randomUUID().toString();

    String keyPrefix2 = "prefix2"; // UUID.randomUUID().toString();

    String keyPrefix3 = "prefix3"; // UUID.randomUUID().toString();

    putRows(this.region, 3, value, keyPrefix1);

    putRows(this.region, 3, value, keyPrefix2);

    putRows(this.region, 3, value, keyPrefix3);

    // this.region.flushCommits();

    putRows(this.region, 3, value2, keyPrefix1);

    putRows(this.region, 3, value2, keyPrefix2);

    putRows(this.region, 3, value2, keyPrefix3);

    System.out.println("Checking values for key: " + keyPrefix1);

    assertEquals("Got back incorrect number of rows from scan", 3,

      getNumberOfRows(keyPrefix1, value2, this.region));

    System.out.println("Checking values for key: " + keyPrefix2);

    assertEquals("Got back incorrect number of rows from scan", 3,

      getNumberOfRows(keyPrefix2, value2, this.region));

    System.out.println("Checking values for key: " + keyPrefix3);

    assertEquals("Got back incorrect number of rows from scan", 3,

      getNumberOfRows(keyPrefix3, value2, this.region));

    deleteColumns(this.region, value2, keyPrefix1);

    deleteColumns(this.region, value2, keyPrefix2);

    deleteColumns(this.region, value2, keyPrefix3);

    System.out.println("Starting important checks.....");

    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,

      0, getNumberOfRows(keyPrefix1, value2, this.region));

    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,

      0, getNumberOfRows(keyPrefix2, value2, this.region));

    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,

      0, getNumberOfRows(keyPrefix3, value2, this.region));

  }

  private void deleteColumns(HRegion r, String value, String keyPrefix)

  throws IOException {

    InternalScanner scanner = buildScanner(keyPrefix, value, r);

    int count = 0;

    boolean more = false;

    List<KeyValue> results = new ArrayList<KeyValue>();

    do {

      more = scanner.next(results);

      if (results != null && !results.isEmpty())

        count++;

      else

        break;

      Delete delete = new Delete(results.get(0).getRow());

      delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));

      r.delete(delete, null, false);

      results.clear();

    } while (more);

    assertEquals("Did not perform correct number of deletes", 3, count);

  }

  private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {

    InternalScanner resultScanner = buildScanner(keyPrefix, value, r);

    int numberOfResults = 0;

    List<KeyValue> results = new ArrayList<KeyValue>();

    boolean more = false;

    do {

      more = resultScanner.next(results);

      if (results != null && !results.isEmpty()) numberOfResults++;

      else break;

      for (KeyValue kv: results) {

        System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(kv.getValue()));

      }

      results.clear();

    } while(more);

    return numberOfResults;

  }

  private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)

  throws IOException {

    // Defaults FilterList.Operator.MUST_PASS_ALL.

    FilterList allFilters = new FilterList();

    allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));

    // Only return rows where this column value exists in the row.

    SingleColumnValueFilter filter =

      new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),

        Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));

    filter.setFilterIfMissing(true);

    allFilters.addFilter(filter);

    Scan scan = new Scan();

    scan.addFamily(Bytes.toBytes("trans-blob"));

    scan.addFamily(Bytes.toBytes("trans-type"));

    scan.addFamily(Bytes.toBytes("trans-date"));

    scan.addFamily(Bytes.toBytes("trans-tags"));

    scan.addFamily(Bytes.toBytes("trans-group"));

    scan.setFilter(allFilters);

    return r.getScanner(scan);

  }

  private void putRows(HRegion r, int numRows, String value, String key)

  throws IOException {

    for (int i = 0; i < numRows; i++) {

      String row = key + "_" + i/* UUID.randomUUID().toString() */;

      System.out.println(String.format("Saving row: %s, with value %s", row,

        value));

      Put put = new Put(Bytes.toBytes(row));

      put.add(Bytes.toBytes("trans-blob"), null,

        Bytes.toBytes("value for blob"));

      put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));

      put.add(Bytes.toBytes("trans-date"), null,

        Bytes.toBytes("20090921010101999"));

      put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"),

        Bytes.toBytes(value));

      put.add(Bytes.toBytes("trans-group"), null,

        Bytes.toBytes("adhocTransactionGroupId"));

      r.put(put);

    }

  }

  public void testFamilyWithAndWithoutColon() throws Exception {

    byte [] b = Bytes.toBytes(getName());

    byte [] cf = Bytes.toBytes("cf");

    initHRegion(b, getName(), cf);

    Put p = new Put(b);

    byte [] cfwithcolon = Bytes.toBytes("cf:");

    p.add(cfwithcolon, cfwithcolon, cfwithcolon);

    boolean exception = false;

    try {

      this.region.put(p);

    } catch (NoSuchColumnFamilyException e) {

      exception = true;

    }

    assertTrue(exception);

    // Can I add it using old style call?

    p = new Put(b);

    p.add(cfwithcolon, System.currentTimeMillis(), cfwithcolon);

    this.region.put(p);

  }

  //////////////////////////////////////////////////////////////////////////////

  // checkAndPut tests

  //////////////////////////////////////////////////////////////////////////////

  public void testCheckAndPut_WithEmptyRowValue() throws IOException {

    byte [] tableName = Bytes.toBytes("testtable");

    byte [] row1 = Bytes.toBytes("row1");

    byte [] fam1 = Bytes.toBytes("fam1");

    byte [] qf1  = Bytes.toBytes("qualifier");

    byte [] emptyVal  = new byte[] {};

    byte [] val1  = Bytes.toBytes("value1");

    byte [] val2  = Bytes.toBytes("value2");

    Integer lockId = null;

   

    //Setting up region

    String method = this.getName();

    initHRegion(tableName, method, fam1);

    //Putting data in key

    Put put = new Put(row1);

    put.add(fam1, qf1, val1);

    //checkAndPut with correct value

    boolean res = region.checkAndPut(row1, fam1, qf1, emptyVal, put, lockId,

        true);

    assertTrue(res);

    // not empty anymore

    res = region.checkAndPut(row1, fam1, qf1, emptyVal, put, lockId, true);

    assertFalse(res);

    put = new Put(row1);

    put.add(fam1, qf1, val2);

    //checkAndPut with correct value

    res = region.checkAndPut(row1, fam1, qf1, val1, put, lockId, true);

    assertTrue(res);

  }

 

  public void testCheckAndPut_WithWrongValue() throws IOException{

    byte [] tableName = Bytes.toBytes("testtable");

    byte [] row1 = Bytes.toBytes("row1");

    byte [] fam1 = Bytes.toBytes("fam1");

    byte [] qf1  = Bytes.toBytes("qualifier");

    byte [] val1  = Bytes.toBytes("value1");

    byte [] val2  = Bytes.toBytes("value2");

    Integer lockId = null;

    //Setting up region

    String method = this.getName();

    initHRegion(tableName, method, fam1);

    //Putting data in key

    Put put = new Put(row1);

    put.add(fam1, qf1, val1);

    region.put(put);

   

    //checkAndPut with wrong value

    boolean res = region.checkAndPut(row1, fam1, qf1, val2, put, lockId, true);

    assertEquals(false, res);

  }

  public void testCheckAndPut_WithCorrectValue() throws IOException{

    byte [] tableName = Bytes.toBytes("testtable");

    byte [] row1 = Bytes.toBytes("row1");

    byte [] fam1 = Bytes.toBytes("fam1");

    byte [] qf1  = Bytes.toBytes("qualifier");

    byte [] val1  = Bytes.toBytes("value1");

    Integer lockId = null;

    //Setting up region

    String method = this.getName();

    initHRegion(tableName, method, fam1);

    //Putting data in key

    Put put = new Put(row1);

    put.add(fam1, qf1, val1);

    region.put(put);

   

    //checkAndPut with correct value

    boolean res = region.checkAndPut(row1, fam1, qf1, val1, put, lockId, true);

    assertEquals(true, res);

  }

 

  public void testCheckAndPut_ThatPutWasWritten() throws IOException{

    byte [] tableName = Bytes.toBytes("testtable");

    byte [] row1 = Bytes.toBytes("row1");

    byte [] fam1 = Bytes.toBytes("fam1");

    byte [] fam2 = Bytes.toBytes("fam2");

    byte [] qf1  = Bytes.toBytes("qualifier");

    byte [] val1  = Bytes.toBytes("value1");

    byte [] val2  = Bytes.toBytes("value2");

    Integer lockId = null;

    byte [][] families = {fam1, fam2};

   

    //Setting up region

    String method = this.getName();

    initHRegion(tableName, method, families);

    //Putting data in the key to check

    Put put = new Put(row1);

    put.add(fam1, qf1, val1);

    region.put(put);

   

    //Creating put to add

    long ts = System.currentTimeMillis();

    KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);

    put = new Put(row1);

    put.add(kv);

   

    //checkAndPut with wrong value

    Store store = region.getStore(fam1);

    store.memstore.kvset.size();

   

    boolean res = region.checkAndPut(row1, fam1, qf1, val1, put, lockId, true);

    assertEquals(true, res);

    store.memstore.kvset.size();

   

    Get get = new Get(row1);

    get.addColumn(fam2, qf1);

    KeyValue [] actual = region.get(get, null).raw();

   

    KeyValue [] expected = {kv};

   

    assertEquals(expected.length, actual.length);

    for(int i=0; i<actual.length; i++) {

      assertEquals(expected[i], actual[i]);

    }

   

  }

 

  //////////////////////////////////////////////////////////////////////////////

  // Delete tests

  //////////////////////////////////////////////////////////////////////////////

  public void testDelete_CheckFamily() throws IOException {

    byte [] tableName = Bytes.toBytes("testtable");

    byte [] row1 = Bytes.toBytes("row1");

    byte [] fam1 = Bytes.toBytes("fam1");

    byte [] fam2 = Bytes.toBytes("fam2");

    byte [] fam3 = Bytes.toBytes("fam3");

    byte [] fam4 = Bytes.toBytes("fam4");

    byte[][] families  = {fam1, fam2, fam3};

    //Setting up region

    String method = this.getName();

    initHRegion(tableName, method, families);

    List<KeyValue> kvs  = new ArrayList<KeyValue>();

    kvs.add(new KeyValue(row1, fam4, null, null));

    //testing existing family

    byte [] family = fam2;

    try {

      region.delete(family, kvs, true);

    } catch (Exception e) {

      assertTrue("Family " +new String(family)+ " does not exist", false);

    }

    //testing non existing family

    boolean ok = false;

    family = fam4;

    try {

      region.delete(family, kvs, true);

    } catch (Exception e) {

      ok = true;

    }

    assertEquals("Family " +new String(family)+ " does exist", true, ok);

  }

  public void testDelete_mixed() throws IOException {

    byte [] tableName = Bytes.toBytes("testtable");

    byte [] fam = Bytes.toBytes("info");

    byte [][] families = {fam};

    String method = this.getName();

    initHRegion(tableName, method, families);

    byte [] row = Bytes.toBytes("table_name");

    // column names

    byte [] serverinfo = Bytes.toBytes("serverinfo");

    byte [] splitA = Bytes.toBytes("splitA");

    byte [] splitB = Bytes.toBytes("splitB");

    // add some data:

    Put put = new Put(row);

    put.add(fam, splitA, Bytes.toBytes("reference_A"));

    region.put(put);

    put = new Put(row);

    put.add(fam, splitB, Bytes.toBytes("reference_B"));

    region.put(put);

    put = new Put(row);

    put.add(fam, serverinfo, Bytes.toBytes("ip_address"));

    region.put(put);

    // ok now delete a split:

    Delete delete = new Delete(row);

    delete.deleteColumns(fam, splitA);

    region.delete(delete, null, true);

    // assert some things:

    Get get = new Get(row).addColumn(fam, serverinfo);

    Result result = region.get(get, null);

    assertEquals(1, result.size());

    get = new Get(row).addColumn(fam, splitA);

    result = region.get(get, null);

    assertEquals(0, result.size());

    get = new Get(row).addColumn(fam, splitB);

    result = region.get(get, null);

    assertEquals(1, result.size());

  }

 

  public void testDeleteRowWithFutureTs() throws IOException {

    byte [] tableName = Bytes.toBytes("testtable");

    byte [] fam = Bytes.toBytes("info");

    byte [][] families = {fam};

    String method = this.getName();

    initHRegion(tableName, method, families);

    byte [] row = Bytes.toBytes("table_name");

    // column names

    byte [] serverinfo = Bytes.toBytes("serverinfo");

    // add data in the far future

    Put put = new Put(row);

    put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP-5,Bytes.toBytes("value"));

    region.put(put);

    // now delete something in the present

    Delete delete = new Delete(row);

    region.delete(delete, null, true);

    // make sure we still see our data

    Get get = new Get(row).addColumn(fam, serverinfo);

    Result result = region.get(get, null);

    assertEquals(1, result.size());

   

    // delete the future row

    delete = new Delete(row,HConstants.LATEST_TIMESTAMP-3,null);

    region.delete(delete, null, true);

    // make sure it is gone

    get = new Get(row).addColumn(fam, serverinfo);

    result = region.get(get, null);

    assertEquals(0, result.size());

  }

  public void testScanner_DeleteOneFamilyNotAnother() throws IOException {

    byte [] tableName = Bytes.toBytes("test_table");

    byte [] fam1 = Bytes.toBytes("columnA");

    byte [] fam2 = Bytes.toBytes("columnB");

    initHRegion(tableName, getName(), fam1, fam2);

    byte [] rowA = Bytes.toBytes("rowA");

    byte [] rowB = Bytes.toBytes("rowB");

    byte [] value = Bytes.toBytes("value");

    Delete delete = new Delete(rowA);

    delete.deleteFamily(fam1);

    region.delete(delete, null, true);

    // now create data.

    Put put = new Put(rowA);

    put.add(fam2, null, value);

    region.put(put);

    put = new Put(rowB);

    put.add(fam1, null, value);

    put.add(fam2, null, value);

    region.put(put);

    Scan scan = new Scan();

    scan.addFamily(fam1).addFamily(fam2);

    InternalScanner s = region.getScanner(scan);

    List<KeyValue> results = new ArrayList<KeyValue>();

    s.next(results);

    assertTrue(Bytes.equals(rowA, results.get(0).getRow()));

    results.clear();

    s.next(results);

    assertTrue(Bytes.equals(rowB, results.get(0).getRow()));

  }

  public void testDeleteColumns_PostInsert() throws IOException,

      InterruptedException {

    Delete delete = new Delete(row);

    delete.deleteColumns(fam1, qual1);

    doTestDelete_AndPostInsert(delete);

  }

  public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {

    Delete delete = new Delete(row);

    delete.deleteFamily(fam1);

    doTestDelete_AndPostInsert(delete);

  }

  public void doTestDelete_AndPostInsert(Delete delete)

      throws IOException, InterruptedException {

    initHRegion(tableName, getName(), fam1);

    Put put = new Put(row);

    put.add(fam1, qual1, value1);

    region.put(put);

    Thread.sleep(10);

   

    // now delete the value:

    region.delete(delete, null, true);

    Thread.sleep(10);

    // ok put data:

    put = new Put(row);

    put.add(fam1, qual1, value2);

    region.put(put);

    // ok get:

    Get get = new Get(row);

    get.addColumn(fam1, qual1);

    Result r = region.get(get, null);

    assertEquals(1, r.size());

    assertByteEquals(value2, r.getValue(fam1, qual1));

    // next:

    Scan scan = new Scan(row);

    scan.addColumn(fam1, qual1);

    InternalScanner s = region.getScanner(scan);

    List<KeyValue> results = new ArrayList<KeyValue>();

    assertEquals(false, s.next(results));

    assertEquals(1, results.size());

    KeyValue kv = results.get(0);

    assertByteEquals(value2, kv.getValue());

    assertByteEquals(fam1, kv.getFamily());

    assertByteEquals(qual1, kv.getQualifier());

    assertByteEquals(row, kv.getRow());

  }

 

  public void testDelete_CheckTimestampUpdated()

  throws IOException {

    byte [] row1 = Bytes.toBytes("row1");

    byte [] col1 = Bytes.toBytes("col1");

    byte [] col2 = Bytes.toBytes("col2");

    byte [] col3 = Bytes.toBytes("col3");

   

    //Setting up region

    String method = this.getName();

    initHRegion(tableName, method, fam1);

   

    //Building checkerList

    List<KeyValue> kvs  = new ArrayList<KeyValue>();

    kvs.add(new KeyValue(row1, fam1, col1, null));

    kvs.add(new KeyValue(row1, fam1, col2, null));

    kvs.add(new KeyValue(row1, fam1, col3, null));

    region.delete(fam1, kvs, true);

    // extract the key values out the memstore:

    // This is kinda hacky, but better than nothing...

    long now = System.currentTimeMillis();

    KeyValue firstKv = region.getStore(fam1).memstore.kvset.first();

    assertTrue(firstKv.getTimestamp() <= now);

    now = firstKv.getTimestamp();

    for (KeyValue kv: region.getStore(fam1).memstore.kvset) {

      assertTrue(kv.getTimestamp() <= now);

      now = kv.getTimestamp();

    }

  }

 

  //////////////////////////////////////////////////////////////////////////////

  // Get tests

  //////////////////////////////////////////////////////////////////////////////

  public void testGet_FamilyChecker() throws IOException {

    byte [] tableName = Bytes.toBytes("testtable");

    byte [] row1 = Bytes.toBytes("row1");

    byte [] fam1 = Bytes.toBytes("fam1");

    byte [] fam2 = Bytes.toBytes("False");

    byte [] col1 = Bytes.toBytes("col1");

   

    //Setting up region

    String method = this.getName();

    initHRegion(tableName, method, fam1);

   

    Get get = new Get(row1);

    get.addColumn(fam2, col1);

   

    //Test

    try {

      region.get(get, null);

    } catch (NoSuchColumnFamilyException e){

      assertFalse(false);

      return;

    }

    assertFalse(true);

  }

  public void testGet_Basic() throws IOException {

    byte [] tableName = Bytes.toBytes("testtable");

    byte [] row1 = Bytes.toBytes("row1");

    byte [] fam1 = Bytes.toBytes("fam1");

    byte [] col1 = Bytes.toBytes("col1");

    byte [] col2 = Bytes.toBytes("col2");

    byte [] col3 = Bytes.toBytes("col3");

    byte [] col4 = Bytes.toBytes("col4");

    byte [] col5 = Bytes.toBytes("col5");

   

    //Setting up region

    String method = this.getName();

    initHRegion(tableName, method, fam1);

   

    //Add to memstore

    Put put = new Put(row1);

    put.add(fam1, col1, null);

    put.add(fam1, col2, null);

    put.add(fam1, col3, null);

    put.add(fam1, col4, null);

    put.add(fam1, col5, null);

    region.put(put);

    Get get = new Get(row1);

    get.addColumn(fam1, col2);

    get.addColumn(fam1, col4);

    //Expected result

    KeyValue kv1 = new KeyValue(row1, fam1, col2);

    KeyValue kv2 = new KeyValue(row1, fam1, col4);

    KeyValue [] expected = {kv1, kv2};

    //Test

    Result res = region.get(get, null);

    assertEquals(expected.length, res.size());

    for(int i=0; i<res.size(); i++){

      assertEquals(0,

          Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));

      assertEquals(0,

          Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));

      assertEquals(0,

          Bytes.compareTo(

              expected[i].getQualifier(), res.raw()[i].getQualifier()));

    }

    // Test using a filter on a Get

    Get g = new Get(row1);

    final int count = 2;

    g.setFilter(new ColumnCountGetFilter(count));

    res = region.get(g, null);

    assertEquals(count, res.size());

  }

  public void testGet_Empty() throws IOException {

    byte [] tableName = Bytes.toBytes("emptytable");

    byte [] row = Bytes.toBytes("row");

    byte [] fam = Bytes.toBytes("fam");

   

    String method = this.getName();

    initHRegion(tableName, method, fam);

   

    Get get = new Get(row);

    get.addFamily(fam);

    Result r = region.get(get, null);

   

    assertTrue(r.isEmpty());

  }