package com.salesforce.phoenix.filter;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import junit.framework.TestCase;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.salesforce.phoenix.query.KeyRange;
import com.salesforce.phoenix.query.QueryConstants;
import com.salesforce.phoenix.schema.ColumnModifier;
import com.salesforce.phoenix.schema.PDataType;
import com.salesforce.phoenix.schema.PDatum;
import com.salesforce.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
import com.salesforce.phoenix.util.ByteUtil;
//reset()
//filterAllRemaining() -> true indicates scan is over, false, keep going on.
//filterRowKey(byte[],int,int) -> true to drop this row, if false, we will also call
//filterKeyValue(KeyValue) -> true to drop this key/value
//filterRow(List) -> allows direct modification of the final list to be submitted
//filterRow() -> last chance to drop entire row based on the sequence of filterValue() calls. Eg: filter a row if it doesn't contain a specified column.
@RunWith(Parameterized.class)
public class SkipScanFilterTest extends TestCase {
private final SkipScanFilter skipper;
private final List<List<KeyRange>> cnf;
private final List<Expectation> expectations;
public SkipScanFilterTest(List<List<KeyRange>> cnf, int[] widths, List<Expectation> expectations) {
this.expectations = expectations;
this.cnf = cnf;
RowKeySchemaBuilder builder = new RowKeySchemaBuilder(widths.length);
for (final int width : widths) {
builder.addField(
new PDatum() {
@Override
public boolean isNullable() {
return width <= 0;
}
@Override
public PDataType getDataType() {
return width <= 0 ? PDataType.VARCHAR : PDataType.CHAR;
}
@Override
public Integer getByteSize() {
return width <= 0 ? null : width;
}
@Override
public Integer getMaxLength() {
return getByteSize();
}
@Override
public Integer getScale() {
return null;
}
@Override
public ColumnModifier getColumnModifier() {
return null;
}
}, width <= 0, null);
}
skipper = new SkipScanFilter(cnf, builder.build());
}
@Test
public void test() {
System.out.println("CNF: " + cnf + "\n" + "Expectations: " + expectations);
for (Expectation expectation : expectations) {
expectation.examine(skipper);
}
}
@Parameters(name="{0} {1} {2}")
public static Collection<Object> data() {
List<Object> testCases = Lists.newArrayList();
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("dzy"), false, Bytes.toBytes("xyz"), false),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("AA"), true, Bytes.toBytes("AB"), false),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("AA"), true, Bytes.toBytes("AB"), false),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("AA"), true, Bytes.toBytes("AB"), false),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("AA"), true, Bytes.toBytes("AB"), false),
}},
new int[]{3,2,2,2,2},
//new SeekNext("abcABABABAB", "abdAAAAAAAA"),
new SeekNext("defAAABABAB", "dzzAAAAAAAA"),
new Finished("xyyABABABAB"))
);
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.VARCHAR.getKeyRange(Bytes.toBytes("j"), false, Bytes.toBytes("k"), true),
}},
new int[]{0},
new SeekNext(Bytes.toBytes("a"), ByteUtil.nextKey(new byte[] {'j',QueryConstants.SEPARATOR_BYTE})),
new Include("ja"),
new Include("jz"),
new Include("k"),
new Finished("ka")));
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("aaa"), true, Bytes.toBytes("aaa"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("aac"), true, Bytes.toBytes("aad"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), true)
}},
new int[]{3},
new SeekNext("aab", "aac"),
new SeekNext("abb", "abc"),
new Include("abc"),
new Include("abe"),
new Include("def"),
new Finished("deg")));
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("aaa"), true, Bytes.toBytes("aaa"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("abc"), false, Bytes.toBytes("def"), true)
}},
new int[]{3},
new SeekNext("aba", "abd"),
new Include("abe"),
new Include("def"),
new Finished("deg")));
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("aaa"), true, Bytes.toBytes("aaa"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("abc"), false, Bytes.toBytes("def"), false)
}},
new int[]{3},
new SeekNext("aba", "abd"),
new Finished("def"))
);
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("dzy"), false, Bytes.toBytes("xyz"), false),
}},
new int[]{3},
new Include("def"),
new SeekNext("deg", "dzz"),
new Include("eee"),
new Finished("xyz"))
);
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("aaa"), true, Bytes.toBytes("aaa"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("abc"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("def"), true, Bytes.toBytes("def"), true),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("AB"), true, Bytes.toBytes("AX"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("EA"), false, Bytes.toBytes("EZ"), false),
PDataType.CHAR.getKeyRange(Bytes.toBytes("PO"), true, Bytes.toBytes("PP"), false),
}},
new int[]{3,2},
new Include("abcAB"),
new SeekNext("abcAY","abcEB"),
new Include("abcEF"),
new SeekNext("abcPP","defAB"),
new SeekNext("defEZ","defPO"),
new Include("defPO"),
new Finished("defPP")
)
);
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("AB"), true, Bytes.toBytes("AX"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("EA"), false, Bytes.toBytes("EZ"), false),
PDataType.CHAR.getKeyRange(Bytes.toBytes("PO"), true, Bytes.toBytes("PP"), false),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("abc"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("def"), true, Bytes.toBytes("def"), true),
}},
new int[]{2,3},
new Include("ABabc"),
new SeekNext("ABdeg","ACabc"),
new Include("AMabc"),
new SeekNext("AYabc","EBabc"),
new Include("EFabc"),
new SeekNext("EZdef","POabc"),
new SeekNext("POabd","POdef"),
new Include("POdef"),
new Finished("PPabc"))
);
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("PO"), true, Bytes.toBytes("PP"), false),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("def"), true, Bytes.toBytes("def"), true),
}},
new int[]{2,3},
new Include("POdef"),
new Finished("POdeg"))
);
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("PO"), true, Bytes.toBytes("PO"), true),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("def"), true, Bytes.toBytes("def"), true),
}},
new int[]{2,3},
new Include("POdef"))
);
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("AAA"), true, Bytes.toBytes("AAA"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("dzy"), false, Bytes.toBytes("xyz"), false),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("AB"), true, Bytes.toBytes("AX"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("EA"), false, Bytes.toBytes("EZ"), false),
PDataType.CHAR.getKeyRange(Bytes.toBytes("PO"), true, Bytes.toBytes("PP"), false),
}},
new int[]{3,2},
new SeekNext("aaaAA", "abcAB"),
new SeekNext("abcZZ", "abdAB"),
new SeekNext("abdZZ", "abeAB"),
new SeekNext(new byte[]{'d','e','a',(byte)0xFF,(byte)0xFF}, new byte[]{'d','e','b','A','B'}),
new Include("defAB"),
new Include("defAC"),
new Include("defAW"),
new Include("defAX"),
new Include("defEB"),
new Include("defPO"),
new SeekNext("degAB", "dzzAB"),
new Include("dzzAX"),
new Include("dzzEY"),
new SeekNext("dzzEZ", "dzzPO"),
new Include("eeeAB"),
new Include("eeeAC"),
new SeekNext("eeeEA", "eeeEB"),
new Include("eeeEF"),
new SeekNext("eeeEZ","eeePO"),
new Finished("xyzAA"))
);
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("aaa"), true, Bytes.toBytes("aaa"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("dzz"), true, Bytes.toBytes("xyz"), false),
}},
new int[]{3},
new SeekNext("abb", "abc"),
new Include("abc"),
new Include("abe"),
new Finished("xyz"))
);
testCases.addAll(
foreach(new KeyRange[][]{{
PDataType.CHAR.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("dzy"), false, Bytes.toBytes("xyz"), false),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("AB"), true, Bytes.toBytes("AX"), true),
PDataType.CHAR.getKeyRange(Bytes.toBytes("EA"), false, Bytes.toBytes("EZ"), false),
PDataType.CHAR.getKeyRange(Bytes.toBytes("PO"), true, Bytes.toBytes("PP"), false),
},
{
PDataType.CHAR.getKeyRange(Bytes.toBytes("100"), true, Bytes.toBytes("250"), false),
PDataType.CHAR.getKeyRange(Bytes.toBytes("700"), false, Bytes.toBytes("901"), false),
}},
new int[]{3,2,3},
new SeekNext("abcEB700", "abcEB701"),
new Include("abcEB701"),
new SeekNext("dzzAB250", "dzzAB701"),
new Finished("zzzAA000"))
);
// TODO variable length columns
// testCases.addAll(
// foreach(new KeyRange[][]{{
// PDataType.CHAR.getKeyRange(Bytes.toBytes("apple"), true, Bytes.toBytes("lemon"), true),
// PDataType.CHAR.getKeyRange(Bytes.toBytes("pear"), false, Bytes.toBytes("yam"), false),
// },
// {
// PDataType.CHAR.getKeyRange(Bytes.toBytes("AB"), true, Bytes.toBytes("AX"), true),
// PDataType.CHAR.getKeyRange(Bytes.toBytes("EA"), false, Bytes.toBytes("EZ"), false),
// PDataType.CHAR.getKeyRange(Bytes.toBytes("PO"), true, Bytes.toBytes("PP"), false),
// },
// {
// PDataType.CHAR.getKeyRange(Bytes.toBytes("100"), true, Bytes.toBytes("250"), false),
// PDataType.CHAR.getKeyRange(Bytes.toBytes("700"), false, Bytes.toBytes("901"), false),
// }},
// new int[]{3,3})
// );
return testCases;
}
private static Collection<?> foreach(KeyRange[][] ranges, int[] widths, Expectation... expectations) {
List<List<KeyRange>> cnf = Lists.transform(Lists.newArrayList(ranges), ARRAY_TO_LIST);
List<Object> ret = Lists.newArrayList();
ret.add(new Object[] {cnf, widths, Arrays.asList(expectations)} );
return ret;
}
private static final Function<KeyRange[], List<KeyRange>> ARRAY_TO_LIST = new Function<KeyRange[], List<KeyRange>>() {
@Override public List<KeyRange> apply(KeyRange[] input) {
return Lists.newArrayList(input);
}
};
static interface Expectation {
void examine(SkipScanFilter skipper);
}
private static final class SeekNext implements Expectation {
private final byte[] rowkey, hint;
public SeekNext(String rowkey, String hint) {
this.rowkey = Bytes.toBytes(rowkey);
this.hint = Bytes.toBytes(hint);
}
public SeekNext(byte[] rowkey, byte[] hint) {
this.rowkey = rowkey;
this.hint = hint;
}
@Override public void examine(SkipScanFilter skipper) {
KeyValue kv = KeyValue.createFirstOnRow(rowkey);
skipper.reset();
assertFalse(skipper.filterAllRemaining());
assertFalse(skipper.filterRowKey(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength()));
assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, skipper.filterKeyValue(kv));
assertEquals(KeyValue.createFirstOnRow(hint), skipper.getNextKeyHint(kv));
}
@Override public String toString() {
return "rowkey=" + Bytes.toStringBinary(rowkey)+", expected seek next using hint: " + Bytes.toStringBinary(hint);
}
}
private static final class Include implements Expectation {
private final byte[] rowkey;
public Include(String rowkey) {
this.rowkey = Bytes.toBytes(rowkey);
}
@Override public void examine(SkipScanFilter skipper) {
KeyValue kv = KeyValue.createFirstOnRow(rowkey);
skipper.reset();
assertFalse(skipper.filterAllRemaining());
assertFalse(skipper.filterRowKey(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength()));
assertEquals(kv.toString(), ReturnCode.INCLUDE, skipper.filterKeyValue(kv));
}
@Override public String toString() {
return "rowkey=" + Bytes.toStringBinary(rowkey)+", expected include";
}
}
private static final class Finished implements Expectation {
private final byte[] rowkey;
public Finished(String rowkey) {
this.rowkey = Bytes.toBytes(rowkey);
}
@Override public void examine(SkipScanFilter skipper) {
KeyValue kv = KeyValue.createFirstOnRow(rowkey);
skipper.reset();
assertEquals(ReturnCode.NEXT_ROW,skipper.filterKeyValue(kv));
skipper.reset();
assertTrue(skipper.filterAllRemaining());
}
@Override public String toString() {
return "rowkey=" + Bytes.toStringBinary(rowkey)+", expected finished";
}
}
}