Examples of org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.Tokenizer

    this.qualifierDeduplicator = USE_HASH_COLUMN_SORTER ? new ByteRangeHashSet()
        : new ByteRangeTreeSet();
    this.timestampEncoder = new LongEncoder();
    this.mvccVersionEncoder = new LongEncoder();
    this.cellTypeEncoder = new CellTypeEncoder();
    this.rowTokenizer = new Tokenizer();
    this.familyTokenizer = new Tokenizer();
    this.qualifierTokenizer = new Tokenizer();
    this.rowWriter = new RowSectionWriter();
    this.familyWriter = new ColumnSectionWriter();
    this.qualifierWriter = new ColumnSectionWriter();


    reset(outputStream, includeMvccVersion);

    List<byte[]> copies = ByteRangeUtils.copyToNewArrays(sortedUniqueColumns);
    Assert.assertTrue(Bytes.isSorted(copies));
    this.blockMeta = new PrefixTreeBlockMeta();
    this.blockMeta.setNumMetaBytes(0);
    this.blockMeta.setNumRowBytes(0);
    this.builder = new Tokenizer();
  }

    List<String> inputs = Lists.newArrayList("a", "ac", "acd", "b");
    testInternal(inputs, 4);
  }


  protected void testInternal(List<String> inputs, int expectedTreeDepth) {
    Tokenizer builder = new Tokenizer();
    for (String s : inputs) {
      ByteRange b = new ByteRange(Bytes.toBytes(s));
      builder.addSorted(b);
    }
    Assert.assertEquals(1, builder.getRoot().getNodeDepth());
    Assert.assertEquals(expectedTreeDepth, builder.getTreeDepth());
  }

    List<byte[]> copies = ByteRangeTool.copyToNewArrays(sortedUniqueColumns);
    Assert.assertTrue(Bytes.isSorted(copies));
    this.blockMeta = new PrefixTreeBlockMeta();
    this.blockMeta.setNumMetaBytes(0);
    this.blockMeta.setNumRowBytes(0);
    this.builder = new Tokenizer();
  }

  private Tokenizer builder;
  private List<byte[]> roundTripped;


  public TestTokenizer(TestTokenizerData sortedByteArrays) {
    this.inputs = sortedByteArrays.getInputs();
    this.builder = new Tokenizer();
    for (byte[] array : inputs) {
      builder.addSorted(new ByteRange(array));
    }
    this.roundTripped = builder.getArrays();
  }

    this.qualifierDeduplicator = USE_HASH_COLUMN_SORTER ? new ByteRangeHashSet()
        : new ByteRangeTreeSet();
    this.timestampEncoder = new LongEncoder();
    this.mvccVersionEncoder = new LongEncoder();
    this.cellTypeEncoder = new CellTypeEncoder();
    this.rowTokenizer = new Tokenizer();
    this.familyTokenizer = new Tokenizer();
    this.qualifierTokenizer = new Tokenizer();
    this.rowWriter = new RowSectionWriter();
    this.familyWriter = new ColumnSectionWriter();
    this.qualifierWriter = new ColumnSectionWriter();


    reset(outputStream, includeMvccVersion);

  private Tokenizer builder;
  private List<byte[]> roundTripped;


  public TestTokenizer(TestTokenizerData sortedByteArrays) {
    this.inputs = sortedByteArrays.getInputs();
    this.builder = new Tokenizer();
    for (byte[] array : inputs) {
      builder.addSorted(new SimpleMutableByteRange(array));
    }
    this.roundTripped = builder.getArrays();
  }

    List<String> inputs = Lists.newArrayList("a", "ac", "acd", "b");
    testInternal(inputs, 4);
  }


  protected void testInternal(List<String> inputs, int expectedTreeDepth) {
    Tokenizer builder = new Tokenizer();
    for (String s : inputs) {
      SimpleMutableByteRange b = new SimpleMutableByteRange(Bytes.toBytes(s));
      builder.addSorted(b);
    }
    Assert.assertEquals(1, builder.getRoot().getNodeDepth());
    Assert.assertEquals(expectedTreeDepth, builder.getTreeDepth());
  }

    this.qualifierDeduplicator = USE_HASH_COLUMN_SORTER ? new ByteRangeHashSet()
        : new ByteRangeTreeSet();
    this.timestampEncoder = new LongEncoder();
    this.mvccVersionEncoder = new LongEncoder();
    this.cellTypeEncoder = new CellTypeEncoder();
    this.rowTokenizer = new Tokenizer();
    this.familyTokenizer = new Tokenizer();
    this.qualifierTokenizer = new Tokenizer();
    this.rowWriter = new RowSectionWriter();
    this.familyWriter = new ColumnSectionWriter();
    this.qualifierWriter = new ColumnSectionWriter();
    initializeTagHelpers();


  protected void initializeTagHelpers() {
    this.tagsRange = new SimpleMutableByteRange();
    this.tagsDeduplicator = USE_HASH_COLUMN_SORTER ? new ByteRangeHashSet()
    : new ByteRangeTreeSet();
    this.tagsTokenizer = new Tokenizer();
    this.tagsWriter = new ColumnSectionWriter();
  }

Related Classes of org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.Tokenizer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.