Package parquet.column.page

Examples of parquet.column.page.Page

87
88
89
90
91
92
93
94
95
96
97
        (int) parentColumnReader.readPositionInBuffer + length, pageHeader.getCompressed_page_size()));

    BytesInput bytesIn = parentColumnReader.parentReader.getCodecFactoryExposer()
        .decompress(BytesInput.from(f, pageHeader.compressed_page_size), pageHeader.getUncompressed_page_size(),
            parentColumnReader.columnChunkMetaData.getCodec());
    currentPage = new Page(
        bytesIn,
        pageHeader.data_page_header.num_values,
        pageHeader.uncompressed_page_size,
        ParquetStorageEngine.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.repetition_level_encoding),
        ParquetStorageEngine.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.definition_level_encoding),
View Full Code Here
388
389
390
391
392
393
394
395
396
 
 
  private void validateContains(MessageType schema, PageReadStore pages, String[] path, int values, BytesInput bytes)
      throws IOException {
    PageReader pageReader = pages.getPageReader(schema.getColumnDescription(path));
    Page page = pageReader.readPage();
    assertEquals(values, page.getValueCount());
    assertArrayEquals(bytes.toByteArray(), page.getBytes().toByteArray());
  }
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
                in.skip(pageHeader.compressed_page_size);
              }
              break;
            case DATA_PAGE:
              valueReadSoFar += pageHeader.data_page_header.getNum_values();
              return new Page(
                      decompressor.decompress(BytesInput.from(in,pageHeader.compressed_page_size), pageHeader.getUncompressed_page_size()),
                      pageHeader.data_page_header.num_values,
                      pageHeader.uncompressed_page_size,
                      parquetMetadataConverter.getEncoding(pageHeader.data_page_header.repetition_level_encoding),
                      parquetMetadataConverter.getEncoding(pageHeader.data_page_header.definition_level_encoding),
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
    BytesInput bytesIn = parentColumnReader.parentReader.getCodecFactoryExposer()
        .decompress( //
            dataReader.getPageAsBytesInput(pageHeader.compressed_page_size), //
            pageHeader.getUncompressed_page_size(), //
            parentColumnReader.columnChunkMetaData.getCodec());
    currentPage = new Page(
        bytesIn,
        pageHeader.data_page_header.num_values,
        pageHeader.uncompressed_page_size,
        ParquetFormatPlugin.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.repetition_level_encoding),
        ParquetFormatPlugin.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.definition_level_encoding),
View Full Code Here
262
263
264
265
266
267
268
269
270

  private void validateContains(MessageType schema, PageReadStore pages, String[] path, int values, BytesInput bytes)
      throws IOException {
    PageReader pageReader = pages.getPageReader(schema.getColumnDescription(path));
    Page page = pageReader.readPage();
    assertEquals(values, page.getValueCount());
    assertArrayEquals(bytes.toByteArray(), page.getBytes().toByteArray());
  }
View Full Code Here
324
325
326
327
328
329
330
331
332

  private void validateContains(MessageType schema, PageReadStore pages, String[] path, int values, BytesInput bytes)
      throws IOException {
    PageReader pageReader = pages.getPageReader(schema.getColumnDescription(path));
    Page page = pageReader.readPage();
    assertEquals(values, page.getValueCount());
    assertArrayEquals(bytes.toByteArray(), page.getBytes().toByteArray());
  }
View Full Code Here
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
    readRepetitionAndDefinitionLevels();
  }

  private void readPage() {
    if (DEBUG) LOG.debug("loading page");
    Page page = pageReader.readPage();

    this.repetitionLevelColumn = page.getRlEncoding().getValuesReader(path, ValuesType.REPETITION_LEVEL);
    this.definitionLevelColumn = page.getDlEncoding().getValuesReader(path, ValuesType.DEFINITION_LEVEL);
    if (page.getValueEncoding().usesDictionary()) {
      if (dictionary == null) {
        throw new ParquetDecodingException(
            "could not read page " + page + " in col " + path + " as the dictionary was missing for encoding " + page.getValueEncoding());
      }
      this.dataColumn = page.getValueEncoding().getDictionaryBasedValuesReader(path, ValuesType.VALUES, dictionary);
    } else {
      this.dataColumn = page.getValueEncoding().getValuesReader(path, ValuesType.VALUES);
    }
    if (page.getValueEncoding().usesDictionary() && converter.hasDictionarySupport()) {
      bindToDictionary(dictionary);
    } else {
      bind(path.getType());
    }
    this.pageValueCount = page.getValueCount();
    this.endOfPageValueCount = readValues + pageValueCount;
    try {
      byte[] bytes = page.getBytes().toByteArray();
      if (DEBUG) LOG.debug("page size " + bytes.length + " bytes and " + pageValueCount + " records");
      if (DEBUG) LOG.debug("reading repetition levels at 0");
      repetitionLevelColumn.initFromPage(pageValueCount, bytes, 0);
      int next = repetitionLevelColumn.getNextOffset();
      if (DEBUG) LOG.debug("reading definition levels at " + next);
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    @Override
    public Page readPage() {
      if (compressedPages.isEmpty()) {
        return null;
      }
      Page compressedPage = compressedPages.remove(0);
      try {
        return new Page(
            decompressor.decompress(compressedPage.getBytes(), compressedPage.getUncompressedSize()),
            compressedPage.getValueCount(),
            compressedPage.getUncompressedSize(),
            compressedPage.getRlEncoding(),
            compressedPage.getDlEncoding(),
            compressedPage.getValueEncoding());
      } catch (IOException e) {
        throw new RuntimeException(e); // TODO: cleanup
      }
    }
View Full Code Here
430
431
432
433
434
435
436
437
438
439
440
                    parquetMetadataConverter.getEncoding(pageHeader.dictionary_page_header.encoding)
                    );
            break;
          case DATA_PAGE:
            pagesInChunk.add(
                new Page(
                    this.readAsBytesInput(pageHeader.compressed_page_size),
                    pageHeader.data_page_header.num_values,
                    pageHeader.uncompressed_page_size,
                    parquetMetadataConverter.getEncoding(pageHeader.data_page_header.repetition_level_encoding),
                    parquetMetadataConverter.getEncoding(pageHeader.data_page_header.definition_level_encoding),
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
              valueReadSoFar += pageHeader.data_page_header.getNum_values();
              ByteBuf buf = allocator.buffer(pageHeader.compressed_page_size);
              lastPage = buf;
              ByteBuffer buffer = buf.nioBuffer(0, pageHeader.compressed_page_size);
              CompatibilityUtil.getBuf(in, buffer, pageHeader.compressed_page_size);
              return new Page(
                      decompressor.decompress(BytesInput.from(buffer, 0, pageHeader.compressed_page_size), pageHeader.getUncompressed_page_size()),
                      pageHeader.data_page_header.num_values,
                      pageHeader.uncompressed_page_size,
                      parquetMetadataConverter.fromParquetStatistics(pageHeader.data_page_header.statistics, columnDescriptor.getType()),
                      parquetMetadataConverter.getEncoding(pageHeader.data_page_header.repetition_level_encoding),
View Full Code Here
TOP

Related Classes of parquet.column.page.Page

  • org.apache.drill.exec.store.parquet.columnreaders.PageReader
  • org.apache.drill.exec.store.parquet.PageReadStatus
  • org.apache.drill.exec.store.parquet.ParquetRecordReaderTest
  • parquet.column.impl.ColumnReaderImpl
  • parquet.hadoop.ColumnChunkIncReadStore$ColumnChunkIncPageReader
  • parquet.hadoop.ColumnChunkPageReadStore$ColumnChunkPageReader
  • parquet.hadoop.ParquetFileReader$Chunk
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.