/*******************************************************************************
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************************/
package org.apache.drill.exec.store;
import com.google.common.collect.Lists;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.store.EventBasedRecordWriter.FieldConverter;
import org.apache.drill.exec.store.parquet.ParquetTypeHelper;
import org.apache.drill.exec.vector.*;
import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
import parquet.io.api.RecordConsumer;
import parquet.schema.MessageType;
import parquet.io.api.Binary;
import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.common.types.TypeProtos;
import org.joda.time.DateTimeUtils;
import java.io.IOException;
import java.lang.UnsupportedOperationException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* Abstract implementation of RecordWriter interface which exposes interface:
* {@link #writeHeader(List)}
* {@link #addField(int,String)}
* to output the data in string format instead of implementing addField for each type holder.
*
* This is useful for text format writers such as CSV, TSV etc.
*/
public abstract class ParquetOutputRecordWriter extends AbstractRecordWriter implements RecordWriter {
private RecordConsumer consumer;
private MessageType schema;
public static final long JULIAN_DAY_EPOC = DateTimeUtils.toJulianDayNumber(0);
public void setUp(MessageType schema, RecordConsumer consumer) {
this.schema = schema;
this.consumer = consumer;
}
@Override
public FieldConverter getNewNullableTinyIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableTinyIntParquetConverter(fieldId, fieldName, reader);
}
public class NullableTinyIntParquetConverter extends FieldConverter {
private NullableTinyIntHolder holder = new NullableTinyIntHolder();
public NullableTinyIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewTinyIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new TinyIntParquetConverter(fieldId, fieldName, reader);
}
public class TinyIntParquetConverter extends FieldConverter {
private NullableTinyIntHolder holder = new NullableTinyIntHolder();
public TinyIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedTinyIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedTinyIntParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedTinyIntParquetConverter extends FieldConverter {
private NullableTinyIntHolder holder = new NullableTinyIntHolder();
public RepeatedTinyIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addInteger(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableUInt1Converter(int fieldId, String fieldName, FieldReader reader) {
return new NullableUInt1ParquetConverter(fieldId, fieldName, reader);
}
public class NullableUInt1ParquetConverter extends FieldConverter {
private NullableUInt1Holder holder = new NullableUInt1Holder();
public NullableUInt1ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewUInt1Converter(int fieldId, String fieldName, FieldReader reader) {
return new UInt1ParquetConverter(fieldId, fieldName, reader);
}
public class UInt1ParquetConverter extends FieldConverter {
private NullableUInt1Holder holder = new NullableUInt1Holder();
public UInt1ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedUInt1Converter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedUInt1ParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedUInt1ParquetConverter extends FieldConverter {
private NullableUInt1Holder holder = new NullableUInt1Holder();
public RepeatedUInt1ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addInteger(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableUInt2Converter(int fieldId, String fieldName, FieldReader reader) {
return new NullableUInt2ParquetConverter(fieldId, fieldName, reader);
}
public class NullableUInt2ParquetConverter extends FieldConverter {
private NullableUInt2Holder holder = new NullableUInt2Holder();
public NullableUInt2ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewUInt2Converter(int fieldId, String fieldName, FieldReader reader) {
return new UInt2ParquetConverter(fieldId, fieldName, reader);
}
public class UInt2ParquetConverter extends FieldConverter {
private NullableUInt2Holder holder = new NullableUInt2Holder();
public UInt2ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedUInt2Converter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedUInt2ParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedUInt2ParquetConverter extends FieldConverter {
private NullableUInt2Holder holder = new NullableUInt2Holder();
public RepeatedUInt2ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addInteger(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableSmallIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableSmallIntParquetConverter(fieldId, fieldName, reader);
}
public class NullableSmallIntParquetConverter extends FieldConverter {
private NullableSmallIntHolder holder = new NullableSmallIntHolder();
public NullableSmallIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewSmallIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new SmallIntParquetConverter(fieldId, fieldName, reader);
}
public class SmallIntParquetConverter extends FieldConverter {
private NullableSmallIntHolder holder = new NullableSmallIntHolder();
public SmallIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedSmallIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedSmallIntParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedSmallIntParquetConverter extends FieldConverter {
private NullableSmallIntHolder holder = new NullableSmallIntHolder();
public RepeatedSmallIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addInteger(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableIntParquetConverter(fieldId, fieldName, reader);
}
public class NullableIntParquetConverter extends FieldConverter {
private NullableIntHolder holder = new NullableIntHolder();
public NullableIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new IntParquetConverter(fieldId, fieldName, reader);
}
public class IntParquetConverter extends FieldConverter {
private NullableIntHolder holder = new NullableIntHolder();
public IntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedIntParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedIntParquetConverter extends FieldConverter {
private NullableIntHolder holder = new NullableIntHolder();
public RepeatedIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addInteger(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableUInt4Converter(int fieldId, String fieldName, FieldReader reader) {
return new NullableUInt4ParquetConverter(fieldId, fieldName, reader);
}
public class NullableUInt4ParquetConverter extends FieldConverter {
private NullableUInt4Holder holder = new NullableUInt4Holder();
public NullableUInt4ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewUInt4Converter(int fieldId, String fieldName, FieldReader reader) {
return new UInt4ParquetConverter(fieldId, fieldName, reader);
}
public class UInt4ParquetConverter extends FieldConverter {
private NullableUInt4Holder holder = new NullableUInt4Holder();
public UInt4ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedUInt4Converter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedUInt4ParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedUInt4ParquetConverter extends FieldConverter {
private NullableUInt4Holder holder = new NullableUInt4Holder();
public RepeatedUInt4ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addInteger(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableFloat4Converter(int fieldId, String fieldName, FieldReader reader) {
return new NullableFloat4ParquetConverter(fieldId, fieldName, reader);
}
public class NullableFloat4ParquetConverter extends FieldConverter {
private NullableFloat4Holder holder = new NullableFloat4Holder();
public NullableFloat4ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addFloat(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewFloat4Converter(int fieldId, String fieldName, FieldReader reader) {
return new Float4ParquetConverter(fieldId, fieldName, reader);
}
public class Float4ParquetConverter extends FieldConverter {
private NullableFloat4Holder holder = new NullableFloat4Holder();
public Float4ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addFloat(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedFloat4Converter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedFloat4ParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedFloat4ParquetConverter extends FieldConverter {
private NullableFloat4Holder holder = new NullableFloat4Holder();
public RepeatedFloat4ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addFloat(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableTimeConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableTimeParquetConverter(fieldId, fieldName, reader);
}
public class NullableTimeParquetConverter extends FieldConverter {
private NullableTimeHolder holder = new NullableTimeHolder();
public NullableTimeParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewTimeConverter(int fieldId, String fieldName, FieldReader reader) {
return new TimeParquetConverter(fieldId, fieldName, reader);
}
public class TimeParquetConverter extends FieldConverter {
private NullableTimeHolder holder = new NullableTimeHolder();
public TimeParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedTimeConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedTimeParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedTimeParquetConverter extends FieldConverter {
private NullableTimeHolder holder = new NullableTimeHolder();
public RepeatedTimeParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addInteger(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableIntervalYearConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableIntervalYearParquetConverter(fieldId, fieldName, reader);
}
public class NullableIntervalYearParquetConverter extends FieldConverter {
private NullableIntervalYearHolder holder = new NullableIntervalYearHolder();
public NullableIntervalYearParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewIntervalYearConverter(int fieldId, String fieldName, FieldReader reader) {
return new IntervalYearParquetConverter(fieldId, fieldName, reader);
}
public class IntervalYearParquetConverter extends FieldConverter {
private NullableIntervalYearHolder holder = new NullableIntervalYearHolder();
public IntervalYearParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedIntervalYearConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedIntervalYearParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedIntervalYearParquetConverter extends FieldConverter {
private NullableIntervalYearHolder holder = new NullableIntervalYearHolder();
public RepeatedIntervalYearParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addInteger(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableDecimal9Converter(int fieldId, String fieldName, FieldReader reader) {
return new NullableDecimal9ParquetConverter(fieldId, fieldName, reader);
}
public class NullableDecimal9ParquetConverter extends FieldConverter {
private NullableDecimal9Holder holder = new NullableDecimal9Holder();
public NullableDecimal9ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewDecimal9Converter(int fieldId, String fieldName, FieldReader reader) {
return new Decimal9ParquetConverter(fieldId, fieldName, reader);
}
public class Decimal9ParquetConverter extends FieldConverter {
private NullableDecimal9Holder holder = new NullableDecimal9Holder();
public Decimal9ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addInteger(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedDecimal9Converter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedDecimal9ParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedDecimal9ParquetConverter extends FieldConverter {
private NullableDecimal9Holder holder = new NullableDecimal9Holder();
public RepeatedDecimal9ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addInteger(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableBigIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableBigIntParquetConverter(fieldId, fieldName, reader);
}
public class NullableBigIntParquetConverter extends FieldConverter {
private NullableBigIntHolder holder = new NullableBigIntHolder();
public NullableBigIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addLong(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewBigIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new BigIntParquetConverter(fieldId, fieldName, reader);
}
public class BigIntParquetConverter extends FieldConverter {
private NullableBigIntHolder holder = new NullableBigIntHolder();
public BigIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addLong(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedBigIntConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedBigIntParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedBigIntParquetConverter extends FieldConverter {
private NullableBigIntHolder holder = new NullableBigIntHolder();
public RepeatedBigIntParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addLong(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableUInt8Converter(int fieldId, String fieldName, FieldReader reader) {
return new NullableUInt8ParquetConverter(fieldId, fieldName, reader);
}
public class NullableUInt8ParquetConverter extends FieldConverter {
private NullableUInt8Holder holder = new NullableUInt8Holder();
public NullableUInt8ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addLong(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewUInt8Converter(int fieldId, String fieldName, FieldReader reader) {
return new UInt8ParquetConverter(fieldId, fieldName, reader);
}
public class UInt8ParquetConverter extends FieldConverter {
private NullableUInt8Holder holder = new NullableUInt8Holder();
public UInt8ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addLong(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedUInt8Converter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedUInt8ParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedUInt8ParquetConverter extends FieldConverter {
private NullableUInt8Holder holder = new NullableUInt8Holder();
public RepeatedUInt8ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addLong(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableFloat8Converter(int fieldId, String fieldName, FieldReader reader) {
return new NullableFloat8ParquetConverter(fieldId, fieldName, reader);
}
public class NullableFloat8ParquetConverter extends FieldConverter {
private NullableFloat8Holder holder = new NullableFloat8Holder();
public NullableFloat8ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addDouble(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewFloat8Converter(int fieldId, String fieldName, FieldReader reader) {
return new Float8ParquetConverter(fieldId, fieldName, reader);
}
public class Float8ParquetConverter extends FieldConverter {
private NullableFloat8Holder holder = new NullableFloat8Holder();
public Float8ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addDouble(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedFloat8Converter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedFloat8ParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedFloat8ParquetConverter extends FieldConverter {
private NullableFloat8Holder holder = new NullableFloat8Holder();
public RepeatedFloat8ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addDouble(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableDateConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableDateParquetConverter(fieldId, fieldName, reader);
}
public class NullableDateParquetConverter extends FieldConverter {
private NullableDateHolder holder = new NullableDateHolder();
public NullableDateParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
// convert from internal Drill date format to Julian Day centered around Unix Epoc
consumer.addInteger((int) (DateTimeUtils.toJulianDayNumber(holder.value) + JULIAN_DAY_EPOC));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewDateConverter(int fieldId, String fieldName, FieldReader reader) {
return new DateParquetConverter(fieldId, fieldName, reader);
}
public class DateParquetConverter extends FieldConverter {
private NullableDateHolder holder = new NullableDateHolder();
public DateParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
// convert from internal Drill date format to Julian Day centered around Unix Epoc
consumer.addInteger((int) (DateTimeUtils.toJulianDayNumber(holder.value) + JULIAN_DAY_EPOC));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedDateConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedDateParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedDateParquetConverter extends FieldConverter {
private NullableDateHolder holder = new NullableDateHolder();
public RepeatedDateParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addInteger((int) (DateTimeUtils.toJulianDayNumber(holder.value) + JULIAN_DAY_EPOC));
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableTimeStampConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableTimeStampParquetConverter(fieldId, fieldName, reader);
}
public class NullableTimeStampParquetConverter extends FieldConverter {
private NullableTimeStampHolder holder = new NullableTimeStampHolder();
public NullableTimeStampParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addLong(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewTimeStampConverter(int fieldId, String fieldName, FieldReader reader) {
return new TimeStampParquetConverter(fieldId, fieldName, reader);
}
public class TimeStampParquetConverter extends FieldConverter {
private NullableTimeStampHolder holder = new NullableTimeStampHolder();
public TimeStampParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addLong(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedTimeStampConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedTimeStampParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedTimeStampParquetConverter extends FieldConverter {
private NullableTimeStampHolder holder = new NullableTimeStampHolder();
public RepeatedTimeStampParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addLong(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableDecimal18Converter(int fieldId, String fieldName, FieldReader reader) {
return new NullableDecimal18ParquetConverter(fieldId, fieldName, reader);
}
public class NullableDecimal18ParquetConverter extends FieldConverter {
private NullableDecimal18Holder holder = new NullableDecimal18Holder();
public NullableDecimal18ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addLong(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewDecimal18Converter(int fieldId, String fieldName, FieldReader reader) {
return new Decimal18ParquetConverter(fieldId, fieldName, reader);
}
public class Decimal18ParquetConverter extends FieldConverter {
private NullableDecimal18Holder holder = new NullableDecimal18Holder();
public Decimal18ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
consumer.addLong(holder.value);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedDecimal18Converter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedDecimal18ParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedDecimal18ParquetConverter extends FieldConverter {
private NullableDecimal18Holder holder = new NullableDecimal18Holder();
public RepeatedDecimal18ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addLong(holder.value);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableTimeStampTZConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableTimeStampTZParquetConverter(fieldId, fieldName, reader);
}
public class NullableTimeStampTZParquetConverter extends FieldConverter {
private NullableTimeStampTZHolder holder = new NullableTimeStampTZHolder();
public NullableTimeStampTZParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
}
}
@Override
public FieldConverter getNewTimeStampTZConverter(int fieldId, String fieldName, FieldReader reader) {
return new TimeStampTZParquetConverter(fieldId, fieldName, reader);
}
public class TimeStampTZParquetConverter extends FieldConverter {
private NullableTimeStampTZHolder holder = new NullableTimeStampTZHolder();
public TimeStampTZParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
}
}
@Override
public FieldConverter getNewRepeatedTimeStampTZConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedTimeStampTZParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedTimeStampTZParquetConverter extends FieldConverter {
private NullableTimeStampTZHolder holder = new NullableTimeStampTZHolder();
public RepeatedTimeStampTZParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableIntervalDayConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableIntervalDayParquetConverter(fieldId, fieldName, reader);
}
public class NullableIntervalDayParquetConverter extends FieldConverter {
private NullableIntervalDayHolder holder = new NullableIntervalDayHolder();
public NullableIntervalDayParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
}
}
@Override
public FieldConverter getNewIntervalDayConverter(int fieldId, String fieldName, FieldReader reader) {
return new IntervalDayParquetConverter(fieldId, fieldName, reader);
}
public class IntervalDayParquetConverter extends FieldConverter {
private NullableIntervalDayHolder holder = new NullableIntervalDayHolder();
public IntervalDayParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
}
}
@Override
public FieldConverter getNewRepeatedIntervalDayConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedIntervalDayParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedIntervalDayParquetConverter extends FieldConverter {
private NullableIntervalDayHolder holder = new NullableIntervalDayHolder();
public RepeatedIntervalDayParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableIntervalConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableIntervalParquetConverter(fieldId, fieldName, reader);
}
public class NullableIntervalParquetConverter extends FieldConverter {
private NullableIntervalHolder holder = new NullableIntervalHolder();
public NullableIntervalParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
}
}
@Override
public FieldConverter getNewIntervalConverter(int fieldId, String fieldName, FieldReader reader) {
return new IntervalParquetConverter(fieldId, fieldName, reader);
}
public class IntervalParquetConverter extends FieldConverter {
private NullableIntervalHolder holder = new NullableIntervalHolder();
public IntervalParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
}
}
@Override
public FieldConverter getNewRepeatedIntervalConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedIntervalParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedIntervalParquetConverter extends FieldConverter {
private NullableIntervalHolder holder = new NullableIntervalHolder();
public RepeatedIntervalParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableDecimal28DenseConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableDecimal28DenseParquetConverter(fieldId, fieldName, reader);
}
public class NullableDecimal28DenseParquetConverter extends FieldConverter {
private NullableDecimal28DenseHolder holder = new NullableDecimal28DenseHolder();
public NullableDecimal28DenseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
}
}
@Override
public FieldConverter getNewDecimal28DenseConverter(int fieldId, String fieldName, FieldReader reader) {
return new Decimal28DenseParquetConverter(fieldId, fieldName, reader);
}
public class Decimal28DenseParquetConverter extends FieldConverter {
private NullableDecimal28DenseHolder holder = new NullableDecimal28DenseHolder();
public Decimal28DenseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
}
}
@Override
public FieldConverter getNewRepeatedDecimal28DenseConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedDecimal28DenseParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedDecimal28DenseParquetConverter extends FieldConverter {
private NullableDecimal28DenseHolder holder = new NullableDecimal28DenseHolder();
public RepeatedDecimal28DenseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableDecimal38DenseConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableDecimal38DenseParquetConverter(fieldId, fieldName, reader);
}
public class NullableDecimal38DenseParquetConverter extends FieldConverter {
private NullableDecimal38DenseHolder holder = new NullableDecimal38DenseHolder();
public NullableDecimal38DenseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
}
}
@Override
public FieldConverter getNewDecimal38DenseConverter(int fieldId, String fieldName, FieldReader reader) {
return new Decimal38DenseParquetConverter(fieldId, fieldName, reader);
}
public class Decimal38DenseParquetConverter extends FieldConverter {
private NullableDecimal38DenseHolder holder = new NullableDecimal38DenseHolder();
public Decimal38DenseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
}
}
@Override
public FieldConverter getNewRepeatedDecimal38DenseConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedDecimal38DenseParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedDecimal38DenseParquetConverter extends FieldConverter {
private NullableDecimal38DenseHolder holder = new NullableDecimal38DenseHolder();
public RepeatedDecimal38DenseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableDecimal38SparseConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableDecimal38SparseParquetConverter(fieldId, fieldName, reader);
}
public class NullableDecimal38SparseParquetConverter extends FieldConverter {
private NullableDecimal38SparseHolder holder = new NullableDecimal38SparseHolder();
public NullableDecimal38SparseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
byte[] bytes = DecimalUtility.getBigDecimalFromSparse(
holder.buffer, holder.start, Decimal38SparseHolder.nDecimalDigits, holder.scale).unscaledValue().toByteArray();
byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL38SPARSE)];
if (holder.getSign(holder.start, holder.buffer)) {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF);
} else {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0);
}
System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length);
consumer.addBinary(Binary.fromByteArray(output));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewDecimal38SparseConverter(int fieldId, String fieldName, FieldReader reader) {
return new Decimal38SparseParquetConverter(fieldId, fieldName, reader);
}
public class Decimal38SparseParquetConverter extends FieldConverter {
private NullableDecimal38SparseHolder holder = new NullableDecimal38SparseHolder();
public Decimal38SparseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
byte[] bytes = DecimalUtility.getBigDecimalFromSparse(
holder.buffer, holder.start, Decimal38SparseHolder.nDecimalDigits, holder.scale).unscaledValue().toByteArray();
byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL38SPARSE)];
if (holder.getSign(holder.start, holder.buffer)) {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF);
} else {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0);
}
System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length);
consumer.addBinary(Binary.fromByteArray(output));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedDecimal38SparseConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedDecimal38SparseParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedDecimal38SparseParquetConverter extends FieldConverter {
private NullableDecimal38SparseHolder holder = new NullableDecimal38SparseHolder();
public RepeatedDecimal38SparseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableDecimal28SparseConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableDecimal28SparseParquetConverter(fieldId, fieldName, reader);
}
public class NullableDecimal28SparseParquetConverter extends FieldConverter {
private NullableDecimal28SparseHolder holder = new NullableDecimal28SparseHolder();
public NullableDecimal28SparseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
reader.read(holder);
byte[] bytes = DecimalUtility.getBigDecimalFromSparse(
holder.buffer, holder.start, Decimal28SparseHolder.nDecimalDigits, holder.scale).unscaledValue().toByteArray();
byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL28SPARSE)];
if (holder.getSign(holder.start, holder.buffer)) {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF);
} else {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0);
}
System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length);
consumer.addBinary(Binary.fromByteArray(output));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewDecimal28SparseConverter(int fieldId, String fieldName, FieldReader reader) {
return new Decimal28SparseParquetConverter(fieldId, fieldName, reader);
}
public class Decimal28SparseParquetConverter extends FieldConverter {
private NullableDecimal28SparseHolder holder = new NullableDecimal28SparseHolder();
public Decimal28SparseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
reader.read(holder);
byte[] bytes = DecimalUtility.getBigDecimalFromSparse(
holder.buffer, holder.start, Decimal28SparseHolder.nDecimalDigits, holder.scale).unscaledValue().toByteArray();
byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL28SPARSE)];
if (holder.getSign(holder.start, holder.buffer)) {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF);
} else {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0);
}
System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length);
consumer.addBinary(Binary.fromByteArray(output));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedDecimal28SparseConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedDecimal28SparseParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedDecimal28SparseParquetConverter extends FieldConverter {
private NullableDecimal28SparseHolder holder = new NullableDecimal28SparseHolder();
public RepeatedDecimal28SparseParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableVarBinaryConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableVarBinaryParquetConverter(fieldId, fieldName, reader);
}
public class NullableVarBinaryParquetConverter extends FieldConverter {
private NullableVarBinaryHolder holder = new NullableVarBinaryHolder();
public NullableVarBinaryParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
reader.read(holder);
DrillBuf buf = holder.buffer;
consumer.startField(fieldName, fieldId);
consumer.addBinary(Binary.fromByteBuffer(holder.buffer.nioBuffer(holder.start, holder.end - holder.start)));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewVarBinaryConverter(int fieldId, String fieldName, FieldReader reader) {
return new VarBinaryParquetConverter(fieldId, fieldName, reader);
}
public class VarBinaryParquetConverter extends FieldConverter {
private NullableVarBinaryHolder holder = new NullableVarBinaryHolder();
public VarBinaryParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
reader.read(holder);
DrillBuf buf = holder.buffer;
consumer.startField(fieldName, fieldId);
consumer.addBinary(Binary.fromByteBuffer(holder.buffer.nioBuffer(holder.start, holder.end - holder.start)));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedVarBinaryConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedVarBinaryParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedVarBinaryParquetConverter extends FieldConverter {
private NullableVarBinaryHolder holder = new NullableVarBinaryHolder();
public RepeatedVarBinaryParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
//consumer.startField(fieldName, fieldId);
consumer.addBinary(Binary.fromByteBuffer(holder.buffer.nioBuffer(holder.start, holder.end - holder.start)));
//consumer.endField(fieldName, fieldId);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableVarCharConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableVarCharParquetConverter(fieldId, fieldName, reader);
}
public class NullableVarCharParquetConverter extends FieldConverter {
private NullableVarCharHolder holder = new NullableVarCharHolder();
public NullableVarCharParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
reader.read(holder);
DrillBuf buf = holder.buffer;
consumer.startField(fieldName, fieldId);
consumer.addBinary(Binary.fromByteBuffer(holder.buffer.nioBuffer(holder.start, holder.end - holder.start)));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewVarCharConverter(int fieldId, String fieldName, FieldReader reader) {
return new VarCharParquetConverter(fieldId, fieldName, reader);
}
public class VarCharParquetConverter extends FieldConverter {
private NullableVarCharHolder holder = new NullableVarCharHolder();
public VarCharParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
reader.read(holder);
DrillBuf buf = holder.buffer;
consumer.startField(fieldName, fieldId);
consumer.addBinary(Binary.fromByteBuffer(holder.buffer.nioBuffer(holder.start, holder.end - holder.start)));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedVarCharConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedVarCharParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedVarCharParquetConverter extends FieldConverter {
private NullableVarCharHolder holder = new NullableVarCharHolder();
public RepeatedVarCharParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
//consumer.startField(fieldName, fieldId);
consumer.addBinary(Binary.fromByteBuffer(holder.buffer.nioBuffer(holder.start, holder.end - holder.start)));
//consumer.endField(fieldName, fieldId);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableVar16CharConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableVar16CharParquetConverter(fieldId, fieldName, reader);
}
public class NullableVar16CharParquetConverter extends FieldConverter {
private NullableVar16CharHolder holder = new NullableVar16CharHolder();
public NullableVar16CharParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
reader.read(holder);
DrillBuf buf = holder.buffer;
consumer.startField(fieldName, fieldId);
consumer.addBinary(Binary.fromByteBuffer(holder.buffer.nioBuffer(holder.start, holder.end - holder.start)));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewVar16CharConverter(int fieldId, String fieldName, FieldReader reader) {
return new Var16CharParquetConverter(fieldId, fieldName, reader);
}
public class Var16CharParquetConverter extends FieldConverter {
private NullableVar16CharHolder holder = new NullableVar16CharHolder();
public Var16CharParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
reader.read(holder);
DrillBuf buf = holder.buffer;
consumer.startField(fieldName, fieldId);
consumer.addBinary(Binary.fromByteBuffer(holder.buffer.nioBuffer(holder.start, holder.end - holder.start)));
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedVar16CharConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedVar16CharParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedVar16CharParquetConverter extends FieldConverter {
private NullableVar16CharHolder holder = new NullableVar16CharHolder();
public RepeatedVar16CharParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
//consumer.startField(fieldName, fieldId);
consumer.addBinary(Binary.fromByteBuffer(holder.buffer.nioBuffer(holder.start, holder.end - holder.start)));
//consumer.endField(fieldName, fieldId);
}
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewNullableBitConverter(int fieldId, String fieldName, FieldReader reader) {
return new NullableBitParquetConverter(fieldId, fieldName, reader);
}
public class NullableBitParquetConverter extends FieldConverter {
private NullableBitHolder holder = new NullableBitHolder();
public NullableBitParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
if (!reader.isSet()) {
return;
}
consumer.startField(fieldName, fieldId);
consumer.addBoolean(holder.value == 1);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewBitConverter(int fieldId, String fieldName, FieldReader reader) {
return new BitParquetConverter(fieldId, fieldName, reader);
}
public class BitParquetConverter extends FieldConverter {
private NullableBitHolder holder = new NullableBitHolder();
public BitParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
consumer.startField(fieldName, fieldId);
consumer.addBoolean(holder.value == 1);
consumer.endField(fieldName, fieldId);
}
}
@Override
public FieldConverter getNewRepeatedBitConverter(int fieldId, String fieldName, FieldReader reader) {
return new RepeatedBitParquetConverter(fieldId, fieldName, reader);
}
public class RepeatedBitParquetConverter extends FieldConverter {
private NullableBitHolder holder = new NullableBitHolder();
public RepeatedBitParquetConverter(int fieldId, String fieldName, FieldReader reader) {
super(fieldId, fieldName, reader);
}
@Override
public void writeField() throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (reader.size() == 0) {
return;
}
consumer.startField(fieldName, fieldId);
for (int i = 0; i < reader.size(); i++) {
reader.read(i, holder);
consumer.addBoolean(holder.value == 1);
}
consumer.endField(fieldName, fieldId);
}
}
}