/*******************************************************************************
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************************/
package org.apache.drill.exec.store;
import com.google.common.collect.Lists;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.store.parquet.ParquetTypeHelper;
import org.apache.drill.exec.vector.*;
import org.apache.drill.common.util.DecimalUtility;
import parquet.io.api.RecordConsumer;
import parquet.schema.MessageType;
import parquet.io.api.Binary;
import io.netty.buffer.ByteBuf;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.common.types.TypeProtos;
import org.joda.time.DateTimeUtils;
import java.io.IOException;
import java.lang.UnsupportedOperationException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* Abstract implementation of RecordWriter interface which exposes interface:
* {@link #writeHeader(List)}
* {@link #addField(int,String)}
* to output the data in string format instead of implementing addField for each type holder.
*
* This is useful for text format writers such as CSV, TSV etc.
*/
public abstract class ParquetOutputRecordWriter implements RecordWriter {
private RecordConsumer consumer;
private MessageType schema;
public static final long JULIAN_DAY_EPOC = DateTimeUtils.toJulianDayNumber(0);
public void setUp(MessageType schema, RecordConsumer consumer) {
this.schema = schema;
this.consumer = consumer;
}
@Override
public void addNullableTinyIntHolder(int fieldId, NullableTinyIntHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addTinyIntHolder(int fieldId, TinyIntHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedTinyIntHolder(int fieldId, RepeatedTinyIntHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addInteger(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableUInt1Holder(int fieldId, NullableUInt1Holder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addUInt1Holder(int fieldId, UInt1Holder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedUInt1Holder(int fieldId, RepeatedUInt1Holder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addInteger(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableUInt2Holder(int fieldId, NullableUInt2Holder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addUInt2Holder(int fieldId, UInt2Holder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedUInt2Holder(int fieldId, RepeatedUInt2Holder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addInteger(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableSmallIntHolder(int fieldId, NullableSmallIntHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addSmallIntHolder(int fieldId, SmallIntHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedSmallIntHolder(int fieldId, RepeatedSmallIntHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addInteger(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableIntHolder(int fieldId, NullableIntHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addIntHolder(int fieldId, IntHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedIntHolder(int fieldId, RepeatedIntHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addInteger(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableUInt4Holder(int fieldId, NullableUInt4Holder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addUInt4Holder(int fieldId, UInt4Holder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedUInt4Holder(int fieldId, RepeatedUInt4Holder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addInteger(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableFloat4Holder(int fieldId, NullableFloat4Holder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addFloat(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addFloat4Holder(int fieldId, Float4Holder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addFloat(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedFloat4Holder(int fieldId, RepeatedFloat4Holder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addFloat(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableTimeHolder(int fieldId, NullableTimeHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addTimeHolder(int fieldId, TimeHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedTimeHolder(int fieldId, RepeatedTimeHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addInteger(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableIntervalYearHolder(int fieldId, NullableIntervalYearHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addIntervalYearHolder(int fieldId, IntervalYearHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedIntervalYearHolder(int fieldId, RepeatedIntervalYearHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addInteger(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableDecimal9Holder(int fieldId, NullableDecimal9Holder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addDecimal9Holder(int fieldId, Decimal9Holder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addInteger(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedDecimal9Holder(int fieldId, RepeatedDecimal9Holder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addInteger(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableBigIntHolder(int fieldId, NullableBigIntHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addLong(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addBigIntHolder(int fieldId, BigIntHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addLong(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedBigIntHolder(int fieldId, RepeatedBigIntHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addLong(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableUInt8Holder(int fieldId, NullableUInt8Holder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addLong(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addUInt8Holder(int fieldId, UInt8Holder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addLong(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedUInt8Holder(int fieldId, RepeatedUInt8Holder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addLong(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableFloat8Holder(int fieldId, NullableFloat8Holder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addDouble(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addFloat8Holder(int fieldId, Float8Holder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addDouble(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedFloat8Holder(int fieldId, RepeatedFloat8Holder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addDouble(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableDateHolder(int fieldId, NullableDateHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
// convert from internal Drill date format to Julian Day centered around Unix Epoc
consumer.addInteger((int) (DateTimeUtils.toJulianDayNumber(valueHolder.value) + JULIAN_DAY_EPOC));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addDateHolder(int fieldId, DateHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
// convert from internal Drill date format to Julian Day centered around Unix Epoc
consumer.addInteger((int) (DateTimeUtils.toJulianDayNumber(valueHolder.value) + JULIAN_DAY_EPOC));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedDateHolder(int fieldId, RepeatedDateHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addInteger((int) (DateTimeUtils.toJulianDayNumber(valueHolder.vector.getAccessor().get(i)) + JULIAN_DAY_EPOC));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableTimeStampHolder(int fieldId, NullableTimeStampHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addLong(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addTimeStampHolder(int fieldId, TimeStampHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addLong(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedTimeStampHolder(int fieldId, RepeatedTimeStampHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addLong(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableDecimal18Holder(int fieldId, NullableDecimal18Holder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addLong(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addDecimal18Holder(int fieldId, Decimal18Holder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addLong(valueHolder.value);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedDecimal18Holder(int fieldId, RepeatedDecimal18Holder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addLong(valueHolder.vector.getAccessor().get(i));
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableTimeStampTZHolder(int fieldId, NullableTimeStampTZHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
}
@Override
public void addTimeStampTZHolder(int fieldId, TimeStampTZHolder valueHolder) throws IOException {
}
@Override
public void addRepeatedTimeStampTZHolder(int fieldId, RepeatedTimeStampTZHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableIntervalDayHolder(int fieldId, NullableIntervalDayHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
}
@Override
public void addIntervalDayHolder(int fieldId, IntervalDayHolder valueHolder) throws IOException {
}
@Override
public void addRepeatedIntervalDayHolder(int fieldId, RepeatedIntervalDayHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableIntervalHolder(int fieldId, NullableIntervalHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
}
@Override
public void addIntervalHolder(int fieldId, IntervalHolder valueHolder) throws IOException {
}
@Override
public void addRepeatedIntervalHolder(int fieldId, RepeatedIntervalHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableDecimal28DenseHolder(int fieldId, NullableDecimal28DenseHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
}
@Override
public void addDecimal28DenseHolder(int fieldId, Decimal28DenseHolder valueHolder) throws IOException {
}
@Override
public void addRepeatedDecimal28DenseHolder(int fieldId, RepeatedDecimal28DenseHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableDecimal38DenseHolder(int fieldId, NullableDecimal38DenseHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
}
@Override
public void addDecimal38DenseHolder(int fieldId, Decimal38DenseHolder valueHolder) throws IOException {
}
@Override
public void addRepeatedDecimal38DenseHolder(int fieldId, RepeatedDecimal38DenseHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableDecimal38SparseHolder(int fieldId, NullableDecimal38SparseHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
Decimal38SparseVector tempVec = new Decimal38SparseVector(MaterializedField.create("", TypeProtos.MajorType.getDefaultInstance()), new TopLevelAllocator());
tempVec.allocateNew(10);
tempVec.getMutator().setSafe(0, valueHolder);
byte[] bytes = DecimalUtility.getBigDecimalFromSparse(
valueHolder.buffer, valueHolder.start, Decimal38SparseHolder.nDecimalDigits, valueHolder.scale).unscaledValue().toByteArray();
byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL38SPARSE)];
if (valueHolder.getSign()) {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF);
} else {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0);
}
System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length);
consumer.addBinary(Binary.fromByteArray(output));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addDecimal38SparseHolder(int fieldId, Decimal38SparseHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
Decimal38SparseVector tempVec = new Decimal38SparseVector(MaterializedField.create("", TypeProtos.MajorType.getDefaultInstance()), new TopLevelAllocator());
tempVec.allocateNew(10);
tempVec.getMutator().setSafe(0, valueHolder);
byte[] bytes = DecimalUtility.getBigDecimalFromSparse(
valueHolder.buffer, valueHolder.start, Decimal38SparseHolder.nDecimalDigits, valueHolder.scale).unscaledValue().toByteArray();
byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL38SPARSE)];
if (valueHolder.getSign()) {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF);
} else {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0);
}
System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length);
consumer.addBinary(Binary.fromByteArray(output));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedDecimal38SparseHolder(int fieldId, RepeatedDecimal38SparseHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableDecimal28SparseHolder(int fieldId, NullableDecimal28SparseHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
Decimal28SparseVector tempVec = new Decimal28SparseVector(MaterializedField.create("", TypeProtos.MajorType.getDefaultInstance()), new TopLevelAllocator());
tempVec.allocateNew(10);
tempVec.getMutator().setSafe(0, valueHolder);
byte[] bytes = DecimalUtility.getBigDecimalFromSparse(
valueHolder.buffer, valueHolder.start, Decimal28SparseHolder.nDecimalDigits, valueHolder.scale).unscaledValue().toByteArray();
byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL28SPARSE)];
if (valueHolder.getSign()) {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF);
} else {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0);
}
System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length);
consumer.addBinary(Binary.fromByteArray(output));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addDecimal28SparseHolder(int fieldId, Decimal28SparseHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
Decimal28SparseVector tempVec = new Decimal28SparseVector(MaterializedField.create("", TypeProtos.MajorType.getDefaultInstance()), new TopLevelAllocator());
tempVec.allocateNew(10);
tempVec.getMutator().setSafe(0, valueHolder);
byte[] bytes = DecimalUtility.getBigDecimalFromSparse(
valueHolder.buffer, valueHolder.start, Decimal28SparseHolder.nDecimalDigits, valueHolder.scale).unscaledValue().toByteArray();
byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL28SPARSE)];
if (valueHolder.getSign()) {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF);
} else {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0);
}
System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length);
consumer.addBinary(Binary.fromByteArray(output));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedDecimal28SparseHolder(int fieldId, RepeatedDecimal28SparseHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableVarBinaryHolder(int fieldId, NullableVarBinaryHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
ByteBuf buf = valueHolder.buffer;
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBinary(Binary.fromByteBuffer(valueHolder.buffer.nioBuffer(valueHolder.start, valueHolder.end - valueHolder.start)));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addVarBinaryHolder(int fieldId, VarBinaryHolder valueHolder) throws IOException {
ByteBuf buf = valueHolder.buffer;
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBinary(Binary.fromByteBuffer(valueHolder.buffer.nioBuffer(valueHolder.start, valueHolder.end - valueHolder.start)));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedVarBinaryHolder(int fieldId, RepeatedVarBinaryHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
VarBinaryHolder singleHolder = new VarBinaryHolder();
valueHolder.vector.getAccessor().get(i, singleHolder);
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBinary(Binary.fromByteBuffer(singleHolder.buffer.nioBuffer(singleHolder.start, singleHolder.end - singleHolder.start)));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableVarCharHolder(int fieldId, NullableVarCharHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
ByteBuf buf = valueHolder.buffer;
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBinary(Binary.fromByteBuffer(valueHolder.buffer.nioBuffer(valueHolder.start, valueHolder.end - valueHolder.start)));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addVarCharHolder(int fieldId, VarCharHolder valueHolder) throws IOException {
ByteBuf buf = valueHolder.buffer;
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBinary(Binary.fromByteBuffer(valueHolder.buffer.nioBuffer(valueHolder.start, valueHolder.end - valueHolder.start)));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedVarCharHolder(int fieldId, RepeatedVarCharHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
VarCharHolder singleHolder = new VarCharHolder();
valueHolder.vector.getAccessor().get(i, singleHolder);
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBinary(Binary.fromByteBuffer(singleHolder.buffer.nioBuffer(singleHolder.start, singleHolder.end - singleHolder.start)));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableVar16CharHolder(int fieldId, NullableVar16CharHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
ByteBuf buf = valueHolder.buffer;
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBinary(Binary.fromByteBuffer(valueHolder.buffer.nioBuffer(valueHolder.start, valueHolder.end - valueHolder.start)));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addVar16CharHolder(int fieldId, Var16CharHolder valueHolder) throws IOException {
ByteBuf buf = valueHolder.buffer;
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBinary(Binary.fromByteBuffer(valueHolder.buffer.nioBuffer(valueHolder.start, valueHolder.end - valueHolder.start)));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedVar16CharHolder(int fieldId, RepeatedVar16CharHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
Var16CharHolder singleHolder = new Var16CharHolder();
valueHolder.vector.getAccessor().get(i, singleHolder);
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBinary(Binary.fromByteBuffer(singleHolder.buffer.nioBuffer(singleHolder.start, singleHolder.end - singleHolder.start)));
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addNullableBitHolder(int fieldId, NullableBitHolder valueHolder) throws IOException {
if (valueHolder.isSet == 0) {
return;
}
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBoolean(valueHolder.value == 1);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addBitHolder(int fieldId, BitHolder valueHolder) throws IOException {
consumer.startField(schema.getFieldName(fieldId), fieldId);
consumer.addBoolean(valueHolder.value == 1);
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
@Override
public void addRepeatedBitHolder(int fieldId, RepeatedBitHolder valueHolder) throws IOException {
// empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
if (valueHolder.start == valueHolder.end)
return;
consumer.startField(schema.getFieldName(fieldId), fieldId);
for (int i = valueHolder.start; i < valueHolder.end; i++) {
consumer.addBoolean(valueHolder.vector.getAccessor().get(i) == 1);
}
consumer.endField(schema.getFieldName(fieldId), fieldId);
}
}