/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hcatalog.mapreduce;
import java.io.IOException;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatUtil;
import org.apache.hcatalog.data.DefaultHCatRecord;
import org.apache.hcatalog.data.HCatRecord;
import org.apache.hcatalog.data.LazyHCatRecord;
import org.apache.hcatalog.data.schema.HCatSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** The HCat wrapper for the underlying RecordReader,
* this ensures that the initialize on
* the underlying record reader is done with the underlying split,
* not with HCatSplit.
*/
class HCatRecordReader extends RecordReader<WritableComparable, HCatRecord> {
private static final Logger LOG = LoggerFactory.getLogger(HCatRecordReader.class);
private InputErrorTracker errorTracker;
WritableComparable currentKey;
Writable currentValue;
HCatRecord currentHCatRecord;
/** The underlying record reader to delegate to. */
private org.apache.hadoop.mapred.RecordReader<WritableComparable, Writable> baseRecordReader;
/** The storage handler used */
private final HCatStorageHandler storageHandler;
private Deserializer deserializer;
private Map<String, String> valuesNotInDataCols;
private HCatSchema outputSchema = null;
private HCatSchema dataSchema = null;
/**
* Instantiates a new hcat record reader.
*/
public HCatRecordReader(HCatStorageHandler storageHandler,
Map<String, String> valuesNotInDataCols) {
this.storageHandler = storageHandler;
this.valuesNotInDataCols = valuesNotInDataCols;
}
/* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.RecordReader#initialize(
* org.apache.hadoop.mapreduce.InputSplit,
* org.apache.hadoop.mapreduce.TaskAttemptContext)
*/
@Override
public void initialize(org.apache.hadoop.mapreduce.InputSplit split,
TaskAttemptContext taskContext) throws IOException, InterruptedException {
HCatSplit hcatSplit = InternalUtil.castToHCatSplit(split);
baseRecordReader = createBaseRecordReader(hcatSplit, storageHandler, taskContext);
createDeserializer(hcatSplit, storageHandler, taskContext);
// Pull the output schema out of the TaskAttemptContext
outputSchema = (HCatSchema) HCatUtil.deserialize(
taskContext.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA));
if (outputSchema == null) {
outputSchema = hcatSplit.getTableSchema();
}
// Pull the table schema out of the Split info
// TODO This should be passed in the TaskAttemptContext instead
dataSchema = hcatSplit.getDataSchema();
errorTracker = new InputErrorTracker(taskContext.getConfiguration());
}
private org.apache.hadoop.mapred.RecordReader createBaseRecordReader(HCatSplit hcatSplit,
HCatStorageHandler storageHandler, TaskAttemptContext taskContext) throws IOException {
JobConf jobConf = HCatUtil.getJobConfFromContext(taskContext);
HCatUtil.copyJobPropertiesToJobConf(hcatSplit.getPartitionInfo().getJobProperties(), jobConf);
org.apache.hadoop.mapred.InputFormat inputFormat =
HCatInputFormat.getMapRedInputFormat(jobConf, storageHandler.getInputFormatClass());
return inputFormat.getRecordReader(hcatSplit.getBaseSplit(), jobConf,
InternalUtil.createReporter(taskContext));
}
private void createDeserializer(HCatSplit hcatSplit, HCatStorageHandler storageHandler,
TaskAttemptContext taskContext) throws IOException {
deserializer = ReflectionUtils.newInstance(storageHandler.getSerDeClass(),
taskContext.getConfiguration());
try {
InternalUtil.initializeDeserializer(deserializer, storageHandler.getConf(),
hcatSplit.getPartitionInfo().getTableInfo(),
hcatSplit.getPartitionInfo().getPartitionSchema());
} catch (SerDeException e) {
throw new IOException("Failed initializing deserializer "
+ storageHandler.getSerDeClass().getName(), e);
}
}
/* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.RecordReader#getCurrentKey()
*/
@Override
public WritableComparable getCurrentKey()
throws IOException, InterruptedException {
return currentKey;
}
/* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.RecordReader#getCurrentValue()
*/
@Override
public HCatRecord getCurrentValue() throws IOException, InterruptedException {
return currentHCatRecord;
}
/* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.RecordReader#getProgress()
*/
@Override
public float getProgress() {
try {
return baseRecordReader.getProgress();
} catch (IOException e) {
LOG.warn("Exception in HCatRecord reader", e);
}
return 0.0f; // errored
}
/**
* Check if the wrapped RecordReader has another record, and if so convert it into an
* HCatRecord. We both check for records and convert here so a configurable percent of
* bad records can be tolerated.
*
* @return if there is a next record
* @throws IOException on error
* @throws InterruptedException on error
*/
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
if (currentKey == null) {
currentKey = baseRecordReader.createKey();
currentValue = baseRecordReader.createValue();
}
while (baseRecordReader.next(currentKey, currentValue)) {
HCatRecord r = null;
Throwable t = null;
errorTracker.incRecords();
try {
Object o = deserializer.deserialize(currentValue);
r = new LazyHCatRecord(o, deserializer.getObjectInspector());
} catch (Throwable throwable) {
t = throwable;
}
if (r == null) {
errorTracker.incErrors(t);
continue;
}
DefaultHCatRecord dr = new DefaultHCatRecord(outputSchema.size());
int i = 0;
for (String fieldName : outputSchema.getFieldNames()) {
if (dataSchema.getPosition(fieldName) != null) {
dr.set(i, r.get(fieldName, dataSchema));
} else {
dr.set(i, valuesNotInDataCols.get(fieldName));
}
i++;
}
currentHCatRecord = dr;
return true;
}
return false;
}
/* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.RecordReader#close()
*/
@Override
public void close() throws IOException {
baseRecordReader.close();
}
/**
* Tracks number of of errors in input and throws a Runtime exception
* if the rate of errors crosses a limit.
* <br/>
* The intention is to skip over very rare file corruption or incorrect
* input, but catch programmer errors (incorrect format, or incorrect
* deserializers etc).
*
* This class was largely copied from Elephant-Bird (thanks @rangadi!)
* https://github.com/kevinweil/elephant-bird/blob/master/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoRecordReader.java
*/
static class InputErrorTracker {
long numRecords;
long numErrors;
double errorThreshold; // max fraction of errors allowed
long minErrors; // throw error only after this many errors
InputErrorTracker(Configuration conf) {
errorThreshold = conf.getFloat(HCatConstants.HCAT_INPUT_BAD_RECORD_THRESHOLD_KEY,
HCatConstants.HCAT_INPUT_BAD_RECORD_THRESHOLD_DEFAULT);
minErrors = conf.getLong(HCatConstants.HCAT_INPUT_BAD_RECORD_MIN_KEY,
HCatConstants.HCAT_INPUT_BAD_RECORD_MIN_DEFAULT);
numRecords = 0;
numErrors = 0;
}
void incRecords() {
numRecords++;
}
void incErrors(Throwable cause) {
numErrors++;
if (numErrors > numRecords) {
// incorrect use of this class
throw new RuntimeException("Forgot to invoke incRecords()?");
}
if (cause == null) {
cause = new Exception("Unknown error");
}
if (errorThreshold <= 0) { // no errors are tolerated
throw new RuntimeException("error while reading input records", cause);
}
LOG.warn("Error while reading an input record ("
+ numErrors + " out of " + numRecords + " so far ): ", cause);
double errRate = numErrors / (double) numRecords;
// will always excuse the first error. We can decide if single
// error crosses threshold inside close() if we want to.
if (numErrors >= minErrors && errRate > errorThreshold) {
LOG.error(numErrors + " out of " + numRecords
+ " crosses configured threshold (" + errorThreshold + ")");
throw new RuntimeException("error rate while reading input records crossed threshold", cause);
}
}
}
}