/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hcatalog.common;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hcatalog.data.Pair;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hcatalog.data.schema.HCatSchema;
import org.apache.hcatalog.data.schema.HCatSchemaUtils;
import org.apache.hcatalog.mapreduce.FosterStorageHandler;
import org.apache.hcatalog.mapreduce.HCatOutputFormat;
import org.apache.hcatalog.mapreduce.HCatStorageHandler;
import org.apache.hcatalog.mapreduce.InputJobInfo;
import org.apache.hcatalog.mapreduce.OutputJobInfo;
import org.apache.hcatalog.mapreduce.PartInfo;
import org.apache.hcatalog.mapreduce.StorerInfo;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.security.auth.login.LoginException;
/**
* @deprecated Use/modify {@link org.apache.hive.hcatalog.common.HCatUtil} instead
*/
public class HCatUtil {
private static final Logger LOG = LoggerFactory.getLogger(HCatUtil.class);
private static volatile HiveClientCache hiveClientCache;
private final static int DEFAULT_HIVE_CACHE_EXPIRY_TIME_SECONDS = 2 * 60;
public static boolean checkJobContextIfRunningFromBackend(JobContext j) {
if (j.getConfiguration().get("mapred.task.id", "").equals("") &&
!("true".equals(j.getConfiguration().get("pig.illustrating")))) {
return false;
}
return true;
}
public static String serialize(Serializable obj) throws IOException {
if (obj == null) {
return "";
}
try {
ByteArrayOutputStream serialObj = new ByteArrayOutputStream();
ObjectOutputStream objStream = new ObjectOutputStream(serialObj);
objStream.writeObject(obj);
objStream.close();
return encodeBytes(serialObj.toByteArray());
} catch (Exception e) {
throw new IOException("Serialization error: " + e.getMessage(), e);
}
}
public static Object deserialize(String str) throws IOException {
if (str == null || str.length() == 0) {
return null;
}
try {
ByteArrayInputStream serialObj = new ByteArrayInputStream(
decodeBytes(str));
ObjectInputStream objStream = new ObjectInputStream(serialObj);
return objStream.readObject();
} catch (Exception e) {
throw new IOException("Deserialization error: " + e.getMessage(), e);
}
}
public static String encodeBytes(byte[] bytes) {
StringBuffer strBuf = new StringBuffer();
for (int i = 0; i < bytes.length; i++) {
strBuf.append((char) (((bytes[i] >> 4) & 0xF) + ('a')));
strBuf.append((char) (((bytes[i]) & 0xF) + ('a')));
}
return strBuf.toString();
}
public static byte[] decodeBytes(String str) {
byte[] bytes = new byte[str.length() / 2];
for (int i = 0; i < str.length(); i += 2) {
char c = str.charAt(i);
bytes[i / 2] = (byte) ((c - 'a') << 4);
c = str.charAt(i + 1);
bytes[i / 2] += (c - 'a');
}
return bytes;
}
public static List<HCatFieldSchema> getHCatFieldSchemaList(
FieldSchema... fields) throws HCatException {
List<HCatFieldSchema> result = new ArrayList<HCatFieldSchema>(
fields.length);
for (FieldSchema f : fields) {
result.add(HCatSchemaUtils.getHCatFieldSchema(f));
}
return result;
}
public static List<HCatFieldSchema> getHCatFieldSchemaList(
List<FieldSchema> fields) throws HCatException {
if (fields == null) {
return null;
} else {
List<HCatFieldSchema> result = new ArrayList<HCatFieldSchema>();
for (FieldSchema f : fields) {
result.add(HCatSchemaUtils.getHCatFieldSchema(f));
}
return result;
}
}
public static HCatSchema extractSchema(Table table) throws HCatException {
return new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));
}
public static HCatSchema extractSchema(Partition partition) throws HCatException {
return new HCatSchema(HCatUtil.getHCatFieldSchemaList(partition.getCols()));
}
public static List<FieldSchema> getFieldSchemaList(
List<HCatFieldSchema> hcatFields) {
if (hcatFields == null) {
return null;
} else {
List<FieldSchema> result = new ArrayList<FieldSchema>();
for (HCatFieldSchema f : hcatFields) {
result.add(HCatSchemaUtils.getFieldSchema(f));
}
return result;
}
}
public static Table getTable(HiveMetaStoreClient client, String dbName, String tableName)
throws NoSuchObjectException, TException, MetaException {
return new Table(client.getTable(dbName, tableName));
}
public static HCatSchema getTableSchemaWithPtnCols(Table table) throws IOException {
HCatSchema tableSchema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));
if (table.getPartitionKeys().size() != 0) {
// add partition keys to table schema
// NOTE : this assumes that we do not ever have ptn keys as columns
// inside the table schema as well!
for (FieldSchema fs : table.getPartitionKeys()) {
tableSchema.append(HCatSchemaUtils.getHCatFieldSchema(fs));
}
}
return tableSchema;
}
/**
* return the partition columns from a table instance
*
* @param table the instance to extract partition columns from
* @return HCatSchema instance which contains the partition columns
* @throws IOException
*/
public static HCatSchema getPartitionColumns(Table table) throws IOException {
HCatSchema cols = new HCatSchema(new LinkedList<HCatFieldSchema>());
if (table.getPartitionKeys().size() != 0) {
for (FieldSchema fs : table.getPartitionKeys()) {
cols.append(HCatSchemaUtils.getHCatFieldSchema(fs));
}
}
return cols;
}
/**
* Validate partition schema, checks if the column types match between the
* partition and the existing table schema. Returns the list of columns
* present in the partition but not in the table.
*
* @param table the table
* @param partitionSchema the partition schema
* @return the list of newly added fields
* @throws IOException Signals that an I/O exception has occurred.
*/
public static List<FieldSchema> validatePartitionSchema(Table table,
HCatSchema partitionSchema) throws IOException {
Map<String, FieldSchema> partitionKeyMap = new HashMap<String, FieldSchema>();
for (FieldSchema field : table.getPartitionKeys()) {
partitionKeyMap.put(field.getName().toLowerCase(), field);
}
List<FieldSchema> tableCols = table.getCols();
List<FieldSchema> newFields = new ArrayList<FieldSchema>();
for (int i = 0; i < partitionSchema.getFields().size(); i++) {
FieldSchema field = HCatSchemaUtils.getFieldSchema(partitionSchema
.getFields().get(i));
FieldSchema tableField;
if (i < tableCols.size()) {
tableField = tableCols.get(i);
if (!tableField.getName().equalsIgnoreCase(field.getName())) {
throw new HCatException(
ErrorType.ERROR_SCHEMA_COLUMN_MISMATCH,
"Expected column <" + tableField.getName()
+ "> at position " + (i + 1)
+ ", found column <" + field.getName()
+ ">");
}
} else {
tableField = partitionKeyMap.get(field.getName().toLowerCase());
if (tableField != null) {
throw new HCatException(
ErrorType.ERROR_SCHEMA_PARTITION_KEY, "Key <"
+ field.getName() + ">");
}
}
if (tableField == null) {
// field present in partition but not in table
newFields.add(field);
} else {
// field present in both. validate type has not changed
TypeInfo partitionType = TypeInfoUtils
.getTypeInfoFromTypeString(field.getType());
TypeInfo tableType = TypeInfoUtils
.getTypeInfoFromTypeString(tableField.getType());
if (!partitionType.equals(tableType)) {
throw new HCatException(
ErrorType.ERROR_SCHEMA_TYPE_MISMATCH, "Column <"
+ field.getName() + ">, expected <"
+ tableType.getTypeName() + ">, got <"
+ partitionType.getTypeName() + ">");
}
}
}
return newFields;
}
/**
* Test if the first FsAction is more permissive than the second. This is
* useful in cases where we want to ensure that a file owner has more
* permissions than the group they belong to, for eg. More completely(but
* potentially more cryptically) owner-r >= group-r >= world-r : bitwise
* and-masked with 0444 => 444 >= 440 >= 400 >= 000 owner-w >= group-w >=
* world-w : bitwise and-masked with &0222 => 222 >= 220 >= 200 >= 000
* owner-x >= group-x >= world-x : bitwise and-masked with &0111 => 111 >=
* 110 >= 100 >= 000
*
* @return true if first FsAction is more permissive than the second, false
* if not.
*/
public static boolean validateMorePermissive(FsAction first, FsAction second) {
if ((first == FsAction.ALL) || (second == FsAction.NONE)
|| (first == second)) {
return true;
}
switch (first) {
case READ_EXECUTE:
return ((second == FsAction.READ) || (second == FsAction.EXECUTE));
case READ_WRITE:
return ((second == FsAction.READ) || (second == FsAction.WRITE));
case WRITE_EXECUTE:
return ((second == FsAction.WRITE) || (second == FsAction.EXECUTE));
}
return false;
}
/**
* Ensure that read or write permissions are not granted without also
* granting execute permissions. Essentially, r-- , rw- and -w- are invalid,
* r-x, -wx, rwx, ---, --x are valid
*
* @param perms The FsAction to verify
* @return true if the presence of read or write permission is accompanied
* by execute permissions
*/
public static boolean validateExecuteBitPresentIfReadOrWrite(FsAction perms) {
if ((perms == FsAction.READ) || (perms == FsAction.WRITE)
|| (perms == FsAction.READ_WRITE)) {
return false;
}
return true;
}
public static Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> getJobTrackerDelegationToken(
Configuration conf, String userName) throws Exception {
// LOG.info("getJobTrackerDelegationToken("+conf+","+userName+")");
JobClient jcl = new JobClient(new JobConf(conf, HCatOutputFormat.class));
Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> t = jcl
.getDelegationToken(new Text(userName));
// LOG.info("got "+t);
return t;
// return null;
}
public static Token<? extends AbstractDelegationTokenIdentifier> extractThriftToken(
String tokenStrForm, String tokenSignature) throws MetaException,
TException, IOException {
// LOG.info("extractThriftToken("+tokenStrForm+","+tokenSignature+")");
Token<? extends AbstractDelegationTokenIdentifier> t = new Token<DelegationTokenIdentifier>();
t.decodeFromUrlString(tokenStrForm);
t.setService(new Text(tokenSignature));
// LOG.info("returning "+t);
return t;
}
/**
* Create an instance of a storage handler defined in storerInfo. If one cannot be found
* then FosterStorageHandler is used to encapsulate the InputFormat, OutputFormat and SerDe.
* This StorageHandler assumes the other supplied storage artifacts are for a file-based storage system.
* @param conf job's configuration will be used to configure the Configurable StorageHandler
* @param storerInfo StorerInfo to definining the StorageHandler and InputFormat, OutputFormat and SerDe
* @return storageHandler instance
* @throws IOException
*/
public static HCatStorageHandler getStorageHandler(Configuration conf, StorerInfo storerInfo) throws IOException {
return getStorageHandler(conf,
storerInfo.getStorageHandlerClass(),
storerInfo.getSerdeClass(),
storerInfo.getIfClass(),
storerInfo.getOfClass());
}
public static HCatStorageHandler getStorageHandler(Configuration conf, PartInfo partitionInfo) throws IOException {
return HCatUtil.getStorageHandler(
conf,
partitionInfo.getStorageHandlerClassName(),
partitionInfo.getSerdeClassName(),
partitionInfo.getInputFormatClassName(),
partitionInfo.getOutputFormatClassName());
}
/**
* Create an instance of a storage handler. If storageHandler == null,
* then surrrogate StorageHandler is used to encapsulate the InputFormat, OutputFormat and SerDe.
* This StorageHandler assumes the other supplied storage artifacts are for a file-based storage system.
* @param conf job's configuration will be used to configure the Configurable StorageHandler
* @param storageHandler fully qualified class name of the desired StorageHandle instance
* @param serDe fully qualified class name of the desired SerDe instance
* @param inputFormat fully qualified class name of the desired InputFormat instance
* @param outputFormat fully qualified class name of the desired outputFormat instance
* @return storageHandler instance
* @throws IOException
*/
public static HCatStorageHandler getStorageHandler(Configuration conf,
String storageHandler,
String serDe,
String inputFormat,
String outputFormat)
throws IOException {
if ((storageHandler == null) || (storageHandler.equals(FosterStorageHandler.class.getName()))) {
try {
FosterStorageHandler fosterStorageHandler =
new FosterStorageHandler(inputFormat, outputFormat, serDe);
fosterStorageHandler.setConf(conf);
return fosterStorageHandler;
} catch (ClassNotFoundException e) {
throw new IOException("Failed to load "
+ "foster storage handler", e);
}
}
try {
Class<? extends HCatStorageHandler> handlerClass =
(Class<? extends HCatStorageHandler>) Class
.forName(storageHandler, true, JavaUtils.getClassLoader());
return (HCatStorageHandler) ReflectionUtils.newInstance(
handlerClass, conf);
} catch (ClassNotFoundException e) {
throw new IOException("Error in loading storage handler."
+ e.getMessage(), e);
}
}
public static Pair<String, String> getDbAndTableName(String tableName) throws IOException {
String[] dbTableNametokens = tableName.split("\\.");
if (dbTableNametokens.length == 1) {
return new Pair<String, String>(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
} else if (dbTableNametokens.length == 2) {
return new Pair<String, String>(dbTableNametokens[0], dbTableNametokens[1]);
} else {
throw new IOException("tableName expected in the form "
+ "<databasename>.<table name> or <table name>. Got " + tableName);
}
}
public static Map<String, String>
getInputJobProperties(HCatStorageHandler storageHandler,
InputJobInfo inputJobInfo) {
TableDesc tableDesc = new TableDesc(storageHandler.getSerDeClass(),
storageHandler.getInputFormatClass(),
storageHandler.getOutputFormatClass(),
inputJobInfo.getTableInfo().getStorerInfo().getProperties());
if (tableDesc.getJobProperties() == null) {
tableDesc.setJobProperties(new HashMap<String, String>());
}
Map<String, String> jobProperties = new HashMap<String, String>();
try {
tableDesc.getJobProperties().put(
HCatConstants.HCAT_KEY_JOB_INFO,
HCatUtil.serialize(inputJobInfo));
storageHandler.configureInputJobProperties(tableDesc,
jobProperties);
} catch (IOException e) {
throw new IllegalStateException(
"Failed to configure StorageHandler", e);
}
return jobProperties;
}
@InterfaceAudience.Private
@InterfaceStability.Evolving
public static void
configureOutputStorageHandler(HCatStorageHandler storageHandler,
Configuration conf,
OutputJobInfo outputJobInfo) {
//TODO replace IgnoreKeyTextOutputFormat with a
//HiveOutputFormatWrapper in StorageHandler
TableDesc tableDesc = new TableDesc(storageHandler.getSerDeClass(),
storageHandler.getInputFormatClass(),
IgnoreKeyTextOutputFormat.class,
outputJobInfo.getTableInfo().getStorerInfo().getProperties());
if (tableDesc.getJobProperties() == null)
tableDesc.setJobProperties(new HashMap<String, String>());
for (Map.Entry<String, String> el : conf) {
tableDesc.getJobProperties().put(el.getKey(), el.getValue());
}
Map<String, String> jobProperties = new HashMap<String, String>();
try {
tableDesc.getJobProperties().put(
HCatConstants.HCAT_KEY_OUTPUT_INFO,
HCatUtil.serialize(outputJobInfo));
storageHandler.configureOutputJobProperties(tableDesc,
jobProperties);
for (Map.Entry<String, String> el : jobProperties.entrySet()) {
conf.set(el.getKey(), el.getValue());
}
} catch (IOException e) {
throw new IllegalStateException(
"Failed to configure StorageHandler", e);
}
}
/**
* Replace the contents of dest with the contents of src
* @param src
* @param dest
*/
public static void copyConf(Configuration src, Configuration dest) {
dest.clear();
for (Map.Entry<String, String> el : src) {
dest.set(el.getKey(), el.getValue());
}
}
/**
* Get or create a hive client depending on whether it exits in cache or not
* @param hiveConf The hive configuration
* @return the client
* @throws MetaException When HiveMetaStoreClient couldn't be created
* @throws IOException
*/
public static HiveMetaStoreClient getHiveClient(HiveConf hiveConf)
throws MetaException, IOException {
// Singleton behaviour: create the cache instance if required. The cache needs to be created lazily and
// using the expiry time available in hiveConf.
if (hiveClientCache == null) {
synchronized (HiveMetaStoreClient.class) {
if (hiveClientCache == null) {
hiveClientCache = new HiveClientCache(hiveConf.getInt(HCatConstants.HCAT_HIVE_CLIENT_EXPIRY_TIME,
DEFAULT_HIVE_CACHE_EXPIRY_TIME_SECONDS));
}
}
}
try {
return hiveClientCache.get(hiveConf);
} catch (LoginException e) {
throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e);
}
}
public static void closeHiveClientQuietly(HiveMetaStoreClient client) {
try {
if (client != null)
client.close();
} catch (Exception e) {
LOG.debug("Error closing metastore client. Ignored the error.", e);
}
}
public static HiveConf getHiveConf(Configuration conf)
throws IOException {
HiveConf hiveConf = new HiveConf(conf, HCatUtil.class);
//copy the hive conf into the job conf and restore it
//in the backend context
if (conf.get(HCatConstants.HCAT_KEY_HIVE_CONF) == null) {
conf.set(HCatConstants.HCAT_KEY_HIVE_CONF,
HCatUtil.serialize(hiveConf.getAllProperties()));
} else {
//Copy configuration properties into the hive conf
Properties properties = (Properties) HCatUtil.deserialize(
conf.get(HCatConstants.HCAT_KEY_HIVE_CONF));
for (Map.Entry<Object, Object> prop : properties.entrySet()) {
if (prop.getValue() instanceof String) {
hiveConf.set((String) prop.getKey(), (String) prop.getValue());
} else if (prop.getValue() instanceof Integer) {
hiveConf.setInt((String) prop.getKey(),
(Integer) prop.getValue());
} else if (prop.getValue() instanceof Boolean) {
hiveConf.setBoolean((String) prop.getKey(),
(Boolean) prop.getValue());
} else if (prop.getValue() instanceof Long) {
hiveConf.setLong((String) prop.getKey(), (Long) prop.getValue());
} else if (prop.getValue() instanceof Float) {
hiveConf.setFloat((String) prop.getKey(),
(Float) prop.getValue());
}
}
}
if (conf.get(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE) != null) {
hiveConf.set("hive.metastore.token.signature",
conf.get(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE));
}
return hiveConf;
}
public static JobConf getJobConfFromContext(JobContext jobContext) {
JobConf jobConf;
// we need to convert the jobContext into a jobConf
// 0.18 jobConf (Hive) vs 0.20+ jobContext (HCat)
// begin conversion..
jobConf = new JobConf(jobContext.getConfiguration());
// ..end of conversion
return jobConf;
}
public static void copyJobPropertiesToJobConf(
Map<String, String> jobProperties, JobConf jobConf) {
for (Map.Entry<String, String> entry : jobProperties.entrySet()) {
jobConf.set(entry.getKey(), entry.getValue());
}
}
public static boolean isHadoop23() {
String version = org.apache.hadoop.util.VersionInfo.getVersion();
if (version.matches("\\b0\\.23\\..+\\b")||version.matches("\\b2\\..*"))
return true;
return false;
}
}