* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* Context class for vectorization execution.

* Main role is to map column names to column indices and serves as a

* factory class for building vectorized expressions out of descriptors.

*

*/

  private static final Log LOG = LogFactory.getLog(

      VectorizationContext.class.getName());

  VectorExpressionDescriptor vMap;

  //columnName to column position map

  private final Map<String, Integer> columnMap;

  private final int firstOutputColumnIndex;

  public static final Pattern decimalTypePattern = Pattern.compile("decimal.*",

      Pattern.CASE_INSENSITIVE);

  //Map column number to type

  private final OutputColumnManager ocm;

  // File key is used by operators to retrieve the scratch vectors

  // from mapWork at runtime. The operators that modify the structure of

  // a vector row batch, need to allocate scratch vectors as well. Every

  // operator that creates a new Vectorization context should set a unique

  // fileKey.

  private String fileKey = null;

  // Set of UDF classes for type casting data types in row-mode.

  private static Set<Class<?>> castExpressionUdfs = new HashSet<Class<?>>();

  static {

    castExpressionUdfs.add(GenericUDFToDecimal.class);

    castExpressionUdfs.add(GenericUDFToBinary.class);

    castExpressionUdfs.add(GenericUDFToDate.class);

    castExpressionUdfs.add(GenericUDFToUnixTimeStamp.class);

    castExpressionUdfs.add(GenericUDFToUtcTimestamp.class);

    castExpressionUdfs.add(GenericUDFToChar.class);

    castExpressionUdfs.add(GenericUDFToVarchar.class);

    castExpressionUdfs.add(GenericUDFTimestamp.class);

    castExpressionUdfs.add(UDFToByte.class);

    castExpressionUdfs.add(UDFToBoolean.class);

    castExpressionUdfs.add(UDFToDouble.class);

    castExpressionUdfs.add(UDFToFloat.class);

    castExpressionUdfs.add(UDFToString.class);

    castExpressionUdfs.add(UDFToInteger.class);

    castExpressionUdfs.add(UDFToLong.class);

    castExpressionUdfs.add(UDFToShort.class);

  }

  public VectorizationContext(Map<String, Integer> columnMap,

      int initialOutputCol) {

    this.columnMap = columnMap;

    this.ocm = new OutputColumnManager(initialOutputCol);

    this.firstOutputColumnIndex = initialOutputCol;

    vMap = new VectorExpressionDescriptor();

  }

  /**

   * This constructor inherits the OutputColumnManger and from

   * the 'parent' constructor, therefore this should be used only by operators

   * that don't create a new vectorized row batch. This should be used only by

   * operators that want to modify the columnName map without changing the row batch.

   */

  public VectorizationContext(VectorizationContext parent) {

    this.columnMap = new HashMap<String, Integer>(parent.columnMap);

    this.ocm = parent.ocm;

    this.firstOutputColumnIndex = parent.firstOutputColumnIndex;

    vMap = new VectorExpressionDescriptor();

  }

  public String getFileKey() {

    return fileKey;

  }

  public void setFileKey(String fileKey) {

    this.fileKey = fileKey;

  }

  protected int getInputColumnIndex(String name) {

    if (!columnMap.containsKey(name)) {

      LOG.error(String.format("The column %s is not in the vectorization context column map.", name));

    }

    return columnMap.get(name);

  }

  protected int getInputColumnIndex(ExprNodeColumnDesc colExpr) {

    return columnMap.get(colExpr.getColumn());

  }

    private final int initialOutputCol;

    private int outputColCount = 0;

    protected OutputColumnManager(int initialOutputCol) {

      this.initialOutputCol = initialOutputCol;

    }

    //The complete list of output columns. These should be added to the

    //Vectorized row batch for processing. The index in the row batch is

    //equal to the index in this array plus initialOutputCol.

    //Start with size 100 and double when needed.

    private String [] outputColumnsTypes = new String[100];

    private final Set<Integer> usedOutputColumns = new HashSet<Integer>();

    int allocateOutputColumn(String columnType) {

      if (initialOutputCol < 0) {

        // This is a test

        return 0;

      }

      int relativeCol = allocateOutputColumnInternal(columnType);

      return initialOutputCol + relativeCol;

    }

    private int allocateOutputColumnInternal(String columnType) {

      for (int i = 0; i < outputColCount; i++) {

        // Re-use an existing, available column of the same required type.

        if (usedOutputColumns.contains(i) ||

            !(outputColumnsTypes)[i].equalsIgnoreCase(columnType)) {

          continue;

        }

        //Use i

        usedOutputColumns.add(i);

        return i;

      }

      //Out of allocated columns

      if (outputColCount < outputColumnsTypes.length) {

        int newIndex = outputColCount;

        outputColumnsTypes[outputColCount++] = columnType;

        usedOutputColumns.add(newIndex);

        return newIndex;

      } else {

        //Expand the array

        outputColumnsTypes = Arrays.copyOf(outputColumnsTypes, 2*outputColCount);

        int newIndex = outputColCount;

        outputColumnsTypes[outputColCount++] = columnType;

        usedOutputColumns.add(newIndex);

        return newIndex;

      }

    }

    void freeOutputColumn(int index) {

      if (initialOutputCol < 0) {

        // This is a test

        return;

      }

      int colIndex = index-initialOutputCol;

      if (colIndex >= 0) {

        usedOutputColumns.remove(index-initialOutputCol);

      }

    }

  }

  private VectorExpression getColumnVectorExpression(ExprNodeColumnDesc

      exprDesc, Mode mode) {

    int columnNum = getInputColumnIndex(exprDesc.getColumn());

    VectorExpression expr = null;

    switch (mode) {

      case FILTER:

        //Important: It will come here only if the column is being used as a boolean

        expr = new SelectColumnIsTrue(columnNum);

        break;

      case PROJECTION:

        expr = new IdentityExpression(columnNum, exprDesc.getTypeString());

        break;

    }

    return expr;

  }

  public VectorExpression[] getVectorExpressions(List<ExprNodeDesc> exprNodes) throws HiveException {

    return getVectorExpressions(exprNodes, Mode.PROJECTION);

  }

  public VectorExpression[] getVectorExpressions(List<ExprNodeDesc> exprNodes, Mode mode)

    throws HiveException {

    int i = 0;

    if (null == exprNodes) {

      return new VectorExpression[0];

    }

    VectorExpression[] ret = new VectorExpression[exprNodes.size()];

    for (ExprNodeDesc e : exprNodes) {

      ret[i++] = getVectorExpression(e, mode);

    }

    return ret;

  }

  public VectorExpression getVectorExpression(ExprNodeDesc exprDesc) throws HiveException {

    return getVectorExpression(exprDesc, Mode.PROJECTION);

  }

  /**

   * Returns a vector expression for a given expression

   * description.

   * @param exprDesc, Expression description

   * @param mode

   * @return {@link VectorExpression}

   * @throws HiveException

   */

  public VectorExpression getVectorExpression(ExprNodeDesc exprDesc, Mode mode) throws HiveException {

    VectorExpression ve = null;

    if (exprDesc instanceof ExprNodeColumnDesc) {

      ve = getColumnVectorExpression((ExprNodeColumnDesc) exprDesc, mode);

    } else if (exprDesc instanceof ExprNodeGenericFuncDesc) {

      ExprNodeGenericFuncDesc expr = (ExprNodeGenericFuncDesc) exprDesc;

      if (isCustomUDF(expr) || isNonVectorizedPathUDF(expr)) {

        ve = getCustomUDFExpression(expr);

      } else {

        // Add cast expression if needed. Child expressions of a udf may return different data types

        // and that would require converting their data types to evaluate the udf.

        // For example decimal column added to an integer column would require integer column to be

        // cast to decimal.

        List<ExprNodeDesc> childExpressions = getChildExpressionsWithImplicitCast(expr.getGenericUDF(),

            exprDesc.getChildren(), exprDesc.getTypeInfo());

        ve = getGenericUdfVectorExpression(expr.getGenericUDF(),

            childExpressions, mode, exprDesc.getTypeInfo());

      }

    } else if (exprDesc instanceof ExprNodeConstantDesc) {

      ve = getConstantVectorExpression(((ExprNodeConstantDesc) exprDesc).getValue(), exprDesc.getTypeInfo(),

          mode);

    }

    if (ve == null) {

      throw new HiveException("Could not vectorize expression: "+exprDesc.getName());

    }

    if (LOG.isDebugEnabled()) {

      LOG.debug("Input Expression = " + exprDesc.getTypeInfo()

          + ", Vectorized Expression = " + ve.toString());

    }

    return ve;

  }

  /**

   * Given a udf and its children, return the common type to which the children's type should be

   * cast.

   */

  private TypeInfo getCommonTypeForChildExpressions(GenericUDF genericUdf, List<ExprNodeDesc> children,

      TypeInfo returnType) {

    TypeInfo commonType;

    if (genericUdf instanceof GenericUDFBaseCompare) {

      // Apply comparison rules

      TypeInfo tLeft = children.get(0).getTypeInfo();

      TypeInfo tRight = children.get(1).getTypeInfo();

      commonType = FunctionRegistry.getCommonClassForComparison(tLeft, tRight);

      if (commonType == null) {

        commonType = returnType;

      }

    } else if (genericUdf instanceof GenericUDFIn) {

      // Cast to the type of the first child

      return children.get(0).getTypeInfo();

    } else {

      // The children type should be converted to return type

      commonType = returnType;

    }

    return commonType;

  }

  /**

   * Add a cast expression to the expression tree if needed. The output of child expressions of a given UDF might

   * need a cast if their return type is different from the return type of the UDF.

   *

   * @param genericUDF The given UDF

   * @param children Child expressions of the UDF that might require a cast.

   * @param returnType The return type of the UDF.

   * @return List of child expressions added with cast.

   */

  private List<ExprNodeDesc> getChildExpressionsWithImplicitCast(GenericUDF genericUDF,

      List<ExprNodeDesc> children, TypeInfo returnType) throws HiveException {

    if (isExcludedFromCast(genericUDF)) {

      // No implicit cast needed

      return children;

    }

    if (children == null) {

      return null;

    }

    TypeInfo commonType = getCommonTypeForChildExpressions(genericUDF, children, returnType);

    if (commonType == null) {

      // Couldn't determine common type, don't cast

      return children;

    }

    List<ExprNodeDesc> childrenWithCasts = new ArrayList<ExprNodeDesc>();

    boolean atleastOneCastNeeded = false;

    for (ExprNodeDesc child : children) {

      ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, commonType);

      if (castExpression != null) {

        atleastOneCastNeeded = true;

        childrenWithCasts.add(castExpression);

      } else {

        childrenWithCasts.add(child);

      }

    }

    if (atleastOneCastNeeded) {

      return childrenWithCasts;

    } else {

      return children;

    }

  }

  private boolean isExcludedFromCast(GenericUDF genericUDF) {

    boolean ret = castExpressionUdfs.contains(genericUDF.getClass())

        || (genericUDF instanceof GenericUDFRound) || (genericUDF instanceof GenericUDFBetween);

    if (ret) {

      return ret;

    }

    if (genericUDF instanceof GenericUDFBridge) {

      Class<?> udfClass = ((GenericUDFBridge) genericUDF).getUdfClass();

      return castExpressionUdfs.contains(udfClass)

          || UDFSign.class.isAssignableFrom(udfClass);

    }

    return false;

  }

  /**

   * Creates a DecimalTypeInfo object with appropriate precision and scale for the given

   * inputTypeInfo.

   */

  private TypeInfo updatePrecision(TypeInfo inputTypeInfo, DecimalTypeInfo returnType) {

    if (!(inputTypeInfo instanceof PrimitiveTypeInfo)) {

      return returnType;

    }

    PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) inputTypeInfo;

    int precision = getPrecisionForType(ptinfo);

    int scale = HiveDecimalUtils.getScaleForType(ptinfo);

    return new DecimalTypeInfo(precision, scale);

  }

  /**

   * The GenericUDFs might need their children output to be cast to the given castType.

   * This method returns a cast expression that would achieve the required casting.

   */

  private ExprNodeDesc getImplicitCastExpression(GenericUDF udf, ExprNodeDesc child, TypeInfo castType)

      throws HiveException {

    TypeInfo inputTypeInfo = child.getTypeInfo();

    String inputTypeString = inputTypeInfo.getTypeName();

    String castTypeString = castType.getTypeName();

    if (inputTypeString.equals(castTypeString)) {

      // Nothing to be done

      return null;

    }

    boolean inputTypeDecimal = false;

    boolean castTypeDecimal = false;

    if (decimalTypePattern.matcher(inputTypeString).matches()) {

      inputTypeDecimal = true;

    }

    if (decimalTypePattern.matcher(castTypeString).matches()) {

      castTypeDecimal = true;

    }

    if (castTypeDecimal && !inputTypeDecimal) {

      // Cast the input to decimal

      // If castType is decimal, try not to lose precision for numeric types.

      castType = updatePrecision(inputTypeInfo, (DecimalTypeInfo) castType);

      GenericUDFToDecimal castToDecimalUDF = new GenericUDFToDecimal();

      castToDecimalUDF.setTypeInfo(castType);

      List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();

      children.add(child);

      ExprNodeDesc desc = new ExprNodeGenericFuncDesc(castType, castToDecimalUDF, children);

      return desc;

    } else if (!castTypeDecimal && inputTypeDecimal) {

      // Cast decimal input to returnType

      GenericUDF genericUdf = getGenericUDFForCast(castType);

      List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();

      children.add(child);

      ExprNodeDesc desc = new ExprNodeGenericFuncDesc(castType, genericUdf, children);

      return desc;

    } else {

      // Casts to exact types including long to double etc. are needed in some special cases.

      if (udf instanceof GenericUDFCoalesce) {

        GenericUDF genericUdf = getGenericUDFForCast(castType);

        List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();

        children.add(child);

        ExprNodeDesc desc = new ExprNodeGenericFuncDesc(castType, genericUdf, children);

        return desc;

      }

    }

    return null;

  }

 

  private int getPrecisionForType(PrimitiveTypeInfo typeInfo) {

    if (isFloatFamily(typeInfo.getTypeName())) {

      return HiveDecimal.MAX_PRECISION;

    }

    return HiveDecimalUtils.getPrecisionForType(typeInfo);

  }

  private GenericUDF getGenericUDFForCast(TypeInfo castType) throws HiveException {

    UDF udfClass = null;

    GenericUDF genericUdf = null;

    switch (((PrimitiveTypeInfo) castType).getPrimitiveCategory()) {

      case BYTE:

        udfClass = new UDFToByte();

        break;

      case SHORT:

        udfClass = new UDFToShort();

        break;

      case INT:

        udfClass = new UDFToInteger();

        break;

      case LONG:

        udfClass = new UDFToLong();

        break;

      case FLOAT:

        udfClass = new UDFToFloat();

        break;

      case DOUBLE:

        udfClass = new UDFToDouble();

        break;

      case STRING:

        udfClass = new UDFToString();

        break;

      case BOOLEAN:

        udfClass = new UDFToBoolean();

        break;

      case DATE:

        genericUdf = new GenericUDFToDate();

        break;

      case TIMESTAMP:

        genericUdf = new GenericUDFToUnixTimeStamp();

        break;

      case BINARY:

        genericUdf = new GenericUDFToBinary();

        break;

      case DECIMAL:

        genericUdf = new GenericUDFToDecimal();

        break;

    }

    if (genericUdf == null) {

      if (udfClass == null) {

        throw new HiveException("Could not add implicit cast for type "+castType.getTypeName());

      }

      genericUdf = new GenericUDFBridge();

      ((GenericUDFBridge) genericUdf).setUdfClassName(udfClass.getClass().getName());

    }

    if (genericUdf instanceof SettableUDF) {

  ((SettableUDF)genericUdf).setTypeInfo(castType);

    }   

    return genericUdf;

  }

  /* Return true if this is one of a small set of functions for which

   * it is significantly easier to use the old code path in vectorized

   * mode instead of implementing a new, optimized VectorExpression.

   *

   * Depending on performance requirements and frequency of use, these

   * may be implemented in the future with an optimized VectorExpression.

   */

  public static boolean isNonVectorizedPathUDF(ExprNodeGenericFuncDesc expr) {

    GenericUDF gudf = expr.getGenericUDF();

    if (gudf instanceof GenericUDFBridge) {

      GenericUDFBridge bridge = (GenericUDFBridge) gudf;

      Class<? extends UDF> udfClass = bridge.getUdfClass();

      if (udfClass.equals(UDFHex.class)

          || udfClass.equals(UDFConv.class)

          || isCastToIntFamily(udfClass) && arg0Type(expr).equals("string")

          || isCastToFloatFamily(udfClass) && arg0Type(expr).equals("string")

          || udfClass.equals(UDFToString.class) &&

               (arg0Type(expr).equals("timestamp")

                   || arg0Type(expr).equals("double")

                   || arg0Type(expr).equals("float"))) {

        return true;

      }

    } else if ((gudf instanceof GenericUDFTimestamp && arg0Type(expr).equals("string"))

            /* GenericUDFCase and GenericUDFWhen are implemented with the UDF Adaptor because

             * of their complexity and generality. In the future, variations of these

             * can be optimized to run faster for the vectorized code path. For example,

             * CASE col WHEN 1 then "one" WHEN 2 THEN "two" ELSE "other" END

             * is an example of a GenericUDFCase that has all constant arguments

             * except for the first argument. This is probably a common case and a

             * good candidate for a fast, special-purpose VectorExpression. Then

             * the UDF Adaptor code path could be used as a catch-all for

             * non-optimized general cases.

             */

            || gudf instanceof GenericUDFCase

            || gudf instanceof GenericUDFWhen) {

      return true;

    }

    return false;

  }

  public static boolean isCastToIntFamily(Class<? extends UDF> udfClass) {

    return udfClass.equals(UDFToByte.class)

        || udfClass.equals(UDFToShort.class)

        || udfClass.equals(UDFToInteger.class)

        || udfClass.equals(UDFToLong.class);

    // Boolean is purposely excluded.

  }

  public static boolean isCastToFloatFamily(Class<? extends UDF> udfClass) {

    return udfClass.equals(UDFToDouble.class)

        || udfClass.equals(UDFToFloat.class);

  }

  // Return the type string of the first argument (argument 0).

  public static String arg0Type(ExprNodeGenericFuncDesc expr) {

    String type = expr.getChildren().get(0).getTypeString();

    return type;

  }

  // Return true if this is a custom UDF or custom GenericUDF.

  // This is for use only in the planner. It will fail in a task.

  public static boolean isCustomUDF(ExprNodeGenericFuncDesc expr) {

    String udfName = expr.getFuncText();

    if (udfName == null) {

      return false;

    }

    FunctionInfo funcInfo = FunctionRegistry.getFunctionInfo(udfName);

    if (funcInfo == null) {

      return false;

    }

    boolean isNativeFunc = funcInfo.isNative();

    return !isNativeFunc;

  }

  /**

   * Handles only the special case of unary operators on a constant.

   * @param exprDesc

   * @return The same expression if no folding done, else return the constant

   *         expression.

   * @throws HiveException

   */

  ExprNodeDesc foldConstantsForUnaryExpression(ExprNodeDesc exprDesc) throws HiveException {

    if (!(exprDesc instanceof ExprNodeGenericFuncDesc)) {

      return exprDesc;

    }

   

    if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) ) {

      return exprDesc;

    }

    ExprNodeConstantDesc foldedChild = null;

    if (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc)) {

      // try recursive folding

      ExprNodeDesc expr = foldConstantsForUnaryExpression(exprDesc.getChildren().get(0));

      if (expr instanceof ExprNodeConstantDesc) {

        foldedChild = (ExprNodeConstantDesc) expr;

      }

    } else {

      foldedChild = (ExprNodeConstantDesc) exprDesc.getChildren().get(0);

    }

    if (foldedChild == null) {

      return exprDesc;

    }

    ObjectInspector childoi = foldedChild.getWritableObjectInspector();

    GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF();

    if (gudf instanceof GenericUDFOPNegative || gudf instanceof GenericUDFOPPositive

        || castExpressionUdfs.contains(gudf.getClass())

        || ((gudf instanceof GenericUDFBridge)

            && castExpressionUdfs.contains(((GenericUDFBridge) gudf).getUdfClass()))) {

      ExprNodeEvaluator<?> evaluator = ExprNodeEvaluatorFactory.get(exprDesc);

      ObjectInspector output = evaluator.initialize(childoi);

      Object constant = evaluator.evaluate(null);

      Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output); 

      return new ExprNodeConstantDesc(exprDesc.getTypeInfo(), java);

     }

    return exprDesc;

  }

  /* Fold simple unary expressions in all members of the input list and return new list

   * containing results.

   */

  private List<ExprNodeDesc> foldConstantsForUnaryExprs(List<ExprNodeDesc> childExpr)

      throws HiveException {

    List<ExprNodeDesc> constantFoldedChildren = new ArrayList<ExprNodeDesc>();

    if (childExpr != null) {

      for (ExprNodeDesc expr : childExpr) {

        expr = this.foldConstantsForUnaryExpression(expr);

        constantFoldedChildren.add(expr);

      }

    }

    return constantFoldedChildren;

  }

  private VectorExpression getConstantVectorExpression(Object constantValue, TypeInfo typeInfo,

      Mode mode) throws HiveException {

    String type = typeInfo.getTypeName();

    String colVectorType = getNormalizedTypeName(type);

    int outCol = -1;

    if (mode == Mode.PROJECTION) {

      outCol = ocm.allocateOutputColumn(colVectorType);

    }

    if (decimalTypePattern.matcher(type).matches()) {

      VectorExpression ve = new ConstantVectorExpression(outCol, (Decimal128) constantValue);

      ve.setOutputType(typeInfo.getTypeName());

      return ve;

    } else if (type.equalsIgnoreCase("long") || type.equalsIgnoreCase("int") ||

        type.equalsIgnoreCase("short") || type.equalsIgnoreCase("byte")) {

      return new ConstantVectorExpression(outCol,

          ((Number) constantValue).longValue());

    } else if (type.equalsIgnoreCase("double") || type.equalsIgnoreCase("float")) {

      return new ConstantVectorExpression(outCol, ((Number) constantValue).doubleValue());

    } else if (type.equalsIgnoreCase("string")) {

      return new ConstantVectorExpression(outCol, ((String) constantValue).getBytes());

    } else if (type.equalsIgnoreCase("boolean")) {

      if (mode == Mode.FILTER) {

        if (((Boolean) constantValue).booleanValue()) {

          return new FilterConstantBooleanVectorExpression(1);

        } else {

          return new FilterConstantBooleanVectorExpression(0);

        }

      } else {

        if (((Boolean) constantValue).booleanValue()) {

          return new ConstantVectorExpression(outCol, 1);

        } else {

          return new ConstantVectorExpression(outCol, 0);

        }

      }

    }

    throw new HiveException("Unsupported constant type: "+type.toString());

  }

  /**

   * Used as a fast path for operations that don't modify their input, like unary +

   * and casting boolean to long. IdentityExpression and its children are always

   * projections.

   */

  private VectorExpression getIdentityExpression(List<ExprNodeDesc> childExprList)

      throws HiveException {

    ExprNodeDesc childExpr = childExprList.get(0);

    int inputCol;

    String colType;

    VectorExpression v1 = null;

    if (childExpr instanceof ExprNodeGenericFuncDesc) {

      v1 = getVectorExpression(childExpr);

      inputCol = v1.getOutputColumn();

      colType = v1.getOutputType();

    } else if (childExpr instanceof ExprNodeColumnDesc) {

      ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) childExpr;

      inputCol = getInputColumnIndex(colDesc.getColumn());

      colType = colDesc.getTypeString();

    } else {

      throw new HiveException("Expression not supported: "+childExpr);

    }

    VectorExpression expr = new IdentityExpression(inputCol, colType);

    if (v1 != null) {

      expr.setChildExpressions(new VectorExpression [] {v1});

    }

    return expr;

  }

  private VectorExpression getVectorExpressionForUdf(Class<?> udf, List<ExprNodeDesc> childExpr, Mode mode,

      TypeInfo returnType) throws HiveException {

    int numChildren = (childExpr == null) ? 0 : childExpr.size();

    if (numChildren > VectorExpressionDescriptor.MAX_NUM_ARGUMENTS) {

      return null;

    }

    VectorExpressionDescriptor.Builder builder = new VectorExpressionDescriptor.Builder();

    builder.setNumArguments(numChildren);

    builder.setMode(mode);

    for (int i = 0; i < numChildren; i++) {

      ExprNodeDesc child = childExpr.get(i);

      builder.setArgumentType(i, child.getTypeString());

      if ((child instanceof ExprNodeGenericFuncDesc) || (child instanceof ExprNodeColumnDesc)) {

        builder.setInputExpressionType(i, InputExpressionType.COLUMN);

      } else if (child instanceof ExprNodeConstantDesc) {

        builder.setInputExpressionType(i, InputExpressionType.SCALAR);

      } else {

        throw new HiveException("Cannot handle expression type: " + child.getClass().getSimpleName());

      }

    }

    VectorExpressionDescriptor.Descriptor descriptor = builder.build();

    Class<?> vclass = this.vMap.getVectorExpressionClass(udf, descriptor);

    if (vclass == null) {

      if (LOG.isDebugEnabled()) {

        LOG.debug("No vector udf found for "+udf.getSimpleName() + ", descriptor: "+descriptor);

      }

      return null;

    }

    Mode childrenMode = getChildrenMode(mode, udf);

    return createVectorExpression(vclass, childExpr, childrenMode, returnType);

  }

  private VectorExpression createVectorExpression(Class<?> vectorClass,

      List<ExprNodeDesc> childExpr, Mode childrenMode, TypeInfo returnType) throws HiveException {

    int numChildren = childExpr == null ? 0: childExpr.size();

    VectorExpression.Type [] inputTypes = new VectorExpression.Type[numChildren];

    List<VectorExpression> children = new ArrayList<VectorExpression>();

    Object[] arguments = new Object[numChildren];

    try {

      for (int i = 0; i < numChildren; i++) {

        ExprNodeDesc child = childExpr.get(i);

        inputTypes[i] = VectorExpression.Type.getValue(child.getTypeInfo().getTypeName());

        if (child instanceof ExprNodeGenericFuncDesc) {

          VectorExpression vChild = getVectorExpression(child, childrenMode);

            children.add(vChild);

            arguments[i] = vChild.getOutputColumn();

        } else if (child instanceof ExprNodeColumnDesc) {

          int colIndex = getInputColumnIndex((ExprNodeColumnDesc) child);

            if (childrenMode == Mode.FILTER) {

              // In filter mode, the column must be a boolean

              children.add(new SelectColumnIsTrue(colIndex));

            }

            arguments[i] = colIndex;

        } else if (child instanceof ExprNodeConstantDesc) {

          Object scalarValue = getVectorTypeScalarValue((ExprNodeConstantDesc) child);

          arguments[i] = scalarValue;

        } else {

          throw new HiveException("Cannot handle expression type: " + child.getClass().getSimpleName());

        }

      }

      VectorExpression  vectorExpression = instantiateExpression(vectorClass, returnType, arguments);

      vectorExpression.setInputTypes(inputTypes);

      if ((vectorExpression != null) && !children.isEmpty()) {

        vectorExpression.setChildExpressions(children.toArray(new VectorExpression[0]));

      }

      return vectorExpression;

    } catch (Exception ex) {

      throw new HiveException(ex);

    } finally {

      for (VectorExpression ve : children) {

        ocm.freeOutputColumn(ve.getOutputColumn());

      }

    }

  }

  private Mode getChildrenMode(Mode mode, Class<?> udf) {

    if (mode.equals(Mode.FILTER) && (udf.equals(GenericUDFOPAnd.class) || udf.equals(GenericUDFOPOr.class))) {

      return Mode.FILTER;

    }

    return Mode.PROJECTION;

  }

  private VectorExpression instantiateExpression(Class<?> vclass, TypeInfo returnType, Object...args)

      throws HiveException {

    VectorExpression ve = null;

    Constructor<?> ctor = getConstructor(vclass);

    int numParams = ctor.getParameterTypes().length;

    int argsLength = (args == null) ? 0 : args.length;

    try {

      if (numParams == 0) {

        ve = (VectorExpression) ctor.newInstance();

      } else if (numParams == argsLength) {

        ve = (VectorExpression) ctor.newInstance(args);

      } else if (numParams == argsLength + 1) {

        // Additional argument is needed, which is the outputcolumn.

        String outType;

        // Special handling for decimal because decimal types need scale and precision parameter.

        // This special handling should be avoided by using returnType uniformly for all cases.

        if (returnType != null) {

          outType = getNormalizedTypeName(returnType.getTypeName()).toLowerCase();

        } else {

          outType = ((VectorExpression) vclass.newInstance()).getOutputType();

        }

        int outputCol = ocm.allocateOutputColumn(outType);

        Object [] newArgs = Arrays.copyOf(args, numParams);

        newArgs[numParams-1] = outputCol;

        ve = (VectorExpression) ctor.newInstance(newArgs);

        ve.setOutputType(outType);

      }

    } catch (Exception ex) {

      throw new HiveException("Could not instantiate " + vclass.getSimpleName(), ex);

    }

    return ve;

  }

  private VectorExpression getGenericUdfVectorExpression(GenericUDF udf,

      List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType) throws HiveException {

    List<ExprNodeDesc> constantFoldedChildren = foldConstantsForUnaryExprs(childExpr);

    childExpr = constantFoldedChildren;

    //First handle special cases

    if (udf instanceof GenericUDFBetween) {

      return getBetweenFilterExpression(childExpr, mode, returnType);

    } else if (udf instanceof GenericUDFIn) {

      return getInExpression(childExpr, mode, returnType);

    } else if (udf instanceof GenericUDFOPPositive) {

      return getIdentityExpression(childExpr);

    } else if (udf instanceof GenericUDFCoalesce) {

      // Coalesce is a special case because it can take variable number of arguments.

      return getCoalesceExpression(childExpr, returnType);

    } else if (udf instanceof GenericUDFBridge) {

      VectorExpression v = getGenericUDFBridgeVectorExpression((GenericUDFBridge) udf, childExpr, mode,

          returnType);

      if (v != null) {

        return v;

      }

    } else if (udf instanceof GenericUDFToDecimal) {

      return getCastToDecimal(childExpr, returnType);

    }

    // Now do a general lookup

    Class<?> udfClass = udf.getClass();

    if (udf instanceof GenericUDFBridge) {

      udfClass = ((GenericUDFBridge) udf).getUdfClass();

    }

    VectorExpression ve = getVectorExpressionForUdf(udfClass, constantFoldedChildren, mode, returnType);

    if (ve == null) {

      throw new HiveException("Udf: "+udf.getClass().getSimpleName()+", is not supported");

    }

    return ve;

  }

  private VectorExpression getCoalesceExpression(List<ExprNodeDesc> childExpr, TypeInfo returnType)

      throws HiveException {

    int[] inputColumns = new int[childExpr.size()];

    VectorExpression[] vectorChildren = null;

    try {

      vectorChildren = getVectorExpressions(childExpr, Mode.PROJECTION);

      int i = 0;

      for (VectorExpression ve : vectorChildren) {

        inputColumns[i++] = ve.getOutputColumn();

      }

      int outColumn = ocm.allocateOutputColumn(getNormalizedTypeName(returnType.getTypeName()));

      VectorCoalesce vectorCoalesce = new VectorCoalesce(inputColumns, outColumn);

      vectorCoalesce.setOutputType(returnType.getTypeName());

      vectorCoalesce.setChildExpressions(vectorChildren);

      return vectorCoalesce;

    } finally {

      // Free the output columns of the child expressions.

      if (vectorChildren != null) {

        for (VectorExpression v : vectorChildren) {

          ocm.freeOutputColumn(v.getOutputColumn());

        }

      }

    }

  }

  /**

   * Create a filter or boolean-valued expression for column IN ( <list-of-constants> )

   */

  private VectorExpression getInExpression(List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType)

      throws HiveException {

    ExprNodeDesc colExpr = childExpr.get(0);

    TypeInfo colTypeInfo = colExpr.getTypeInfo();

    String colType = colExpr.getTypeString();

    // prepare arguments for createVectorExpression

    List<ExprNodeDesc> childrenForInList =

        foldConstantsForUnaryExprs(childExpr.subList(1, childExpr.size()));

    /* This method assumes that the IN list has no NULL entries. That is enforced elsewhere,

     * in the Vectorizer class. If NULL is passed in as a list entry, behavior is not defined.

     * If in the future, NULL values are allowed in the IN list, be sure to handle 3-valued

     * logic correctly. E.g. NOT (col IN (null)) should be considered UNKNOWN, so that would

     * become FALSE in the WHERE clause, and cause the row in question to be filtered out.

     * See the discussion in Jira HIVE-5583.

     */

    VectorExpression expr = null;

    // determine class

    Class<?> cl = null;

    if (isIntFamily(colType)) {

      cl = (mode == Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);

      long[] inVals = new long[childrenForInList.size()];

      for (int i = 0; i != inVals.length; i++) {

        inVals[i] = getIntFamilyScalarAsLong((ExprNodeConstantDesc) childrenForInList.get(i));

      }

      expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);

      ((ILongInExpr) expr).setInListValues(inVals);

    } else if (isTimestampFamily(colType)) {

      cl = (mode == Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);

      long[] inVals = new long[childrenForInList.size()];

      for (int i = 0; i != inVals.length; i++) {

        inVals[i] = getTimestampScalar(childrenForInList.get(i));

      }

      expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);

      ((ILongInExpr) expr).setInListValues(inVals);

    } else if (isStringFamily(colType)) {

      cl = (mode == Mode.FILTER ? FilterStringColumnInList.class : StringColumnInList.class);

      byte[][] inVals = new byte[childrenForInList.size()][];

      for (int i = 0; i != inVals.length; i++) {

        inVals[i] = getStringScalarAsByteArray((ExprNodeConstantDesc) childrenForInList.get(i));

      }

      expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);

      ((IStringInExpr) expr).setInListValues(inVals);

    } else if (isFloatFamily(colType)) {

      cl = (mode == Mode.FILTER ? FilterDoubleColumnInList.class : DoubleColumnInList.class);

      double[] inValsD = new double[childrenForInList.size()];

      for (int i = 0; i != inValsD.length; i++) {

        inValsD[i] = getNumericScalarAsDouble(childrenForInList.get(i));

      }

      expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);

      ((IDoubleInExpr) expr).setInListValues(inValsD);

    } else if (isDecimalFamily(colType)) {

      cl = (mode == Mode.FILTER ? FilterDecimalColumnInList.class : DecimalColumnInList.class);

      Decimal128[] inValsD = new Decimal128[childrenForInList.size()];

      for (int i = 0; i != inValsD.length; i++) {

        inValsD[i] = (Decimal128) getVectorTypeScalarValue(

            (ExprNodeConstantDesc)  childrenForInList.get(i));

      }

      expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);

      ((IDecimalInExpr) expr).setInListValues(inValsD);

    } else if (isDateFamily(colType)) {

      cl = (mode == Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);

      long[] inVals = new long[childrenForInList.size()];

      for (int i = 0; i != inVals.length; i++) {

        inVals[i] = (Integer) getVectorTypeScalarValue((ExprNodeConstantDesc) childrenForInList.get(i));

      }

      expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);

      ((ILongInExpr) expr).setInListValues(inVals);

    }

    // Return the desired VectorExpression if found. Otherwise, return null to cause

    // execution to fall back to row mode.

    return expr;

  }

  private byte[] getStringScalarAsByteArray(ExprNodeConstantDesc exprNodeConstantDesc)

      throws HiveException {

    Object o = getScalarValue(exprNodeConstantDesc);

    if (!(o instanceof byte[])) {

      throw new HiveException("Expected constant argument of type string");

    }

    return (byte[]) o;

  }

  /**

   * Invoke special handling for expressions that can't be vectorized by regular

   * descriptor based lookup.

   */

  private VectorExpression getGenericUDFBridgeVectorExpression(GenericUDFBridge udf,

      List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType) throws HiveException {

    Class<? extends UDF> cl = udf.getUdfClass();

    if (isCastToIntFamily(cl)) {

      return getCastToLongExpression(childExpr);

    } else if (cl.equals(UDFToBoolean.class)) {

      return getCastToBoolean(childExpr);

    } else if (isCastToFloatFamily(cl)) {

      return getCastToDoubleExpression(cl, childExpr, returnType);

    } else if (cl.equals(UDFToString.class)) {

      return getCastToString(childExpr, returnType);

    }

    return null;

  }

  private VectorExpression getCastToDecimal(List<ExprNodeDesc> childExpr, TypeInfo returnType)

      throws HiveException {

    ExprNodeDesc child = childExpr.get(0);

    String inputType = childExpr.get(0).getTypeString();

    if (child instanceof ExprNodeConstantDesc) {

      // Return a constant vector expression

      Object constantValue = ((ExprNodeConstantDesc) child).getValue();

      Decimal128 decimalValue = castConstantToDecimal(constantValue, child.getTypeInfo());

      return getConstantVectorExpression(decimalValue, returnType, Mode.PROJECTION);

    }

    if (isIntFamily(inputType)) {

      return createVectorExpression(CastLongToDecimal.class, childExpr, Mode.PROJECTION, returnType);