Examples of org.apache.hadoop.hive.ql.udf.generic.GenericUDF

Package org.apache.hadoop.hive.ql.udf.generic

Examples of org.apache.hadoop.hive.ql.udf.generic.GenericUDF

org.apache.hadoop.hive.ql.udf.generic.GenericUDF
A Generic User-defined function (GenericUDF) for the use with Hive. New GenericUDF classes need to inherit from this GenericUDF class. The GenericUDF are superior to normal UDFs in the following ways: 1. It can accept arguments of complex types, and return complex types. 2. It can accept variable length of arguments. 3. It can accept an infinite number of function signature - for example, it's easy to write a GenericUDF that accepts array, array> and so on (arbitrary levels of nesting). 4. It can do short-circuit evaluations using DeferedObject.

    List<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>();
    ExprNodeColumnDesc colDesc1 = new ExprNodeColumnDesc(Long.class, "a", "table", false);
    ExprNodeColumnDesc colDesc2 = new ExprNodeColumnDesc(Long.class, "b", "table", false);
    ExprNodeColumnDesc colDesc3 = new ExprNodeColumnDesc(Long.class, "c", "table", false);
    ExprNodeGenericFuncDesc plusDesc = new ExprNodeGenericFuncDesc();
    GenericUDF gudf = new GenericUDFOPPlus();


    plusDesc.setGenericUDF(gudf);
    List<ExprNodeDesc> children = new  ArrayList<ExprNodeDesc>();
    children.add(colDesc1);
    children.add(colDesc2);

View Full Code Here

        TypeInfoFactory.booleanTypeInfo,
        FunctionRegistry.getGenericUDFForAnd(),
        residuals);
    }


    GenericUDF genericUDF = expr.getGenericUDF();
    if (!(genericUDF instanceof GenericUDFBaseCompare)) {
      return expr;
    }
    ExprNodeDesc expr1 = (ExprNodeDesc) nodeOutputs[0];
    ExprNodeDesc expr2 = (ExprNodeDesc) nodeOutputs[1];
    ExprNodeDesc[] extracted = ExprNodeDescUtils.extractComparePair(expr1, expr2);
    if (extracted == null) {
      return expr;
    }
    if (extracted.length > 2) {
      genericUDF = genericUDF.flip();
    }


    String udfName = genericUDF.getUdfName();
    if (!udfNames.contains(genericUDF.getUdfName())) {
      return expr;
    }


    ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) extracted[0];
    ExprNodeConstantDesc constantDesc = (ExprNodeConstantDesc) extracted[1];

View Full Code Here

        AnnotateStatsProcCtx aspCtx, List<String> neededCols) throws CloneNotSupportedException {
      long newNumRows = 0;
      Statistics andStats = null;
      if (pred instanceof ExprNodeGenericFuncDesc) {
        ExprNodeGenericFuncDesc genFunc = (ExprNodeGenericFuncDesc) pred;
        GenericUDF udf = genFunc.getGenericUDF();


        // for AND condition cascadingly update stats
        if (udf instanceof GenericUDFOPAnd) {
          andStats = stats.clone();
          aspCtx.setAndExprStats(andStats);

View Full Code Here

      long numRows = stats.getNumRows();


      if (child instanceof ExprNodeGenericFuncDesc) {


        ExprNodeGenericFuncDesc genFunc = (ExprNodeGenericFuncDesc) child;
        GenericUDF udf = genFunc.getGenericUDF();


        if (udf instanceof GenericUDFOPEqual || udf instanceof GenericUDFOPEqualNS) {
          String colName = null;
          String tabAlias = null;
          boolean isConst = false;

View Full Code Here

      FunctionInfo fi = FunctionRegistry.getFunctionInfo(udfName);
      if (fi == null) {
        throw new UDFArgumentException(udfName + " not found.");
      }


      GenericUDF genericUDF = fi.getGenericUDF();
      if (genericUDF == null) {
        throw new UDFArgumentException(udfName
            + " is an aggregation function or a table function.");
      }

View Full Code Here

            throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(expr));
          }
        }


        // getGenericUDF() actually clones the UDF. Just call it once and reuse.
        GenericUDF genericUDF = fi.getGenericUDF();


        if (!fi.isNative()) {
          ctx.getUnparseTranslator().addIdentifierTranslation(
              (ASTNode) expr.getChild(0));
        }

View Full Code Here

      } else {
        throw new IllegalStateException("Unexpected non-null ExprNodeConstantDesc: "
          + expr.getExprString());
      }
    } else if (expr instanceof ExprNodeGenericFuncDesc) {
      GenericUDF udf = ((ExprNodeGenericFuncDesc)expr).getGenericUDF();
      boolean isAnd = udf instanceof GenericUDFOPAnd;
      if (isAnd || udf instanceof GenericUDFOPOr) {
        List<ExprNodeDesc> children = expr.getChildren();
        ExprNodeDesc left = children.get(0);
        children.set(0, compactExpr(left));

View Full Code Here

      this.topExpr = topExpr;
    }


    public void visit(ExprNodeGenericFuncDesc fnExpr) throws HiveException
    {
      GenericUDF fn = fnExpr.getGenericUDF();
      if (fn instanceof GenericUDFLead || fn instanceof GenericUDFLag )
      {
        llInfo.addLLFuncExprForTopExpr(topExpr, fnExpr);
      }
    }

View Full Code Here

    ExprNodeConstantDesc folded = foldConstant(((ExprNodeGenericFuncDesc) expr));
    return folded == null ? expr : folded;
  }


  private static ExprNodeConstantDesc foldConstant(ExprNodeGenericFuncDesc func) {
    GenericUDF udf = func.getGenericUDF();
    if (!FunctionRegistry.isDeterministic(udf) || FunctionRegistry.isStateful(udf)) {
      return null;
    }
    try {
      // If the UDF depends on any external resources, we can't fold because the
      // resources may not be available at compile time.
      if (udf instanceof GenericUDFBridge) {
        UDF internal = ReflectionUtils.newInstance(((GenericUDFBridge) udf).getUdfClass(), null);
        if (internal.getRequiredFiles() != null || internal.getRequiredJars() != null) {
          return null;
        }
      } else {
        if (udf.getRequiredFiles() != null || udf.getRequiredJars() != null) {
          return null;
        }
      }


      if (func.getChildren() != null) {

View Full Code Here

    conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
    conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "5000");
    InputFormat<?, ?> in = new OrcInputFormat();
    FileInputFormat.setInputPaths(conf, testFilePath.toString());


    GenericUDF udf = new GenericUDFOPEqualOrLessThan();
    List<ExprNodeDesc> childExpr = Lists.newArrayList();
    ExprNodeColumnDesc col = new ExprNodeColumnDesc(Long.class, "userid", "T", false);
    ExprNodeConstantDesc con = new ExprNodeConstantDesc(100);
    childExpr.add(col);
    childExpr.add(con);

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.hadoop.hive.ql.udf.generic.GenericUDF

com.sap.hadoop.windowing.query2.translate.WindowingExprNodeEvaluatorFactory$FindLeadLagFuncExprs

com.sap.hadoop.windowing.query2.translate.WindowingTypeCheckProcFactory$DefaultExprProcessor

org.apache.drill.exec.expr.fn.HiveFunctionRegistry

org.apache.hadoop.hive.accumulo.predicate.AccumuloRangeGenerator

org.apache.hadoop.hive.ql.exec.ExprNodeGenericFuncEvaluator

org.apache.hadoop.hive.ql.exec.Utilities

org.apache.hadoop.hive.ql.exec.vector.TestVectorizationContext

org.apache.hadoop.hive.ql.exec.vector.TestVectorSelectOperator

org.apache.hadoop.hive.ql.exec.vector.udf.TestVectorUDFAdaptor

org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.