Examples of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator

Package org.apache.hadoop.hive.ql.udf.generic

Examples of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator

org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator
A Generic User-defined aggregation function (GenericUDAF) for the use with Hive. New GenericUDAF classes need to inherit from this GenericUDAF class. The GenericUDAF are superior to normal UDAFs in the following ways: 1. It can accept arguments of complex types, and return complex types. 2. It can accept variable length of arguments. 3. It can accept an infinite number of function signature - for example, it's easy to write a GenericUDAF that accepts array, array> and so on (arbitrary levels of nesting).

      if (isDistinct) {
        numDistinctUDFs++;
      }
      boolean isAllColumns = value.getType() == HiveParser.TOK_FUNCTIONSTAR;
      Mode amode = groupByDescModeToUDAFMode(mode, isDistinct);
      GenericUDAFEvaluator genericUDAFEvaluator = null;
      // For distincts, partial aggregations have not been done
      if (distPartAgg) {
        genericUDAFEvaluator = getGenericUDAFEvaluator(aggName, aggParameters,
            value, isDistinct, isAllColumns);
        assert (genericUDAFEvaluator != null);

View Full Code Here


      boolean isDistinct = value.getType() == HiveParser.TOK_FUNCTIONDI;
      boolean isAllColumns = value.getType() == HiveParser.TOK_FUNCTIONSTAR;
      Mode amode = groupByDescModeToUDAFMode(mode, isDistinct);


      GenericUDAFEvaluator genericUDAFEvaluator = getGenericUDAFEvaluator(
          aggName, aggParameters, value, isDistinct, isAllColumns);
      assert (genericUDAFEvaluator != null);
      GenericUDAFInfo udaf = getGenericUDAFInfo(genericUDAFEvaluator, amode,
          aggParameters);
      aggregations.add(new AggregationDesc(aggName.toLowerCase(),

View Full Code Here

      String aggName = value.getChild(0).getText();


      boolean isDistinct = value.getType() == HiveParser.TOK_FUNCTIONDI;
      boolean isStar = value.getType() == HiveParser.TOK_FUNCTIONSTAR;
      Mode amode = groupByDescModeToUDAFMode(mode, isDistinct);
      GenericUDAFEvaluator genericUDAFEvaluator = genericUDAFEvaluators
          .get(entry.getKey());
      assert (genericUDAFEvaluator != null);
      GenericUDAFInfo udaf = getGenericUDAFInfo(genericUDAFEvaluator, amode,
          aggParameters);
      aggregations

View Full Code Here

   */
  public boolean isDistinctLike() {
    ArrayList<AggregationDesc> aggregators = getAggregators();
    for (AggregationDesc ad : aggregators) {
      if (!ad.getDistinct()) {
        GenericUDAFEvaluator udafEval = ad.getGenericUDAFEvaluator();
        UDFType annot = AnnotationUtils.getAnnotation(udafEval.getClass(), UDFType.class);
        if (annot == null || !annot.distinctLike()) {
          return false;
        }
      }
    }

View Full Code Here

            List<ExprNodeDesc> paraList = aggregationDesc.getParameters();
            List<ObjectInspector> parametersOIList = new ArrayList<ObjectInspector>();
            for (ExprNodeDesc expr : paraList) {
              parametersOIList.add(expr.getWritableObjectInspector());
            }
            GenericUDAFEvaluator evaluator = FunctionRegistry.getGenericUDAFEvaluator(
                "sum", parametersOIList, false, false);
            aggregationDesc.setGenericUDAFEvaluator(evaluator);
            aggregationDesc.setGenericUDAFName("sum");
          }
        }

View Full Code Here

    }
  }


  Object evaluateWindowFunction(WindowFunctionDef wFn,
      PTFPartitionIterator<Object> pItr) throws HiveException {
    GenericUDAFEvaluator fEval = wFn.getWFnEval();
    Object[] args = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()];
    AggregationBuffer aggBuffer = fEval.getNewAggregationBuffer();
    while(pItr.hasNext())
    {
      Object row = pItr.next();
      int i =0;
      if ( wFn.getArgs() != null ) {
        for(PTFExpressionDef arg : wFn.getArgs())
        {
          args[i++] = arg.getExprEvaluator().evaluate(row);
        }
      }
      fEval.aggregate(aggBuffer, args);
    }
    Object out = fEval.evaluate(aggBuffer);
    out = ObjectInspectorUtils.copyToStandardObject(out, wFn.getOI());
    return out;
  }

View Full Code Here

    int followingSpan = 0;


    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
      WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i);
      WindowFrameDef wdwFrame = wFnDef.getWindowFrame();
      GenericUDAFEvaluator fnEval = wFnDef.getWFnEval();
      boolean streamingPossible = streamingPossible(cfg, wFnDef);
      GenericUDAFEvaluator streamingEval = streamingPossible ? fnEval
          .getWindowingEvaluator(wdwFrame) : null;
      if (streamingEval != null
          && streamingEval instanceof ISupportStreamingModeForWindowing) {
        continue;
      }

View Full Code Here

    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();


    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
      WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i);
      WindowFrameDef wdwFrame = wFnDef.getWindowFrame();
      GenericUDAFEvaluator fnEval = wFnDef.getWFnEval();
      GenericUDAFEvaluator streamingEval = fnEval
          .getWindowingEvaluator(wdwFrame);
      if (streamingEval != null) {
        wFnDef.setWFnEval(streamingEval);
        if (wFnDef.isPivotResult()) {
          ListObjectInspector listOI = (ListObjectInspector) wFnDef.getOI();

View Full Code Here


    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();


    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
      WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i);
      GenericUDAFEvaluator fnEval = wFn.getWFnEval();


      int a = 0;
      if (wFn.getArgs() != null) {
        for (PTFExpressionDef arg : wFn.getArgs()) {
          streamingState.funcArgs[i][a++] = arg.getExprEvaluator().evaluate(row);
        }
      }


      if (fnEval instanceof ISupportStreamingModeForWindowing) {
        fnEval.aggregate(streamingState.aggBuffers[i], streamingState.funcArgs[i]);
        Object out = ((ISupportStreamingModeForWindowing) fnEval)
            .getNextResult(streamingState.aggBuffers[i]);
        if (out != null) {
          streamingState.fnOutputs[i]
              .add(out == ISupportStreamingModeForWindowing.NULL_RESULT ? null

View Full Code Here

    }


    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();
    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
      WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i);
      GenericUDAFEvaluator fnEval = wFn.getWFnEval();


      int numRowsRemaining = wFn.getWindowFrame().getEnd().getAmt();
      if (fnEval instanceof ISupportStreamingModeForWindowing) {
        fnEval.terminate(streamingState.aggBuffers[i]);


        WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFn
            .getName());
        if (!wFnInfo.isSupportsWindow()) {
          numRowsRemaining = ((ISupportStreamingModeForWindowing) fnEval)

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator

brickhouse.udf.hll.HyperLogLogUDAFTest

com.sap.hadoop.windowing.functions2.table.WindowingTableFunction

com.sap.hadoop.windowing.query2.translate.WindowFunctionTranslation

org.apache.hadoop.hive.ql.exec.FunctionRegistry

org.apache.hadoop.hive.ql.optimizer.index.RewriteQueryUsingAggregateIndex$NewQueryGroupbySchemaProc

org.apache.hadoop.hive.ql.parse.PTFTranslator

org.apache.hadoop.hive.ql.parse.SemanticAnalyzer

org.apache.hadoop.hive.ql.parse.SemanticAnalyzer$OptiqBasedPlanner

org.apache.hadoop.hive.ql.plan.GroupByDesc

org.apache.hadoop.hive.ql.plan.PTFDeserializer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.