Examples of TableFunctionResolver

com.sap.hadoop.windowing.functions2.TableFunctionResolver
Based on Hive {@link GenericUDAFResolver}. Break up the responsibility of the old AbstractTableFunction class into a Resolver and Evaluator. The Resolver is responsible for:
1. setting up the {@link tableFunctionEvaluator}
2. Setting up the The raw and output ObjectInspectors of the Evaluator.
3. The Evaluator also holds onto the {@link TableFunctionDef}. This provides information about the arguments to the function, the shape of the Input partition and the Partitioning details.
The Resolver for a function is obtained from the {@link FunctionRegistry}. The Resolver is initialized by the following 4 step process:
1. The initialize method is called; which is passed the {@link QueryDef} and the {@link TableFunctionDef}.
2. The resolver is then asked to setup the Raw ObjectInspector. This is only required if the Function reshapes the raw input.
3. Once the Resolver has had a chance to compute the shape of the Raw Input that is fed to the partitioning machinery; the translator sets up the partitioning details on the tableFuncDef.
4. finally the resolver is asked to setup the output ObjectInspector.
org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver
Based on Hive {@link GenericUDAFResolver}. Break up the responsibility of the old AbstractTableFunction class into a Resolver and Evaluator. The Resolver is responsible for:
1. setting up the {@link tableFunctionEvaluator}
2. Setting up the The raw and output ObjectInspectors of the Evaluator.
3. The Evaluator also holds onto the {@link TableFunctionDef}. This provides information about the arguments to the function, the shape of the Input partition and the Partitioning details.
The Resolver for a function is obtained from the {@link FunctionRegistry}. The Resolver is initialized by the following 4 step process:
1. The initialize method is called; which is passed the {@link PTFDesc} and the {@link TableFunctionDef}.
2. The resolver is then asked to setup the Raw ObjectInspector. This is only required if the Function reshapes the raw input.
3. Once the Resolver has had a chance to compute the shape of the Raw Input that is fed to the partitioning machinery; the translator sets up the partitioning details on the tableFuncDef.
4. finally the resolver is asked to setup the output ObjectInspector.

Examples of com.sap.hadoop.windowing.functions2.TableFunctionResolver

   */
  private static TableFuncDef translate(QueryDef qDef, TableFuncSpec tSpec, QueryInputDef inputDef) throws WindowingException
  {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    
    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(tSpec.getName());
    if ( tFn == null)
    {
      throw new WindowingException(sprintf("Unknown Table Function %s", tSpec.getName()));
    }
    
    TableFuncDef tDef = new TableFuncDef();
    tDef.setSpec(tSpec);
    tDef.setInput(inputDef);
    InputInfo iInfo = tInfo.getInputInfo(inputDef); 
    
    /*
     * translate args
     */
    ArrayList<ASTNode> args = tSpec.getArgs();
    if ( args != null)
    {
      for(ASTNode expr : args)
      {
        ArgDef argDef = translateTableFunctionArg(qDef, tDef, iInfo,  expr);
        tDef.addArg(argDef);
      }
    }
    
    tFn.initialize(qDef, tDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();    
    tDef.setFunction(tEval);
    tFn.setupRawInputOI();
    tDef.setWindow(WindowSpecTranslation.translateWindow(qDef, tDef));
    tFn.setupOutputOI();
    TranslateUtils.setupSerdeAndOI(tDef, inputDef, tInfo, tEval);


    return tDef;
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver


  private PartitionedTableFunctionDef translate(PartitionedTableFunctionSpec spec,
      PTFInputDef inpDef,
      int inpNum)
      throws SemanticException {
    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(spec.getName());
    if (tFn == null) {
      throw new SemanticException(String.format("Unknown Table Function %s",
          spec.getName()));
    }
    PartitionedTableFunctionDef def = new PartitionedTableFunctionDef();
    def.setInput(inpDef);
    def.setName(spec.getName());
    def.setResolverClassName(tFn.getClass().getName());
    def.setAlias(spec.getAlias() == null ? "ptf_" + inpNum : spec.getAlias());
    def.setExpressionTreeString(spec.getAstNode().toStringTree());
    def.setTransformsRawInput(tFn.transformsRawInput());
    /*
     * translate args
     */
    ArrayList<ASTNode> args = spec.getArgs();
    if (args != null)
    {
      for (ASTNode expr : args)
      {
        PTFExpressionDef argDef = null;
        try {
          argDef = buildExpressionDef(inpDef.getOutputShape(), expr);
        } catch (HiveException he) {
          throw new SemanticException(he);
        }
        def.addArg(argDef);
      }
    }


    tFn.initialize(hCfg, ptfDesc, def);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    def.setTFunction(tEval);
    def.setCarryForwardNames(tFn.carryForwardNames());
    tFn.setupRawInputOI();


    if (tFn.transformsRawInput()) {
      StructObjectInspector rawInOutOI = tEval.getRawInputOI();
      ArrayList<String> rawInOutColNames = tFn.getRawInputColumnNames();
      RowResolver rawInRR = buildRowResolverForPTF(def.getName(),
          spec.getAlias(),
          rawInOutOI,
          rawInOutColNames,
          inpDef.getOutputShape().getRr());
      ShapeDetails rawInpShape = setupTableFnShape(def.getName(),
          inpDef.getOutputShape(),
          rawInOutOI,
          rawInOutColNames,
          rawInRR);
      def.setRawInputShape(rawInpShape);
    }
    else {
      def.setRawInputShape(inpDef.getOutputShape());
    }


    translatePartitioning(def, spec);
    tFn.setupOutputOI();


    StructObjectInspector outputOI = tEval.getOutputOI();
    ArrayList<String> outColNames = tFn.getOutputColumnNames();
    RowResolver outRR = buildRowResolverForPTF(def.getName(),
        spec.getAlias(),
        outputOI,
        outColNames,
        def.getRawInputShape().getRr());

View Full Code Here

Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver

    /*
     * 2. setup resolve, make connections
     */
    TableFunctionEvaluator tEval = def.getTFunction();
    // TableFunctionResolver tResolver = FunctionRegistry.getTableFunctionResolver(def.getName());
    TableFunctionResolver tResolver = constructResolver(def.getResolverClassName());
    tResolver.initialize(ptfDesc, def, tEval);


    /*
     * 3. give Evaluator chance to setup for RawInput execution; setup RawInput shape
     */
    if (tEval.isTransformsRawInput()) {
      tResolver.initializeRawInputOI();
      initialize(def.getRawInputShape(), tEval.getRawInputOI());
    } else {
      def.setRawInputShape(inpShape);
    }


    inpShape = def.getRawInputShape();


    /*
     * 4. give Evaluator chance to setup for Output execution; setup Output shape.
     */
    tResolver.initializeOutputOI();
    initialize(def.getOutputShape(), tEval.getOutputOI());
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver


  private PartitionedTableFunctionDef translate(PartitionedTableFunctionSpec spec,
      PTFInputDef inpDef,
      int inpNum)
      throws SemanticException {
    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(spec.getName());
    if (tFn == null) {
      throw new SemanticException(String.format("Unknown Table Function %s",
          spec.getName()));
    }
    PartitionedTableFunctionDef def = new PartitionedTableFunctionDef();
    def.setInput(inpDef);
    def.setName(spec.getName());
    def.setResolverClassName(tFn.getClass().getName());
    def.setAlias(spec.getAlias() == null ? "ptf_" + inpNum : spec.getAlias());
    def.setExpressionTreeString(spec.getAstNode().toStringTree());
    def.setTransformsRawInput(tFn.transformsRawInput());
    /*
     * translate args
     */
    List<ASTNode> args = spec.getArgs();
    if (args != null)
    {
      for (ASTNode expr : args)
      {
        PTFExpressionDef argDef = null;
        try {
          argDef = buildExpressionDef(inpDef.getOutputShape(), expr);
        } catch (HiveException he) {
          throw new SemanticException(he);
        }
        def.addArg(argDef);
      }
    }


    tFn.initialize(hCfg, ptfDesc, def);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    def.setTFunction(tEval);
    def.setCarryForwardNames(tFn.carryForwardNames());
    tFn.setupRawInputOI();


    if (tFn.transformsRawInput()) {
      StructObjectInspector rawInOutOI = tEval.getRawInputOI();
      List<String> rawInOutColNames = tFn.getRawInputColumnNames();
      RowResolver rawInRR = buildRowResolverForPTF(def.getName(),
          spec.getAlias(),
          rawInOutOI,
          rawInOutColNames,
          inpDef.getOutputShape().getRr());
      ShapeDetails rawInpShape = setupTableFnShape(def.getName(),
          inpDef.getOutputShape(),
          rawInOutOI,
          rawInOutColNames,
          rawInRR);
      def.setRawInputShape(rawInpShape);
    }
    else {
      def.setRawInputShape(inpDef.getOutputShape());
    }


    translatePartitioning(def, spec);
    tFn.setupOutputOI();


    StructObjectInspector outputOI = tEval.getOutputOI();
    List<String> outColNames = tFn.getOutputColumnNames();
    RowResolver outRR = buildRowResolverForPTF(def.getName(),
        spec.getAlias(),
        outputOI,
        outColNames,
        def.getRawInputShape().getRr());

View Full Code Here

Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver

    /*
     * 2. setup resolve, make connections
     */
    TableFunctionEvaluator tEval = def.getTFunction();
    // TableFunctionResolver tResolver = FunctionRegistry.getTableFunctionResolver(def.getName());
    TableFunctionResolver tResolver = constructResolver(def.getResolverClassName());
    tResolver.initialize(ptfDesc, def, tEval);


    /*
     * 3. give Evaluator chance to setup for RawInput execution; setup RawInput shape
     */
    if (tEval.isTransformsRawInput()) {
      tResolver.initializeRawInputOI();
      initialize(def.getRawInputShape(), tEval.getRawInputOI());
    } else {
      def.setRawInputShape(inpShape);
    }


    inpShape = def.getRawInputShape();


    /*
     * 4. give Evaluator chance to setup for Output execution; setup Output shape.
     */
    tResolver.initializeOutputOI();
    initialize(def.getOutputShape(), tEval.getOutputOI());
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver


  private PartitionedTableFunctionDef translate(PartitionedTableFunctionSpec spec,
      PTFInputDef inpDef,
      int inpNum)
      throws SemanticException {
    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(spec.getName());
    if (tFn == null) {
      throw new SemanticException(String.format("Unknown Table Function %s",
          spec.getName()));
    }
    PartitionedTableFunctionDef def = new PartitionedTableFunctionDef();
    def.setInput(inpDef);
    def.setName(spec.getName());
    def.setResolverClassName(tFn.getClass().getName());
    def.setAlias(spec.getAlias() == null ? "ptf_" + inpNum : spec.getAlias());
    def.setExpressionTreeString(spec.getAstNode().toStringTree());
    def.setTransformsRawInput(tFn.transformsRawInput());
    /*
     * translate args
     */
    List<ASTNode> args = spec.getArgs();
    if (args != null)
    {
      for (ASTNode expr : args)
      {
        PTFExpressionDef argDef = null;
        try {
          argDef = buildExpressionDef(inpDef.getOutputShape(), expr);
        } catch (HiveException he) {
          throw new SemanticException(he);
        }
        def.addArg(argDef);
      }
    }


    tFn.initialize(hCfg, ptfDesc, def);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    def.setTFunction(tEval);
    def.setCarryForwardNames(tFn.carryForwardNames());
    tFn.setupRawInputOI();


    if (tFn.transformsRawInput()) {
      StructObjectInspector rawInOutOI = tEval.getRawInputOI();
      List<String> rawInOutColNames = tFn.getRawInputColumnNames();
      RowResolver rawInRR = buildRowResolverForPTF(def.getName(),
          spec.getAlias(),
          rawInOutOI,
          rawInOutColNames,
          inpDef.getOutputShape().getRr());
      ShapeDetails rawInpShape = setupTableFnShape(def.getName(),
          inpDef.getOutputShape(),
          rawInOutOI,
          rawInOutColNames,
          rawInRR);
      def.setRawInputShape(rawInpShape);
    }
    else {
      def.setRawInputShape(inpDef.getOutputShape());
    }


    translatePartitioning(def, spec);
    tFn.setupOutputOI();


    StructObjectInspector outputOI = tEval.getOutputOI();
    List<String> outColNames = tFn.getOutputColumnNames();
    RowResolver outRR = buildRowResolverForPTF(def.getName(),
        spec.getAlias(),
        outputOI,
        outColNames,
        def.getRawInputShape().getRr());

View Full Code Here

Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver

    /*
     * 2. setup resolve, make connections
     */
    TableFunctionEvaluator tEval = def.getTFunction();
    //TableFunctionResolver tResolver = FunctionRegistry.getTableFunctionResolver(def.getName());
    TableFunctionResolver tResolver = constructResolver(def.getResolverClassName());
    tResolver.initialize(ptfDesc, def, tEval);


    /*
     * 3. give Evaluator chance to setup for RawInput execution; setup RawInput shape
     */
    if (tEval.isTransformsRawInput())
    {
      tResolver.initializeRawInputOI();
      initialize(def.getRawInputShape(), tEval.getRawInputOI());
    }
    else {
      def.setRawInputShape(inpShape);
    }


    inpShape = def.getRawInputShape();


    /*
     * 4. give Evaluator chance to setup for Output execution; setup Output shape.
     */
    tResolver.initializeOutputOI();
    initialize(def.getOutputShape(), tEval.getOutputOI());
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver


  private PartitionedTableFunctionDef translate(PartitionedTableFunctionSpec spec,
      PTFInputDef inpDef,
      int inpNum)
      throws SemanticException {
    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(spec.getName());
    if (tFn == null) {
      throw new SemanticException(String.format("Unknown Table Function %s",
          spec.getName()));
    }
    PartitionedTableFunctionDef def = new PartitionedTableFunctionDef();
    def.setInput(inpDef);
    def.setName(spec.getName());
    def.setResolverClassName(tFn.getClass().getName());
    def.setAlias(spec.getAlias() == null ? "ptf_" + inpNum : spec.getAlias());
    def.setExpressionTreeString(spec.getAstNode().toStringTree());
    def.setTransformsRawInput(tFn.transformsRawInput());
    /*
     * translate args
     */
    List<ASTNode> args = spec.getArgs();
    if (args != null)
    {
      for (ASTNode expr : args)
      {
        PTFExpressionDef argDef = null;
        try {
          argDef = buildExpressionDef(inpDef.getOutputShape(), expr);
        } catch (HiveException he) {
          throw new SemanticException(he);
        }
        def.addArg(argDef);
      }
    }


    tFn.initialize(hCfg, ptfDesc, def);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    def.setTFunction(tEval);
    def.setCarryForwardNames(tFn.carryForwardNames());
    tFn.setupRawInputOI();


    if (tFn.transformsRawInput()) {
      StructObjectInspector rawInOutOI = tEval.getRawInputOI();
      List<String> rawInOutColNames = tFn.getRawInputColumnNames();
      RowResolver rawInRR = buildRowResolverForPTF(def.getName(),
          spec.getAlias(),
          rawInOutOI,
          rawInOutColNames,
          inpDef.getOutputShape().getRr());
      ShapeDetails rawInpShape = setupTableFnShape(def.getName(),
          inpDef.getOutputShape(),
          rawInOutOI,
          rawInOutColNames,
          rawInRR);
      def.setRawInputShape(rawInpShape);
    }
    else {
      def.setRawInputShape(inpDef.getOutputShape());
    }


    translatePartitioning(def, spec);
    tFn.setupOutputOI();


    StructObjectInspector outputOI = tEval.getOutputOI();
    List<String> outColNames = tFn.getOutputColumnNames();
    RowResolver outRR = buildRowResolverForPTF(def.getName(),
        spec.getAlias(),
        outputOI,
        outColNames,
        def.getRawInputShape().getRr());

View Full Code Here

Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver

    /*
     * 2. setup resolve, make connections
     */
    TableFunctionEvaluator tEval = def.getTFunction();
    // TableFunctionResolver tResolver = FunctionRegistry.getTableFunctionResolver(def.getName());
    TableFunctionResolver tResolver = constructResolver(def.getResolverClassName());
    tResolver.initialize(ptfDesc, def, tEval);


    /*
     * 3. give Evaluator chance to setup for RawInput execution; setup RawInput shape
     */
    if (tEval.isTransformsRawInput()) {
      tResolver.initializeRawInputOI();
      initialize(def.getRawInputShape(), tEval.getRawInputOI());
    } else {
      def.setRawInputShape(inpShape);
    }


    inpShape = def.getRawInputShape();


    /*
     * 4. give Evaluator chance to setup for Output execution; setup Output shape.
     */
    tResolver.initializeOutputOI();
    initialize(def.getOutputShape(), tEval.getOutputOI());
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver


  private PartitionedTableFunctionDef translate(PartitionedTableFunctionSpec spec,
      PTFInputDef inpDef,
      int inpNum)
      throws SemanticException {
    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(spec.getName());
    if (tFn == null) {
      throw new SemanticException(String.format("Unknown Table Function %s",
          spec.getName()));
    }
    PartitionedTableFunctionDef def = new PartitionedTableFunctionDef();
    def.setInput(inpDef);
    def.setName(spec.getName());
    def.setResolverClassName(tFn.getClass().getName());
    def.setAlias(spec.getAlias() == null ? "ptf_" + inpNum : spec.getAlias());
    def.setExpressionTreeString(spec.getAstNode().toStringTree());
    def.setTransformsRawInput(tFn.transformsRawInput());
    /*
     * translate args
     */
    ArrayList<ASTNode> args = spec.getArgs();
    if (args != null)
    {
      for (ASTNode expr : args)
      {
        PTFExpressionDef argDef = null;
        try {
          argDef = buildExpressionDef(inpDef.getOutputShape(), expr);
        } catch (HiveException he) {
          throw new SemanticException(he);
        }
        def.addArg(argDef);
      }
    }


    tFn.initialize(hCfg, ptfDesc, def);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    def.setTFunction(tEval);
    def.setCarryForwardNames(tFn.carryForwardNames());
    tFn.setupRawInputOI();


    if (tFn.transformsRawInput()) {
      StructObjectInspector rawInOutOI = tEval.getRawInputOI();
      ArrayList<String> rawInOutColNames = tFn.getRawInputColumnNames();
      RowResolver rawInRR = buildRowResolverForPTF(def.getName(),
          spec.getAlias(),
          rawInOutOI,
          rawInOutColNames,
          inpDef.getOutputShape().getRr());
      ShapeDetails rawInpShape = setupTableFnShape(def.getName(),
          inpDef.getOutputShape(),
          rawInOutOI,
          rawInOutColNames,
          rawInRR);
      def.setRawInputShape(rawInpShape);
    }
    else {
      def.setRawInputShape(inpDef.getOutputShape());
    }


    translatePartitioning(def, spec);
    tFn.setupOutputOI();


    StructObjectInspector outputOI = tEval.getOutputOI();
    ArrayList<String> outColNames = tFn.getOutputColumnNames();
    RowResolver outRR = buildRowResolverForPTF(def.getName(),
        spec.getAlias(),
        outputOI,
        outColNames,
        def.getRawInputShape().getRr());

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.