Examples of PartitionedTableFunctionDef


Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef

    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(spec.getName());
    if (tFn == null) {
      throw new SemanticException(String.format("Unknown Table Function %s",
          spec.getName()));
    }
    PartitionedTableFunctionDef def = new PartitionedTableFunctionDef();
    def.setInput(inpDef);
    def.setName(spec.getName());
    def.setResolverClassName(tFn.getClass().getName());
    def.setAlias(spec.getAlias() == null ? "ptf_" + inpNum : spec.getAlias());
    def.setExpressionTreeString(spec.getAstNode().toStringTree());
    def.setTransformsRawInput(tFn.transformsRawInput());
    /*
     * translate args
     */
    ArrayList<ASTNode> args = spec.getArgs();
    if (args != null)
    {
      for (ASTNode expr : args)
      {
        PTFExpressionDef argDef = null;
        try {
          argDef = buildExpressionDef(inpDef.getOutputShape(), expr);
        } catch (HiveException he) {
          throw new SemanticException(he);
        }
        def.addArg(argDef);
      }
    }

    tFn.initialize(hCfg, ptfDesc, def);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    def.setTFunction(tEval);
    def.setCarryForwardNames(tFn.carryForwardNames());
    tFn.setupRawInputOI();

    if (tFn.transformsRawInput()) {
      StructObjectInspector rawInOutOI = tEval.getRawInputOI();
      ArrayList<String> rawInOutColNames = tFn.getRawInputColumnNames();
      RowResolver rawInRR = buildRowResolverForPTF(def.getName(),
          spec.getAlias(),
          rawInOutOI,
          rawInOutColNames,
          inpDef.getOutputShape().getRr());
      ShapeDetails rawInpShape = setupTableFnShape(def.getName(),
          inpDef.getOutputShape(),
          rawInOutOI,
          rawInOutColNames,
          rawInRR);
      def.setRawInputShape(rawInpShape);
    }
    else {
      def.setRawInputShape(inpDef.getOutputShape());
    }

    translatePartitioning(def, spec);
    tFn.setupOutputOI();

    StructObjectInspector outputOI = tEval.getOutputOI();
    ArrayList<String> outColNames = tFn.getOutputColumnNames();
    RowResolver outRR = buildRowResolverForPTF(def.getName(),
        spec.getAlias(),
        outputOI,
        outColNames,
        def.getRawInputShape().getRr());
    ShapeDetails outputShape = setupTableFnShape(def.getName(),
        inpDef.getOutputShape(),
        outputOI,
        outColNames,
        outRR);
    def.setOutputShape(outputShape);

    return def;
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef

     * Input -> ReduceSink
     *
     * Here the ExprNodeDescriptors in the QueryDef are based on the Input Operator's RR.
     */
    {
      PartitionedTableFunctionDef tabDef = ptfDesc.getStartOfChain();

      /*
       * a. add Map-side PTF Operator if needed
       */
      if (tabDef.isTransformsRawInput() )
      {
        RowResolver ptfMapRR = tabDef.getRawInputShape().getRr();

        ptfDesc.setMapSide(true);
        input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc,
            new RowSchema(ptfMapRR.getColumnInfos()),
            input), ptfMapRR);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef

     */
    @Override
    public void setupOutputOI() throws SemanticException
    {
      NPath evaluator = (NPath) getEvaluator();
      PartitionedTableFunctionDef tDef = evaluator.getTableDef();

      ArrayList<PTFExpressionDef> args = tDef.getArgs();
      int argsNum = args == null ? 0 : args.size();

      if ( argsNum < 4 )
      {
        throwErrorWithSignature("at least 4 arguments required");
      }

      validateAndSetupPatternStr(evaluator, args);
      validateAndSetupSymbolInfo(evaluator, args, argsNum);
      validateAndSetupResultExprStr(evaluator, args, argsNum);
      setupSymbolFunctionChain(evaluator);

      /*
       * setup OI for input to resultExpr select list
       */
      RowResolver selectListInputRR = NPath.createSelectListRR(evaluator, tDef.getInput());

      /*
       * parse ResultExpr Str and setup OI.
       */
      ResultExpressionParser resultExprParser =
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef

    @Override
    public void initializeOutputOI() throws HiveException {
      try {
        NPath evaluator = (NPath) getEvaluator();
        PartitionedTableFunctionDef tDef = evaluator.getTableDef();

        ArrayList<PTFExpressionDef> args = tDef.getArgs();
        int argsNum = args.size();

        validateAndSetupPatternStr(evaluator, args);
        validateAndSetupSymbolInfo(evaluator, args, argsNum);
        validateAndSetupResultExprStr(evaluator, args, argsNum);
        setupSymbolFunctionChain(evaluator);

        /*
         * setup OI for input to resultExpr select list
         */
        StructObjectInspector selectListInputOI = NPath.createSelectListOI( evaluator,
            tDef.getInput());
        ResultExprInfo resultExprInfo = evaluator.resultExprInfo;
        ArrayList<ObjectInspector> selectListExprOIs = new ArrayList<ObjectInspector>();
        resultExprInfo.resultExprEvals = new ArrayList<ExprNodeEvaluator>();

        for(int i=0 ; i < resultExprInfo.resultExprNodes.size(); i++) {
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef

    inputPart = createFirstPartitionForChain(
        inputObjInspectors[0], hiveConf, isMapOperator);

    if (isMapOperator)
    {
      PartitionedTableFunctionDef tDef = conf.getStartOfChain();
      outputObjInspector = tDef.getRawInputShape().getOI();
    }
    else
    {
      outputObjInspector = conf.getFuncDef().getOutputShape().getOI();
    }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef

    }
  }

  protected void processMapFunction() throws HiveException
  {
    PartitionedTableFunctionDef tDef = conf.getStartOfChain();
    PTFPartition outPart = tDef.getTFunction().transformRawInput(inputPart);
    PTFPartitionIterator<Object> pItr = outPart.iterator();
    while (pItr.hasNext())
    {
      Object oRow = pItr.next();
      forward(oRow, outputObjInspector);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef

      {
        break;
      }
    }

    PartitionedTableFunctionDef currFnDef;
    while (!fnDefs.isEmpty())
    {
      currFnDef = fnDefs.pop();
      part = currFnDef.getTFunction().execute(part);
    }
    return part;
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef

   * @throws HiveException
   */
  public PTFPartition createFirstPartitionForChain(ObjectInspector oi,
      HiveConf hiveConf, boolean isMapSide) throws HiveException
  {
    PartitionedTableFunctionDef tabDef = conf.getStartOfChain();
    TableFunctionEvaluator tEval = tabDef.getTFunction();
    String partClassName = tEval.getPartitionClass();
    int partMemSize = tEval.getPartitionMemSize();

    PTFPartition part = null;
    SerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() :
      tabDef.getRawInputShape().getSerde();
    part = new PTFPartition(partClassName, partMemSize, serde,
        (StructObjectInspector) oi);
    return part;

  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef

     */
    @Override
    public void setupOutputOI() throws SemanticException
    {
      MatchPath evaluator = (MatchPath) getEvaluator();
      PartitionedTableFunctionDef tDef = evaluator.getTableDef();

      ArrayList<PTFExpressionDef> args = tDef.getArgs();
      int argsNum = args == null ? 0 : args.size();

      if ( argsNum < 4 )
      {
        throwErrorWithSignature("at least 4 arguments required");
      }

      validateAndSetupPatternStr(evaluator, args);
      validateAndSetupSymbolInfo(evaluator, args, argsNum);
      validateAndSetupResultExprStr(evaluator, args, argsNum);
      setupSymbolFunctionChain(evaluator);

      /*
       * setup OI for input to resultExpr select list
       */
      RowResolver selectListInputRR = MatchPath.createSelectListRR(evaluator, tDef.getInput());

      /*
       * parse ResultExpr Str and setup OI.
       */
      ResultExpressionParser resultExprParser =
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef

    @Override
    public void initializeOutputOI() throws HiveException {
      try {
        MatchPath evaluator = (MatchPath) getEvaluator();
        PartitionedTableFunctionDef tDef = evaluator.getTableDef();

        ArrayList<PTFExpressionDef> args = tDef.getArgs();
        int argsNum = args.size();

        validateAndSetupPatternStr(evaluator, args);
        validateAndSetupSymbolInfo(evaluator, args, argsNum);
        validateAndSetupResultExprStr(evaluator, args, argsNum);
        setupSymbolFunctionChain(evaluator);

        /*
         * setup OI for input to resultExpr select list
         */
        StructObjectInspector selectListInputOI = MatchPath.createSelectListOI( evaluator,
            tDef.getInput());
        ResultExprInfo resultExprInfo = evaluator.resultExprInfo;
        ArrayList<ObjectInspector> selectListExprOIs = new ArrayList<ObjectInspector>();
        resultExprInfo.resultExprEvals = new ArrayList<ExprNodeEvaluator>();

        for(int i=0 ; i < resultExprInfo.resultExprNodes.size(); i++) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.