Package org.apache.hadoop.hive.ql.plan.ptf

Examples of org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef


     * Input -> ReduceSink
     *
     * Here the ExprNodeDescriptors in the QueryDef are based on the Input Operator's RR.
     */
    {
      PartitionedTableFunctionDef tabDef = ptfDesc.getStartOfChain();

      /*
       * a. add Map-side PTF Operator if needed
       */
      if (tabDef.isTransformsRawInput() )
      {
        RowResolver ptfMapRR = tabDef.getRawInputShape().getRr();

        ptfDesc.setMapSide(true);
        input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc,
            new RowSchema(ptfMapRR.getColumnInfos()),
            input), ptfMapRR);
View Full Code Here


     */
    @Override
    public void setupOutputOI() throws SemanticException
    {
      MatchPath evaluator = (MatchPath) getEvaluator();
      PartitionedTableFunctionDef tDef = evaluator.getTableDef();

      List<PTFExpressionDef> args = tDef.getArgs();
      int argsNum = args == null ? 0 : args.size();

      if ( argsNum < 4 )
      {
        throwErrorWithSignature("at least 4 arguments required");
      }

      validateAndSetupPatternStr(evaluator, args);
      validateAndSetupSymbolInfo(evaluator, args, argsNum);
      validateAndSetupResultExprStr(evaluator, args, argsNum);
      setupSymbolFunctionChain(evaluator);

      /*
       * setup OI for input to resultExpr select list
       */
      RowResolver selectListInputRR = MatchPath.createSelectListRR(evaluator, tDef.getInput());

      /*
       * parse ResultExpr Str and setup OI.
       */
      ResultExpressionParser resultExprParser =
View Full Code Here

    @Override
    public void initializeOutputOI() throws HiveException {
      try {
        MatchPath evaluator = (MatchPath) getEvaluator();
        PartitionedTableFunctionDef tDef = evaluator.getTableDef();

        List<PTFExpressionDef> args = tDef.getArgs();
        int argsNum = args.size();

        validateAndSetupPatternStr(evaluator, args);
        validateAndSetupSymbolInfo(evaluator, args, argsNum);
        validateAndSetupResultExprStr(evaluator, args, argsNum);
        setupSymbolFunctionChain(evaluator);

        /*
         * setup OI for input to resultExpr select list
         */
        StructObjectInspector selectListInputOI = MatchPath.createSelectListOI( evaluator,
            tDef.getInput());
        ResultExprInfo resultExprInfo = evaluator.resultExprInfo;
        ArrayList<ObjectInspector> selectListExprOIs = new ArrayList<ObjectInspector>();
        resultExprInfo.resultExprEvals = new ArrayList<ExprNodeEvaluator>();

        for(int i=0 ; i < resultExprInfo.resultExprNodes.size(); i++) {
View Full Code Here

    reconstructQueryDef(hiveConf);
    inputPart = createFirstPartitionForChain(
        inputObjInspectors[0], hiveConf, isMapOperator);

    if (isMapOperator) {
      PartitionedTableFunctionDef tDef = conf.getStartOfChain();
      outputObjInspector = tDef.getRawInputShape().getOI();
    } else {
      outputObjInspector = conf.getFuncDef().getOutputShape().getOI();
    }

    setupKeysWrapper(inputObjInspectors[0]);
View Full Code Here

      forward(oRow, outputObjInspector);
    }
  }

  protected void processMapFunction() throws HiveException {
    PartitionedTableFunctionDef tDef = conf.getStartOfChain();
    PTFPartition outPart = tDef.getTFunction().transformRawInput(inputPart);
    PTFPartitionIterator<Object> pItr = outPart.iterator();
    while (pItr.hasNext()) {
      Object oRow = pItr.next();
      forward(oRow, outputObjInspector);
    }
View Full Code Here

    while (iDef instanceof PartitionedTableFunctionDef) {
      fnDefs.push((PartitionedTableFunctionDef) iDef);
      iDef = ((PartitionedTableFunctionDef) iDef).getInput();
    }

    PartitionedTableFunctionDef currFnDef;
    while (!fnDefs.isEmpty()) {
      currFnDef = fnDefs.pop();
      part = currFnDef.getTFunction().execute(part);
    }
    return part;
  }
View Full Code Here

   * @return
   * @throws HiveException
   */
  public PTFPartition createFirstPartitionForChain(ObjectInspector oi,
      HiveConf hiveConf, boolean isMapSide) throws HiveException {
    PartitionedTableFunctionDef tabDef = conf.getStartOfChain();
    TableFunctionEvaluator tEval = tabDef.getTFunction();

    PTFPartition part = null;
    SerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() :
      tabDef.getRawInputShape().getSerde();
    StructObjectInspector outputOI = isMapSide ? tabDef.getInput().getOutputShape().getOI() :
      tabDef.getRawInputShape().getOI();
    part = PTFPartition.create(conf.getCfg(),
        serde,
        (StructObjectInspector) oi,
        outputOI);

View Full Code Here

    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(spec.getName());
    if (tFn == null) {
      throw new SemanticException(String.format("Unknown Table Function %s",
          spec.getName()));
    }
    PartitionedTableFunctionDef def = new PartitionedTableFunctionDef();
    def.setInput(inpDef);
    def.setName(spec.getName());
    def.setResolverClassName(tFn.getClass().getName());
    def.setAlias(spec.getAlias() == null ? "ptf_" + inpNum : spec.getAlias());
    def.setExpressionTreeString(spec.getAstNode().toStringTree());
    def.setTransformsRawInput(tFn.transformsRawInput());
    /*
     * translate args
     */
    List<ASTNode> args = spec.getArgs();
    if (args != null)
    {
      for (ASTNode expr : args)
      {
        PTFExpressionDef argDef = null;
        try {
          argDef = buildExpressionDef(inpDef.getOutputShape(), expr);
        } catch (HiveException he) {
          throw new SemanticException(he);
        }
        def.addArg(argDef);
      }
    }

    tFn.initialize(hCfg, ptfDesc, def);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    def.setTFunction(tEval);
    def.setCarryForwardNames(tFn.carryForwardNames());
    tFn.setupRawInputOI();

    if (tFn.transformsRawInput()) {
      StructObjectInspector rawInOutOI = tEval.getRawInputOI();
      List<String> rawInOutColNames = tFn.getRawInputColumnNames();
      RowResolver rawInRR = buildRowResolverForPTF(def.getName(),
          spec.getAlias(),
          rawInOutOI,
          rawInOutColNames,
          inpDef.getOutputShape().getRr());
      ShapeDetails rawInpShape = setupTableFnShape(def.getName(),
          inpDef.getOutputShape(),
          rawInOutOI,
          rawInOutColNames,
          rawInRR);
      def.setRawInputShape(rawInpShape);
    }
    else {
      def.setRawInputShape(inpDef.getOutputShape());
    }

    translatePartitioning(def, spec);
    tFn.setupOutputOI();

    StructObjectInspector outputOI = tEval.getOutputOI();
    List<String> outColNames = tFn.getOutputColumnNames();
    RowResolver outRR = buildRowResolverForPTF(def.getName(),
        spec.getAlias(),
        outputOI,
        outColNames,
        def.getRawInputShape().getRr());
    ShapeDetails outputShape = setupTableFnShape(def.getName(),
        inpDef.getOutputShape(),
        outputOI,
        outColNames,
        outRR);
    def.setOutputShape(outputShape);

    return def;
  }
View Full Code Here

    boolean isStreaming() {
      return tabFn.canAcceptInputAsStream();
    }

    void initializeStreaming(Configuration cfg, boolean isMapSide) throws HiveException {
      PartitionedTableFunctionDef tabDef = tabFn.getTableDef();
      PTFInputDef inputDef = tabDef.getInput();
      ObjectInspector inputOI = conf.getStartOfChain() == tabDef ?
          inputObjInspectors[0] : inputDef.getOutputShape().getOI();

      tabFn.initializeStreaming(cfg, (StructObjectInspector) inputOI, isMapSide);
View Full Code Here

     * @param isMapSide
     * @return
     * @throws HiveException
     */
    private void createInputPartition() throws HiveException {
      PartitionedTableFunctionDef tabDef = tabFn.getTableDef();
      PTFInputDef inputDef = tabDef.getInput();
      ObjectInspector inputOI = conf.getStartOfChain() == tabDef ?
          inputObjInspectors[0] : inputDef.getOutputShape().getOI();

      SerDe serde = conf.isMapSide() ? tabDef.getInput().getOutputShape().getSerde() :
        tabDef.getRawInputShape().getSerde();
      StructObjectInspector outputOI = conf.isMapSide() ? tabDef.getInput().getOutputShape().getOI() :
        tabDef.getRawInputShape().getOI();
      inputPart = PTFPartition.create(conf.getCfg(),
          serde,
          (StructObjectInspector) inputOI,
          outputOI);
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.