* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

  private static final Log LOG = LogFactory.getLog(StatsRulesProcFactory.class.getName());

  private static final boolean isDebugEnabled = LOG.isDebugEnabled();

  /**

   * Collect basic statistics like number of rows, data size and column level statistics from the

   * table. Also sets the state of the available statistics. Basic and column statistics can have

   * one of the following states COMPLETE, PARTIAL, NONE. In case of partitioned table, the basic

   * and column stats are aggregated together to table level statistics. Column statistics will not

   * be collected if hive.stats.fetch.column.stats is set to false. If basic statistics is not

   * available then number of rows will be estimated from file size and average row size (computed

   * from schema).

   */

  public static class TableScanStatsRule extends DefaultStatsRule implements NodeProcessor {

    @Override

    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,

        Object... nodeOutputs) throws SemanticException {

      TableScanOperator tsop = (TableScanOperator) nd;

      AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx;

      PrunedPartitionList partList =

          aspCtx.getParseContext().getPrunedPartitions(tsop.getName(), tsop);

      Table table = aspCtx.getParseContext().getTopToTable().get(tsop);

      try {

        // gather statistics for the first time and the attach it to table scan operator

        Statistics stats = StatsUtils.collectStatistics(aspCtx.getConf(), partList, table, tsop);

        tsop.setStatistics(stats.clone());

        if (isDebugEnabled) {

          LOG.debug("[0] STATS-" + tsop.toString() + " (" + table.getTableName() + "): " +

              stats.extendedToString());

        }

      } catch (CloneNotSupportedException e) {

        throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg());

      } catch (HiveException e) {

        LOG.debug(e);

        throw new SemanticException(e);

      }

      return null;

    }

  }

  /**

   * SELECT operator doesn't change the number of rows emitted from the parent operator. It changes

   * the size of each tuple emitted. In a typical case, where only subset of columns are selected

   * the average row size will reduce as some of the columns will be pruned. In order to accurately

   * compute the average row size, column level statistics is required. Column level statistics

   * stores average size of values in column which can be used to more reliably estimate the

   * reduction in size of each tuple. In the absence of column level statistics, size of columns

   * will be based on data type. For primitive data types size from

   * {@link org.apache.hadoop.hive.ql.util.JavaDataModel} will be used and for variable length data

   * types worst case will be assumed.

   * <p>

   * <i>For more information, refer 'Estimating The Cost Of Operations' chapter in

   * "Database Systems: The Complete Book" by Garcia-Molina et. al.</i>

   * </p>

   */

  public static class SelectStatsRule extends DefaultStatsRule implements NodeProcessor {

    @Override

    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,

        Object... nodeOutputs) throws SemanticException {

      SelectOperator sop = (SelectOperator) nd;

      Operator<? extends OperatorDesc> parent = sop.getParentOperators().get(0);

      Statistics parentStats = parent.getStatistics();

      AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx;

      HiveConf conf = aspCtx.getConf();

      Statistics stats = null;

      if (parentStats != null) {

        try {

          stats = parentStats.clone();

        } catch (CloneNotSupportedException e) {

          throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg());

        }

      }

      try {

        if (satisfyPrecondition(parentStats)) {

          // this will take care of mapping between input column names and output column names. The

          // returned column stats will have the output column names.

          List<ColStatistics> colStats = StatsUtils.getColStatisticsFromExprMap(conf, parentStats,

              sop.getColumnExprMap(), sop.getSchema());

          stats.setColumnStats(colStats);

          // in case of select(*) the data size does not change

          if (!sop.getConf().isSelectStar() && !sop.getConf().isSelStarNoCompute()) {

            long dataSize = StatsUtils.getDataSizeFromColumnStats(stats.getNumRows(), colStats);

            stats.setDataSize(dataSize);

          }

          sop.setStatistics(stats);

          if (isDebugEnabled) {

            LOG.debug("[0] STATS-" + sop.toString() + ": " + stats.extendedToString());

          }

        } else {

          if (parentStats != null) {

            sop.setStatistics(parentStats.clone());

            if (isDebugEnabled) {

              LOG.debug("[1] STATS-" + sop.toString() + ": " + parentStats.extendedToString());

            }

          }

        }

      } catch (CloneNotSupportedException e) {

        throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg());

      }

      return null;

    }

  }

  /**

   * FILTER operator does not change the average row size but it does change the number of rows

   * emitted. The reduction in the number of rows emitted is dependent on the filter expression.

   * <ul>

   * <i>Notations:</i>

   * <li>T(S) - Number of tuples in relations S</li>

   * <li>V(S,A) - Number of distinct values of attribute A in relation S</li>

   * </ul>

   * <ul>

   * <i>Rules:</i> <b>

   * <li>Column equals a constant</li></b> T(S) = T(R) / V(R,A)

   * <p>

   * <b>

   * <li>Inequality conditions</li></b> T(S) = T(R) / 3

   * <p>

   * <b>

   * <li>Not equals comparison</li></b> - Simple formula T(S) = T(R)

   * <p>

   * - Alternate formula T(S) = T(R) (V(R,A) - 1) / V(R,A)

   * <p>

   * <b>

   * <li>NOT condition</li></b> T(S) = 1 - T(S'), where T(S') is the satisfying condition

   * <p>

   * <b>

   * <li>Multiple AND conditions</li></b> Cascadingly apply the rules 1 to 3 (order doesn't matter)

   * <p>

   * <b>

   * <li>Multiple OR conditions</li></b> - Simple formula is to evaluate conditions independently

   * and sum the results T(S) = m1 + m2

   * <p>

   * - Alternate formula T(S) = T(R) * ( 1 - ( 1 - m1/T(R) ) * ( 1 - m2/T(R) ))

   * <p>

   * where, m1 is the number of tuples that satisfy condition1 and m2 is the number of tuples that

   * satisfy condition2

   * </ul>

   * <p>

   * <i>Worst case:</i> If no column statistics are available, then evaluation of predicate

   * expression will assume worst case (i.e; half the input rows) for each of predicate expression.

   * <p>

   * <i>For more information, refer 'Estimating The Cost Of Operations' chapter in

   * "Database Systems: The Complete Book" by Garcia-Molina et. al.</i>

   * </p>

   */

  public static class FilterStatsRule extends DefaultStatsRule implements NodeProcessor {

    @Override

    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,

        Object... nodeOutputs) throws SemanticException {

      AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx;

      FilterOperator fop = (FilterOperator) nd;

      Operator<? extends OperatorDesc> parent = fop.getParentOperators().get(0);

      Statistics parentStats = parent.getStatistics();

      List<String> neededCols = null;

      if (parent instanceof TableScanOperator) {

        TableScanOperator tsop = (TableScanOperator) parent;

        neededCols = tsop.getNeededColumns();

      }

      try {

        if (parentStats != null) {

          ExprNodeDesc pred = fop.getConf().getPredicate();

          // evaluate filter expression and update statistics

          long newNumRows = evaluateExpression(parentStats, pred, aspCtx,

              neededCols, fop);

          Statistics st = parentStats.clone();

          if (satisfyPrecondition(parentStats)) {

            // update statistics based on column statistics.

            // OR conditions keeps adding the stats independently, this may

            // result in number of rows getting more than the input rows in

            // which case stats need not be updated

            if (newNumRows <= parentStats.getNumRows()) {

              updateStats(st, newNumRows, true, fop);

            }

            if (isDebugEnabled) {

              LOG.debug("[0] STATS-" + fop.toString() + ": " + st.extendedToString());

            }

          } else {

            // update only the basic statistics in the absence of column statistics

            if (newNumRows <= parentStats.getNumRows()) {

              updateStats(st, newNumRows, false, fop);

            }

            if (isDebugEnabled) {

              LOG.debug("[1] STATS-" + fop.toString() + ": " + st.extendedToString());

            }

          }

          fop.setStatistics(st);

          aspCtx.setAndExprStats(null);

        }

      } catch (CloneNotSupportedException e) {

        throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg());

      }

      return null;

    }

    private long evaluateExpression(Statistics stats, ExprNodeDesc pred,

        AnnotateStatsProcCtx aspCtx, List<String> neededCols,

        FilterOperator fop) throws CloneNotSupportedException {

      long newNumRows = 0;

      Statistics andStats = null;

      if (pred instanceof ExprNodeGenericFuncDesc) {

        ExprNodeGenericFuncDesc genFunc = (ExprNodeGenericFuncDesc) pred;

        GenericUDF udf = genFunc.getGenericUDF();

        // for AND condition cascadingly update stats

        if (udf instanceof GenericUDFOPAnd) {

          andStats = stats.clone();

          aspCtx.setAndExprStats(andStats);

          // evaluate children

          for (ExprNodeDesc child : genFunc.getChildren()) {

            newNumRows = evaluateChildExpr(aspCtx.getAndExprStats(), child,

                aspCtx, neededCols, fop);

            if (satisfyPrecondition(aspCtx.getAndExprStats())) {

              updateStats(aspCtx.getAndExprStats(), newNumRows, true, fop);

            } else {

              updateStats(aspCtx.getAndExprStats(), newNumRows, false, fop);

            }

          }

        } else if (udf instanceof GenericUDFOPOr) {

          // for OR condition independently compute and update stats

          for (ExprNodeDesc child : genFunc.getChildren()) {

            newNumRows = StatsUtils.safeAdd(

                evaluateChildExpr(stats, child, aspCtx, neededCols, fop), newNumRows);

          }

        } else if (udf instanceof GenericUDFOPNot) {

          newNumRows = evaluateNotExpr(stats, pred, aspCtx, neededCols, fop);

        } else {

          // single predicate condition

          newNumRows = evaluateChildExpr(stats, pred, aspCtx, neededCols, fop);

        }

      } else if (pred instanceof ExprNodeColumnDesc) {

        // can be boolean column in which case return true count

        ExprNodeColumnDesc encd = (ExprNodeColumnDesc) pred;

        String colName = encd.getColumn();

        String tabAlias = encd.getTabAlias();

        String colType = encd.getTypeString();

        if (colType.equalsIgnoreCase(serdeConstants.BOOLEAN_TYPE_NAME)) {

          ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName);

          if (cs != null) {

            return cs.getNumTrues();

          }

        }

        // if not boolean column return half the number of rows

        return stats.getNumRows() / 2;

      } else if (pred instanceof ExprNodeConstantDesc) {

        // special case for handling false constants

        ExprNodeConstantDesc encd = (ExprNodeConstantDesc) pred;

        if (encd.getValue().equals(false)) {

          return 0;

        } else {

          return stats.getNumRows();

        }

      }

      return newNumRows;

    }

    private long evaluateNotExpr(Statistics stats, ExprNodeDesc pred,

        AnnotateStatsProcCtx aspCtx, List<String> neededCols, FilterOperator fop)

        throws CloneNotSupportedException {

      long numRows = stats.getNumRows();

      // if the evaluate yields true then pass all rows else pass 0 rows

      if (pred instanceof ExprNodeGenericFuncDesc) {

        ExprNodeGenericFuncDesc genFunc = (ExprNodeGenericFuncDesc) pred;

        for (ExprNodeDesc leaf : genFunc.getChildren()) {

          if (leaf instanceof ExprNodeGenericFuncDesc) {

            // GenericUDF

            long newNumRows = 0;

            for (ExprNodeDesc child : genFunc.getChildren()) {

              newNumRows = evaluateChildExpr(stats, child, aspCtx, neededCols,

                  fop);

            }

            return numRows - newNumRows;

          } else if (leaf instanceof ExprNodeConstantDesc) {

            ExprNodeConstantDesc encd = (ExprNodeConstantDesc) leaf;

            if (encd.getValue().equals(true)) {

              return 0;

            } else {

              return numRows;

            }

          } else if (leaf instanceof ExprNodeColumnDesc) {

            // NOT on boolean columns is possible. in which case return false count.

            ExprNodeColumnDesc encd = (ExprNodeColumnDesc) leaf;

            String colName = encd.getColumn();

            String tabAlias = encd.getTabAlias();

            String colType = encd.getTypeString();

            if (colType.equalsIgnoreCase(serdeConstants.BOOLEAN_TYPE_NAME)) {

              ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName);

              if (cs != null) {

                return cs.getNumFalses();

              }

            }

            // if not boolean column return half the number of rows

            return numRows / 2;

          }

        }

      }

      // worst case

      return numRows / 2;

    }

    private long evaluateColEqualsNullExpr(Statistics stats, ExprNodeDesc pred) {

      long numRows = stats.getNumRows();

      if (pred instanceof ExprNodeGenericFuncDesc) {

        ExprNodeGenericFuncDesc genFunc = (ExprNodeGenericFuncDesc) pred;

        for (ExprNodeDesc leaf : genFunc.getChildren()) {

          if (leaf instanceof ExprNodeColumnDesc) {

            ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) leaf;

            String colName = colDesc.getColumn();

            String tabAlias = colDesc.getTabAlias();

            ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName);

            if (cs != null) {

              return cs.getNumNulls();

            }

          }

        }

      }

      // worst case

      return numRows / 2;

    }

    private long evaluateChildExpr(Statistics stats, ExprNodeDesc child,

        AnnotateStatsProcCtx aspCtx, List<String> neededCols,

        FilterOperator fop) throws CloneNotSupportedException {

      long numRows = stats.getNumRows();

      if (child instanceof ExprNodeGenericFuncDesc) {

        ExprNodeGenericFuncDesc genFunc = (ExprNodeGenericFuncDesc) child;

        GenericUDF udf = genFunc.getGenericUDF();

        if (udf instanceof GenericUDFOPEqual ||

            udf instanceof GenericUDFOPEqualNS) {

          String colName = null;

          String tabAlias = null;

          boolean isConst = false;

          Object prevConst = null;

          for (ExprNodeDesc leaf : genFunc.getChildren()) {

            if (leaf instanceof ExprNodeConstantDesc) {

              // constant = constant expressions. We shouldn't be getting this

              // after constant folding

              if (isConst) {

                // special case: if both constants are not equal then return 0

                if (prevConst != null &&

                    !prevConst.equals(((ExprNodeConstantDesc)leaf).getValue())) {

                  return 0;

                }

                return numRows;

              }

              // if the first argument is const then just set the flag and continue

              if (colName == null) {

                isConst = true;

                prevConst = ((ExprNodeConstantDesc) leaf).getValue();

                continue;

              }

              // if column name is not contained in needed column list then it

              // is a partition column. We do not need to evaluate partition columns

              // in filter expression since it will be taken care by partitio pruner

              if (neededCols != null && !neededCols.contains(colName)) {

                return numRows;

              }

              ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName);

              if (cs != null) {

                long dvs = cs.getCountDistint();

                numRows = dvs == 0 ? numRows / 2 : numRows / dvs;

                return numRows;

              }

            } else if (leaf instanceof ExprNodeColumnDesc) {

              ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) leaf;

              colName = colDesc.getColumn();

              tabAlias = colDesc.getTabAlias();

              // if const is first argument then evaluate the result

              if (isConst) {

                // if column name is not contained in needed column list then it

                // is a partition column. We do not need to evaluate partition columns

                // in filter expression since it will be taken care by partitio pruner

                if (neededCols != null && neededCols.indexOf(colName) == -1) {

                  return numRows;

                }

                ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName);

                if (cs != null) {

                  long dvs = cs.getCountDistint();

                  numRows = dvs == 0 ? numRows / 2 : numRows / dvs;

                  return numRows;

                }

              }

            }

          }

        } else if (udf instanceof GenericUDFOPNotEqual) {

          return numRows;

        } else if (udf instanceof GenericUDFOPEqualOrGreaterThan

            || udf instanceof GenericUDFOPEqualOrLessThan || udf instanceof GenericUDFOPGreaterThan

            || udf instanceof GenericUDFOPLessThan) {

          return numRows / 3;

        } else if (udf instanceof GenericUDFOPNotNull) {

          long newNumRows = evaluateColEqualsNullExpr(stats, genFunc);

          return stats.getNumRows() - newNumRows;

        } else if (udf instanceof GenericUDFOPNull) {

          return evaluateColEqualsNullExpr(stats, genFunc);

        } else if (udf instanceof GenericUDFOPAnd || udf instanceof GenericUDFOPOr

            || udf instanceof GenericUDFOPNot) {

          return evaluateExpression(stats, genFunc, aspCtx, neededCols, fop);

        }

      }

      // worst case

      return numRows / 2;

    }

  }

   * GROUPBY operator changes the number of rows. The number of rows emitted by GBY operator will be

   * atleast 1 or utmost T(R) (number of rows in relation T) based on the aggregation. A better

   * estimate can be found if we have column statistics on the columns that we are grouping on.

   * <p>

   * Suppose if we are grouping by attributes A,B,C and if statistics for columns A,B,C are

   * available then a better estimate can be found by taking the smaller of product of V(R,[A,B,C])

   * (product of distinct cardinalities of A,B,C) and T(R)/2.

   * <p>

   * T(R) = min (T(R)/2 , V(R,[A,B,C]) ---> [1]

   * <p>

   * In the presence of grouping sets, map-side GBY will emit more rows depending on the size of

   * grouping set (input rows * size of grouping set). These rows will get reduced because of

   * map-side hash aggregation. Hash aggregation is an optimization in hive to reduce the number of

   * rows shuffled between map and reduce stage. This optimization will be disabled if the memory

   * used for hash aggregation exceeds 90% of max available memory for hash aggregation. The number

   * of rows emitted from map-side will vary if hash aggregation is enabled throughout execution or

   * disabled. In the presence of grouping sets, following rules will be applied

   * <p>

   * If <b>hash-aggregation is enabled</b>, for query SELECT * FROM table GROUP BY (A,B) WITH CUBE

   * <p>

   * T(R) = min(T(R)/2, T(R, GBY(A,B)) + T(R, GBY(A)) + T(R, GBY(B)) + 1))

   * <p>

   * where, GBY(A,B), GBY(B), GBY(B) are the GBY rules mentioned above [1]

   * <p>

   * If <b>hash-aggregation is disabled</b>, apply the GBY rule [1] and then multiply the result by

   * number of elements in grouping set T(R) = T(R) * length_of_grouping_set. Since we do not know

   * if hash-aggregation is enabled or disabled during compile time, we will assume worst-case i.e,

   * hash-aggregation is disabled

   * <p>

   * NOTE: The number of rows from map-side GBY operator is dependent on map-side parallelism i.e,

   * number of mappers. The map-side parallelism is expected from hive config

   * "hive.stats.map.parallelism". If the config is not set then default parallelism of 1 will be

   * assumed.

   * <p>

   * <i>Worst case:</i> If no column statistics are available, then T(R) = T(R)/2 will be used as

   * heuristics.

   * <p>

   * <i>For more information, refer 'Estimating The Cost Of Operations' chapter in

   * "Database Systems: The Complete Book" by Garcia-Molina et. al.</i>

   * </p>

   */

  public static class GroupByStatsRule extends DefaultStatsRule implements NodeProcessor {

    @Override

    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,

        Object... nodeOutputs) throws SemanticException {

      GroupByOperator gop = (GroupByOperator) nd;

      Operator<? extends OperatorDesc> parent = gop.getParentOperators().get(0);

      Statistics parentStats = parent.getStatistics();

      // parent stats are not populated yet

      if (parentStats == null) {

        return null;

      }

      AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx;

      HiveConf conf = aspCtx.getConf();

      long maxSplitSize = HiveConf.getLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE);

      List<AggregationDesc> aggDesc = gop.getConf().getAggregators();

      Map<String, ExprNodeDesc> colExprMap = gop.getColumnExprMap();

      RowSchema rs = gop.getSchema();

      Statistics stats = null;

      List<ColStatistics> colStats = StatsUtils.getColStatisticsFromExprMap(conf, parentStats,

          colExprMap, rs);

      long cardinality;

      long parallelism = 1L;

      boolean mapSide = false;

      boolean mapSideHashAgg = false;

      long inputSize = 1L;

      boolean containsGroupingSet = gop.getConf().isGroupingSetsPresent();

      long sizeOfGroupingSet =

          containsGroupingSet ? gop.getConf().getListGroupingSets().size() : 1L;

      // There are different cases for Group By depending on map/reduce side, hash aggregation,

      // grouping sets and column stats. If we don't have column stats, we just assume hash

      // aggregation is disabled. Following are the possible cases and rule for cardinality

      // estimation

      // MAP SIDE:

      // Case 1: NO column stats, NO hash aggregation, NO grouping sets — numRows

      // Case 2: NO column stats, NO hash aggregation, grouping sets — numRows * sizeOfGroupingSet

      // Case 3: column stats, hash aggregation, NO grouping sets — Min(numRows / 2, ndvProduct * parallelism)

      // Case 4: column stats, hash aggregation, grouping sets — Min((numRows * sizeOfGroupingSet) / 2, ndvProduct * parallelism * sizeOfGroupingSet)

      // Case 5: column stats, NO hash aggregation, NO grouping sets — numRows

      // Case 6: column stats, NO hash aggregation, grouping sets — numRows * sizeOfGroupingSet

      // REDUCE SIDE:

      // Case 7: NO column stats — numRows / 2

      // Case 8: column stats, grouping sets — Min(numRows, ndvProduct * sizeOfGroupingSet)

      // Case 9: column stats, NO grouping sets - Min(numRows, ndvProduct)

      if (gop.getChildOperators().get(0) instanceof ReduceSinkOperator ||

          gop.getChildOperators().get(0) instanceof AppMasterEventOperator) {

        mapSide = true;

        // consider approximate map side parallelism to be table data size

        // divided by max split size

        TableScanOperator top = OperatorUtils.findSingleOperatorUpstream(gop,

            TableScanOperator.class);

        // if top is null then there are multiple parents (RS as well), hence

        // lets use parent statistics to get data size. Also maxSplitSize should

        // be updated to bytes per reducer (1GB default)

        if (top == null) {

          inputSize = parentStats.getDataSize();

          maxSplitSize = HiveConf.getLongVar(conf, HiveConf.ConfVars.BYTESPERREDUCER);

        } else {

          inputSize = top.getConf().getStatistics().getDataSize();

        }

        parallelism = (int) Math.ceil((double) inputSize / maxSplitSize);

      }

      if (isDebugEnabled) {

        LOG.debug("STATS-" + gop.toString() + ": inputSize: " + inputSize + " maxSplitSize: " +

            maxSplitSize + " parallelism: " + parallelism + " containsGroupingSet: " +

            containsGroupingSet + " sizeOfGroupingSet: " + sizeOfGroupingSet);

      }

      try {

        // satisfying precondition means column statistics is available

        if (satisfyPrecondition(parentStats)) {

          // check if map side aggregation is possible or not based on column stats

          mapSideHashAgg = checkMapSideAggregation(gop, colStats, conf);

          if (isDebugEnabled) {

            LOG.debug("STATS-" + gop.toString() + " mapSideHashAgg: " + mapSideHashAgg);

          }

          stats = parentStats.clone();

          stats.setColumnStats(colStats);

          long ndvProduct = 1;

          final long parentNumRows = stats.getNumRows();

          // compute product of distinct values of grouping columns

          for (ColStatistics cs : colStats) {

            if (cs != null) {

              long ndv = cs.getCountDistint();

              if (cs.getNumNulls() > 0) {

                ndv = StatsUtils.safeAdd(ndv, 1);

              }

              ndvProduct = StatsUtils.safeMult(ndvProduct, ndv);

            } else {

              if (parentStats.getColumnStatsState().equals(Statistics.State.COMPLETE)) {

                // the column must be an aggregate column inserted by GBY. We

                // don't have to account for this column when computing product

                // of NDVs

                continue;

              } else {

                // partial column statistics on grouping attributes case.

                // if column statistics on grouping attribute is missing, then

                // assume worst case.

                // GBY rule will emit half the number of rows if ndvProduct is 0

                ndvProduct = 0;

              }

              break;

            }

          }

          // if ndvProduct is 0 then column stats state must be partial and we are missing

          // column stats for a group by column

          if (ndvProduct == 0) {

            ndvProduct = parentNumRows / 2;

            if (isDebugEnabled) {

              LOG.debug("STATS-" + gop.toString() + ": ndvProduct became 0 as some column does not" +

                  " have stats. ndvProduct changed to: " + ndvProduct);

            }

          }

          if (mapSide) {

            // MAP SIDE

            if (mapSideHashAgg) {

              if (containsGroupingSet) {

                // Case 4: column stats, hash aggregation, grouping sets

                cardinality = Math.min(

                    (StatsUtils.safeMult(parentNumRows, sizeOfGroupingSet)) / 2,

                    StatsUtils.safeMult(StatsUtils.safeMult(ndvProduct, parallelism), sizeOfGroupingSet));

                if (isDebugEnabled) {

                  LOG.debug("[Case 4] STATS-" + gop.toString() + ": cardinality: " + cardinality);

                }

              } else {

                // Case 3: column stats, hash aggregation, NO grouping sets

                cardinality = Math.min(parentNumRows / 2, StatsUtils.safeMult(ndvProduct, parallelism));

                if (isDebugEnabled) {

                  LOG.debug("[Case 3] STATS-" + gop.toString() + ": cardinality: " + cardinality);

                }

              }

            } else {

              if (containsGroupingSet) {

                // Case 6: column stats, NO hash aggregation, grouping sets

                cardinality = StatsUtils.safeMult(parentNumRows, sizeOfGroupingSet);

                if (isDebugEnabled) {

                  LOG.debug("[Case 6] STATS-" + gop.toString() + ": cardinality: " + cardinality);

                }

              } else {

                // Case 5: column stats, NO hash aggregation, NO grouping sets

                cardinality = parentNumRows;

                if (isDebugEnabled) {

                  LOG.debug("[Case 5] STATS-" + gop.toString() + ": cardinality: " + cardinality);

                }

              }

            }

          } else {

            // REDUCE SIDE

            // in reduce side GBY, we don't know if the grouping set was present or not. so get it

            // from map side GBY

            GroupByOperator mGop = OperatorUtils.findSingleOperatorUpstream(parent, GroupByOperator.class);

            if (mGop != null) {

              containsGroupingSet = mGop.getConf().isGroupingSetsPresent();

              sizeOfGroupingSet = mGop.getConf().getListGroupingSets().size();

            }

            if (containsGroupingSet) {

              // Case 8: column stats, grouping sets

              cardinality = Math.min(parentNumRows, StatsUtils.safeMult(ndvProduct, sizeOfGroupingSet));

              if (isDebugEnabled) {

                LOG.debug("[Case 8] STATS-" + gop.toString() + ": cardinality: " + cardinality);

              }

            } else {

              // Case 9: column stats, NO grouping sets

              cardinality = Math.min(parentNumRows, ndvProduct);

              if (isDebugEnabled) {

                LOG.debug("[Case 9] STATS-" + gop.toString() + ": cardinality: " + cardinality);

              }

            }

          }

          // update stats, but don't update NDV as it will not change

          updateStats(stats, cardinality, true, gop, false);

        } else {

          // NO COLUMN STATS

          if (parentStats != null) {

            stats = parentStats.clone();

            final long parentNumRows = stats.getNumRows();

            // if we don't have column stats, we just assume hash aggregation is disabled

            if (mapSide) {

              // MAP SIDE

              if (containsGroupingSet) {

                // Case 2: NO column stats, NO hash aggregation, grouping sets

                cardinality = StatsUtils.safeMult(parentNumRows, sizeOfGroupingSet);

                if (isDebugEnabled) {

                  LOG.debug("[Case 2] STATS-" + gop.toString() + ": cardinality: " + cardinality);

                }

              } else {

                // Case 1: NO column stats, NO hash aggregation, NO grouping sets

                cardinality = parentNumRows;

                if (isDebugEnabled) {

                  LOG.debug("[Case 1] STATS-" + gop.toString() + ": cardinality: " + cardinality);

                }

              }

            } else {

              // REDUCE SIDE

              // Case 7: NO column stats

              cardinality = parentNumRows / 2;

              if (isDebugEnabled) {

                LOG.debug("[Case 7] STATS-" + gop.toString() + ": cardinality: " + cardinality);

              }

            }

            updateStats(stats, cardinality, false, gop);

          }

        }

        // if UDAFs are present, new columns needs to be added

        if (!aggDesc.isEmpty() && stats != null) {

          List<ColStatistics> aggColStats = Lists.newArrayList();

          for (ColumnInfo ci : rs.getSignature()) {

            // if the columns in row schema is not contained in column

            // expression map, then those are the aggregate columns that

            // are added GBY operator. we will estimate the column statistics

            // for those newly added columns

            if (!colExprMap.containsKey(ci.getInternalName())) {

              String colName = ci.getInternalName();

              String tabAlias = ci.getTabAlias();

              String colType = ci.getTypeName();

              ColStatistics cs = new ColStatistics(tabAlias, colName, colType);

              cs.setCountDistint(stats.getNumRows());

              cs.setNumNulls(0);

              cs.setAvgColLen(StatsUtils.getAvgColLenOfFixedLengthTypes(colType));

              aggColStats.add(cs);

            }

          }

          // add the new aggregate column and recompute data size

          if (aggColStats.size() > 0) {

            stats.addToColumnStats(aggColStats);

            // only if the column stats is available, update the data size from

            // the column stats

            if (!stats.getColumnStatsState().equals(Statistics.State.NONE)) {

              updateStats(stats, stats.getNumRows(), true, gop);

            }

          }

          // if UDAF present and if column expression map is empty then it must

          // be full aggregation query like count(*) in which case number of

          // rows will be 1

          if (colExprMap.isEmpty()) {

            stats.setNumRows(1);

            updateStats(stats, 1, true, gop);

          }

        }

        gop.setStatistics(stats);

        if (isDebugEnabled && stats != null) {

          LOG.debug("[0] STATS-" + gop.toString() + ": " + stats.extendedToString());

        }

      } catch (CloneNotSupportedException e) {

        throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg());

      }

      return null;

    }

    /**

     * This method does not take into account many configs used at runtime to

     * disable hash aggregation like HIVEMAPAGGRHASHMINREDUCTION. This method

     * roughly estimates the number of rows and size of each row to see if it

     * can fit in hashtable for aggregation.

     * @param gop - group by operator

     * @param colStats - column stats for key columns

     * @param conf - hive conf

     * @return

     */

    private boolean checkMapSideAggregation(GroupByOperator gop,

        List<ColStatistics> colStats, HiveConf conf) {

      List<AggregationDesc> aggDesc = gop.getConf().getAggregators();

      GroupByDesc desc = gop.getConf();

      GroupByDesc.Mode mode = desc.getMode();

      if (mode.equals(GroupByDesc.Mode.HASH)) {

        float hashAggMem = conf.getFloatVar(HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY);

        float hashAggMaxThreshold = conf.getFloatVar(HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);

        // get available map memory

        long totalMemory = StatsUtils.getAvailableMemory(conf) * 1000L * 1000L;

        long maxMemHashAgg = Math.round(totalMemory * hashAggMem * hashAggMaxThreshold);

        // estimated number of rows will be product of NDVs

        long numEstimatedRows = 1;

        // estimate size of key from column statistics

        long avgKeySize = 0;

        for (ColStatistics cs : colStats) {

          if (cs != null) {

            numEstimatedRows = StatsUtils.safeMult(numEstimatedRows, cs.getCountDistint());

            avgKeySize += Math.ceil(cs.getAvgColLen());

          }

        }

        // average value size will be sum of all sizes of aggregation buffers

        long avgValSize = 0;

        // go over all aggregation buffers and see they implement estimable

        // interface if so they aggregate the size of the aggregation buffer

        GenericUDAFEvaluator[] aggregationEvaluators;

        aggregationEvaluators = new GenericUDAFEvaluator[aggDesc.size()];

        // get aggregation evaluators

        for (int i = 0; i < aggregationEvaluators.length; i++) {

          AggregationDesc agg = aggDesc.get(i);

          aggregationEvaluators[i] = agg.getGenericUDAFEvaluator();

        }

        // estimate size of aggregation buffer

        for (int i = 0; i < aggregationEvaluators.length; i++) {

          // each evaluator has constant java object overhead

          avgValSize += gop.javaObjectOverHead;

          GenericUDAFEvaluator.AggregationBuffer agg = null;

          try {

            agg = aggregationEvaluators[i].getNewAggregationBuffer();

          } catch (HiveException e) {

            // in case of exception assume unknown type (256 bytes)

            avgValSize += gop.javaSizeUnknownType;

          }

          // aggregate size from aggregation buffers

          if (agg != null) {

            if (GenericUDAFEvaluator.isEstimable(agg)) {

              avgValSize += ((GenericUDAFEvaluator.AbstractAggregationBuffer) agg)

                  .estimate();

            } else {

              // if the aggregation buffer is not estimable then get all the

              // declared fields and compute the sizes from field types

              Field[] fArr = ObjectInspectorUtils

                  .getDeclaredNonStaticFields(agg.getClass());

              for (Field f : fArr) {

                long avgSize = StatsUtils

                    .getAvgColLenOfFixedLengthTypes(f.getType().getName());

                avgValSize += avgSize == 0 ? gop.javaSizeUnknownType : avgSize;

              }

            }

          }

        }

        // total size of each hash entry

        long hashEntrySize = gop.javaHashEntryOverHead + avgKeySize + avgValSize;

        // estimated hash table size

        long estHashTableSize = StatsUtils.safeMult(numEstimatedRows, hashEntrySize);

        if (estHashTableSize < maxMemHashAgg) {

          return true;

        }

      }

      // worst-case, hash aggregation disabled

      return false;

    }

    private long applyGBYRule(long numRows, long dvProd) {

      long newNumRows = numRows;

      // to avoid divide by 2 to become 0

      if (numRows > 1) {

        if (dvProd != 0) {

          newNumRows = Math.min(numRows / 2, dvProd);

        } else {

          newNumRows = numRows / 2;

        }

      }

      return newNumRows;

    }

  }

  /**

   * JOIN operator can yield any of the following three cases <li>The values of join keys are

   * disjoint in both relations in which case T(RXS) = 0 (we need histograms for this)</li> <li>Join

   * key is primary key on relation R and foreign key on relation S in which case every tuple in S

   * will have a tuple in R T(RXS) = T(S) (we need histograms for this)</li> <li>Both R & S relation

   * have same value for join-key. Ex: bool column with all true values T(RXS) = T(R) * T(S) (we

   * need histograms for this. counDistinct = 1 and same value)</li>

   * <p>

   * In the absence of histograms, we can use the following general case

   * <p>

   * <b>2 Relations, 1 attribute</b>

   * <p>

   * T(RXS) = (T(R)*T(S))/max(V(R,Y), V(S,Y)) where Y is the join attribute

   * <p>

   * <b>2 Relations, 2 attributes</b>

   * <p>

   * T(RXS) = T(R)*T(S)/max(V(R,y1), V(S,y1)) * max(V(R,y2), V(S,y2)), where y1 and y2 are the join

   * attributes

   * <p>

   * <b>3 Relations, 1 attributes</b>

   * <p>

   * T(RXSXQ) = T(R)*T(S)*T(Q)/top2largest(V(R,y), V(S,y), V(Q,y)), where y is the join attribute

   * <p>

   * <b>3 Relations, 2 attributes</b>

   * <p>

   * T(RXSXQ) = T(R)*T(S)*T(Q)/top2largest(V(R,y1), V(S,y1), V(Q,y1)) * top2largest(V(R,y2), V(S,y2), V(Q,y2)),

   * where y1 and y2 are the join attributes

   * <p>

   * <i>Worst case:</i> If no column statistics are available, then T(RXS) = joinFactor * max(T(R),

   * T(S)) * (numParents - 1) will be used as heuristics. joinFactor is from hive.stats.join.factor

   * hive config. In the worst case, since we do not know any information about join keys (and hence

   * which of the 3 cases to use), we let it to the user to provide the join factor.

   * <p>

   * <i>For more information, refer 'Estimating The Cost Of Operations' chapter in

   * "Database Systems: The Complete Book" by Garcia-Molina et. al.</i>

   * </p>

   */

  public static class JoinStatsRule extends DefaultStatsRule implements NodeProcessor {

    private boolean pkfkInferred = false;

    private long newNumRows = 0;

    private List<Operator<? extends OperatorDesc>> parents;

    private CommonJoinOperator<? extends JoinDesc> jop;

    private int numAttr = 1;

    @Override

    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,

        Object... nodeOutputs) throws SemanticException {

      jop = (CommonJoinOperator<? extends JoinDesc>) nd;

      parents = jop.getParentOperators();

      AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx;

      HiveConf conf = aspCtx.getConf();

      boolean allStatsAvail = true;

      boolean allSatisfyPreCondition = true;

      for (Operator<? extends OperatorDesc> op : parents) {

        if (op.getStatistics() == null) {

          allStatsAvail = false;

        }

      }

      if (allStatsAvail) {

        for (Operator<? extends OperatorDesc> op : parents) {

          if (!satisfyPrecondition(op.getStatistics())) {

            allSatisfyPreCondition = false;

          }

        }

        if (allSatisfyPreCondition) {

          // statistics object that is combination of statistics from all

          // relations involved in JOIN

          Statistics stats = new Statistics();

          Map<String, Long> rowCountParents = new HashMap<String, Long>();

          List<Long> distinctVals = Lists.newArrayList();

          int numParent = parents.size();

          Map<String, ColStatistics> joinedColStats = Maps.newHashMap();

          Map<Integer, List<String>> joinKeys = Maps.newHashMap();

          List<Long> rowCounts = Lists.newArrayList();

          // detect if there are multiple attributes in join key

          ReduceSinkOperator rsOp = (ReduceSinkOperator) jop.getParentOperators().get(0);

          List<String> keyExprs = rsOp.getConf().getOutputKeyColumnNames();

          numAttr = keyExprs.size();

          // infer PK-FK relationship in single attribute join case

          pkfkInferred = false;

          inferPKFKRelationship();

          // get the join keys from parent ReduceSink operators

          for (int pos = 0; pos < parents.size(); pos++) {

            ReduceSinkOperator parent = (ReduceSinkOperator) jop.getParentOperators().get(pos);

            Statistics parentStats = parent.getStatistics();

            keyExprs = parent.getConf().getOutputKeyColumnNames();

            // Parent RS may have column statistics from multiple parents.

            // Populate table alias to row count map, this will be used later to

            // scale down/up column statistics based on new row count

            // NOTE: JOIN with UNION as parent of RS will not have table alias

            // propagated properly. UNION operator does not propagate the table

            // alias of subqueries properly to expression nodes. Hence union20.q

            // will have wrong number of rows.

            Set<String> tableAliases = StatsUtils.getAllTableAlias(parent.getColumnExprMap());

            for (String tabAlias : tableAliases) {

              rowCountParents.put(tabAlias, parentStats.getNumRows());

            }

            rowCounts.add(parentStats.getNumRows());

            // compute fully qualified join key column names. this name will be

            // used to quickly look-up for column statistics of join key.

            // TODO: expressions in join condition will be ignored. assign

            // internal name for expressions and estimate column statistics for expression.

            List<String> fqCols = StatsUtils.getFullyQualifedReducerKeyNames(keyExprs,

                parent.getColumnExprMap());

            joinKeys.put(pos, fqCols);

            // get column statistics for all output columns

            for (ColStatistics cs : parentStats.getColumnStats()) {

              joinedColStats.put(cs.getFullyQualifiedColName(), cs);

            }

            // since new statistics is derived from all relations involved in

            // JOIN, we need to update the state information accordingly

            stats.updateColumnStatsState(parentStats.getColumnStatsState());

          }

          // compute denominator i.e, max(V(R,Y), V(S,Y)) in case of single

          // attribute join, else max(V(R,y1), V(S,y1)) * max(V(R,y2), V(S,y2))

          // in case of multi-attribute join

          long denom = 1;

          if (numAttr > 1) {

            List<Long> perAttrDVs = Lists.newArrayList();

            for (int idx = 0; idx < numAttr; idx++) {

              for (Integer i : joinKeys.keySet()) {

                String col = joinKeys.get(i).get(idx);

                ColStatistics cs = joinedColStats.get(col);

                if (cs != null) {

                  perAttrDVs.add(cs.getCountDistint());

                }

              }

              distinctVals.add(getDenominator(perAttrDVs));

              perAttrDVs.clear();

            }

            if (numAttr > numParent) {

              // To avoid denominator getting larger and aggressively reducing

              // number of rows, we will ease out denominator.

              denom = getEasedOutDenominator(distinctVals);

            } else {

              for (Long l : distinctVals) {

                denom = StatsUtils.safeMult(denom, l);

              }

            }

          } else {

            for (List<String> jkeys : joinKeys.values()) {

              for (String jk : jkeys) {

                ColStatistics cs = joinedColStats.get(jk);

                if (cs != null) {

                  distinctVals.add(cs.getCountDistint());

                }

              }

            }

            denom = getDenominator(distinctVals);

          }

          // Update NDV of joined columns to be min(V(R,y), V(S,y))

          updateJoinColumnsNDV(joinKeys, joinedColStats, numAttr);

          // column statistics from different sources are put together and rename

          // fully qualified column names based on output schema of join operator

          Map<String, ExprNodeDesc> colExprMap = jop.getColumnExprMap();

          RowSchema rs = jop.getSchema();

          List<ColStatistics> outColStats = Lists.newArrayList();

          Map<String, String> outInTabAlias = new HashMap<String, String>();

          for (ColumnInfo ci : rs.getSignature()) {

            String key = ci.getInternalName();

            ExprNodeDesc end = colExprMap.get(key);

            if (end instanceof ExprNodeColumnDesc) {

              String colName = ((ExprNodeColumnDesc) end).getColumn();

              String tabAlias = ((ExprNodeColumnDesc) end).getTabAlias();

              String fqColName = StatsUtils.getFullyQualifiedColumnName(tabAlias, colName);

              ColStatistics cs = joinedColStats.get(fqColName);

              String outColName = key;

              String outTabAlias = ci.getTabAlias();

              outInTabAlias.put(outTabAlias, tabAlias);

              if (cs != null) {

                cs.setColumnName(outColName);

                cs.setTableAlias(outTabAlias);

              }

              outColStats.add(cs);

            }

          }

          // update join statistics

          stats.setColumnStats(outColStats);

          long newRowCount = pkfkInferred ? newNumRows : computeNewRowCount(rowCounts, denom);

          updateStatsForJoinType(stats, newRowCount, jop, rowCountParents,outInTabAlias);

          jop.setStatistics(stats);

          if (isDebugEnabled) {

            LOG.debug("[0] STATS-" + jop.toString() + ": " + stats.extendedToString());

          }

        } else {

          // worst case when there are no column statistics

          float joinFactor = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVE_STATS_JOIN_FACTOR);

          int numParents = parents.size();

          List<Long> parentRows = Lists.newArrayList();