Package org.apache.hadoop.hive.ql.exec.vector

Examples of org.apache.hadoop.hive.ql.exec.vector.VectorizationContext


    public Map<String, Map<Integer, String>> getAllScratchColumnVectorTypeMaps() {
      Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps =
          new HashMap<String, Map<Integer, String>>();
      for (String onefile : scratchColumnContext.keySet()) {
        VectorizationContext vc = scratchColumnContext.get(onefile);
        Map<Integer, String> cmap = vc.getScratchColumnTypeMap();
        allScratchColumnVectorTypeMaps.put(onefile, cmap);
      }
      return allScratchColumnVectorTypeMaps;
    }
View Full Code Here


    public Map<String, Map<String, Integer>> getAllColumnVectorMaps() {
      Map<String, Map<String, Integer>> allColumnVectorMaps =
          new HashMap<String, Map<String, Integer>>();
      for(String oneFile: scratchColumnContext.keySet()) {
        VectorizationContext vc = scratchColumnContext.get(oneFile);
        Map<String, Integer> cmap = vc.getProjectionColumnMap();
        allColumnVectorMaps.put(oneFile, cmap);
      }
      return allColumnVectorMaps;
    }
View Full Code Here

      return allColumnVectorMaps;
    }

    public VectorizationContext walkStackToFindVectorizationContext(Stack<Node> stack,
            Operator<? extends OperatorDesc> op) throws SemanticException {
      VectorizationContext vContext = null;
      if (stack.size() <= 1) {
        throw new SemanticException(
            String.format("Expected operator stack for operator %s to have at least 2 operators",
                  op.getName()));
      }
View Full Code Here

          if (vectorOp != op) {
            opsDone.add(vectorOp);
          }
          if (vectorOp instanceof VectorizationContextRegion) {
            VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
            VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext();
            vContextsByOp.put(op, vOutContext);
            scratchColumnContext.put(vOutContext.getFileKey(), vOutContext);
          }
        }
      } catch (HiveException e) {
        throw new SemanticException(e);
      }
View Full Code Here

        Object... nodeOutputs) throws SemanticException {

      Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
      LOG.info("MapWorkVectorizationNodeProcessor processing Operator: " + op.getName() + "...");

      VectorizationContext vContext = null;

      if (op instanceof TableScanOperator) {
        vContext = getVectorizationContext(op, physicalContext);
        for (String onefile : mWork.getPathToAliases().keySet()) {
          List<String> aliases = mWork.getPathToAliases().get(onefile);
          for (String alias : aliases) {
            Operator<? extends OperatorDesc> opRoot = mWork.getAliasToWork().get(alias);
            if (op == opRoot) {
              // The same vectorization context is copied multiple times into
              // the MapWork scratch columnMap
              // Each partition gets a copy
              //
              vContext.setFileKey(onefile);
              scratchColumnContext.put(onefile, vContext);
              if (LOG.isDebugEnabled()) {
                LOG.debug("Vectorized MapWork operator " + op.getName() + " vectorization context " + vContext.toString());
              }
              break;
            }
          }
        }
        vContextsByOp.put(op, vContext);
      } else {
        vContext = walkStackToFindVectorizationContext(stack, op);
        if (vContext == null) {
          throw new SemanticException(
              String.format("Did not find vectorization context for operator %s in operator stack",
                      op.getName()));
        }
      }

      assert vContext != null;

      // When Vectorized GROUPBY outputs rows instead of vectorized row batchs, we don't
      // vectorize the operators below it.
      if (nonVectorizableChildOfGroupBy(op)) {
        // No need to vectorize
        if (!opsDone.contains(op)) {
            opsDone.add(op);
          }
        return null;
      }

      Operator<? extends OperatorDesc> vectorOp = doVectorize(op, vContext);

      if (LOG.isDebugEnabled()) {
        LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " vectorization context " + vContext.toString());
        if (vectorOp instanceof VectorizationContextRegion) {
          VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
          VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext();
          LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " added vectorization context " + vContext.toString());
        }
      }

      return null;
View Full Code Here

      Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
      LOG.info("ReduceWorkVectorizationNodeProcessor processing Operator: " +
              op.getName() + "...");

      VectorizationContext vContext = null;

      boolean saveRootVectorOp = false;

      if (op.getParentOperators().size() == 0) {
        LOG.info("ReduceWorkVectorizationNodeProcessor process reduceColumnNames " + reduceColumnNames.toString());

        vContext = new VectorizationContext(reduceColumnNames);
        vContext.setFileKey("_REDUCE_SHUFFLE_");
        scratchColumnContext.put("_REDUCE_SHUFFLE_", vContext);
        reduceShuffleVectorizationContext = vContext;
        saveRootVectorOp = true;

        if (LOG.isDebugEnabled()) {
          LOG.debug("Vectorized ReduceWork reduce shuffle vectorization context " + vContext.toString());
        }
      } else {
        vContext = walkStackToFindVectorizationContext(stack, op);
        if (vContext == null) {
          // If we didn't find a context among the operators, assume the top -- reduce shuffle's
          // vectorization context.
          vContext = reduceShuffleVectorizationContext;
        }
      }

      assert vContext != null;

      // When Vectorized GROUPBY outputs rows instead of vectorized row batchs, we don't
      // vectorize the operators below it.
      if (nonVectorizableChildOfGroupBy(op)) {
        // No need to vectorize
        if (!opsDone.contains(op)) {
          opsDone.add(op);
        }
        return null;
      }

      Operator<? extends OperatorDesc> vectorOp = doVectorize(op, vContext);

      if (LOG.isDebugEnabled()) {
        LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " vectorization context " + vContext.toString());
        if (vectorOp instanceof VectorizationContextRegion) {
          VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
          VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext();
          LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " added vectorization context " + vContext.toString());
        }
      }
      if (vectorOp instanceof VectorGroupByOperator) {
        VectorGroupByOperator groupBy = (VectorGroupByOperator) vectorOp;
View Full Code Here

    columns.add("col1");
    columns.add("col2");
    columns.add("col3");

    //Generate vectorized expression
    vContext = new VectorizationContext(columns);
  }
View Full Code Here

  boolean validateExprNodeDesc(ExprNodeDesc desc, VectorExpressionDescriptor.Mode mode) {
    if (!validateExprNodeDescRecursive(desc)) {
      return false;
    }
    try {
      VectorizationContext vc = new ValidatorVectorizationContext();
      if (vc.getVectorExpression(desc, mode) == null) {
        // TODO: this cannot happen - VectorizationContext throws in such cases.
        return false;
      }
    } catch (Exception e) {
      if (LOG.isDebugEnabled()) {
View Full Code Here

      if (!isVirtualColumn(c)) {
        cmap.put(c.getInternalName(), columnCount++);
      }
    }

    VectorizationContext vc =  new VectorizationContext(cmap, columnCount);
    return vc;
  }
View Full Code Here

    public Map<String, Map<Integer, String>> getScratchColumnVectorTypes() {
      Map<String, Map<Integer, String>> scratchColumnVectorTypes =
          new HashMap<String, Map<Integer, String>>();
      for (String onefile : scratchColumnContext.keySet()) {
        VectorizationContext vc = scratchColumnContext.get(onefile);
        Map<Integer, String> cmap = vc.getOutputColumnTypeMap();
        scratchColumnVectorTypes.put(onefile, cmap);
      }
      return scratchColumnVectorTypes;
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.ql.exec.vector.VectorizationContext

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.