Examples of com.odiago.flumebase.plan.PlanContext

com.odiago.flumebase.plan.PlanContext
Container for state associated with the plan-formation process when operating over the statement AST.

      stmt.accept(new TypeChecker(mRootSymbolTable));
      stmt.accept(new ReplaceWindows()); // Must be after TC.
      stmt.accept(new JoinKeyVisitor()); // Must be after TC.
      stmt.accept(new JoinNameVisitor());
      stmt.accept(new IdentifyAggregates()); // Must be after TC.
      PlanContext planContext = new PlanContext();
      planContext.setConf(planConf);
      planContext.setSymbolTable(mRootSymbolTable);
      PlanContext retContext = stmt.createExecPlan(planContext);
      msgBuilder.append(retContext.getMsgBuilder().toString());
      FlowSpecification spec = retContext.getFlowSpec();
      if (null != spec) {
        spec.setQuery(query);
        spec.setConf(planConf);
        // Given a flow specification from the AST, run it through
        // necessary post-processing and optimization phases.
        spec.bfs(new PropagateSchemas());
        if (retContext.isExplain()) {
          // We just should explain this flow, but not actually add it.
          msgBuilder.append("Execution plan:\n");
          msgBuilder.append(spec.toString());
          msgBuilder.append("\n");
        } else {

View Full Code Here

  public PlanContext createExecPlan(PlanContext planContext) {
    RecordSource leftSrc = getLeft();
    RecordSource rightSrc = getRight();


    // Create separate execution plans to gather data from our upstream sources.
    PlanContext leftContext = getSubPlan(leftSrc, planContext);
    PlanContext rightContext = getSubPlan(rightSrc, planContext);


    // Add our upstream source plans to our graph.
    FlowSpecification flowSpec = planContext.getFlowSpec();
    flowSpec.addNodesFromDAG(leftContext.getFlowSpec());
    flowSpec.addNodesFromDAG(rightContext.getFlowSpec());
    
    // Get the true field names that represent keys on the left and right
    // sides of the join.
    String leftName = leftSrc.getSourceName();
    AssignedSymbol leftSym = (AssignedSymbol) getLeftKey().resolveAliases();
    TypedField leftKey = new TypedField(leftSym.getAssignedName(), leftSym.getType());


    String rightName = rightSrc.getSourceName();
    AssignedSymbol rightSym = (AssignedSymbol) getRightKey().resolveAliases();
    TypedField rightKey = new TypedField(rightSym.getAssignedName(), rightSym.getType());


    WindowSpec window = null;
    try {
      // This should evaluate to itself, but make sure to resolve it anyway.
      assert mWindowExpr.isConstant();
      window = (WindowSpec) mWindowExpr.eval(new EmptyEventWrapper());
    } catch (IOException ioe) {
      // mWindowExpr should be constant, so this should be impossible.
      LOG.error("IOException calculating window expression: " + ioe);
      // Signal error by returning a null flow specification anyway.
      planContext.setFlowSpec(null);
      return planContext;
    }


    HashJoinNode joinNode = new HashJoinNode(leftName, rightName, leftKey, rightKey,
        window, getSourceName(), leftContext.getOutFields(), rightContext.getOutFields(),
        planContext.getConf());


    // Set this node to expect multiple input schemas.
    List<Schema> inputSchemas = new ArrayList<Schema>();
    inputSchemas.add(leftContext.getSchema());
    inputSchemas.add(rightContext.getSchema());
    joinNode.setAttr(PlanNode.MULTI_INPUT_SCHEMA_ATTR, inputSchemas);


    flowSpec.attachToLastLayer(joinNode);


    // Create an output context defining our fields, etc.
    PlanContext outContext = new PlanContext(planContext);


    SymbolTable outTable = SymbolTable.mergeSymbols(leftContext.getSymbolTable(),
        rightContext.getSymbolTable(), planContext.getSymbolTable());
    outContext.setSymbolTable(outTable);


    List<TypedField> outputFields = new ArrayList<TypedField>();
    outputFields.addAll(leftContext.getOutFields());
    outputFields.addAll(rightContext.getOutFields());
    outputFields = distinctFields(outputFields);
    outContext.setOutFields(outputFields);


    Schema outSchema = createFieldSchema(outputFields);
    outContext.setSchema(outSchema);
    joinNode.setAttr(PlanNode.OUTPUT_SCHEMA_ATTR, outSchema);


    return outContext;
  }

View Full Code Here

  public PlanContext createExecPlan(PlanContext planContext) {
    SQLStatement source = getSource();
    Expr where = getWhereConditions();


    // Create an execution plan for the source(s) of this SELECT stream.
    PlanContext sourceOutCtxt = getSubPlan(source, planContext);
    SymbolTable srcOutSymbolTable = sourceOutCtxt.getSymbolTable();


    // Now incorporate that entire plan into our plan.
    FlowSpecification flowSpec = planContext.getFlowSpec();
    flowSpec.addNodesFromDAG(sourceOutCtxt.getFlowSpec());


    // List of all fields required as output from the source node.
    List<TypedField> allRequiredFields = new ArrayList<TypedField>();


    // All fields carried forward by the aggregation layer from the source layer.
    List<TypedField> groupByPropagateFields = new ArrayList<TypedField>();


    // Another list holds all the fields which the EvaluateExprsNode will need to
    // propagate from the initial source layer forward.
    List<TypedField> exprPropagateFields = new ArrayList<TypedField>();


    // List of all fields with their input names that should be read by the ProjectionNode.
    // This is exprPropagateFields + fields emitted by the expr layer.
    List<TypedField> projectionInputs = new ArrayList<TypedField>();


    // List of all fields returned from the ProjectionNode; this layer
    // uses the translated names from the "x AS y" clauses.
    List<TypedField> projectionOutputs = new ArrayList<TypedField>();


    // Create a list containing the (ordered) set of fields we want emitted to the console.
    List<TypedField> consoleFields = new ArrayList<TypedField>();


    // Populate the field lists defined above
    calculateRequiredFields(srcOutSymbolTable, sourceOutCtxt.getOutFields(),
        allRequiredFields, groupByPropagateFields, exprPropagateFields,
        projectionInputs, projectionOutputs, consoleFields);


    if (where != null) {
      // Non-null filter conditions; apply the filter to all of our sources.

View Full Code Here

  /**
   * Create the output PlanContext that should be returned by createExecPlan().
   */
  private PlanContext createReturnedContext(PlanContext planContext,
      List<TypedField> outputFields) {
    PlanContext outContext = planContext;
    FlowSpecification flowSpec = planContext.getFlowSpec();
    if (planContext.isRoot()) {
      String selectTarget = planContext.getConf().get(CLIENT_SELECT_TARGET_KEY,
          DEFAULT_CLIENT_SELECT_TARGET);
      if (CONSOLE_SELECT_TARGET.equals(selectTarget)) {
        // SELECT statements that are root queries go to the output node.


        // This output node may emit Avro records to a Flume node. These records
        // should use more user-friendly names for the fields than the anonymized
        // field names we use internally. Create a final schema for the output
        // plan node.
        String outputName = getOutputName();
        List<TypedField> outSchemaFields = new ArrayList<TypedField>();
        List<TypedField> distinctOutFields = distinctFields(outputFields);
        for (TypedField outField : distinctOutFields) {
          String safeName = avroSafeName(outField.getDisplayName());
          outSchemaFields.add(new TypedField(safeName, outField.getType()));
        }
        Schema finalSchema = createFieldSchema(outSchemaFields, outputName);
        OutputNode outputNode = new OutputNode(outputFields, outSchemaFields, outputName);
        outputNode.setAttr(PlanNode.OUTPUT_SCHEMA_ATTR, finalSchema);
        flowSpec.attachToLastLayer(outputNode);
      } else {
        // Client has specified that outputs of this root query go to a named memory buffer.
        flowSpec.attachToLastLayer(new MemoryOutputNode(selectTarget,
            distinctFields(outputFields)));
      }
    } else {
      // If the initial projection contained both explicitly selected fields as
      // well as implicitly selected fields (e.g., for the WHERE clause), attach another
      // projection layer that extracts only the explicitly selected fields.


      // SELECT as a sub-query needs to create an output context with a
      // symbol table that contains the fields we expose through projection. 
      // We also need to set the output field names and output schema in our
      // returned context.
      outContext = new PlanContext(planContext);
      SymbolTable inTable = planContext.getSymbolTable();
      SymbolTable outTable = new HashSymbolTable(inTable);
      outputFields = distinctFields(outputFields);
      outTable.addAll(mFieldSymbols);
      Schema outputSchema = createFieldSchema(outputFields);
      ProjectionNode cleanupProjection = new ProjectionNode(outputFields, outputFields);
      cleanupProjection.setAttr(PlanNode.OUTPUT_SCHEMA_ATTR, outputSchema);
      flowSpec.attachToLastLayer(cleanupProjection);


      outContext.setSymbolTable(outTable);
      outContext.setSchema(outputSchema);
      outContext.setOutFields(outputFields);
    }


    return outContext;
  }

View Full Code Here

    // the symbol table at plan resolution time.


    // The output PlanContext contains a new symbol table defining the fields
    // of this source.


    PlanContext outContext = new PlanContext(planContext);
    SymbolTable inTable = planContext.getSymbolTable();
    SymbolTable outTable = mSymbols;
    outContext.setSymbolTable(outTable);


    // streamSym is guaranteed to be a non-null StreamSymbol by the typechecker.
    StreamSymbol streamSym = (StreamSymbol) inTable.resolve(mSourceName).resolveAliases();
    List<TypedField> fields = streamSym.getFields();
    List<String> fieldNames = new ArrayList<String>();
    for (TypedField field : fields) {
      String fieldName = field.getAvroName();
      if (!fieldNames.contains(fieldName)) {
        fieldNames.add(fieldName);
      }
    }


    // Create an Avro output schema for this node, specifying all the fields
    // we can emit.  Use our internal symbol (mSymbols a.k.a. outTable) to
    // create more precise TypedFields that use the proper avro names.
    List<TypedField> outFields = new ArrayList<TypedField>();
    for (String fieldName : fieldNames) {
      AssignedSymbol sym = (AssignedSymbol) outTable.resolve(fieldName).resolveAliases();
      outFields.add(new TypedField(fieldName, sym.getType(), sym.getAssignedName(), fieldName));
    }


    PlanNode node = new NamedSourceNode(mSourceName, outFields);
    planContext.getFlowSpec().addRoot(node);
    Schema outSchema = createFieldSchema(outFields);
    outContext.setSchema(outSchema);
    outContext.setOutFields(outFields);
    node.setAttr(PlanNode.OUTPUT_SCHEMA_ATTR, outSchema);


    return outContext;
  }

View Full Code Here

    StringBuilder sb = planContext.getMsgBuilder();
    sb.append("Parse tree:\n");
    getChildStmt().format(sb, 0);
    sb.append("\n");


    PlanContext retContext = new PlanContext(planContext);
    retContext.setExplain(true);
    return retContext;
  }

View Full Code Here

   */
  protected PlanContext getSubPlan(SQLStatement subStmt, PlanContext planContext) {
    // Create an execution plan to build the source (it may be a single node
    // representing a Flume source or file, or it may be an entire DAG because
    // we use another SELECT statement as a source) inside a new context.
    PlanContext sourceInCtxt = new PlanContext(planContext);
    sourceInCtxt.setRoot(false);
    sourceInCtxt.setFlowSpec(new FlowSpecification(planContext.getConf()));
    return subStmt.createExecPlan(sourceInCtxt);
  }

View Full Code Here

    case Flow:
    default:
      sb.append("Do not know how to show item: " + mTarget);
    }


    PlanContext retContext = new PlanContext(planContext);
    retContext.setFlowSpec(null);
    return retContext;
  }

View Full Code Here

TOP

Related Classes of com.odiago.flumebase.plan.PlanContext

com.odiago.flumebase.exec.HashSymbolTable

com.odiago.flumebase.exec.local.LocalEnvironment

com.odiago.flumebase.parser.ExplainStmt

com.odiago.flumebase.parser.JoinedSource

com.odiago.flumebase.parser.LiteralSource

com.odiago.flumebase.parser.RecordSource

com.odiago.flumebase.parser.SelectStmt

com.odiago.flumebase.parser.ShowStmt

org.apache.hadoop.conf.Configuration

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.