StreamSymbol streamSym = new StreamSymbol(createStream);
if (!streamSym.getEventParser().validate(streamSym)) {
// Fails final check of parameters
// TODO: The EventParser is giving better info in its LOG; but this
// should really be communicated back to the user.
throw new DAGOperatorException(
"Stream cannot be created with the specified parameters.");
} else if (mRootSymbolTable.resolve(streamName) != null) {
// TODO: Allow CREATE OR REPLACE STREAM to override this.
throw new DAGOperatorException("Object already exists at top level: " + streamName);
} else {
mRootSymbolTable.addSymbol(streamSym);
mSubmitterSession.sendInfo("CREATE STREAM");
}
if (createStream.getType().equals(StreamSourceType.File)
&& streamSym.getFormatSpec().getParam(FileSourceElement.TIMESTAMP_COL_KEY) == null) {
// We're reading from a file, and making up timestamps based on read time.
// Warn the user that timestamps will change between queries.
StringBuilder sb = new StringBuilder();
sb.append("Warning: File-based streams will set event timestamps based on read time.\n");
sb.append("To specify timestamps explictly, set the ");
sb.append(FileSourceElement.TIMESTAMP_COL_KEY);
sb.append(" event format property.");
mSubmitterSession.sendInfo(sb.toString());
}
} else if (node instanceof DescribeNode) {
// Look up the referenced object in the symbol table and describe it immediately.
DescribeNode describe = (DescribeNode) node;
Symbol sym = mRootSymbolTable.resolve(describe.getIdentifier());
mSubmitterSession.sendInfo(sym.toString());
} else if (node instanceof DropNode) {
// Perform the operation here.
// Remove the objet from our symbol table.
DropNode dropNode = (DropNode) node;
String name = dropNode.getName();
Symbol sym = mRootSymbolTable.resolve(name);
if (null == sym) {
// Shouldn't happen; the type checker already accepted this statement.
throw new DAGOperatorException("No such object at top level: " + name);
}
EntityTarget targetType = dropNode.getType();
// Perform the operation.
mRootSymbolTable.remove(name);
mSubmitterSession.sendInfo("DROP " + targetType.toString().toUpperCase());
} else if (node instanceof NamedSourceNode) {
NamedSourceNode namedInput = (NamedSourceNode) node;
String streamName = namedInput.getStreamName();
Symbol symbol = mRootSymbolTable.resolve(streamName).resolveAliases();
if (null == symbol) {
throw new DAGOperatorException("No symbol for stream: " + streamName);
}
if (!(symbol instanceof StreamSymbol)) {
throw new DAGOperatorException("Identifier " + streamName + " has type: "
+ symbol.getType() + ", not STREAM.");
}
StreamSymbol streamSymbol = (StreamSymbol) symbol;
switch (streamSymbol.getSourceType()) {
case File:
String fileName = streamSymbol.getSource();
newElem = new FileSourceElement(newContext, fileName, streamSymbol.isLocal(),
namedInput.getFields(), streamSymbol);
break;
case Source:
if (!streamSymbol.isLocal()) {
throw new DAGOperatorException("Do not know how to handle a non-local source yet.");
}
String flumeSource = streamSymbol.getSource();
long flowIdNum = mFlowId.getId();
String flowSourceId = "flumebase-flow-" + flowIdNum + "-" + streamSymbol.getName();
newElem = new LocalFlumeSourceElement(newContext, flowSourceId,
mFlumeConfig, flumeSource, (Schema) namedInput.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR),
namedInput.getFields(), streamSymbol);
if (!streamSymbol.isLocal()) {
LOG.info("Created local Flume logical node: " + flowSourceId);
LOG.info("You may need to connect upstream Flume elements to this source.");
}
// Mark Flume as required to execute this flow.
mLocalFlow.setFlumeRequired(true);
break;
case Memory:
newElem = new LocalInMemSourceElement(newContext,
namedInput.getFields(), (InMemStreamSymbol) streamSymbol);
break;
case Node:
String nodeSourceId = "flumebase-flow-" + mFlowId.getId() + "-" + streamSymbol.getName();
newElem = new FlumeNodeElement(newContext, nodeSourceId,
mFlumeConfig, streamSymbol.getSource(),
(Schema) namedInput.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR),
namedInput.getFields(), streamSymbol);
LOG.info("Created local Flume receiver context: " + nodeSourceId);
LOG.info("This will be connected to upstream Flume node: " + streamSymbol.getSource());
// Mark Flume as required to execute this flow.
mLocalFlow.setFlumeRequired(true);
break;
default:
throw new DAGOperatorException("Unhandled stream source type: "
+ streamSymbol.getSourceType());
}
} else if (node instanceof FilterNode) {
FilterNode filterNode = (FilterNode) node;
Expr filterExpr = filterNode.getFilterExpr();
newElem = new FilterElement(newContext, filterExpr);
} else if (node instanceof ProjectionNode) {
ProjectionNode projNode = (ProjectionNode) node;
Schema outSchema = (Schema) projNode.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR);
newElem = new ProjectionElement(newContext, outSchema, projNode.getInputFields(),
projNode.getOutputFields());
} else if (node instanceof AggregateNode) {
AggregateNode aggNode = (AggregateNode) node;
newElem = new BucketedAggregationElement(newContext, aggNode);
} else if (node instanceof EvaluateExprsNode) {
EvaluateExprsNode evalNode = (EvaluateExprsNode) node;
Schema outSchema = (Schema) evalNode.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR);
newElem = new EvaluationElement(newContext, evalNode.getExprs(),
evalNode.getPropagateFields(), outSchema);
} else if (node instanceof HashJoinNode) {
HashJoinNode joinNode = (HashJoinNode) node;
newElem = new HashJoinElement(newContext, joinNode);
} else {
throw new DAGOperatorException("Cannot create FlowElement for PlanNode of type: "
+ node.getClass().getName());
}
if (null != newElem) {
// Wrap the FlowElement in a DAGNode.