package ch.uzh.ifi.ddis.ifp.streams;
/*
* #%L
* Esper implementation of Streams Nodes
* %%
* Copyright (C) 2013 University of Zurich, Department of Informatics
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-2.0.html>.
* #L%
*/
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import stream.AbstractProcessor;
import stream.Data;
import stream.ProcessContext;
import stream.annotations.Parameter;
import stream.io.Queue;
import stream.io.Sink;
import com.espertech.esper.client.Configuration;
import com.espertech.esper.client.EPAdministrator;
import com.espertech.esper.client.EPRuntime;
import com.espertech.esper.client.EPServiceProvider;
import com.espertech.esper.client.EPServiceProviderManager;
import com.espertech.esper.client.EPStatement;
import com.espertech.esper.client.EventType;
import com.espertech.esper.client.soda.EPStatementObjectModel;
import com.espertech.esper.client.time.CurrentTimeEvent;
/**
* <p>
* Abstract class that provides basic Esper support for the Streams platform.
* </p>
* <p>
* <a href="http://esper.codehaus.org/">Esper</a> is a Complex Event Processing
* (CEP) platform for the Java and .Net languages. Esper must be started from
* custom code. It provides parallel execution but no distribution. The Streams
* stream processing platform allows to distribute Esper over different
* processes.
* </p>
* <p>
* This package on the other hand allows integrating Complex Event Processing
* with a high level query language, i.e., the Event Processing Language (EPL).
* The EPL is a SQL-like language. For further information please refer to the
* <a href=
* "http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html_single/#epl-intro"
* >Esper documentation</a>.
* </p>
* <p>
* The configuration of an Esper Processor follows the Xml configuration of the
* Streams platform. The configuration may include an <a href=
* "http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html_single/#configuration-xml"
* >Esper config</a>. EPL statements are added in textual form in the config
* file.
* </p>
* <p>
* Example Configuration:
* <pre>
* <ch.uzh.ifi.ddis.ifp.streams.EsperProcessor
* output="queue1,queue2">
* <config>
* <esper-configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
* xmlns="http://www.espertech.com/schema/esper"
* xsi:schemaLocation="http://www.espertech.com/schema/esper http://www.espertech.com/schema/esper/esper-configuration-2.0.xsd">
*
* <engine-settings>
* <defaults>
* <threading>
* <internal-timer msec-resolution="1" enabled="false" />
* </threading>
* </defaults>
* </engine-settings>
*
* <event-type name="LeftEvent">
* <java-util-map start-timestamp-property-name="timestamp">
* <map-property name="timestamp" class="long" />
* <map-property name="id" class="string" />
* </java-util-map>
* </event-type>
* <event-type name="RightEvent">
* <java-util-map start-timestamp-property-name="timestamp">
* <map-property name="timestamp" class="long" />
* <map-property name="id" class="string" />
* </java-util-map>
* </event-type>
*
* </esper-configuration>
* <statement output="queue2">select id from LeftEvent</statement>
* <statement>select * from RightEvent</statement>
* </config>
* </ch.uzh.ifi.ddis.ifp.streams.EsperProcessor>
* </pre>
* </p>
*
* @author Thomas Scharrenbach
* @version 0.0.1
* @since 0.0.1
*
*/
public class EsperProcessor extends AbstractProcessor implements
stream.Configurable {
private static final Logger _log = LoggerFactory
.getLogger(EsperProcessor.class);
public static final String ESPER_CONFIG_LOCAL_NAME = "esper-configuration";
public static final String ESPER_NS = "http://www.espertech.com/schema/esper";
public static final String ESPER_STATEMENT_LOCAL_NAME = "statement";
public static final String EVENT_TYPE_KEY = "EPEventType";
//
//
//
protected transient EPServiceProvider _epService;
protected transient EPRuntime _epRuntime;
private long _currentTime;
private long _initialTime;
private final Configuration _configuration;
private final List<String> _epStatements;
/**
* Maps Esper event types to property names of start or end timestamps.
*/
private final Map<String, String> _startTimestampMap;
private final Map<String, String> _endTimestampMap;
//
// Fields that are parameters.
//
private String _epProviderURI;
private Map<String, Sink> _sinksMap;
private final Map<String, String> _statementSinksMap;
//
//
//
public EsperProcessor() {
_configuration = new Configuration();
_epStatements = new ArrayList<String>();
_statementSinksMap = new HashMap<String, String>();
_startTimestampMap = new HashMap<String, String>();
_endTimestampMap = new HashMap<String, String>();
_currentTime = Long.MIN_VALUE;
}
//
//
//
/**
* Initializes the Esper service.
*/
@Override
public void init(ProcessContext context) throws Exception {
super.init(context);
initEPService();
}
private void initEPService() {
_log.debug("Initializing {} ...", this.getClass());
final String providerUri = getProviderUri();
if (providerUri == null || providerUri.isEmpty()) {
_log.debug("Creating new Esper service from default provider.");
_epService = EPServiceProviderManager
.getDefaultProvider(_configuration);
}
//
else {
_log.debug("Creating new Esper service from named provider: {}",
providerUri);
_epService = EPServiceProviderManager.getProvider(providerUri,
_configuration);
}
_log.debug("Adding statements to Esper service");
final EPAdministrator epAdmin = _epService.getEPAdministrator();
for (String epStatement : _epStatements) {
_log.debug("Compiling Esper statement {}", epStatement);
final EPStatementObjectModel stmtModel = epAdmin
.compileEPL(epStatement);
final EPStatement stmt = epAdmin.create(stmtModel);
final String stmtName = Integer.toString(epStatement.hashCode());
// TODO fast hack disabling queues, since they are currently not
// supported by Streams-Storm.
final Sink sink = _sinksMap.get(_statementSinksMap.get(stmtName));
final String[] propertyNames = stmt.getEventType()
.getPropertyNames();
if (sink != null) {
final EsperStatementSubscriber subscriber = new EsperStatementSubscriber(
sink, propertyNames);
_log.debug("Adding subscriber to statement {}", stmtName);
stmt.setSubscriber(subscriber);
}
}
_log.debug("Finished adding statements to Esper service");
_log.debug("Mapping event types to timestamp properties, if any");
for (EventType eventType : epAdmin.getConfiguration().getEventTypes()) {
final String startTimestampProperty = eventType
.getStartTimestampPropertyName();
final String endTimestampProperty = eventType
.getEndTimestampPropertyName();
final String timestampProperty = endTimestampProperty == null ? startTimestampProperty
: endTimestampProperty;
final String eventTypeName = eventType.getName();
_log.debug("Timestamp property for event type {}: {}",
eventTypeName, timestampProperty);
_startTimestampMap.put(eventTypeName, startTimestampProperty);
_endTimestampMap.put(eventTypeName, endTimestampProperty);
}
_log.debug("Finished mapping event types to timestamp properties, if any");
_epRuntime = _epService.getEPRuntime();
_log.debug("Finished initalizing {}.", this.getClass());
}
@Override
public Data process(Data input) {
final String mapEventTypeName = (String) input.get("@stream");
final Data event = input.createCopy();
event.remove("@stream");
event.remove("@stream:id");
// If this event defines a start time, then adjust the current time if
// necessary and replace the string value with the long value.
final String startTimeKey = _startTimestampMap.get(mapEventTypeName);
if (startTimeKey != null) {
final long dataStartTime = Long.parseLong(input.get(startTimeKey)
.toString());
event.put(startTimeKey, dataStartTime);
if (dataStartTime > _currentTime) {
_currentTime = dataStartTime;
final CurrentTimeEvent timeEvent = new CurrentTimeEvent(
_currentTime);
_log.debug("Sending time event new time: {}", _currentTime);
_epRuntime.sendEvent(timeEvent);
}
}
// If this event defines an end time, then replace the string value with
// the long value.
final String endTimeKey = _endTimestampMap.get(mapEventTypeName);
if (endTimeKey != null) {
final long dataEndTime = Long.parseLong(input.get(endTimeKey)
.toString());
event.put(endTimeKey, dataEndTime);
}
_epRuntime.sendEvent(event, mapEventTypeName);
return null;
}
@Override
public void resetState() throws Exception {
_epService.destroy();
initEPService();
}
@Override
public void finish() throws Exception {
_log.debug("Finishing {} ...", this.getClass());
if (_epService != null) {
_epService.destroy();
}
_log.debug("Finished finishing {}", this.getClass());
}
//
// Methods from Configurable.
//
@Override
public void configure(Element document) {
// TODO fast hack, since Streams does not support namespaces.
// final NodeList esperConfigNodeList = document.getElementsByTagNameNS(
// ESPER_NS, ESPER_CONFIG_LOCAL_NAME);
final NodeList esperConfigNodeList = document
.getElementsByTagName(ESPER_CONFIG_LOCAL_NAME);
for (int i = 0; i < esperConfigNodeList.getLength(); ++i) {
_log.debug("Configuring Esper with xml node.");
final Node esperConfigNode = esperConfigNodeList.item(i);
_log.debug("Esper xml configuration: {}",
esperConfigNode.cloneNode(true));
try {
final Document esperConfigDocument = DocumentBuilderFactory
.newInstance().newDocumentBuilder().newDocument();
esperConfigDocument.appendChild(esperConfigDocument.importNode(
esperConfigNode, true));
_configuration.configure(esperConfigDocument);
} catch (ParserConfigurationException e) {
_log.debug("Error configuring Esper with xml node.");
throw new RuntimeException(e);
}
_log.debug("Finished configuring Esper with xml node.");
}
final NodeList esperStatementNodeList = document
.getElementsByTagName(ESPER_STATEMENT_LOCAL_NAME);
for (int i = 0; i < esperStatementNodeList.getLength(); ++i) {
final Node esperStatementNode = esperStatementNodeList.item(i);
final String esperStatement = esperStatementNode.getTextContent();
_log.debug("Adding statement for Esper: {}", esperStatement);
if (esperStatement == null) {
_log.error("Empty Esper statement node.");
throw new RuntimeException(
"If specified, Esper statement must not be empty!");
}
//
else {
_epStatements.add(esperStatement);
if (esperStatementNode.hasAttributes()) {
final Node outputAttr = esperStatementNode.getAttributes()
.getNamedItem("output");
if (outputAttr != null) {
// Note: _queue is not yet set. We cannot check whether
// a queue actually exists.
final String output = outputAttr.getTextContent();
final String statementName = Integer
.toString(esperStatement.hashCode());
_statementSinksMap.put(statementName, output);
}
}
}
}
}
//
//
//
public void setOutput(Queue[] queues) {
_sinksMap = new HashMap<String, Sink>();
for (int i = 0; i < queues.length; ++i) {
_sinksMap.put(queues[i].getId(), queues[i]);
}
return;
}
public Configuration getConfiguration() {
return _configuration;
}
@Parameter(defaultValue = "", description = "The URI of the Esper Runtime.", required = false)
public String getProviderUri() {
return _epProviderURI;
}
public void setProviderUri(String providerUri) {
_epProviderURI = providerUri;
}
public List<String> getEpStatements() {
return _epStatements;
}
public long getInitialTime() {
return _initialTime;
}
public void setInitialTime(long initialTime) {
_initialTime = initialTime;
_currentTime = initialTime;
}
}