package ch.uzh.ifi.ddis.ifp.streams;
/*
* #%L
* Esper implementation of Streams Nodes
* %%
* Copyright (C) 2013 University of Zurich, Department of Informatics
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-2.0.html>.
* #L%
*/
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import stream.AbstractProcessor;
import stream.Data;
import stream.ProcessContext;
import stream.annotations.Parameter;
import stream.io.Queue;
import stream.io.Sink;
import com.espertech.esper.client.Configuration;
import com.espertech.esper.client.EPAdministrator;
import com.espertech.esper.client.EPRuntime;
import com.espertech.esper.client.EPServiceProvider;
import com.espertech.esper.client.EPServiceProviderManager;
import com.espertech.esper.client.EPStatement;
import com.espertech.esper.client.EventType;
import com.espertech.esper.client.soda.EPStatementObjectModel;
import com.espertech.esper.client.time.CurrentTimeEvent;
/**
* <p>
* Abstract class that provides basic Esper support for the Streams platform.
* </p>
* <p>
* <a href="http://esper.codehaus.org/">Esper</a> is a Complex Event Processing
* (CEP) platform for the Java and .Net languages. Esper must be started from
* custom code. It provides parallel execution but no distribution. The Streams
* stream processing platform allows to distribute Esper over different
* processes.
* </p>
* <p>
* This package on the other hand allows integrating Complex Event Processing
* with a high level query language, i.e., the Event Processing Language (EPL).
* The EPL is a SQL-like language. For further information please refer to the
* <a href=
* "http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html_single/#epl-intro"
* >Esper documentation</a>.
* </p>
* <p>
* The configuration of an Esper Processor follows the Xml configuration of the
* Streams platform. The configuration may include an <a href=
* "http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html_single/#configuration-xml"
* >Esper config</a>. EPL statements are added in textual form in the config
* file.
* </p>
* <p>
* Example Configuration:
*
* <pre>
* <ch.uzh.ifi.ddis.ifp.streams.EsperProcessor
* output="queue1,queue2">
* <config>
* <esper-configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
* xmlns="http://www.espertech.com/schema/esper"
* xsi:schemaLocation="http://www.espertech.com/schema/esper http://www.espertech.com/schema/esper/esper-configuration-2.0.xsd">
*
* <engine-settings>
* <defaults>
* <threading>
* <internal-timer msec-resolution="1" enabled="false" />
* </threading>
* </defaults>
* </engine-settings>
*
* <event-type name="LeftEvent">
* <java-util-map start-timestamp-property-name="timestamp">
* <map-property name="timestamp" class="long" />
* <map-property name="id" class="string" />
* </java-util-map>
* </event-type>
* <event-type name="RightEvent">
* <java-util-map start-timestamp-property-name="timestamp">
* <map-property name="timestamp" class="long" />
* <map-property name="id" class="string" />
* </java-util-map>
* </event-type>
*
* </esper-configuration>
* <statement output="queue2">select id from LeftEvent</statement>
* <statement>select * from RightEvent</statement>
* </config>
* </ch.uzh.ifi.ddis.ifp.streams.EsperProcessor>
* </pre>
*
* </p>
*
* @author Thomas Scharrenbach
* @version 0.2.4
* @since 0.0.1
*
*/
public class EsperProcessor extends AbstractProcessor implements
stream.Configurable {
private static final Logger _log = LoggerFactory
.getLogger(EsperProcessor.class);
public static final String ESPER_CONFIG_LOCAL_NAME = "esper-configuration";
public static final String ESPER_NS = "http://www.espertech.com/schema/esper";
public static final String ESPER_STATEMENT_LOCAL_NAME = "statement";
public static final String EVENT_TYPE_KEY = "EPEventType";
//
//
//
protected transient EPServiceProvider _epService;
protected transient EPRuntime _epRuntime;
private long _currentTime;
private long _initialTime;
private final Configuration _configuration;
/**
* Maps Esper event types to property names of start or end timestamps.
*/
private final Map<String, String> _startTimestampMap;
private final Map<String, String> _endTimestampMap;
//
// Fields that are parameters.
//
private String _epProviderURI;
private Map<String, Sink> _sinksMap;
private final Set<StatementBean> _esperStatements;
//
//
//
public EsperProcessor() {
_configuration = new Configuration();
_startTimestampMap = new HashMap<String, String>();
_endTimestampMap = new HashMap<String, String>();
_currentTime = Long.MIN_VALUE;
_esperStatements = new LinkedHashSet<StatementBean>();
}
//
//
//
/**
* Initializes the Esper service.
*/
@Override
public void init(ProcessContext context) throws Exception {
super.init(context);
initEPService();
}
private void initEPService() {
_log.info("Started initializing {} ...", this.getClass());
final String providerUri = getProviderUri();
if (providerUri == null || providerUri.isEmpty()) {
_log.debug("Creating new Esper service from default provider.");
_epService = EPServiceProviderManager
.getDefaultProvider(_configuration);
}
//
else {
_log.debug("Creating new Esper service from named provider: {}",
providerUri);
_epService = EPServiceProviderManager.getProvider(providerUri,
_configuration);
}
_log.debug("Adding statements to Esper service");
final EPAdministrator epAdmin = _epService.getEPAdministrator();
// Add all statements to the Esper engine.
// Note that subscribers are added only for those statements that
// declare an output sink.
for (StatementBean epStatement : _esperStatements) {
_log.info("Compiling statement {}", epStatement);
_log.debug("Compiling Esper statement {}", epStatement.getEsperStatement());
// Create a template for the statement in the current Esper engine.
final EPStatementObjectModel stmtModel = epAdmin
.compileEPL(epStatement.getEsperStatement());
// Create the actual statement in the current Esper engine from the
// statement model.
final String stmtName = epStatement.getName();
final EPStatement stmt = (stmtName == null ? epAdmin
.create(stmtModel) : epAdmin.create(stmtModel, stmtName));
// If an output sink was defined, then we add a subscriber.
if (epStatement.getOutput() != null) {
final String sinkName = epStatement.getOutput();
final Sink sink = _sinksMap.get(sinkName);
final String[] propertyNames = stmt.getEventType()
.getPropertyNames();
if (sink == null) {
_log.warn(
"Statement {} declares the sink {} "
+ "but the corresponding Esper processor does not.",
epStatement, sinkName);
} else {
final EsperStatementSubscriber subscriber = new EsperStatementSubscriber(
sink, propertyNames);
_log.info("Adding subscriber {} to statement {}", sink.getId(),
epStatement);
stmt.setSubscriber(subscriber);
}
}
}
_log.debug("Finished adding statements to Esper service");
_log.debug("Mapping event types to timestamp properties, if any");
for (EventType eventType : epAdmin.getConfiguration().getEventTypes()) {
final String startTimestampProperty = eventType
.getStartTimestampPropertyName();
final String endTimestampProperty = eventType
.getEndTimestampPropertyName();
final String timestampProperty = endTimestampProperty == null ? startTimestampProperty
: endTimestampProperty;
final String eventTypeName = eventType.getName();
_log.debug("Timestamp property for event type {}: {}",
eventTypeName, timestampProperty);
_startTimestampMap.put(eventTypeName, startTimestampProperty);
_endTimestampMap.put(eventTypeName, endTimestampProperty);
}
_log.debug("Finished mapping event types to timestamp properties, if any");
_epRuntime = _epService.getEPRuntime();
_log.info("Finished initalizing {}.", this.getClass());
}
/**
* <p>
* Decodes a {@link stream.Data} input item and sends it to the Esper
* engine. The result of the processing in the Esper engine is output to
* {@link stream.io.Sink} objects asynchonously.
* </p>
* <p>
* The method determines the type of input by evaluating the field
* "@stream". It sends a copy of the input {@link stream.Data} item to the
* Esper {@link EPRuntime}.It removes the values for "@stream" and
* "@stream:id" from the copied {@link stream.Data} item before sending it.
* </p>
* <p>
* The the input {@link stream.Data} item defines a start time, then this
* start time is compared with the current data time. If the time stamp of
* the input {@link stream.Data} item is larger than the current data time,
* then the current data time is set to the input {@link stream.Data} item's
* start time. A time event with the {@link stream.Data} item's start time
* is sent to the Esper {@link EPRuntime}.
* </p>
*
* @return null, since this implementation works asynchronously.
*/
@Override
public Data process(Data input) {
final String mapEventTypeName = (String) input.get("@stream");
final Data event = input.createCopy();
event.remove("@stream");
event.remove("@stream:id");
// If this event defines a start time, then adjust the current time if
// necessary and replace the string value with the long value.
final String startTimeKey = _startTimestampMap.get(mapEventTypeName);
if (startTimeKey != null) {
final long dataStartTime = Long.parseLong(input.get(startTimeKey)
.toString());
event.put(startTimeKey, dataStartTime);
if (dataStartTime > _currentTime) {
_currentTime = dataStartTime;
final CurrentTimeEvent timeEvent = new CurrentTimeEvent(
_currentTime);
_log.debug("Sending time event new time: {}", _currentTime);
_epRuntime.sendEvent(timeEvent);
}
}
// If this event defines an end time, then replace the string value with
// the long value.
final String endTimeKey = _endTimestampMap.get(mapEventTypeName);
if (endTimeKey != null) {
final long dataEndTime = Long.parseLong(input.get(endTimeKey)
.toString());
event.put(endTimeKey, dataEndTime);
}
_epRuntime.sendEvent(event, mapEventTypeName);
return null;
}
/**
* <p>
* Calls {@link EPServiceProvider#destroy()} and {@link #initEPService()}
* afterwards.
* </p>
*
* @throws delegates
* any {@link Exception} that might be thrown during the calls
* of this method.
*/
@Override
public void resetState() throws Exception {
super.resetState();
_epService.destroy();
initEPService();
}
/**
* <p>
* Calls {@link EPServiceProvider#destroy()}.
* </p>
*
* @throws delegates
* any {@link Exception} that might be thrown during the calls
* of this method.
*/
@Override
public void finish() throws Exception {
_log.debug("Finishing {} ...", this.getClass());
if (_epService != null) {
_epService.destroy();
}
_log.debug("Finished finishing {}", this.getClass());
}
//
// Methods from Configurable.
//
/**
* <p>
* Parses the <configuration> ... </configuration> tag if
* provided.
* </p>
* <p>
* The configuration may contain the following elements:
* <ul>
* <li>An Esper configuration element <esper-configuration> ...
* </esper-configuration> .</li>
* <li>An arbitrary number of Esper statements, i.e., each statement
* enclosed by <statement> ... </statement> .</li>
* </ul>
* </p>
*/
@Override
public void configure(Element document) {
// TODO fast hack, since Streams does not support namespaces.
// final NodeList esperConfigNodeList = document.getElementsByTagNameNS(
// ESPER_NS, ESPER_CONFIG_LOCAL_NAME);
final NodeList esperConfigNodeList = document
.getElementsByTagName(ESPER_CONFIG_LOCAL_NAME);
for (int i = 0; i < esperConfigNodeList.getLength(); ++i) {
_log.debug("Configuring Esper with xml node.");
final Node esperConfigNode = esperConfigNodeList.item(i);
_log.debug("Esper xml configuration: {}",
esperConfigNode.cloneNode(true));
try {
final Document esperConfigDocument = DocumentBuilderFactory
.newInstance().newDocumentBuilder().newDocument();
esperConfigDocument.appendChild(esperConfigDocument.importNode(
esperConfigNode, true));
_configuration.configure(esperConfigDocument);
} catch (ParserConfigurationException e) {
_log.debug("Error configuring Esper with xml node.");
throw new RuntimeException(e);
}
_log.debug("Finished configuring Esper with xml node.");
}
final NodeList esperStatementNodeList = document
.getElementsByTagName(ESPER_STATEMENT_LOCAL_NAME);
for (int i = 0; i < esperStatementNodeList.getLength(); ++i) {
final Node esperStatementNode = esperStatementNodeList.item(i);
try {
_esperStatements.add(new StatementBeanFactory()
.createStatementBean(esperStatementNode));
} catch (Exception e) {
throw new RuntimeException(
"Error reading Esper statement from configuration!", e);
}
}
}
//
//
//
public void setOutput(Queue[] queues) {
_sinksMap = new HashMap<String, Sink>();
for (int i = 0; i < queues.length; ++i) {
_sinksMap.put(queues[i].getId(), queues[i]);
}
return;
}
public Configuration getConfiguration() {
return _configuration;
}
@Parameter(defaultValue = "", description = "The URI of the Esper Runtime.", required = false)
public String getProviderUri() {
return _epProviderURI;
}
public void setProviderUri(String providerUri) {
_epProviderURI = providerUri;
}
public long getInitialTime() {
return _initialTime;
}
public void setInitialTime(long initialTime) {
_initialTime = initialTime;
_currentTime = initialTime;
}
}