package streams.esper;
/*
* #%L
* Esper implementation of Streams Nodes
* %%
* Copyright (C) 2013 University of Zurich, Department of Informatics
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-2.0.html>.
* #L%
*/
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import stream.Processor;
import stream.annotations.Parameter;
import stream.io.Sink;
import stream.service.Service;
import com.espertech.esper.client.Configuration;
import com.espertech.esper.client.EPAdministrator;
import com.espertech.esper.client.EPRuntime;
import com.espertech.esper.client.EPServiceProvider;
import com.espertech.esper.client.EPServiceProviderManager;
import com.espertech.esper.client.EPStatement;
import com.espertech.esper.client.EventType;
import com.espertech.esper.client.soda.EPStatementObjectModel;
import com.espertech.esper.client.soda.Stream;
import com.espertech.esper.client.time.CurrentTimeEvent;
import com.espertech.esper.client.time.TimerEvent;
/**
* <p>
* Abstract class that provides basic Esper support for the Streams platform.
* </p>
* <p>
* <a href="http://esper.codehaus.org/">Esper</a> is a Complex Event Processing
* (CEP) platform for the Java and .Net languages. Esper must be started from
* custom code. It provides parallel execution but no distribution. The Streams
* stream processing platform allows to distribute Esper over different
* processes.
* </p>
* <p>
* This package on the other hand allows integrating Complex Event Processing
* with a high level query language, i.e., the Event Processing Language (EPL).
* The EPL is a SQL-like language. For further information please refer to the
* <a href=
* "http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html_single/#epl-intro"
* >Esper documentation</a>.
* </p>
* <p>
* The configuration of an Esper engine as a Streams {@link Service} follows the
* Xml configuration of the Streams platform. The configuration may include an
* <a href=
* "http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html_single/#configuration-xml"
* >Esper config</a>. EPL statements are added as Streams {@link Processor} in
* the form of Streams-Esper {@link Query} objects.
* </p>
* <p>
* Example Configuration:
*
* <pre>
* <Service class="streams.esper.EsperEngineService" id="esperEngine01">
* <config>
* <esper-configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
* xmlns="http://www.espertech.com/schema/esper"
* xsi:schemaLocation="http://www.espertech.com/schema/esper http://www.espertech.com/schema/esper/esper-configuration-2.0.xsd">
*
* <engine-settings>
* <defaults>
* <threading>
* <internal-timer msec-resolution="1" enabled="false" />
* </threading>
* </defaults>
* </engine-settings>
*
* <event-type name="LeftEvent">
* <java-util-map start-timestamp-property-name="timestamp">
* <map-property name="timestamp" class="long" />
* <map-property name="id" class="string" />
* </java-util-map>
* </event-type>
* <event-type name="RightEvent">
* <java-util-map start-timestamp-property-name="timestamp">
* <map-property name="timestamp" class="long" />
* <map-property name="id" class="string" />
* </java-util-map>
* </event-type>
*
* </esper-configuration>
* </config>
* </Service>
* </pre>
*
* </p>
*
* @author Thomas Scharrenbach
*
* @version 0.3.0
* @since 0.3.0
*
* @see streams.esper.Query
*
*/
public class EsperEngine implements stream.service.Service, stream.Configurable {
//
// Static fields and constants.
//
/**
* @since 0.3.0
* @version 0.3.0
*/
private static final Logger _log = LoggerFactory
.getLogger(EsperEngine.class);
/**
* @since 0.3.0
* @version 0.3.0
*/
public static final String ESPER_CONFIG_LOCAL_NAME = "esper-configuration";
/**
* @since 0.3.0
* @version 0.3.0
*/
public static final String ESPER_NS = "http://www.espertech.com/schema/esper";
/**
* @since 0.3.0
* @version 0.3.0
*/
public static final String EVENT_TYPE_KEY = "EPEventType";
/**
* @since 0.3.0
* @version 0.3.0
*/
public static final String ESPER_STATEMENT_LOCAL_NAME = "statement";
/**
* @since 0.3.0
* @version 0.3.0
*/
public static final String DEFAULT_ID = "default";
/**
* The default amount of difference in time up to which events are still
* considered for processing.
*
* @since 0.3.0
* @version 0.3.0
*
*/
public static final long DEFAULT_TIME_TOLERANCE = 1000;
private static final Map<String, EsperEngine> _registry = new HashMap<String, EsperEngine>();
//
// Static methods
//
/**
* Is being called from inside {@link #setId(String)}.
*
* @param engine
*/
private static void registerEngine(EsperEngine engine) {
if (_registry.containsKey(engine.getId())) {
final String errorMessage = String.format(
"Engine '%s' already registered!", engine.getId());
throw new IllegalArgumentException(errorMessage);
}
//
else {
_log.info("Registering engine {}", engine.getId());
_registry.put(engine.getId(), engine);
}
}
/**
* <p>
* If the engine with the specified id has not yet been registered but the
* id is equal to the default id, then a new engine with the default id will
* be created and registered and finally returned.
* </p>
*
* @param id
* @return
* @since 0.3.0
* @version 0.3.0
*/
public static EsperEngine getEsperEngine(String id) {
EsperEngine result = _registry.get(id);
if (result == null && DEFAULT_ID.equals(id)) {
synchronized (_registry) {
_log.info("Requesting default engine, but default engine was not yet registered. Creating it...");
result = new EsperEngine();
result.setId(DEFAULT_ID);
try {
result.init();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
return result;
}
/**
* @author Christian Bockermann
*
* @since 0.3.0
* @version 0.3.0
*
* @param name
* @return
*/
protected static Class<?> classForName(String name) {
//
// the default packages to look for classes...
//
String[] pkgs = new String[] { "", "java.lang" };
for (String pkg : pkgs) {
String className = name;
if (!pkg.isEmpty())
className = pkg + "." + name;
try {
Class<?> clazz = Class.forName(className);
if (clazz != null)
return clazz;
} catch (Exception e) {
}
}
return null;
}
//
//
//
protected transient EPServiceProvider _epService;
protected transient EPRuntime _epRuntime;
private long _currentTime;
private long _initialTime;
private final Configuration _configuration;
private long _itemsCounter;
private long _timeTolerance;
private String _id;
private final Map<String, Class<?>> _typesMap = new LinkedHashMap<String, Class<?>>();
/**
* Maps Esper event types to property names of start or end timestamps.
*/
private final Map<String, String> _startTimestampMap;
private final Map<String, String> _endTimestampMap;
//
// Fields that are parameters.
//
private String _epProviderURI;
private final List<EsperStatementBean> _staticStatements;
private Integer _shutdownCount;
//
//
//
/**
* <p>
* Initializes the following components:
* <ul>
* <li>an empty Esper {@link Configuration},</li>
* <li>the maps for types with a start and an end timsestamp,</li>
* <li>the list of Esper statements ( {@link Query} ), and</li>
* <li>the default values for the time tolerance.</li>
* </ul>
* </p>
*
* @since 0.3.0
* @version 0.3.0
*/
public EsperEngine() {
_configuration = new Configuration();
_startTimestampMap = new HashMap<String, String>();
_endTimestampMap = new HashMap<String, String>();
_staticStatements = new ArrayList<EsperStatementBean>();
_timeTolerance = DEFAULT_TIME_TOLERANCE;
}
//
//
//
/**
* Initializes the Esper service.
*
* @since 0.3.0
* @version 0.3.0
*/
public void init() throws Exception {
_log.info("Started initializing {} ...", this.getClass());
_shutdownCount = 0;
_itemsCounter = 0;
_currentTime = Long.MIN_VALUE;
final String providerUri = getProviderUri();
if (providerUri == null || providerUri.isEmpty()) {
_log.debug("Creating new Esper service from default provider.");
_epService = EPServiceProviderManager
.getDefaultProvider(_configuration);
}
//
else {
_log.debug("Creating new Esper service from named provider: {}",
providerUri);
_epService = EPServiceProviderManager.getProvider(providerUri,
_configuration);
}
_log.info("Declaring types to Esper service");
for (Entry<String, Class<?>> typesEntry : _typesMap.entrySet()) {
_configuration.addEventType(typesEntry.getKey(),
typesEntry.getValue());
}
_log.info("Finished declaring types to Esper service");
_log.info("Adding static queries");
for (EsperStatementBean statement : _staticStatements) {
addEsperQuery(statement);
}
_log.info("Finished adding static queries");
_log.debug("Creating Esper runtime...");
_epRuntime = _epService.getEPRuntime();
_log.debug("Finished creating Esper runtime...");
_log.info("Finished initalizing {}.", this.getClass());
}
/**
* <p>
* </p>
*
* @throws Exception
*
* @since 0.3.2
* @version 0.3.2
*/
public void notifyShutdown() throws Exception {
synchronized (_shutdownCount) {
--_shutdownCount;
}
if (_shutdownCount <= 0) {
_log.info("Started destroying Esper engine...");
try {
_epService.destroy();
} catch (Exception e) {
_log.error("Error destroying Esper engine!");
throw e;
}
_log.info("Finished destroying Esper engine.");
}
}
/**
* Extracts the stamps from the specified data item.
*
* @param input
* @param event
* @param mapEventTypeName
* @return
*/
private boolean checkTimestamps(stream.Data input, stream.Data event,
String mapEventTypeName) {
// If this event defines a start time, then adjust the current time if
// necessary and replace the string value with the long value.
final String startTimeKey = _startTimestampMap.get(mapEventTypeName);
if (startTimeKey != null) {
final long dataStartTime = ((Number) input.get(startTimeKey))
.longValue();
event.put(startTimeKey, dataStartTime);
if (dataStartTime > _currentTime) {
TimerEvent timeEvent = null;
// If first timestamp to set, then advance to data time.
if (_currentTime != Long.MIN_VALUE) {
_currentTime = dataStartTime;
timeEvent = new CurrentTimeEvent(_currentTime
- _timeTolerance);
_log.debug("Sending time event new time: {}", _currentTime);
_log.debug("Data items per time interval: {}",
_itemsCounter);
_itemsCounter = 0;
}
// Advance to new data time.
else {
_currentTime = dataStartTime;
timeEvent = new CurrentTimeEvent(_currentTime
- _timeTolerance);
_log.debug("Setting start time: {}", _currentTime);
_log.debug("Data items per time interval: {}",
_itemsCounter);
_itemsCounter = 0;
}
_epRuntime.sendEvent(timeEvent);
}
// Data items that fall outside the time limit are ignored.
else if (dataStartTime < _currentTime - _timeTolerance) {
if (_log.isDebugEnabled()) {
_log.debug("Time inconsistency! {} Tolerance: {}",
(_currentTime - dataStartTime), _timeTolerance);
return false;
}
}
}
// If this event defines an end time, then replace the string value with
// the long value.
final String endTimeKey = _endTimestampMap.get(mapEventTypeName);
if (endTimeKey != null) {
final long dataEndTime = Long.parseLong(input.get(endTimeKey)
.toString());
event.put(endTimeKey, dataEndTime);
}
return true;
}
/**
* <p>
* Decodes a {@link stream.Data} input item and sends it to the Esper
* engine. The result of the processing in the Esper engine is output to
* {@link stream.io.Sink} objects asynchonously.
* </p>
* <p>
* The method determines the type of input by evaluating the field
* "@stream". It sends a copy of the input {@link stream.Data} item to the
* Esper {@link EPRuntime}.It removes the values for "@stream" and
* "@stream:id" from the copied {@link stream.Data} item before sending it.
* </p>
* <p>
* The the input {@link stream.Data} item defines a start time, then this
* start time is compared with the current data time. If the time stamp of
* the input {@link stream.Data} item is larger than the current data time,
* then the current data time is set to the input {@link stream.Data} item's
* start time. A time event with the {@link stream.Data} item's start time
* is sent to the Esper {@link EPRuntime}.
* </p>
*
* @since 0.3.0
* @version 0.3.0
*
* @return null, since this implementation works asynchronously.
*/
// @Override
public boolean write(stream.Data input) throws Exception {
final streams.esper.EsperData event = new EsperData(input.createCopy());
// remove the streams keys since they might interfer with Esper
event.remove("@stream");
event.remove("@stream:id");
final Object mapEventTypeName = input.get("@esperType");
// In case the name of the Esper type was provided for a data item, then
// determine its sender from the engine and send the data via the
// sender.
if (mapEventTypeName != null) {
if (checkTimestamps(input, event, (String) mapEventTypeName)) {
_epRuntime.getEventSender((String) mapEventTypeName).sendEvent(
event);
}
}
// For events for which we do not know the type we simply try to send
// them to the engine.
else {
_epRuntime.sendEvent(event);
}
++_itemsCounter;
return true;
}
/**
* <p>
* Calls {@link EPServiceProvider#destroy()}.
* </p>
*
* @since 0.3.0
* @version 0.3.0
*
* @throws delegates
* any {@link Exception} that might be thrown during the calls
* of this method.
*/
@Override
public void reset() throws Exception {
_log.info("Started resetting Esper engine '{}' ...", this.getProviderUri());
if (_epService != null) {
_epService.destroy();
}
this.init();
_log.info("Finished resetting Esper engine '{}' .", this.getProviderUri());
}
//
// Methods from Configurable.
//
/**
* <p>
* Parses the <configuration> ... </configuration> tag if
* provided.
* </p>
* <p>
* The configuration may contain the following elements:
* <ul>
* <li>An Esper configuration element <esper-configuration> ...
* </esper-configuration> .</li>
* <li>An arbitrary number of Esper statements, i.e., each statement
* enclosed by <statement> ... </statement> .</li>
* </ul>
* </p>
*
* @since 0.3.0
* @version 0.3.0
*
*
*/
@Override
public void configure(Element document) {
// TODO fast hack, since Streams does not support namespaces.
// final NodeList esperConfigNodeList = document.getElementsByTagNameNS(
// ESPER_NS, ESPER_CONFIG_LOCAL_NAME);
final NodeList esperConfigNodeList = document
.getElementsByTagName(ESPER_CONFIG_LOCAL_NAME);
for (int i = 0; i < esperConfigNodeList.getLength(); ++i) {
_log.debug("Configuring Esper with xml node.");
final Node esperConfigNode = esperConfigNodeList.item(i);
_log.debug("Esper xml configuration: {}",
esperConfigNode.cloneNode(true));
try {
final Document esperConfigDocument = DocumentBuilderFactory
.newInstance().newDocumentBuilder().newDocument();
esperConfigDocument.appendChild(esperConfigDocument.importNode(
esperConfigNode, true));
_configuration.configure(esperConfigDocument);
} catch (ParserConfigurationException e) {
_log.debug("Error configuring Esper with xml node.");
throw new RuntimeException(e);
}
_log.debug("Finished configuring Esper with xml node.");
}
// Extract the statements given for the engine.
final NodeList esperStatementNodeList = document
.getElementsByTagName(ESPER_STATEMENT_LOCAL_NAME);
for (int i = 0; i < esperStatementNodeList.getLength(); ++i) {
final Element esperStatementNode = (Element) esperStatementNodeList
.item(i);
try {
final EsperStatementBean epStatement = new EsperStatementBean();
if (esperStatementNode.hasAttribute("name")) {
epStatement
.setName(esperStatementNode.getAttribute("name"));
}
if (esperStatementNode.hasAttribute("removeBackticks")) {
epStatement.setRemoveBackticks(Boolean
.parseBoolean(esperStatementNode
.getAttribute("removeBackticks")));
}
// if (esperStatementNode.hasAttribute("output")) {
// String[] output = esperStatementNode.getAttribute("output")
// .split(",");
// for (int outputIdx = 0; outputIdx < output.length;
// ++outputIdx) {
//
// }
// epStatement.setOutput();
// }
epStatement.setStatement(esperStatementNode.getTextContent()
.trim());
addStaticEsperStatement(epStatement);
} catch (Exception e) {
throw new RuntimeException(
"Error reading Esper statement from configuration!", e);
}
}
}
/**
* Adds a static statment to the Esper engine. Static statments will be
* addded to the engine before external statements are added via
* {@link #addEsperQuery(EsperStatementBean)}.
*
* @since 0.3.0
* @version 0.3.0
*
* @param epStatement
*/
private void addStaticEsperStatement(EsperStatementBean epStatement) {
_staticStatements.add(epStatement);
}
public void addEsperQuery(EsperStatementBean epStatement,
boolean increaseShutdownCount) {
++_shutdownCount;
addEsperQuery(epStatement);
}
/**
* <p>
* Add a statement to the Esper engine.
* </p>
* <p>
* Note that subscribers are added only for those statements that declare an
* output sink.
* </p>
*
* @since 0.3.0
* @version 0.3.0
*
* @param epStatement
*/
public void addEsperQuery(EsperStatementBean epStatement) {
_log.info("Compiling statement {}", epStatement);
_log.debug("Compiling Esper statement {}", epStatement.getStatement());
final EPAdministrator epAdmin = _epService.getEPAdministrator();
// Create a template for the statement in the current Esper engine.
final EPStatementObjectModel stmtModel = epAdmin.compileEPL(epStatement
.getStatement());
final EsperStreamEventTypeVisitor esperStreamVisitor = new EsperStreamEventTypeVisitor(
_epService.getEPAdministrator().getConfiguration());
for (Stream s : stmtModel.getFromClause().getStreams()) {
esperStreamVisitor.visitStream(s);
}
// Create the actual statement in the current Esper engine from the
// statement model.
final String stmtName = epStatement.getName();
final EPStatement stmt = (stmtName == null ? epAdmin.create(stmtModel)
: epAdmin.create(stmtModel, stmtName));
// If an output sink was defined, then we add a subscriber.
if (epStatement.getOutput() != null) {
final Sink[] sinksList = epStatement.getOutput();
if (sinksList == null) {
_log.warn("Statement {} has no sinks ");
} else {
final String[] propertyNames = stmt.getEventType()
.getPropertyNames();
final EsperStatementSubscriber subscriber = (epStatement
.isRemoveBackticks() ? new EsperTrimmedStatementSubscriber(
Arrays.asList(sinksList), propertyNames)
: new EsperStatementSubscriber(
Arrays.asList(sinksList), propertyNames));
_log.info("Adding subscriber {} to statement {}", sinksList,
epStatement);
stmt.setSubscriber(subscriber);
}
}
mapTimestampProperties();
_log.info("Finished compiling statement {}", epStatement);
}
/**
* @since 0.3.0
* @version 0.3.0
*/
private void mapTimestampProperties() {
_log.debug("Mapping event types to timestamp properties, if any");
final EPAdministrator epAdmin = _epService.getEPAdministrator();
for (EventType eventType : epAdmin.getConfiguration().getEventTypes()) {
final String startTimestampProperty = eventType
.getStartTimestampPropertyName();
final String endTimestampProperty = eventType
.getEndTimestampPropertyName();
final String timestampProperty = endTimestampProperty == null ? startTimestampProperty
: endTimestampProperty;
final String eventTypeName = eventType.getName();
_log.debug("Timestamp property for event type {}: {}",
eventTypeName, timestampProperty);
_startTimestampMap.put(eventTypeName, startTimestampProperty);
_endTimestampMap.put(eventTypeName, endTimestampProperty);
}
_log.debug("Finished mapping event types to timestamp properties, if any");
}
//
//
//
/**
* Getter for the Esper configuration.
*
* @since 0.3.0
* @version 0.3.0
*
* @return
*/
public Configuration getConfiguration() {
return _configuration;
}
/**
* Getter for the URI of the Esper engine.
*
* @since 0.3.0
* @version 0.3.0
*
* @return
*/
@Parameter(defaultValue = "", description = "The URI of the Esper Runtime.", required = false)
public String getProviderUri() {
return _epProviderURI;
}
public void setProviderUri(String providerUri) {
_epProviderURI = providerUri;
}
/**
* Getter for the initial timestamp of the Esper engine.
*
* @since 0.3.0
* @version 0.3.0
*
* @return
*/
public long getInitialTime() {
return _initialTime;
}
@Parameter(required = false)
public void setInitialTime(long initialTime) {
_initialTime = initialTime;
_currentTime = initialTime;
}
/**
* <p>
* Getter for the maximal difference a timestamp may reach into the past.
* </p>
*
* @since 0.3.0
* @version 0.3.0
*
* @return
*/
public long getTimeTolerance() {
return _timeTolerance;
}
@Parameter(name = "timeTolerance", defaultValue = "100000", description = "The tolerance "
+ "how many msecs a time event may reach into the past "
+ "to be still processed.", required = false)
public void setTimeTolerance(long timeTolerance) {
_timeTolerance = timeTolerance;
}
public void setId(String id) {
if (_id != null) {
final String errorMessage = String.format(
"Parameter %s already defined.", "id");
throw new IllegalArgumentException(errorMessage);
}
_id = id;
registerEngine(this);
}
/**
* <p>
* Getter for the id of this service.
* </p>
*
* @since 0.3.0
* @version 0.3.0
*
* @return
*/
public String getId() {
return _id;
}
/**
* <p>
* Getter for the map of Esper types to Java classes via a streams
* attribute.
* </p>
*
* @author Christian Bockermann
*
* @since 0.3.0
* @version 0.3.0
*
* @return
*/
public String[] getTypes() {
final List<String> result = new ArrayList<String>();
Iterator<String> it = _typesMap.keySet().iterator();
while (it.hasNext()) {
String key = it.next();
Class<?> clazz = (Class<?>) _typesMap.get(key);
result.add(String.format("%s:%s", key, clazz));
}
return result.toArray(new String[result.size()]);
}
/**
*
* @author Matthias Weidlich, Christian Bockermann
*
* @since 0.3.0
* @version 0.3.0
*
* @param types
*/
@Parameter(required = false, description = "Simple key:value mapping of properties")
public void setTypes(String[] types) {
_typesMap.clear();
for (String typeDefinition : types) {
int idx = typeDefinition.indexOf(":");
// Parse attribute value.
if (idx > 0) {
String key = typeDefinition.substring(0, idx);
String type = typeDefinition.substring(idx + 1);
Class<?> clazz = classForName(type);
if (clazz != null) {
_log.debug("Defining type class '{}' for key '{}'", key,
clazz);
_typesMap.put(key, clazz);
}
// Could not find a matching class in the class path.
else {
final String errorMessage = String.format(
"Failed to locate class for type '%s'!", type);
throw new IllegalArgumentException(errorMessage);
}
}
// Attribute values must be of the format "key:value".
else {
final String errorMessage = String
.format("Type definition contains no colon!");
throw new IllegalArgumentException(errorMessage);
}
}
_log.debug("Types: {}", (Object[]) types);
}
//
//
//
}