* #%L

* Esper implementation of Streams Nodes

* %%

* Copyright (C) 2013 University of Zurich, Department of Informatics

* %%

* This program is free software: you can redistribute it and/or modify

* it under the terms of the GNU General Public License as

* published by the Free Software Foundation, either version 2 of the

* License, or (at your option) any later version.

*

* This program is distributed in the hope that it will be useful,

* but WITHOUT ANY WARRANTY; without even the implied warranty of

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

* GNU General Public License for more details.

*

* You should have received a copy of the GNU General Public

* License along with this program.  If not, see

* <http://www.gnu.org/licenses/gpl-2.0.html>.

* #L%

*/

* <p>

* Abstract class that provides basic Esper support for the Streams platform.

* </p>

* <p>

* <a href="http://esper.codehaus.org/">Esper</a> is a Complex Event Processing

* (CEP) platform for the Java and .Net languages. Esper must be started from

* custom code. It provides parallel execution but no distribution. The Streams

* stream processing platform allows to distribute Esper over different

* processes.

* </p>

* <p>

* This package on the other hand allows integrating Complex Event Processing

* with a high level query language, i.e., the Event Processing Language (EPL).

* The EPL is a SQL-like language. For further information please refer to the

* <a href=

* "http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html_single/#epl-intro"

* >Esper documentation</a>.

* </p>

* <p>

* The configuration of an Esper Processor follows the Xml configuration of the

* Streams platform. The configuration may include an <a href=

* "http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html_single/#configuration-xml"

* >Esper config</a>. EPL statements are added in textual form in the config

* file.

* </p>

* <p>

* Example Configuration:

*

* <pre>

* &lt;ch.uzh.ifi.ddis.ifp.streams.EsperProcessor

*     output="queue1,queue2"&gt;

*     &lt;config&gt;

*       &lt;esper-configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

*         xmlns="http://www.espertech.com/schema/esper"

*         xsi:schemaLocation="http://www.espertech.com/schema/esper http://www.espertech.com/schema/esper/esper-configuration-2.0.xsd"&gt;

*

*         &lt;engine-settings&gt;

*           &lt;defaults&gt;

*             &lt;threading&gt;

*               &lt;internal-timer msec-resolution="1" enabled="false" /&gt;

*             &lt;/threading&gt;

*           &lt;/defaults&gt;

*         &lt;/engine-settings&gt;

*

*         &lt;event-type name="LeftEvent"&gt;

*           &lt;java-util-map start-timestamp-property-name="timestamp"&gt;

*             &lt;map-property name="timestamp" class="long" /&gt;

*             &lt;map-property name="id" class="string" /&gt;

*           &lt;/java-util-map&gt;

*         &lt;/event-type&gt;

*         &lt;event-type name="RightEvent"&gt;

*           &lt;java-util-map start-timestamp-property-name="timestamp"&gt;

*             &lt;map-property name="timestamp" class="long" /&gt;

*             &lt;map-property name="id" class="string" /&gt;

*           &lt;/java-util-map&gt;

*         &lt;/event-type&gt;

*

*       &lt;/esper-configuration&gt;

*       &lt;statement output="queue2"&gt;select id from LeftEvent&lt;/statement&gt;

*       &lt;statement&gt;select * from RightEvent&lt;/statement&gt;

*      &lt;/config&gt;

*   &lt;/ch.uzh.ifi.ddis.ifp.streams.EsperProcessor&gt;

* </pre>

*

* </p>

*

* @author Thomas Scharrenbach

* @version 0.0.1

* @since 0.0.1

*

*/

    stream.Configurable {

  private static final Logger _log = LoggerFactory

      .getLogger(EsperProcessor.class);

  public static final String ESPER_CONFIG_LOCAL_NAME = "esper-configuration";

  public static final String ESPER_NS = "http://www.espertech.com/schema/esper";

  public static final String ESPER_STATEMENT_LOCAL_NAME = "statement";

  public static final String EVENT_TYPE_KEY = "EPEventType";

  //

  //

  //

  protected transient EPServiceProvider _epService;

  protected transient EPRuntime _epRuntime;

  private long _currentTime;

  private long _initialTime;

  private final Configuration _configuration;

  private final List<String> _epStatements;

  /**

   * Maps Esper event types to property names of start or end timestamps.

   */

  private final Map<String, String> _startTimestampMap;

  private final Map<String, String> _endTimestampMap;

  //

  // Fields that are parameters.

  //

  private String _epProviderURI;

  private Map<String, Sink> _sinksMap;

  private final Map<String, String> _statementSinksMap;

  //

  //

  //

  public EsperProcessor() {

    _configuration = new Configuration();

    _epStatements = new ArrayList<String>();

    _statementSinksMap = new HashMap<String, String>();

    _startTimestampMap = new HashMap<String, String>();

    _endTimestampMap = new HashMap<String, String>();

    _currentTime = Long.MIN_VALUE;

  }

  //

  //

  //

  /**

   * Initializes the Esper service.

   */

  @Override

  public void init(ProcessContext context) throws Exception {

    super.init(context);

    initEPService();

  }

  private void initEPService() {

    _log.debug("Initializing {} ...", this.getClass());

    final String providerUri = getProviderUri();

    if (providerUri == null || providerUri.isEmpty()) {

      _log.debug("Creating new Esper service from default provider.");

      _epService = EPServiceProviderManager

          .getDefaultProvider(_configuration);

    }

    //

    else {

      _log.debug("Creating new Esper service from named provider: {}",

          providerUri);

      _epService = EPServiceProviderManager.getProvider(providerUri,

          _configuration);

    }

    _log.debug("Adding statements to Esper service");

    final EPAdministrator epAdmin = _epService.getEPAdministrator();

    // Add all statements to the Esper engine.

    // Note that subscribers are added only for those statements that

    // declare an output sink.

    for (String epStatement : _epStatements) {

      _log.debug("Compiling Esper statement {}", epStatement);

      // Create a template for the statement in the current Esper engine.

      final EPStatementObjectModel stmtModel = epAdmin

          .compileEPL(epStatement);

      // Create the actual statement in the current Esper engine from the

      // statement model.

      final EPStatement stmt = epAdmin.create(stmtModel);

      // Compute the hash value of the statement.

      // We use this to reference the statement's sink, if it declares

      // any.

      final String stmtName = Integer.toString(epStatement.hashCode());

      final Sink sink = _sinksMap.get(_statementSinksMap.get(stmtName));

      final String[] propertyNames = stmt.getEventType()

          .getPropertyNames();

      // Only add a subscriber, if the statement delcares to send its

      // output to a queue.

      if (sink != null) {

        final EsperStatementSubscriber subscriber = new EsperStatementSubscriber(

            sink, propertyNames);

        _log.debug("Adding subscriber to statement {}", stmtName);

        stmt.setSubscriber(subscriber);

      }

    }

    _log.debug("Finished adding statements to Esper service");

    _log.debug("Mapping event types to timestamp properties, if any");

    for (EventType eventType : epAdmin.getConfiguration().getEventTypes()) {

      final String startTimestampProperty = eventType

          .getStartTimestampPropertyName();

      final String endTimestampProperty = eventType

          .getEndTimestampPropertyName();

      final String timestampProperty = endTimestampProperty == null ? startTimestampProperty

          : endTimestampProperty;

      final String eventTypeName = eventType.getName();

      _log.debug("Timestamp property for event type {}: {}",

          eventTypeName, timestampProperty);

      _startTimestampMap.put(eventTypeName, startTimestampProperty);

      _endTimestampMap.put(eventTypeName, endTimestampProperty);

    }

    _log.debug("Finished mapping event types to timestamp properties, if any");

    _epRuntime = _epService.getEPRuntime();

    _log.debug("Finished initalizing {}.", this.getClass());

  }

  /**

   * <p>

   * Decodes a {@link stream.Data} input item and sends it to the Esper

   * engine. The result of the processing in the Esper engine is output to

   * {@link stream.io.Sink} objects asynchonously.

   * </p>

   * <p>

   * The method determines the type of input by evaluating the field

   * "@stream". It sends a copy of the input {@link stream.Data} item to the

   * Esper {@link EPRuntime}.It removes the values for "@stream" and

   * "@stream:id" from the copied {@link stream.Data} item before sending it.

   * </p>

   * <p>

   * The the input {@link stream.Data} item defines a start time, then this

   * start time is compared with the current data time. If the time stamp of

   * the input {@link stream.Data} item is larger than the current data time,

   * then the current data time is set to the input {@link stream.Data} item's

   * start time. A time event with the {@link stream.Data} item's start time

   * is sent to the Esper {@link EPRuntime}.

   * </p>

   *

   * @return null, since this implementation works asynchronously.

   */

  @Override

  public Data process(Data input) {

    final String mapEventTypeName = (String) input.get("@stream");

    final Data event = input.createCopy();

    event.remove("@stream");

    event.remove("@stream:id");

    // If this event defines a start time, then adjust the current time if

    // necessary and replace the string value with the long value.

    final String startTimeKey = _startTimestampMap.get(mapEventTypeName);

    if (startTimeKey != null) {

      final long dataStartTime = Long.parseLong(input.get(startTimeKey)

          .toString());

      event.put(startTimeKey, dataStartTime);

      if (dataStartTime > _currentTime) {

        _currentTime = dataStartTime;

        final CurrentTimeEvent timeEvent = new CurrentTimeEvent(

            _currentTime);

        _log.debug("Sending time event new time: {}", _currentTime);

        _epRuntime.sendEvent(timeEvent);

      }

    }

    // If this event defines an end time, then replace the string value with

    // the long value.

    final String endTimeKey = _endTimestampMap.get(mapEventTypeName);

    if (endTimeKey != null) {

      final long dataEndTime = Long.parseLong(input.get(endTimeKey)

          .toString());

      event.put(endTimeKey, dataEndTime);

    }

    _epRuntime.sendEvent(event, mapEventTypeName);

    return null;

  }

  /**

   * <p>

   * Calls {@link EPServiceProvider#destroy()} and {@link #initEPService()}

   * afterwards.

   * </p>

   *

   * @throws delegates

   *             any {@link Exception} that might be thrown during the calls

   *             of this method.

   */

  @Override

  public void resetState() throws Exception {

    super.resetState();

    _epService.destroy();

    initEPService();

  }

  /**

   * <p>

   * Calls {@link EPServiceProvider#destroy()}.

   * </p>

   *

   * @throws delegates

   *             any {@link Exception} that might be thrown during the calls

   *             of this method.

   */

  @Override

  public void finish() throws Exception {

    _log.debug("Finishing {} ...", this.getClass());

    if (_epService != null) {

      _epService.destroy();

    }

    _log.debug("Finished finishing {}", this.getClass());

  }

  //

  // Methods from Configurable.

  //

  /**

   * <p>

   * Parses the &lt;configuration&gt; ... &lt;/configuration&gt; tag if

   * provided.

   * </p>

   * <p>

   * The configuration may contain the following elements:

   * <ul>

   * <li>An Esper configuration element &lt;esper-configuration&gt; ...

   * &lt;/esper-configuration&gt; .</li>

   * <li>An arbitrary number of Esper statements, i.e., each statement

   * enclosed by &lt;statement&gt; ... &lt;/statement&gt; .</li>

   * </ul>

   * </p>

   */

  @Override

  public void configure(Element document) {

    // TODO fast hack, since Streams does not support namespaces.

    // final NodeList esperConfigNodeList = document.getElementsByTagNameNS(

    // ESPER_NS, ESPER_CONFIG_LOCAL_NAME);

    final NodeList esperConfigNodeList = document

        .getElementsByTagName(ESPER_CONFIG_LOCAL_NAME);

    for (int i = 0; i < esperConfigNodeList.getLength(); ++i) {

      _log.debug("Configuring Esper with xml node.");

      final Node esperConfigNode = esperConfigNodeList.item(i);

      _log.debug("Esper xml configuration: {}",

          esperConfigNode.cloneNode(true));

      try {

        final Document esperConfigDocument = DocumentBuilderFactory

            .newInstance().newDocumentBuilder().newDocument();

        esperConfigDocument.appendChild(esperConfigDocument.importNode(

            esperConfigNode, true));

        _configuration.configure(esperConfigDocument);

      } catch (ParserConfigurationException e) {

        _log.debug("Error configuring Esper with xml node.");

        throw new RuntimeException(e);

      }

      _log.debug("Finished configuring Esper with xml node.");

    }

    final NodeList esperStatementNodeList = document

        .getElementsByTagName(ESPER_STATEMENT_LOCAL_NAME);

    for (int i = 0; i < esperStatementNodeList.getLength(); ++i) {

      final Node esperStatementNode = esperStatementNodeList.item(i);

      final String esperStatement = esperStatementNode.getTextContent();

      _log.debug("Adding statement for Esper: {}", esperStatement);

      if (esperStatement == null) {

        _log.error("Empty Esper statement node.");

        throw new RuntimeException(

            "If specified, Esper statement must not be empty!");

      }

      //

      else {

        _epStatements.add(esperStatement);

        if (esperStatementNode.hasAttributes()) {

          final Node outputAttr = esperStatementNode.getAttributes()

              .getNamedItem("output");

          if (outputAttr != null) {

            // Note: _queue is not yet set. We cannot check whether

            // a queue actually exists.

            final String output = outputAttr.getTextContent();

            final String statementName = Integer

                .toString(esperStatement.hashCode());

            _statementSinksMap.put(statementName, output);

          }

        }

      }

    }

  }

  //

  //

  //

  public void setOutput(Queue[] queues) {

    _sinksMap = new HashMap<String, Sink>();

    for (int i = 0; i < queues.length; ++i) {

      _sinksMap.put(queues[i].getId(), queues[i]);

    }

    return;

  }

  public Configuration getConfiguration() {

    return _configuration;

  }

  @Parameter(defaultValue = "", description = "The URI of the Esper Runtime.", required = false)

  public String getProviderUri() {

    return _epProviderURI;

  }

  public void setProviderUri(String providerUri) {

    _epProviderURI = providerUri;

  }

  public List<String> getEpStatements() {

    return _epStatements;

  }

  public long getInitialTime() {

    return _initialTime;

  }

  public void setInitialTime(long initialTime) {

    _initialTime = initialTime;

    _currentTime = initialTime;

  }

}