package com.linkedin.databus.client;
/*
*
* Copyright 2013 LinkedIn Corp. All rights reserved
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.avro.Schema;
import org.apache.log4j.Logger;
import com.linkedin.databus.client.pub.SCN;
import com.linkedin.databus.core.Checkpoint;
import com.linkedin.databus.core.DbusEventBuffer;
import com.linkedin.databus.core.DbusLogAccumulator;
import com.linkedin.databus.core.util.IdNamePair;
import com.linkedin.databus2.core.container.request.RegisterResponseEntry;
import com.linkedin.databus2.core.container.request.RegisterResponseMetadataEntry;
import com.linkedin.databus2.schemas.SchemaId;
import com.linkedin.databus2.schemas.SchemaRegistryService;
import com.linkedin.databus2.schemas.VersionedSchemaSet;
public class DispatcherState
{
public static final String MODULE = DispatcherState.class.getName();
public static final Logger LOG = Logger.getLogger(MODULE);
public enum StateId
{
//Initial state when is the dispatcher is started
INITIAL,
//Start reading from event buffer and dispatch events.
START_DISPATCH_EVENTS,
//First event of a new window (end of window for the previous window has been seen)
EXPECT_EVENT_WINDOW,
//Start of a window (which will be followed by data events)
START_STREAM_EVENT_WINDOW,
//End of events for a source within a window
START_STREAM_SOURCE,
//Beginning of data events to be streamed
EXPECT_STREAM_DATA_EVENTS,
//End of events for a source within a window (which maybe followed by more sources)
END_STREAM_SOURCE,
//End of a window
END_STREAM_EVENT_WINDOW,
//Rollback is triggered (after error in the consumer)
ROLLBACK,
//After rollback, events are replayed to the consumer
REPLAY_DATA_EVENTS,
//Not used (???)
STOP_DISPATCH_EVENTS,
//Shutdown the dispatcher
CLOSED
}
private StateId _stateId;
//START_DISPATCH_EVENTS
private final Map<Long, IdNamePair> _sources = new HashMap<Long, IdNamePair>();
//payload schemas
private static final VersionedSchemaSet _schemaSet = new VersionedSchemaSet(true);
//metadata schemas
private static final VersionedSchemaSet _metadataSchemasSet = new VersionedSchemaSet();
private DbusEventBuffer.DbusEventIterator _eventsIterator;
private DbusEventBuffer.DbusEventIterator _lastSuccessfulIterator;
// Looks like _payloadSchemaMap is a member variable purely for testing purposes. Keeping it thus can
// introduce bugs like DDSDBUS-3271. Need to remove the member variable.
private final Map<Long, List<RegisterResponseEntry>> _payloadSchemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
private DbusEventBuffer _buffer;
//EXPECT_EVENT_WINDOW extends START_DISPATCH_EVENTS
//START_STREAM_EVENT_WINDOW extends DISPATCH_EVENTS
private volatile SCN _startWinScn;
private boolean _eventsSeen;
//START_STREAM_SOURCE extends START_STREAM_EVENT_WINDOW
private IdNamePair _currentSource;
private Schema _currentSourceSchema;
//EXPECT_STREAM_DATA_EVENTS extends START_STREAM_SOURCE
//END_STREAM_SOURCE extends START_STREAM_SOURCE
private volatile SCN _endWinScn;
//END_STREAM_EVENT_WINDOW extends START_STREAM_EVENT_WINDOW
//CHECKPOINT extends END_STREAM_EVENT_WINDOW
private Checkpoint _lastSuccessfulCheckpoint;
private SCN _lastSuccessfulScn;
//ROLLBACK extends DISPATCH_EVENTS
//REPLAY_DATA_EVENTS extends ROLLBACK
//STOP_DISPATCH_EVENTS
private DbusEventAvroDecoder _eventDecoder;
private boolean _scnRegress = false;
private DispatcherState()
{
super();
_stateId = StateId.INITIAL;
}
private DispatcherState(DbusEventBuffer buffer)
{
super();
_stateId = StateId.INITIAL;
_buffer = buffer;
}
public static DispatcherState create()
{
return new DispatcherState();
}
public static DispatcherState create(DbusEventBuffer eventBuffer, String iteratorName)
{
DispatcherState result = new DispatcherState(eventBuffer);
result.createEventsIterator(iteratorName);
return result;
}
private void setLastSuccessfulIterator(DbusEventBuffer.DbusEventIterator newValue)
{
final boolean isDebugEnabled = LOG.isDebugEnabled();
if (isDebugEnabled)
{
DbusLogAccumulator.addLog("Changing _lastSuccessfulIterator from: " + _lastSuccessfulIterator, LOG);
}
if (null != _lastSuccessfulIterator)
{
_lastSuccessfulIterator.close();
}
if (null == newValue)
{
_lastSuccessfulIterator = null;
}
else
{
_lastSuccessfulIterator = newValue.copy(_lastSuccessfulIterator, newValue.getIdentifier() + ".save");
}
if (isDebugEnabled)
{
DbusLogAccumulator.addLog("Changing _lastSuccessfulIterator to: " + _lastSuccessfulIterator, LOG);
}
}
public DispatcherState switchToStartDispatchEvents()
{
_stateId = StateId.START_DISPATCH_EVENTS;
return this;
}
private void createEventsIterator(String iteratorName)
{
if (null == _eventsIterator)
{
_stateId = StateId.START_DISPATCH_EVENTS;
_lastSuccessfulCheckpoint = null;
_lastSuccessfulScn = null;
resetSourceInfo();
_eventsIterator = _buffer.acquireIterator(iteratorName);
LOG.info("start dispatch from: " + _eventsIterator);
setLastSuccessfulIterator(_eventsIterator);
}
}
private void refreshSchemas(List<RegisterResponseMetadataEntry> metadataSchemaList)
{
final boolean isDebugEnabled = LOG.isDebugEnabled();
try
{
for (Map.Entry<Long, List<RegisterResponseEntry>> e: _payloadSchemaMap.entrySet())
{
for (RegisterResponseEntry r : e.getValue())
{
final long id = r.getId();
String schemaName = null;
if (_sources.containsKey(id))
{
schemaName = _sources.get(r.getId()).getName();
}
else
{
LOG.error("Obtained a RegisterResponseEntry with schema that has no sourceId set. id = " + id);
continue;
}
String schema = r.getSchema();
if (_schemaSet.add(schemaName, r.getVersion(), schema))
{
LOG.info("Registering schema name=" + schemaName + " id=" + e.getKey().toString() +
" version=" + r.getVersion());
if (isDebugEnabled)
{
String msg = "Registering schema name=" + schemaName + " id=" + e.getKey().toString() +
" version=" + r.getVersion() + ": " + schema;
DbusLogAccumulator.addLog(msg, LOG);
}
}
else
{
if (isDebugEnabled)
{
String msg = "Schema already known: " + schemaName + " version " + r.getId();
DbusLogAccumulator.addLog(msg, LOG);
}
}
}
}
//Refresh metadata schema map
if ((metadataSchemaList != null) && !metadataSchemaList.isEmpty())
{
for (RegisterResponseMetadataEntry e: metadataSchemaList)
{
SchemaId id = new SchemaId(e.getCrc32());
if (_metadataSchemasSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE,e.getVersion(),id,e.getSchema()))
{
LOG.info("Added metadata schema version " + e.getVersion() + ",schemaID=0x" + id);
}
else
{
if (isDebugEnabled)
{
String msg = "Metadata schema version " + e.getVersion() + ",schemaId=0x" + id + " already exists";
DbusLogAccumulator.addLog(msg, LOG);
}
}
}
}
else
{
if (isDebugEnabled)
{
String msg = "Metadata schema is empty";
DbusLogAccumulator.addLog(msg, LOG);
}
}
_eventDecoder = new DbusEventAvroDecoder(_schemaSet,_metadataSchemasSet);
}
catch (Exception e)
{
LOG.error("Error adding schema", e);
}
}
public void resetIterators()
{
if (null != _lastSuccessfulIterator)
{
setLastSuccessfulIterator(null);
_lastSuccessfulScn = null;
_lastSuccessfulCheckpoint = null;
}
if (null != _eventsIterator)
{
DbusEventBuffer eventBuffer = _eventsIterator.getEventBuffer();
String iteratorName = _eventsIterator.getIdentifier();
_eventsIterator.close();
_eventsIterator = eventBuffer.acquireIterator(iteratorName);
if (LOG.isDebugEnabled())
{
String msg = "Reset event iterator to: " + _eventsIterator;
DbusLogAccumulator.addLog(msg, LOG);
}
resetSourceInfo();
}
}
public void switchToExpectEventWindow()
{
_stateId = StateId.EXPECT_EVENT_WINDOW;
}
public DispatcherState switchToStopDispatch()
{
_stateId = StateId.STOP_DISPATCH_EVENTS;
setEventsIterator(null);
return this;
}
public DispatcherState switchToClosed()
{
_stateId = StateId.CLOSED;
setLastSuccessfulIterator(null);
setEventsIterator(null);
return this;
}
public synchronized void switchToStartStreamEventWindow(SCN startWinScn)
{
_stateId = StateId.START_STREAM_EVENT_WINDOW;
_startWinScn = startWinScn;
_eventsSeen = true;
resetSourceInfo();
}
public synchronized void switchToEndStreamEventWindow(SCN endWinScn)
{
_stateId = StateId.END_STREAM_EVENT_WINDOW;
_endWinScn = endWinScn;
_eventsSeen = false;
}
public void switchToStartStreamSource(IdNamePair source, Schema sourceSchema)
{
_stateId = StateId.START_STREAM_SOURCE;
_currentSource = source;
_currentSourceSchema = sourceSchema;
}
public void switchToEndStreamSource()
{
_stateId = StateId.END_STREAM_SOURCE;
}
public void switchToExpectStreamDataEvents()
{
_stateId = StateId.EXPECT_STREAM_DATA_EVENTS;
}
public void storeCheckpoint(Checkpoint cp, SCN scn)
{
_lastSuccessfulCheckpoint = cp;
_lastSuccessfulScn = scn;
setLastSuccessfulIterator(_eventsIterator);
}
public void switchToRollback()
{
_stateId = StateId.ROLLBACK;
_eventsSeen = false;
}
private void setEventsIterator(DbusEventBuffer.DbusEventIterator newValue)
{
String iterName = null == newValue ? "dispatcher iterator" :
newValue.getIdentifier();
if (null != _eventsIterator)
{
iterName = _eventsIterator.getIdentifier();
if (LOG.isDebugEnabled())
{
String msg = "Closing dispatcher iterator: " + _eventsIterator;
DbusLogAccumulator.addLog(msg, LOG);
}
_eventsIterator.close();
}
if (null == newValue)
{
_eventsIterator = null;
if (LOG.isDebugEnabled())
{
String msg = "Dispatcher iterator set to null";
DbusLogAccumulator.addLog(msg, LOG);
}
}
else
{
_eventsIterator = newValue.copy(_eventsIterator, iterName);
if (LOG.isDebugEnabled())
{
String msg = "New dispatcher iterator: " + _eventsIterator;
DbusLogAccumulator.addLog(msg, LOG);
}
}
}
public void switchToReplayDataEvents()
{
_stateId = StateId.REPLAY_DATA_EVENTS;
resetSourceInfo();
setEventsIterator(_lastSuccessfulIterator);
}
public void resetSourceInfo()
{
_currentSource = null;
_currentSourceSchema = null;
}
public StateId getStateId()
{
return _stateId;
}
public DbusEventBuffer.DbusEventIterator getEventsIterator()
{
return _eventsIterator;
}
public Map<Long, IdNamePair> getSources()
{
return _sources;
}
public DbusEventAvroDecoder getEventDecoder()
{
return _eventDecoder;
}
public synchronized SCN getStartWinScn()
{
return _startWinScn;
}
public IdNamePair getCurrentSource()
{
return _currentSource;
}
public Schema getCurrentSourceSchema()
{
return _currentSourceSchema;
}
public synchronized SCN getEndWinScn()
{
return _endWinScn;
}
public Checkpoint getLastSuccessfulCheckpoint()
{
return _lastSuccessfulCheckpoint;
}
public VersionedSchemaSet getSchemaSet()
{
return _schemaSet;
}
public SCN getLastSuccessfulScn()
{
return _lastSuccessfulScn;
}
public DbusEventBuffer.DbusEventIterator getLastSuccessfulIterator()
{
return _lastSuccessfulIterator;
}
@Override
public String toString()
{
return "DispatcherState:" + _stateId.toString();
}
public Map<Long, List<RegisterResponseEntry>> getSchemaMap()
{
return _payloadSchemaMap;
}
public void removeEvents()
{
DbusEventBuffer.DbusEventIterator iter = getEventsIterator();
if (!iter.equivalent(_lastSuccessfulIterator))
{
if (_lastSuccessfulIterator == null)
{
LOG.warn("Last Successful Iterator was null. Rollback will not be possible!");
}
else
{
LOG.info("Invalidating last successful iterator " + "last = " +
_lastSuccessfulIterator + " this iterator= " + iter);
setLastSuccessfulIterator(null);
}
}
iter.remove();
}
public boolean isSCNRegress()
{
return _scnRegress;
}
public void setSCNRegress(boolean scnRegress)
{
_scnRegress = scnRegress;
}
public boolean isEventsSeen()
{
return _eventsSeen;
}
public void setEventsSeen(boolean hasSeenDataEvents)
{
_eventsSeen = hasSeenDataEvents;
}
protected DispatcherState addSources(Collection<IdNamePair> sources)
{
for (IdNamePair source: sources)
{
_sources.put(source.getId(), source);
}
return this;
}
protected DispatcherState addSchemas(Map<Long, List<RegisterResponseEntry>> schemaMap)
{
return addSchemas(schemaMap,null);
}
protected DispatcherState addSchemas(Map<Long, List<RegisterResponseEntry>> schemaMap,
List<RegisterResponseMetadataEntry> metadataSchemaList)
{
_payloadSchemaMap.putAll(schemaMap);
refreshSchemas(metadataSchemaList);
return this;
}
}