package com.linkedin.databus.client.generic;
/*
*
* Copyright 2013 LinkedIn Corp. All rights reserved
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
import java.io.IOException;
import org.apache.avro.Schema;
import org.apache.log4j.Logger;
import com.linkedin.databus.client.ClientFileBasedEventTrackingCallback;
import com.linkedin.databus.client.ClientFileBasedMetadataTrackingCallback;
import com.linkedin.databus.client.DbusEventAvroDecoder;
import com.linkedin.databus.client.consumer.AbstractDatabusCombinedConsumer;
import com.linkedin.databus.client.pub.ConsumerCallbackResult;
import com.linkedin.databus.client.pub.DbusEventDecoder;
import com.linkedin.databus.client.pub.SCN;
import com.linkedin.databus.core.DbusEvent;
import com.linkedin.databus.core.FileBasedEventTrackingCallback;
import com.linkedin.databus.core.util.ConfigBuilder;
import com.linkedin.databus.core.util.InvalidConfigException;
public class DatabusFileLoggingConsumer extends AbstractDatabusCombinedConsumer
implements DatabusConsumerPauseInterface
{
public final static String MODULE = DatabusFileLoggingConsumer.class.getName();
public final static Logger LOG = Logger.getLogger(MODULE);
private ClientFileBasedEventTrackingCallback _fileBasedDecodedValueCallback = null;
private ClientFileBasedMetadataTrackingCallback _fileBasedMetadataCallback = null;
private FileBasedEventTrackingCallback _fileBasedRawEventCallback = null;
private boolean _isPaused;
private String _EventPattern = null;
public static class StaticConfig
{
private final String _valueDumpFile;
private final String _metadataDumpFile;
private final String _eventDumpFile;
private boolean _append;
/** The file in which to store the payload values in JSON format. If null, no values are to be stored. */
public String getValueDumpFile()
{
return _valueDumpFile;
}
/** The file in which to store the decoded metadata info from v2 events. If null, no metadata are to be stored. */
public String getMetadataDumpFile()
{
return _metadataDumpFile;
}
/** The file in which to store the raw (undecoded) event in JSON format. If null, no raw events will be stored. */
public String getEventDumpFile()
{
return _eventDumpFile;
}
public boolean isAppendOnly()
{
return _append;
}
//NOT USED?
// public StaticConfig(String valueDumpFile, boolean append)
// {
// this(valueDumpFile, null, append);
// }
public StaticConfig(String valueDumpFile, String metadataDumpFile, String eventDumpFile, boolean append)
{
_valueDumpFile = valueDumpFile;
_metadataDumpFile = metadataDumpFile;
_eventDumpFile = eventDumpFile;
_append = append;
}
}
public static class StaticConfigBuilder implements ConfigBuilder<StaticConfig>
{
private String _valueDumpFile;
private String _metadataDumpFile;
private String _eventDumpFile;
private boolean _appendOnly = false; // by default file logging is not append-only
public String getValueDumpFile()
{
return _valueDumpFile;
}
public String getMetadataDumpFile()
{
return _metadataDumpFile;
}
public String getEventDumpFile()
{
return _eventDumpFile;
}
public void setValueDumpFile(String valueDumpFile)
{
_valueDumpFile = valueDumpFile;
}
public void setMetadataDumpFile(String metadataDumpFile)
{
_metadataDumpFile = metadataDumpFile;
}
public void setEventDumpFile(String eventDumpFile)
{
_eventDumpFile = eventDumpFile;
}
public boolean getAppendOnly()
{
return _appendOnly;
}
public void setAppendOnly(boolean appendOnly)
{
this._appendOnly = appendOnly;
}
@Override
public StaticConfig build() throws InvalidConfigException
{
return new StaticConfig(_valueDumpFile, _metadataDumpFile, _eventDumpFile, _appendOnly);
}
}
//NOT USED?
//public DatabusFileLoggingConsumer() throws IOException
//{
// this((String)null, false);
//}
//NOT USED?
//public DatabusFileLoggingConsumer(StaticConfigBuilder configBuilder)
// throws IOException, InvalidConfigException
//{
// this(configBuilder.build());
//}
public DatabusFileLoggingConsumer(StaticConfig config) throws IOException
{
this(config.getValueDumpFile(), config.getMetadataDumpFile(), config.getEventDumpFile(), config.isAppendOnly());
}
public DatabusFileLoggingConsumer(String valueDumpFile, boolean appendOnly) throws IOException
{
this(valueDumpFile, null, null, appendOnly);
}
public DatabusFileLoggingConsumer(String valueDumpFile,
String metadataDumpFile,
String eventDumpFile,
boolean appendOnly)
throws IOException
{
LOG.info("DatabusFileLoggingConsumer instantiated with payload-value dump file: " + valueDumpFile +
", metadata dump file: " + metadataDumpFile +
", raw-event dump file: " + eventDumpFile +
", appendOnly: " + appendOnly);
if (valueDumpFile != null)
{
_fileBasedDecodedValueCallback = new ClientFileBasedEventTrackingCallback(valueDumpFile, appendOnly);
_fileBasedDecodedValueCallback.init();
}
if (metadataDumpFile != null)
{
_fileBasedMetadataCallback = new ClientFileBasedMetadataTrackingCallback(metadataDumpFile, appendOnly);
_fileBasedMetadataCallback.init();
}
if (eventDumpFile != null)
{
_fileBasedRawEventCallback = new FileBasedEventTrackingCallback(eventDumpFile, appendOnly);
_fileBasedRawEventCallback.init();
}
}
@Override
public ConsumerCallbackResult onCheckpoint(SCN checkpointScn)
{
waitIfPaused();
LOG.info("startEvents:" + checkpointScn.toString());
return ConsumerCallbackResult.SUCCESS;
}
protected void LogTypedValue(DbusEvent e, DbusEventDecoder eventDecoder)
{
LOG.info("Log Typed Value");
}
@Override
public ConsumerCallbackResult onDataEvent(DbusEvent e, DbusEventDecoder eventDecoder)
{
waitIfPaused();
if (!e.isValid())
{
throw new RuntimeException("Got invalid event!!!");
}
if (_fileBasedDecodedValueCallback != null)
{
_fileBasedDecodedValueCallback.dumpEventValue(e, eventDecoder);
}
if( _fileBasedMetadataCallback != null && eventDecoder instanceof DbusEventAvroDecoder )
{
_fileBasedMetadataCallback.dumpEventMetadata(e, (DbusEventAvroDecoder) eventDecoder);
}
if (_fileBasedRawEventCallback != null)
{
_fileBasedRawEventCallback.onEvent(e);
}
LogTypedValue(e, eventDecoder);
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onEndDataEventSequence(SCN endScn) {
waitIfPaused();
LOG.info("endEvents:" + endScn);
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onEndSource(String source, Schema sourceSchema) {
waitIfPaused();
LOG.info("endSource:" + source);
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onRollback(SCN startScn) {
waitIfPaused();
LOG.info("rollback:"+ startScn);
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onStartDataEventSequence(SCN startScn) {
waitIfPaused();
LOG.info("startDataEventSequence:" + startScn);
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onStartSource(String source, Schema sourceSchema) {
waitIfPaused();
LOG.info("startSource:" + source);
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onBootstrapEvent(DbusEvent e, DbusEventDecoder eventDecoder)
{
waitIfPaused();
if (!e.isValid())
{
throw new RuntimeException("Got invalid event!!!");
}
if (_fileBasedDecodedValueCallback != null)
{
_fileBasedDecodedValueCallback.dumpEventValue(e, eventDecoder);
}
if (_fileBasedRawEventCallback != null)
{
_fileBasedRawEventCallback.onEvent(e);
}
LogTypedValue(e, eventDecoder);
printBootstrapEventInfo(BootstrapStage.OnBootstrapEvent, e.toString());
return ConsumerCallbackResult.SUCCESS;
}
@Override
public synchronized void pause()
{
_isPaused = true;
LOG.info("Consumer is set to pause!");
}
@Override
public synchronized void resume()
{
_isPaused = false;
notifyAll();
LOG.info("Consumer is set to resume!");
}
@Override
public synchronized void waitIfPaused()
{
while (_isPaused)
{
LOG.info("Consumer is paused!");
try
{
wait();
}
catch (InterruptedException e)
{
// resume waiting, nothing to do
}
}
}
@Override
public ConsumerCallbackResult onStartBootstrapSequence(SCN startScn)
{
waitIfPaused();
printBootstrapEventInfo(BootstrapStage.StartBootstrapSequence, startScn.toString());
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onEndBootstrapSequence(SCN endScn)
{
waitIfPaused();
printBootstrapEventInfo(BootstrapStage.EndBootstrapSequence, endScn.toString());
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onStartBootstrapSource(String name, Schema sourceSchema)
{
waitIfPaused();
printBootstrapEventInfo(BootstrapStage.StartBootstrapSource, name);
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onEndBootstrapSource(String name, Schema sourceSchema)
{
waitIfPaused();
printBootstrapEventInfo(BootstrapStage.EndBootstrapSource, name);
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onBootstrapCheckpoint(SCN batchCheckpointScn)
{
waitIfPaused();
printBootstrapEventInfo(BootstrapStage.OnCheckpointEvent, batchCheckpointScn.toString());
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onError(Throwable err)
{
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onBootstrapError(Throwable err)
{
return ConsumerCallbackResult.SUCCESS;
}
enum StreamStage
{
StartDataEventSequence,
EndDataEventSequence,
OnStreamEvent,
OnCheckpointEvent,
StartStreamSource,
EndStreamSource,
InvalidStage
}
enum BootstrapStage
{
StartBootstrapSequence,
EndBootstrapSequence,
/* StartSnapshotBatch,
EndSnapshotBatch,
StartCatchupBatch,
EndCatchupBatch,
*/ OnBootstrapEvent,
OnCheckpointEvent,
StartBootstrapSource,
EndBootstrapSource
}
protected void printBootstrapEventInfo(BootstrapStage stage, String info)
{
// There are integration tests that rely on this message (they look for "EndBootstrapSequence:" in the logs)
LOG.info(stage + ": " + info);
}
@Override
public ConsumerCallbackResult onStartBootstrap()
{
waitIfPaused();
LOG.info("startBootstrap");
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onStopBootstrap()
{
waitIfPaused();
LOG.info("stopBootstrap");
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onBootstrapRollback(SCN batchCheckpointScn)
{
waitIfPaused();
LOG.info("startBootstrapRollback:" + batchCheckpointScn.toString());
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onStartConsumption()
{
waitIfPaused();
LOG.info("startConsumption");
return ConsumerCallbackResult.SUCCESS;
}
@Override
public ConsumerCallbackResult onStopConsumption()
{
waitIfPaused();
LOG.info("stopConsumption");
return ConsumerCallbackResult.SUCCESS;
}
public String getEventPattern()
{
return _EventPattern;
}
public Boolean isCheckingEventPattern()
{
return (_EventPattern != null) ;
}
public void setEventPattern(String checkEventPattern)
{
this._EventPattern = checkEventPattern;
}
}