/*******************************************************************************
* Copyright (c) 2010, 2011 LogSaw project and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* LogSaw project committers - initial API and implementation
*******************************************************************************/
package net.sf.logsaw.dialect.pattern;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import net.sf.logsaw.core.config.IConfigOption;
import net.sf.logsaw.core.config.IConfigOptionVisitor;
import net.sf.logsaw.core.config.model.StringConfigOption;
import net.sf.logsaw.core.dialect.ILogEntryCollector;
import net.sf.logsaw.core.dialect.ILogFieldProvider;
import net.sf.logsaw.core.dialect.support.ALogDialect;
import net.sf.logsaw.core.dialect.support.FilteringFieldProvider;
import net.sf.logsaw.core.field.ALogEntryField;
import net.sf.logsaw.core.field.LogEntry;
import net.sf.logsaw.core.logresource.IHasEncoding;
import net.sf.logsaw.core.logresource.IHasLocale;
import net.sf.logsaw.core.logresource.IHasTimeZone;
import net.sf.logsaw.core.logresource.IHasTimestampPattern;
import net.sf.logsaw.core.logresource.ILogResource;
import net.sf.logsaw.dialect.pattern.internal.Messages;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.LineIterator;
import org.eclipse.core.runtime.Assert;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.MultiStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.osgi.util.NLS;
/**
* @author Philipp Nanz
*/
public abstract class APatternDialect extends ALogDialect implements IHasTimestampPattern {
public static final StringConfigOption OPTION_PATTERN =
new StringConfigOption("pattern", "Conversion Pattern"); //$NON-NLS-1$ //$NON-NLS-2$
private IConversionPatternTranslator patternTranslator;
private ILogFieldProvider fieldProvider;
private List<ConversionRule> rules;
private Pattern internalPatternFirstLine;
private Pattern internalPatternFull;
private String timestampPattern;
/**
* @return the patternTranslator
*/
public IConversionPatternTranslator getPatternTranslator() {
return patternTranslator;
}
/**
* @param patternTranslator the patternTranslator to set
*/
public void setPatternTranslator(
IConversionPatternTranslator patternTranslator) {
this.patternTranslator = patternTranslator;
}
/**
* @return the rules
*/
public List<ConversionRule> getRules() {
return rules;
}
/**
* @param rules the rules to set
*/
public void setRules(List<ConversionRule> rules) {
this.rules = rules;
}
/**
* @return the internalPatternFirstLine
*/
public Pattern getInternalPatternFirstLine() {
return internalPatternFirstLine;
}
/**
* @param internalPatternFirstLine the internalPatternFirstLine to set
*/
public void setInternalPatternFirstLine(Pattern internalPatternFirstLine) {
this.internalPatternFirstLine = internalPatternFirstLine;
}
/**
* @return the internalPatternFull
*/
public Pattern getInternalPatternFull() {
return internalPatternFull;
}
/**
* @param internalPatternFull the internalPatternFull to set
*/
public void setInternalPatternFull(Pattern internalPatternFull) {
this.internalPatternFull = internalPatternFull;
}
/* (non-Javadoc)
* @see net.sf.logsaw.core.framework.ILogDialect#getFieldProvider()
*/
@Override
public final ILogFieldProvider getFieldProvider() {
return fieldProvider;
}
/**
* @param fieldProvider the fieldProvider to set
*/
public void setFieldProvider(ILogFieldProvider fieldProvider) {
this.fieldProvider = fieldProvider;
}
/* (non-Javadoc)
* @see net.sf.logsaw.core.logresource.IHasTimestampPattern#getTimestampPattern()
*/
@Override
public String getTimestampPattern() {
return timestampPattern;
}
/**
* Creates the pattern translator.
* @return the pattern translator
*/
protected abstract IConversionPatternTranslator doCreatePatternTranslator();
/**
* Creates the base field provider.
* @return the base field provider
*/
protected abstract ILogFieldProvider doCreateFieldProvider();
/**
* Returns the default conversion pattern or <code>null</code>
* @return the default conversion pattern
*/
protected String doGetDefaultConversionPattern() {
return null;
}
/**
* Calculates the <code>followedByQuotedString</code> for the specified conversion rules.
* @param externalPattern the external pattern
* @param rules the conversion rules
*/
protected void fillFollowedByQuotedString(String externalPattern, List<ConversionRule> rules) {
// Determine whether rules are followed by quoted string, allowing use of special Regex lazy modifiers
int idx = 0;
ConversionRule prevRule = null;
for (ConversionRule rule : rules) {
if ((rule.getBeginIndex() > idx) && (prevRule != null)) {
// Previous rule is followed by a quoted string, allowing special regex flags
prevRule.setFollowedByQuotedString(true);
}
idx = rule.getBeginIndex();
idx += rule.getLength();
prevRule = rule;
}
if ((externalPattern.length() > idx) && (prevRule != null)) {
// Previous rule is followed by a quoted string, allowing special regex flags
prevRule.setFollowedByQuotedString(true);
}
}
/**
* Converts the given external pattern to the internal Regex pattern using the specified conversion rules.
* @param externalPattern the external pattern
* @param patternTranslator the conversion pattern translator
* @param rules the conversion rules
* @return the internal pattern
* @throws CoreException if an error occurred
*/
protected String toRegexPattern(String externalPattern, IConversionPatternTranslator patternTranslator,
List<ConversionRule> rules, boolean firstLineOnly) throws CoreException {
// Build the internal Regex pattern
StringBuilder sb = new StringBuilder();
int idx = 0;
for (ConversionRule rule : rules) {
if (firstLineOnly && rule.isLineBreak()) {
// That's it
return sb.toString();
}
if (rule.getBeginIndex() > idx) {
// Escape chars with special meaning
sb.append(Pattern.quote(externalPattern.substring(idx, rule.getBeginIndex())));
}
idx = rule.getBeginIndex();
String regex = patternTranslator.getRegexPatternForRule(rule);
Assert.isNotNull(regex, "regex"); //$NON-NLS-1$
sb.append(regex);
idx += rule.getLength();
}
if (externalPattern.length() > idx) {
// Append suffix
sb.append(Pattern.quote(externalPattern.substring(idx)));
}
return sb.toString();
}
/**
* Extracts the available fields from the given rules.
* @param rules the conversion rules
* @return the list of fields
* @throws CoreException if an error occurred
*/
protected List<ALogEntryField<?, ?>> extractFields(List<ConversionRule> rules) throws CoreException {
List<ALogEntryField<?, ?>> ret = new ArrayList<ALogEntryField<?,?>>();
for (ConversionRule rule : rules) {
ALogEntryField<?, ?> fld = getPatternTranslator().getFieldForRule(rule);
if (fld != null) {
ret.add(fld);
}
}
return ret;
}
/* (non-Javadoc)
* @see net.sf.logsaw.core.framework.support.AConfigurableLogDialect#configureDefaults()
*/
@Override
public void configureDefaults() throws CoreException {
String externalPattern = doGetDefaultConversionPattern();
if (externalPattern != null) {
// Configure default pattern (if any)
configure(OPTION_PATTERN, externalPattern);
}
super.configureDefaults();
}
/* (non-Javadoc)
* @see net.sf.logsaw.core.framework.ALogDialect#configure(net.sf.logsaw.core.config.IConfigOption, java.lang.Object)
*/
@Override
public <T> void configure(IConfigOption<T> option, T value)
throws CoreException {
option.visit(new IConfigOptionVisitor() {
/* (non-Javadoc)
* @see net.sf.logsaw.core.config.IConfigOptionVisitor#visit(net.sf.logsaw.core.config.StringConfigOption, java.lang.String)
*/
@Override
public void visit(StringConfigOption opt, String value) throws CoreException {
if (OPTION_TIMESTAMP_PATTERN.equals(opt)) {
Assert.isNotNull(value, "timestampPattern"); //$NON-NLS-1$
timestampPattern = value;
} else if (OPTION_PATTERN.equals(opt)) {
Assert.isNotNull(value, "externalPattern"); //$NON-NLS-1$
getLogger().info("Configuring conversion pattern " + value); //$NON-NLS-1$
// Create pattern translator
IConversionPatternTranslator translator = doCreatePatternTranslator();
Assert.isNotNull(translator, "patternTranslator"); //$NON-NLS-1$
setPatternTranslator(translator);
// Extract rules from external pattern
value = translator.prepare(value);
Assert.isNotNull(value, "externalPattern"); //$NON-NLS-1$
List<ConversionRule> rules = translator.extractRules(value);
Assert.isNotNull(rules, "rules"); //$NON-NLS-1$
if (rules.isEmpty()) {
throw new CoreException(new Status(IStatus.ERROR, PatternDialectPlugin.PLUGIN_ID,
Messages.APatternDialect_error_invalidPattern));
}
for (ConversionRule rule : rules) {
// Apply default modifiers
translator.applyDefaults(rule, APatternDialect.this);
// Rewrite rules
translator.rewrite(rule, APatternDialect.this);
}
fillFollowedByQuotedString(value, rules);
setRules(rules);
// Convert rules to Regex (internal) pattern
try {
Pattern internalPatternFirstLine = Pattern.compile(toRegexPattern(value, translator, rules, true));
getLogger().debug("Internal Pattern (first line): " + internalPatternFirstLine.pattern()); //$NON-NLS-1$
setInternalPatternFirstLine(internalPatternFirstLine);
Pattern internalPatternFull = Pattern.compile(toRegexPattern(value, translator, rules, false));
getLogger().debug("Internal Pattern (full): " + internalPatternFull.pattern()); //$NON-NLS-1$
setInternalPatternFull(internalPatternFull);
} catch (PatternSyntaxException e) {
throw new CoreException(new Status(IStatus.ERROR, PatternDialectPlugin.PLUGIN_ID,
NLS.bind(Messages.APatternDialect_error_failedToTranslateToRegex, value)));
}
// Setup field provider
List<ALogEntryField<?, ?>> fields = extractFields(rules);
ILogFieldProvider innerProvider = doCreateFieldProvider();
Assert.isNotNull(innerProvider, "fieldProvider"); //$NON-NLS-1$
setFieldProvider(new FilteringFieldProvider(innerProvider, fields));
}
}
}, value);
super.configure(option, value);
}
/* (non-Javadoc)
* @see net.sf.logsaw.core.config.IConfigurableObject#validate(net.sf.logsaw.core.config.IConfigOption, java.lang.Object)
*/
@Override
public <T> void validate(IConfigOption<T> option, T value)
throws CoreException {
option.visit(new IConfigOptionVisitor() {
/* (non-Javadoc)
* @see net.sf.logsaw.core.config.IConfigOptionVisitor#visit(net.sf.logsaw.core.config.StringConfigOption, java.lang.String)
*/
@Override
public void visit(StringConfigOption opt, String value) throws CoreException {
if (OPTION_PATTERN.equals(opt)) {
Assert.isNotNull(value, "externalPattern"); //$NON-NLS-1$
getLogger().info("Validating conversion pattern " + value); //$NON-NLS-1$
// Create pattern translator
IConversionPatternTranslator translator = doCreatePatternTranslator();
Assert.isNotNull(translator, "patternTranslator"); //$NON-NLS-1$
// Extract rules from external pattern
value = translator.prepare(value);
Assert.isNotNull(value, "externalPattern"); //$NON-NLS-1$
List<ConversionRule> rules = translator.extractRules(value);
Assert.isNotNull(rules, "rules"); //$NON-NLS-1$
if (rules.isEmpty()) {
throw new CoreException(new Status(IStatus.ERROR, PatternDialectPlugin.PLUGIN_ID,
Messages.APatternDialect_error_invalidPattern));
}
for (ConversionRule rule : rules) {
// Apply default modifiers
translator.applyDefaults(rule, APatternDialect.this);
// Rewrite rules
translator.rewrite(rule, APatternDialect.this);
}
fillFollowedByQuotedString(value, rules);
// Convert rules to Regex (internal) pattern
try {
Pattern internalPatternFirstLine = Pattern.compile(toRegexPattern(value, translator, rules, true));
getLogger().debug("Internal Pattern (first line): " + internalPatternFirstLine.pattern()); //$NON-NLS-1$
Pattern internalPatternFull = Pattern.compile(toRegexPattern(value, translator, rules, false));
getLogger().debug("Internal Pattern (full): " + internalPatternFull.pattern()); //$NON-NLS-1$
} catch (PatternSyntaxException e) {
throw new CoreException(new Status(IStatus.ERROR, PatternDialectPlugin.PLUGIN_ID,
NLS.bind(Messages.APatternDialect_error_failedToTranslateToRegex, value)));
}
}
}
}, value);
super.validate(option, value);
}
/* (non-Javadoc)
* @see net.sf.logsaw.core.framework.AConfigurableLogDialect#getSupportedConfigOptions()
*/
@Override
public List<IConfigOption<?>> getRequiredConfigOptions() {
List<IConfigOption<?>> ret = new ArrayList<IConfigOption<?>>();
ret.add(OPTION_PATTERN);
return ret;
}
/* (non-Javadoc)
* @see net.sf.logsaw.core.framework.ILogDialect#parse(net.sf.logsaw.core.framework.ILogResource, java.io.InputStream, net.sf.logsaw.core.framework.ILogEntryCollector)
*/
@Override
public void parse(ILogResource log, InputStream input,
ILogEntryCollector collector) throws CoreException {
Assert.isNotNull(log, "log"); //$NON-NLS-1$
Assert.isNotNull(input, "input"); //$NON-NLS-1$
Assert.isNotNull(collector, "collector"); //$NON-NLS-1$
Assert.isTrue(isConfigured(), "Dialect should be configured by now"); //$NON-NLS-1$
try {
LogEntry currentEntry = null;
IHasEncoding enc = (IHasEncoding) log.getAdapter(IHasEncoding.class);
IHasLocale loc = (IHasLocale) log.getAdapter(IHasLocale.class);
if (loc != null) {
// Apply the locale
getPatternTranslator().applyLocale(loc.getLocale(), rules);
}
IHasTimeZone tz = (IHasTimeZone) log.getAdapter(IHasTimeZone.class);
if (tz != null) {
// Apply the timezone
getPatternTranslator().applyTimeZone(tz.getTimeZone(), rules);
}
LineIterator iter = IOUtils.lineIterator(input, enc.getEncoding());
int minLinesPerEntry = getPatternTranslator().getMinLinesPerEntry();
int lineNo = 0;
int moreLinesToCome = 0;
try {
String line = null;
while (iter.hasNext()) {
lineNo++;
if (minLinesPerEntry == 1) {
// Simple case
line = iter.nextLine();
} else {
String s = iter.nextLine();
if (moreLinesToCome == 0) {
Matcher m = getInternalPatternFirstLine().matcher(s);
if (m.find()) {
// First line
line = s;
moreLinesToCome = minLinesPerEntry - 1;
continue;
} else {
// Some crazy stuff
line = s;
}
} else if (iter.hasNext() && (moreLinesToCome > 1)) {
// Some middle line
line += IOUtils.LINE_SEPARATOR + s;
moreLinesToCome--;
continue;
} else {
// Last line
line += IOUtils.LINE_SEPARATOR + s;
if (!iter.hasNext()) {
line += IOUtils.LINE_SEPARATOR;
}
moreLinesToCome = 0;
}
}
// Error handling
List<IStatus> statuses = null;
boolean fatal = false; // determines whether to interrupt parsing
Matcher m = getInternalPatternFull().matcher(line);
if (m.find()) {
// The next line matches, so flush the previous entry and continue
if (currentEntry != null) {
collector.collect(currentEntry);
currentEntry = null;
}
currentEntry = new LogEntry();
for (int i = 0; i < m.groupCount(); i++) {
try {
getPatternTranslator().extractField(currentEntry, getRules().get(i),
m.group(i + 1));
} catch (CoreException e) {
// Mark for interruption
fatal = fatal || e.getStatus().matches(IStatus.ERROR);
// Messages will be displayed later
if (statuses == null) {
statuses = new ArrayList<IStatus>();
}
if (e.getStatus().isMultiStatus()) {
Collections.addAll(statuses, e.getStatus().getChildren());
} else {
statuses.add(e.getStatus());
}
}
}
// We encountered errors or warnings
if (statuses != null && !statuses.isEmpty()) {
currentEntry = null; // Stop propagation
IStatus status = new MultiStatus(PatternDialectPlugin.PLUGIN_ID,
0, statuses.toArray(new IStatus[statuses.size()]),
NLS.bind(Messages.APatternDialect_error_failedToParseLine, lineNo), null);
if (fatal) {
// Interrupt parsing in case of error
throw new CoreException(status);
} else {
collector.addMessage(status);
}
}
} else if (currentEntry != null) {
// Append to message
String msg = currentEntry.get(getFieldProvider().getMessageField());
currentEntry.put(getFieldProvider().getMessageField(), msg + IOUtils.LINE_SEPARATOR + line);
}
if (collector.isCanceled()) {
// Cancel parsing
break;
}
}
if (currentEntry != null) {
// Collect left over entry
collector.collect(currentEntry);
}
} finally {
LineIterator.closeQuietly(iter);
}
} catch (Exception e) {
throw new CoreException(new Status(IStatus.ERROR, PatternDialectPlugin.PLUGIN_ID,
NLS.bind(Messages.APatternDialect_error_failedToParseFile,
new Object[] {log.getName(), e.getLocalizedMessage()}), e));
}
}
}