package org.apache.blur.server;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static org.apache.blur.utils.BlurConstants.BLUR_FIELDTYPE;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLURINDEX_CLASS;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_SIMILARITY;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_READ_INTERCEPTOR;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMITS;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_REFRESHS;
import static org.apache.blur.utils.BlurConstants.SUPER;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.blur.BlurConfiguration;
import org.apache.blur.analysis.FieldManager;
import org.apache.blur.analysis.FieldTypeDefinition;
import org.apache.blur.analysis.HdfsFieldManager;
import org.apache.blur.analysis.NoStopWordStandardAnalyzer;
import org.apache.blur.log.Log;
import org.apache.blur.log.LogFactory;
import org.apache.blur.lucene.search.FairSimilarity;
import org.apache.blur.manager.ReadInterceptor;
import org.apache.blur.manager.indexserver.BlurIndexWarmup;
import org.apache.blur.manager.writer.BlurIndex;
import org.apache.blur.manager.writer.BlurIndexCloser;
import org.apache.blur.manager.writer.BlurIndexSimpleWriter;
//import org.apache.blur.manager.writer.BlurNRTIndex;
import org.apache.blur.manager.writer.SharedMergeScheduler;
import org.apache.blur.thrift.generated.ScoreType;
import org.apache.blur.thrift.generated.TableDescriptor;
import org.apache.blur.utils.BlurConstants;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
public class TableContext {
private static final Log LOG = LogFactory.getLog(TableContext.class);
private static final String LOGS = "logs";
private static final String TYPES = "types";
private static ConcurrentHashMap<String, TableContext> _cache = new ConcurrentHashMap<String, TableContext>();
private static Configuration _systemConfiguration;
private static BlurConfiguration _systemBlurConfiguration;
private static final ReadInterceptor DEFAULT_INTERCEPTOR = new ReadInterceptor(null) {
@Override
public Filter getFilter() {
return null;
}
};
private Path _tablePath;
private Path _walTablePath;
private String _defaultFieldName;
private String _table;
private IndexDeletionPolicy _indexDeletionPolicy;
private Similarity _similarity;
private Configuration _configuration;
private TableDescriptor _descriptor;
private long _timeBetweenCommits;
private long _timeBetweenRefreshs;
private ScoreType _defaultScoreType;
private Term _defaultPrimeDocTerm;
private FieldManager _fieldManager;
private BlurConfiguration _blurConfiguration;
private ReadInterceptor _readInterceptor;
protected TableContext() {
}
public static void clear() {
_cache.clear();
}
public static void clear(String table) {
_cache.remove(table);
}
public static TableContext create(TableDescriptor tableDescriptor) {
if (tableDescriptor == null) {
throw new NullPointerException("TableDescriptor can not be null.");
}
String name = tableDescriptor.getName();
if (name == null) {
throw new NullPointerException("Table name in the TableDescriptor can not be null.");
}
String tableUri = tableDescriptor.getTableUri();
if (tableUri == null) {
throw new NullPointerException("Table uri in the TableDescriptor can not be null.");
}
TableContext tableContext = _cache.get(name);
if (tableContext != null) {
return tableContext;
}
LOG.info("Creating table context for table [{0}]", name);
Configuration configuration = getSystemConfiguration();
BlurConfiguration blurConfiguration = getSystemBlurConfiguration();
Map<String, String> tableProperties = tableDescriptor.getTableProperties();
if (tableProperties != null) {
for (Entry<String, String> prop : tableProperties.entrySet()) {
configuration.set(prop.getKey(), prop.getValue());
blurConfiguration.set(prop.getKey(), prop.getValue());
}
}
tableContext = new TableContext();
tableContext._configuration = configuration;
tableContext._blurConfiguration = blurConfiguration;
tableContext._tablePath = new Path(tableUri);
tableContext._walTablePath = new Path(tableContext._tablePath, LOGS);
tableContext._defaultFieldName = SUPER;
tableContext._table = name;
tableContext._descriptor = tableDescriptor;
tableContext._timeBetweenCommits = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_COMMITS, 60000);
tableContext._timeBetweenRefreshs = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_REFRESHS, 5000);
tableContext._defaultPrimeDocTerm = new Term(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE);
tableContext._defaultScoreType = ScoreType.SUPER;
boolean strict = tableDescriptor.isStrictTypes();
String defaultMissingFieldType = tableDescriptor.getDefaultMissingFieldType();
boolean defaultMissingFieldLessIndexing = tableDescriptor.isDefaultMissingFieldLessIndexing();
Map<String, String> defaultMissingFieldProps = emptyIfNull(tableDescriptor.getDefaultMissingFieldProps());
Path storagePath = new Path(tableContext._tablePath, TYPES);
try {
HdfsFieldManager hdfsFieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(), storagePath,
configuration, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing, defaultMissingFieldProps);
loadCustomTypes(tableContext, blurConfiguration, hdfsFieldManager);
hdfsFieldManager.loadFromStorage();
tableContext._fieldManager = hdfsFieldManager;
} catch (IOException e) {
throw new RuntimeException(e);
}
Class<?> c1 = configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE,
KeepOnlyLastCommitDeletionPolicy.class);
tableContext._indexDeletionPolicy = (IndexDeletionPolicy) configure(ReflectionUtils.newInstance(c1, configuration),
tableContext);
Class<?> c2 = configuration.getClass(BLUR_SHARD_INDEX_SIMILARITY, FairSimilarity.class);
tableContext._similarity = (Similarity) configure(ReflectionUtils.newInstance(c2, configuration), tableContext);
String readInterceptorClass = blurConfiguration.get(BLUR_SHARD_READ_INTERCEPTOR);
if (readInterceptorClass == null || readInterceptorClass.trim().isEmpty()) {
tableContext._readInterceptor = DEFAULT_INTERCEPTOR;
} else {
try {
@SuppressWarnings("unchecked")
Class<? extends ReadInterceptor> clazz = (Class<? extends ReadInterceptor>) Class.forName(readInterceptorClass);
Constructor<? extends ReadInterceptor> constructor = clazz
.getConstructor(new Class[] { BlurConfiguration.class });
tableContext._readInterceptor = constructor.newInstance(blurConfiguration);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
tableContext._similarity = (Similarity) configure(ReflectionUtils.newInstance(c2, configuration), tableContext);
// DEFAULT_INTERCEPTOR
_cache.put(name, tableContext);
return tableContext;
}
@SuppressWarnings("unchecked")
private static void loadCustomTypes(TableContext tableContext, BlurConfiguration blurConfiguration,
FieldManager fieldManager) {
Set<Entry<String, String>> entrySet = blurConfiguration.getProperties().entrySet();
TableDescriptor descriptor = tableContext._descriptor;
for (Entry<String, String> entry : entrySet) {
String key = entry.getKey();
if (key.startsWith(BLUR_FIELDTYPE)) {
String className = entry.getValue();
descriptor.putToTableProperties(key, className);
LOG.info("Attempting to load new type [{0}]", className);
Class<? extends FieldTypeDefinition> clazz;
try {
clazz = (Class<? extends FieldTypeDefinition>) Class.forName(className);
FieldTypeDefinition fieldTypeDefinition = clazz.newInstance();
fieldManager.registerType(clazz);
LOG.info("Sucessfully loaded new type [{0}] with name [{1}]", className, fieldTypeDefinition.getName());
} catch (ClassNotFoundException e) {
LOG.error("The field type definition class [{0}] was not found. Check the classpath.", e, className);
} catch (InstantiationException e) {
LOG.error("Could not create the field type definition [{0}].", e, className);
} catch (IllegalAccessException e) {
LOG.error("Unknown exception while trying to load field type definition [{0}].", e, className);
}
}
}
}
private static Map<String, String> emptyIfNull(Map<String, String> defaultMissingFieldProps) {
if (defaultMissingFieldProps == null) {
return new HashMap<String, String>();
}
return defaultMissingFieldProps;
}
private static Object configure(Object o, TableContext tableContext) {
if (o instanceof Configurable) {
((Configurable) o).setTableContext(tableContext);
}
return o;
}
public IndexDeletionPolicy getIndexDeletionPolicy() {
return _indexDeletionPolicy;
}
public Similarity getSimilarity() {
return _similarity;
}
public long getTimeBetweenCommits() {
return _timeBetweenCommits;
}
public long getTimeBetweenRefreshs() {
return _timeBetweenRefreshs;
}
public FieldManager getFieldManager() {
return _fieldManager;
}
public String getTable() {
return _table;
}
public Configuration getConfiguration() {
return _configuration;
}
public TableDescriptor getDescriptor() {
return _descriptor;
}
public Path getTablePath() {
return _tablePath;
}
public Path getWalTablePath() {
return _walTablePath;
}
public String getDefaultFieldName() {
return _defaultFieldName;
}
public Term getDefaultPrimeDocTerm() {
return _defaultPrimeDocTerm;
}
public ScoreType getDefaultScoreType() {
return _defaultScoreType;
}
public long getTimeBetweenWALSyncsNanos() {
return TimeUnit.MILLISECONDS.toNanos(10);
}
public BlurConfiguration getBlurConfiguration() {
return _blurConfiguration;
}
public static synchronized Configuration getSystemConfiguration() {
if (_systemConfiguration == null) {
_systemConfiguration = new Configuration();
}
return new Configuration(_systemConfiguration);
}
public static void setSystemConfiguration(Configuration systemConfiguration) {
TableContext._systemConfiguration = systemConfiguration;
}
public static synchronized BlurConfiguration getSystemBlurConfiguration() {
if (_systemBlurConfiguration == null) {
try {
_systemBlurConfiguration = new BlurConfiguration();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return _systemBlurConfiguration.clone();
}
public static void setSystemBlurConfiguration(BlurConfiguration systemBlurConfiguration) {
TableContext._systemBlurConfiguration = systemBlurConfiguration;
}
@SuppressWarnings("unchecked")
public BlurIndex newInstanceBlurIndex(ShardContext shardContext, Directory dir, SharedMergeScheduler mergeScheduler,
ExecutorService searchExecutor, BlurIndexCloser indexCloser, BlurIndexWarmup indexWarmup) throws IOException {
String className = _blurConfiguration.get(BLUR_SHARD_BLURINDEX_CLASS, BlurIndexSimpleWriter.class.getName());
Class<? extends BlurIndex> clazz;
try {
clazz = (Class<? extends BlurIndex>) Class.forName(className);
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
Constructor<? extends BlurIndex> constructor = findConstructor(clazz);
try {
return constructor.newInstance(shardContext, dir, mergeScheduler, searchExecutor, indexCloser, indexWarmup);
} catch (InstantiationException e) {
throw new IOException(e);
} catch (IllegalAccessException e) {
throw new IOException(e);
} catch (IllegalArgumentException e) {
throw new IOException(e);
} catch (InvocationTargetException e) {
throw new IOException(e);
}
}
private Constructor<? extends BlurIndex> findConstructor(Class<? extends BlurIndex> clazz) throws IOException {
try {
return clazz.getConstructor(new Class[] { ShardContext.class, Directory.class, SharedMergeScheduler.class,
ExecutorService.class, BlurIndexCloser.class, BlurIndexWarmup.class });
} catch (NoSuchMethodException e) {
throw new IOException(e);
} catch (SecurityException e) {
throw new IOException(e);
}
}
public ReadInterceptor getReadInterceptor() {
return _readInterceptor;
}
}