/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* $Id: IndexManager.java 517983 2007-03-14 03:21:58Z vgritsenko $
*/
package org.apache.xindice.core.indexer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.xindice.Stopwatch;
import org.apache.xindice.core.Collection;
import org.apache.xindice.core.DBException;
import org.apache.xindice.core.data.Key;
import org.apache.xindice.core.data.RecordSet;
import org.apache.xindice.util.Configuration;
import org.apache.xindice.util.ConfigurationCallback;
import org.apache.xindice.util.ObjectStack;
import org.apache.xindice.util.SimpleConfigurable;
import org.apache.xindice.util.XindiceException;
import org.apache.xindice.xml.SymbolTable;
import org.apache.xindice.xml.sax.CompressionHandler;
import org.apache.xindice.xml.sax.SAXEventGenerator;
import org.w3c.dom.Document;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.Locator;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TimerTask;
import java.util.WeakHashMap;
import java.util.Timer;
/**
* IndexManager is a class that manages Indexes. Good description, eh?
* I should win a Pulitzer Prize for that one.
*
* @version $Revision: 517983 $, $Date: 2007-03-13 23:21:58 -0400 (Tue, 13 Mar 2007) $
*/
public final class IndexManager extends SimpleConfigurable {
private static final Log log = LogFactory.getLog(IndexManager.class);
private static final String[] EMPTY_STRINGS = new String[0];
private static final IndexerInfo[] EMPTY_INDEXERS = new IndexerInfo[0];
private static final String INDEX = "index";
private static final String NAME = "name";
private static final String CLASS = "class";
private static final int STATUS_READY = 0;
private static final int STATUS_BUSY = 1;
private static final int ACTION_CREATE = 0;
private static final int ACTION_UPDATE = 1;
private static final int ACTION_DELETE = 2;
private Map patternMap = new HashMap(); // IndexPattern to IndexerInfo
private Map indexes = new HashMap(); // String to IndexerInfo
private Map bestIndexers = new HashMap(); // String to Map of IndexPattern to Indexer
private IndexerInfo[] idxList = EMPTY_INDEXERS;
private Collection collection;
private Timer timer;
private SymbolTable symbols;
private final List newIndexers = new ArrayList(); // of IndexerInfo
private int taskCount; // counter of scheduled tasks
private final Object lock = new Object(); // lock object for manipulating taskCounter
/**
* Create IndexManager for a given collection
*
* @param collection Collection for this IndexManager
* @throws DBException if can't get collection's symbols
*/
public IndexManager(Collection collection, Timer timer) throws DBException {
this.collection = collection;
this.symbols = collection.getSymbols();
this.timer = timer;
}
/**
* Configure index manager, register all indexes specified in the configuration
*
* @param config IndexManager configuration
*/
public void setConfig(Configuration config) throws XindiceException {
super.setConfig(config);
config.processChildren(INDEX, new ConfigurationCallback() {
public void process(Configuration cfg) {
String className = cfg.getAttribute(CLASS);
try {
register(Class.forName(className), cfg);
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn("Failed to register index with class '" + className + "' for collection '" + collection.getCanonicalName() + "'", e);
}
}
}
});
}
/**
* list returns a list of the Indexers that this IndexerManager has
* registered.
*
* @return An array containing the Indexer names
*/
public synchronized String[] list() {
return (String[]) indexes.keySet().toArray(EMPTY_STRINGS);
}
/**
* drop physically removes the specified Indexer and any
* associated system resources that the Indexer uses.
*
* @param name The Indexer to drop
* @return Whether or not the Indexer was dropped
*/
public synchronized boolean drop(final String name) {
// Get indexer
Indexer idx = get(name);
// Unregister and remove from coniguration
unregister(name);
config.processChildren(INDEX, new ConfigurationCallback() {
public void process(Configuration cfg) {
try {
if (cfg.getAttribute(NAME).equals(name)) {
cfg.delete();
}
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn("ignored exception", e);
}
}
}
});
// Drop indexer
boolean res = false;
try {
res = idx.drop();
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn("ignored exception", e);
}
}
return res;
}
/**
* Drop all indexers
*/
public synchronized void drop() {
// Get a copy of idxList
IndexerInfo[] idx = idxList;
// Drop indexes
for (int i = 0; i < idx.length; i++) {
drop(idx[i].name);
}
}
/**
* create creates a new Indexer object and any associated
* system resources that the Indexer will need.
*
* @param cfg The Indexer's configuration
* @return The Indexer that was created
*/
public synchronized Indexer create(Configuration cfg) throws DBException {
String name = cfg.getAttribute(NAME);
try {
// Check for duplicates
Configuration[] cfgs = config.getChildren();
for (int i = 0; i < cfgs.length; i++) {
if (cfgs[i].getAttribute(NAME).equals(name)) {
throw new DuplicateIndexException("Duplicate Index '" + name + "' in collection '" + collection.getCanonicalName() + "'");
}
}
String className = cfg.getAttribute(CLASS);
Indexer idx = register(Class.forName(className), cfg);
config.add(cfg);
return idx;
} catch (DBException e) {
throw e;
} catch (Exception e) {
throw new CannotCreateException("Cannot create index '" + name + "' in " + collection.getCanonicalName(), e);
}
}
/**
* Closes all indexers managed by this index manager.
*/
public synchronized void close() {
// wait for all scheduled tasks to finish
synchronized(lock) {
while (taskCount > 0) {
try {
lock.wait();
} catch (InterruptedException e) {
// ignore
}
}
}
// close all indexers
for (int i = 0; i < idxList.length; i++) {
try {
idxList[i].indexer.close();
} catch (DBException e) {
if (log.isWarnEnabled()) {
log.warn("Failed to close indexer " + idxList[i].name + " on collection " + collection.getCanonicalName(), e);
}
}
}
}
public synchronized Indexer register(Class c, Configuration cfg) throws DBException {
String name = null;
try {
Indexer idx = (Indexer) c.newInstance();
initialize(idx, cfg);
name = idx.getName();
if (name == null || name.trim().equals("")) {
throw new CannotCreateException("No name specified");
}
IndexPattern pattern = new IndexPattern(symbols, idx.getPattern(), null);
String style = idx.getIndexStyle();
IndexerInfo info = new IndexerInfo(name, style, pattern, idx);
if (!idx.exists()) {
idx.create();
idx.open();
info.status = STATUS_BUSY;
synchronized (newIndexers) {
newIndexers.add(info);
}
synchronized (lock) {
taskCount++;
try {
// Schedule new task
timer.schedule(new PopulateIndexersTimerTask(), 0);
} catch (RuntimeException e) {
// If failed to schedule the task, decrease the counter.
taskCount--;
throw e;
} catch (Error e) {
// If failed to schedule the task, decrease the counter.
taskCount--;
throw e;
}
if (log.isDebugEnabled()) {
log.debug("Scheduled new task, count is " + taskCount);
}
}
} else {
info.status = STATUS_READY;
idx.open();
}
indexes.put(name, info);
patternMap.put(pattern, info);
Map tbl = (Map) bestIndexers.get(style);
if (tbl != null) {
tbl.clear();
}
idxList = (IndexerInfo[]) indexes.values().toArray(EMPTY_INDEXERS);
return idx;
} catch (DBException e) {
throw e;
} catch (Exception e) {
throw new CannotCreateException("Cannot create Index '" + name + "' in " + collection.getCanonicalName(), e);
}
}
public synchronized void unregister(String name) {
IndexerInfo idx = (IndexerInfo) indexes.remove(name);
String style = idx.style;
patternMap.remove(idx.pattern);
Map tbl = (Map) bestIndexers.get(style);
if (tbl != null) {
tbl.clear();
}
idxList = (IndexerInfo[]) indexes.values().toArray(EMPTY_INDEXERS);
}
private void initialize(Indexer idx, Configuration cfg) throws XindiceException {
idx.setCollection(collection);
idx.setConfig(cfg);
}
private void populateNewIndexers() throws DBException {
IndexerInfo[] list;
synchronized (newIndexers) {
list = (IndexerInfo[]) newIndexers.toArray(EMPTY_INDEXERS);
newIndexers.clear();
}
if (list.length > 0) {
if (log.isTraceEnabled()) {
for (int i = 0; i < list.length; i++) {
log.trace("Index Creation: " + list[i].indexer.getName());
}
}
Stopwatch sw = new Stopwatch("Populated Indexes", true);
RecordSet rs = collection.getFiler().getRecordSet();
while (rs.hasMoreRecords()) {
// Read only key, we don't need filer-level value
Key key = rs.getNextKey();
Object value = collection.getEntry(key);
if (value instanceof Document) {
try {
new SAXHandler(key, (Document)value, ACTION_CREATE, list);
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn("Failed to index document " + key, e);
}
}
}
}
for (int i = 0; i < list.length; i++) {
try {
list[i].indexer.flush();
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn("ignored exception", e);
}
}
list[i].status = STATUS_READY;
}
sw.stop();
if (log.isDebugEnabled()) {
for (int i = 0; i < list.length; i++) {
log.debug("Index Complete: " + list[i].indexer.getName());
}
log.debug(sw.toString());
}
}
}
/**
* get retrieves an Indexer by name.
*
* @param name The Indexer name
* @return The Indexer
*/
public synchronized Indexer get(String name) {
IndexerInfo info = (IndexerInfo) indexes.get(name);
return info != null ? info.indexer : null;
}
/**
* getBestIndexer retrieves the best Indexer to use for the specified
* IndexPattern.
*
* @param style The Indexer Style (ex: Node, Value)
* @param pattern The IndexPattern to use
* @return The best Indexer (or null)
*/
public Indexer getBestIndexer(String style, IndexPattern pattern) {
Map tbl = (Map) bestIndexers.get(style);
if (tbl == null) {
tbl = new WeakHashMap(); // FIXME: Review usage of WeakHashMap
bestIndexers.put(style, tbl);
}
Indexer idx = (Indexer) tbl.get(pattern);
if (idx == null) {
int highScore = 0;
Iterator i = indexes.values().iterator();
while (i.hasNext()) {
IndexerInfo info = (IndexerInfo) i.next();
if (info.status != STATUS_READY || !info.indexer.getIndexStyle().equals(style)) {
continue;
}
int score = pattern.getMatchLevel(info.pattern);
if (score > highScore) {
idx = info.indexer;
highScore = score;
}
}
tbl.put(pattern, idx);
}
return idx;
}
public void addDocument(Key key, Document doc) {
if (idxList.length > 0) {
new SAXHandler(key, doc, ACTION_CREATE);
for (int i = 0; i < idxList.length; i++) {
try {
idxList[i].indexer.flush();
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn("ignored exception", e);
}
}
}
}
}
public void removeDocument(Key key, Document doc) {
if (idxList.length > 0) {
new SAXHandler(key, doc, ACTION_DELETE);
for (int i = 0; i < idxList.length; i++) {
try {
idxList[i].indexer.flush();
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn("ignored exception", e);
}
}
}
}
}
/**
* IndexerInfo
*/
private class IndexerInfo {
public String name;
public String style;
public IndexPattern pattern;
public Indexer indexer;
public int status;
public IndexerInfo(String name, String style, IndexPattern pattern, Indexer indexer) {
this.name = name;
this.style = style;
this.pattern = pattern;
this.indexer = indexer;
}
}
/**
* SAXHandler actually performs the work of adding and removing Indexer
* entries.
*/
private class SAXHandler implements ContentHandler, CompressionHandler {
private ObjectStack stack = new ObjectStack();
private IndexerInfo[] list;
public Key key;
public Document doc;
public int action;
public StackInfo info; // Current State
public SAXHandler(Key key, Document doc, int action, IndexerInfo[] list) {
this.list = list;
this.key = key;
this.doc = doc;
this.action = action;
try {
SAXEventGenerator events = new SAXEventGenerator(symbols, doc);
events.setContentHandler(this);
events.setProperty(HANDLER, this);
events.start();
if (action == ACTION_CREATE || action == ACTION_UPDATE) {
collection.flushSymbolTable();
}
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn("ignored exception", e);
}
}
}
public SAXHandler(Key key, Document doc, int action) {
this(key, doc, action, idxList);
}
// These are all NO-OPs
public void setDocumentLocator(Locator locator) {
}
public void startDocument() {
}
public void endDocument() {
}
public void startPrefixMapping(String prefix, String uri) {
}
public void endPrefixMapping(String prefix) {
}
public void ignorableWhitespace(char ch[], int start, int length) {
}
public void processingInstruction(String target, String data) {
}
public void skippedEntity(String name) {
}
public void symbols(SymbolTable symbols) {
}
public void dataBytes(byte[] data) {
}
public void processEntry(IndexPattern pattern, String value, int pos, int len) {
for (int i = 0; i < list.length; i++) {
if (pattern.getMatchLevel(list[i].pattern) > 0) {
try {
switch (action) {
case ACTION_CREATE:
case ACTION_UPDATE:
list[i].indexer.add(value, key, pos, len, pattern.getElementID(), pattern.getAttributeID());
break;
case ACTION_DELETE:
list[i].indexer.remove(value, key, pos, len, pattern.getElementID(), pattern.getAttributeID());
break;
default:
if (log.isWarnEnabled()) {
log.warn("invalid action : " + action);
}
}
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn("ignored exception", e);
}
}
}
}
}
public void startElement(String namespaceURI, String localName, String qName, Attributes atts) {
// Modify the stack info to normalize the symbolID
if (namespaceURI != null && namespaceURI.length() > 0) {
String elemNSID = SymbolTable.getNormalizedLocalName(localName, namespaceURI);
info.symbolID = symbols.getSymbol(elemNSID, namespaceURI, true);
}
int size = atts.getLength();
for (int i = 0; i < size; i++) {
String nsURI = atts.getURI(i);
if (nsURI != null && nsURI.length() > 0) {
String attrNSID = "ns" + Integer.toString(nsURI.hashCode()) + ":" + atts.getLocalName(i);
short id = symbols.getSymbol(attrNSID, nsURI, true);
processEntry(new IndexPattern(symbols, info.symbolID, id), atts.getValue(i), info.pos, info.len);
} else {
short id = symbols.getSymbol(atts.getQName(i));
processEntry(new IndexPattern(symbols, info.symbolID, id), atts.getValue(i), info.pos, info.len);
}
}
}
public void endElement(String namespaceURI, String localName, String qName) {
if (info.sb != null) {
processEntry(new IndexPattern(symbols, info.symbolID), info.sb.toString(), info.pos, info.len);
}
info = (StackInfo) stack.pop();
}
public void characters(char ch[], int start, int length) {
String val = new String(ch).trim();
if (info.sb == null) {
info.sb = new StringBuffer(ch.length);
} else if (info.sb.length() > 0) {
info.sb.append(' ');
}
info.sb.append(val);
}
public void symbolID(short symbolID) {
if (info != null) {
stack.push(info);
}
info = new StackInfo(symbolID);
}
public void dataLocation(int pos, int len) {
info.pos = pos;
info.len = len;
}
}
/**
* StackInfo
*/
private class StackInfo {
public short symbolID;
public StringBuffer sb = null;
public int pos = -1;
public int len = -1;
public StackInfo(short symbolID) {
this.symbolID = symbolID;
}
}
private class PopulateIndexersTimerTask extends TimerTask {
public void run() {
try {
populateNewIndexers();
} catch (DBException e) {
if (log.isWarnEnabled()) {
log.warn("ignored exception", e);
}
} finally {
synchronized (lock) {
taskCount--;
if (log.isDebugEnabled()) {
log.debug("Task completed, count is " + taskCount);
}
if (taskCount == 0) {
lock.notifyAll();
}
}
}
}
}
}