Package org.jivesoftware.openfire.archive

Source Code of org.jivesoftware.openfire.archive.ArchiveIndexer$RebuildFuture

/**
* $Revision$
* $Date$
*
* Copyright (C) 2008 Jive Software. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.jivesoftware.openfire.archive;

import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.Writer;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimerTask;
import java.util.TreeSet;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexModifier;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.dom4j.DocumentFactory;
import org.dom4j.io.OutputFormat;
import org.dom4j.io.XMLWriter;
import org.jivesoftware.database.DbConnectionManager;
import org.jivesoftware.openfire.reporting.util.TaskEngine;
import org.jivesoftware.util.JiveConstants;
import org.jivesoftware.util.JiveGlobals;
import org.jivesoftware.util.XMLProperties;
import org.picocontainer.Startable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xmpp.packet.JID;

/**
* Indexes archived conversations. If conversation archiving is not enabled,
* this class does nothing. The search index is maintained in the <tt>monitoring/search</tt>
* directory of the Openfire home directory. It's automatically updated with the latest
* conversation content as long as conversation archiving is enabled. The index update
* interval is controllec by the Jive property "conversation.search.updateInterval" and
* the default value is 15 minutes.
*
* @see ArchiveSearcher
* @author Matt Tucker
*/
public class ArchiveIndexer implements Startable {

  private static final Logger Log = LoggerFactory.getLogger(ArchiveIndexer.class);

    private static final String ALL_CONVERSATIONS =
            "SELECT conversationID, isExternal FROM ofConversation";
    private static final String NEW_CONVERSATIONS =
            "SELECT DISTINCT conversationID FROM ofMessageArchive WHERE sentDate > ?";
    private static final String CONVERSATION_METADATA =
            "SELECT isExternal FROM ofConversation WHERE conversationID=?";
    private static final String CONVERSATION_MESSAGES =
            "SELECT conversationID, sentDate, fromJID, toJID, body FROM ofMessageArchive " +
            "WHERE conversationID IN ? ORDER BY conversationID";

    private File searchDir;
    private TaskEngine taskEngine;
    private ConversationManager conversationManager;
    private XMLProperties indexProperties;
    private Directory directory;
    private IndexSearcher searcher;
    private Lock writerLock;
    private boolean stopped = false;

    private boolean rebuildInProgress = false;
    private RebuildFuture rebuildFuture;

    private long lastModified = 0;

    private TimerTask indexUpdater;

    /**
     * Constructs a new archive indexer.
     *
     * @param conversationManager a ConversationManager instance.
     * @param taskEngine a task engine instance.
     */
    public ArchiveIndexer(ConversationManager conversationManager, TaskEngine taskEngine) {
        this.conversationManager = conversationManager;
        this.taskEngine = taskEngine;
    }

    public void start() {
        searchDir = new File(JiveGlobals.getHomeDirectory() +
                    File.separator + "monitoring" + File.separator + "search");
        if (!searchDir.exists()) {
            searchDir.mkdirs();
        }
        boolean indexCreated = false;
        try {
            loadPropertiesFile(searchDir);
            // If the index already exists, use it.
            if (IndexReader.indexExists(searchDir)) {
                directory = FSDirectory.getDirectory(searchDir, false);
            }
            // Otherwise, create a new index.
            else {
                directory = FSDirectory.getDirectory(searchDir, true);
                indexCreated = true;
            }
        }
        catch (IOException ioe) {
            Log.error(ioe.getMessage(), ioe);
        }
        writerLock = new ReentrantLock(true);

        // Force the directory unlocked if it's locked (due to non-clean app shut-down,
        // for example).
        try {
            if (IndexReader.isLocked(directory)) {
                Log.warn("Archiving search index was locked, probably due to non-clean " +
                        "application shutdown.");
                IndexReader.unlock(directory);
            }
        }
        catch (IOException ioe) {
            Log.error(ioe.getMessage(), ioe);
        }

        String modified = indexProperties.getProperty("lastModified");
        if (modified != null) {
            try {
                lastModified = Long.parseLong(modified);
            }
            catch (NumberFormatException nfe) {
                // Ignore.
            }
        }
        // If the index has never been updated, build it from scratch.
        if (lastModified == 0 || indexCreated) {
            taskEngine.submit(new Runnable() {
                public void run() {
                    rebuildIndex();
                }
            });
        }

        indexUpdater = new TimerTask() {
            @Override
      public void run() {
                updateIndex();
            }
        };
        int updateInterval = JiveGlobals.getIntProperty("conversation.search.updateInterval", 15);
        taskEngine.scheduleAtFixedRate(indexUpdater, JiveConstants.MINUTE * 5,
                JiveConstants.MINUTE * updateInterval);
    }

    public void stop() {
        stopped = true;
        indexUpdater.cancel();
        if (searcher != null) {
            try {
                searcher.close();
            }
            catch (Exception e) {
                Log.error(e.getMessage(), e);
            }
            searcher = null;
        }
        try {
            directory.close();
        }
        catch (Exception e) {
            Log.error(e.getMessage(), e);
        }
        directory = null;
        indexProperties = null;
        conversationManager = null;
        searchDir = null;
        rebuildFuture = null;
    }

    /**
     * Returns the total size of the search index (in bytes).
     *
     * @return the total size of the search index (in bytes).
     */
    public long getIndexSize() {
        File [] files = searchDir.listFiles(new FilenameFilter() {
            public boolean accept(File dir, String name) {
                // Ignore the index properties file since it's not part of the index.
                return !name.equals("indexprops.xml");
            }
        });
        if (files == null) {
            // Search folder does not exist so size of index is 0
            return 0;
        }
        long size = 0;
        for (File file : files) {
            size += file.length();
        }
        return size;
    }

    /**
     * Updates the search index with all new conversation data since the last index update.
     */
    public void updateIndex() {
        // Immediately return if the service has been stopped.
        if (stopped) {
            return;
        }
        // Do nothing if archiving is disabled.
        if (!conversationManager.isArchivingEnabled()) {
            return;
        }
        // If we're currently rebuilding the index, return.
        if (rebuildInProgress) {
            return;
        }
        writerLock.lock();
        IndexModifier writer = null;
        try {
            writer = new IndexModifier(directory, new StandardAnalyzer(), false);
            List<Long> conversationIDs = new ArrayList<Long>();
            Connection con = null;
            PreparedStatement pstmt = null;
            ResultSet rs = null;
            try {
                con = DbConnectionManager.getConnection();
                pstmt = con.prepareStatement(NEW_CONVERSATIONS);
                pstmt.setLong(1, lastModified);
                rs = pstmt.executeQuery();
                while (rs.next()) {
                    conversationIDs.add(rs.getLong(1));
                }
            }
            catch (SQLException sqle) {
                Log.error(sqle.getMessage(), sqle);
            }
            finally {
                DbConnectionManager.closeConnection(rs, pstmt, con);
            }

            // Delete any conversations found -- they may have already been indexed, but
            // updated since then.
            for (long conversationID : conversationIDs) {
                writer.deleteDocuments(new Term("conversationID", Long.toString(conversationID)));
            }

            // Load meta-data for each conversation.
            Map<Long, Boolean> externalMetaData = new HashMap<Long, Boolean>();
            for (long conversationID : conversationIDs) {
                try {
                    con = DbConnectionManager.getConnection();
                    pstmt = con.prepareStatement(CONVERSATION_METADATA);
                    pstmt.setLong(1, conversationID);
                    rs = pstmt.executeQuery();
                    while (rs.next()) {
                        externalMetaData.put(conversationID, rs.getInt(1) == 1);
                    }
                }
                catch (SQLException sqle) {
                    Log.error(sqle.getMessage(), sqle);
                }
                finally {
                    DbConnectionManager.closeConnection(rs, pstmt, con);
                }
            }

            // Now index all the new conversations.
            long newestDate = indexConversations(conversationIDs, externalMetaData, writer, false);

            writer.optimize();

            // Done indexing so store a last modified date.
            if (newestDate != -1) {
                lastModified = newestDate;
                indexProperties.setProperty("lastModified", Long.toString(lastModified));
            }
        }
        catch (IOException ioe) {
            Log.error(ioe.getMessage(), ioe);
        }
        finally {
            if (writer != null) {
                try {
                    writer.close();
                }
                catch (Exception e) {
                    Log.error(e.getMessage(), e);
                }
            }
            writerLock.unlock();
        }
    }

    /**
     * Rebuilds the search index with all archived conversation data. This method returns
     * a Future that represents the status of the index rebuild process (also available
     * via {@link #getIndexRebuildProgress()}). The integer value
     * (values 0 through 100) represents the percentage of work done. If message archiving
     * is disabled, this method will return <tt>null</tt>.
     *
     * @return a Future to indicate the status of rebuilding the index or <tt>null</tt> if
     *      rebuilding the index is not possible.
     */
    public synchronized Future<Integer> rebuildIndex() {
        // Immediately return if the service has been stopped.
        if (stopped) {
            return null;
        }
        // If a rebuild is already happening, return.
        if (rebuildInProgress) {
            return null;
        }
        rebuildInProgress = true;
        // Do nothing if archiving is disabled.
        if (!conversationManager.isArchivingEnabled()) {
            return null;
        }

        // Create a future to track the index rebuild progress.
        rebuildFuture = new RebuildFuture();

        // Create a runnable that will perform the actual rebuild work.
        Runnable rebuildTask = new Runnable() {

            public void run() {
                List<Long> conversationIDs = new ArrayList<Long>();
                Map<Long, Boolean> externalMetaData = new HashMap<Long, Boolean>();
                Connection con = null;
                PreparedStatement pstmt = null;
                ResultSet rs = null;
                try {
                    con = DbConnectionManager.getConnection();
                    pstmt = con.prepareStatement(ALL_CONVERSATIONS);
                    rs = pstmt.executeQuery();
                    while (rs.next()) {
                        long conversationID = rs.getLong(1);
                        conversationIDs.add(conversationID);
                        externalMetaData.put(conversationID, rs.getInt(2) == 1);
                    }
                }
                catch (SQLException sqle) {
                    Log.error(sqle.getMessage(), sqle);
                }
                finally {
                    DbConnectionManager.closeConnection(rs, pstmt, con);
                }

                if (!conversationIDs.isEmpty()) {
                    // Index the conversations.
                    writerLock.lock();
                    IndexModifier writer = null;
                    try {
                        writer = new IndexModifier(directory, new StandardAnalyzer(), true);
                        long newestDate = indexConversations(conversationIDs, externalMetaData,
                                writer, true);
                        writer.optimize();

                        // Done indexing so store a last modified date.
                        if (newestDate != -1) {
                            lastModified = newestDate;
                            indexProperties.setProperty("lastModified", Long.toString(lastModified));
                        }
                    }
                    catch (IOException ioe) {
                        Log.error(ioe.getMessage(), ioe);
                    }
                    finally {
                        if (writer != null) {
                            try {
                                writer.close();
                            }
                            catch (Exception e) {
                                Log.error(e.getMessage(), e);
                            }
                        }
                        writerLock.unlock();
                    }
                }
                // Done rebuilding the index, so reset state.
                rebuildFuture = null;
                rebuildInProgress = false;
            }
        };
        taskEngine.submit(rebuildTask);

        return rebuildFuture;
    }

    /**
     * Returns a Future representing the status of an index rebuild operation. This is the
     * same Future returned by the {@link #rebuildIndex()} method; access is provided via
     * this method as a convenience. If the index is not currently being rebuilt, this method
     * will return <tt>null</tt>.
     *
     * @return a Future that represents the index rebuild status or <tt>null</tt> if the
     *      index is not being rebuilt.
     */
    public Future<Integer> getIndexRebuildProgress() {
        return rebuildFuture;
    }

    /**
     * Indexes a set of conversations. Each conversation is stored as a single Lucene document
     * by appending message bodies together. The date of the newest message indexed is
     * returned, or -1 if no conversations are indexed.
     *
     * @param conversationIDs the ID's of the conversations to index.
     * @param externalMetaData meta-data about whether each conversation involves a participant on
     *      an external server.
     * @param writer an IndexModifier to add the documents to.
     * @param indexRebuild true if this is an index rebuild operation.
     * @return the date of the newest message archived.
     */
    private long indexConversations(List<Long> conversationIDs, Map<Long, Boolean> externalMetaData,
            IndexModifier writer, boolean indexRebuild) throws IOException
    {
        if (conversationIDs.isEmpty()) {
            return -1;
        }

        // Keep track of how many conversations we index for index rebuild stats.
        int indexedConversations = 0;

        long newestDate = -1;
        // Index 250 items at a time.
        final int OP_SIZE = 250;
        int n = ((conversationIDs.size() - 1) / OP_SIZE);
        if (n == 0) {
            n = 1;
        }
        for (int i = 0; i < n; i++) {
            StringBuilder inSQL = new StringBuilder();
            inSQL.append(" (");
            int start = i * OP_SIZE;
            int end = (start + OP_SIZE > conversationIDs.size()) ? conversationIDs.size() : start + OP_SIZE;
            if (end > conversationIDs.size()) {
                end = conversationIDs.size();
            }
            inSQL.append(conversationIDs.get(start));
            for (int j = start + 1; j < end; j++) {
                inSQL.append(", ").append(conversationIDs.get(j));
            }
            inSQL.append(")");
            // Get the messages.
            Connection con = null;
            PreparedStatement pstmt = null;
            ResultSet rs = null;
            try {
                con = DbConnectionManager.getConnection();
                pstmt = con.prepareStatement(CONVERSATION_MESSAGES.replaceAll("\\?", inSQL.toString()));
                rs = pstmt.executeQuery();
                long conversationID = -1;
                long date = -1;
                Set<String> jids = null;
                StringBuilder text = null;
                // Loop through each message. Each conversation is a single document. So, as
                // we find each conversation we save off the last chunk of content as a document.
                while (rs.next()) {
                    long id = rs.getLong(1);
                    if (id != conversationID) {
                        if (conversationID != -1) {
                            // Index the previously defined doc.
                            boolean external = externalMetaData.get(conversationID);
                            indexDocument(writer, conversationID, external, date, jids, text.toString());
                        }
                        // Reset the variables to index the next conversation.
                        conversationID = id;
                        date = rs.getLong(2);
                        jids = new TreeSet<String>();
                        // Get the JID's. Each JID may be stored in full format. We convert
                        // to bare JID for indexing so that searching is possible.
                        jids.add(new JID(rs.getString(3)).toBareJID());
                        jids.add(new JID(rs.getString(4)).toBareJID());
                        text = new StringBuilder();
                    }
                    // Make sure that we record the earliest date of the conversation start
                    // for consistency.
                    long msgDate = rs.getLong(2);
                    if (msgDate < date) {
                        date = msgDate;
                    }
                    // See if this is the newest message found so far.
                    if (msgDate > newestDate) {
                        newestDate = msgDate;
                    }
                    // Add the body of the current message to the buffer.
                    text.append(DbConnectionManager.getLargeTextField(rs, 5)).append("\n");
                }
                // Finally, index the last document found.
                if (conversationID != -1) {
                    // Index the previously defined doc.
                    boolean external = externalMetaData.get(conversationID);
                    indexDocument(writer, conversationID, external, date, jids, text.toString());
                }
                // If this is an index rebuild, we need to track the percentage done.
                if (indexRebuild) {
                    indexedConversations++;
                    rebuildFuture.setPercentageDone(indexedConversations/conversationIDs.size());
                }
            }
            catch (SQLException sqle) {
                Log.error(sqle.getMessage(), sqle);
            }
            finally {
                DbConnectionManager.closeConnection(rs, pstmt, con);
            }
        }
        return newestDate;
    }

    /**
     * Indexes a single conversation.
     *
     * @param writer the index modifier.
     * @param conversationID the ID of the conversation to index.
     * @param external true if the conversation has a participant from an external server.
     * @param date the date the conversation was started.
     * @param jids the JIDs of the users in the conversation.
     * @param text the full text of the conversation.
     * @throws IOException if an IOException occurs.
     */
    private void indexDocument(IndexModifier writer, long conversationID, boolean external,
            long date, Set<String> jids, String text) throws IOException
    {
        Document document = new Document();
        document.add(new Field("conversationID", String.valueOf(conversationID),
                Field.Store.YES, Field.Index.UN_TOKENIZED));
        document.add(new Field("external", String.valueOf(external),
                Field.Store.YES, Field.Index.UN_TOKENIZED));
        document.add(new Field("date", DateTools.timeToString(date, DateTools.Resolution.DAY),
                Field.Store.YES, Field.Index.UN_TOKENIZED));
        for (String jid : jids) {
            document.add(new Field("jid", jid, Field.Store.YES, Field.Index.TOKENIZED));
        }
        document.add(new Field("text", text, Field.Store.NO, Field.Index.TOKENIZED));
        writer.addDocument(document);
    }

    /**
     * Returns an IndexSearcher to search the archive index.
     *
     * @return an IndexSearcher.
     * @throws IOException if an IOException occurs.
     */
    synchronized IndexSearcher getSearcher() throws IOException {
        // If the searcher hasn't been instantiated, create it.
        if (searcher == null) {
            searcher = new IndexSearcher(directory);
        }
        // See if the searcher needs to be closed due to the index being updated.
        else if (!searcher.getIndexReader().isCurrent()) {
            searcher.close();
            searcher = new IndexSearcher(directory);
        }
        return searcher;
    }

    /**
     * Loads a property manager for search properties if it isn't already
     * loaded. If an XML file for the search properties isn't already
     * created, it will attempt to make a file with default values.
     */
    private void loadPropertiesFile(File searchDir) throws IOException {
        File indexPropertiesFile = new File(searchDir, "indexprops.xml");

        // Make sure the file actually exists. If it doesn't, a new file
        // will be created.
        // If it doesn't exists we have to create it.
        if (!indexPropertiesFile.exists()) {
            org.dom4j.Document doc = DocumentFactory.getInstance().createDocument(
                    DocumentFactory.getInstance().createElement("search"));
            // Now, write out to the file.
            Writer out = null;
            try {
                // Use JDOM's XMLOutputter to do the writing and formatting.
                out = new FileWriter(indexPropertiesFile);
                XMLWriter outputter = new XMLWriter(out, OutputFormat.createPrettyPrint());
                outputter.write(doc);
                outputter.flush();
            }
            catch (Exception e) {
                Log.error(e.getMessage(), e);
            }
            finally {
                try {
                    if (out != null) {
                        out.close();
                    }
                }
                catch (Exception e) {
                    // Ignore.
                }
            }
        }
        indexProperties = new XMLProperties(indexPropertiesFile);
    }

    /**
     * A Future class to track the status of index rebuilding.
     */
    private class RebuildFuture implements Future<Integer> {

        private int percentageDone = 0;

        public boolean cancel(boolean mayInterruptIfRunning) {
            // Don't allow cancels.
            return false;
        }

        public boolean isCancelled() {
            return false;
        }

        public boolean isDone() {
            return percentageDone == 100;
        }

        public Integer get() throws InterruptedException, ExecutionException {
            return percentageDone;
        }

        public Integer get(long timeout, TimeUnit unit) throws InterruptedException,
                ExecutionException, TimeoutException
        {
            return percentageDone;
        }

        /**
         * Sets the percentage done.
         *
         * @param percentageDone the percentage done.
         */
        public void setPercentageDone(int percentageDone) {
            if (percentageDone < 0 || percentageDone > 100) {
                throw new IllegalArgumentException("Invalid value: " + percentageDone);
            }
            this.percentageDone = percentageDone;
        }
    }
}
TOP

Related Classes of org.jivesoftware.openfire.archive.ArchiveIndexer$RebuildFuture

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.