/*
* Copyright 2002-2005 Uwyn bvba/sprl <info[remove] at uwyn dot com>
* Distributed under the terms of the GNU Lesser General Public
* License, v2.1 or later
*
* $Id: SearchTool.java 2004 2005-06-04 14:38:50Z gbevin $
*/
package com.uwyn.drone.tools;
import com.uwyn.rife.database.*;
import java.util.*;
import com.uwyn.drone.DroneConfig;
import com.uwyn.drone.core.Bot;
import com.uwyn.drone.core.Channel;
import com.uwyn.drone.protocol.ServerMessage;
import com.uwyn.drone.webui.elements.pub.Search;
import com.uwyn.rife.config.Config;
import com.uwyn.rife.database.queries.Query;
import com.uwyn.rife.database.queries.Select;
import com.uwyn.rife.rep.Participant;
import com.uwyn.rife.rep.Rep;
import com.uwyn.rife.scheduler.Scheduler;
import com.uwyn.rife.search.dam.indexqueuequerymanagers.IndexQueueFactory;
import com.uwyn.rife.search.dam.indexqueuequerymanagers.exceptions.IndexQueryManagerInstallationException;
import com.uwyn.rife.search.executors.IndexQueueAction;
import com.uwyn.rife.search.executors.IndexQueueExecutor;
import com.uwyn.rife.site.ValidationError;
import com.uwyn.rife.tools.ExceptionUtils;
import com.uwyn.rife.tools.InnerClassException;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.logging.Logger;
import org.apache.lucene.document.DateField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.RangeQuery;
public final class SearchTool extends com.uwyn.rife.search.tools.SearchTool
{
private static final SimpleDateFormat SORT_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss");
private static final int FETCH_SIZE = 1000;
private org.apache.lucene.search.Query mKeywordQuery = null;
static
{
SORT_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone(DroneConfig.getTimezone()));
}
public SearchTool()
{
super(Datasources.getRepInstance().getDatasource(Config.getRepInstance().getString("DRONE_DATASOURCE", Config.getRepInstance().getString("DATASOURCE"))));
}
public void reindexFull(final SearchToolListener listener)
{
System.out.println("Full reindex beginning...");
long fullIndexTime = System.currentTimeMillis();
Participant participant = Rep.getParticipant("scheduler");
Scheduler scheduler = null;
IndexQueueExecutor indexqueue = null;
if (participant != null)
{
scheduler = (Scheduler)participant.getObject("noop");
if (scheduler != null)
{
indexqueue = (IndexQueueExecutor)scheduler.getExecutor("IndexQueue");
if (indexqueue != null)
{
indexqueue.customQueueAction(new IndexQueueAction() {
public void performAction()
{
performReindex(listener);
}
});
}
}
}
System.out.println("Full reindex finished; took: "+(System.currentTimeMillis() - fullIndexTime)+"ms");
}
private void performReindex(final SearchToolListener listener)
{
Datasource d = Datasources.getRepInstance().getDatasource(Config.getRepInstance().getString("DRONE_DATASOURCE", Config.getRepInstance().getString("DATASOURCE")));
try
{
// try and create the db structure
IndexQueueFactory.getInstance(d).create();
}
catch (IndexQueryManagerInstallationException e1)
{
Logger
.getLogger("com.uwyn.drone.tools")
.warning("Tried to install IndexQueueDbStructure but it failed; " +
"probably already installed.");
}
final Select get_messages = new Select(d);
get_messages
.field("moment")
.field("botname")
.field("channel")
.field("servername")
.field("nickname")
.field("username")
.field("hostname")
.field("message")
.from("log");
final Select count_messages = new Select(d);
count_messages
.field("count(*)")
.from("log");
class Scope
{
int count = 0;
long start_time = System.currentTimeMillis();
String current_channel = null;
String current_servername = null;
String current_botname = null;
}
final HashMap index_writer_cache = new HashMap();
final Scope s = new Scope();
final DbQueryManager manager = new DbQueryManager(d);
final Calendar beginningOfDay = Calendar.getInstance();
final int total_count = manager.executeGetFirstInt(count_messages);
if (listener != null)
{
listener.indexStep(0, total_count);
}
manager.inTransaction(new DbTransactionUserWithoutResult() {
public void useTransactionWithoutResult() throws InnerClassException
{
manager.executeFetchAll(get_messages, new DbRowProcessor() {
public boolean processRow(ResultSet resultSet) throws SQLException
{
String servername = resultSet.getString("servername").toLowerCase();
String botname = resultSet.getString("botname").toLowerCase();
String channel = resultSet.getString("channel").substring(1).toLowerCase();
String key = getIndexName(botname, servername, channel);
Document doc = new Document();
Timestamp moment = resultSet.getTimestamp("moment");
IndexWriter writer = (IndexWriter)index_writer_cache.get(key);
if (writer == null)
{
writer = getIndexWriter(key, true);
index_writer_cache.put(key, writer);
}
beginningOfDay.setTime(moment);
beginningOfDay.setTimeZone(TimeZone.getTimeZone(DroneConfig.getTimezone()));
beginningOfDay.set(Calendar.HOUR, 0);
beginningOfDay.set(Calendar.MINUTE, 0);
beginningOfDay.set(Calendar.SECOND, 0);
beginningOfDay.set(Calendar.MILLISECOND, 0);
long dayInMillis = beginningOfDay.getTimeInMillis();
long timeInMillis = moment.getTime() - dayInMillis;
// need to store channel without the '#'
doc.add(new Field("moment", DateField.dateToString(resultSet.getTimestamp("moment")), true, true, true));
doc.add(new Field("momentDateSort", String.valueOf(dayInMillis), false, true, false));
doc.add(new Field("momentTimeSort", String.valueOf(timeInMillis), false, true, false));
doc.add(new Field("botname", botname, true, true, true));
doc.add(new Field("channel", channel, true, true, true));
doc.add(new Field("servername", servername, true, true, true));
doc.add(new Field("nickname", resultSet.getString("nickname"), true, true, true));
doc.add(new Field("username", resultSet.getString("username"), true, true, true));
doc.add(new Field("hostname", resultSet.getString("hostname"), true, true, true));
doc.add(new Field("message", resultSet.getString("message"), true, true, true));
try
{
writer.addDocument(doc, getAnalyzer());
}
catch (IOException e)
{
Logger
.getLogger("com.uwyn.drone.tools")
.severe(ExceptionUtils.getExceptionStackTrace(e));
cleanupIndexWriter(index_writer_cache.values(), e);
return false;
}
s.count++;
if (listener != null)
{
listener.indexStep(s.count, total_count);
}
if (0 == s.count % FETCH_SIZE)
{
long time = System.currentTimeMillis() - s.start_time;
System.out.println("rebuilded index up to "+s.count+"; took "+time+"ms");
s.start_time = System.currentTimeMillis();
}
return true;
}
}, new DbPreparedStatementHandler() {
public DbPreparedStatement getPreparedStatement(Query query, DbConnection connection)
{
DbPreparedStatement stmt = connection.getPreparedStatement(query);
stmt.setFetchSize(FETCH_SIZE);
return stmt;
}
});
}
});
cleanupIndexWriter(index_writer_cache.values(), null);
}
private void cleanupIndexWriter(Collection cache, Throwable e)
{
Iterator iter = cache.iterator();
while (iter.hasNext())
{
cleanupIndexWriter((IndexWriter)iter.next(), e);
}
}
private String getIndexName(String botname, String servername, String channel)
{
return botname+"-"+servername+"-"+channel;
}
public void indexServerMessage(Date moment, Bot bot, Channel channel, ServerMessage message)
{
String botname = bot.getName();
String channelname = channel.getName().substring(1);
String servername = channel.getServer().getServerName();
Document doc = new Document();
Calendar beginningOfDay = Calendar.getInstance();
beginningOfDay.setTime(moment);
beginningOfDay.setTimeZone(TimeZone.getTimeZone(DroneConfig.getTimezone()));
beginningOfDay.set(Calendar.HOUR, 0);
beginningOfDay.set(Calendar.MINUTE, 0);
beginningOfDay.set(Calendar.SECOND, 0);
beginningOfDay.set(Calendar.MILLISECOND, 0);
long dayInMillis = beginningOfDay.getTimeInMillis();
long timeInMillis = moment.getTime() - dayInMillis;
doc.add(new Field("moment", DateField.dateToString(moment), true, true, true));
doc.add(new Field("momentDateSort", String.valueOf(dayInMillis), false, true, false));
doc.add(new Field("momentTimeSort", String.valueOf(timeInMillis), false, true, false));
doc.add(new Field("botname", botname, true, true, true));
doc.add(new Field("channel", channelname, true, true, true));
doc.add(new Field("servername", servername, true, true, true));
doc.add(new Field("nickname", message.getPrefix().getNickName(), true, true, true));
doc.add(new Field("username", message.getPrefix().getUser(), true, true, true));
doc.add(new Field("hostname", message.getPrefix().getHost(), true, true, true));
doc.add(new Field("message", message.getTrailing(), true, true, true));
addDocumentToQueue(getIndexName(botname, servername, channelname), doc);
}
public BooleanQuery getSearchQuery(Search.SearchBean bean)
{
BooleanQuery query = new BooleanQuery();
if (bean.getKeyword() != null &&
!bean.getKeyword().equals(""))
{
try
{
mKeywordQuery = QueryParser.parse(bean.getKeyword(), "message", getAnalyzer());
query.add(mKeywordQuery, true, false);
}
catch (org.apache.lucene.queryParser.ParseException e)
{
bean.addValidationError(new ValidationError.INVALID("keyword"));
}
}
if (bean.getDate() != null &&
!bean.getDate().equals(""))
{
try
{
Date parsed = Search.INPUT_DATE_FORMAT_SHORT.parse(bean.getDate());
Calendar cal = Calendar.getInstance();
cal.setTimeZone(TimeZone.getTimeZone(DroneConfig.getTimezone()));
cal.setTime(parsed);
Term begin = new Term("moment", DateField.dateToString(cal.getTime()));
cal.set(Calendar.HOUR, 23);
cal.set(Calendar.MINUTE, 59);
cal.set(Calendar.SECOND, 59);
Term end = new Term("moment", DateField.dateToString(cal.getTime()));
query.add(new RangeQuery(begin, end, true), true, false);
}
catch (Exception e)
{
bean.addValidationError(new ValidationError.WRONGFORMAT("date"));
}
}
boolean beginDatePresent = (bean.getBeginDate() != null && !bean.getBeginDate().equals(""));
boolean endDatePresent = (bean.getEndDate() != null && !bean.getEndDate().equals(""));
if (beginDatePresent && endDatePresent)
{
String beginDateFormatted = null;
String endDateFormatted = null;
try
{
Date beginDateParsed = Search.INPUT_DATE_FORMAT_SHORT.parse(bean.getBeginDate());
Date endDateParsed = Search.INPUT_DATE_FORMAT_SHORT.parse(bean.getEndDate());
Calendar cal = Calendar.getInstance();
cal.setTime(endDateParsed);
cal.set(Calendar.HOUR, 23);
cal.set(Calendar.MINUTE, 59);
cal.set(Calendar.SECOND, 59);
beginDateFormatted = DateField.dateToString(beginDateParsed);
endDateFormatted = DateField.dateToString(cal.getTime());
}
catch (ParseException e)
{
bean.addValidationError(new ValidationError.WRONGFORMAT("beginDate"));
bean.addValidationError(new ValidationError.WRONGFORMAT("endDate"));
}
if (bean.getValidationErrors().size() == 0)
{
Term begin = new Term("moment", beginDateFormatted);
Term end = new Term("moment", endDateFormatted);
query.add(new RangeQuery(begin, end, true), true, false);
}
}
else if (!beginDatePresent && !endDatePresent)
{
// noop; means they didn't try a date range
}
else
{
bean.addValidationError(new ValidationError.INCOMPLETE("dateRange"));
}
if (bean.getUser() != null &&
!bean.getUser().equals(""))
{
try
{
query.add(QueryParser.parse(bean.getUser(), "nickname", getAnalyzer()), true, false);
}
catch (org.apache.lucene.queryParser.ParseException e)
{
bean.addValidationError(new ValidationError.INVALID("user"));
}
}
if (bean.getChannel() != null &&
!bean.getChannel().equals("") &&
!bean.getChannel().equals("All"))
{
try
{
int pos = bean.getChannel().indexOf("#");
String botname = bean.getChannel().substring(0, pos - 3);
String channel = bean.getChannel().substring(pos + 1);
query.add(QueryParser.parse(channel, "channel", getAnalyzer()), true, false);
query.add(QueryParser.parse(botname, "botname", getAnalyzer()), true, false);
}
catch (org.apache.lucene.queryParser.ParseException e)
{
bean.addValidationError(new ValidationError.INVALID("channel"));
}
}
if (bean.getValidationErrors().size() == 0)
{
return query;
}
else
{
return null;
}
}
public org.apache.lucene.search.Query getKeywordQuery()
{
return mKeywordQuery;
}
public void setKeywordQuery(org.apache.lucene.search.Query keywordQuery)
{
mKeywordQuery = keywordQuery;
}
}