package net.fortytwo.twitlogic.rdfagents;
import net.fortytwo.rdfagents.RDFAgents;
import net.fortytwo.rdfagents.messaging.Commitment;
import net.fortytwo.rdfagents.messaging.LocalFailure;
import net.fortytwo.rdfagents.messaging.subscribe.PubsubProvider;
import net.fortytwo.rdfagents.model.AgentId;
import net.fortytwo.rdfagents.model.Dataset;
import net.fortytwo.rdfagents.model.RDFAgent;
import net.fortytwo.twitlogic.TweetFilterCriterion;
import net.fortytwo.twitlogic.TwitLogic;
import net.fortytwo.twitlogic.flow.Filter;
import net.fortytwo.twitlogic.flow.Handler;
import net.fortytwo.twitlogic.logging.TweetPersistedLogger;
import net.fortytwo.twitlogic.logging.TweetReceivedLogger;
import net.fortytwo.twitlogic.model.Tweet;
import net.fortytwo.twitlogic.model.User;
import net.fortytwo.twitlogic.persistence.TweetDeleter;
import net.fortytwo.twitlogic.persistence.TweetPersister;
import net.fortytwo.twitlogic.persistence.TweetStore;
import net.fortytwo.twitlogic.persistence.TweetStoreException;
import net.fortytwo.twitlogic.services.twitter.HandlerException;
import net.fortytwo.twitlogic.services.twitter.CustomTwitterClient;
import net.fortytwo.twitlogic.services.twitter.TwitterClientException;
import net.fortytwo.twitlogic.syntax.Matcher;
import net.fortytwo.twitlogic.syntax.MultiMatcher;
import net.fortytwo.twitlogic.syntax.TopicSniffer;
import net.fortytwo.twitlogic.syntax.TweetAnnotator;
import net.fortytwo.twitlogic.syntax.afterthought.DemoAfterthoughtMatcher;
import net.fortytwo.twitlogic.util.properties.PropertyException;
import org.openrdf.model.Statement;
import org.openrdf.model.Value;
import org.openrdf.model.ValueFactory;
import org.openrdf.sail.NotifyingSail;
import org.openrdf.sail.NotifyingSailConnection;
import org.openrdf.sail.Sail;
import org.openrdf.sail.SailConnectionListener;
import org.openrdf.sail.SailException;
import org.openrdf.sail.helpers.NotifyingSailBase;
import org.openrdf.sail.helpers.NotifyingSailWrapper;
import java.util.Collection;
import java.util.Date;
import java.util.LinkedList;
import java.util.Properties;
import java.util.Set;
import java.util.logging.Logger;
/**
* User: josh
* Date: 6/1/11
* Time: 4:41 PM
*/
public class TwitLogicPubsubProvider extends PubsubProvider<Value, Dataset> {
private static final Logger LOGGER = TwitLogic.getLogger(TwitLogicPubsubProvider.class);
private boolean active = false;
private long minimumUpdateInterval = 0;
private long lastUpdate = 0;
private Sail sail;
public TwitLogicPubsubProvider(final RDFAgent agent,
final Properties config) throws Exception {
super(agent);
TwitLogic.setConfiguration(config);
final Handler<Dataset> handler = new Handler<Dataset>() {
public boolean isOpen() {
return true;
}
public void handle(Dataset dataset) throws HandlerException {
try {
handleDataset(dataset);
} catch (LocalFailure e) {
throw new HandlerException(e);
}
}
};
// Do this here so that the Sail is initialized before the runner thread starts.
final TweetGenerator g = new TweetGenerator();
new Thread(new Runnable() {
@Override
public void run() {
try {
// TODO: shut down the generator
g.run(handler);
} catch (Throwable e) {
active = false;
LOGGER.severe("TwitLogic thread failed (stack trace follows)\n" + RDFAgents.stackTraceToString(e));
}
}
}).start();
}
public void setMinimumUpdateInterval(final long minimumUpdateInterval) {
this.minimumUpdateInterval = minimumUpdateInterval;
}
public boolean isActive() {
return active;
}
public Sail getSail() {
return sail;
}
@Override
protected Commitment considerSubscriptionRequestInternal(final Value topic,
final AgentId initiator) {
return new Commitment(Commitment.Decision.AGREE_AND_NOTIFY, null);
}
public class TweetGenerator {
private final Collection<Statement> buffer = new LinkedList<Statement>();
private final TweetStore store;
public TweetGenerator() throws TweetStoreException, SailException {
SailConnectionListener listener = new SailConnectionListener() {
@Override
public void statementAdded(Statement statement) {
buffer.add(statement);
}
@Override
public void statementRemoved(Statement statement) {
// Do nothing.
}
};
Sail baseSail = TweetStore.createSail();
NotifyingSail b;
if (baseSail instanceof NotifyingSail) {
b = (NotifyingSail) baseSail;
} else {
b = new NotifyingSailWrapper();
((NotifyingSailWrapper) b).setBaseSail(baseSail);
}
sail = new WrapperNotifyingSail(b, listener);
sail.initialize();
store = new TweetStore(sail);
}
public void run(final Handler<Dataset> datasetHandler) throws TweetStoreException, TwitterClientException, PropertyException {
store.initialize();
try {
final TweetPersister persister = new TweetPersister(store, null);
try {
Handler<Tweet> handler = new Handler<Tweet>() {
@Override
public boolean isOpen() {
return persister.isOpen() && datasetHandler.isOpen();
}
@Override
public void handle(final Tweet tweet) throws HandlerException {
System.out.println("got this tweet: " + tweet);
buffer.clear();
persister.handle(tweet);
if (0 < buffer.size()) {
Collection<Statement> c = new LinkedList<Statement>();
c.addAll(buffer);
datasetHandler.handle(new Dataset(c));
}
}
};
CustomTwitterClient client = new CustomTwitterClient();
TweetPersistedLogger pLogger = new TweetPersistedLogger(client.getStatistics(), handler);
TweetFilterCriterion crit = new TweetFilterCriterion(TwitLogic.getConfiguration());
Filter<Tweet> f = new Filter<Tweet>(crit, pLogger);
// Add a "topic sniffer".
TopicSniffer topicSniffer = new TopicSniffer(f);
// Add a tweet annotator.
Matcher matcher = new MultiMatcher(
new DemoAfterthoughtMatcher());
final Handler<Tweet> annotator
= new TweetAnnotator(matcher, topicSniffer);
Handler<Tweet> deleter = new TweetDeleter(store);
TweetReceivedLogger rLogger = new TweetReceivedLogger(client.getStatistics(), annotator);
Set<User> users = TwitLogic.findFollowList(client);
Set<String> terms = TwitLogic.findTrackTerms();
if (0 < users.size() || 0 < terms.size()) {
client.processFilterStream(users, terms, null, rLogger, deleter, 0);
} else {
client.processSampleStream(rLogger, deleter);
}
} finally {
persister.close();
}
} finally {
store.shutDown();
}
}
}
private class WrapperNotifyingSail extends NotifyingSailBase {
private final NotifyingSail base;
private final SailConnectionListener listener;
public WrapperNotifyingSail(final NotifyingSail base, SailConnectionListener listener) {
this.base = base;
this.listener = listener;
}
@Override
protected void shutDownInternal() throws SailException {
base.shutDown();
}
@Override
protected NotifyingSailConnection getConnectionInternal() throws SailException {
NotifyingSailConnection c = base.getConnection();
c.addConnectionListener(listener);
return c;
}
@Override
public boolean isWritable() throws SailException {
return base.isWritable();
}
@Override
public ValueFactory getValueFactory() {
return base.getValueFactory();
}
}
protected void handleDataset(final Dataset d) throws LocalFailure {
Set<Value> t = getTopics();
long now = new Date().getTime();
if (now - lastUpdate >= minimumUpdateInterval) {
lastUpdate = now;
for (Statement s : d.getStatements()) {
if (t.contains(s.getSubject())) {
produceUpdate(s.getSubject(), d);
} else if (t.contains(s.getObject())) {
produceUpdate(s.getObject(), d);
}
}
}
}
}